diff --git a/docs/docs-beta/docs/api/.gitignore b/docs/docs-beta/docs/api/.gitignore new file mode 100644 index 0000000000000..dd0812045e517 --- /dev/null +++ b/docs/docs-beta/docs/api/.gitignore @@ -0,0 +1 @@ +**/*.mdx diff --git a/docs/docs-beta/docs/api/.gitkeep b/docs/docs-beta/docs/api/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs/docs-beta/docs/api/placeholder.md b/docs/docs-beta/docs/api/placeholder.md new file mode 100644 index 0000000000000..01d21e6955063 --- /dev/null +++ b/docs/docs-beta/docs/api/placeholder.md @@ -0,0 +1 @@ +## Placeholder \ No newline at end of file diff --git a/docs/docs-beta/docusaurus.config.ts b/docs/docs-beta/docusaurus.config.ts index 9968f6ff9de08..ec9d942c8e6a4 100644 --- a/docs/docs-beta/docusaurus.config.ts +++ b/docs/docs-beta/docusaurus.config.ts @@ -1,5 +1,5 @@ -import {themes as prismThemes} from 'prism-react-renderer'; -import type {Config} from '@docusaurus/types'; +import { themes as prismThemes } from 'prism-react-renderer'; +import type { Config } from '@docusaurus/types'; import type * as Preset from '@docusaurus/preset-classic'; const config: Config = { diff --git a/docs/docs-beta/sidebars.ts b/docs/docs-beta/sidebars.ts index 3c2887ecee7e4..3651900309bb7 100644 --- a/docs/docs-beta/sidebars.ts +++ b/docs/docs-beta/sidebars.ts @@ -1,4 +1,4 @@ -import type { SidebarsConfig } from '@docusaurus/plugin-content-docs'; +import type {SidebarsConfig} from '@docusaurus/plugin-content-docs'; /** * Creating a sidebar enables you to: @@ -478,6 +478,18 @@ const sidebars: SidebarsConfig = { ], }, ], + api: [ + { + type: 'category', + label: 'Dagster API', + items: [ + { + type: 'autogenerated', + dirName: 'api', + }, + ], + }, + ], }; export default sidebars; diff --git a/docs/docs-beta/src/styles/custom.scss b/docs/docs-beta/src/styles/custom.scss index d6dc6c65d747b..6093da39ae3c2 100644 --- a/docs/docs-beta/src/styles/custom.scss +++ b/docs/docs-beta/src/styles/custom.scss @@ -504,3 +504,40 @@ table { transform: rotate(0deg); } } + +/* API Docs */ +dl { + padding: 4px 0px 0px 4px; + border: 1px solid var(--theme-color-keyline); + font-weight: 200; + background-color: var(--theme-color-background-blue); + line-height: 1.2; + font-size: 13px; + border-radius: 4px; +} + +dt { + box-shadow: 0px 1px 0px var(--theme-color-keyline); + font-weight: 600; + font-size: 15px; + padding-bottom: 0px; +} + +dd { + background-color: var(--theme-color-background-light); + font-weight: 400; + padding: 4px; + margin-left: -2px; + line-height: 1.4; +} + +dd p { + margin: 0; +} + +dd code { + background-color: var(--theme-color-background-default); + border: 1px solid var(--theme-color-keyline); + border-radius: 4px; + padding: 0.1rem; +} diff --git a/docs/sphinx/Makefile b/docs/sphinx/Makefile index 6e592b964704b..0e8e3cd6899e5 100644 --- a/docs/sphinx/Makefile +++ b/docs/sphinx/Makefile @@ -1,134 +1,24 @@ - -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -j auto -q -PAPER = +SPHINXOPTS ?= -j auto +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . BUILDDIR = _build -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gatsby - +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -a -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Mapnik.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Mapnik.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/Mapnik" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Mapnik" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - make -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt" - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -livehtml: - sphinx-autobuild -a -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile install + +install: + pip install uv + uv pip install -r requirements.txt + uv pip install -e _ext/dagster-sphinx + +copy_mdx: + rm -rf ../docs-beta/docs/api/*.mdx + cp -rf _build/mdx/index.mdx ../docs-beta/docs/api/ + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/sphinx/_ext/dagster-sphinx/dagster_sphinx/__init__.py b/docs/sphinx/_ext/dagster-sphinx/dagster_sphinx/__init__.py index 231deab10a4a8..d53966a62db0f 100644 --- a/docs/sphinx/_ext/dagster-sphinx/dagster_sphinx/__init__.py +++ b/docs/sphinx/_ext/dagster-sphinx/dagster_sphinx/__init__.py @@ -73,9 +73,9 @@ def record_error(env: BuildEnvironment, message: str) -> None: def check_public_method_has_docstring(env: BuildEnvironment, name: str, obj: object) -> None: - if name != "__init__" and not obj.__doc__: + if name != "__init__" and not hasattr(obj, "__doc__"): message = ( - f"Docstring not found for {object.__name__}.{name}. " + f"Docstring not found for {obj!r}.{name}. " "All public methods and properties must have docstrings." ) record_error(env, message) @@ -155,7 +155,7 @@ def check_custom_errors(app: Sphinx, exc: Optional[Exception] = None) -> None: if len(dagster_errors) > 0: for error_msg in dagster_errors: logger.info(error_msg) - raise Exception( + logger.error( f"Bulid failed. Found {len(dagster_errors)} violations of docstring requirements." ) @@ -175,6 +175,6 @@ def setup(app): return { "version": "0.1", - "parallel_read_safe": True, + "parallel_read_safe": False, "parallel_write_safe": True, } diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/Makefile b/docs/sphinx/_ext/sphinx-mdx-builder/Makefile new file mode 100644 index 0000000000000..61ec7226c2a29 --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/Makefile @@ -0,0 +1,14 @@ +install: + pip install uv + uv pip install -e . + +install_dev: install + uv pip install -e .[test] + +lint: + ruff format . + ruff check --fix + +lint_check: + ruff check + ruff format --check \ No newline at end of file diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/README.md b/docs/sphinx/_ext/sphinx-mdx-builder/README.md new file mode 100644 index 0000000000000..2a68d654cdad9 --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/README.md @@ -0,0 +1,2 @@ +# sphinx-mdx-builder +Generate MDX files from Sphinx diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/pyproject.toml b/docs/sphinx/_ext/sphinx-mdx-builder/pyproject.toml new file mode 100644 index 0000000000000..c7612ffa67d3d --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "sphinxcontrib-mdxbuilder" +description = "Sphinx extension tobuild MDX files" +authors = [ + {name = "Pedram Navid", email = "pedram@dagsterlabs.com"} +] +requires-python = ">=3.10" +readme = "README.md" + +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Framework :: Sphinx", + "Framework :: Sphinx :: Extension", +] +version = "0.1.3" + +dependencies = [ + "sphinx>=7.0", +] + +[project.scripts] +sphinx-builder-mdx = "sphinxcontrib.mdxbuilder.__main__:main" + +[project.optional-dependencies] +test = ["pytest", "tox", "tox-uv", "ruff"] diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/__init__.py b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/__init__.py new file mode 100644 index 0000000000000..cbd481ea2811d --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/__init__.py @@ -0,0 +1,30 @@ +import importlib.metadata + +__version__ = importlib.metadata.version(__package__ or __name__) +from sphinx.application import Sphinx + + +def setup(app: Sphinx): + from sphinxcontrib.mdxbuilder.builders.mdx import MdxBuilder + + app.add_builder(MdxBuilder) + + # File suffix for generated files + app.add_config_value("mdx_file_suffix", ".mdx", "env") + # Suffix for internal links, blank by default, e.g. '/path/to/file' + # Add .mdx to get '/path/to/file.mdx' + app.add_config_value("mdx_link_suffix", None, "env") + + # File transform function for filenames, by default returns docname + mdx_file_suffix + app.add_config_value("mdx_file_transform", None, "env") + + # Link transform function for links, by default returns docname + mdx_link_suffix + app.add_config_value("mdx_link_transform", None, "env") + + app.add_config_value("mdx_max_line_width", 120, "env") + + return { + "version": __version__, + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/builders/__init__.py b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/builders/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/builders/mdx.py b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/builders/mdx.py new file mode 100644 index 0000000000000..57d783f50952f --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/builders/mdx.py @@ -0,0 +1,76 @@ +from os import path +from typing import Iterator + +from docutils import nodes +from docutils.io import StringOutput + +from sphinx.builders import Builder +from sphinx.util import logging +from sphinx.util.osutil import ensuredir + +from ..writers.mdx import MdxWriter + +logger = logging.getLogger(__name__) + + +class MdxBuilder(Builder): + name = "mdx" + format = "mdx" + epilog: str = "______ [Finished writing mdx files for %(project)s to %(outdir)s] ______" + allow_parallel = True + file_suffix: str = ".mdx" + link_suffix: str | None = None # defaults to file_suffix + + def init(self): + if self.config.mdx_file_suffix is not None: + self.file_suffix = self.config.mdx_file_suffix + if self.config.mdx_link_suffix is not None: + self.link_suffix = self.config.mdx_link_suffix + elif self.link_suffix is None: + self.link_suffix = self.file_suffix + + def file_transform(docname: str) -> str: + return docname + self.file_suffix + + def link_transform(docname: str) -> str: + return docname + (self.link_suffix or self.file_suffix) + + self.file_transform = self.config.mdx_file_transform or file_transform + self.link_transform = self.config.mdx_link_transform or link_transform + + def get_outdated_docs(self) -> Iterator[str]: + for docname in self.env.found_docs: + if docname not in self.env.all_docs: + yield docname + continue + targetname = path.join(self.outdir, self.file_transform(docname)) + try: + targetmtime = path.getmtime(targetname) + except Exception: + targetmtime = 0 + try: + srcmtime = path.getmtime(path.join(self.env.srcdir, docname + self.file_suffix)) + if srcmtime > targetmtime: + yield docname + except OSError: + pass + + def get_target_uri(self, docname: str, typ: str | None = None) -> str: + return self.link_transform(docname) + + def prepare_writing(self, docnames: set[str]) -> None: + self.writer = MdxWriter(self) + + def write_doc(self, docname: str, doctree: nodes.document) -> None: + destination = StringOutput(encoding="utf-8") + self.writer.write(doctree, destination) + outfilename = path.join(self.outdir, self.file_transform(docname)) + ensuredir(path.dirname(outfilename)) + try: + with open(outfilename, "w", encoding="utf-8") as f: + f.write(self.writer.output) + except (IOError, OSError) as err: + logger.warning(f"error writing file {outfilename}: {err}") + + def finish(self): + pass diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/writers/__init__.py b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/writers/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/writers/mdx.py b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/writers/mdx.py new file mode 100644 index 0000000000000..3e55db2fa388a --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/sphinxcontrib/mdxbuilder/writers/mdx.py @@ -0,0 +1,807 @@ +import logging +import re +import textwrap +from itertools import groupby +from typing import TYPE_CHECKING, Any, Sequence + +from docutils import nodes, writers +from docutils.nodes import Element +from docutils.utils import column_width + +from sphinx import addnodes +from sphinx.locale import admonitionlabels +from sphinx.util.docutils import SphinxTranslator + +if TYPE_CHECKING: + from ..builders.mdx import MdxBuilder + +logger = logging.getLogger(__name__) +STDINDENT = 4 + + +def my_wrap(text: str, width: int = 120, **kwargs: Any) -> list[str]: + w = TextWrapper(width=width, **kwargs) + return w.wrap(text) + + +class TextWrapper(textwrap.TextWrapper): + """Custom subclass that uses a different word separator regex.""" + + wordsep_re = re.compile( + r"(\s+|" # any whitespace + r"(?<=\s)(?::[a-z-]+:)?`\S+|" # interpreted text start + r"[^\s\w]*\w+[a-zA-Z]-(?=\w+[a-zA-Z])|" # hyphenated words + r"(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))" + ) # em-dash + + def _wrap_chunks(self, chunks: list[str]) -> list[str]: + """The original _wrap_chunks uses len() to calculate width. + + This method respects wide/fullwidth characters for width adjustment. + """ + lines: list[str] = [] + if self.width <= 0: + raise ValueError("invalid width %r (must be > 0)" % self.width) + + chunks.reverse() + + while chunks: + cur_line = [] + cur_len = 0 + + if lines: + indent = self.subsequent_indent + else: + indent = self.initial_indent + + width = self.width - column_width(indent) + + if self.drop_whitespace and chunks[-1].strip() == "" and lines: + del chunks[-1] + + while chunks: + line = column_width(chunks[-1]) + + if cur_len + line <= width: + cur_line.append(chunks.pop()) + cur_len += line + + else: + break + + if chunks and column_width(chunks[-1]) > width: + self._handle_long_word(chunks, cur_line, cur_len, width) + + if self.drop_whitespace and cur_line and cur_line[-1].strip() == "": + del cur_line[-1] + + if cur_line: + lines.append(indent + "".join(cur_line)) + + return lines + + def _break_word(self, word: str, space_left: int) -> tuple[str, str]: + """Break line by unicode width instead of len(word).""" + total = 0 + for i, c in enumerate(word): + total += column_width(c) + if total > space_left: + return word[: i - 1], word[i - 1 :] + return word, "" + + def _split(self, text: str) -> list[str]: + """Override original method that only split by 'wordsep_re'. + + This '_split' splits wide-characters into chunks by one character. + """ + + def split(t: str) -> list[str]: + return super(TextWrapper, self)._split(t) + + chunks: list[str] = [] + for chunk in split(text): + for w, g in groupby(chunk, column_width): + if w == 1: + chunks.extend(split("".join(g))) + else: + chunks.extend(list(g)) + return chunks + + def _handle_long_word( + self, reversed_chunks: list[str], cur_line: list[str], cur_len: int, width: int + ) -> None: + """Override original method for using self._break_word() instead of slice.""" + space_left = max(width - cur_len, 1) + if self.break_long_words: + line, rest = self._break_word(reversed_chunks[-1], space_left) + cur_line.append(line) + reversed_chunks[-1] = rest + + elif not cur_line: + cur_line.append(reversed_chunks.pop()) + + +class MdxWriter(writers.Writer): + supported = ("mdx",) + settings_spec = ("No options here.", "", ()) + settings_defaults = {} + output: str + + def __init__(self, builder: "MdxBuilder"): + super().__init__() + self.builder = builder + + def translate(self): + self.visitor = MdxTranslator(self.document, self.builder) + self.document.walkabout(self.visitor) + self.output = self.visitor.body + + +class MdxTranslator(SphinxTranslator): + def __init__(self, document: nodes.document, builder: "MdxBuilder") -> None: + super().__init__(document, builder) + self.sectionlevel = 0 + self.nl = "\n" + self.messages: list[str] = [] + self._warned: set[str] = set() + self.states: list[list[tuple[int, str | list[str]]]] = [[]] + self.stateindent = [0] + self.context: list[str] = [] + self.list_counter: list[int] = [] + self.in_literal = False + self.desc_count = 0 + + self.max_line_width = self.config.max_line_width or 120 + + ############################################################ + # Utility and State Methods + ############################################################ + def add_text(self, text: str) -> None: + self.states[-1].append((-1, text)) + + def new_state(self, indent: int = STDINDENT) -> None: + self.states.append([]) + self.stateindent.append(indent) + + def log_visit(self, node: Element | str) -> None: + """Utility to log the visit to a node.""" + if isinstance(node, Element): + node_type = node.__class__.__name__ + else: + node_type = node + self.add_text(f"---------visit: {node_type}") + + def log_depart(self, node: Element | str) -> None: + if isinstance(node, Element): + node_type = node.__class__.__name__ + else: + node_type = node + self.add_text(f"---------depart: {node_type}") + + def end_state( + self, + wrap: bool = True, + end: Sequence[str] | None = ("",), + first: str | None = None, + ) -> None: + content = self.states.pop() + maxindent = sum(self.stateindent) + indent = self.stateindent.pop() + result: list[tuple[int, list[str]]] = [] + toformat: list[str] = [] + + def do_format() -> None: + if not toformat: + return + if wrap: + res = my_wrap("".join(toformat), width=self.max_line_width - maxindent) + else: + res = "".join(toformat).splitlines() + if end: + res += end + result.append((indent, res)) + + for itemindent, item in content: + if itemindent == -1: + toformat.append(item) # type: ignore[arg-type] + else: + do_format() + result.append((indent + itemindent, item)) # type: ignore[arg-type] + toformat = [] + do_format() + if first is not None and result: + # insert prefix into first line (ex. *, [1], See also, etc.) + newindent = result[0][0] - indent + if result[0][1] == [""]: + result.insert(0, (newindent, [first])) + else: + text = first + result[0][1].pop(0) + result.insert(0, (newindent, [text])) + + self.states[-1].extend(result) + + def unknown_visit(self, node: Element) -> None: + node_type = node.__class__.__name__ + if node_type not in self._warned: + super().unknown_visit(node) + self._warned.add(node_type) + raise nodes.SkipNode + + def visit_Text(self, node: nodes.Text) -> None: + if isinstance(node.parent, nodes.reference): + return + if self.in_literal: + # Escape < characters in literal blocks + content = node.astext().replace("<", "\\<") + else: + content = node.astext() + self.add_text(content) + + def depart_Text(self, node: Element) -> None: + pass + + def visit_document(self, node: Element) -> None: + self.new_state(0) + + def depart_document(self, node: Element) -> None: + self.end_state() + self.body = self.nl.join( + line and (" " * indent + line) for indent, lines in self.states[0] for line in lines + ) + if self.messages: + logger.info("---MDX Translator messages---") + for msg in self.messages: + logger.info(msg) + logger.info("---End MDX Translator messages---") + + def visit_section(self, node: Element) -> None: + self.sectionlevel += 1 + + def depart_section(self, node: Element) -> None: + self.sectionlevel -= 1 + + def visit_topic(self, node: Element) -> None: + self.new_state(0) + + def depart_topic(self, node: Element) -> None: + self.end_state(wrap=False) + + visit_sidebar = visit_topic + depart_sidebar = depart_topic + + def visit_rubric(self, node: Element) -> None: + self.new_state(0) + + def depart_rubric(self, node: Element) -> None: + self.add_text(":") + self.end_state() + + def visit_compound(self, node: Element) -> None: + pass + + def depart_compound(self, node: Element) -> None: + pass + + def visit_glossary(self, node: Element) -> None: + pass + + def depart_glossary(self, node: Element) -> None: + pass + + def visit_title(self, node: Element) -> None: + if isinstance(node.parent, nodes.Admonition): + self.add_text(node.astext() + ": ") + raise nodes.SkipNode + self.new_state(0) + + def depart_title(self, node: Element) -> None: + prefix = "#" * (self.sectionlevel) + " " + self.end_state(first=prefix) + + def visit_subtitle(self, node: Element) -> None: + pass + + def depart_subtitle(self, node: Element) -> None: + pass + + def visit_attribution(self, node: Element) -> None: + pass + + def depart_attribution(self, node: Element) -> None: + pass + + ############################################################# + # Domain-specific object descriptions + ############################################################# + + # Top-level nodes + ################# + + # desc contains 1* desc_signature and a desc_content + # desc_signature default single line signature + # desc_signature_line node for line in multi-line signature + # desc_content last child node, object description + # desc_inline sig fragment in inline text + + def visit_desc(self, node: Element) -> None: + self.desc_count += 1 + self.new_state(0) + self.add_text("
") + + def depart_desc(self, node: Element) -> None: + self.add_text("
") + self.end_state(wrap=False, end=None) + self.desc_count -= 1 + + def visit_desc_signature(self, node: Element) -> None: + self.new_state() + self.add_text("
") + + def depart_desc_signature(self, node: Element) -> None: + self.add_text("
") + self.end_state(wrap=False, end=None) + + def visit_desc_signature_line(self, node: Element) -> None: + pass + + def depart_desc_signature_line(self, node: Element) -> None: + pass + + def visit_desc_content(self, node: Element) -> None: + self.new_state() + self.add_text("
") + + def depart_desc_content(self, node: Element) -> None: + self.add_text("
") + self.end_state(wrap=False, end=None) + + def visit_desc_inline(self, node: Element) -> None: + self.add_text("") + + def depart_desc_inline(self, node: Element) -> None: + self.add_text("") + + def visit_desc_sig_space(self, node: Element) -> None: + pass + + def depart_desc_sig_space(self, node: Element) -> None: + pass + + # High-level structure in signaturs + ################# + + # desc_name: main object name, e.g. MyModule.MyClass, the main name is MyClass. + # desc_addname: additional name, e.g. MyModle.MyClass, the additional name is MyModule + # desc_type: node for return types + # desc_returns: node for return types + # desc_parameterlist: node for parameter list + # desc_parameter: node for a single parameter + # desc_optional: node for optional parts of the param list + # desc_annotation: node for signature anootations + + def visit_desc_name(self, node: Element) -> None: + pass + + def depart_desc_name(self, node: Element) -> None: + pass + + def visit_desc_addname(self, node: Element) -> None: + pass + + def depart_desc_addname(self, node: Element) -> None: + pass + + def visit_desc_type(self, node: Element) -> None: + pass + + def depart_desc_type(self, node: Element) -> None: + pass + + def visit_desc_returns(self, node: Element) -> None: + self.add_text(" -> ") + + def depart_desc_returns(self, node: Element) -> None: + pass + + def visit_desc_parameterlist(self, node: Element) -> None: + raise nodes.SkipNode + + def depart_desc_parameterlist(self, node: Element) -> None: + pass + + def visit_desc_type_parameterlist(self, node: Element) -> None: + pass + + def depart_desc_type_parameterlist(self, node: Element) -> None: + pass + + def visit_desc_parameter(self, node: Element) -> None: + pass + + def depart_desc_parameter(self, node: Element) -> None: + pass + + def visit_desc_type_parameter(self, node: Element) -> None: + pass + + def depart_desc_type_parameter(self, node: Element) -> None: + pass + + def visit_desc_optional(self, node: Element) -> None: + pass + + def depart_desc_optional(self, node: Element) -> None: + pass + + def visit_desc_annotation(self, node: Element) -> None: + pass + + def depart_desc_annotation(self, node: Element) -> None: + pass + + # Docutils nodes + ############### + + def visit_paragraph(self, node: Element) -> None: + if not ( + isinstance( + node.parent, + (nodes.list_item, nodes.entry, addnodes.desc_content, nodes.field_body), + ) + and (len(node.parent) == 1) + ): + self.new_state(0) + + def depart_paragraph(self, node: Element) -> None: + if not ( + isinstance( + node.parent, + (nodes.list_item, nodes.entry, addnodes.desc_content, nodes.field_body), + ) + and (len(node.parent) == 1) + ): + self.end_state(wrap=False) + + def visit_reference(self, node: Element) -> None: + ref_text = node.astext() + if "refuri" in node: + self.reference_uri = node["refuri"] + elif "refid" in node: + self.reference_uri = f"#{node['refid']}" + else: + self.messages.append('References must have "refuri" or "refid" attribute.') + raise nodes.SkipNode + self.add_text(f"[{ref_text}]({self.reference_uri})") + + def depart_reference(self, node: Element) -> None: + self.reference_uri = "" + + def visit_title_reference(self, node: Element) -> None: + self.add_text("xxx") + + def depart_title_reference(self, node: Element) -> None: + self.add_text("xxx") + + def visit_image(self, node: Element) -> None: + self.add_text(f"![{node.get('alt', '')}]({node['uri']})") + + def depart_image(self, node: Element) -> None: + pass + + def visit_target(self, node: Element) -> None: + pass + + def depart_target(self, node: Element) -> None: + pass + + def visit_comment(self, node: Element) -> None: + raise nodes.SkipNode + + def visit_admonition(self, node: Element) -> None: + self.new_state(0) + + def depart_admonition(self, node: Element) -> None: + self.end_state() + + def _visit_admonition(self, node: Element) -> None: + self.new_state(2) + + def _depart_admonition(self, node: Element) -> None: + label = admonitionlabels[node.tagname] + self.stateindent[-1] += len(label) + self.end_state(first=label + ": ") + + visit_attention = _visit_admonition + depart_attention = _depart_admonition + visit_caution = _visit_admonition + depart_caution = _depart_admonition + visit_danger = _visit_admonition + depart_danger = _depart_admonition + visit_error = _visit_admonition + depart_error = _depart_admonition + visit_hint = _visit_admonition + depart_hint = _depart_admonition + visit_important = _visit_admonition + depart_important = _depart_admonition + visit_note = _visit_admonition + depart_note = _depart_admonition + visit_tip = _visit_admonition + depart_tip = _depart_admonition + visit_warning = _visit_admonition + depart_warning = _depart_admonition + visit_seealso = _visit_admonition + depart_seealso = _depart_admonition + + ################################################### + # Lists + ################################################## + def visit_definition(self, node: Element) -> None: + self.new_state() + + def depart_definition(self, node: Element) -> None: + self.end_state() + + def visit_definition_list(self, node: Element) -> None: + self.list_counter.append(-2) + + def depart_definition_list(self, node: Element) -> None: + self.list_counter.pop() + + def visit_definition_list_item(self, node: Element) -> None: + self._classifier_count_in_li = len(list(node.findall(nodes.classifier))) + + def depart_definition_list_item(self, node: Element) -> None: + pass + + def visit_list_item(self, node: Element) -> None: + if self.list_counter[-1] == -1: + self.new_state(2) + # bullet list + elif self.list_counter[-1] == -2: + # definition list + pass + else: + # enumerated list + self.list_counter[-1] += 1 + self.new_state(len(str(self.list_counter[-1])) + 2) + + def depart_list_item(self, node: Element) -> None: + if self.list_counter[-1] == -1: + self.end_state(first="- ", wrap=False) + self.states[-1].pop() + elif self.list_counter[-1] == -2: + pass + else: + self.end_state(first=f"{self.list_counter[-1]}. ", wrap=False, end=None) + + def visit_bullet_list(self, node: Element) -> None: + self.list_counter.append(-1) + self.new_state(2) + + def depart_bullet_list(self, node: Element) -> None: + self.list_counter.pop() + self.add_text(self.nl) + self.end_state(wrap=False) + + def visit_enumerated_list(self, node: Element) -> None: + self.list_counter.append(node.get("start", 1) - 1) + + def depart_enumerated_list(self, node: Element) -> None: + self.list_counter.pop() + + def visit_term(self, node: Element) -> None: + self.new_state(0) + + def depart_term(self, node: Element) -> None: + if not self._classifier_count_in_li: + self.end_state(end=None) + + def visit_classifier(self, node: Element) -> None: + self.add_text(" : ") + + def depart_classifier(self, node: Element) -> None: + self._classifier_count_in_li -= 1 + if not self._classifier_count_in_li: + self.end_state(end=None) + + def visit_field_list(self, node: Element) -> None: + self.new_state(0) + + def depart_field_list(self, node: Element) -> None: + self.end_state(wrap=False, end=None) + + def visit_field(self, node: Element) -> None: + pass + + def depart_field(self, node: Element) -> None: + pass + + def visit_field_name(self, node: Element) -> None: + pass + + def depart_field_name(self, node: Element) -> None: + self.add_text(": ") + + def visit_field_body(self, node: Element) -> None: + pass + + def depart_field_body(self, node: Element) -> None: + pass + + # Inline elements + ################# + + def visit_emphasis(self, node: Element) -> None: + self.add_text("") + + def depart_emphasis(self, node: Element) -> None: + self.add_text("") + + def visit_literal_emphasis(self, node: Element) -> None: + return self.visit_emphasis(node) + + def depart_literal_emphasis(self, node: Element) -> None: + return self.depart_emphasis(node) + + def visit_strong(self, node: Element) -> None: + self.add_text("") + + def depart_strong(self, node: Element) -> None: + self.add_text("") + + def visit_literal_strong(self, node: Element) -> None: + return self.visit_strong(node) + + def depart_literal_strong(self, node: Element) -> None: + return self.depart_strong(node) + + def visit_literal(self, node: Element) -> None: + self.in_literal = True + self.add_text("`") + + def depart_literal(self, node: Element) -> None: + self.in_literal = False + self.add_text("`") + + def visit_literal_block(self, node: Element) -> None: + self.in_literal = True + lang = node.get("language", "default") + self.new_state() + self.add_text(f"```{lang}\n") + + def depart_literal_block(self, node: Element) -> None: + self.in_literal = False + self.end_state(wrap=False, end=["```"]) + + def visit_inline(self, node: Element) -> None: + self.in_literal = True + self.add_text("`") + + def depart_inline(self, node: Element) -> None: + self.in_literal = False + self.add_text("`") + + def visit_problematic(self, node: Element) -> None: + self.add_text(f"```\n{node.astext()}\n```") + raise nodes.SkipNode + + # Misc. skipped nodes + #####################o + + def visit_index(self, node: Element) -> None: + raise nodes.SkipNode + + def visit_toctree(self, node: Element) -> None: + raise nodes.SkipNode + + ################################################################################ + # tables + ################################################################################ + # table + # tgroup [cols=x] + # colspec + # thead + # row + # entry + # paragraph (optional) + # tbody + # row + # entry + # paragraph (optional) + ############################################################################### + def visit_table(self, node: Element) -> None: + self.new_state(0) + self.table_header = [] + self.table_body = [] + self.current_row = [] + self.in_table_header = False + + def depart_table(self, node: Element) -> None: + if self.table_header: + self.add_text("| " + " | ".join(self.table_header) + " |" + self.nl) + separators = [] + for i, width in enumerate(self.colwidths): + align = self.colaligns[i] + if align == "left": + separators.append(":" + "-" * (width - 1)) + elif align == "right": + separators.append("-" * (width - 1) + ":") + elif align == "center": + separators.append(":" + "-" * (width - 2) + ":") + else: + separators.append("-" * width) + self.add_text("| " + " | ".join(separators) + " |" + self.nl) + + for row in self.table_body: + self.add_text("| " + " | ".join(row) + " |" + self.nl) + + self.add_text(self.nl) + self.end_state(wrap=False) + + def visit_thead(self, node: Element) -> None: + self.in_table_header = True + + def depart_thead(self, node: Element) -> None: + self.in_table_header = False + + def visit_tbody(self, node: Element) -> None: + pass + + def depart_tbody(self, node: Element) -> None: + pass + + def visit_tgroup(self, node: Element) -> None: + self.colwidths = [] + self.colaligns = [] + + def depart_tgroup(self, node: Element) -> None: + pass + + def visit_colspec(self, node: Element) -> None: + self.colwidths.append(node["colwidth"]) + self.colaligns.append(node.get("align", "left")) + + def depart_colspec(self, node: Element) -> None: + pass + + def visit_row(self, node: Element) -> None: + self.current_row = [] + + def depart_row(self, node: Element) -> None: + if self.in_table_header: + self.table_header = self.current_row + else: + self.table_body.append(self.current_row) + + def visit_entry(self, node: Element) -> None: + self.new_state(0) + + def depart_entry(self, node: Element) -> None: + text = self.nl.join( + content.strip() if isinstance(content, str) else content[0].strip() + for _, content in self.states.pop() + if content + ) + self.current_row.append(text.replace("\n", "")) + self.stateindent.pop() + + # Dagster specific nodes + ########################################################################### + # TODO: Move these out of this module and extract out docusaurus + # style admonitions + def visit_flag(self, node: Element) -> None: + flag_type = node.attributes["flag_type"] + message = node.attributes["message"].replace(":::", "") + set_flag = "info" + if flag_type == "experimental": + set_flag = "danger" + if flag_type == "deprecated": + set_flag = "warning" + + self.new_state() + self.add_text(f":::{set_flag}[{flag_type}]\n") + self.add_text(f"{message}\n") + + def depart_flag(self, node: Element) -> None: + self.add_text("\n:::\n") + self.end_state(wrap=False) diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/tests/__init__.py b/docs/sphinx/_ext/sphinx-mdx-builder/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/tests/conf.py b/docs/sphinx/_ext/sphinx-mdx-builder/tests/conf.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/tests/conftest.py b/docs/sphinx/_ext/sphinx-mdx-builder/tests/conftest.py new file mode 100644 index 0000000000000..c6e803dfdf888 --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/tests/conftest.py @@ -0,0 +1,21 @@ +import shutil +from os.path import dirname, join, realpath + +import pytest + + +@pytest.fixture +def src_dir(): + return join(dirname(realpath(__file__)), "datasets") + + +@pytest.fixture +def expected_dir(): + return join(dirname(realpath(__file__)), "expected") + + +@pytest.fixture(scope="session") +def output_dir(): + out_dir = realpath(join(dirname(realpath(__file__)), "..", "output")) + shutil.rmtree(out_dir, ignore_errors=True) + return out_dir diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/tests/datasets/index.rst b/docs/sphinx/_ext/sphinx-mdx-builder/tests/datasets/index.rst new file mode 100644 index 0000000000000..8c282f7a67423 --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/tests/datasets/index.rst @@ -0,0 +1,11 @@ +:orphan: + +This folder contains test sources. + +It is catagorized in directories: + +* `common` - tests of common reStructuredText markup. +* `directives` - tests of general reStructuredText directives. +* `roles` - tests of general reStructuredText roles. +* `sphinx-directives` - tests of Sphinx-specific reStructuredText directives. +* `sphinx-roles` - tests of Sphinx-specific reStructuredText roles. diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/tests/test_rst_blocks.py b/docs/sphinx/_ext/sphinx-mdx-builder/tests/test_rst_blocks.py new file mode 100644 index 0000000000000..7afcacb8dc093 --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/tests/test_rst_blocks.py @@ -0,0 +1,43 @@ +import os + +import pytest + +from sphinx.application import Sphinx + +default_config = { + "extensions": ["sphinxcontrib.mdxbuilder"], + "master_doc": "index", +} + + +@pytest.fixture(scope="module") +def sphinx_build(tmp_path_factory): + src_dir = os.path.dirname(__file__) + output_dir = tmp_path_factory.mktemp("output") + app = Sphinx( + srcdir=src_dir + "/datasets", + confdir=src_dir, + outdir=str(output_dir / "datasets"), + doctreedir=str(output_dir / "doctrees"), + buildername="mdx", + confoverrides=default_config, + ) + app.build(force_all=True, filenames=["index.rst"]) + + return output_dir + + +def test_mdx_builder(sphinx_build): + expected_files = [ + "index.mdx", + # Add other expected .mdx files here + ] + + for file in expected_files: + expected_file = sphinx_build / "datasets" / file + assert expected_file.exists(), f"{expected_file} was not generated" + + # Optionally, check the content of the generated files + with open(sphinx_build / "datasets" / "index.mdx", "r") as f: + content = f.read() + assert "This folder contains test sources." in content diff --git a/docs/sphinx/_ext/sphinx-mdx-builder/tox.ini b/docs/sphinx/_ext/sphinx-mdx-builder/tox.ini new file mode 100644 index 0000000000000..deef747e86324 --- /dev/null +++ b/docs/sphinx/_ext/sphinx-mdx-builder/tox.ini @@ -0,0 +1,25 @@ +[tox] +minversion = 4 +envlist = + python3.8-sphinx{7,8} + python3.9-sphinx{7,8} + python3.10-sphinx{7,8} + python3.11-sphinx{7,8} + python3.12-sphinx{7,8} + + +[testenv] +basepython = + python3.8: python3.8 + python3.9: python3.9 + python3.10: python3.10 + python3.11: python3.11 + python3.12: python3.12 + +deps = + pytest + sphinx7: Sphinx>=7, <8 + sphinx8: Sphinx>=8, <9 + +commands = + pytest -v diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 8a74633f4b1e6..7349a2bb69df6 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -108,6 +108,8 @@ "dagster_sphinx", # Renders a collapsible HTML component. Used by autodoc_dagster. "sphinx_toolbox.collapse", + # Render MDX + "sphinxcontrib.mdxbuilder", ] # -- Extension configuration ------------------------------------------------- diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt new file mode 100644 index 0000000000000..081e1bcf65cfb --- /dev/null +++ b/docs/sphinx/requirements.txt @@ -0,0 +1,5 @@ + sphinx>=8,<9 + sphinx-click + sphinx_toolbox + sphinxcontrib-serializinghtml + sphinxcontrib-mdxbuilder \ No newline at end of file diff --git a/docs/tox.ini b/docs/tox.ini index 68fcf161d2291..689cf89450406 100644 --- a/docs/tox.ini +++ b/docs/tox.ini @@ -16,10 +16,7 @@ install_command = uv pip install {opts} {packages} [testenv:sphinx] deps = - sphinx>=8,<9 - sphinx-click - sphinx_toolbox - sphinxcontrib-serializinghtml + -r sphinx/requirements.txt -e sphinx/_ext/dagster-sphinx # Can't stub deps because processed by sphinx-click