diff --git a/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java b/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java index 0f51a643..5b58ecc6 100644 --- a/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java +++ b/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java @@ -22,88 +22,88 @@ public void setUp() { @Test public void testConvertCommonmarkToRstWithTitleAndParagraph() { String html = "

Title

Paragraph

"; - String expected = "\n\nTitle\n=====\nParagraph\n"; + String expected = "Title\n=====\nParagraph"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithImportantNote() { String html = "Important note"; - String expected = "\n\n.. important::\n Important note\n"; + String expected = ".. important::\n Important note"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithList() { String html = ""; - String expected = "\n\n* Item 1\n\n* Item 2\n\n"; + String expected = "* Item 1\n* Item 2"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithMixedElements() { String html = "

Title

Paragraph

"; - String expected = "\n\nTitle\n=====\nParagraph\n\n* Item 1\n\n* Item 2\n\n"; + String expected = "Title\n=====\nParagraph\n\n\n* Item 1\n* Item 2"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithNestedElements() { String html = "

Title

Paragraph with bold text

"; - String expected = "\n\nTitle\n=====\nParagraph with **bold** text\n"; + String expected = "Title\n=====\nParagraph with **bold** text"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithAnchorTag() { String html = "Link"; - String expected = "\n`Link `_"; + String expected = "`Link `_"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithBoldTag() { String html = "Bold text"; - String expected = "\n**Bold text**"; + String expected = "**Bold text**"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithItalicTag() { String html = "Italic text"; - String expected = "\n*Italic text*"; + String expected = "*Italic text*"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithCodeTag() { String html = "code snippet"; - String expected = "\n``code snippet``"; + String expected = "``code snippet``"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithNoteTag() { String html = "Note text"; - String expected = "\n\n.. note::\n Note text\n"; + String expected = ".. note::\n Note text"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } @Test public void testConvertCommonmarkToRstWithNestedList() { String html = ""; - String expected = "\n\n* Item 1\n * Subitem 1\n\n* Item 2\n\n"; + String expected = "* Item 1\n\n * Subitem 1\n* Item 2"; String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result); + assertEquals(expected, result.trim()); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 93f561ba..b726c42e 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -24,6 +24,7 @@ import software.amazon.smithy.python.codegen.integrations.PythonIntegration; import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; import software.amazon.smithy.python.codegen.sections.*; +import software.amazon.smithy.python.codegen.writer.MarkdownToRstDocConverter; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -60,6 +61,8 @@ private void generateService(PythonWriter writer) { var docs = service.getTrait(DocumentationTrait.class) .map(StringTrait::getValue) .orElse("Client for " + serviceSymbol.getName()); + String rstDocs = + MarkdownToRstDocConverter.getInstance().convertCommonmarkToRst(docs); writer.writeDocs(() -> { writer.write(""" $L @@ -68,7 +71,7 @@ private void generateService(PythonWriter writer) { endpoint for HTTP services or auth credentials. :param plugins: A list of callables that modify the configuration dynamically. These - can be used to set defaults, for example.""", docs); + can be used to set defaults, for example.""", rstDocs); }); var defaultPlugins = new LinkedHashSet(); @@ -866,14 +869,17 @@ private void writeSharedOperationInit(PythonWriter writer, OperationShape operat .orElse("The operation's input."); writer.write(""" - :param input: $L - + $L + """,docs); + writer.write(""); + writer.write(":param input: $L", inputDocs); + writer.write(""); + writer.write(""" :param plugins: A list of callables that modify the configuration dynamically. Changes made by these plugins only apply for the duration of the operation execution and will not affect any other operation invocations. + """); - $L - """, inputDocs, docs); }); var defaultPlugins = new LinkedHashSet(); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java index 4106207f..899848e5 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java @@ -334,7 +334,7 @@ private static void writeConf( } } - autodoc_typehints = 'both' + autodoc_typehints = 'description' """, projectName, version, projectName); }); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java index d7cc4dd1..e0e3d2dc 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java @@ -24,11 +24,13 @@ import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.ClientOptionalTrait; import software.amazon.smithy.model.traits.DefaultTrait; import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.ErrorTrait; import software.amazon.smithy.model.traits.InputTrait; import software.amazon.smithy.model.traits.OutputTrait; +import software.amazon.smithy.model.traits.RequiredTrait; import software.amazon.smithy.model.traits.SensitiveTrait; import software.amazon.smithy.model.traits.StreamingTrait; import software.amazon.smithy.python.codegen.CodegenUtils; @@ -316,8 +318,14 @@ private boolean hasDocs() { private void writeMemberDocs(MemberShape member) { member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> { + String descriptionPrefix = ""; + if (member.hasTrait(RequiredTrait.class) && !member.hasTrait(ClientOptionalTrait.class)) { + descriptionPrefix = "**[Required]** - "; + } + String memberName = symbolProvider.toMemberName(member); - String docs = writer.formatDocs(String.format(":param %s: %s", memberName, trait.getValue())); + String docs = writer.formatDocs(String.format(":param %s: %s%s", + memberName, descriptionPrefix, trait.getValue())); writer.write(docs); }); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java index 811d938c..52415c8d 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java @@ -6,6 +6,8 @@ import static org.jsoup.nodes.Document.OutputSettings.Syntax.html; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.commonmark.node.BlockQuote; import org.commonmark.node.FencedCodeBlock; import org.commonmark.node.Heading; @@ -21,6 +23,7 @@ import org.jsoup.nodes.TextNode; import org.jsoup.select.NodeVisitor; import software.amazon.smithy.utils.SetUtils; +import software.amazon.smithy.utils.SimpleCodeWriter; import software.amazon.smithy.utils.SmithyInternalApi; /** @@ -51,8 +54,11 @@ public static MarkdownToRstDocConverter getInstance() { } public String convertCommonmarkToRst(String commonmark) { - String html = - HtmlRenderer.builder().escapeHtml(false).build().render(MARKDOWN_PARSER.parse(commonmark)); + String html = HtmlRenderer.builder().escapeHtml(false).build().render(MARKDOWN_PARSER.parse(commonmark)); + //Replace the outer HTML paragraph tag with a div tag + Pattern pattern = Pattern.compile("^

(.*)

$", Pattern.DOTALL); + Matcher matcher = pattern.matcher(html); + html = matcher.replaceAll("
$1
"); Document document = Jsoup.parse(html); RstNodeVisitor visitor = new RstNodeVisitor(); document.body().traverse(visitor); @@ -60,8 +66,7 @@ public String convertCommonmarkToRst(String commonmark) { } private static class RstNodeVisitor implements NodeVisitor { - //TODO migrate away from StringBuilder to use a SimpleCodeWriter - private final StringBuilder sb = new StringBuilder(); + SimpleCodeWriter writer = new SimpleCodeWriter(); private boolean inList = false; private int listDepth = 0; @@ -71,47 +76,73 @@ public void head(Node node, int depth) { TextNode textNode = (TextNode) node; String text = textNode.text(); if (!text.trim().isEmpty()) { - sb.append(text); - // Account for services making a paragraph tag that's empty except - // for a newline + if (text.startsWith(":param ")) { + int secondColonIndex = text.indexOf(':', 1); + writer.write(text.substring(0, secondColonIndex + 1)); + //TODO right now the code generator gives us a mixture of + // RST and HTML (for instance :param xyz:

docs + //

). Since we standardize to html above, that

tag + // starts a newline. We account for that with this if/else + // statement, but we should refactor this in the future to + // have a more elegant codepath. + if (secondColonIndex +1 == text.strip().length()) { + writer.indent(); + writer.ensureNewline(); + } else { + writer.ensureNewline(); + writer.indent(); + writer.write(text.substring(secondColonIndex + 1)); + writer.dedent(); + } + } else { + writer.writeInline(text); + } + // Account for services making a paragraph tag that's empty except + // for a newline } else if (node.parent() instanceof Element && ((Element) node.parent()).tagName().equals("p")) { - sb.append(text.replaceAll("[ \\t]+", "")); + writer.writeInline(text.replaceAll("[ \\t]+", "")); } } else if (node instanceof Element) { Element element = (Element) node; switch (element.tagName()) { case "a": - sb.append("`"); + writer.writeInline("`"); break; case "b": case "strong": - sb.append("**"); + writer.writeInline("**"); break; case "i": case "em": - sb.append("*"); + writer.writeInline("*"); break; case "code": - sb.append("``"); + writer.writeInline("``"); break; case "important": - sb.append("\n.. important::\n "); + writer.ensureNewline(); + writer.write(""); + writer.openBlock(".. important::"); break; case "note": - sb.append("\n.. note::\n "); + writer.ensureNewline(); + writer.write(""); + writer.openBlock(".. note::"); break; case "ul": + if (inList) { + writer.indent(); + } inList = true; listDepth++; - sb.append("\n"); + writer.ensureNewline(); + writer.write(""); break; case "li": - if (inList) { - sb.append(" ".repeat(listDepth - 1)).append("* "); - } + writer.writeInline("* "); break; case "h1": - sb.append("\n"); + writer.ensureNewline(); break; default: break; @@ -125,41 +156,47 @@ public void tail(Node node, int depth) { Element element = (Element) node; switch (element.tagName()) { case "a": - sb.append(" <").append(element.attr("href")).append(">`_"); + String href = element.attr("href"); + if (!href.isEmpty()) { + writer.writeInline(" <").writeInline(href).writeInline(">`_"); + } else { + writer.writeInline("`"); + } break; case "b": case "strong": - sb.append("**"); + writer.writeInline("**"); break; case "i": case "em": - sb.append("*"); + writer.writeInline("*"); break; case "code": - sb.append("``"); + writer.writeInline("``"); break; case "important": case "note": + writer.closeBlock(""); + break; case "p": - sb.append("\n"); + writer.ensureNewline(); + writer.write(""); break; case "ul": listDepth--; if (listDepth == 0) { inList = false; + } else { + writer.dedent(); } - if (sb.charAt(sb.length() - 1) != '\n') { - sb.append("\n\n"); - } + writer.ensureNewline(); break; case "li": - if (sb.charAt(sb.length() - 1) != '\n') { - sb.append("\n\n"); - } + writer.ensureNewline(); break; case "h1": String title = element.text(); - sb.append("\n").append("=".repeat(title.length())).append("\n"); + writer.ensureNewline().writeInline("=".repeat(title.length())).ensureNewline(); break; default: break; @@ -169,7 +206,7 @@ public void tail(Node node, int depth) { @Override public String toString() { - return sb.toString(); + return writer.toString(); } } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java index 0e590113..6d3ecac9 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java @@ -174,6 +174,9 @@ private static void wrapLine(String line, StringBuilder wrappedText) { while (line.length() > MAX_LINE_LENGTH) { int wrapAt = findWrapPosition(line, MAX_LINE_LENGTH); wrappedText.append(indentStr).append(line, 0, wrapAt).append("\n"); + if (line.startsWith("* ")) { + indentStr += " "; + } line = line.substring(wrapAt).trim(); if (line.isEmpty()) { return; @@ -190,6 +193,7 @@ private static int findWrapPosition(String line, int maxLineLength) { wrapAt = line.length(); } else { // Ensure we don't break a link + //TODO account for earlier backticks on the same line as a link int linkStart = line.lastIndexOf("`", wrapAt); int linkEnd = line.indexOf("`_", wrapAt); if (linkStart != -1 && (linkEnd != -1 && linkEnd > linkStart)) {