diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d594bf6e..ad10133c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,11 @@ jobs: activate-environment: true enable-cache: true + - name: Install pandoc + uses: pandoc/actions/setup@86321b6dd4675f5014c611e05088e10d4939e09e + with: + version: 3.8.2 + - name: Setup workspace run: | make install diff --git a/README.md b/README.md index 502d20217..a8422154d 100644 --- a/README.md +++ b/README.md @@ -112,12 +112,13 @@ With both files your project directory should look like this: The code generator libraries have not been published yet, so you'll need to build it yourself. To build and run the generator, you will need -the following prerequisites: +the following prerequisites installed in your environment: * [uv](https://docs.astral.sh/uv/) * The [Smithy CLI](https://smithy.io/2.0/guides/smithy-cli/cli_installation.html) * JDK 17 or newer * make +* [pandoc](https://pandoc.org/installing.html) CLI This project uses [uv](https://docs.astral.sh/uv/) for managing all things python. Once you have it installed, run the following command to check that it's ready to use: @@ -169,6 +170,16 @@ if __name__ == "__main__": asyncio.run(main()) ``` +#### pandoc CLI + +Smithy [documentation traits](https://smithy.io/2.0/spec/documentation-traits.html#documentation-trait) are modeled in one of two formats: + +- **Raw HTML** for AWS services +- **CommonMark** for all other Smithy-based services (may include embedded HTML) + +The code generator uses [pandoc](https://pandoc.org/) to normalize and convert this +content into Markdown suitable for Google-style Python docstrings. + #### Is Java really required? Only for now. Once the generator has been published, the Smithy CLI will be able diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java deleted file mode 100644 index 0141801b3..000000000 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.aws.codegen; - -import static software.amazon.smithy.python.codegen.SymbolProperties.OPERATION_METHOD; - -import java.util.List; -import software.amazon.smithy.model.traits.InputTrait; -import software.amazon.smithy.model.traits.OutputTrait; -import software.amazon.smithy.python.codegen.GenerationContext; -import software.amazon.smithy.python.codegen.integrations.PythonIntegration; -import software.amazon.smithy.python.codegen.sections.*; -import software.amazon.smithy.python.codegen.writer.PythonWriter; -import software.amazon.smithy.utils.CodeInterceptor; -import software.amazon.smithy.utils.CodeSection; - -public class AwsRstDocFileGenerator implements PythonIntegration { - - @Override - public List> interceptors( - GenerationContext context - ) { - return List.of( - // We generate custom RST files for each member that we want to have - // its own page. This gives us much more fine-grained control of - // what gets generated than just using automodule or autoclass on - // the client would alone. - new OperationGenerationInterceptor(context), - new StructureGenerationInterceptor(context), - new ErrorGenerationInterceptor(context), - new UnionGenerationInterceptor(context), - new UnionMemberGenerationInterceptor(context)); - } - - /** - * Utility method to generate a header for documentation files. - * - * @param title The title of the section. - * @return A formatted header string. - */ - private static String generateHeader(String title) { - return String.format("%s%n%s%n%n", title, "=".repeat(title.length())); - } - - private static final class OperationGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public OperationGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return OperationSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, OperationSection section) { - var operation = section.operation(); - var operationSymbol = context.symbolProvider().toSymbol(operation).expectProperty(OPERATION_METHOD); - var input = context.model().expectShape(operation.getInputShape()); - var inputSymbol = context.symbolProvider().toSymbol(input); - var output = context.model().expectShape(operation.getOutputShape()); - var outputSymbol = context.symbolProvider().toSymbol(output); - - String operationName = operationSymbol.getName(); - String inputSymbolName = inputSymbol.toString(); - String outputSymbolName = outputSymbol.toString(); - String serviceName = context.symbolProvider().toSymbol(section.service()).getName(); - String docsFileName = String.format("docs/client/%s.rst", operationName); - String fullOperationReference = String.format("%s.client.%s.%s", - context.settings().moduleName(), - serviceName, - operationName); - - context.writerDelegator().useFileWriter(docsFileName, "", fileWriter -> { - fileWriter.write(generateHeader(operationName)); - fileWriter.write(".. automethod:: " + fullOperationReference + "\n\n"); - fileWriter.write(".. toctree::\n :hidden:\n :maxdepth: 2\n\n"); - fileWriter.write("=================\nInput:\n=================\n\n"); - fileWriter.write(".. autoclass:: " + inputSymbolName + "\n :members:\n"); - fileWriter.write("=================\nOutput:\n=================\n\n"); - fileWriter.write(".. autoclass:: " + outputSymbolName + "\n :members:\n"); - }); - } - } - - private static final class StructureGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public StructureGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return StructureSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, StructureSection section) { - var shape = section.structure(); - var symbol = context.symbolProvider().toSymbol(shape); - String docsFileName = String.format("docs/models/%s.rst", - symbol.getName()); - if (!shape.hasTrait(InputTrait.class) && !shape.hasTrait(OutputTrait.class)) { - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(generateHeader(symbol.getName())); - writer.write(".. autoclass:: " + symbol.toString() + "\n :members:\n"); - }); - } - } - } - - private static final class ErrorGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public ErrorGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return ErrorSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, ErrorSection section) { - var symbol = section.errorSymbol(); - String docsFileName = String.format("docs/models/%s.rst", - symbol.getName()); - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(generateHeader(symbol.getName())); - writer.write(".. autoexception:: " + symbol.toString() + "\n :members:\n :show-inheritance:\n"); - }); - } - } - - private static final class UnionGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public UnionGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return UnionSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, UnionSection section) { - String parentName = section.parentName(); - String docsFileName = String.format("docs/models/%s.rst", parentName); - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(".. _" + parentName + ":\n\n"); - writer.write(generateHeader(parentName)); - writer.write( - ".. autodata:: " + context.symbolProvider().toSymbol(section.unionShape()).toString() + " \n"); - }); - } - } - - private static final class UnionMemberGenerationInterceptor - implements CodeInterceptor.Appender { - - private final GenerationContext context; - - public UnionMemberGenerationInterceptor(GenerationContext context) { - this.context = context; - } - - @Override - public Class sectionType() { - return UnionMemberSection.class; - } - - @Override - public void append(PythonWriter pythonWriter, UnionMemberSection section) { - var memberSymbol = section.memberSymbol(); - String symbolName = memberSymbol.getName(); - String docsFileName = String.format("docs/models/%s.rst", symbolName); - context.writerDelegator().useFileWriter(docsFileName, "", writer -> { - writer.write(".. _" + symbolName + ":\n\n"); - writer.write(generateHeader(symbolName)); - writer.write(".. autoclass:: " + memberSymbol.toString() + " \n"); - }); - } - } -} diff --git a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index 2a8cfe6d6..a338df30c 100644 --- a/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -8,4 +8,3 @@ software.amazon.smithy.python.aws.codegen.AwsProtocolsIntegration software.amazon.smithy.python.aws.codegen.AwsServiceIdIntegration software.amazon.smithy.python.aws.codegen.AwsUserAgentIntegration software.amazon.smithy.python.aws.codegen.AwsStandardRegionalEndpointsIntegration -software.amazon.smithy.python.aws.codegen.AwsRstDocFileGenerator diff --git a/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java b/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java deleted file mode 100644 index 8f4a7bd73..000000000 --- a/codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.aws.codegen; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import software.amazon.smithy.python.codegen.writer.MarkdownToRstDocConverter; - -public class MarkdownToRstDocConverterTest { - - private MarkdownToRstDocConverter markdownToRstDocConverter; - - @BeforeEach - public void setUp() { - markdownToRstDocConverter = MarkdownToRstDocConverter.getInstance(); - } - - @Test - public void testConvertCommonmarkToRstWithTitleAndParagraph() { - String html = "

Title

Paragraph

"; - String expected = "Title\n=====\nParagraph"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithImportantNote() { - String html = "Important note"; - String expected = ".. important::\n Important note"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithList() { - String html = "
  • Item 1
  • Item 2
"; - String expected = "* Item 1\n* Item 2"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithMixedElements() { - String html = "

Title

Paragraph

  • Item 1
  • Item 2
"; - String expected = "Title\n=====\nParagraph\n\n\n* Item 1\n* Item 2"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithNestedElements() { - String html = "

Title

Paragraph with bold text

"; - String expected = "Title\n=====\nParagraph with **bold** text"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithAnchorTag() { - String html = "Link"; - String expected = "`Link `_"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithBoldTag() { - String html = "Bold text"; - String expected = "**Bold text**"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithItalicTag() { - String html = "Italic text"; - String expected = "*Italic text*"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithCodeTag() { - String html = "code snippet"; - String expected = "``code snippet``"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithNoteTag() { - String html = "Note text"; - String expected = ".. note::\n Note text"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithNestedList() { - String html = "
  • Item 1
    • Subitem 1
  • Item 2
"; - String expected = "* Item 1\n\n * Subitem 1\n* Item 2"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } - - @Test - public void testConvertCommonmarkToRstWithFormatSpecifierCharacters() { - // Test that Smithy format specifier characters ($) are properly escaped and treated as literal text - String html = "

Testing $placeholder_one and $placeholder_two

"; - String expected = "Testing $placeholder_one and $placeholder_two"; - String result = markdownToRstDocConverter.convertCommonmarkToRst(html); - assertEquals(expected, result.trim()); - } -} diff --git a/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts b/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts index f50917c67..9dd1a1f95 100644 --- a/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts +++ b/codegen/buildSrc/src/main/kotlin/smithy-python.java-conventions.gradle.kts @@ -39,6 +39,7 @@ dependencies { testRuntimeOnly(libs.junit.jupiter.engine) testRuntimeOnly(libs.junit.platform.launcher) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) compileOnly("com.github.spotbugs:spotbugs-annotations:${spotbugs.toolVersion.get()}") testCompileOnly("com.github.spotbugs:spotbugs-annotations:${spotbugs.toolVersion.get()}") } diff --git a/codegen/core/build.gradle.kts b/codegen/core/build.gradle.kts index 04f9064dd..bc8da8191 100644 --- a/codegen/core/build.gradle.kts +++ b/codegen/core/build.gradle.kts @@ -15,6 +15,4 @@ dependencies { implementation(libs.smithy.protocol.test.traits) // We have this because we're using RestJson1 as a 'generic' protocol. implementation(libs.smithy.aws.traits) - implementation(libs.jsoup) - implementation(libs.commonmark) } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 95c6e0d5a..01630c1a8 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -22,8 +22,6 @@ import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.python.codegen.integrations.PythonIntegration; import software.amazon.smithy.python.codegen.integrations.RuntimeClientPlugin; -import software.amazon.smithy.python.codegen.sections.*; -import software.amazon.smithy.python.codegen.writer.MarkdownToRstDocConverter; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -59,19 +57,8 @@ private void generateService(PythonWriter writer) { writer.openBlock("class $L:", "", serviceSymbol.getName(), () -> { var docs = service.getTrait(DocumentationTrait.class) .map(StringTrait::getValue) - .orElse("Client for " + serviceSymbol.getName()); - String rstDocs = - MarkdownToRstDocConverter.getInstance().convertCommonmarkToRst(docs); - writer.writeDocs(() -> { - writer.write(""" - $L - - :param config: Optional configuration for the client. Here you can set things like the - endpoint for HTTP services or auth credentials. - - :param plugins: A list of callables that modify the configuration dynamically. These - can be used to set defaults, for example.""", rstDocs); - }); + .orElse("Client for " + service.getId().getName()); + writer.writeDocs(docs, context); var defaultPlugins = new LinkedHashSet(); @@ -87,10 +74,11 @@ private void generateService(PythonWriter writer) { writer.addImport("smithy_core.retries", "RetryStrategyResolver"); writer.write(""" def __init__(self, config: $1T | None = None, plugins: list[$2T] | None = None): + $3C self._config = config or $1T() client_plugins: list[$2T] = [ - $3C + $4C ] if plugins: client_plugins.extend(plugins) @@ -99,7 +87,11 @@ def __init__(self, config: $1T | None = None, plugins: list[$2T] | None = None): plugin(self._config) self._retry_strategy_resolver = RetryStrategyResolver() - """, configSymbol, pluginSymbol, writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); + """, + configSymbol, + pluginSymbol, + writer.consumer(w -> writeConstructorDocs(w, serviceSymbol.getName())), + writer.consumer(w -> writeDefaultPlugins(w, defaultPlugins))); var topDownIndex = TopDownIndex.of(model); var eventStreamIndex = EventStreamIndex.of(model); @@ -120,6 +112,22 @@ private void writeDefaultPlugins(PythonWriter writer, Collection { + writer.write(""" + Constructor for `$L`. + + Args: + config: + Optional configuration for the client. Here you can set things like + the endpoint for HTTP services or auth credentials. + plugins: + A list of callables that modify the configuration dynamically. These + can be used to set defaults, for example. + """, clientName); + }); + } + /** * Generates the function for a single operation. */ @@ -134,7 +142,6 @@ private void generateOperation(PythonWriter writer, OperationShape operation) { var output = model.expectShape(operation.getOutputShape()); var outputSymbol = symbolProvider.toSymbol(output); - writer.pushState(new OperationSection(service, operation)); writer.putContext("input", inputSymbol); writer.putContext("output", outputSymbol); writer.putContext("plugin", pluginSymbol); @@ -148,32 +155,48 @@ private void generateOperation(PythonWriter writer, OperationShape operation) { ${C|} return await pipeline(call) """, - writer.consumer(w -> writeSharedOperationInit(w, operation, input))); - writer.popState(); + writer.consumer(w -> writeSharedOperationInit(w, operation, input, output))); + } + + private void writeSharedOperationInit(PythonWriter writer, OperationShape operation, Shape input, Shape output) { + writeSharedOperationInit(writer, operation, input, output, null); } - private void writeSharedOperationInit(PythonWriter writer, OperationShape operation, Shape input) { - writer.writeDocs(() -> { - var docs = writer.formatDocs(operation.getTrait(DocumentationTrait.class) + private void writeSharedOperationInit( + PythonWriter writer, + OperationShape operation, + Shape input, + Shape output, + String eventStreamOutputDocs + ) { + writer.writeMultiLineDocs(() -> { + var operationDocs = writer.formatDocs(operation.getTrait(DocumentationTrait.class) .map(StringTrait::getValue) .orElse(String.format("Invokes the %s operation.", - operation.getId().getName()))); + operation.getId().getName())), + context); - var inputDocs = input.getTrait(DocumentationTrait.class) - .map(StringTrait::getValue) - .orElse("The operation's input."); + var inputSymbolName = symbolProvider.toSymbol(input).getName(); + var outputSymbolName = symbolProvider.toSymbol(output).getName(); + var inputDocs = String.format("An instance of `%s`.", inputSymbolName); + var outputDocs = eventStreamOutputDocs != null ? eventStreamOutputDocs + : String.format("An instance of `%s`.", outputSymbolName); writer.write(""" $L - """, docs); - writer.write(""); - writer.write(":param input: $L", inputDocs); - writer.write(""); - writer.write(""" - :param plugins: A list of callables that modify the configuration dynamically. - Changes made by these plugins only apply for the duration of the operation - execution and will not affect any other operation invocations."""); + Args: + input: + $L + plugins: + A list of callables that modify the configuration dynamically. + Changes made by these plugins only apply for the duration of the + operation execution and will not affect any other operation + invocations. + + Returns: + ${L|} + """, operationDocs, inputDocs, outputDocs); }); var defaultPlugins = new LinkedHashSet(); @@ -243,7 +266,6 @@ raise TypeError( } private void generateEventStreamOperation(PythonWriter writer, OperationShape operation) { - writer.pushState(new OperationSection(service, operation)); writer.addDependency(SmithyPythonDependency.SMITHY_CORE); writer.addDependency(SmithyPythonDependency.SMITHY_AWS_CORE.withOptionalDependencies("eventstream")); var operationSymbol = symbolProvider.toSymbol(operation); @@ -283,6 +305,7 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op if (inputStreamSymbol.isPresent()) { if (outputStreamSymbol.isPresent()) { writer.addImport("smithy_core.aio.eventstream", "DuplexEventStream"); + var outputDocs = "A `DuplexEventStream` for bidirectional streaming."; writer.write(""" async def ${operationName:L}( self, @@ -297,9 +320,10 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op ${outputStreamDeserializer:T}().deserialize ) """, - writer.consumer(w -> writeSharedOperationInit(w, operation, input))); + writer.consumer(w -> writeSharedOperationInit(w, operation, input, output, outputDocs))); } else { writer.addImport("smithy_core.aio.eventstream", "InputEventStream"); + var outputDocs = "An `InputEventStream` for client-to-server streaming."; writer.write(""" async def ${operationName:L}( self, @@ -311,10 +335,12 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op call, ${inputStream:T} ) - """, writer.consumer(w -> writeSharedOperationInit(w, operation, input))); + """, + writer.consumer(w -> writeSharedOperationInit(w, operation, input, output, outputDocs))); } } else { writer.addImport("smithy_core.aio.eventstream", "OutputEventStream"); + var outputDocs = "An `OutputEventStream` for server-to-client streaming."; writer.write(""" async def ${operationName:L}( self, @@ -328,9 +354,7 @@ private void generateEventStreamOperation(PythonWriter writer, OperationShape op ${outputStreamDeserializer:T}().deserialize ) """, - writer.consumer(w -> writeSharedOperationInit(w, operation, input))); + writer.consumer(w -> writeSharedOperationInit(w, operation, input, output, outputDocs))); } - - writer.popState(); } } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java index 994fc9fad..b039dd582 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java @@ -280,6 +280,17 @@ private static ZonedDateTime parseHttpDate(Node value) { return instant.atZone(ZoneId.of("UTC")); } + /** + * Determines whether the service being generated is an AWS service. + * + * @param context The generation context. + * @return Returns true if the service is an AWS service, false otherwise. + */ + public static boolean isAwsService(GenerationContext context) { + var service = context.model().expectShape(context.settings().service()); + return service.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class); + } + /** * Writes an accessor for a structure member, handling defaultedness and nullability. * diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java index 62330fa78..0a06f15bf 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java @@ -587,7 +587,7 @@ private void writeTestBlock( writer.write("@mark.xfail()"); } writer.openBlock("async def test_$L() -> None:", "", CaseUtils.toSnakeCase(testName), () -> { - testCase.getDocumentation().ifPresent(writer::writeDocs); + testCase.getDocumentation().ifPresent(docs -> writer.writeDocs(docs, context)); f.run(); }); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index 71fde8b10..c8400568d 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -100,24 +100,6 @@ public final class SmithyPythonDependency { Type.TEST_DEPENDENCY, false); - /** - * library used for documentation generation - */ - public static final PythonDependency SPHINX = new PythonDependency( - "sphinx", - ">=8.2.3", - Type.DOCS_DEPENDENCY, - false); - - /** - * sphinx theme - */ - public static final PythonDependency SPHINX_PYDATA_THEME = new PythonDependency( - "pydata-sphinx-theme", - ">=0.16.1", - Type.DOCS_DEPENDENCY, - false); - private SmithyPythonDependency() {} /** diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java index b17847ba9..de03d42d0 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ConfigGenerator.java @@ -274,8 +274,8 @@ public void run() { context.writerDelegator().useFileWriter(plugin.getDefinitionFile(), plugin.getNamespace(), writer -> { writer.addStdlibImport("typing", "Callable"); writer.addStdlibImport("typing", "TypeAlias"); - writer.writeComment("A callable that allows customizing the config object on each request."); writer.write("$L: TypeAlias = Callable[[$T], None]", plugin.getName(), config); + writer.writeDocs("A callable that allows customizing the config object on each request.", context); }); } @@ -355,13 +355,11 @@ def __init__( ${C|} ): ${C|} - ${C|} """, configSymbol.getName(), serviceId, writer.consumer(w -> writePropertyDeclarations(w, finalProperties)), writer.consumer(w -> writeInitParams(w, finalProperties)), - writer.consumer(w -> documentProperties(w, finalProperties)), writer.consumer(w -> initializeProperties(w, finalProperties))); writer.popState(); } @@ -372,6 +370,8 @@ private void writePropertyDeclarations(PythonWriter writer, Collection pro } } - private void documentProperties(PythonWriter writer, Collection properties) { - writer.writeDocs(() -> { - var iter = properties.iterator(); - writer.write("\nConstructor.\n"); - while (iter.hasNext()) { - var property = iter.next(); - var docs = writer.formatDocs(String.format(":param %s: %s", property.name(), property.documentation())); - - if (iter.hasNext()) { - docs += "\n"; - } - - writer.write(docs); - } - }); - } - private void initializeProperties(PythonWriter writer, Collection properties) { for (ConfigProperty property : properties) { property.initialize(writer); @@ -422,11 +405,14 @@ public void write(PythonWriter writer, String previousText, ConfigSection sectio writer.write(""" def set_auth_scheme(self, scheme: AuthScheme[Any, Any, Any, Any]) -> None: - \"""Sets the implementation of an auth scheme. + \""" + Sets the implementation of an auth scheme. Using this method ensures the correct key is used. - :param scheme: The auth scheme to add. + Args: + scheme: + The auth scheme to add. \""" self.auth_schemes[scheme.scheme_id] = scheme """); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java index 8f0d2f804..354054503 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java @@ -32,14 +32,16 @@ public void run() { writer.addStdlibImport("enum", "StrEnum"); writer.openBlock("class $L(StrEnum):", "", enumSymbol.getName(), () -> { shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { - writer.writeDocs(writer.formatDocs(trait.getValue())); + writer.writeDocs(trait.getValue(), context); }); for (MemberShape member : shape.members()) { var name = context.symbolProvider().toMemberName(member); var value = member.expectTrait(EnumValueTrait.class).expectStringValue(); writer.write("$L = $S", name, value); - member.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeDocs(trait.getValue())); + member.getTrait(DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(trait.getValue(), context); + }); } }); }); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java index 7fbb6bb5d..8816f9d38 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java @@ -29,14 +29,16 @@ public void run() { writer.addStdlibImport("enum", "IntEnum"); writer.openBlock("class $L(IntEnum):", "", enumSymbol.getName(), () -> { directive.shape().getTrait(DocumentationTrait.class).ifPresent(trait -> { - writer.writeDocs(writer.formatDocs(trait.getValue())); + writer.writeDocs(trait.getValue(), directive.context()); }); for (MemberShape member : directive.shape().members()) { - member.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeComment(trait.getValue())); var name = directive.symbolProvider().toMemberName(member); var value = member.expectTrait(EnumValueTrait.class).expectIntValue(); writer.write("$L = $L\n", name, value); + member.getTrait(DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(trait.getValue(), directive.context()); + }); } }); }); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java index 3b6a062ae..a4fcf04c1 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ServiceErrorGenerator.java @@ -32,7 +32,8 @@ public void run() { writer.addImport("smithy_core.exceptions", "ModeledError"); writer.write(""" class $L(ModeledError): - ""\"Base error for all errors in the service. + ""\" + Base error for all errors in the service. Some exceptions do not extend from this class, including synthetic, implicit, and shared exception types. diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java index 343fccfd0..6e88bae27 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java @@ -14,7 +14,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import software.amazon.smithy.aws.traits.ServiceTrait; import software.amazon.smithy.codegen.core.CodegenException; import software.amazon.smithy.codegen.core.SymbolDependency; import software.amazon.smithy.codegen.core.WriterDelegator; @@ -40,7 +39,6 @@ public static void generateSetup( PythonSettings settings, GenerationContext context ) { - writeDocsSkeleton(settings, context); var dependencies = gatherDependencies(context.writerDelegator().getDependencies().stream()); writePyproject(settings, context.writerDelegator(), dependencies); writeReadme(settings, context); @@ -149,11 +147,7 @@ private static void writePyproject( Optional.ofNullable(dependencies.get(PythonDependency.Type.TEST_DEPENDENCY.getType())) .map(Map::values); - Optional> docsDeps = - Optional.ofNullable(dependencies.get(PythonDependency.Type.DOCS_DEPENDENCY.getType())) - .map(Map::values); - - if (testDeps.isPresent() || docsDeps.isPresent()) { + if (testDeps.isPresent()) { writer.write("[dependency-groups]"); } @@ -161,22 +155,12 @@ private static void writePyproject( writer.openBlock("test = [", "]\n", () -> writeDependencyList(writer, deps)); }); - docsDeps.ifPresent(deps -> { - writer.openBlock("docs = [", "]\n", () -> writeDependencyList(writer, deps)); - }); - // TODO: remove the pyright global suppressions after the serde redo is done writer.write(""" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" - [tool.hatch.build.targets.bdist] - exclude = [ - "tests", - "docs", - ] - [tool.pyright] typeCheckingMode = "strict" reportPrivateUsage = false @@ -269,238 +253,9 @@ private static void writeReadme( ### Documentation $L - """, documentation); + """, writer.formatDocs(documentation, context)); }); writer.popState(); }); } - - /** - * Write the files required for sphinx doc generation - */ - private static void writeDocsSkeleton( - PythonSettings settings, - GenerationContext context - ) { - //TODO Add a configurable flag to disable the generation of the sphinx files - //TODO Add a configuration that will allow users to select a sphinx theme - context.writerDelegator().useFileWriter("pyproject.toml", "", writer -> { - writer.addDependency(SmithyPythonDependency.SPHINX); - writer.addDependency(SmithyPythonDependency.SPHINX_PYDATA_THEME); - }); - var service = context.model().expectShape(settings.service()); - String projectName = service.getTrait(TitleTrait.class) - .map(StringTrait::getValue) - .orElseGet(() -> service.getTrait(ServiceTrait.class) - .map(ServiceTrait::getSdkId) - .orElse(context.settings().service().getName())); - writeConf(settings, context, projectName); - writeIndexes(context, projectName); - writeDocsReadme(context); - writeMakeBat(context); - writeMakeFile(context); - } - - /** - * Write a conf.py file. - * A conf.py file is a configuration file used by Sphinx, a documentation - * generation tool for Python projects. This file contains settings and - * configurations that control the behavior and appearance of the generated - * documentation. - */ - private static void writeConf( - PythonSettings settings, - GenerationContext context, - String projectName - ) { - String version = settings.moduleVersion(); - context.writerDelegator().useFileWriter("docs/conf.py", "", writer -> { - writer.write(""" - import os - import sys - sys.path.insert(0, os.path.abspath('..')) - - project = '$L' - author = 'Amazon Web Services' - release = '$L' - - extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - ] - - templates_path = ['_templates'] - exclude_patterns = [] - - autodoc_default_options = { - 'exclude-members': 'deserialize,deserialize_kwargs,serialize,serialize_members' - } - - html_theme = 'pydata_sphinx_theme' - html_theme_options = { - "logo": { - "text": "$L", - } - } - - autodoc_typehints = 'description' - """, projectName, version, projectName); - }); - } - - /** - * Write a make.bat file. - * A make.bat file is a batch script used on Windows to build Sphinx documentation. - * This script sets up the environment and runs the Sphinx build commands. - * - * @param context The generation context containing the writer delegator. - */ - private static void writeMakeBat( - GenerationContext context - ) { - context.writerDelegator().useFileWriter("docs/make.bat", "", writer -> { - writer.write(""" - @ECHO OFF - - pushd %~dp0 - - REM Command file for Sphinx documentation - - if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build - ) - set BUILDDIR=build - set SERVICESDIR=source/reference/services - set SPHINXOPTS=-j auto - set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . - - if "%1" == "" goto help - - if "%1" == "clean" ( - rmdir /S /Q %BUILDDIR% - goto end - ) - - if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - echo. - echo "Build finished. The HTML pages are in %BUILDDIR%/html." - goto end - ) - - :help - %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - - :end - popd - """); - }); - } - - /** - * Write a Makefile. - * A Makefile is used on Unix-based systems to build Sphinx documentation. - * This file contains rules for cleaning the build directory and generating HTML documentation. - * - * @param context The generation context containing the writer delegator. - */ - private static void writeMakeFile( - GenerationContext context - ) { - context.writerDelegator().useFileWriter("docs/Makefile", "", writer -> { - writer.write(""" - SPHINXBUILD = sphinx-build - BUILDDIR = build - SERVICESDIR = source/reference/services - SPHINXOPTS = -j auto - ALLSPHINXOPTS = -d $$(BUILDDIR)/doctrees $$(SPHINXOPTS) . - - clean: - \t-rm -rf $$(BUILDDIR)/* - - html: - \t$$(SPHINXBUILD) -b html $$(ALLSPHINXOPTS) $$(BUILDDIR)/html - \t@echo - \t@echo "Build finished. The HTML pages are in $$(BUILDDIR)/html." - """); - }); - } - - /** - * Write the main index files for the documentation. - * This method creates the main index.rst file and additional index files for - * the client and models sections. - * - * @param context The generation context containing the writer delegator. - */ - private static void writeIndexes(GenerationContext context, String projectName) { - // Write the main index file for the documentation - context.writerDelegator().useFileWriter("docs/index.rst", "", writer -> { - writer.write(""" - $L - $L - - .. toctree:: - :maxdepth: 2 - :titlesonly: - :glob: - - */index - """, projectName, "=".repeat(projectName.length())); - }); - - // Write the index file for the client section - writeIndexFile(context, "docs/client/index.rst", "Client"); - - // Write the index file for the models section - writeIndexFile(context, "docs/models/index.rst", "Models"); - } - - /** - * Write the readme in the docs folder describing instructions for generation - * - * @param context The generation context containing the writer delegator. - */ - private static void writeDocsReadme( - GenerationContext context - ) { - context.writerDelegator().useFileWriter("docs/README.md", writer -> { - writer.write(""" - ## Generating Documentation - - Sphinx is used for documentation. You can generate HTML locally with the - following: - - ``` - $$ uv pip install --group docs . - $$ cd docs - $$ make html - ``` - """); - }); - } - - /** - * Helper method to write an index file with the given title. - * This method creates an index file at the specified file path with the provided title. - * - * @param context The generation context. - * @param filePath The file path of the index file. - * @param title The title of the index file. - */ - private static void writeIndexFile(GenerationContext context, String filePath, String title) { - context.writerDelegator().useFileWriter(filePath, "", writer -> { - writer.write(""" - $L - ======= - .. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - * - """, title); - }); - } - } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java index 498938f36..55d4daf67 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java @@ -20,11 +20,9 @@ import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.StructureShape; -import software.amazon.smithy.model.traits.ClientOptionalTrait; import software.amazon.smithy.model.traits.DefaultTrait; import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.ErrorTrait; -import software.amazon.smithy.model.traits.RequiredTrait; import software.amazon.smithy.model.traits.RetryableTrait; import software.amazon.smithy.model.traits.SensitiveTrait; import software.amazon.smithy.model.traits.StreamingTrait; @@ -32,8 +30,6 @@ import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; import software.amazon.smithy.python.codegen.SymbolProperties; -import software.amazon.smithy.python.codegen.sections.ErrorSection; -import software.amazon.smithy.python.codegen.sections.StructureSection; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -97,8 +93,6 @@ public void run() { private void renderStructure() { writer.addStdlibImport("dataclasses", "dataclass"); var symbol = symbolProvider.toSymbol(shape); - writer.pushState(new StructureSection(shape)); - writer.write(""" @dataclass(kw_only=True) class $L: @@ -112,12 +106,10 @@ class $L: """, symbol.getName(), - writer.consumer(this::writeClassDocs), + writer.consumer(w -> writeClassDocs()), writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), writer.consumer(w -> generateDeserializeMethod())); - - writer.popState(); } private void renderError() { @@ -128,7 +120,6 @@ private void renderError() { var fault = errorTrait.getValue(); var symbol = symbolProvider.toSymbol(shape); var baseError = CodegenUtils.getServiceError(settings); - writer.pushState(new ErrorSection(symbol)); writer.putContext("retryable", false); writer.putContext("throttling", false); @@ -160,12 +151,17 @@ class $1L($2T): symbol.getName(), baseError, fault, - writer.consumer(this::writeClassDocs), + writer.consumer(w -> writeClassDocs()), writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), writer.consumer(w -> generateDeserializeMethod())); - writer.popState(); + } + private void writeClassDocs() { + var docs = shape.getTrait(DocumentationTrait.class) + .map(DocumentationTrait::getValue) + .orElse("Dataclass for " + shape.getId().getName() + " structure."); + writer.writeDocs(docs, context); } private void writeProperties() { @@ -179,21 +175,17 @@ private void writeProperties() { } else { writer.putContext("sensitive", false); } - var docs = member.getMemberTrait(model, DocumentationTrait.class) - .map(DocumentationTrait::getValue) - .map(writer::formatDocs) - .orElse(null); - writer.putContext("docs", docs); var memberName = symbolProvider.toMemberName(member); writer.putContext("quote", recursiveShapes.contains(target) ? "'" : ""); writer.write(""" $L: ${quote:L}$T${quote:L}\ ${?sensitive} = field(repr=False)${/sensitive} - ${?docs}""\"${docs:L}""\"${/docs} + $C """, memberName, - symbolProvider.toSymbol(member)); + symbolProvider.toSymbol(member), + writer.consumer(w -> writeMemberDocs(member))); writer.popState(); } @@ -227,11 +219,6 @@ private void writeProperties() { writer.putContext("defaultKey", defaultKey); writer.putContext("defaultValue", defaultValue); writer.putContext("useField", requiresField); - var docs = member.getMemberTrait(model, DocumentationTrait.class) - .map(DocumentationTrait::getValue) - .map(writer::formatDocs) - .orElse(null); - writer.putContext("docs", docs); writer.putContext("quote", recursiveShapes.contains(target) ? "'" : ""); @@ -242,15 +229,18 @@ private void writeProperties() { ${?useField}\ field(${?sensitive}repr=False, ${/sensitive}${defaultKey:L}=${defaultValue:L})\ ${/useField} - ${?docs}""\"${docs:L}""\"${/docs} - """, memberName, symbolProvider.toSymbol(member)); + $C + """, + memberName, + symbolProvider.toSymbol(member), + writer.consumer(w -> writeMemberDocs(member))); writer.popState(); } } - private void writeClassDocs(PythonWriter writer) { - shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { - writer.writeDocs(writer.formatDocs(trait.getValue()).trim()); + private void writeMemberDocs(MemberShape member) { + member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(trait.getValue(), context); }); } @@ -304,34 +294,6 @@ private String getDefaultValue(PythonWriter writer, MemberShape member) { }; } - private boolean hasDocs() { - if (shape.hasTrait(DocumentationTrait.class)) { - return true; - } - for (MemberShape member : shape.members()) { - if (member.getMemberTrait(model, DocumentationTrait.class).isPresent()) { - return true; - } - } - return false; - } - - private void writeMemberDocs(MemberShape member) { - member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> { - String descriptionPrefix = ""; - if (member.hasTrait(RequiredTrait.class) && !member.hasTrait(ClientOptionalTrait.class)) { - descriptionPrefix = "**[Required]** - "; - } - - String memberName = symbolProvider.toMemberName(member); - String docs = writer.formatDocs(String.format(":param %s: %s%s", - memberName, - descriptionPrefix, - trait.getValue())); - writer.write(docs); - }); - } - private void generateSerializeMethod() { writer.pushState(); writer.addImport("smithy_core.serializers", "ShapeSerializer"); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java index c24c6590f..b3beb2cf3 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java @@ -15,8 +15,6 @@ import software.amazon.smithy.model.traits.StringTrait; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.SymbolProperties; -import software.amazon.smithy.python.codegen.sections.UnionMemberSection; -import software.amazon.smithy.python.codegen.sections.UnionSection; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -64,7 +62,6 @@ public void run() { var target = model.expectShape(member.getTarget()); var targetSymbol = symbolProvider.toSymbol(target); - writer.pushState(new UnionMemberSection(memberSymbol)); writer.write(""" @dataclass class $1L: @@ -86,7 +83,7 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: memberSymbol.getName(), writer.consumer(w -> member.getMemberTrait(model, DocumentationTrait.class) .map(StringTrait::getValue) - .ifPresent(w::writeDocs)), + .ifPresent(docs -> w.writeDocs(docs, context))), targetSymbol, schemaSymbol, writer.consumer(w -> target.accept( @@ -95,7 +92,6 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: new MemberDeserializerGenerator(context, w, member, "deserializer"))) ); - writer.popState(); } // Note that the unknown variant doesn't implement __eq__. This is because @@ -103,12 +99,12 @@ def deserialize(cls, deserializer: ShapeDeserializer) -> Self: // Since the underlying value is unknown and un-comparable, that is the only // realistic implementation. var unknownSymbol = symbolProvider.toSymbol(shape).expectProperty(SymbolProperties.UNION_UNKNOWN); - writer.pushState(new UnionMemberSection(unknownSymbol)); writer.addImport("smithy_core.exceptions", "SerializationError"); writer.write(""" @dataclass class $1L: - \"""Represents an unknown variant. + \""" + Represents an unknown variant. If you receive this value, you will need to update your library to receive the parsed value. @@ -130,14 +126,13 @@ raise NotImplementedError() """, unknownSymbol.getName()); memberNames.add(unknownSymbol.getName()); - writer.popState(); - writer.pushState(new UnionSection(shape, parentName, memberNames)); // We need to use the old union syntax until we either migrate away from // Sphinx or Sphinx fixes the issue upstream: https://github.com/sphinx-doc/sphinx/issues/10785 - writer.write("$L = Union[$L]\n", parentName, String.join(" | ", memberNames)); - shape.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeDocs(trait.getValue())); - writer.popState(); + writer.write("$L = Union[$L]", parentName, String.join(" | ", memberNames)); + shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { + writer.writeDocs(trait.getValue(), context); + }); generateDeserializer(); writer.popState(); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/ErrorSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/ErrorSection.java deleted file mode 100644 index 4b8831f0f..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/ErrorSection.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.sections; - -import software.amazon.smithy.codegen.core.Symbol; -import software.amazon.smithy.utils.CodeSection; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * A section that controls writing an error. - */ -@SmithyInternalApi -public record ErrorSection(Symbol errorSymbol) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/OperationSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/OperationSection.java deleted file mode 100644 index ceab19545..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/OperationSection.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.sections; - -import software.amazon.smithy.model.shapes.OperationShape; -import software.amazon.smithy.model.shapes.Shape; -import software.amazon.smithy.utils.CodeSection; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * A section that controls writing an operation. - */ -@SmithyInternalApi -public record OperationSection(Shape service, OperationShape operation) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/StructureSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/StructureSection.java deleted file mode 100644 index 23713e651..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/StructureSection.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.sections; - -import software.amazon.smithy.model.shapes.StructureShape; -import software.amazon.smithy.utils.CodeSection; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * A section that controls writing a structure. - */ -@SmithyInternalApi -public record StructureSection(StructureShape structure) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java deleted file mode 100644 index bd6d999e6..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionMemberSection.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.sections; - -import software.amazon.smithy.codegen.core.Symbol; -import software.amazon.smithy.utils.CodeSection; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * A section that controls writing a union member. - */ -@SmithyInternalApi -public record UnionMemberSection(Symbol memberSymbol) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionSection.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionSection.java deleted file mode 100644 index 55138f2f6..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/sections/UnionSection.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.sections; - -import java.util.ArrayList; -import software.amazon.smithy.model.shapes.UnionShape; -import software.amazon.smithy.utils.CodeSection; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * A section that controls writing a union. - */ -@SmithyInternalApi -public record UnionSection( - UnionShape unionShape, - String parentName, - ArrayList memberNames) implements CodeSection {} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownConverter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownConverter.java new file mode 100644 index 000000000..eed730e9f --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownConverter.java @@ -0,0 +1,215 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.writer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import software.amazon.smithy.codegen.core.CodegenException; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Converts CommonMark/HTML documentation to Markdown for Python docstrings. + * + * This converter uses the pandoc CLI tool to convert documentation from CommonMark/HTML + * format to Markdown suitable for Python docstrings with Google-style formatting. + * + * Pandoc must be installed and available. + */ +@SmithyInternalApi +public final class MarkdownConverter { + + private static final int PANDOC_WRAP_COLUMNS = 72; + private static final int TIMEOUT_SECONDS = 10; + + // List of HTML tags to exclude from documentation (including their content). + private static final List EXCLUDED_TAGS = List.of("fullname"); + + // Private constructor to prevent instantiation + private MarkdownConverter() {} + + /** + * Converts HTML or CommonMark strings to Markdown format using pandoc. + * + * For AWS services, documentation is in HTML format with raw HTML tags. + * For generic services, documentation is in CommonMark format which can + * include embedded HTML. + * + * @param input The input string (HTML or CommonMark) + * @param context The generation context to determine service type + * @return Markdown formatted string + */ + public static String convert(String input, GenerationContext context) { + if (input == null || input.isEmpty()) { + return ""; + } + + try { + input = preProcessPandocInput(input); + + if (!CodegenUtils.isAwsService(context)) { + // Commonmark may include embedded HTML so we first normalize the input to HTML format + input = convertWithPandoc(input, "commonmark", "html"); + } + + // The "html+raw_html" format preserves unrecognized html tags (e.g. , ) + // in Markdown output. We convert these tags to admonitions in postProcressPandocOutput() + String output = convertWithPandoc(input, "html+raw_html", "markdown"); + + return postProcessPandocOutput(output); + } catch (IOException | InterruptedException e) { + throw new CodegenException("Failed to convert documentation using pandoc: " + e.getMessage(), e); + } + } + + /** + * Pre-processes input before passing to pandoc. + * + * @param input The raw input text + * @return Pre-processed input ready for pandoc conversion + */ + private static String preProcessPandocInput(String input) { + // Trim leading and trailing spaces in hrefs i.e href=" https://example.com " + Pattern p = Pattern.compile("href\\s*=\\s*\"([^\"]*)\""); + input = p.matcher(input).replaceAll(match -> "href=\"" + match.group(1).trim() + "\""); + + // Remove excluded HTML tags and their content + for (String tagName : EXCLUDED_TAGS) { + input = removeHtmlTag(input, tagName); + } + + return input; + } + + /** + * Removes HTML tags and their content from the input string + * + * @param text The text to process + * @param tagName The tag name to remove (e.g., "fullname") + * @return Text with tags and their content removed + */ + private static String removeHtmlTag(String text, String tagName) { + // Remove content completely + Pattern p = Pattern.compile( + "<" + Pattern.quote(tagName) + ">[\\s\\S]*?"); + return p.matcher(text).replaceAll(""); + } + + /** + * Calls pandoc CLI to convert documentation. + * + * @param input The input string + * @param fromFormat The input format (e.g., "html+raw_html" or "commonmark") + * @param toFormat The output format (e.g., "markdown") + * @return Converted Markdown string + * @throws IOException if process I/O fails + * @throws InterruptedException if process is interrupted + */ + private static String convertWithPandoc(String input, String fromFormat, String toFormat) + throws IOException, InterruptedException { + ProcessBuilder processBuilder = new ProcessBuilder( + "pandoc", + "--from=" + fromFormat, + "--to=" + toFormat, + "--wrap=auto", + "--columns=" + PANDOC_WRAP_COLUMNS); + processBuilder.redirectErrorStream(true); + + Process process = processBuilder.start(); + + // Write input to pandoc's stdin + try (var outputStream = process.getOutputStream()) { + outputStream.write(input.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); + } + + // Read output from pandoc's stdout + StringBuilder output = new StringBuilder(); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + output.append(line).append("\n"); + } + } + + // Wait for process to complete + boolean finished = process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + throw new CodegenException("Pandoc process timed out after " + TIMEOUT_SECONDS + " seconds"); + } + + int exitCode = process.exitValue(); + if (exitCode != 0) { + throw new CodegenException( + "Pandoc failed with exit code " + exitCode + ": " + output.toString().trim()); + } + + return output.toString(); + } + + /** + * Post-processes pandoc output for Python docstrings. + * + * @param output The raw output from pandoc + * @return Post-processed Markdown suitable for Python docstrings + */ + private static String postProcessPandocOutput(String output) { + // Remove empty lines at the start and end + output = output.trim(); + + // Remove unnecessary backslash escapes that pandoc adds for markdown + // These characters don't need escaping in Python docstrings + // Handles: [ ] ' { } ( ) < > ` @ _ * | ! ~ $ + output = output.replaceAll("\\\\([\\[\\]'{}()<>`@_*|!~$])", "$1"); + + // Replace and tags with admonitions for mkdocstrings + output = replaceAdmonitionTags(output, "note", "Note"); + output = replaceAdmonitionTags(output, "important", "Warning"); + + // Escape Smithy format specifiers + return output.replace("$", "$$"); + } + + /** + * Replaces admonition tags (e.g. note, important) with Google-style format. + * + * @param text The text to process + * @param tagName The tag name to replace (e.g., "note", "important") + * @param label The label to use (e.g., "Note", "Warning") + * @return Text with replaced admonitions + */ + private static String replaceAdmonitionTags(String text, String tagName, String label) { + // Match content across multiple lines + Pattern pattern = Pattern.compile("<" + tagName + ">\\s*([\\s\\S]*?)\\s*"); + Matcher matcher = pattern.matcher(text); + StringBuffer result = new StringBuffer(); + + while (matcher.find()) { + // Extract the content between tags + String content = matcher.group(1).trim(); + + // Indent each line with 4 spaces + String[] lines = content.split("\n"); + StringBuilder indented = new StringBuilder(label + ":\n"); + for (String line : lines) { + indented.append(" ").append(line.trim()).append("\n"); + } + + matcher.appendReplacement(result, Matcher.quoteReplacement(indented.toString().trim())); + } + matcher.appendTail(result); + + return result.toString(); + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java deleted file mode 100644 index 883e9b132..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.writer; - -import static org.jsoup.nodes.Document.OutputSettings.Syntax.html; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.commonmark.node.BlockQuote; -import org.commonmark.node.FencedCodeBlock; -import org.commonmark.node.Heading; -import org.commonmark.node.HtmlBlock; -import org.commonmark.node.ListBlock; -import org.commonmark.node.ThematicBreak; -import org.commonmark.parser.Parser; -import org.commonmark.renderer.html.HtmlRenderer; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.nodes.Node; -import org.jsoup.nodes.TextNode; -import org.jsoup.select.NodeVisitor; -import software.amazon.smithy.utils.SetUtils; -import software.amazon.smithy.utils.SimpleCodeWriter; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * Add a runtime plugin to convert the HTML docs that are provided by services into RST - */ -@SmithyInternalApi -public class MarkdownToRstDocConverter { - private static final Parser MARKDOWN_PARSER = Parser.builder() - .enabledBlockTypes(SetUtils.of( - Heading.class, - HtmlBlock.class, - ThematicBreak.class, - FencedCodeBlock.class, - BlockQuote.class, - ListBlock.class)) - .build(); - - // Singleton instance - private static final MarkdownToRstDocConverter DOC_CONVERTER = new MarkdownToRstDocConverter(); - - // Private constructor to prevent instantiation - private MarkdownToRstDocConverter() { - // Constructor - } - - public static MarkdownToRstDocConverter getInstance() { - return DOC_CONVERTER; - } - - public String convertCommonmarkToRst(String commonmark) { - String html = HtmlRenderer.builder().escapeHtml(false).build().render(MARKDOWN_PARSER.parse(commonmark)); - //Replace the outer HTML paragraph tag with a div tag - Pattern pattern = Pattern.compile("^

(.*)

$", Pattern.DOTALL); - Matcher matcher = pattern.matcher(html); - html = matcher.replaceAll("
$1
"); - Document document = Jsoup.parse(html); - RstNodeVisitor visitor = new RstNodeVisitor(); - document.body().traverse(visitor); - return "\n" + visitor; - } - - private static class RstNodeVisitor implements NodeVisitor { - SimpleCodeWriter writer = new SimpleCodeWriter(); - private boolean inList = false; - private int listDepth = 0; - - @Override - public void head(Node node, int depth) { - if (node instanceof TextNode) { - TextNode textNode = (TextNode) node; - String text = textNode.text(); - if (!text.trim().isEmpty()) { - if (text.startsWith(":param ")) { - int secondColonIndex = text.indexOf(':', 1); - writer.write("$L", text.substring(0, secondColonIndex + 1)); - //TODO right now the code generator gives us a mixture of - // RST and HTML (for instance :param xyz:

docs - //

). Since we standardize to html above, that

tag - // starts a newline. We account for that with this if/else - // statement, but we should refactor this in the future to - // have a more elegant codepath. - if (secondColonIndex + 1 == text.strip().length()) { - writer.indent(); - writer.ensureNewline(); - } else { - writer.ensureNewline(); - writer.indent(); - writer.write("$L", text.substring(secondColonIndex + 1)); - writer.dedent(); - } - } else { - writer.writeInline("$L", text); - } - // Account for services making a paragraph tag that's empty except - // for a newline - } else if (node.parent() != null && ((Element) node.parent()).tagName().equals("p")) { - writer.writeInline("$L", text.replaceAll("[ \\t]+", "")); - } - } else if (node instanceof Element) { - Element element = (Element) node; - switch (element.tagName()) { - case "a": - writer.writeInline("`"); - break; - case "b": - case "strong": - writer.writeInline("**"); - break; - case "i": - case "em": - writer.writeInline("*"); - break; - case "code": - writer.writeInline("``"); - break; - case "important": - writer.ensureNewline(); - writer.write(""); - writer.openBlock(".. important::"); - break; - case "note": - writer.ensureNewline(); - writer.write(""); - writer.openBlock(".. note::"); - break; - case "ul": - if (inList) { - writer.indent(); - } - inList = true; - listDepth++; - writer.ensureNewline(); - writer.write(""); - break; - case "li": - writer.writeInline("* "); - break; - case "h1": - writer.ensureNewline(); - break; - default: - break; - } - } - } - - @Override - public void tail(Node node, int depth) { - if (node instanceof Element) { - Element element = (Element) node; - switch (element.tagName()) { - case "a": - String href = element.attr("href"); - if (!href.isEmpty()) { - writer.writeInline(" <").writeInline("$L", href).writeInline(">`_"); - } else { - writer.writeInline("`"); - } - break; - case "b": - case "strong": - writer.writeInline("**"); - break; - case "i": - case "em": - writer.writeInline("*"); - break; - case "code": - writer.writeInline("``"); - break; - case "important": - case "note": - writer.closeBlock(""); - break; - case "p": - writer.ensureNewline(); - writer.write(""); - break; - case "ul": - listDepth--; - if (listDepth == 0) { - inList = false; - } else { - writer.dedent(); - } - writer.ensureNewline(); - break; - case "li": - writer.ensureNewline(); - break; - case "h1": - String title = element.text(); - writer.ensureNewline().writeInline("$L", "=".repeat(title.length())).ensureNewline(); - break; - default: - break; - } - } - } - - @Override - public String toString() { - return writer.toString(); - } - } -} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java index 5ac42637e..435b8463a 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java @@ -23,7 +23,7 @@ import software.amazon.smithy.model.node.NumberNode; import software.amazon.smithy.model.node.ObjectNode; import software.amazon.smithy.model.node.StringNode; -import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; import software.amazon.smithy.utils.SmithyUnstableApi; import software.amazon.smithy.utils.StringUtils; @@ -39,10 +39,8 @@ public final class PythonWriter extends SymbolWriter { private static final Logger LOGGER = Logger.getLogger(PythonWriter.class.getName()); - private static final MarkdownToRstDocConverter DOC_CONVERTER = MarkdownToRstDocConverter.getInstance(); private final String fullPackageName; - private final String commentStart; private boolean addLogger = false; /** @@ -52,24 +50,12 @@ public final class PythonWriter extends SymbolWriter write(formatDocs(docs))); + public PythonWriter writeDocs(String docs, GenerationContext context) { + String formatted = formatDocs(docs, context); + if (formatted.contains("\n")) { + writeMultiLineDocs(() -> write(formatted)); + } else { + writeSingleLineDocs(() -> write(formatted)); + } return this; } @@ -174,92 +170,17 @@ public PythonWriter trimTrailingWhitespaces() { return this; } - private static final int MAX_LINE_LENGTH = CodegenUtils.MAX_PREFERRED_LINE_LENGTH - 8; - /** - * Formats a given Commonmark string and wraps it for use in a doc - * comment. + * Formats documentation from CommonMark or HTML to Google-style Markdown for Python docstrings. + * + *

For AWS services, expects HTML input. For generic clients, expects CommonMark input. * * @param docs Documentation to format. + * @param context The generation context used to determine service type and formatting. * @return Formatted documentation. */ - public String formatDocs(String docs) { - String rstDocs = DOC_CONVERTER.convertCommonmarkToRst(docs); - return wrapRST(rstDocs).toString().replace("$", "$$"); - } - - public static String wrapRST(String text) { - StringBuilder wrappedText = new StringBuilder(); - String[] lines = text.split("\n"); - for (String line : lines) { - wrapLine(line, wrappedText); - } - return wrappedText.toString(); - } - - private static void wrapLine(String line, StringBuilder wrappedText) { - int indent = getIndentationLevel(line); - String indentStr = " ".repeat(indent); - line = line.trim(); - - while (line.length() > MAX_LINE_LENGTH) { - int wrapAt = findWrapPosition(line, MAX_LINE_LENGTH); - wrappedText.append(indentStr).append(line, 0, wrapAt).append("\n"); - if (line.startsWith("* ")) { - indentStr += " "; - } - line = line.substring(wrapAt).trim(); - if (line.isEmpty()) { - return; - } - } - wrappedText.append(indentStr).append(line).append("\n"); - } - - private static int findWrapPosition(String line, int maxLineLength) { - // Find the last space before maxLineLength - int wrapAt = line.lastIndexOf(' ', maxLineLength); - if (wrapAt == -1) { - // If no space found, don't wrap - wrapAt = line.length(); - } else { - // Ensure we don't break a link - //TODO account for earlier backticks on the same line as a link - int linkStart = line.lastIndexOf("`", wrapAt); - int linkEnd = line.indexOf("`_", wrapAt); - if (linkStart != -1 && (linkEnd != -1 && linkEnd > linkStart)) { - linkEnd = line.indexOf("`_", linkStart); - if (linkEnd != -1) { - wrapAt = linkEnd + 2; - } else { - // No matching `_` found, keep the original wrap position - wrapAt = line.lastIndexOf(' ', maxLineLength); - if (wrapAt == -1) { - wrapAt = maxLineLength; - } - } - } - } - // Include trailing punctuation before a space in the previous line - int nextSpace = line.indexOf(' ', wrapAt); - if (nextSpace != -1) { - int i = wrapAt; - while (i < nextSpace && !Character.isLetterOrDigit(line.charAt(i))) { - i++; - } - if (i == nextSpace) { - wrapAt = nextSpace; - } - } - return wrapAt; - } - - private static int getIndentationLevel(String line) { - int indent = 0; - while (indent < line.length() && Character.isWhitespace(line.charAt(indent))) { - indent++; - } - return indent; + public String formatDocs(String docs, GenerationContext context) { + return MarkdownConverter.convert(docs, context); } /** @@ -284,7 +205,7 @@ public PythonWriter openComment(Runnable runnable) { * @return Returns the writer. */ public PythonWriter writeComment(String comment) { - return openComment(() -> write(formatDocs(comment.replace("\n", " ")))); + return openComment(() -> write(comment.replace("\n", " "))); } /** @@ -397,18 +318,12 @@ public PythonWriter maybeWrite(boolean shouldWrite, Object content, Object... ar @Override public String toString() { - String contents = getImportContainer().toString(); + String header = "# Code generated by smithy-python-codegen DO NOT EDIT.\n\n"; + String imports = getImportContainer().toString(); + String logger = addLogger ? "\nlogger = logging.getLogger(__name__)\n\n" : ""; + String mainContent = super.toString(); - if (addLogger) { - contents += "\nlogger = logging.getLogger(__name__)\n\n"; - } - - contents += super.toString(); - if (!commentStart.equals("")) { - String header = String.format("%s Code generated by smithy-python-codegen DO NOT EDIT.%n%n", commentStart); - contents = header + contents; - } - return contents; + return header + imports + logger + mainContent; } /** diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/CodegenUtilsTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/CodegenUtilsTest.java new file mode 100644 index 000000000..df3848d02 --- /dev/null +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/CodegenUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; + +public class CodegenUtilsTest { + + @Test + public void testIsAwsServiceWithAwsServiceTrait() { + // Create a mock context with an AWS service + GenerationContext context = createMockContext(true); + + // Should return true for AWS services + assertTrue(CodegenUtils.isAwsService(context)); + } + + @Test + public void testIsAwsServiceWithoutAwsServiceTrait() { + // Create a mock context with a non-AWS service + GenerationContext context = createMockContext(false); + + // Should return false for non-AWS services + assertFalse(CodegenUtils.isAwsService(context)); + } + + /** + * Helper method to create a mock GenerationContext with a service shape. + * + * @param hasAwsServiceTrait whether the service has the AWS service trait + * @return a mocked GenerationContext + */ + private GenerationContext createMockContext(boolean hasAwsServiceTrait) { + GenerationContext context = mock(GenerationContext.class); + Model model = mock(Model.class); + PythonSettings settings = mock(PythonSettings.class); + + ShapeId serviceId = ShapeId.from("test.service#TestService"); + ServiceShape serviceShape = mock(ServiceShape.class); + + when(context.model()).thenReturn(model); + when(context.settings()).thenReturn(settings); + when(settings.service()).thenReturn(serviceId); + when(model.expectShape(serviceId)).thenReturn(serviceShape); + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)) + .thenReturn(hasAwsServiceTrait); + + return context; + } +} diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/MarkdownConverterTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/MarkdownConverterTest.java new file mode 100644 index 000000000..028ec1aaf --- /dev/null +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/writer/MarkdownConverterTest.java @@ -0,0 +1,275 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.writer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.PythonSettings; + +public class MarkdownConverterTest { + + @Test + public void testConvertHtmlToMarkdownWithTitleAndParagraph() { + String html = "

Title

Paragraph

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should contain markdown heading and paragraph + String expected = """ + # Title + + Paragraph"""; + assertEquals(expected, result); + } + + @Test + public void testConvertHtmlToMarkdownWithList() { + String html = "
  • Item 1
  • Item 2
"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should contain markdown list + String expected = """ + - Item 1 + - Item 2"""; + assertEquals(expected, result); + } + + @Test + public void testConvertHtmlToMarkdownWithBoldTag() { + String html = "Bold text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("**Bold text**", result); + } + + @Test + public void testConvertHtmlToMarkdownWithItalicTag() { + String html = "Italic text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("*Italic text*", result); + } + + @Test + public void testConvertHtmlToMarkdownWithCodeTag() { + String html = "code snippet"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("`code snippet`", result); + } + + @Test + public void testConvertHtmlToMarkdownWithAnchorTag() { + String html = "Link"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("[Link](https://example.com)", result); + } + + @Test + public void testConvertHtmlToMarkdownWithNestedList() { + String html = "
  • Item 1
    • Subitem 1
  • Item 2
"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should contain nested markdown list with proper indentation + String expected = """ + - Item 1 + - Subitem 1 + - Item 2"""; + assertEquals(expected, result); + } + + @Test + public void testConvertHtmlToMarkdownWithFormatSpecifierCharacters() { + // Test that Smithy format specifier characters ($) are properly escaped + String html = "

Testing $placeholderOne and $placeholderTwo

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // $ should be escaped to $$ + assertEquals("Testing $$placeholderOne and $$placeholderTwo", result); + } + + @Test + public void testConvertCommonmarkWithEmbeddedHtmlForNonAwsService() { + // For non-AWS services, input is CommonMark which may include embedded HTML + // This tests the important path where commonmark -> html -> markdown conversion happens + String commonmarkWithHtml = "# Title\n\nParagraph with **bold** text and embedded HTML."; + String result = MarkdownConverter.convert(commonmarkWithHtml, createMockContext(false)); + // Should properly handle both markdown syntax and embedded HTML + String expected = """ + # Title + + Paragraph with **bold** text and *embedded HTML*."""; + assertEquals(expected, result); + } + + @Test + public void testConvertPureCommonmarkForNonAwsService() { + // For non-AWS services with pure CommonMark (no embedded HTML) + String commonmark = "# Title\n\nParagraph with **bold** and *italic* text.\n\n- List item 1\n- List item 2"; + String result = MarkdownConverter.convert(commonmark, createMockContext(false)); + // Should preserve the markdown structure (pandoc uses single space after dash) + String expected = """ + # Title + + Paragraph with **bold** and *italic* text. + + - List item 1 + - List item 2"""; + assertEquals(expected, result); + } + + @Test + public void testConvertRemovesUnnecessaryBackslashEscapes() { + // Pandoc adds escapes for these characters but they're not needed in Python docstrings + String html = "

Text with [brackets] and {braces} and (parens)

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should not have backslash escapes for these characters + assertEquals("Text with [brackets] and {braces} and (parens)", result.trim()); + } + + @Test + public void testConvertMixedElements() { + String html = "

Title

Paragraph

  • Item 1
  • Item 2
"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + String expected = "# Title\n\nParagraph\n\n- Item 1\n- Item 2"; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertNestedElements() { + String html = "

Title

Paragraph with bold text

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + String expected = """ + # Title + + Paragraph with **bold** text"""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertMultilineText() { + // Create a string with content > 72 chars to trigger wrapping + String html = + "This is a very long line that exceeds seventy-two characters and should wrap into two lines."; + String result = MarkdownConverter.convert(html, createMockContext(true)); + String expected = """ + This is a very long line that exceeds seventy-two characters and should + wrap into two lines."""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertHtmlToMarkdownWithNoteTag() { + String html = "Note text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should convert to admonition format + String expected = """ + Note: + Note text"""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertHtmlToMarkdownWithImportantTag() { + String html = "Important text"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Should convert to warning admonition + String expected = """ + Warning: + Important text"""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertMultilineAdmonitionTag() { + // Create a note with content > 72 chars to trigger wrapping + String longLine = + "This is a very long line that exceeds seventy-two characters and should wrap into two lines."; + String html = "" + longLine + ""; + String result = MarkdownConverter.convert(html, createMockContext(true)); + + // Expected: first line up to 72 chars, rest on second line, both indented + String expected = """ + Note: + This is a very long line that exceeds seventy-two characters and should + wrap into two lines."""; + assertEquals(expected, result.trim()); + } + + @Test + public void testConvertExcludesFullnameTag() { + String html = "

Some text

AWS Service Name

More text

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + String expected = """ + Some text + + More text"""; + assertEquals(expected, result); + } + + @Test + public void testConvertBoldWithNestedFormatting() { + String html = "Test test Link"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("**`Test` test [Link](https://testing.com)**", result); + } + + @Test + public void testConvertHrefWithSpaces() { + String html = "

Link

"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + // Leading and trailing spaces must be trimmed + assertEquals("[Link](https://testing.com)", result); + } + + @Test + public void testConvertLinkWithNestedCode() { + String html = "Link"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("[`Link`](https://testing.com)", result); + } + + @Test + public void testConvertCodeWithNestedLinkSpaces() { + String html = " Link "; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("` `[`Link`](https://testing.com)` `", result); + } + + @Test + public void testConvertCodeWithNestedLinkNoSpaces() { + String html = "Link"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("[`Link`](https://testing.com)", result); + } + + @Test + public void testConvertCodeWithNestedEmptyLink() { + String html = "Link"; + String result = MarkdownConverter.convert(html, createMockContext(true)); + assertEquals("`Link`", result); + } + + private GenerationContext createMockContext(boolean isAwsService) { + GenerationContext context = mock(GenerationContext.class); + Model model = mock(Model.class); + PythonSettings settings = mock(PythonSettings.class); + + ShapeId serviceId = ShapeId.from("test.service#TestService"); + ServiceShape serviceShape = mock(ServiceShape.class); + + when(context.model()).thenReturn(model); + when(context.settings()).thenReturn(settings); + when(settings.service()).thenReturn(serviceId); + when(model.expectShape(serviceId)).thenReturn(serviceShape); + + if (isAwsService) { + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)).thenReturn(true); + } else { + when(serviceShape.hasTrait(software.amazon.smithy.aws.traits.ServiceTrait.class)).thenReturn(false); + } + + return context; + } +} diff --git a/codegen/gradle/libs.versions.toml b/codegen/gradle/libs.versions.toml index 4ccc1b007..4b1765361 100644 --- a/codegen/gradle/libs.versions.toml +++ b/codegen/gradle/libs.versions.toml @@ -1,13 +1,12 @@ [versions] junit5 = "6.0.1" smithy = "1.64.0" +mockito = "5.20.0" test-logger-plugin = "4.0.0" spotbugs = "6.0.22" spotless = "8.1.0" smithy-gradle-plugins = "1.3.0" dep-analysis = "3.4.1" -jsoup = "1.21.2" -commonmark = "0.17.0" [libraries] smithy-model = { module = "software.amazon.smithy:smithy-model", version.ref = "smithy" } @@ -23,6 +22,7 @@ junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.re junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit5" } junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit5" } junit-platform-launcher = { module = "org.junit.platform:junit-platform-launcher" } +mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } # plugin artifacts for buildsrc plugins test-logger-plugin = { module = "com.adarshr:gradle-test-logger-plugin", version.ref = "test-logger-plugin" } @@ -30,9 +30,6 @@ spotbugs = { module = "com.github.spotbugs.snom:spotbugs-gradle-plugin", version spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotless" } dependency-analysis = { module = "com.autonomousapps:dependency-analysis-gradle-plugin", version.ref = "dep-analysis" } -jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } -commonmark = { module = "com.atlassian.commonmark:commonmark", version.ref ="commonmark" } - [plugins] smithy-gradle-base = { id = "software.amazon.smithy.gradle.smithy-base", version.ref = "smithy-gradle-plugins" } smithy-gradle-jar = { id = "software.amazon.smithy.gradle.smithy-jar", version.ref = "smithy-gradle-plugins" }