diff --git a/src/main/java/com/hubspot/jinjava/Jinjava.java b/src/main/java/com/hubspot/jinjava/Jinjava.java index a5d834b48..f3ac8a4fb 100644 --- a/src/main/java/com/hubspot/jinjava/Jinjava.java +++ b/src/main/java/com/hubspot/jinjava/Jinjava.java @@ -250,7 +250,7 @@ public RenderResult renderForResult( .getInterpreterFactory() .newInstance(this, context, renderConfig); try { - String result = interpreter.render(template); + String result = stripTrailingNewlineIfNeeded(interpreter.render(template)); return new RenderResult( result, interpreter.getContext(), @@ -293,6 +293,18 @@ public RenderResult renderForResult( } } + /** + * Strips a single trailing newline from the rendered output when + * {@code keepTrailingNewline} is {@code false} in {@link Config}, + * matching Python Jinja2's default behaviour. + */ + private String stripTrailingNewlineIfNeeded(String output) { + if (!globalConfig.isKeepTrailingNewline() && output.endsWith("\n")) { + return output.substring(0, output.length() - 1); + } + return output; + } + /** * Creates a new interpreter instance using the global context and global config * diff --git a/src/main/java/com/hubspot/jinjava/JinjavaConfig.java b/src/main/java/com/hubspot/jinjava/JinjavaConfig.java index 0ef3c53d6..5a31cbf43 100644 --- a/src/main/java/com/hubspot/jinjava/JinjavaConfig.java +++ b/src/main/java/com/hubspot/jinjava/JinjavaConfig.java @@ -216,6 +216,17 @@ public boolean isEnableFilterChainOptimization() { return false; } + /** + * When {@code false} (default), a single trailing newline is stripped from the rendered + * output, matching Python Jinja2's default. + * When {@code true}, the trailing newline of + * the rendered output is preserved — matching Jinjava's historical behaviour. + */ + @Value.Default + public boolean isKeepTrailingNewline() { + return false; + } + @Value.Default public ObjectMapper getObjectMapper() { ObjectMapper objectMapper = new ObjectMapper().registerModule(new Jdk8Module()); diff --git a/src/main/java/com/hubspot/jinjava/LegacyOverrides.java b/src/main/java/com/hubspot/jinjava/LegacyOverrides.java index be16ae950..3357a7022 100644 --- a/src/main/java/com/hubspot/jinjava/LegacyOverrides.java +++ b/src/main/java/com/hubspot/jinjava/LegacyOverrides.java @@ -19,6 +19,7 @@ public class LegacyOverrides { .withUseTrimmingForNotesAndExpressions(true) .withKeepNullableLoopValues(true) .withIteratorOnlyReverseFilter(true) + .withHandleBackslashInQuotesOnly(true) .build(); public static final LegacyOverrides ALL = new LegacyOverrides.Builder() .withEvaluateMapKeys(true) @@ -31,6 +32,7 @@ public class LegacyOverrides { .withUseTrimmingForNotesAndExpressions(true) .withKeepNullableLoopValues(true) .withIteratorOnlyReverseFilter(true) + .withHandleBackslashInQuotesOnly(true) .build(); private final boolean evaluateMapKeys; private final boolean iterateOverMapKeys; @@ -42,6 +44,7 @@ public class LegacyOverrides { private final boolean useTrimmingForNotesAndExpressions; private final boolean keepNullableLoopValues; private final boolean iteratorOnlyReverseFilter; + private final boolean handleBackslashInQuotesOnly; private LegacyOverrides(Builder builder) { evaluateMapKeys = builder.evaluateMapKeys; @@ -54,6 +57,7 @@ private LegacyOverrides(Builder builder) { useTrimmingForNotesAndExpressions = builder.useTrimmingForNotesAndExpressions; keepNullableLoopValues = builder.keepNullableLoopValues; iteratorOnlyReverseFilter = builder.iteratorOnlyReverseFilter; + handleBackslashInQuotesOnly = builder.handleBackslashInQuotesOnly; } public static Builder newBuilder() { @@ -100,6 +104,10 @@ public boolean isIteratorOnlyReverseFilter() { return iteratorOnlyReverseFilter; } + public boolean isHandleBackslashInQuotesOnly() { + return handleBackslashInQuotesOnly; + } + public static class Builder { private boolean evaluateMapKeys = false; @@ -112,6 +120,7 @@ public static class Builder { private boolean useTrimmingForNotesAndExpressions = false; private boolean keepNullableLoopValues = false; private boolean iteratorOnlyReverseFilter = false; + private boolean handleBackslashInQuotesOnly = false; private Builder() {} @@ -134,7 +143,8 @@ public static Builder from(LegacyOverrides legacyOverrides) { legacyOverrides.useTrimmingForNotesAndExpressions ) .withKeepNullableLoopValues(legacyOverrides.keepNullableLoopValues) - .withIteratorOnlyReverseFilter(legacyOverrides.iteratorOnlyReverseFilter); + .withIteratorOnlyReverseFilter(legacyOverrides.iteratorOnlyReverseFilter) + .withHandleBackslashInQuotesOnly(legacyOverrides.handleBackslashInQuotesOnly); } public Builder withEvaluateMapKeys(boolean evaluateMapKeys) { @@ -192,5 +202,10 @@ public Builder withIteratorOnlyReverseFilter(boolean iteratorOnlyReverseFilter) this.iteratorOnlyReverseFilter = iteratorOnlyReverseFilter; return this; } + + public Builder withHandleBackslashInQuotesOnly(boolean handleBackslashInQuotesOnly) { + this.handleBackslashInQuotesOnly = handleBackslashInQuotesOnly; + return this; + } } } diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java index 6dadc5cf6..0e5df631c 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java @@ -68,6 +68,11 @@ public class StringTokenScanner extends AbstractIterator { private final char[] lineStmtPrefix; private final char[] lineCommentPrefix; + // When true, backslash is treated as an escape character only inside quoted + // string literals, matching Jinja2 behaviour. When false (legacy default), + // the scanner consumes backslash + next char unconditionally. + private final boolean backslashInQuotesOnly; + // Remembers where the current opening delimiter began so the emitted block/comment // token image starts from the opener (not the content), letting parse() strip the // correct number of delimiter characters from both ends. @@ -97,6 +102,8 @@ public StringTokenScanner(String input, JinjavaConfig config) { String lcp = symbols.getLineCommentPrefix(); lineCommentPrefix = (lcp != null && !lcp.isEmpty()) ? lcp.toCharArray() : null; + + backslashInQuotesOnly = config.getLegacyOverrides().isHandleBackslashInQuotesOnly(); } // ── Core scanning loop ──────────────────────────────────────────────────── @@ -202,9 +209,7 @@ private Token scanInsideComment() { */ private Token scanInsideBlock(char c) { if (inQuote != 0) { - // Inside a quoted string: a backslash escapes the next character so a - // delimiter or quote character following it does not prematurely close - // the block or the string. + // Inside a quoted string: a backslash always escapes the next character. if (c == '\\') { currPost += (currPost + 1 < length) ? 2 : 1; return DELIMITER_MATCHED; @@ -215,8 +220,9 @@ private Token scanInsideBlock(char c) { currPost++; return DELIMITER_MATCHED; } - // Outside a quoted string: a backslash escapes the next character. - if (c == '\\') { + // Outside a quoted string: only consume the backslash if the legacy + // flag is enabled; otherwise leave it for the expression parser. + if (c == '\\' && !backslashInQuotesOnly) { currPost += (currPost + 1 < length) ? 2 : 1; return DELIMITER_MATCHED; } diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java index a72c931ac..de3b6e040 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java @@ -49,6 +49,11 @@ public class TokenScanner extends AbstractIterator { private final TokenScannerSymbols symbols; private final WhitespaceControlParser whitespaceControlParser; + // When true, backslash is treated as an escape character only inside quoted + // string literals, matching Jinja2 behaviour. When false (legacy default), + // the scanner consumes backslash + next char unconditionally. + private final boolean backslashInQuotesOnly; + public TokenScanner(String input, JinjavaConfig config) { this.config = config; @@ -71,6 +76,7 @@ public TokenScanner(String input, JinjavaConfig config) { config.getLegacyOverrides().isParseWhitespaceControlStrictly() ? WhitespaceControlParser.STRICT : WhitespaceControlParser.LENIENT; + backslashInQuotesOnly = config.getLegacyOverrides().isHandleBackslashInQuotesOnly(); } private Token getNextToken() { @@ -82,10 +88,14 @@ private Token getNextToken() { } if (inBlock > 0) { - if (c == '\\') { + if (c == '\\' && !backslashInQuotesOnly) { ++currPost; continue; } else if (inQuote != 0) { + if (c == '\\') { + ++currPost; + continue; + } if (inQuote == c) { inQuote = 0; } diff --git a/src/test/java/com/hubspot/jinjava/TrailingNewlineTest.java b/src/test/java/com/hubspot/jinjava/TrailingNewlineTest.java new file mode 100644 index 000000000..9d43062d5 --- /dev/null +++ b/src/test/java/com/hubspot/jinjava/TrailingNewlineTest.java @@ -0,0 +1,73 @@ +package com.hubspot.jinjava; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.ImmutableMap; +import com.hubspot.jinjava.Jinjava; +import java.util.HashMap; +import org.junit.Test; + +public class TrailingNewlineTest { + + private static final String TEMPLATE_WITH_TRAILING_NEWLINE = "hello\n"; + private static final String TEMPLATE_WITHOUT_TRAILING_NEWLINE = "hello"; + private static final String TEMPLATE_MULTIPLE_TRAILING_NEWLINES = "hello\n\n"; + + // ── keepTrailingNewline=true (legacy default: preserve \n) ───────────────── + + @Test + public void itKeepsTrailingNewlineIsTrue() { + Jinjava jinjava = new Jinjava( + JinjavaConfig.newBuilder().withKeepTrailingNewline(true).build() + ); + assertThat(jinjava.render(TEMPLATE_WITH_TRAILING_NEWLINE, new HashMap<>())) + .isEqualTo("hello\n"); + } + + @Test + public void itStripsTrailingNewlineDefault() { + // Defaults keepTrailingNewline=false (matching Python behaviour) + Jinjava jinjava = new Jinjava(); + assertThat(jinjava.render(TEMPLATE_WITH_TRAILING_NEWLINE, new HashMap<>())) + .isEqualTo("hello"); + } + + // ── keepTrailingNewline=false (Python-compatible: strip trailing \n) ──────── + + @Test + public void itStripsTrailingNewlineIsFalse() { + Jinjava jinjava = new Jinjava( + JinjavaConfig.newBuilder().withKeepTrailingNewline(false).build() + ); + + assertThat(jinjava.render(TEMPLATE_WITH_TRAILING_NEWLINE, new HashMap<>())) + .isEqualTo("hello"); + } + + // ── Edge cases ────────────────────────────────────────────────────────────── + + @Test + public void itDoesNotAffectOutputWithNoTrailingNewline() { + Jinjava jinjava = new Jinjava( + JinjavaConfig.newBuilder().withKeepTrailingNewline(true).build() + ); + + assertThat(jinjava.render(TEMPLATE_WITHOUT_TRAILING_NEWLINE, new HashMap<>())) + .isEqualTo("hello"); + } + + @Test + public void itStripsOnlyOneTrailingNewlineNotMultiple() { + // Python only strips a single trailing newline, not all of them. + Jinjava jinjava = new Jinjava(); + assertThat(jinjava.render(TEMPLATE_MULTIPLE_TRAILING_NEWLINES, new HashMap<>())) + .isEqualTo("hello\n"); + } + + @Test + public void itStripsTrailingNewlineFromRenderedExpressions() { + Jinjava jinjava = new Jinjava(); + assertThat(jinjava.render("{{ greeting }}\n", ImmutableMap.of("greeting", "hello"))) + .isEqualTo("hello"); + } +} diff --git a/src/test/java/com/hubspot/jinjava/el/ext/AstFilterChainTest.java b/src/test/java/com/hubspot/jinjava/el/ext/AstFilterChainTest.java index c71050414..0544d1a5c 100644 --- a/src/test/java/com/hubspot/jinjava/el/ext/AstFilterChainTest.java +++ b/src/test/java/com/hubspot/jinjava/el/ext/AstFilterChainTest.java @@ -27,7 +27,11 @@ public class AstFilterChainTest { public void setup() { jinjava = new Jinjava( - BaseJinjavaTest.newConfigBuilder().withEnableFilterChainOptimization(true).build() + BaseJinjavaTest + .newConfigBuilder() + .withEnableFilterChainOptimization(true) + .withKeepTrailingNewline(true) + .build() ); context = new HashMap<>(); @@ -123,6 +127,7 @@ public void itSkipsDisabledFilterAndContinuesChain() { BaseJinjavaTest .newConfigBuilder() .withEnableFilterChainOptimization(true) + .withKeepTrailingNewline(true) .withDisabled(disabled) .build() ); diff --git a/src/test/java/com/hubspot/jinjava/interpret/LegacyOperatorPrecedenceTest.java b/src/test/java/com/hubspot/jinjava/interpret/LegacyOperatorPrecedenceTest.java index c9849a9a0..20c4729e4 100644 --- a/src/test/java/com/hubspot/jinjava/interpret/LegacyOperatorPrecedenceTest.java +++ b/src/test/java/com/hubspot/jinjava/interpret/LegacyOperatorPrecedenceTest.java @@ -21,6 +21,7 @@ public void setUp() throws Exception { new Jinjava( BaseJinjavaTest .newConfigBuilder() + .withKeepTrailingNewline(true) .withLegacyOverrides(LegacyOverrides.NONE) .build() ); @@ -28,6 +29,7 @@ public void setUp() throws Exception { new Jinjava( BaseJinjavaTest .newConfigBuilder() + .withKeepTrailingNewline(true) .withLegacyOverrides( LegacyOverrides.newBuilder().withUseNaturalOperatorPrecedence(true).build() ) diff --git a/src/test/java/com/hubspot/jinjava/interpret/LegacyWhitespaceControlParsingTest.java b/src/test/java/com/hubspot/jinjava/interpret/LegacyWhitespaceControlParsingTest.java index 48bbed0c9..c9cfdc8ce 100644 --- a/src/test/java/com/hubspot/jinjava/interpret/LegacyWhitespaceControlParsingTest.java +++ b/src/test/java/com/hubspot/jinjava/interpret/LegacyWhitespaceControlParsingTest.java @@ -21,6 +21,7 @@ public void setUp() throws Exception { new Jinjava( BaseJinjavaTest .newConfigBuilder() + .withKeepTrailingNewline(true) .withLegacyOverrides(LegacyOverrides.NONE) .build() ); @@ -28,6 +29,7 @@ public void setUp() throws Exception { new Jinjava( BaseJinjavaTest .newConfigBuilder() + .withKeepTrailingNewline(true) .withLegacyOverrides( LegacyOverrides.newBuilder().withParseWhitespaceControlStrictly(true).build() ) diff --git a/src/test/java/com/hubspot/jinjava/lib/filter/SliceFilterTest.java b/src/test/java/com/hubspot/jinjava/lib/filter/SliceFilterTest.java index 571a7b0bd..62e0ea820 100644 --- a/src/test/java/com/hubspot/jinjava/lib/filter/SliceFilterTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/filter/SliceFilterTest.java @@ -7,6 +7,7 @@ import com.google.common.collect.Lists; import com.google.common.io.Resources; import com.hubspot.jinjava.BaseJinjavaTest; +import com.hubspot.jinjava.Jinjava; import com.hubspot.jinjava.interpret.RenderResult; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -14,10 +15,19 @@ import java.util.Random; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; +import org.junit.Before; import org.junit.Test; public class SliceFilterTest extends BaseJinjavaTest { + @Before + public void setup() { + jinjava = + new Jinjava( + BaseJinjavaTest.newConfigBuilder().withKeepTrailingNewline(true).build() + ); + } + @Test public void itSlicesLists() throws Exception { Document dom = Jsoup.parseBodyFragment( diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/ForTagTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/ForTagTest.java index 5333b5f7c..1895a25a4 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/ForTagTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/ForTagTest.java @@ -11,6 +11,8 @@ import com.google.common.io.Resources; import com.hubspot.jinjava.BaseInterpretingTest; import com.hubspot.jinjava.BaseJinjavaTest; +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.JinjavaConfig; import com.hubspot.jinjava.LegacyOverrides; import com.hubspot.jinjava.interpret.InterpretException; import com.hubspot.jinjava.interpret.JinjavaInterpreter; @@ -40,6 +42,11 @@ public class ForTagTest extends BaseInterpretingTest { @Override public void baseSetup() { super.baseSetup(); + + jinjava = + new Jinjava( + BaseJinjavaTest.newConfigBuilder().withKeepTrailingNewline(true).build() + ); tag = new ForTag(); try { diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/BackslashHandlingTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/BackslashHandlingTest.java new file mode 100644 index 000000000..0515e4e10 --- /dev/null +++ b/src/test/java/com/hubspot/jinjava/tree/parse/BackslashHandlingTest.java @@ -0,0 +1,234 @@ +package com.hubspot.jinjava.tree.parse; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableMap; +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.LegacyOverrides; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import org.junit.Test; + +/** + * Tests for backslash handling inside block/variable/comment delimiters, + * covering both the char-based (DefaultTokenScannerSymbols) and string-based + * (StringTokenScannerSymbols) scanning paths, with the + * {@link LegacyOverrides#isHandleBackslashInQuotesOnly()} flag both off (legacy) + * and on (Jinja2-compatible). + */ +public class BackslashHandlingTest { + + // ── Jinjava instances ────────────────────────────────────────────────────── + + /** Char-based scanner, legacy backslash behaviour (flag = false). */ + private static Jinjava charLegacy() { + return new Jinjava( + JinjavaConfig + .newBuilder() + .withLegacyOverrides(LegacyOverrides.newBuilder().build()) + .build() + ); + } + + /** Char-based scanner, Jinja2-compatible backslash behaviour (flag = true). */ + private static Jinjava charNew() { + return new Jinjava( + JinjavaConfig + .newBuilder() + .withLegacyOverrides( + LegacyOverrides.newBuilder().withHandleBackslashInQuotesOnly(true).build() + ) + .build() + ); + } + + /** String-based scanner, legacy backslash behaviour (flag = false). */ + private static Jinjava stringLegacy() { + return new Jinjava( + JinjavaConfig + .newBuilder() + .withTokenScannerSymbols(StringTokenScannerSymbols.builder().build()) + .withLegacyOverrides(LegacyOverrides.newBuilder().build()) + .build() + ); + } + + /** String-based scanner, Jinja2-compatible backslash behaviour (flag = true). */ + private static Jinjava stringNew() { + return new Jinjava( + JinjavaConfig + .newBuilder() + .withTokenScannerSymbols(StringTokenScannerSymbols.builder().build()) + .withLegacyOverrides( + LegacyOverrides.newBuilder().withHandleBackslashInQuotesOnly(true).build() + ) + .build() + ); + } + + // ── Backslash inside a quoted string ────────────────────────────────────── + // + // Both legacy and new behaviour must handle escaped quotes inside strings + // correctly — \" should not close the string. + + @Test + public void charLegacy_escapedQuoteInsideString() { + assertThat(charLegacy().render("{{ \"he said \\\"hi\\\"\" }}", new HashMap<>())) + .isEqualTo("he said \"hi\""); + } + + @Test + public void charNew_escapedQuoteInsideString() { + assertThat(charNew().render("{{ \"he said \\\"hi\\\"\" }}", new HashMap<>())) + .isEqualTo("he said \"hi\""); + } + + @Test + public void stringLegacy_escapedQuoteInsideString() { + assertThat(stringLegacy().render("{{ \"he said \\\"hi\\\"\" }}", new HashMap<>())) + .isEqualTo("he said \"hi\""); + } + + @Test + public void stringNew_escapedQuoteInsideString() { + assertThat(stringNew().render("{{ \"he said \\\"hi\\\"\" }}", new HashMap<>())) + .isEqualTo("he said \"hi\""); + } + + // ── Backslash outside a quoted string ───────────────────────────────────── + // + // Template under test: "prefix {{ x \}} suffix }}" + // + // We test the scanner token structure directly rather than going through + // render(), because the expression "x \..." is always a JUEL lexical error + // regardless of mode. What differs between modes is which token boundaries + // the scanner produces — and that is what we assert on. + // + // Legacy (backslashInQuotesOnly = false): + // Scanner consumes '\' and skips the following '}'. The first '}}' is not + // recognized as a closer. The block runs until the second '}}', so the + // token sequence is: + // TEXT "prefix " | EXPR "{{ x \}} suffix }}" + // + // New (backslashInQuotesOnly = true): + // Scanner leaves '\' untouched. The first '}}' is recognized as the closer. + // The token sequence is: + // TEXT "prefix " | EXPR "{{ x \}}" | TEXT " suffix }}" + + private static final String BACKSLASH_TEMPLATE = "prefix {{ x \\}} suffix }}"; + + @Test + public void charLegacy_backslashConsumesOneDelimiterChar_blockRunsToSecondCloser() { + List tokens = scanAll( + new TokenScanner(BACKSLASH_TEMPLATE, charLegacy().getGlobalConfig()) + ); + assertThat(tokens).hasSize(2); + assertThat(tokens.get(0)).isInstanceOf(TextToken.class); + assertThat(tokens.get(0).image).isEqualTo("prefix "); + assertThat(tokens.get(1)).isInstanceOf(ExpressionToken.class); + assertThat(tokens.get(1).image).isEqualTo("{{ x \\}} suffix }}"); + } + + @Test + public void charNew_backslashIgnored_blockClosesAtFirstDelimiter() { + List tokens = scanAll( + new TokenScanner(BACKSLASH_TEMPLATE, charNew().getGlobalConfig()) + ); + assertThat(tokens).hasSize(3); + assertThat(tokens.get(0)).isInstanceOf(TextToken.class); + assertThat(tokens.get(0).image).isEqualTo("prefix "); + assertThat(tokens.get(1)).isInstanceOf(ExpressionToken.class); + assertThat(tokens.get(1).image).isEqualTo("{{ x \\}}"); + assertThat(tokens.get(2)).isInstanceOf(TextToken.class); + assertThat(tokens.get(2).image).isEqualTo(" suffix }}"); + } + + @Test + public void stringLegacy_backslashConsumesOneDelimiterChar_blockRunsToSecondCloser() { + List tokens = scanAll( + new StringTokenScanner(BACKSLASH_TEMPLATE, stringLegacy().getGlobalConfig()) + ); + assertThat(tokens).hasSize(2); + assertThat(tokens.get(0)).isInstanceOf(TextToken.class); + assertThat(tokens.get(0).image).isEqualTo("prefix "); + assertThat(tokens.get(1)).isInstanceOf(ExpressionToken.class); + assertThat(tokens.get(1).image).isEqualTo("{{ x \\}} suffix }}"); + } + + @Test + public void stringNew_backslashIgnored_blockClosesAtFirstDelimiter() { + List tokens = scanAll( + new StringTokenScanner(BACKSLASH_TEMPLATE, stringNew().getGlobalConfig()) + ); + assertThat(tokens).hasSize(3); + assertThat(tokens.get(0)).isInstanceOf(TextToken.class); + assertThat(tokens.get(0).image).isEqualTo("prefix "); + assertThat(tokens.get(1)).isInstanceOf(ExpressionToken.class); + assertThat(tokens.get(1).image).isEqualTo("{{ x \\}}"); + assertThat(tokens.get(2)).isInstanceOf(TextToken.class); + assertThat(tokens.get(2).image).isEqualTo(" suffix }}"); + } + + private static List scanAll(AbstractIterator scanner) { + List tokens = new ArrayList<>(); + scanner.forEachRemaining(tokens::add); + return tokens; + } + + // ── Backslash in a plain variable expression ─────────────────────────────── + // + // The most common real-world case: a Windows path or similar string passed + // directly as a variable value. The backslash is in the *value*, not the + // template, so scanner behaviour is irrelevant — both modes should render + // identically. + + @Test + public void backslashInVariableValueIsUnaffectedByFlag_char() { + ImmutableMap ctx = ImmutableMap.of("path", "C:\\Users\\foo"); + assertThat(charLegacy().render("{{ path }}", ctx)).isEqualTo("C:\\Users\\foo"); + assertThat(charNew().render("{{ path }}", ctx)).isEqualTo("C:\\Users\\foo"); + } + + @Test + public void backslashInVariableValueIsUnaffectedByFlag_string() { + ImmutableMap ctx = ImmutableMap.of("path", "C:\\Users\\foo"); + assertThat(stringLegacy().render("{{ path }}", ctx)).isEqualTo("C:\\Users\\foo"); + assertThat(stringNew().render("{{ path }}", ctx)).isEqualTo("C:\\Users\\foo"); + } + + // ── New behaviour: simple expressions are unaffected ────────────────────── + // + // Expressions with no backslash should behave identically under both modes. + + @Test + public void charNew_simpleExpressionUnchanged() { + assertThat(charNew().render("{{ greeting }}", ImmutableMap.of("greeting", "hello"))) + .isEqualTo("hello"); + } + + @Test + public void stringNew_simpleExpressionUnchanged() { + assertThat(stringNew().render("{{ greeting }}", ImmutableMap.of("greeting", "hello"))) + .isEqualTo("hello"); + } + + // ── LegacyOverrides preset assertions ───────────────────────────────────── + + @Test + public void allPresetDoesNotEnableNewBackslashHandling() { + assertThat(LegacyOverrides.ALL.isHandleBackslashInQuotesOnly()).isTrue(); + } + + @Test + public void threePointZeroPresetDoesNotEnableNewBackslashHandling() { + assertThat(LegacyOverrides.THREE_POINT_0.isHandleBackslashInQuotesOnly()).isTrue(); + } + + @Test + public void nonePresetKeepsLegacyBackslashHandling() { + assertThat(LegacyOverrides.NONE.isHandleBackslashInQuotesOnly()).isFalse(); + } +} diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java index a03fe836e..b7af6c074 100644 --- a/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java +++ b/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java @@ -256,6 +256,7 @@ public void itRespectsTrimBlocksWithAngleSymbols() { .newConfigBuilder() .withTokenScannerSymbols(ANGLE_SYMBOLS) .withTrimBlocks(true) + .withKeepTrailingNewline(true) .build() ); // Without trimBlocks the newline after <% if show %> would appear in output. @@ -274,6 +275,7 @@ public void itRespectsTrimBlocksWithLatexSymbols() { .newConfigBuilder() .withTokenScannerSymbols(LATEX_SYMBOLS) .withTrimBlocks(true) + .withKeepTrailingNewline(true) .build() ); String result = j.render( @@ -291,6 +293,7 @@ public void itRespectsLstripBlocksWithAngleSymbols() { .withTokenScannerSymbols(ANGLE_SYMBOLS) .withLstripBlocks(true) .withTrimBlocks(true) + .withKeepTrailingNewline(true) .build() ); // Leading spaces before the tag are stripped by lstripBlocks (TreeParser). @@ -310,6 +313,7 @@ public void itRespectsLstripBlocksWithLatexSymbols() { .withTokenScannerSymbols(LATEX_SYMBOLS) .withLstripBlocks(true) .withTrimBlocks(true) + .withKeepTrailingNewline(true) .build() ); String result = j.render( @@ -503,7 +507,11 @@ public void itHandlesBothLinePrefixesTogether() { private Jinjava jinjavaWith(StringTokenScannerSymbols symbols) { return new Jinjava( - BaseJinjavaTest.newConfigBuilder().withTokenScannerSymbols(symbols).build() + BaseJinjavaTest + .newConfigBuilder() + .withKeepTrailingNewline(true) + .withTokenScannerSymbols(symbols) + .build() ); } } diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java index 4261b4fc2..2c5c1c8e4 100644 --- a/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java +++ b/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java @@ -8,6 +8,7 @@ import com.google.common.io.Resources; import com.hubspot.jinjava.BaseJinjavaTest; import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.LegacyOverrides; import com.hubspot.jinjava.features.BuiltInFeatures; import com.hubspot.jinjava.features.FeatureConfig; import com.hubspot.jinjava.features.FeatureStrategies; @@ -319,6 +320,13 @@ public void testLstripBlocks() { @Test public void itTreatsEscapedQuotesSameWhenNotInQuotes() { + config = + BaseJinjavaTest + .newConfigBuilder() + .withLegacyOverrides( + LegacyOverrides.newBuilder().withHandleBackslashInQuotesOnly(false).build() + ) + .build(); List tokens = tokens("tag-with-all-escaped-quotes"); assertThat(tokens).hasSize(8); assertThat(tokens.stream().map(Token::getType).collect(Collectors.toList()))