From f10c02e12a15ca3d5acf17e732812172acd819ce Mon Sep 17 00:00:00 2001 From: Jonathan Hedley Date: Sat, 22 Aug 2026 07:11:43 +1000 Subject: [PATCH 1/2] Escape supplementary characters for non-UTF charsets Supplementary characters are now checked as complete code points before being emitted with non-UTF, non-ASCII charsets. The previous low-16-bit check could treat U+100A3 as U+00A3 under ISO-8859-1, then emit the original character unchanged. This still preserves the existing ASCII and UTF fast paths. --- CHANGES.md | 1 + src/main/java/org/jsoup/nodes/Entities.java | 20 ++++++++---- .../java/org/jsoup/nodes/ElementTest.java | 19 ++++++++++++ .../java/org/jsoup/nodes/EntitiesTest.java | 31 +++++++++++++++++++ 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 59d361a05c..911a52776a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,7 @@ * Valid HTML names that are not XML QNames, such as `a:b:c`, are normalized. Attributes that still cannot be represented are skipped, and unrepresentable elements no longer change the surrounding tree. * Fixed `W3CDom` conversion of programmatically created or renamed elements whose names can be represented in a jsoup HTML DOM but are not valid XML names, such as `1abc`. These names are now normalized (e.g. `_1abc`) instead of causing a `NullPointerException`. [#2560](https://github.com/jhy/jsoup/issues/2560) * Fixed XML doctype serialization when a system identifier contains a double quote, which could otherwise produce invalid XML. [#2571](https://github.com/jhy/jsoup/issues/2571) +* Supplementary Unicode characters are now escaped correctly when serializing with non-UTF, non-ASCII output charsets such as ISO-8859-1. Previously, characters could be emitted unescaped when their low 16-bit value was representable by the configured charset, causing replacement or corruption when the output was encoded. * Fixed the JDK `HttpClient` implementation to accept responses without a `Content-Type` header, matching the `HttpURLConnection` implementation. [#2549](https://github.com/jhy/jsoup/pull/2549) * Fixed HTTP response content-type matching to handle media types case-insensitively and recognize structured `+xml` suffixes, including vendor-specific media types. [#2550](https://github.com/jhy/jsoup/pull/2550) * Corrected multipart form encoding to percent-escape CR and LF in field names and filenames, matching the HTML form submission specification. Multipart file content-types containing CR or LF are now rejected with a `ValidationException`. [#2555](https://github.com/jhy/jsoup/pull/2555) diff --git a/src/main/java/org/jsoup/nodes/Entities.java b/src/main/java/org/jsoup/nodes/Entities.java index f633cb6419..376a3cacdb 100644 --- a/src/main/java/org/jsoup/nodes/Entities.java +++ b/src/main/java/org/jsoup/nodes/Entities.java @@ -8,6 +8,7 @@ import org.jsoup.parser.CharacterReader; import org.jsoup.parser.Parser; +import java.nio.CharBuffer; import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import java.util.ArrayList; @@ -266,11 +267,11 @@ private static void appendEscaped(int codePoint, QuietAppendable accum, int opti accum.append(c); break; default: - if (c < 0x20 || !canEncode(coreCharset, c, fallback)) appendEncoded(accum, escapeMode, codePoint); + if (c < 0x20 || !canEncode(coreCharset, codePoint, fallback)) appendEncoded(accum, escapeMode, codePoint); else accum.append(c); } } else { - if (canEncode(coreCharset, c, fallback)) { + if (canEncode(coreCharset, codePoint, fallback)) { // reads into charBuf - we go through these steps to avoid GC objects as much as possible (would be a new String and a new char[2] for each character) char[] chars = charBuf.get(); int len = Character.toChars(codePoint, chars, 0); @@ -339,15 +340,22 @@ static String unescape(String string, boolean strict) { * Alterslash: 3013, 28 * Jsoup: 167, 2 */ - private static boolean canEncode(final CoreCharset charset, final char c, final CharsetEncoder fallback) { + private static boolean canEncode(final CoreCharset charset, final int codePoint, final CharsetEncoder fallback) { // todo add more charset tests if impacted by Android's bad perf in canEncode switch (charset) { case ascii: - return c < 0x80; + return codePoint < 0x80; case utf: - return !(c >= Character.MIN_SURROGATE && c < (Character.MAX_SURROGATE + 1)); // !Character.isSurrogate(c); but not in Android 10 desugar + // reject unpaired UTF-16 surrogate code units; valid supplementary code points are outside this range + return codePoint < Character.MIN_SURROGATE || codePoint > Character.MAX_SURROGATE; default: - return fallback.canEncode(c); + if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT) + return fallback.canEncode((char) codePoint); + + // check the complete UTF-16 pair; checking only the low 16 bits could accept an unencodable code point + char[] chars = charBuf.get(); + int len = Character.toChars(codePoint, chars, 0); + return fallback.canEncode(CharBuffer.wrap(chars, 0, len)); } } diff --git a/src/test/java/org/jsoup/nodes/ElementTest.java b/src/test/java/org/jsoup/nodes/ElementTest.java index 6b908f9173..3f07a09753 100644 --- a/src/test/java/org/jsoup/nodes/ElementTest.java +++ b/src/test/java/org/jsoup/nodes/ElementTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -857,6 +858,24 @@ public void testAddNewText() { assertEquals("

Hello

there & now >", TextUtil.stripNewlines(div.html())); } + @Test + public void escapesSupplementaryTextForNonUtf() { + Document doc = Document.createShell(""); + doc.outputSettings().charset(StandardCharsets.ISO_8859_1).prettyPrint(false); + doc.body().appendElement("p").text(new String(Character.toChars(0x100A3))); + + assertEquals("

𐂣

", doc.body().html()); + } + + @Test + public void escapesSupplementaryAttributeForNonUtf() { + Document doc = Document.createShell(""); + doc.outputSettings().charset(StandardCharsets.ISO_8859_1).prettyPrint(false); + doc.body().appendElement("p").attr("data-probe", new String(Character.toChars(0x100A3))); + + assertEquals("

", doc.body().html()); + } + @Test public void testPrependText() { Document doc = Jsoup.parse("

Hello

"); diff --git a/src/test/java/org/jsoup/nodes/EntitiesTest.java b/src/test/java/org/jsoup/nodes/EntitiesTest.java index 6180e27ce5..5d53073206 100644 --- a/src/test/java/org/jsoup/nodes/EntitiesTest.java +++ b/src/test/java/org/jsoup/nodes/EntitiesTest.java @@ -4,6 +4,8 @@ import org.jsoup.parser.Parser; import org.junit.jupiter.api.Test; +import java.nio.charset.StandardCharsets; + import static org.jsoup.nodes.Document.OutputSettings; import static org.jsoup.nodes.Entities.EscapeMode.*; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -79,11 +81,40 @@ public class EntitiesTest { } @Test public void escapeSupplementaryCharacter() { + // Supplementary code points are escaped for ASCII but preserved for UTF-8 String text = new String(Character.toChars(135361)); String escapedAscii = Entities.escape(text, new OutputSettings().charset("ascii").escapeMode(base)); assertEquals("𡃁", escapedAscii); String escapedUtf = Entities.escape(text, new OutputSettings().charset("UTF-8").escapeMode(base)); assertEquals(text, escapedUtf); + + // The low 16 bits of U+1D800 form a surrogate code unit; UTF-8 must still preserve the full code point + String textWithSurrogateCodeUnit = new String(Character.toChars(0x1D800)); + assertEquals(textWithSurrogateCodeUnit, + Entities.escape(textWithSurrogateCodeUnit, new OutputSettings().charset("UTF-8").escapeMode(base))); + } + + @Test public void escapeSupplementaryNonUtf() { + // U+100A3 is supplementary, but its low 16 bits are U+00A3, which ISO-8859-1 can encode + String text = new String(Character.toChars(0x100A3)); + String escaped = Entities.escape(text, new OutputSettings() + .charset(StandardCharsets.ISO_8859_1) + .escapeMode(base)); + + assertEquals("𐂣", escaped); + } + + @Test public void escapeSupplementarySequenceNonUtf() { + // A non-UTF encoder must check the complete supplementary character, not just its low 16 bits. + // before this fix, those low 16-bit checks decided U+100A3 and U+100A9 were encodable, so the original + // code points were emitted raw rather than escaped + String text = "A" + new String(Character.toChars(0x100A3)) + + " & " + new String(Character.toChars(0x100A9)) + " Z"; + String escaped = Entities.escape(text, new OutputSettings() + .charset(StandardCharsets.ISO_8859_1) + .escapeMode(base)); + + assertEquals("A𐂣 & 𐂩 Z", escaped); } @Test public void notMissingMultis() { From ae8c6d21fb1b4588adf059a7d6c24333bbc7364d Mon Sep 17 00:00:00 2001 From: Jonathan Hedley Date: Sat, 22 Aug 2026 07:15:51 +1000 Subject: [PATCH 2/2] Changelog URL For #2578 / f10c02e12a15ca3d5acf17e732812172acd819ce --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 911a52776a..ce7474e9dc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,7 +24,7 @@ * Valid HTML names that are not XML QNames, such as `a:b:c`, are normalized. Attributes that still cannot be represented are skipped, and unrepresentable elements no longer change the surrounding tree. * Fixed `W3CDom` conversion of programmatically created or renamed elements whose names can be represented in a jsoup HTML DOM but are not valid XML names, such as `1abc`. These names are now normalized (e.g. `_1abc`) instead of causing a `NullPointerException`. [#2560](https://github.com/jhy/jsoup/issues/2560) * Fixed XML doctype serialization when a system identifier contains a double quote, which could otherwise produce invalid XML. [#2571](https://github.com/jhy/jsoup/issues/2571) -* Supplementary Unicode characters are now escaped correctly when serializing with non-UTF, non-ASCII output charsets such as ISO-8859-1. Previously, characters could be emitted unescaped when their low 16-bit value was representable by the configured charset, causing replacement or corruption when the output was encoded. +* Supplementary Unicode characters are now escaped correctly when serializing with non-UTF, non-ASCII output charsets such as ISO-8859-1. Previously, characters could be emitted unescaped when their low 16-bit value was representable by the configured charset, causing replacement or corruption when the output was encoded. [#2578](https://github.com/jhy/jsoup/issues/2578) * Fixed the JDK `HttpClient` implementation to accept responses without a `Content-Type` header, matching the `HttpURLConnection` implementation. [#2549](https://github.com/jhy/jsoup/pull/2549) * Fixed HTTP response content-type matching to handle media types case-insensitively and recognize structured `+xml` suffixes, including vendor-specific media types. [#2550](https://github.com/jhy/jsoup/pull/2550) * Corrected multipart form encoding to percent-escape CR and LF in field names and filenames, matching the HTML form submission specification. Multipart file content-types containing CR or LF are now rejected with a `ValidationException`. [#2555](https://github.com/jhy/jsoup/pull/2555)