diff --git a/CHANGES.md b/CHANGES.md index 59d361a05c..ce7474e9dc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,7 @@ * Valid HTML names that are not XML QNames, such as `a:b:c`, are normalized. Attributes that still cannot be represented are skipped, and unrepresentable elements no longer change the surrounding tree. * Fixed `W3CDom` conversion of programmatically created or renamed elements whose names can be represented in a jsoup HTML DOM but are not valid XML names, such as `1abc`. These names are now normalized (e.g. `_1abc`) instead of causing a `NullPointerException`. [#2560](https://github.com/jhy/jsoup/issues/2560) * Fixed XML doctype serialization when a system identifier contains a double quote, which could otherwise produce invalid XML. [#2571](https://github.com/jhy/jsoup/issues/2571) +* Supplementary Unicode characters are now escaped correctly when serializing with non-UTF, non-ASCII output charsets such as ISO-8859-1. Previously, characters could be emitted unescaped when their low 16-bit value was representable by the configured charset, causing replacement or corruption when the output was encoded. [#2578](https://github.com/jhy/jsoup/issues/2578) * Fixed the JDK `HttpClient` implementation to accept responses without a `Content-Type` header, matching the `HttpURLConnection` implementation. [#2549](https://github.com/jhy/jsoup/pull/2549) * Fixed HTTP response content-type matching to handle media types case-insensitively and recognize structured `+xml` suffixes, including vendor-specific media types. [#2550](https://github.com/jhy/jsoup/pull/2550) * Corrected multipart form encoding to percent-escape CR and LF in field names and filenames, matching the HTML form submission specification. Multipart file content-types containing CR or LF are now rejected with a `ValidationException`. [#2555](https://github.com/jhy/jsoup/pull/2555) diff --git a/src/main/java/org/jsoup/nodes/Entities.java b/src/main/java/org/jsoup/nodes/Entities.java index f633cb6419..376a3cacdb 100644 --- a/src/main/java/org/jsoup/nodes/Entities.java +++ b/src/main/java/org/jsoup/nodes/Entities.java @@ -8,6 +8,7 @@ import org.jsoup.parser.CharacterReader; import org.jsoup.parser.Parser; +import java.nio.CharBuffer; import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import java.util.ArrayList; @@ -266,11 +267,11 @@ private static void appendEscaped(int codePoint, QuietAppendable accum, int opti accum.append(c); break; default: - if (c < 0x20 || !canEncode(coreCharset, c, fallback)) appendEncoded(accum, escapeMode, codePoint); + if (c < 0x20 || !canEncode(coreCharset, codePoint, fallback)) appendEncoded(accum, escapeMode, codePoint); else accum.append(c); } } else { - if (canEncode(coreCharset, c, fallback)) { + if (canEncode(coreCharset, codePoint, fallback)) { // reads into charBuf - we go through these steps to avoid GC objects as much as possible (would be a new String and a new char[2] for each character) char[] chars = charBuf.get(); int len = Character.toChars(codePoint, chars, 0); @@ -339,15 +340,22 @@ static String unescape(String string, boolean strict) { * Alterslash: 3013, 28 * Jsoup: 167, 2 */ - private static boolean canEncode(final CoreCharset charset, final char c, final CharsetEncoder fallback) { + private static boolean canEncode(final CoreCharset charset, final int codePoint, final CharsetEncoder fallback) { // todo add more charset tests if impacted by Android's bad perf in canEncode switch (charset) { case ascii: - return c < 0x80; + return codePoint < 0x80; case utf: - return !(c >= Character.MIN_SURROGATE && c < (Character.MAX_SURROGATE + 1)); // !Character.isSurrogate(c); but not in Android 10 desugar + // reject unpaired UTF-16 surrogate code units; valid supplementary code points are outside this range + return codePoint < Character.MIN_SURROGATE || codePoint > Character.MAX_SURROGATE; default: - return fallback.canEncode(c); + if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT) + return fallback.canEncode((char) codePoint); + + // check the complete UTF-16 pair; checking only the low 16 bits could accept an unencodable code point + char[] chars = charBuf.get(); + int len = Character.toChars(codePoint, chars, 0); + return fallback.canEncode(CharBuffer.wrap(chars, 0, len)); } } diff --git a/src/test/java/org/jsoup/nodes/ElementTest.java b/src/test/java/org/jsoup/nodes/ElementTest.java index 6b908f9173..3f07a09753 100644 --- a/src/test/java/org/jsoup/nodes/ElementTest.java +++ b/src/test/java/org/jsoup/nodes/ElementTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -857,6 +858,24 @@ public void testAddNewText() { assertEquals("
Hello
there & now >", TextUtil.stripNewlines(div.html())); } + @Test + public void escapesSupplementaryTextForNonUtf() { + Document doc = Document.createShell(""); + doc.outputSettings().charset(StandardCharsets.ISO_8859_1).prettyPrint(false); + doc.body().appendElement("p").text(new String(Character.toChars(0x100A3))); + + assertEquals("𐂣
", doc.body().html()); + } + + @Test + public void escapesSupplementaryAttributeForNonUtf() { + Document doc = Document.createShell(""); + doc.outputSettings().charset(StandardCharsets.ISO_8859_1).prettyPrint(false); + doc.body().appendElement("p").attr("data-probe", new String(Character.toChars(0x100A3))); + + assertEquals("", doc.body().html()); + } + @Test public void testPrependText() { Document doc = Jsoup.parse("Hello