[pull] master from jhy:master

CHANGES.md

@@ -24,6 +24,7 @@
   * Valid HTML names that are not XML QNames, such as `a:b:c`, are normalized. Attributes that still cannot be represented are skipped, and unrepresentable elements no longer change the surrounding tree.
 * Fixed `W3CDom` conversion of programmatically created or renamed elements whose names can be represented in a jsoup HTML DOM but are not valid XML names, such as `1abc`. These names are now normalized (e.g. `_1abc`) instead of causing a `NullPointerException`. [#2560](https://github.com/jhy/jsoup/issues/2560)
 * Fixed XML doctype serialization when a system identifier contains a double quote, which could otherwise produce invalid XML. [#2571](https://github.com/jhy/jsoup/issues/2571)
+* Supplementary Unicode characters are now escaped correctly when serializing with non-UTF, non-ASCII output charsets such as ISO-8859-1. Previously, characters could be emitted unescaped when their low 16-bit value was representable by the configured charset, causing replacement or corruption when the output was encoded. [#2578](https://github.com/jhy/jsoup/issues/2578)
 * Fixed the JDK `HttpClient` implementation to accept responses without a `Content-Type` header, matching the `HttpURLConnection` implementation. [#2549](https://github.com/jhy/jsoup/pull/2549)
 * Fixed HTTP response content-type matching to handle media types case-insensitively and recognize structured `+xml` suffixes, including vendor-specific media types. [#2550](https://github.com/jhy/jsoup/pull/2550)
 * Corrected multipart form encoding to percent-escape CR and LF in field names and filenames, matching the HTML form submission specification. Multipart file content-types containing CR or LF are now rejected with a `ValidationException`. [#2555](https://github.com/jhy/jsoup/pull/2555)

src/main/java/org/jsoup/nodes/Entities.java

@@ -8,6 +8,7 @@
 import org.jsoup.parser.CharacterReader;
 import org.jsoup.parser.Parser;
 
+import java.nio.CharBuffer;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetEncoder;
 import java.util.ArrayList;
@@ -266,11 +267,11 @@ private static void appendEscaped(int codePoint, QuietAppendable accum, int opti
                     accum.append(c);
                     break;
                 default:
-                    if (c < 0x20 || !canEncode(coreCharset, c, fallback)) appendEncoded(accum, escapeMode, codePoint);
+                    if (c < 0x20 || !canEncode(coreCharset, codePoint, fallback)) appendEncoded(accum, escapeMode, codePoint);
                     else accum.append(c);
             }
         } else {
-            if (canEncode(coreCharset, c, fallback)) {
+            if (canEncode(coreCharset, codePoint, fallback)) {
                 // reads into charBuf - we go through these steps to avoid GC objects as much as possible (would be a new String and a new char[2] for each character)
                 char[] chars = charBuf.get();
                 int len = Character.toChars(codePoint, chars, 0);
@@ -339,15 +340,22 @@ static String unescape(String string, boolean strict) {
      * Alterslash: 3013, 28
      * Jsoup: 167, 2
      */
-    private static boolean canEncode(final CoreCharset charset, final char c, final CharsetEncoder fallback) {
+    private static boolean canEncode(final CoreCharset charset, final int codePoint, final CharsetEncoder fallback) {
         // todo add more charset tests if impacted by Android's bad perf in canEncode
         switch (charset) {
             case ascii:
-                return c < 0x80;
+                return codePoint < 0x80;
             case utf:
-                return !(c >= Character.MIN_SURROGATE && c < (Character.MAX_SURROGATE + 1)); // !Character.isSurrogate(c); but not in Android 10 desugar
+                // reject unpaired UTF-16 surrogate code units; valid supplementary code points are outside this range
+                return codePoint < Character.MIN_SURROGATE || codePoint > Character.MAX_SURROGATE;
             default:
-                return fallback.canEncode(c);
+                if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT)
+                    return fallback.canEncode((char) codePoint);
+
+                // check the complete UTF-16 pair; checking only the low 16 bits could accept an unencodable code point
+                char[] chars = charBuf.get();
+                int len = Character.toChars(codePoint, chars, 0);
+                return fallback.canEncode(CharBuffer.wrap(chars, 0, len));
         }
     }
 

src/test/java/org/jsoup/nodes/ElementTest.java

@@ -19,6 +19,7 @@
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -857,6 +858,24 @@ public void testAddNewText() {
         assertEquals("<p>Hello</p>there &amp; now &gt;", TextUtil.stripNewlines(div.html()));
     }
 
+    @Test
+    public void escapesSupplementaryTextForNonUtf() {
+        Document doc = Document.createShell("");
+        doc.outputSettings().charset(StandardCharsets.ISO_8859_1).prettyPrint(false);
+        doc.body().appendElement("p").text(new String(Character.toChars(0x100A3)));
+
+        assertEquals("<p>&#x100a3;</p>", doc.body().html());
+    }
+
+    @Test
+    public void escapesSupplementaryAttributeForNonUtf() {
+        Document doc = Document.createShell("");
+        doc.outputSettings().charset(StandardCharsets.ISO_8859_1).prettyPrint(false);
+        doc.body().appendElement("p").attr("data-probe", new String(Character.toChars(0x100A3)));
+
+        assertEquals("<p data-probe=\"&#x100a3;\"></p>", doc.body().html());
+    }
+
     @Test
     public void testPrependText() {
         Document doc = Jsoup.parse("<div id=1><p>Hello</p></div>");

src/test/java/org/jsoup/nodes/EntitiesTest.java

@@ -4,6 +4,8 @@
 import org.jsoup.parser.Parser;
 import org.junit.jupiter.api.Test;
 
+import java.nio.charset.StandardCharsets;
+
 import static org.jsoup.nodes.Document.OutputSettings;
 import static org.jsoup.nodes.Entities.EscapeMode.*;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
@@ -79,11 +81,40 @@ public class EntitiesTest {
     }
 
     @Test public void escapeSupplementaryCharacter() {
+        // Supplementary code points are escaped for ASCII but preserved for UTF-8
         String text = new String(Character.toChars(135361));
         String escapedAscii = Entities.escape(text, new OutputSettings().charset("ascii").escapeMode(base));
         assertEquals("𡃁", escapedAscii);
         String escapedUtf = Entities.escape(text, new OutputSettings().charset("UTF-8").escapeMode(base));
         assertEquals(text, escapedUtf);
+
+        // The low 16 bits of U+1D800 form a surrogate code unit; UTF-8 must still preserve the full code point
+        String textWithSurrogateCodeUnit = new String(Character.toChars(0x1D800));
+        assertEquals(textWithSurrogateCodeUnit,
+            Entities.escape(textWithSurrogateCodeUnit, new OutputSettings().charset("UTF-8").escapeMode(base)));
+    }
+
+    @Test public void escapeSupplementaryNonUtf() {
+        // U+100A3 is supplementary, but its low 16 bits are U+00A3, which ISO-8859-1 can encode
+        String text = new String(Character.toChars(0x100A3));
+        String escaped = Entities.escape(text, new OutputSettings()
+            .charset(StandardCharsets.ISO_8859_1)
+            .escapeMode(base));
+
+        assertEquals("𐂣", escaped);
+    }
+
+    @Test public void escapeSupplementarySequenceNonUtf() {
+        // A non-UTF encoder must check the complete supplementary character, not just its low 16 bits.
+        // before this fix, those low 16-bit checks decided U+100A3 and U+100A9 were encodable, so the original
+        // code points were emitted raw rather than escaped
+        String text = "A" + new String(Character.toChars(0x100A3))
+            + " & " + new String(Character.toChars(0x100A9)) + " Z";
+        String escaped = Entities.escape(text, new OutputSettings()
+            .charset(StandardCharsets.ISO_8859_1)
+            .escapeMode(base));
+
+        assertEquals("A𐂣 & 𐂩 Z", escaped);
     }
 
     @Test public void notMissingMultis() {