From e1bf7d0187a351f16335ca63e2c26a4573587797 Mon Sep 17 00:00:00 2001 From: yinbing Date: Thu, 30 Jul 2026 12:18:26 -0700 Subject: [PATCH 1/6] read and deserialize json file support all encoding types regarding UTF-8, UTF-16, UTF-32, BOM, LE/BE --- .gitattributes | 5 + proxy_agent_shared/src/error.rs | 3 + proxy_agent_shared/src/misc_helpers.rs | 190 +++++++++++++++++- .../testdata/encodings/utf16be_bom.json | Bin 0 -> 142 bytes .../testdata/encodings/utf16be_no_bom.json | Bin 0 -> 140 bytes .../testdata/encodings/utf16le_bom.json | Bin 0 -> 142 bytes .../testdata/encodings/utf16le_no_bom.json | Bin 0 -> 140 bytes .../testdata/encodings/utf32be_bom.json | Bin 0 -> 280 bytes .../testdata/encodings/utf32be_no_bom.json | Bin 0 -> 276 bytes .../testdata/encodings/utf32le_bom.json | Bin 0 -> 280 bytes .../testdata/encodings/utf32le_no_bom.json | Bin 0 -> 276 bytes .../testdata/encodings/utf8_bom.json | 1 + .../testdata/encodings/utf8_no_bom.json | 1 + 13 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 proxy_agent_shared/testdata/encodings/utf16be_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf16be_no_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf16le_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf16le_no_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf32be_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf32be_no_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf32le_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf32le_no_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf8_bom.json create mode 100644 proxy_agent_shared/testdata/encodings/utf8_no_bom.json diff --git a/.gitattributes b/.gitattributes index a2e13784..43c149dc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -29,3 +29,8 @@ Cargo.lock eol=lf *.png binary *.wim binary *.zip binary + +# json_read_from_file encoding fixtures. These are UTF-16/UTF-32 and +# BOM-prefixed files whose exact bytes are the thing under test, so they must +# never be text-normalized despite the `*.json text` rule above. +proxy_agent_shared/testdata/encodings/* binary diff --git a/proxy_agent_shared/src/error.rs b/proxy_agent_shared/src/error.rs index 61360f3f..d7ab917e 100644 --- a/proxy_agent_shared/src/error.rs +++ b/proxy_agent_shared/src/error.rs @@ -57,6 +57,9 @@ pub enum Error { #[error("Parse datetime string error: {0}")] ParseDateTimeStringError(String), + #[error("Failed to decode file '{0}': {1}")] + DecodeFile(String, String), + #[error( "Failed to get proxy agent aggregate status (server error: {0}; status file error: {1})" )] diff --git a/proxy_agent_shared/src/misc_helpers.rs b/proxy_agent_shared/src/misc_helpers.rs index dfe4172d..a78870de 100644 --- a/proxy_agent_shared/src/misc_helpers.rs +++ b/proxy_agent_shared/src/misc_helpers.rs @@ -8,6 +8,7 @@ use regex::Regex; use serde::de::DeserializeOwned; use serde::Serialize; use std::{ + borrow::Cow, fs::{self, File}, path::{Path, PathBuf}, process::Command, @@ -245,26 +246,134 @@ where Ok(()) } +/// Reads `file_path`, decodes it using the detected text encoding and +/// deserializes the resulting JSON into `T`. +/// +/// Supports UTF-8, UTF-16LE, UTF-16BE, UTF-32LE and UTF-32BE, each with or +/// without a BOM - the 10 encodings a JSON file produced by an arbitrary +/// editor or tool can realistically use. Any BOM is consumed while decoding and +/// never reaches serde_json, which would fail if the json payload contains BOM prefix. pub fn json_read_from_file(file_path: &Path) -> Result where T: DeserializeOwned, { - // Read the whole file to bytes so we can transparently skip an optional - // UTF-8 BOM (EF BB BF). serde_json does not strip a BOM and would otherwise - // fail the parse with "expected value at line 1 column 1" for any file - // produced by editors / tools that default to BOM-prefixed UTF-8 (e.g. - // Windows PowerShell 5.1's `Set-Content -Encoding UTF8`, Notepad, VS Code's - // "UTF-8 with BOM"). let bytes = fs::read(file_path)?; - let payload = match bytes.as_slice() { - [0xEF, 0xBB, 0xBF, rest @ ..] => rest, - rest => rest, - }; - let obj: T = serde_json::from_slice(payload)?; + let text = decode_json_text(&bytes, file_path)?; + let obj: T = serde_json::from_str(&text)?; Ok(obj) } +/// Detects the text encoding of `bytes` and returns it as +/// (code unit width in bytes, big endian, BOM length in bytes). +/// width: 1 for UTF-8, 2 for UTF-16, 4 for UTF-32 +/// big_endian: true for BE, false for LE +/// bom length length of bom +/// +/// Wider BOMs must be tested first: the UTF-32LE BOM (FF FE 00 00) starts with +/// the UTF-16LE BOM (FF FE), so a shortest-first scan would mis-detect a +/// UTF-32LE file as UTF-16LE. +fn detect_json_encoding(bytes: &[u8]) -> (usize, bool, usize) { + if bytes.starts_with(&[0x00, 0x00, 0xFE, 0xFF]) { + (4, true, 4) // UTF-32BE with BOM + } else if bytes.starts_with(&[0xFF, 0xFE, 0x00, 0x00]) { + (4, false, 4) // UTF-32LE with BOM + } else if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) { + (1, false, 3) // UTF-8 with BOM + } else if bytes.starts_with(&[0xFE, 0xFF]) { + (2, true, 2) // UTF-16BE with BOM + } else if bytes.starts_with(&[0xFF, 0xFE]) { + (2, false, 2) // UTF-16LE with BOM + } else { + // No BOM. A JSON document always starts with an ASCII character (`[`, + // `{`, `"`, a digit or whitespace), so the NUL padding around the first + // code unit identifies both the width and the byte order. The 4-byte + // patterns are checked first because they are a superset of the 2-byte + // ones. + let is_nul = |i: usize| bytes.get(i) == Some(&0x00); + let is_text = |i: usize| matches!(bytes.get(i), Some(b) if *b != 0x00); + + if is_nul(0) && is_nul(1) && is_nul(2) && is_text(3) { + (4, true, 0) // 00 00 00 xx -> UTF-32BE + } else if is_text(0) && is_nul(1) && is_nul(2) && is_nul(3) { + (4, false, 0) // xx 00 00 00 -> UTF-32LE + } else if is_nul(0) && is_text(1) { + (2, true, 0) // 00 xx -> UTF-16BE + } else if is_text(0) && is_nul(1) { + (2, false, 0) // xx 00 -> UTF-16LE + } else { + (1, false, 0) // anything else, including plain ASCII / UTF-8 + } + } +} + +/// Decodes `bytes` into UTF-8 text using the encoding detected by +/// [`detect_json_encoding`], skipping the BOM when present. +/// UTF-8 input is borrowed as-is, so the common case does not allocate. +fn decode_json_text<'a>(bytes: &'a [u8], file_path: &Path) -> Result> { + let decode_error = |detail: String| Error::DecodeFile(path_to_string(file_path), detail); + + let (code_unit_len, big_endian, bom_len) = detect_json_encoding(bytes); + let payload = &bytes[bom_len..]; + + match code_unit_len { + 1 => std::str::from_utf8(payload) + .map(Cow::Borrowed) + .map_err(|e| decode_error(format!("content is not valid UTF-8: {e}"))), + 2 => { + if !payload.len().is_multiple_of(2) { + return Err(decode_error(format!( + "UTF-16 content is truncated: {} bytes is not a whole number of 16-bit code units", + payload.len() + ))); + } + + let code_units = payload.chunks_exact(2).map(|chunk| { + let unit = [chunk[0], chunk[1]]; + if big_endian { + u16::from_be_bytes(unit) + } else { + u16::from_le_bytes(unit) + } + }); + + // `decode_utf16` pairs surrogates, so astral-plane characters + // (emoji) are reassembled correctly; a lone surrogate is rejected + // rather than silently replaced. + char::decode_utf16(code_units) + .collect::>() + .map(Cow::Owned) + .map_err(|e| decode_error(format!("UTF-16 content has an unpaired surrogate: {e}"))) + } + _ => { + if !payload.len().is_multiple_of(4) { + return Err(decode_error(format!( + "UTF-32 content is truncated: {} bytes is not a whole number of 32-bit code units", + payload.len() + ))); + } + + payload + .chunks_exact(4) + .map(|chunk| { + let unit = [chunk[0], chunk[1], chunk[2], chunk[3]]; + let scalar = if big_endian { + u32::from_be_bytes(unit) + } else { + u32::from_le_bytes(unit) + }; + char::from_u32(scalar).ok_or_else(|| { + decode_error(format!( + "UTF-32 content has an invalid scalar value: {scalar:#010X}" + )) + }) + }) + .collect::>() + .map(Cow::Owned) + } + } +} + pub fn json_clone(obj: &T) -> Result where T: Serialize + DeserializeOwned, @@ -644,6 +753,65 @@ mod tests { _ = fs::remove_dir_all(&temp_test_path); } + #[test] + fn json_read_from_file_supports_all_ten_encodings_test() { + // Latin-1 accent + CJK + an astral-plane emoji (a surrogate pair in + // UTF-16) so multi-byte decoding and surrogate pairing are exercised, + // not just the ASCII fast path. This is the exact `message` value + // stored in every fixture file under testdata/encodings. + const NON_ASCII_MESSAGE: &str = "caf\u{00e9} \u{6d4b}\u{8bd5} \u{1F600}"; + + #[derive(Serialize, Deserialize, PartialEq, Debug)] + struct EncodingTestStruct { + name: String, + code: i32, + message: String, + enabled: bool, + } + + let testdata_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("testdata") + .join("encodings"); + + let expected = EncodingTestStruct { + name: "EncodingTest".to_string(), + code: 7, + message: NON_ASCII_MESSAGE.to_string(), + enabled: true, + }; + + // Pre-created fixture files, one per supported encoding. They hold the + // same JSON document, byte-for-byte encoded differently. The + // UTF-32LE-with-BOM file is the ambiguous one: its BOM starts with the + // UTF-16LE BOM. + let fixtures = [ + "utf8_bom.json", + "utf8_no_bom.json", + "utf16le_bom.json", + "utf16le_no_bom.json", + "utf16be_bom.json", + "utf16be_no_bom.json", + "utf32le_bom.json", + "utf32le_no_bom.json", + "utf32be_bom.json", + "utf32be_no_bom.json", + ]; + + for file_name in fixtures { + let file_path = testdata_dir.join(file_name); + assert!( + file_path.exists(), + "missing encoding fixture file: {}", + file_path.display() + ); + + let actual = super::json_read_from_file::(&file_path) + .unwrap_or_else(|e| panic!("{file_name}: {e}")); + + assert_eq!(expected, actual, "{file_name}: decoded payload differs"); + } + } + #[test] fn path_to_string_test() { let path = "path_to_string_test"; diff --git a/proxy_agent_shared/testdata/encodings/utf16be_bom.json b/proxy_agent_shared/testdata/encodings/utf16be_bom.json new file mode 100644 index 0000000000000000000000000000000000000000..2a0812aa2af49a98cd4549b0dc6b8da12f09170c GIT binary patch literal 142 zcmezOpP`yTi6M_6ks+5M709vzVpkwPnIWGcg&`BjN@oZG$`mt{0A+N57$O5vYYr9x z=`04qM4(!j8OcCi8pBHlg8|tV@PDkWk?0Gtbo`R$WLa-XGmel1hUc@LVz;G3?)Dr9Uz9tK-8LpML;@> zfiMxM7G_2=ke9~rl0hNYyZb7G!VTMd3{d^4VB3-ya)4@6fMQlalZzNifxKD(xriDC literal 0 HcmV?d00001 diff --git a/proxy_agent_shared/testdata/encodings/utf16le_bom.json b/proxy_agent_shared/testdata/encodings/utf16le_bom.json new file mode 100644 index 0000000000000000000000000000000000000000..1424668e10259965412383da9a5605d4b1941c95 GIT binary patch literal 142 zcmezWubM%LA&()EA(tT)$g%=rS0F!`A)g_IArr_-X9xkx6f=|nWpscTA_Gxt4i*9F zEC#|vpjwz2$v|Ej!%GGQ2JhUf-3ko0HyG|g^rwPtOJc|Ys!ajgSi(@mPzvPLG5`SQ CnH&ND literal 0 HcmV?d00001 diff --git a/proxy_agent_shared/testdata/encodings/utf16le_no_bom.json b/proxy_agent_shared/testdata/encodings/utf16le_no_bom.json new file mode 100644 index 0000000000000000000000000000000000000000..30c48539177cd71758162414ed9c8a5523e1f343 GIT binary patch literal 140 zcmb`5C2 literal 0 HcmV?d00001 diff --git a/proxy_agent_shared/testdata/encodings/utf32be_no_bom.json b/proxy_agent_shared/testdata/encodings/utf32be_no_bom.json new file mode 100644 index 0000000000000000000000000000000000000000..aad7d33ed632b8d3562c0890bf733a8590780cab GIT binary patch literal 276 zcmY+ov}3aD=%7D%f+y4fZalnD^v!i|kH%(|);!bMyPobe6R}o|vPg e`7!>^@tXa;ZGZaLmU9bp?w|hWz8CbccE=YtiyH<2 literal 0 HcmV?d00001 diff --git a/proxy_agent_shared/testdata/encodings/utf32le_no_bom.json b/proxy_agent_shared/testdata/encodings/utf32le_no_bom.json new file mode 100644 index 0000000000000000000000000000000000000000..62a347f02ab1d070e122dfe863d475bb7d330df1 GIT binary patch literal 276 zcmY+~TQ{bJZNTMC%Q1XmN(Q6LzrYfdTez=9u^7bIa^bdeeTnhja7$&UBWwBfcn5)BG9# b=a^QDx9v~=I&kh`&i&K>-1m+rtiACA3$Gao literal 0 HcmV?d00001 diff --git a/proxy_agent_shared/testdata/encodings/utf8_bom.json b/proxy_agent_shared/testdata/encodings/utf8_bom.json new file mode 100644 index 00000000..2cdc89e8 --- /dev/null +++ b/proxy_agent_shared/testdata/encodings/utf8_bom.json @@ -0,0 +1 @@ +{"name":"EncodingTest","code":7,"message":"café 测试 😀","enabled":true} \ No newline at end of file diff --git a/proxy_agent_shared/testdata/encodings/utf8_no_bom.json b/proxy_agent_shared/testdata/encodings/utf8_no_bom.json new file mode 100644 index 00000000..e81d358b --- /dev/null +++ b/proxy_agent_shared/testdata/encodings/utf8_no_bom.json @@ -0,0 +1 @@ +{"name":"EncodingTest","code":7,"message":"café 测试 😀","enabled":true} \ No newline at end of file From c3763c69f8a04f19fec4483c24f8b2856764bd1f Mon Sep 17 00:00:00 2001 From: yinbing Date: Thu, 30 Jul 2026 12:46:35 -0700 Subject: [PATCH 2/6] spell fixes --- proxy_agent_shared/src/misc_helpers.rs | 8 ++++---- .../encodings/utf16be_bom.json | Bin .../encodings/utf16be_no_bom.json | Bin .../encodings/utf16le_bom.json | Bin .../encodings/utf16le_no_bom.json | Bin .../encodings/utf32be_bom.json | Bin .../encodings/utf32be_no_bom.json | Bin .../encodings/utf32le_bom.json | Bin .../encodings/utf32le_no_bom.json | Bin .../{testdata => test_data}/encodings/utf8_bom.json | 0 .../encodings/utf8_no_bom.json | 0 11 files changed, 4 insertions(+), 4 deletions(-) rename proxy_agent_shared/{testdata => test_data}/encodings/utf16be_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf16be_no_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf16le_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf16le_no_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf32be_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf32be_no_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf32le_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf32le_no_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf8_bom.json (100%) rename proxy_agent_shared/{testdata => test_data}/encodings/utf8_no_bom.json (100%) diff --git a/proxy_agent_shared/src/misc_helpers.rs b/proxy_agent_shared/src/misc_helpers.rs index a78870de..645d785d 100644 --- a/proxy_agent_shared/src/misc_helpers.rs +++ b/proxy_agent_shared/src/misc_helpers.rs @@ -758,7 +758,7 @@ mod tests { // Latin-1 accent + CJK + an astral-plane emoji (a surrogate pair in // UTF-16) so multi-byte decoding and surrogate pairing are exercised, // not just the ASCII fast path. This is the exact `message` value - // stored in every fixture file under testdata/encodings. + // stored in every fixture file under test_data/encodings. const NON_ASCII_MESSAGE: &str = "caf\u{00e9} \u{6d4b}\u{8bd5} \u{1F600}"; #[derive(Serialize, Deserialize, PartialEq, Debug)] @@ -769,8 +769,8 @@ mod tests { enabled: bool, } - let testdata_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("testdata") + let test_data_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("test_data") .join("encodings"); let expected = EncodingTestStruct { @@ -798,7 +798,7 @@ mod tests { ]; for file_name in fixtures { - let file_path = testdata_dir.join(file_name); + let file_path = test_data_dir.join(file_name); assert!( file_path.exists(), "missing encoding fixture file: {}", diff --git a/proxy_agent_shared/testdata/encodings/utf16be_bom.json b/proxy_agent_shared/test_data/encodings/utf16be_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf16be_bom.json rename to proxy_agent_shared/test_data/encodings/utf16be_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf16be_no_bom.json b/proxy_agent_shared/test_data/encodings/utf16be_no_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf16be_no_bom.json rename to proxy_agent_shared/test_data/encodings/utf16be_no_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf16le_bom.json b/proxy_agent_shared/test_data/encodings/utf16le_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf16le_bom.json rename to proxy_agent_shared/test_data/encodings/utf16le_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf16le_no_bom.json b/proxy_agent_shared/test_data/encodings/utf16le_no_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf16le_no_bom.json rename to proxy_agent_shared/test_data/encodings/utf16le_no_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf32be_bom.json b/proxy_agent_shared/test_data/encodings/utf32be_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf32be_bom.json rename to proxy_agent_shared/test_data/encodings/utf32be_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf32be_no_bom.json b/proxy_agent_shared/test_data/encodings/utf32be_no_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf32be_no_bom.json rename to proxy_agent_shared/test_data/encodings/utf32be_no_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf32le_bom.json b/proxy_agent_shared/test_data/encodings/utf32le_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf32le_bom.json rename to proxy_agent_shared/test_data/encodings/utf32le_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf32le_no_bom.json b/proxy_agent_shared/test_data/encodings/utf32le_no_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf32le_no_bom.json rename to proxy_agent_shared/test_data/encodings/utf32le_no_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf8_bom.json b/proxy_agent_shared/test_data/encodings/utf8_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf8_bom.json rename to proxy_agent_shared/test_data/encodings/utf8_bom.json diff --git a/proxy_agent_shared/testdata/encodings/utf8_no_bom.json b/proxy_agent_shared/test_data/encodings/utf8_no_bom.json similarity index 100% rename from proxy_agent_shared/testdata/encodings/utf8_no_bom.json rename to proxy_agent_shared/test_data/encodings/utf8_no_bom.json From 7ba846237ce51e2c382a3e5fd80aa17d0c0d9acc Mon Sep 17 00:00:00 2001 From: yinbing Date: Thu, 30 Jul 2026 12:53:52 -0700 Subject: [PATCH 3/6] spelling fixes --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 43c149dc..7596475a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,4 +33,4 @@ Cargo.lock eol=lf # json_read_from_file encoding fixtures. These are UTF-16/UTF-32 and # BOM-prefixed files whose exact bytes are the thing under test, so they must # never be text-normalized despite the `*.json text` rule above. -proxy_agent_shared/testdata/encodings/* binary +proxy_agent_shared/test_data/encodings/* binary From 77422eeb2fd9ffa1502a4f9d471d0cb1e86cc3e7 Mon Sep 17 00:00:00 2001 From: yinbing Date: Tue, 4 Aug 2026 08:11:04 -0700 Subject: [PATCH 4/6] ut test update --- .gitattributes | 5 - proxy_agent_shared/src/misc_helpers.rs | 125 ++++++++++++++---- .../test_data/encodings/utf16be_bom.json | Bin 142 -> 0 bytes .../test_data/encodings/utf16be_no_bom.json | Bin 140 -> 0 bytes .../test_data/encodings/utf16le_bom.json | Bin 142 -> 0 bytes .../test_data/encodings/utf16le_no_bom.json | Bin 140 -> 0 bytes .../test_data/encodings/utf32be_bom.json | Bin 280 -> 0 bytes .../test_data/encodings/utf32be_no_bom.json | Bin 276 -> 0 bytes .../test_data/encodings/utf32le_bom.json | Bin 280 -> 0 bytes .../test_data/encodings/utf32le_no_bom.json | Bin 276 -> 0 bytes .../test_data/encodings/utf8_bom.json | 1 - .../test_data/encodings/utf8_no_bom.json | 1 - 12 files changed, 98 insertions(+), 34 deletions(-) delete mode 100644 proxy_agent_shared/test_data/encodings/utf16be_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf16be_no_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf16le_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf16le_no_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf32be_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf32be_no_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf32le_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf32le_no_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf8_bom.json delete mode 100644 proxy_agent_shared/test_data/encodings/utf8_no_bom.json diff --git a/.gitattributes b/.gitattributes index 7596475a..a2e13784 100644 --- a/.gitattributes +++ b/.gitattributes @@ -29,8 +29,3 @@ Cargo.lock eol=lf *.png binary *.wim binary *.zip binary - -# json_read_from_file encoding fixtures. These are UTF-16/UTF-32 and -# BOM-prefixed files whose exact bytes are the thing under test, so they must -# never be text-normalized despite the `*.json text` rule above. -proxy_agent_shared/test_data/encodings/* binary diff --git a/proxy_agent_shared/src/misc_helpers.rs b/proxy_agent_shared/src/misc_helpers.rs index 645d785d..8a2cc46e 100644 --- a/proxy_agent_shared/src/misc_helpers.rs +++ b/proxy_agent_shared/src/misc_helpers.rs @@ -757,8 +757,7 @@ mod tests { fn json_read_from_file_supports_all_ten_encodings_test() { // Latin-1 accent + CJK + an astral-plane emoji (a surrogate pair in // UTF-16) so multi-byte decoding and surrogate pairing are exercised, - // not just the ASCII fast path. This is the exact `message` value - // stored in every fixture file under test_data/encodings. + // not just the ASCII fast path. const NON_ASCII_MESSAGE: &str = "caf\u{00e9} \u{6d4b}\u{8bd5} \u{1F600}"; #[derive(Serialize, Deserialize, PartialEq, Debug)] @@ -769,10 +768,65 @@ mod tests { enabled: bool, } - let test_data_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("test_data") - .join("encodings"); + #[derive(Clone, Copy)] + enum Encoding { + Utf8, + Utf16Le, + Utf16Be, + Utf32Le, + Utf32Be, + } + + /// Encodes `text` into the raw bytes written to each test file. + fn encode(text: &str, encoding: Encoding, with_bom: bool) -> Vec { + let mut bytes = Vec::new(); + + if with_bom { + bytes.extend_from_slice(match encoding { + Encoding::Utf8 => &[0xEF, 0xBB, 0xBF][..], + Encoding::Utf16Le => &[0xFF, 0xFE][..], + Encoding::Utf16Be => &[0xFE, 0xFF][..], + Encoding::Utf32Le => &[0xFF, 0xFE, 0x00, 0x00][..], + Encoding::Utf32Be => &[0x00, 0x00, 0xFE, 0xFF][..], + }); + } + + match encoding { + Encoding::Utf8 => bytes.extend_from_slice(text.as_bytes()), + Encoding::Utf16Le => { + for unit in text.encode_utf16() { + bytes.extend_from_slice(&unit.to_le_bytes()); + } + } + Encoding::Utf16Be => { + for unit in text.encode_utf16() { + bytes.extend_from_slice(&unit.to_be_bytes()); + } + } + Encoding::Utf32Le => { + for ch in text.chars() { + bytes.extend_from_slice(&(ch as u32).to_le_bytes()); + } + } + Encoding::Utf32Be => { + for ch in text.chars() { + bytes.extend_from_slice(&(ch as u32).to_be_bytes()); + } + } + } + + bytes + } + let mut temp_test_path = env::temp_dir(); + temp_test_path.push("json_read_from_file_supports_all_ten_encodings_test"); + // clean up and ignore the clean up errors + _ = fs::remove_dir_all(&temp_test_path); + super::try_create_folder(&temp_test_path).unwrap(); + + let json = format!( + r#"{{"name":"EncodingTest","code":7,"message":"{NON_ASCII_MESSAGE}","enabled":true}}"# + ); let expected = EncodingTestStruct { name: "EncodingTest".to_string(), code: 7, @@ -780,36 +834,53 @@ mod tests { enabled: true, }; - // Pre-created fixture files, one per supported encoding. They hold the - // same JSON document, byte-for-byte encoded differently. The - // UTF-32LE-with-BOM file is the ambiguous one: its BOM starts with the - // UTF-16LE BOM. - let fixtures = [ - "utf8_bom.json", - "utf8_no_bom.json", - "utf16le_bom.json", - "utf16le_no_bom.json", - "utf16be_bom.json", - "utf16be_no_bom.json", - "utf32le_bom.json", - "utf32le_no_bom.json", - "utf32be_bom.json", - "utf32be_no_bom.json", + // The same JSON document written 10 times, byte-for-byte encoded + // differently. The UTF-32LE-with-BOM case is the ambiguous one: its BOM + // starts with the UTF-16LE BOM. + let combinations = [ + ("utf8_bom.json", Encoding::Utf8, true), + ("utf8_no_bom.json", Encoding::Utf8, false), + ("utf16le_bom.json", Encoding::Utf16Le, true), + ("utf16le_no_bom.json", Encoding::Utf16Le, false), + ("utf16be_bom.json", Encoding::Utf16Be, true), + ("utf16be_no_bom.json", Encoding::Utf16Be, false), + ("utf32le_bom.json", Encoding::Utf32Le, true), + ("utf32le_no_bom.json", Encoding::Utf32Le, false), + ("utf32be_bom.json", Encoding::Utf32Be, true), + ("utf32be_no_bom.json", Encoding::Utf32Be, false), ]; - for file_name in fixtures { - let file_path = test_data_dir.join(file_name); - assert!( - file_path.exists(), - "missing encoding fixture file: {}", - file_path.display() - ); + for (file_name, encoding, with_bom) in combinations { + let file_path = temp_test_path.join(file_name); + fs::write(&file_path, encode(&json, encoding, with_bom)).unwrap(); let actual = super::json_read_from_file::(&file_path) .unwrap_or_else(|e| panic!("{file_name}: {e}")); assert_eq!(expected, actual, "{file_name}: decoded payload differs"); } + + // Odd byte count cannot be a whole number of UTF-16 code units. + let truncated = temp_test_path.join("truncated_utf16.json"); + fs::write(&truncated, [0x7B, 0x00, 0x22]).unwrap(); + let error = super::json_read_from_file::(&truncated) + .unwrap_err() + .to_string(); + assert!(error.contains("truncated"), "{error}"); + + // 0x0011_0000 is one past the highest Unicode scalar value. + let bad_scalar = temp_test_path.join("bad_utf32_scalar.json"); + fs::write( + &bad_scalar, + [0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00], + ) + .unwrap(); + let error = super::json_read_from_file::(&bad_scalar) + .unwrap_err() + .to_string(); + assert!(error.contains("invalid scalar value"), "{error}"); + + _ = fs::remove_dir_all(&temp_test_path); } #[test] diff --git a/proxy_agent_shared/test_data/encodings/utf16be_bom.json b/proxy_agent_shared/test_data/encodings/utf16be_bom.json deleted file mode 100644 index 2a0812aa2af49a98cd4549b0dc6b8da12f09170c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 142 zcmezOpP`yTi6M_6ks+5M709vzVpkwPnIWGcg&`BjN@oZG$`mt{0A+N57$O5vYYr9x z=`04qM4(!j8OcCi8pBHlg8|tV@PDkWk?0Gtbo`R$WLa-XGmel1hUc@LVz;G3?)Dr9Uz9tK-8LpML;@> zfiMxM7G_2=ke9~rl0hNYyZb7G!VTMd3{d^4VB3-ya)4@6fMQlalZzNifxKD(xriDC diff --git a/proxy_agent_shared/test_data/encodings/utf16le_bom.json b/proxy_agent_shared/test_data/encodings/utf16le_bom.json deleted file mode 100644 index 1424668e10259965412383da9a5605d4b1941c95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 142 zcmezWubM%LA&()EA(tT)$g%=rS0F!`A)g_IArr_-X9xkx6f=|nWpscTA_Gxt4i*9F zEC#|vpjwz2$v|Ej!%GGQ2JhUf-3ko0HyG|g^rwPtOJc|Ys!ajgSi(@mPzvPLG5`SQ CnH&ND diff --git a/proxy_agent_shared/test_data/encodings/utf16le_no_bom.json b/proxy_agent_shared/test_data/encodings/utf16le_no_bom.json deleted file mode 100644 index 30c48539177cd71758162414ed9c8a5523e1f343..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 140 zcmb`5C2 diff --git a/proxy_agent_shared/test_data/encodings/utf32be_no_bom.json b/proxy_agent_shared/test_data/encodings/utf32be_no_bom.json deleted file mode 100644 index aad7d33ed632b8d3562c0890bf733a8590780cab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 276 zcmY+ov}3aD=%7D%f+y4fZalnD^v!i|kH%(|);!bMyPobe6R}o|vPg e`7!>^@tXa;ZGZaLmU9bp?w|hWz8CbccE=YtiyH<2 diff --git a/proxy_agent_shared/test_data/encodings/utf32le_no_bom.json b/proxy_agent_shared/test_data/encodings/utf32le_no_bom.json deleted file mode 100644 index 62a347f02ab1d070e122dfe863d475bb7d330df1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 276 zcmY+~TQ{bJZNTMC%Q1XmN(Q6LzrYfdTez=9u^7bIa^bdeeTnhja7$&UBWwBfcn5)BG9# b=a^QDx9v~=I&kh`&i&K>-1m+rtiACA3$Gao diff --git a/proxy_agent_shared/test_data/encodings/utf8_bom.json b/proxy_agent_shared/test_data/encodings/utf8_bom.json deleted file mode 100644 index 2cdc89e8..00000000 --- a/proxy_agent_shared/test_data/encodings/utf8_bom.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"EncodingTest","code":7,"message":"café 测试 😀","enabled":true} \ No newline at end of file diff --git a/proxy_agent_shared/test_data/encodings/utf8_no_bom.json b/proxy_agent_shared/test_data/encodings/utf8_no_bom.json deleted file mode 100644 index e81d358b..00000000 --- a/proxy_agent_shared/test_data/encodings/utf8_no_bom.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"EncodingTest","code":7,"message":"café 测试 😀","enabled":true} \ No newline at end of file From 4cff3b1ac52c87c7f484a67fbca1a2f02ad2423b Mon Sep 17 00:00:00 2001 From: yinbing Date: Tue, 4 Aug 2026 09:03:45 -0700 Subject: [PATCH 5/6] review comments --- proxy_agent_shared/src/misc_helpers.rs | 118 ++++++++++++++++--------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/proxy_agent_shared/src/misc_helpers.rs b/proxy_agent_shared/src/misc_helpers.rs index 8a2cc46e..f14f74ed 100644 --- a/proxy_agent_shared/src/misc_helpers.rs +++ b/proxy_agent_shared/src/misc_helpers.rs @@ -258,74 +258,108 @@ where T: DeserializeOwned, { let bytes = fs::read(file_path)?; - let text = decode_json_text(&bytes, file_path)?; + let text = decode_text(&bytes) + .map_err(|detail| Error::DecodeFile(path_to_string(file_path), detail))?; let obj: T = serde_json::from_str(&text)?; Ok(obj) } -/// Detects the text encoding of `bytes` and returns it as -/// (code unit width in bytes, big endian, BOM length in bytes). -/// width: 1 for UTF-8, 2 for UTF-16, 4 for UTF-32 -/// big_endian: true for BE, false for LE -/// bom length length of bom +/// The Unicode transformation format of the detected encoding. +#[derive(Clone, Copy)] +enum TextEncoding { + /// UTF-8 (and plain ASCII) - 1 byte per code unit. + Utf8, + /// UTF-16 - 2 bytes per code unit. + Utf16, + /// UTF-32 - 4 bytes per code unit. + Utf32, +} + +/// The text encoding detected from the leading bytes of a file. +struct DetectedEncoding { + /// The Unicode transformation format. + text_encoding: TextEncoding, + /// True for big endian, false for little endian. Not meaningful for UTF-8. + big_endian: bool, + /// Length of the BOM in bytes, 0 when the file has no BOM. + bom_len: usize, +} + +impl DetectedEncoding { + const fn new(text_encoding: TextEncoding, big_endian: bool, bom_len: usize) -> Self { + Self { + text_encoding, + big_endian, + bom_len, + } + } +} + +/// Detects the text encoding of `bytes`. +/// +/// A BOM is a Unicode construct rather than a JSON one, so BOM detection here is +/// format agnostic. The BOM-less fallback, however, assumes the document starts +/// with an ASCII character - true for JSON, XML and most text config formats. /// /// Wider BOMs must be tested first: the UTF-32LE BOM (FF FE 00 00) starts with /// the UTF-16LE BOM (FF FE), so a shortest-first scan would mis-detect a /// UTF-32LE file as UTF-16LE. -fn detect_json_encoding(bytes: &[u8]) -> (usize, bool, usize) { +fn detect_text_encoding(bytes: &[u8]) -> DetectedEncoding { if bytes.starts_with(&[0x00, 0x00, 0xFE, 0xFF]) { - (4, true, 4) // UTF-32BE with BOM + DetectedEncoding::new(TextEncoding::Utf32, true, 4) // UTF-32BE with BOM } else if bytes.starts_with(&[0xFF, 0xFE, 0x00, 0x00]) { - (4, false, 4) // UTF-32LE with BOM + DetectedEncoding::new(TextEncoding::Utf32, false, 4) // UTF-32LE with BOM } else if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) { - (1, false, 3) // UTF-8 with BOM + DetectedEncoding::new(TextEncoding::Utf8, false, 3) // UTF-8 with BOM } else if bytes.starts_with(&[0xFE, 0xFF]) { - (2, true, 2) // UTF-16BE with BOM + DetectedEncoding::new(TextEncoding::Utf16, true, 2) // UTF-16BE with BOM } else if bytes.starts_with(&[0xFF, 0xFE]) { - (2, false, 2) // UTF-16LE with BOM + DetectedEncoding::new(TextEncoding::Utf16, false, 2) // UTF-16LE with BOM } else { - // No BOM. A JSON document always starts with an ASCII character (`[`, - // `{`, `"`, a digit or whitespace), so the NUL padding around the first - // code unit identifies both the width and the byte order. The 4-byte - // patterns are checked first because they are a superset of the 2-byte - // ones. + // No BOM. The document is expected to start with an ASCII character + // (for JSON that is `[`, `{`, `"`, a digit or whitespace), so the NUL + // padding around the first code unit identifies both the width and the + // byte order. The 4-byte patterns are checked first because they are a + // superset of the 2-byte ones. let is_nul = |i: usize| bytes.get(i) == Some(&0x00); let is_text = |i: usize| matches!(bytes.get(i), Some(b) if *b != 0x00); if is_nul(0) && is_nul(1) && is_nul(2) && is_text(3) { - (4, true, 0) // 00 00 00 xx -> UTF-32BE + DetectedEncoding::new(TextEncoding::Utf32, true, 0) // 00 00 00 xx -> UTF-32BE } else if is_text(0) && is_nul(1) && is_nul(2) && is_nul(3) { - (4, false, 0) // xx 00 00 00 -> UTF-32LE + DetectedEncoding::new(TextEncoding::Utf32, false, 0) // xx 00 00 00 -> UTF-32LE } else if is_nul(0) && is_text(1) { - (2, true, 0) // 00 xx -> UTF-16BE + DetectedEncoding::new(TextEncoding::Utf16, true, 0) // 00 xx -> UTF-16BE } else if is_text(0) && is_nul(1) { - (2, false, 0) // xx 00 -> UTF-16LE + DetectedEncoding::new(TextEncoding::Utf16, false, 0) // xx 00 -> UTF-16LE } else { - (1, false, 0) // anything else, including plain ASCII / UTF-8 + DetectedEncoding::new(TextEncoding::Utf8, false, 0) // anything else, including plain ASCII / UTF-8 } } } /// Decodes `bytes` into UTF-8 text using the encoding detected by -/// [`detect_json_encoding`], skipping the BOM when present. +/// [`detect_text_encoding`], skipping the BOM when present. /// UTF-8 input is borrowed as-is, so the common case does not allocate. -fn decode_json_text<'a>(bytes: &'a [u8], file_path: &Path) -> Result> { - let decode_error = |detail: String| Error::DecodeFile(path_to_string(file_path), detail); - - let (code_unit_len, big_endian, bom_len) = detect_json_encoding(bytes); - let payload = &bytes[bom_len..]; - - match code_unit_len { - 1 => std::str::from_utf8(payload) +/// +/// On failure it returns only a description of what made the content +/// undecodable; the caller attaches the source of the bytes. +fn decode_text(bytes: &[u8]) -> std::result::Result, String> { + let encoding = detect_text_encoding(bytes); + let big_endian = encoding.big_endian; + let payload = &bytes[encoding.bom_len..]; + + match encoding.text_encoding { + TextEncoding::Utf8 => std::str::from_utf8(payload) .map(Cow::Borrowed) - .map_err(|e| decode_error(format!("content is not valid UTF-8: {e}"))), - 2 => { + .map_err(|e| format!("content is not valid UTF-8: {e}")), + TextEncoding::Utf16 => { if !payload.len().is_multiple_of(2) { - return Err(decode_error(format!( + return Err(format!( "UTF-16 content is truncated: {} bytes is not a whole number of 16-bit code units", payload.len() - ))); + )); } let code_units = payload.chunks_exact(2).map(|chunk| { @@ -343,14 +377,14 @@ fn decode_json_text<'a>(bytes: &'a [u8], file_path: &Path) -> Result>() .map(Cow::Owned) - .map_err(|e| decode_error(format!("UTF-16 content has an unpaired surrogate: {e}"))) + .map_err(|e| format!("UTF-16 content has an unpaired surrogate: {e}")) } - _ => { + TextEncoding::Utf32 => { if !payload.len().is_multiple_of(4) { - return Err(decode_error(format!( + return Err(format!( "UTF-32 content is truncated: {} bytes is not a whole number of 32-bit code units", payload.len() - ))); + )); } payload @@ -363,12 +397,10 @@ fn decode_json_text<'a>(bytes: &'a [u8], file_path: &Path) -> Result>() + .collect::>() .map(Cow::Owned) } } From 62b1f31bd9d59d280f7d27cbcd5bd4475f96526e Mon Sep 17 00:00:00 2001 From: yinbing Date: Tue, 4 Aug 2026 09:12:02 -0700 Subject: [PATCH 6/6] unit test --- proxy_agent_shared/src/misc_helpers.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/proxy_agent_shared/src/misc_helpers.rs b/proxy_agent_shared/src/misc_helpers.rs index f14f74ed..673d4591 100644 --- a/proxy_agent_shared/src/misc_helpers.rs +++ b/proxy_agent_shared/src/misc_helpers.rs @@ -915,6 +915,26 @@ mod tests { _ = fs::remove_dir_all(&temp_test_path); } + #[test] + fn detect_text_encoding_short_input_test() { + use super::TextEncoding; + + /// True when `bytes` is detected as plain UTF-8 with no BOM. + fn is_utf8_no_bom(bytes: &[u8]) -> bool { + let detected = super::detect_text_encoding(bytes); + matches!(detected.text_encoding, TextEncoding::Utf8) + && !detected.big_endian + && detected.bom_len == 0 + } + + // 1. An empty input is UTF-8 with no BOM. + assert!(is_utf8_no_bom(&[]), "empty input"); + // 2. A single ASCII byte is UTF-8 with no BOM. + assert!(is_utf8_no_bom(&[0]), "single NUL byte"); + // 3. Two bytes that are invalid + assert!(is_utf8_no_bom(&[1, 2]), "two invalid bytes"); + } + #[test] fn path_to_string_test() { let path = "path_to_string_test";