From d2bf0ad56c2c6ca3ea3a461396afec993151d418 Mon Sep 17 00:00:00 2001 From: ghosteau Date: Thu, 3 Sep 2026 00:26:28 -0400 Subject: [PATCH 1/2] fix: repair mojibake introduced by the American English pass That pass was scripted with PowerShell, and Get-Content -Raw decodes as the ANSI codepage on Windows PowerShell 5.1 rather than UTF-8. Every non-ASCII character round-tripped through cp1252 and was re-encoded, so the ellipsis and arrow characters turned into "a-hat" sequences -- visible in the legend ("768D -> 3D") and in the projection loading message. The result is still valid UTF-8, so it compiled, passed tests, and shipped. cp1252 has undefined byte values that decode to U+FFFD, so the transform loses information and cannot be reversed byte-for-byte. The five affected files are instead restored from 4f7c116, the commit before the pass; nothing has touched them since. Set-Content -Encoding utf8 had also added BOMs to three further files, which are stripped here. Adds scripts/check_encoding.py and wires it into CI as its own job. This class of corruption is invisible to both the type-checker and the test suite -- the only symptom is garbage in the rendered UI -- so it needs a check that looks at the bytes. Verified it fails on reintroduced corruption and passes on the repaired tree. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 13 +++ backend/app/core/visualizer.py | 10 +- frontend/src/components/Legend.tsx | 4 +- frontend/src/components/SearchPanel.tsx | 2 +- frontend/src/components/VizControls.tsx | 6 +- frontend/src/components/scene/PointCloud.tsx | 4 +- frontend/src/lib/api.ts | 2 +- frontend/src/lib/layout.ts | 2 +- frontend/src/store/useStore.ts | 12 +-- scripts/check_encoding.py | 97 ++++++++++++++++++++ 10 files changed, 131 insertions(+), 21 deletions(-) create mode 100644 scripts/check_encoding.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 452ff38..1008475 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,19 @@ concurrency: cancel-in-progress: true jobs: + encoding: + name: Text encoding + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + # Cheap, and catches a corruption that is invisible to the compiler and + # the test suite: double-encoded UTF-8 still parses fine and only shows + # up as garbage characters in the rendered UI. + - run: python scripts/check_encoding.py + backend: name: Backend (pytest) runs-on: ubuntu-latest diff --git a/backend/app/core/visualizer.py b/backend/app/core/visualizer.py index 7eb6179..4be0738 100644 --- a/backend/app/core/visualizer.py +++ b/backend/app/core/visualizer.py @@ -1,16 +1,16 @@ -"""Embedding analysis: the :class:`LoadedModel` value object. +"""Embedding analysis: the :class:`LoadedModel` value object. A :class:`LoadedModel` is an immutable-after-construction snapshot of one transformer's token embedding space, together with derived metadata and an LRU cache of UMAP projections. All of the geometric queries the API exposes (neighbors, comparisons, search, statistics) are methods here, operating purely -on NumPy arrays — there is no global state and no HTTP awareness, which makes +on NumPy arrays — there is no global state and no HTTP awareness, which makes the maths trivial to unit-test and safe to share across concurrent requests. The split of responsibilities is deliberate: -* :class:`LoadedModel` — *what* we know about an already-loaded model. -* :class:`~app.core.model_manager.ModelManager` — *when* models are loaded, +* :class:`LoadedModel` — *what* we know about an already-loaded model. +* :class:`~app.core.model_manager.ModelManager` — *when* models are loaded, cached, and evicted. """ @@ -68,7 +68,7 @@ def __init__( ) -> None: self.name = name # Record the full-vocabulary dimensions for reporting, but only retain - # the analyzed subset below — holding the entire embedding matrix as + # the analyzed subset below — holding the entire embedding matrix as # well would roughly double memory for large models. self.vocabulary_size = int(embeddings.shape[0]) self.embedding_dimension = int(embeddings.shape[1]) diff --git a/frontend/src/components/Legend.tsx b/frontend/src/components/Legend.tsx index ca673a7..c8b14d6 100644 --- a/frontend/src/components/Legend.tsx +++ b/frontend/src/components/Legend.tsx @@ -1,4 +1,4 @@ -/** Projection legend and client-side dataset export. */ +/** Projection legend and client-side dataset export. */ import { useStore } from "../store/useStore"; import { TOKEN_COLORS, TOKEN_TYPE_LABELS } from "../lib/tokenColors"; @@ -75,7 +75,7 @@ export function Legend() {
{vocab ? `${vocab.toLocaleString()} vocabulary / ` : ""} - {vizData.statistics.original_dimension}D → {vizData.statistics.reduced_dimension}D / UMAP {vizData.config.metric} + {vizData.statistics.original_dimension}D → {vizData.statistics.reduced_dimension}D / UMAP {vizData.config.metric}
); diff --git a/frontend/src/components/SearchPanel.tsx b/frontend/src/components/SearchPanel.tsx index 29445dd..12a9c4f 100644 --- a/frontend/src/components/SearchPanel.tsx +++ b/frontend/src/components/SearchPanel.tsx @@ -1,4 +1,4 @@ -/** Debounced token lookup across the complete analyzed vocabulary subset. */ +/** Debounced token lookup across the complete analyzed vocabulary subset. */ import { useEffect, useState } from "react"; import { formatToken } from "../lib/tokenFormat"; diff --git a/frontend/src/components/VizControls.tsx b/frontend/src/components/VizControls.tsx index cac8e98..b200ece 100644 --- a/frontend/src/components/VizControls.tsx +++ b/frontend/src/components/VizControls.tsx @@ -1,4 +1,4 @@ -/** UMAP projection controls; re-projects on demand once a model is loaded. */ +/** UMAP projection controls; re-projects on demand once a model is loaded. */ import { useStore } from "../store/useStore"; import { Field, SegToggle, Slider } from "./ui"; @@ -78,10 +78,10 @@ export function VizControls() { disabled={disabled} onClick={() => void regenerate().catch(() => undefined)} > - {loadState === "visualizing" ? "Computing…" : dirty ? "Apply projection" : "Recompute"} + {loadState === "visualizing" ? "Computing…" : dirty ? "Apply projection" : "Recompute"} - {/* Visible-points is a purely client-side declutter — no re-projection. + {/* Visible-points is a purely client-side declutter — no re-projection. The full projection is always retained so any token can still be searched, inspected, and compared. */} {totalPoints > 0 && ( diff --git a/frontend/src/components/scene/PointCloud.tsx b/frontend/src/components/scene/PointCloud.tsx index 0fc77e9..5de5ebd 100644 --- a/frontend/src/components/scene/PointCloud.tsx +++ b/frontend/src/components/scene/PointCloud.tsx @@ -1,11 +1,11 @@ -/** +/** * The interactive token point cloud. * * Renders the visible tokens as a single GPU-friendly THREE.Points object (one * draw call). Normalized coordinates are shared via the store so the camera rig * uses identical positions. Hover/click use r3f's points raycasting * (`event.index`). The current selection and its neighbors are emphasised with - * an overlay that reads from the *full* position buffer — so a token can be + * an overlay that reads from the *full* position buffer — so a token can be * highlighted even when it sits outside the visible-points cutoff. */ diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 35e98fb..4d4cc3d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,4 +1,4 @@ -/** +/** * Typed client for the Embeddings Visualizer backend. * * Every query endpoint is model-keyed (the backend has no notion of a single diff --git a/frontend/src/lib/layout.ts b/frontend/src/lib/layout.ts index b83f94e..ed86514 100644 --- a/frontend/src/lib/layout.ts +++ b/frontend/src/lib/layout.ts @@ -1,4 +1,4 @@ -/** +/** * Geometry helpers for laying out the projected token cloud. * * UMAP output lives in an arbitrary coordinate range, so we normalize it into a diff --git a/frontend/src/store/useStore.ts b/frontend/src/store/useStore.ts index 10d8d03..956cd0f 100644 --- a/frontend/src/store/useStore.ts +++ b/frontend/src/store/useStore.ts @@ -1,4 +1,4 @@ -/** +/** * Central application state (Zustand). * * Holds the model-loading lifecycle, the active visualization payload, the @@ -161,7 +161,7 @@ export const useStore = create((set, get) => ({ const model = get().selectedModelId.trim(); if (!model) return; - set({ loadState: "loading", loadProgress: "Requesting model…", comparison: null }); + set({ loadState: "loading", loadProgress: "Requesting model…", comparison: null }); // Poll the status endpoint for live progress while /load runs server-side. // The load happens in a worker thread, so the event loop keeps serving @@ -186,7 +186,7 @@ export const useStore = create((set, get) => ({ set({ loadedModel: model, loadState: "visualizing", - loadProgress: "Projecting embeddings… (first projection can take ~30s)", + loadProgress: "Projecting embeddings… (first projection can take ~30s)", accentGlowHex: theme.glowHex, themeLabel: theme.label, }); @@ -197,7 +197,7 @@ export const useStore = create((set, get) => ({ .then((statistics) => set({ statistics })) .catch(() => undefined); set({ loadState: "ready" }); - get().pushToast("success", `${model} loaded — ${get().vizData?.tokens.length ?? 0} tokens projected.`); + get().pushToast("success", `${model} loaded — ${get().vizData?.tokens.length ?? 0} tokens projected.`); } catch (e) { const msg = e instanceof ApiError ? e.message : "Failed to load model"; set({ loadState: "error", loadProgress: msg }); @@ -211,7 +211,7 @@ export const useStore = create((set, get) => ({ const model = get().loadedModel; if (!model) return; const wasReady = get().loadState === "ready"; - if (wasReady) set({ loadState: "visualizing", loadProgress: "Re-projecting…" }); + if (wasReady) set({ loadState: "visualizing", loadProgress: "Re-projecting…" }); try { const vizData = await api.createVisualization(model, get().config); set({ @@ -253,7 +253,7 @@ export const useStore = create((set, get) => ({ } }, - // Select a token AND fly the camera to it — used when inspecting a token that + // Select a token AND fly the camera to it — used when inspecting a token that // may be off-screen (e.g. chosen from search or comparison results). focusOn: async (index) => { set({ focus: { index, nonce: ++focusSeq } }); diff --git a/scripts/check_encoding.py b/scripts/check_encoding.py new file mode 100644 index 0000000..3b2a6e2 --- /dev/null +++ b/scripts/check_encoding.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Fail if any tracked text file has broken text encoding. + +Guards against a specific, easy-to-miss corruption: reading a UTF-8 file with +a tool that assumes the legacy Windows ANSI codepage, then writing it back as +UTF-8. That double-encodes every non-ASCII character -- "…" becomes "…" and +"→" becomes "→" -- and because the result is still *valid* UTF-8, nothing +downstream complains. It compiles, it passes tests, and it only shows up as +garbage in the rendered UI. + +It also rejects UTF-8 BOMs, which some Windows editors and PowerShell's +Set-Content add silently. + +Run from the repository root: python scripts/check_encoding.py +""" + +from __future__ import annotations + +import subprocess +import sys +import unicodedata + +TEXT_SUFFIXES = ( + ".ts", ".tsx", ".js", ".jsx", ".py", ".md", ".json", ".yml", ".yaml", + ".css", ".html", ".txt", ".example", ".toml", ".cfg", ".sh", +) + +# Each of these is what a common UTF-8 character looks like after being +# misread as cp1252. None of them is plausible in hand-written source. +MOJIBAKE = ("â€", "â†", "’", "“", "â", "é", "è", "ï", "·", "¶", "°") + +REPLACEMENT = "�" # U+FFFD, left behind when a decode gives up +BOM = b"\xef\xbb\xbf" + + +def tracked_text_files() -> list[str]: + out = subprocess.run( + ["git", "ls-files"], capture_output=True, text=True, check=True + ).stdout + return [line for line in out.splitlines() if line.endswith(TEXT_SUFFIXES)] + + +def main() -> int: + problems: list[str] = [] + scanned = 0 + + for rel in tracked_text_files(): + try: + raw = open(rel, "rb").read() + except FileNotFoundError: + continue # tracked but deleted in the working tree + scanned += 1 + + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + problems.append(f"{rel}: not valid UTF-8 ({exc})") + continue + + if raw.startswith(BOM): + problems.append(f"{rel}: starts with a UTF-8 BOM") + + for marker in MOJIBAKE: + if marker in text: + line = next( + (i for i, l in enumerate(text.splitlines(), 1) if marker in l), + 0, + ) + problems.append( + f"{rel}:{line}: mojibake {marker!r} " + f"(UTF-8 read as cp1252, then re-encoded)" + ) + break + + if REPLACEMENT in text: + problems.append(f"{rel}: contains U+FFFD, so characters were lost") + + print(f"checked {scanned} tracked text files") + + if problems: + print("\nEncoding problems found:\n") + for problem in problems: + print(f" {problem}") + print( + "\nRe-read the affected file as UTF-8 and rewrite it as UTF-8. On " + "Windows PowerShell 5.1 that means passing -Encoding utf8 to " + "Get-Content as well as Set-Content -- the default for reading is " + "the ANSI codepage, which is what causes this." + ) + return 1 + + print("no encoding problems") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 329b04f4263b409fd23bd8888d8f6eb9042a9157 Mon Sep 17 00:00:00 2001 From: ghosteau Date: Thu, 3 Sep 2026 00:33:53 -0400 Subject: [PATCH 2/2] fix: keep the encoding checker ASCII so it stops flagging itself The first CI run failed on the checker's own source. Its patterns are, by definition, the character sequences it hunts for, so spelling them literally made the file trip its own check -- and the U+FFFD constant did the same. Excluding the file from the scan would have been the easy fix, but then genuine corruption in it would go unnoticed. Instead the patterns are built from numeric code points, so the source stays pure ASCII and the checker can scan itself like any other file. Verified: source is ASCII, the code points still decode to the intended pairs, all four sample corruptions are matched, exit 1 on injected corruption and exit 0 on the clean tree. Co-Authored-By: Claude Opus 5 --- scripts/check_encoding.py | 67 +++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/scripts/check_encoding.py b/scripts/check_encoding.py index 3b2a6e2..c5b38d6 100644 --- a/scripts/check_encoding.py +++ b/scripts/check_encoding.py @@ -3,14 +3,21 @@ Guards against a specific, easy-to-miss corruption: reading a UTF-8 file with a tool that assumes the legacy Windows ANSI codepage, then writing it back as -UTF-8. That double-encodes every non-ASCII character -- "…" becomes "…" and -"→" becomes "→" -- and because the result is still *valid* UTF-8, nothing -downstream complains. It compiles, it passes tests, and it only shows up as -garbage in the rendered UI. +UTF-8. That double-encodes every non-ASCII character -- an ellipsis becomes a +three-character sequence starting with a-circumflex -- and because the result +is still *valid* UTF-8, nothing downstream complains. It compiles, it passes +tests, and it only shows up as garbage in the rendered UI. It also rejects UTF-8 BOMs, which some Windows editors and PowerShell's Set-Content add silently. +This file is deliberately pure ASCII: the characters it searches for are built +from numeric code points rather than typed out. Spelled literally they would +appear in this file's own source and the checker would flag itself -- which is +exactly what happened the first time it ran in CI. Staying ASCII means it can +scan itself like any other file instead of needing an exemption, so real +corruption here is still caught. Keep it that way. + Run from the repository root: python scripts/check_encoding.py """ @@ -18,18 +25,33 @@ import subprocess import sys -import unicodedata TEXT_SUFFIXES = ( ".ts", ".tsx", ".js", ".jsx", ".py", ".md", ".json", ".yml", ".yaml", ".css", ".html", ".txt", ".example", ".toml", ".cfg", ".sh", ) -# Each of these is what a common UTF-8 character looks like after being -# misread as cp1252. None of them is plausible in hand-written source. -MOJIBAKE = ("â€", "â†", "’", "“", "â", "é", "è", "ï", "·", "¶", "°") +# What common UTF-8 lead bytes look like once misread as cp1252. A UTF-8 +# sequence beginning C2/C3 (Latin-1 supplement) or E2 (punctuation, arrows) +# decodes under cp1252 to one of these pairs. None is plausible in +# hand-written source text. +_MOJIBAKE_CODEPOINTS = ( + (0x00E2, 0x20AC), # E2 80 xx: ellipsis, em dash, curly quotes + (0x00E2, 0x2020), # E2 86 xx: arrows + (0x00C3, 0x00A2), # C3 A2: a-circumflex + (0x00C3, 0x00A9), # C3 A9: e-acute + (0x00C3, 0x00A8), # C3 A8: e-grave + (0x00C3, 0x00AF), # C3 AF: i-diaeresis + (0x00C2, 0x00B7), # C2 B7: middle dot + (0x00C2, 0x00B6), # C2 B6: pilcrow + (0x00C2, 0x00B0), # C2 B0: degree sign + (0x00C2, 0x00A0), # C2 A0: non-breaking space +) + +MOJIBAKE = tuple("".join(chr(point) for point in pair) + for pair in _MOJIBAKE_CODEPOINTS) -REPLACEMENT = "�" # U+FFFD, left behind when a decode gives up +REPLACEMENT = chr(0xFFFD) # left behind when a decode gives up BOM = b"\xef\xbb\xbf" @@ -40,13 +62,26 @@ def tracked_text_files() -> list[str]: return [line for line in out.splitlines() if line.endswith(TEXT_SUFFIXES)] +def first_line_containing(text: str, needle: str) -> int: + for number, line in enumerate(text.splitlines(), 1): + if needle in line: + return number + return 0 + + +def describe(text: str) -> str: + """Render a needle as escapes, so the report stays ASCII and unambiguous.""" + return "".join(f"\\u{ord(char):04x}" for char in text) + + def main() -> int: problems: list[str] = [] scanned = 0 for rel in tracked_text_files(): try: - raw = open(rel, "rb").read() + with open(rel, "rb") as handle: + raw = handle.read() except FileNotFoundError: continue # tracked but deleted in the working tree scanned += 1 @@ -62,18 +97,18 @@ def main() -> int: for marker in MOJIBAKE: if marker in text: - line = next( - (i for i, l in enumerate(text.splitlines(), 1) if marker in l), - 0, - ) problems.append( - f"{rel}:{line}: mojibake {marker!r} " + f"{rel}:{first_line_containing(text, marker)}: " + f"mojibake {describe(marker)} " f"(UTF-8 read as cp1252, then re-encoded)" ) break if REPLACEMENT in text: - problems.append(f"{rel}: contains U+FFFD, so characters were lost") + problems.append( + f"{rel}:{first_line_containing(text, REPLACEMENT)}: " + f"contains U+FFFD, so characters were lost" + ) print(f"checked {scanned} tracked text files")