Fix mojibake

.github/workflows/ci.yml

@@ -12,6 +12,19 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  encoding:
+    name: Text encoding
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      # Cheap, and catches a corruption that is invisible to the compiler and
+      # the test suite: double-encoded UTF-8 still parses fine and only shows
+      # up as garbage characters in the rendered UI.
+      - run: python scripts/check_encoding.py
+
   backend:
     name: Backend (pytest)
     runs-on: ubuntu-latest

backend/app/core/visualizer.py

@@ -1,16 +1,16 @@
-"""Embedding analysis: the :class:`LoadedModel` value object.
+"""Embedding analysis: the :class:`LoadedModel` value object.
 
 A :class:`LoadedModel` is an immutable-after-construction snapshot of one
 transformer's token embedding space, together with derived metadata and an
 LRU cache of UMAP projections. All of the geometric queries the API exposes
 (neighbors, comparisons, search, statistics) are methods here, operating purely
-on NumPy arrays — there is no global state and no HTTP awareness, which makes
+on NumPy arrays — there is no global state and no HTTP awareness, which makes
 the maths trivial to unit-test and safe to share across concurrent requests.
 
 The split of responsibilities is deliberate:
 
-* :class:`LoadedModel` — *what* we know about an already-loaded model.
-* :class:`~app.core.model_manager.ModelManager` — *when* models are loaded,
+* :class:`LoadedModel` — *what* we know about an already-loaded model.
+* :class:`~app.core.model_manager.ModelManager` — *when* models are loaded,
   cached, and evicted.
 """
 
@@ -68,7 +68,7 @@ def __init__(
     ) -> None:
         self.name = name
         # Record the full-vocabulary dimensions for reporting, but only retain
-        # the analyzed subset below — holding the entire embedding matrix as
+        # the analyzed subset below — holding the entire embedding matrix as
         # well would roughly double memory for large models.
         self.vocabulary_size = int(embeddings.shape[0])
         self.embedding_dimension = int(embeddings.shape[1])

frontend/src/components/Legend.tsx

@@ -1,4 +1,4 @@
-/** Projection legend and client-side dataset export. */
+/** Projection legend and client-side dataset export. */
 
 import { useStore } from "../store/useStore";
 import { TOKEN_COLORS, TOKEN_TYPE_LABELS } from "../lib/tokenColors";
@@ -75,7 +75,7 @@ export function Legend() {
 
       <div className="mt-3 border-t border-line pt-2 font-mono text-[10px] leading-relaxed text-faint">
         {vocab ? `${vocab.toLocaleString()} vocabulary / ` : ""}
-        {vizData.statistics.original_dimension}D → {vizData.statistics.reduced_dimension}D / UMAP {vizData.config.metric}
+        {vizData.statistics.original_dimension}D → {vizData.statistics.reduced_dimension}D / UMAP {vizData.config.metric}
       </div>
     </div>
   );

frontend/src/components/SearchPanel.tsx

@@ -1,4 +1,4 @@
-/** Debounced token lookup across the complete analyzed vocabulary subset. */
+/** Debounced token lookup across the complete analyzed vocabulary subset. */
 
 import { useEffect, useState } from "react";
 import { formatToken } from "../lib/tokenFormat";

frontend/src/components/VizControls.tsx

@@ -1,4 +1,4 @@
-/** UMAP projection controls; re-projects on demand once a model is loaded. */
+/** UMAP projection controls; re-projects on demand once a model is loaded. */
 
 import { useStore } from "../store/useStore";
 import { Field, SegToggle, Slider } from "./ui";
@@ -78,10 +78,10 @@ export function VizControls() {
         disabled={disabled}
         onClick={() => void regenerate().catch(() => undefined)}
       >
-        {loadState === "visualizing" ? "Computing…" : dirty ? "Apply projection" : "Recompute"}
+        {loadState === "visualizing" ? "Computing…" : dirty ? "Apply projection" : "Recompute"}
       </button>
 
-      {/* Visible-points is a purely client-side declutter — no re-projection.
+      {/* Visible-points is a purely client-side declutter — no re-projection.
           The full projection is always retained so any token can still be
           searched, inspected, and compared. */}
       {totalPoints > 0 && (

frontend/src/components/scene/PointCloud.tsx

@@ -1,11 +1,11 @@
-/**
+/**
  * The interactive token point cloud.
  *
  * Renders the visible tokens as a single GPU-friendly THREE.Points object (one
  * draw call). Normalized coordinates are shared via the store so the camera rig
  * uses identical positions. Hover/click use r3f's points raycasting
  * (`event.index`). The current selection and its neighbors are emphasised with
- * an overlay that reads from the *full* position buffer — so a token can be
+ * an overlay that reads from the *full* position buffer — so a token can be
  * highlighted even when it sits outside the visible-points cutoff.
  */
 

frontend/src/lib/api.ts

@@ -1,4 +1,4 @@
-/**
+/**
  * Typed client for the Embeddings Visualizer backend.
  *
  * Every query endpoint is model-keyed (the backend has no notion of a single

frontend/src/lib/layout.ts

@@ -1,4 +1,4 @@
-/**
+/**
  * Geometry helpers for laying out the projected token cloud.
  *
  * UMAP output lives in an arbitrary coordinate range, so we normalize it into a

frontend/src/store/useStore.ts

@@ -1,4 +1,4 @@
-/**
+/**
  * Central application state (Zustand).
  *
  * Holds the model-loading lifecycle, the active visualization payload, the
@@ -161,7 +161,7 @@ export const useStore = create<AppState>((set, get) => ({
     const model = get().selectedModelId.trim();
     if (!model) return;
 
-    set({ loadState: "loading", loadProgress: "Requesting model…", comparison: null });
+    set({ loadState: "loading", loadProgress: "Requesting model…", comparison: null });
 
     // Poll the status endpoint for live progress while /load runs server-side.
     // The load happens in a worker thread, so the event loop keeps serving
@@ -186,7 +186,7 @@ export const useStore = create<AppState>((set, get) => ({
       set({
         loadedModel: model,
         loadState: "visualizing",
-        loadProgress: "Projecting embeddings… (first projection can take ~30s)",
+        loadProgress: "Projecting embeddings… (first projection can take ~30s)",
         accentGlowHex: theme.glowHex,
         themeLabel: theme.label,
       });
@@ -197,7 +197,7 @@ export const useStore = create<AppState>((set, get) => ({
         .then((statistics) => set({ statistics }))
         .catch(() => undefined);
       set({ loadState: "ready" });
-      get().pushToast("success", `${model} loaded — ${get().vizData?.tokens.length ?? 0} tokens projected.`);
+      get().pushToast("success", `${model} loaded — ${get().vizData?.tokens.length ?? 0} tokens projected.`);
     } catch (e) {
       const msg = e instanceof ApiError ? e.message : "Failed to load model";
       set({ loadState: "error", loadProgress: msg });
@@ -211,7 +211,7 @@ export const useStore = create<AppState>((set, get) => ({
     const model = get().loadedModel;
     if (!model) return;
     const wasReady = get().loadState === "ready";
-    if (wasReady) set({ loadState: "visualizing", loadProgress: "Re-projecting…" });
+    if (wasReady) set({ loadState: "visualizing", loadProgress: "Re-projecting…" });
     try {
       const vizData = await api.createVisualization(model, get().config);
       set({
@@ -253,7 +253,7 @@ export const useStore = create<AppState>((set, get) => ({
     }
   },
 
-  // Select a token AND fly the camera to it — used when inspecting a token that
+  // Select a token AND fly the camera to it — used when inspecting a token that
   // may be off-screen (e.g. chosen from search or comparison results).
   focusOn: async (index) => {
     set({ focus: { index, nonce: ++focusSeq } });

scripts/check_encoding.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Fail if any tracked text file has broken text encoding.
+
+Guards against a specific, easy-to-miss corruption: reading a UTF-8 file with
+a tool that assumes the legacy Windows ANSI codepage, then writing it back as
+UTF-8. That double-encodes every non-ASCII character -- an ellipsis becomes a
+three-character sequence starting with a-circumflex -- and because the result
+is still *valid* UTF-8, nothing downstream complains. It compiles, it passes
+tests, and it only shows up as garbage in the rendered UI.
+
+It also rejects UTF-8 BOMs, which some Windows editors and PowerShell's
+Set-Content add silently.
+
+This file is deliberately pure ASCII: the characters it searches for are built
+from numeric code points rather than typed out. Spelled literally they would
+appear in this file's own source and the checker would flag itself -- which is
+exactly what happened the first time it ran in CI. Staying ASCII means it can
+scan itself like any other file instead of needing an exemption, so real
+corruption here is still caught. Keep it that way.
+
+Run from the repository root:  python scripts/check_encoding.py
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+TEXT_SUFFIXES = (
+    ".ts", ".tsx", ".js", ".jsx", ".py", ".md", ".json", ".yml", ".yaml",
+    ".css", ".html", ".txt", ".example", ".toml", ".cfg", ".sh",
+)
+
+# What common UTF-8 lead bytes look like once misread as cp1252. A UTF-8
+# sequence beginning C2/C3 (Latin-1 supplement) or E2 (punctuation, arrows)
+# decodes under cp1252 to one of these pairs. None is plausible in
+# hand-written source text.
+_MOJIBAKE_CODEPOINTS = (
+    (0x00E2, 0x20AC),  # E2 80 xx: ellipsis, em dash, curly quotes
+    (0x00E2, 0x2020),  # E2 86 xx: arrows
+    (0x00C3, 0x00A2),  # C3 A2: a-circumflex
+    (0x00C3, 0x00A9),  # C3 A9: e-acute
+    (0x00C3, 0x00A8),  # C3 A8: e-grave
+    (0x00C3, 0x00AF),  # C3 AF: i-diaeresis
+    (0x00C2, 0x00B7),  # C2 B7: middle dot
+    (0x00C2, 0x00B6),  # C2 B6: pilcrow
+    (0x00C2, 0x00B0),  # C2 B0: degree sign
+    (0x00C2, 0x00A0),  # C2 A0: non-breaking space
+)
+
+MOJIBAKE = tuple("".join(chr(point) for point in pair)
+                 for pair in _MOJIBAKE_CODEPOINTS)
+
+REPLACEMENT = chr(0xFFFD)  # left behind when a decode gives up
+BOM = b"\xef\xbb\xbf"
+
+
+def tracked_text_files() -> list[str]:
+    out = subprocess.run(
+        ["git", "ls-files"], capture_output=True, text=True, check=True
+    ).stdout
+    return [line for line in out.splitlines() if line.endswith(TEXT_SUFFIXES)]
+
+
+def first_line_containing(text: str, needle: str) -> int:
+    for number, line in enumerate(text.splitlines(), 1):
+        if needle in line:
+            return number
+    return 0
+
+
+def describe(text: str) -> str:
+    """Render a needle as escapes, so the report stays ASCII and unambiguous."""
+    return "".join(f"\\u{ord(char):04x}" for char in text)
+
+
+def main() -> int:
+    problems: list[str] = []
+    scanned = 0
+
+    for rel in tracked_text_files():
+        try:
+            with open(rel, "rb") as handle:
+                raw = handle.read()
+        except FileNotFoundError:
+            continue  # tracked but deleted in the working tree
+        scanned += 1
+
+        try:
+            text = raw.decode("utf-8")
+        except UnicodeDecodeError as exc:
+            problems.append(f"{rel}: not valid UTF-8 ({exc})")
+            continue
+
+        if raw.startswith(BOM):
+            problems.append(f"{rel}: starts with a UTF-8 BOM")
+
+        for marker in MOJIBAKE:
+            if marker in text:
+                problems.append(
+                    f"{rel}:{first_line_containing(text, marker)}: "
+                    f"mojibake {describe(marker)} "
+                    f"(UTF-8 read as cp1252, then re-encoded)"
+                )
+                break
+
+        if REPLACEMENT in text:
+            problems.append(
+                f"{rel}:{first_line_containing(text, REPLACEMENT)}: "
+                f"contains U+FFFD, so characters were lost"
+            )
+
+    print(f"checked {scanned} tracked text files")
+
+    if problems:
+        print("\nEncoding problems found:\n")
+        for problem in problems:
+            print(f"  {problem}")
+        print(
+            "\nRe-read the affected file as UTF-8 and rewrite it as UTF-8. On "
+            "Windows PowerShell 5.1 that means passing -Encoding utf8 to "
+            "Get-Content as well as Set-Content -- the default for reading is "
+            "the ANSI codepage, which is what causes this."
+        )
+        return 1
+
+    print("no encoding problems")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())