diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 452ff38..1008475 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,19 @@ concurrency: cancel-in-progress: true jobs: + encoding: + name: Text encoding + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + # Cheap, and catches a corruption that is invisible to the compiler and + # the test suite: double-encoded UTF-8 still parses fine and only shows + # up as garbage characters in the rendered UI. + - run: python scripts/check_encoding.py + backend: name: Backend (pytest) runs-on: ubuntu-latest diff --git a/backend/app/core/visualizer.py b/backend/app/core/visualizer.py index 7eb6179..4be0738 100644 --- a/backend/app/core/visualizer.py +++ b/backend/app/core/visualizer.py @@ -1,16 +1,16 @@ -"""Embedding analysis: the :class:`LoadedModel` value object. +"""Embedding analysis: the :class:`LoadedModel` value object. A :class:`LoadedModel` is an immutable-after-construction snapshot of one transformer's token embedding space, together with derived metadata and an LRU cache of UMAP projections. All of the geometric queries the API exposes (neighbors, comparisons, search, statistics) are methods here, operating purely -on NumPy arrays — there is no global state and no HTTP awareness, which makes +on NumPy arrays — there is no global state and no HTTP awareness, which makes the maths trivial to unit-test and safe to share across concurrent requests. The split of responsibilities is deliberate: -* :class:`LoadedModel` — *what* we know about an already-loaded model. -* :class:`~app.core.model_manager.ModelManager` — *when* models are loaded, +* :class:`LoadedModel` — *what* we know about an already-loaded model. +* :class:`~app.core.model_manager.ModelManager` — *when* models are loaded, cached, and evicted. """ @@ -68,7 +68,7 @@ def __init__( ) -> None: self.name = name # Record the full-vocabulary dimensions for reporting, but only retain - # the analyzed subset below — holding the entire embedding matrix as + # the analyzed subset below — holding the entire embedding matrix as # well would roughly double memory for large models. self.vocabulary_size = int(embeddings.shape[0]) self.embedding_dimension = int(embeddings.shape[1]) diff --git a/frontend/src/components/Legend.tsx b/frontend/src/components/Legend.tsx index ca673a7..c8b14d6 100644 --- a/frontend/src/components/Legend.tsx +++ b/frontend/src/components/Legend.tsx @@ -1,4 +1,4 @@ -/** Projection legend and client-side dataset export. */ +/** Projection legend and client-side dataset export. */ import { useStore } from "../store/useStore"; import { TOKEN_COLORS, TOKEN_TYPE_LABELS } from "../lib/tokenColors"; @@ -75,7 +75,7 @@ export function Legend() {
{vocab ? `${vocab.toLocaleString()} vocabulary / ` : ""} - {vizData.statistics.original_dimension}D → {vizData.statistics.reduced_dimension}D / UMAP {vizData.config.metric} + {vizData.statistics.original_dimension}D → {vizData.statistics.reduced_dimension}D / UMAP {vizData.config.metric}
); diff --git a/frontend/src/components/SearchPanel.tsx b/frontend/src/components/SearchPanel.tsx index 29445dd..12a9c4f 100644 --- a/frontend/src/components/SearchPanel.tsx +++ b/frontend/src/components/SearchPanel.tsx @@ -1,4 +1,4 @@ -/** Debounced token lookup across the complete analyzed vocabulary subset. */ +/** Debounced token lookup across the complete analyzed vocabulary subset. */ import { useEffect, useState } from "react"; import { formatToken } from "../lib/tokenFormat"; diff --git a/frontend/src/components/VizControls.tsx b/frontend/src/components/VizControls.tsx index cac8e98..b200ece 100644 --- a/frontend/src/components/VizControls.tsx +++ b/frontend/src/components/VizControls.tsx @@ -1,4 +1,4 @@ -/** UMAP projection controls; re-projects on demand once a model is loaded. */ +/** UMAP projection controls; re-projects on demand once a model is loaded. */ import { useStore } from "../store/useStore"; import { Field, SegToggle, Slider } from "./ui"; @@ -78,10 +78,10 @@ export function VizControls() { disabled={disabled} onClick={() => void regenerate().catch(() => undefined)} > - {loadState === "visualizing" ? "Computing…" : dirty ? "Apply projection" : "Recompute"} + {loadState === "visualizing" ? "Computing…" : dirty ? "Apply projection" : "Recompute"} - {/* Visible-points is a purely client-side declutter — no re-projection. + {/* Visible-points is a purely client-side declutter — no re-projection. The full projection is always retained so any token can still be searched, inspected, and compared. */} {totalPoints > 0 && ( diff --git a/frontend/src/components/scene/PointCloud.tsx b/frontend/src/components/scene/PointCloud.tsx index 0fc77e9..5de5ebd 100644 --- a/frontend/src/components/scene/PointCloud.tsx +++ b/frontend/src/components/scene/PointCloud.tsx @@ -1,11 +1,11 @@ -/** +/** * The interactive token point cloud. * * Renders the visible tokens as a single GPU-friendly THREE.Points object (one * draw call). Normalized coordinates are shared via the store so the camera rig * uses identical positions. Hover/click use r3f's points raycasting * (`event.index`). The current selection and its neighbors are emphasised with - * an overlay that reads from the *full* position buffer — so a token can be + * an overlay that reads from the *full* position buffer — so a token can be * highlighted even when it sits outside the visible-points cutoff. */ diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 35e98fb..4d4cc3d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,4 +1,4 @@ -/** +/** * Typed client for the Embeddings Visualizer backend. * * Every query endpoint is model-keyed (the backend has no notion of a single diff --git a/frontend/src/lib/layout.ts b/frontend/src/lib/layout.ts index b83f94e..ed86514 100644 --- a/frontend/src/lib/layout.ts +++ b/frontend/src/lib/layout.ts @@ -1,4 +1,4 @@ -/** +/** * Geometry helpers for laying out the projected token cloud. * * UMAP output lives in an arbitrary coordinate range, so we normalize it into a diff --git a/frontend/src/store/useStore.ts b/frontend/src/store/useStore.ts index 10d8d03..956cd0f 100644 --- a/frontend/src/store/useStore.ts +++ b/frontend/src/store/useStore.ts @@ -1,4 +1,4 @@ -/** +/** * Central application state (Zustand). * * Holds the model-loading lifecycle, the active visualization payload, the @@ -161,7 +161,7 @@ export const useStore = create((set, get) => ({ const model = get().selectedModelId.trim(); if (!model) return; - set({ loadState: "loading", loadProgress: "Requesting model…", comparison: null }); + set({ loadState: "loading", loadProgress: "Requesting model…", comparison: null }); // Poll the status endpoint for live progress while /load runs server-side. // The load happens in a worker thread, so the event loop keeps serving @@ -186,7 +186,7 @@ export const useStore = create((set, get) => ({ set({ loadedModel: model, loadState: "visualizing", - loadProgress: "Projecting embeddings… (first projection can take ~30s)", + loadProgress: "Projecting embeddings… (first projection can take ~30s)", accentGlowHex: theme.glowHex, themeLabel: theme.label, }); @@ -197,7 +197,7 @@ export const useStore = create((set, get) => ({ .then((statistics) => set({ statistics })) .catch(() => undefined); set({ loadState: "ready" }); - get().pushToast("success", `${model} loaded — ${get().vizData?.tokens.length ?? 0} tokens projected.`); + get().pushToast("success", `${model} loaded — ${get().vizData?.tokens.length ?? 0} tokens projected.`); } catch (e) { const msg = e instanceof ApiError ? e.message : "Failed to load model"; set({ loadState: "error", loadProgress: msg }); @@ -211,7 +211,7 @@ export const useStore = create((set, get) => ({ const model = get().loadedModel; if (!model) return; const wasReady = get().loadState === "ready"; - if (wasReady) set({ loadState: "visualizing", loadProgress: "Re-projecting…" }); + if (wasReady) set({ loadState: "visualizing", loadProgress: "Re-projecting…" }); try { const vizData = await api.createVisualization(model, get().config); set({ @@ -253,7 +253,7 @@ export const useStore = create((set, get) => ({ } }, - // Select a token AND fly the camera to it — used when inspecting a token that + // Select a token AND fly the camera to it — used when inspecting a token that // may be off-screen (e.g. chosen from search or comparison results). focusOn: async (index) => { set({ focus: { index, nonce: ++focusSeq } }); diff --git a/scripts/check_encoding.py b/scripts/check_encoding.py new file mode 100644 index 0000000..c5b38d6 --- /dev/null +++ b/scripts/check_encoding.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Fail if any tracked text file has broken text encoding. + +Guards against a specific, easy-to-miss corruption: reading a UTF-8 file with +a tool that assumes the legacy Windows ANSI codepage, then writing it back as +UTF-8. That double-encodes every non-ASCII character -- an ellipsis becomes a +three-character sequence starting with a-circumflex -- and because the result +is still *valid* UTF-8, nothing downstream complains. It compiles, it passes +tests, and it only shows up as garbage in the rendered UI. + +It also rejects UTF-8 BOMs, which some Windows editors and PowerShell's +Set-Content add silently. + +This file is deliberately pure ASCII: the characters it searches for are built +from numeric code points rather than typed out. Spelled literally they would +appear in this file's own source and the checker would flag itself -- which is +exactly what happened the first time it ran in CI. Staying ASCII means it can +scan itself like any other file instead of needing an exemption, so real +corruption here is still caught. Keep it that way. + +Run from the repository root: python scripts/check_encoding.py +""" + +from __future__ import annotations + +import subprocess +import sys + +TEXT_SUFFIXES = ( + ".ts", ".tsx", ".js", ".jsx", ".py", ".md", ".json", ".yml", ".yaml", + ".css", ".html", ".txt", ".example", ".toml", ".cfg", ".sh", +) + +# What common UTF-8 lead bytes look like once misread as cp1252. A UTF-8 +# sequence beginning C2/C3 (Latin-1 supplement) or E2 (punctuation, arrows) +# decodes under cp1252 to one of these pairs. None is plausible in +# hand-written source text. +_MOJIBAKE_CODEPOINTS = ( + (0x00E2, 0x20AC), # E2 80 xx: ellipsis, em dash, curly quotes + (0x00E2, 0x2020), # E2 86 xx: arrows + (0x00C3, 0x00A2), # C3 A2: a-circumflex + (0x00C3, 0x00A9), # C3 A9: e-acute + (0x00C3, 0x00A8), # C3 A8: e-grave + (0x00C3, 0x00AF), # C3 AF: i-diaeresis + (0x00C2, 0x00B7), # C2 B7: middle dot + (0x00C2, 0x00B6), # C2 B6: pilcrow + (0x00C2, 0x00B0), # C2 B0: degree sign + (0x00C2, 0x00A0), # C2 A0: non-breaking space +) + +MOJIBAKE = tuple("".join(chr(point) for point in pair) + for pair in _MOJIBAKE_CODEPOINTS) + +REPLACEMENT = chr(0xFFFD) # left behind when a decode gives up +BOM = b"\xef\xbb\xbf" + + +def tracked_text_files() -> list[str]: + out = subprocess.run( + ["git", "ls-files"], capture_output=True, text=True, check=True + ).stdout + return [line for line in out.splitlines() if line.endswith(TEXT_SUFFIXES)] + + +def first_line_containing(text: str, needle: str) -> int: + for number, line in enumerate(text.splitlines(), 1): + if needle in line: + return number + return 0 + + +def describe(text: str) -> str: + """Render a needle as escapes, so the report stays ASCII and unambiguous.""" + return "".join(f"\\u{ord(char):04x}" for char in text) + + +def main() -> int: + problems: list[str] = [] + scanned = 0 + + for rel in tracked_text_files(): + try: + with open(rel, "rb") as handle: + raw = handle.read() + except FileNotFoundError: + continue # tracked but deleted in the working tree + scanned += 1 + + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + problems.append(f"{rel}: not valid UTF-8 ({exc})") + continue + + if raw.startswith(BOM): + problems.append(f"{rel}: starts with a UTF-8 BOM") + + for marker in MOJIBAKE: + if marker in text: + problems.append( + f"{rel}:{first_line_containing(text, marker)}: " + f"mojibake {describe(marker)} " + f"(UTF-8 read as cp1252, then re-encoded)" + ) + break + + if REPLACEMENT in text: + problems.append( + f"{rel}:{first_line_containing(text, REPLACEMENT)}: " + f"contains U+FFFD, so characters were lost" + ) + + print(f"checked {scanned} tracked text files") + + if problems: + print("\nEncoding problems found:\n") + for problem in problems: + print(f" {problem}") + print( + "\nRe-read the affected file as UTF-8 and rewrite it as UTF-8. On " + "Windows PowerShell 5.1 that means passing -Encoding utf8 to " + "Get-Content as well as Set-Content -- the default for reading is " + "the ANSI codepage, which is what causes this." + ) + return 1 + + print("no encoding problems") + return 0 + + +if __name__ == "__main__": + sys.exit(main())