From d527d153b45a2460748cd65af17e5048e5bfd542 Mon Sep 17 00:00:00 2001 From: StuBehan Date: Wed, 8 Jul 2026 00:29:37 +0100 Subject: [PATCH 1/2] feat: add speech normalization for md files --- docs/speech-normalization.md | 120 ++++++++++++++++ stackvox/__init__.py | 3 +- stackvox/text.py | 268 +++++++++++++++++++++++++++++++++++ tests/test_text.py | 155 ++++++++++++++++++++ 4 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 docs/speech-normalization.md create mode 100644 stackvox/text.py create mode 100644 tests/test_text.py diff --git a/docs/speech-normalization.md b/docs/speech-normalization.md new file mode 100644 index 0000000..5cf9a6b --- /dev/null +++ b/docs/speech-normalization.md @@ -0,0 +1,120 @@ +# Proposal: speech-text normalization in StackVox + +**Status:** draft for review · **Author:** Stu Behan (with Claude) · target StackVox `0.6.0` + +## 1. Why + +StackVox synthesizes whatever string it's handed. Two consumers now need the +same *pre*-step — turning real-world text into something that *sounds* right — +and they've already grown separate, diverging copies of it: + +| Consumer | File | Input | Has | Lacks | +|---|---|---|---|---| +| behan.codes read-aloud | `blog/tools/read-aloud.py` | Markdown post files | pronunciation dict, unit/number expansion, decimals→"point", pause shaping, `say:` directive, title split | emoji stripping, tables-as-CSV | +| stackone-speaklast | `.../stackone-speaklast/scripts/extract-response.py` (`clean_markdown`) | Claude Code responses | emoji stripping, tables→comma, reference links, snake_case-safe emphasis | pronunciation dict, units/numbers, decimals, pauses | + +Net effect: speaklast currently says "ree-dees", "stack-own", and "one, one +hundred ninety-eight point nine" — every bug the blog already fixed. Copying the +dict and rules into speaklast is the trap this proposal avoids. + +**Key insight:** the part that keeps changing is the *pronunciation entries* +(`agy→antigravity`, `Behan→Bayan`) — and those are per-consumer config that was +never going to live in StackVox anyway. The *mechanism* (apply a dict, expand +units/numbers, markdown→prose, shape pauses) has been stable. So we can extract +the mechanism now and let each caller keep evolving its own dictionary. + +## 2. Goals / non-goals + +**Goals** +- One normalization mechanism in StackVox, usable as a library call and from the CLI. +- Configurable by flags + an injected pronunciation dict. +- Backward compatible — existing `speak`/`synthesize` behaviour unchanged unless opted in. + +**Non-goals** +- Baking domain pronunciations (agy, StackOne, Redis…) into StackVox. +- Owning the blog's `say:` authoring directive (a Markdown-authoring convention; Claude responses never contain it). +- Changing default synthesis behaviour or the daemon protocol. + +## 3. Proposed API + +New module `stackvox/text.py`, one primary entry point: + +```python +def normalize_for_speech( + text: str, + *, + markdown: bool = True, # strip Markdown structure to prose + pronunciations: dict[str, str] | None = None, # {written: spoken}, whole-word, case-insensitive + expand_units: bool = True, # £/p/kWh/MPG/kg/km, ÷ × = + expand_numbers: bool = True, # thousands commas removed; decimals → "X point d d" + pauses: bool = True, # dash → beat (…); comma before "(" + tables: str = "drop", # "drop" | "csv" (how Markdown tables are voiced) + strip_emoji: bool = False, + terminal_stops: bool = True, # ensure each line ends in . / ? / : so a pause lands +) -> str: + ... +``` + +Re-exported from `stackvox/__init__.py` (`from stackvox.text import normalize_for_speech`). +Individual stages also exposed for composability (e.g. `expand_numbers(text)`, +`apply_pronunciations(text, mapping)`), but `normalize_for_speech` is the one most callers use. + +### Pipeline order (order matters — encodes real bugs we hit) + +1. **Markdown** (if `markdown`): fenced/inline code, images, links→text, reference links, headings, blockquotes, horizontal rules, list markers, emphasis (leave lone `_` for snake_case), tables per `tables`. +2. **Emoji** (if `strip_emoji`). +3. **Numbers** (if `expand_numbers`): strip thousands commas (`1,198.9`→`1198.9`) *then* decimals→words (`1198.9`→`1198 point 9`). +4. **Units/symbols** (if `expand_units`): currency **before** decimals is impossible if decimals ran first, so units run here and the decimal pass must see `£1.63`→`1.63 pounds` first → **currency/units run before the decimal split**. (This is why `read-aloud.py` orders units → decimals.) +5. **Pauses** (if `pauses`): ` - `/`—`→` … `; word`(`→`word, (`. +6. **Pronunciations**: whole-word, case-insensitive, from `pronunciations`. +7. **Whitespace collapse**; **terminal stops** (if `terminal_stops`). + +> Ordering note to preserve: **currency/unit expansion must precede the +> decimal-point split**, or `£1.63` becomes `£1 point 63`. See `read-aloud.py`. + +## 4. CLI surface + +Backward-compatible additions to the existing `speak`/`say`: + +``` +stackvox speak --normalize [--markdown] --file post.txt # normalize, then synth +stackvox normalize --file resp.md # print normalized text only (pipe/debug) +``` + +`--normalize` off by default, so notification phrases synth exactly as today. + +## 5. What moves vs. what stays + +| | Moves into StackVox | Stays in the caller | +|---|---|---| +| **Core** | markdown→prose, numbers, units, pauses, pronunciation *mechanism*, emoji, terminal stops | — | +| **Blog** | — | reads `.md` + front matter; the `say:` directive; frontmatter-title split + **silence splice** in `generate-audio.sh`; its dict; `tables="drop"` | +| **speaklast** | — | transcript discovery + last-response extraction; its **Claude-tuned** dict; `tables="csv"`, `strip_emoji=True`; drops its own `clean_markdown()` | + +### Illustrative dictionaries (stay per-consumer) + +- **Blog:** `agy, 1M, 175K, xhigh, SessionStart, PermissionRequest, StackOne, OAuth, Behan, Redis` +- **speaklast (to tune by listening):** `StackOne, Redis, OAuth, CLI, npm, async, repo, MCP, …` — Claude leans on tool/lib/identifier names, so this dict will grow differently. + +## 6. Backward compatibility & rollout + +- Additive + opt-in → no behaviour change for current users. Bump **minor → 0.6.0** (conventional commits), add `tests/test_text.py` to satisfy the coverage gate. +- **Distribution wrinkle:** both consumers currently run the StackVox **bundled in the Stack Nudge app** (`~/.stack-nudge/venv`, 0.5.0), *not* this clone. Until 0.6.0 ships in the bundle: + - dev/local: `pip install -e ~/stackone/stackvox` and point the callers at that venv (blog's `generate-audio.sh` already resolves a `stackvox` binary; add a clone/venv candidate); + - each caller keeps a thin local fallback (its current cleaner) if the installed StackVox predates `normalize_for_speech`, so nothing breaks on an old bundle. + +## 7. Migration steps + +1. **StackVox:** add `stackvox/text.py` + tests + `--normalize`/`normalize` CLI; export; release `0.6.0`. +2. **Blog:** replace the transform functions in `read-aloud.py` with a `normalize_for_speech(md, pronunciations=BLOG_DICT, tables="drop")` call; keep front-matter/`say:`/title-split; keep `generate-audio.sh`'s silence splice. +3. **speaklast:** replace `clean_markdown()` with `normalize_for_speech(resp, pronunciations=CLAUDE_DICT, tables="csv", strip_emoji=True)`. +4. Verify each by listening (blog posts + a few Claude responses). + +## 8. Decisions (resolved 2026-07-07) + +- **`terminal_stops`** — lives in the lib, **default on**. ✔ +- **Default `tables`** — **`drop`** (blog default); speaklast passes `tables="csv"`. ✔ +- **Locale** — add a **`locale` param, default `"en-GB"`**; currency/unit rules are keyed by locale (en-GB shipped first, others added later). ✔ +- **Emoji** — **`strip_emoji=False`** by default; speaklast sets `True`. ✔ +- **Distribution** — ship the updated StackVox **through the Stack Nudge app bundle** (both consumers run the bundled venv), so no editable-install fallback is needed in the callers once 0.6.0 is bundled. ✔ +- **Frontmatter-title split** — stays caller-side (blog), alongside the silence splice. diff --git a/stackvox/__init__.py b/stackvox/__init__.py index d178624..af683bb 100644 --- a/stackvox/__init__.py +++ b/stackvox/__init__.py @@ -5,6 +5,7 @@ from stackvox import daemon from stackvox.engine import Stackvox, speak, synthesize +from stackvox.text import normalize_for_speech try: __version__ = _pkg_version("stackvox") @@ -13,4 +14,4 @@ # bootstrap before `pip install -e .` has completed). __version__ = "0.0.0+unknown" -__all__ = ["Stackvox", "daemon", "speak", "synthesize"] +__all__ = ["Stackvox", "daemon", "normalize_for_speech", "speak", "synthesize"] diff --git a/stackvox/text.py b/stackvox/text.py new file mode 100644 index 0000000..a13819c --- /dev/null +++ b/stackvox/text.py @@ -0,0 +1,268 @@ +"""Turn real-world text — Markdown, Claude responses, prose — into something +that *sounds* right when synthesized. + +StackVox speaks whatever string it's given; this module is the pre-step that +makes that string speakable: strip Markdown structure, expand units and +numbers, shape pauses, and apply a caller-supplied pronunciation dictionary. + +The primary entry point is :func:`normalize_for_speech`. Each stage is also +exposed for composition and testing. See ``docs/speech-normalization.md``. +""" + +from __future__ import annotations + +import re + +__all__ = [ + "normalize_for_speech", + "markdown_to_paragraphs", + "strip_emoji", + "strip_thousands_separators", + "decimals_to_words", + "expand_units", + "apply_pronunciations", + "shape_pauses", + "ensure_terminal_stop", +] + +# --------------------------------------------------------------------------- # +# Emoji # +# --------------------------------------------------------------------------- # + +_EMOJI = re.compile( + "[" + "\U0001f000-\U0001faff" # symbols, emoticons, pictographs, supplemental + "\U00002600-\U000027bf" # misc symbols + dingbats + "\U00002190-\U000021ff" # arrows + "\U00002b00-\U00002bff" # misc symbols and arrows + "\U0000fe00-\U0000fe0f" # variation selectors + "]+", + flags=re.UNICODE, +) + + +def strip_emoji(text: str) -> str: + return _EMOJI.sub("", text) + + +# --------------------------------------------------------------------------- # +# Units & symbols (locale-keyed) # +# --------------------------------------------------------------------------- # +# (pattern, replacement). Digit-glued units capture the leading digit and +# re-emit it, so "65kg" -> "65 kilograms". IMPORTANT: currency/unit expansion +# must run BEFORE the decimal-point split, or "£1.63" becomes "£1 point 63". + +_UNIT_RULES: dict[str, list[tuple[str, str]]] = { + "en-GB": [ + (r"£\s?(\d[\d,]*(?:\.\d+)?)", r"\1 pounds"), # £1.63 -> 1.63 pounds + (r"\bkWh\b", "kilowatt hours"), + (r"\bMPG\b", "miles per gallon"), + (r"(\d)\s?kg\b", r"\1 kilograms"), + (r"(\d)\s?km\b", r"\1 kilometres"), + (r"(\d)p\b", r"\1 pence"), # 25p, 167.14p + (r"\s*÷\s*", " divided by "), + (r"\s*×\s*", " times "), + (r"\s*=\s*", " equals "), + ], +} +DEFAULT_LOCALE = "en-GB" + + +def expand_units(text: str, locale: str = DEFAULT_LOCALE) -> str: + for pattern, repl in _UNIT_RULES.get(locale, _UNIT_RULES[DEFAULT_LOCALE]): + text = re.sub(pattern, repl, text) + return text + + +# --------------------------------------------------------------------------- # +# Numbers # +# --------------------------------------------------------------------------- # + + +def strip_thousands_separators(text: str) -> str: + """1,198.9 -> 1198.9 (so the whole number is read as one, not split).""" + return re.sub(r"(?<=\d),(?=\d)", "", text) + + +def decimals_to_words(text: str) -> str: + """1198.9 -> "1198 point 9"; 770.72 -> "770 point 7 2". Removes the bare + "." between digits, which TTS can otherwise read as a full stop.""" + return re.sub( + r"(\d+)\.(\d+)", + lambda m: m.group(1) + " point " + " ".join(m.group(2)), + text, + ) + + +# --------------------------------------------------------------------------- # +# Pronunciations # +# --------------------------------------------------------------------------- # + + +def apply_pronunciations(text: str, mapping: dict[str, str] | None) -> str: + """Whole-word, case-insensitive spoken-form substitutions.""" + for written, spoken in (mapping or {}).items(): + text = re.sub(rf"\b{re.escape(written)}\b", spoken, text, flags=re.IGNORECASE) + return text + + +# --------------------------------------------------------------------------- # +# Pauses # +# --------------------------------------------------------------------------- # + + +def shape_pauses(text: str) -> str: + """Give punctuation the beats it deserves: a dash used as punctuation reads + as a rushed nothing, and a "(" runs onto the previous word.""" + text = text.replace("→", " to ") + text = re.sub(r"\s*[—–]\s*", " ... ", text) # em / en dash + text = re.sub(r"\s+--?\s+", " ... ", text) # spaced ASCII hyphen(s) + text = re.sub(r"(\w)\s*\(", r"\1, (", text) # comma before "(" + return text + + +def ensure_terminal_stop(text: str) -> str: + """Guarantee terminal punctuation so a pause lands before the next line — + StackVox pauses on punctuation, not on line breaks.""" + text = text.rstrip() + return text if text.endswith((".", "!", "?", ":", "…")) else text + "." + + +# --------------------------------------------------------------------------- # +# Markdown -> prose # +# --------------------------------------------------------------------------- # + + +def _strip_md_inline(line: str) -> str: + line = re.sub(r"!\[[^\]]*\]\([^)]*\)", "", line) # images + line = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", line) # inline links -> text + line = re.sub(r"\[([^\]]+)\]\[[^\]]*\]", r"\1", line) # reference links -> text + line = re.sub(r"`([^`]+)`", r"\1", line) # inline code -> text + line = re.sub(r"(\*\*|__|~~|\*)", "", line) # emphasis (leave lone _ for snake_case) + line = re.sub(r"", "", line) # HTML comments + line = re.sub(r"<[^>]+>", "", line) # stray HTML tags + return line + + +def markdown_to_paragraphs(text: str, *, tables: str = "drop", strip_emoji_flag: bool = False) -> list[str]: + """Reduce Markdown to a list of speakable paragraphs. Headings and list + items become their own paragraphs (so each gets its own pause). Tables are + dropped or rendered comma-separated per ``tables``.""" + text = re.sub(r"(?ms)^[ \t]*(```|~~~).*?^[ \t]*\1[ \t]*$", "\n", text) # fenced code + text = re.sub(r"```+|~~~+", " ", text) # stray fences + if strip_emoji_flag: + text = strip_emoji(text) + + paragraphs: list[str] = [] + current: list[str] = [] + + def flush() -> None: + if current: + paragraphs.append(" ".join(current)) + current.clear() + + for raw in text.splitlines(): + row = raw.strip() + + # table separator row ( |---|---| ) — always dropped + if re.fullmatch(r"\|?[\s:|-]*-[\s:|-]*\|?", row): + flush() + continue + # full table row + if len(row) >= 2 and row.startswith("|") and row.endswith("|"): + flush() + if tables == "csv": + cells = [c.strip() for c in _strip_md_inline(row.strip("|")).split("|")] + joined = ", ".join(c for c in cells if c) + if joined: + paragraphs.append(joined) + continue + # horizontal rule + if re.fullmatch(r"([-*_])(?:\s*\1){2,}", row): + flush() + continue + + is_heading = re.match(r"^\s{0,3}#{1,6}\s+", raw) + is_item = re.match(r"^\s*([-*+]|\d+[.)])\s+", raw) + cleaned = _strip_md_inline(re.sub(r"^\s{0,3}(#{1,6}\s*|>\s?)", "", raw)) + cleaned = re.sub(r"^\s*([-*+]|\d+[.)])\s+", "", cleaned).strip() + + if is_heading or is_item: # each stands alone -> its own pause + flush() + if cleaned: + paragraphs.append(cleaned) + elif cleaned: + current.append(cleaned) + else: + flush() + + flush() + return paragraphs + + +# --------------------------------------------------------------------------- # +# Orchestrator # +# --------------------------------------------------------------------------- # + + +def _shape_paragraph( + text: str, + *, + pronunciations: dict[str, str] | None, + expand_units_flag: bool, + expand_numbers_flag: bool, + pauses_flag: bool, + locale: str, +) -> str: + if expand_numbers_flag: + text = strip_thousands_separators(text) + if pauses_flag: + text = shape_pauses(text) + if pronunciations: + text = apply_pronunciations(text, pronunciations) + if expand_units_flag: # units BEFORE decimals (see note above) + text = expand_units(text, locale) + if expand_numbers_flag: + text = decimals_to_words(text) + return re.sub(r"[ \t]{2,}", " ", text).strip() + + +def normalize_for_speech( + text: str, + *, + markdown: bool = True, + pronunciations: dict[str, str] | None = None, + expand_units: bool = True, + expand_numbers: bool = True, + pauses: bool = True, + tables: str = "drop", + strip_emoji: bool = False, + terminal_stops: bool = True, + locale: str = DEFAULT_LOCALE, +) -> str: + """Normalize ``text`` into speakable prose. Returns paragraphs joined by + newlines. See ``docs/speech-normalization.md`` for the full contract.""" + expand_units_flag, expand_numbers_flag = expand_units, expand_numbers + + if markdown: + paragraphs = markdown_to_paragraphs(text, tables=tables, strip_emoji_flag=strip_emoji) + else: + # `strip_emoji` (the bool kwarg) shadows the module function here, so + # reach for the underlying pattern directly. + body = _EMOJI.sub("", text) if strip_emoji else text + paragraphs = [p.strip() for p in re.split(r"\n\s*\n", body) if p.strip()] + + out = [] + for para in paragraphs: + shaped = _shape_paragraph( + para, + pronunciations=pronunciations, + expand_units_flag=expand_units_flag, + expand_numbers_flag=expand_numbers_flag, + pauses_flag=pauses, + locale=locale, + ) + if not shaped: + continue + out.append(ensure_terminal_stop(shaped) if terminal_stops else shaped) + return "\n".join(out) diff --git a/tests/test_text.py b/tests/test_text.py new file mode 100644 index 0000000..47077b0 --- /dev/null +++ b/tests/test_text.py @@ -0,0 +1,155 @@ +"""Tests for stackvox.text — speech-text normalization.""" + +from stackvox.text import ( + apply_pronunciations, + decimals_to_words, + ensure_terminal_stop, + expand_units, + markdown_to_paragraphs, + normalize_for_speech, + shape_pauses, + strip_emoji, + strip_thousands_separators, +) + +# --- numbers --------------------------------------------------------------- + + +def test_thousands_separators_removed(): + assert strip_thousands_separators("1,198.9 and 2,294.7") == "1198.9 and 2294.7" + + +def test_decimals_spoken_digit_by_digit(): + assert decimals_to_words("1198.9") == "1198 point 9" + assert decimals_to_words("770.72") == "770 point 7 2" + + +def test_year_is_not_a_decimal(): + assert decimals_to_words("built in 2023") == "built in 2023" + + +# --- units ----------------------------------------------------------------- + + +def test_currency_and_pence(): + assert expand_units("£1.63") == "1.63 pounds" + assert expand_units("25p per litre") == "25 pence per litre" + assert expand_units("167.14p") == "167.14 pence" + + +def test_kwh_spaced_and_glued_units(): + assert expand_units("13.5 kWh") == "13.5 kilowatt hours" + assert expand_units("per kWh") == "per kilowatt hours" + assert expand_units("65kg") == "65 kilograms" + assert expand_units("a 4km trip") == "a 4 kilometres trip" + + +def test_math_symbols(): + assert expand_units("770 ÷ 2 × 3 = 5").strip() == "770 divided by 2 times 3 equals 5" + + +def test_unknown_locale_falls_back_to_en_gb(): + assert expand_units("£5", locale="xx-YY") == "5 pounds" + + +# --- pronunciations -------------------------------------------------------- + + +def test_pronunciations_whole_word_case_insensitive(): + m = {"Redis": "Reddis", "agy": "antigravity"} + assert apply_pronunciations("Redis and AGY", m) == "Reddis and antigravity" + + +def test_pronunciations_do_not_match_substrings(): + # "StackOne" must not partially rewrite "StackOneHQ" + assert apply_pronunciations("StackOneHQ", {"StackOne": "stack one"}) == "StackOneHQ" + + +# --- pauses ---------------------------------------------------------------- + + +def test_dash_becomes_ellipsis_pause(): + assert shape_pauses("in control - once I started") == "in control ... once I started" + + +def test_comma_inserted_before_paren(): + assert shape_pauses("Claude Code (CC)") == "Claude Code, (CC)" + + +def test_ensure_terminal_stop(): + assert ensure_terminal_stop("About") == "About." + assert ensure_terminal_stop("Ready?") == "Ready?" + assert ensure_terminal_stop("A list:") == "A list:" + + +# --- emoji ----------------------------------------------------------------- + + +def test_strip_emoji(): + assert ( + strip_emoji("done ✅ and 🚀 go").replace(" ", " ").strip() + == "done and go".replace(" ", " ").strip() + ) + + +# --- markdown -------------------------------------------------------------- + + +def test_markdown_headings_and_lists_are_own_paragraphs(): + md = "# Title\n\nBody line.\n\n- one\n- two" + assert markdown_to_paragraphs(md) == ["Title", "Body line.", "one", "two"] + + +def test_markdown_links_and_code_reduced_to_text(): + md = "See [the docs](https://x.example) and run `make build`." + assert markdown_to_paragraphs(md) == ["See the docs and run make build."] + + +def test_fenced_code_dropped(): + md = "Before.\n\n```py\nprint('hi')\n```\n\nAfter." + assert markdown_to_paragraphs(md) == ["Before.", "After."] + + +def test_tables_dropped_by_default(): + md = "| A | B |\n| --- | --- |\n| 1 | 2 |" + assert markdown_to_paragraphs(md) == [] + + +def test_tables_as_csv(): + md = "| A | B |\n| --- | --- |\n| 1 | 2 |" + assert markdown_to_paragraphs(md, tables="csv") == ["A, B", "1, 2"] + + +# --- end to end ------------------------------------------------------------ + + +def test_currency_runs_before_decimal_split(): + # The ordering bug guard: £1.63 -> "1.63 pounds" -> "1 point 6 3 pounds", + # never "£1 point 63". + out = normalize_for_speech("It cost £1.63 today.", markdown=False) + assert "1 point 6 3 pounds" in out + assert "£" not in out + + +def test_blog_style_normalization(): + md = "## The Verdict\n\nIt drew 770.72 kWh for £192.68 - a loss." + out = normalize_for_speech(md, pronunciations={}, tables="drop") + assert out == ("The Verdict.\nIt drew 770 point 7 2 kilowatt hours for 192 point 6 8 pounds ... a loss.") + + +def test_speaklast_style_normalization(): + md = "I updated `Redis` ✅ and the CLI (v2)." + out = normalize_for_speech( + md, + pronunciations={"Redis": "Reddis", "CLI": "C L I"}, + tables="csv", + strip_emoji=True, + ) + assert "Reddis" in out + assert "C L I, (v2)" in out + assert "✅" not in out + + +def test_plain_text_mode_splits_paragraphs(): + out = normalize_for_speech("First para.\n\nSecond para.", markdown=False) + assert out == "First para.\nSecond para." From 265bc563d357d89b0d78141aa2c4afbb957b9d60 Mon Sep 17 00:00:00 2001 From: StuBehan Date: Wed, 8 Jul 2026 15:47:46 +0100 Subject: [PATCH 2/2] fix(text): harden markdown parsing for speech --- stackvox/text.py | 64 ++++++++++++++++++++++++++++++++++++++-------- tests/test_text.py | 42 ++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 11 deletions(-) diff --git a/stackvox/text.py b/stackvox/text.py index a13819c..c0797c3 100644 --- a/stackvox/text.py +++ b/stackvox/text.py @@ -138,9 +138,15 @@ def _strip_md_inline(line: str) -> str: line = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", line) # inline links -> text line = re.sub(r"\[([^\]]+)\]\[[^\]]*\]", r"\1", line) # reference links -> text line = re.sub(r"`([^`]+)`", r"\1", line) # inline code -> text - line = re.sub(r"(\*\*|__|~~|\*)", "", line) # emphasis (leave lone _ for snake_case) line = re.sub(r"", "", line) # HTML comments - line = re.sub(r"<[^>]+>", "", line) # stray HTML tags + line = re.sub(r"]*>", "", line) # real tags + autolinks; leaves "a < b" + line = re.sub(r"https?://\S+", "", line) # bare URLs: strip rather than read them aloud + # Emphasis: only strip *paired, boundary-flanked* markers, so code-ish tokens + # that aren't emphasis in CommonMark — *args, **kwargs, __init__, snake_case — + # survive intact. Underscores are left alone entirely (TTS doesn't voice them). + line = re.sub(r"(? None: if current: paragraphs.append(" ".join(current)) current.clear() + def emit_csv_row(cells_source: str) -> None: + cells = [c.strip() for c in _strip_md_inline(cells_source.strip("|")).split("|")] + joined = ", ".join(c for c in cells if c) + if joined: + paragraphs.append(joined) + for raw in text.splitlines(): row = raw.strip() - # table separator row ( |---|---| ) — always dropped + if not row: # blank line ends the current paragraph and any table block + flush() + in_table = False + continue + + # link reference definition ( [id]: https://… ) — invisible when rendered + if re.match(r"^\[[^\]]+\]:\s*\S", row): + flush() + continue + + # table separator row ( |---|---| or --- | --- ) if re.fullmatch(r"\|?[\s:|-]*-[\s:|-]*\|?", row): + # Only a *table* separator if the line above held cells; otherwise it's + # a `---` horizontal rule / setext underline (no table context). + if current and "|" in current[-1]: + header = current.pop() + flush() + if tables == "csv": + emit_csv_row(header) + in_table = True + else: + flush() + in_table = False + continue + + # setext underline ( === ) — drop so it isn't voiced as "equals equals…" + if re.fullmatch(r"=+", row): flush() continue - # full table row - if len(row) >= 2 and row.startswith("|") and row.endswith("|"): + + # table row: outer-pipe form, or a bare-pipe row within a table block + outer_pipe = len(row) >= 2 and row.startswith("|") and row.endswith("|") + if outer_pipe or (in_table and "|" in row): flush() if tables == "csv": - cells = [c.strip() for c in _strip_md_inline(row.strip("|")).split("|")] - joined = ", ".join(c for c in cells if c) - if joined: - paragraphs.append(joined) + emit_csv_row(row) continue - # horizontal rule + + # horizontal rule ( *** / ___ ; --- is handled by the separator branch ) if re.fullmatch(r"([-*_])(?:\s*\1){2,}", row): flush() + in_table = False continue + in_table = False is_heading = re.match(r"^\s{0,3}#{1,6}\s+", raw) is_item = re.match(r"^\s*([-*+]|\d+[.)])\s+", raw) - cleaned = _strip_md_inline(re.sub(r"^\s{0,3}(#{1,6}\s*|>\s?)", "", raw)) + cleaned = _strip_md_inline(re.sub(r"^\s{0,3}(#{1,6}\s*|(?:>\s?)+)", "", raw)) cleaned = re.sub(r"^\s*([-*+]|\d+[.)])\s+", "", cleaned).strip() + if is_item: # drop task-list checkboxes ( - [ ] / - [x] ) + cleaned = re.sub(r"^\[[ xX]\]\s+", "", cleaned).strip() if is_heading or is_item: # each stands alone -> its own pause flush() diff --git a/tests/test_text.py b/tests/test_text.py index 47077b0..e7bf735 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -120,6 +120,48 @@ def test_tables_as_csv(): assert markdown_to_paragraphs(md, tables="csv") == ["A, B", "1, 2"] +def test_tables_without_outer_pipes_are_recognized(): + md = "A | B\n--- | ---\n1 | 2" + assert markdown_to_paragraphs(md) == [] + assert markdown_to_paragraphs(md, tables="csv") == ["A, B", "1, 2"] + + +def test_code_identifiers_are_not_treated_as_emphasis(): + # __init__, **kwargs and *args are not emphasis in CommonMark — keep them. + md = "The `__init__` method takes **kwargs and *args." + assert markdown_to_paragraphs(md) == ["The __init__ method takes **kwargs and *args."] + + +def test_paired_emphasis_markers_are_stripped(): + md = "This is **bold** and *italic* text." + assert markdown_to_paragraphs(md) == ["This is bold and italic text."] + + +def test_angle_brackets_in_prose_are_kept(): + # Only real HTML tags are stripped, not comparisons. + assert markdown_to_paragraphs("Voltage: x < y and y > z.") == ["Voltage: x < y and y > z."] + assert markdown_to_paragraphs("Wrap x now.") == ["Wrap x now."] + + +def test_bare_urls_and_reference_definitions_are_dropped(): + md = "See [the docs][docs] here.\n\n[docs]: https://example.com/path" + assert markdown_to_paragraphs(md) == ["See the docs here."] + # whitespace left by the stripped URL is collapsed by the full pipeline + assert normalize_for_speech("Go to https://example.com now.") == "Go to now." + + +def test_setext_underline_is_dropped(): + assert markdown_to_paragraphs("The Verdict\n===========\n\nBody.") == ["The Verdict", "Body."] + + +def test_task_list_checkboxes_are_stripped(): + assert markdown_to_paragraphs("- [ ] todo\n- [x] done") == ["todo", "done"] + + +def test_nested_blockquote_markers_are_stripped(): + assert markdown_to_paragraphs("> > deeply quoted") == ["deeply quoted"] + + # --- end to end ------------------------------------------------------------