summarize.py

#!/usr/bin/env python3
"""Extractive transcript summarization with zero external dependencies.

This module turns long transcript text into skimmable notes by selecting the
most representative sentences verbatim from the input. It is an *extractive*
summarizer — it never generates new prose, never paraphrases, and never calls
an LLM or any network service. The algorithm is the classic word-frequency
approach:

1. Split the text into sentences with a regular expression.
2. Build a frequency table of lowercased whitespace tokens, ignoring a small
   English stopword list and bare punctuation.
3. Score each sentence by the average frequency of its content words.
4. Return the top-N sentences, re-ordered to match their original positions.

Limitations (by design, to stay stdlib-only):

* Tokenization is whitespace-based. Languages written without spaces between
  words (Chinese, Japanese) degrade to per-run tokens, so frequency scoring is
  coarse for them; Korean is space-separated and fares better, though particle
  suffixes are not stripped. CJK input is handled without crashing and CJK
  sentence terminators (。！？) are recognized.
* Sentence splitting is heuristic; unusual abbreviations may split early.
"""

import re
from dataclasses import dataclass, field

# Sentence terminators: ASCII . ! ? plus CJK ideographic full stop and
# full-width ! / ?. A sentence ends at one or more terminators followed by
# closing quotes/brackets, then whitespace or end-of-text.
_SENTENCE_END_RE = re.compile(
    r"(?<=[.!?。！？])[\"'”’)\]]*\s+"
)

# Strip leading/trailing punctuation from tokens before frequency counting.
_TOKEN_STRIP_RE = re.compile(r"^\W+|\W+$", re.UNICODE)

# Small English stopword list: enough to keep glue words from dominating the
# frequency table without pulling in any external corpus.
_STOPWORDS = frozenset(
    """
    a an and are as at be but by for from had has have he her his i if in is
    it its me my nor not of on or our she so that the their them then there
    they this to us was we were what when which who will with you your
    """.split()
)


@dataclass
class Summary:
    """Result of an extractive summarization run.

    Attributes:
        summary_text: The selected sentences joined with single spaces, in
            their original document order.
        key_sentences: The selected sentences as a list, in original order.
        word_count: Number of whitespace-separated tokens in the *input*
            text (not the summary).
    """

    summary_text: str
    key_sentences: list = field(default_factory=list)
    word_count: int = 0


def _split_sentences(text):
    """Split ``text`` into sentences using punctuation heuristics.

    Args:
        text: Raw input text.

    Returns:
        A list of non-empty sentence strings with surrounding whitespace
        stripped. Newlines without terminal punctuation do not split
        sentences, so transcripts with hard-wrapped lines stay intact.
    """
    normalized = re.sub(r"\s+", " ", text).strip()
    if not normalized:
        return []
    parts = _SENTENCE_END_RE.split(normalized)
    return [part.strip() for part in parts if part.strip()]


def _content_words(sentence):
    """Extract lowercased content words from a sentence.

    Args:
        sentence: A single sentence string.

    Returns:
        A list of lowercased whitespace tokens with surrounding punctuation
        stripped, excluding stopwords and empty tokens.
    """
    words = []
    for raw in sentence.split():
        token = _TOKEN_STRIP_RE.sub("", raw).lower()
        if token and token not in _STOPWORDS:
            words.append(token)
    return words


def _score_sentences(sentences):
    """Score sentences by average content-word frequency.

    Args:
        sentences: List of sentence strings.

    Returns:
        A list of float scores parallel to ``sentences``. Sentences with no
        content words score 0.0. Using the *average* (rather than the sum)
        keeps long sentences from winning on length alone.
    """
    frequencies = {}
    tokenized = []
    for sentence in sentences:
        words = _content_words(sentence)
        tokenized.append(words)
        for word in words:
            frequencies[word] = frequencies.get(word, 0) + 1

    scores = []
    for words in tokenized:
        if words:
            scores.append(sum(frequencies[w] for w in words) / len(words))
        else:
            scores.append(0.0)
    return scores


def summarize_text(text, max_sentences=5):
    """Produce an extractive summary of ``text``.

    Selects the ``max_sentences`` highest-scoring sentences (ties broken by
    earlier position) and returns them in their original order, so the
    summary reads chronologically — important for transcripts.

    Args:
        text: The transcript or document text to summarize. May be empty.
        max_sentences: Maximum number of sentences to include in the
            summary. Must be at least 1.

    Returns:
        A :class:`Summary`. For empty or whitespace-only input the summary
        is empty with ``word_count`` 0. If the text has ``max_sentences`` or
        fewer sentences, all of them are returned unchanged (short input is
        effectively passed through).

    Raises:
        ValueError: If ``max_sentences`` is less than 1.
    """
    if max_sentences < 1:
        raise ValueError("max_sentences must be at least 1")

    word_count = len(text.split())
    sentences = _split_sentences(text)
    if not sentences:
        return Summary(summary_text="", key_sentences=[], word_count=0)

    if len(sentences) <= max_sentences:
        selected = list(sentences)
    else:
        scores = _score_sentences(sentences)
        ranked = sorted(
            range(len(sentences)), key=lambda i: (-scores[i], i)
        )
        chosen = sorted(ranked[:max_sentences])
        selected = [sentences[i] for i in chosen]

    return Summary(
        summary_text=" ".join(selected),
        key_sentences=selected,
        word_count=word_count,
    )


def summarize_segments(segments, max_sentences=5):
    """Summarize a sequence of transcript segments.

    Accepts any iterable of duck-typed segment objects exposing a ``.text``
    attribute (e.g. timestamped transcription segments). Segment texts are
    joined with single spaces before summarization, so sentences that span
    segment boundaries are reassembled.

    Args:
        segments: Iterable of objects with a ``.text`` string attribute.
        max_sentences: Maximum number of sentences in the summary. Must be
            at least 1.

    Returns:
        A :class:`Summary` over the joined segment text.

    Raises:
        ValueError: If ``max_sentences`` is less than 1.
        AttributeError: If a segment lacks a ``.text`` attribute.
    """
    joined = " ".join(segment.text for segment in segments)
    return summarize_text(joined, max_sentences=max_sentences)