From 2da18ea81ccb6444cadc78476ce507fb536c38b8 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:12:33 +0000 Subject: [PATCH 01/11] refactor(parser): remove unused FOLLOW infrastructure and renumber phases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rule.follow, Rule.follow_built, Rule.calc_first, Rule.first, Rule.first_built, Rule.calc_follow, and compute_follow_sets are removed. Since T6 the LALR(1) lookahead table drives all reduce decisions; FOLLOW sets were no longer used. Phase numbering updated: old Phase 4→3, 5→4, 6→5. Co-Authored-By: Claude Sonnet 4.6 --- plare/parser.py | 121 +++---------------------------------- tests/test_first_follow.py | 27 +-------- 2 files changed, 11 insertions(+), 137 deletions(-) diff --git a/plare/parser.py b/plare/parser.py index c30bea5..6976467 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -423,7 +423,7 @@ def force_update(self, state: int, symbol: Symbol, action: Action[T]) -> None: class Rule[T]: - """All RHS alternatives for a single non-terminal, together with its FIRST/FOLLOW sets. + """All RHS alternatives for a single non-terminal. ``Rule`` is the unit of grammar specification. One ``Rule`` object aggregates every production ``A → rhs₁ | rhs₂ | …`` for a given @@ -431,16 +431,13 @@ class Rule[T]: Attributes: left: The non-terminal name (LHS). - rights: List of ``(rhs_symbols, maker)`` pairs, one per alternative. - first: FIRST(A) — populated by ``calc_first``. - follow: FOLLOW(A) — populated by ``calc_follow``. + rights: List of ``(rhs_symbols, maker, prec_override)`` triples. + definition_indices: Grammar-wide ordinal for each production alternative. """ left: str rights: list[tuple[list[Symbol], Maker[T], int | None]] definition_indices: list[int] - first: set[type[Token]] - follow: set[type[Token]] def __init__( self, @@ -458,41 +455,6 @@ def __init__( for right, action, args, prec_override in rights ] self.definition_indices = list(range(start_index, start_index + len(rights))) - self.first_built = False - self.follow_built = False - - def calc_first(self, rules: dict[str, Rule[T]]) -> set[type[Token]]: - """Return FIRST(A), computing it via ``compute_first_sets`` if needed. - - Args: - rules: Complete grammar mapping non-terminal name → ``Rule``. - - Returns: - The FIRST set for this non-terminal (also stored in ``self.first``). - """ - if not self.first_built: - fs = compute_first_sets(rules) - for n, r in rules.items(): - r.first = fs[n] - r.first_built = True - return self.first - - def calc_follow(self, rules: dict[str, Rule[T]]) -> set[type[Token]]: - """Return FOLLOW(A), computing it via ``compute_follow_sets`` if needed. - - Args: - rules: Complete grammar mapping non-terminal name → ``Rule``. - - Returns: - The FOLLOW set for this non-terminal (also stored in ``self.follow``). - """ - if not self.follow_built: - fs = {n: r.first for n, r in rules.items()} - fw = compute_follow_sets(rules, fs) - for n, r in rules.items(): - r.follow = fw[n] - r.follow_built = True - return self.follow def __hash__(self) -> int: return hash(self.left) @@ -560,56 +522,6 @@ def compute_first_sets[T](rules: dict[str, Rule[T]]) -> dict[str, set[type[Token return first -def compute_follow_sets[T]( - rules: dict[str, Rule[T]], - first_sets: dict[str, set[type[Token]]], -) -> dict[str, set[type[Token]]]: - """Compute FOLLOW sets for all non-terminals via worklist fixed-point iteration. - - Seeds EOS into every augmented start symbol, then propagates terminals - through productions until no FOLLOW set changes. Requires FIRST sets - to have been computed first. - - Args: - rules: Complete grammar mapping non-terminal name → ``Rule``. - first_sets: Precomputed FIRST sets (from ``compute_first_sets``). - - Returns: - Mapping from non-terminal name to its FOLLOW set. - """ - follow: dict[str, set[type[Token]]] = {name: set() for name in rules} - for name in rules: - if isinstance(name, StartVariable): - follow[name].add(EOS) - changed = True - while changed: - changed = False - for lhs, rule in rules.items(): - for right, _, _ in rule.rights: - for i, sym in enumerate(right): - if not isinstance(sym, str): - continue - trailer: set[type[Token]] = set() - all_nullable = True - for next_sym in right[i + 1 :]: - if isinstance(next_sym, type): - trailer.add(next_sym) - all_nullable = False - break - else: - next_first = first_sets[next_sym] - trailer.update(next_first - {EPSILON}) - if EPSILON not in next_first: - all_nullable = False - break - if all_nullable: - trailer.update(follow[lhs]) - added = trailer - follow[sym] - if added: - follow[sym].update(added) - changed = True - return follow - def symbol_sort_key(s: Symbol) -> tuple[int, str]: """Return a sort key that gives a stable total order over grammar symbols. @@ -971,21 +883,8 @@ def __init__( # ── Phase 2: Compute FIRST sets ────────────────────────────────────── # FIRST(A) is needed to propagate ε through nullable non-terminals - # when computing FOLLOW sets in Phase 3. + # during LALR(1) lookahead propagation in Phase 4. first_sets = compute_first_sets(rules) - for name, rule in rules.items(): - rule.first = first_sets[name] - rule.first_built = True - - # ── Phase 3: Compute FOLLOW sets ───────────────────────────────────── - # FOLLOW sets are stored on each Rule for the public API - # (Rule.follow, compute_follow_sets) but are no longer used by Phase 6 - # to place reduce actions. Phase 5 computes tighter per-item LALR(1) - # lookaheads for that purpose. - follow_sets = compute_follow_sets(rules, first_sets) - for name, rule in rules.items(): - rule.follow = follow_sets[name] - rule.follow_built = True all_items = {left: rule.items for left, rule in rules.items()} all_tokens = set[type[Token]]() @@ -993,7 +892,7 @@ def __init__( for right, _, _ in rule.rights: all_tokens.update(t for t in right if isinstance(t, type)) - # ── Phase 4: Build LR(0) canonical collection ──────────────────────── + # ── Phase 3: Build LR(0) canonical collection ──────────────────────── # BFS over the LR(0) automaton. ``state_index`` maps a frozenset of # items to the assigned state id, giving O(1) deduplication instead of # a linear scan. ``worklist`` is a deque so processing order is @@ -1030,10 +929,10 @@ def __init__( if is_new: bfs.append(target_state) - # ── Phase 5: Compute LALR(1) per-item lookaheads ──────────────────── - # Build a flat goto_map from the edges collected in Phase 4 and call + # ── Phase 4: Compute LALR(1) per-item lookaheads ──────────────────── + # Build a flat goto_map from the edges collected in Phase 3 and call # compute_lalr1_lookaheads (ASU §9.6). entry_state_ids[i] equals i - # because entry states are the first interned during Phase 4 BFS. + # because entry states are the first interned during Phase 3 BFS. goto_map: dict[tuple[int, Symbol], int] = { (src.id, sym): tgt.id for src, sym, tgt in edges } @@ -1048,7 +947,7 @@ def __init__( if (state.id, item) in lookahead_table } - # ── Phase 6: Populate action/goto table ────────────────────────────── + # ── Phase 5: Populate action/goto table ────────────────────────────── # Shift and Goto actions come directly from the automaton edges. self.table = Table(len(state_list)) for prev, symbol, next in edges: @@ -1060,7 +959,7 @@ def __init__( # Reduce and Accept actions come from complete items (dot at end). # LALR(1): state.lookaheads[item] holds the per-item lookahead set - # computed in Phase 5. A reduce for A → α fires only on the tokens + # computed in Phase 4. A reduce for A → α fires only on the tokens # in that set, which is a subset of FOLLOW(A) and avoids spurious # conflicts caused by FOLLOW-set inflation. # Conflicts are resolved by precedence and associativity: diff --git a/tests/test_first_follow.py b/tests/test_first_follow.py index b109e28..51ac9ce 100644 --- a/tests/test_first_follow.py +++ b/tests/test_first_follow.py @@ -1,4 +1,4 @@ -"""Unit tests for compute_first_sets and compute_follow_sets.""" +"""Unit tests for compute_first_sets.""" from __future__ import annotations @@ -7,13 +7,10 @@ import pytest from plare.parser import ( - EOS, EPSILON, Rule, - StartVariable, Symbol, compute_first_sets, - compute_follow_sets, ) from plare.token import Token @@ -185,25 +182,3 @@ def test_dragon_book_expression_grammar_first() -> None: assert first["F"] == {LParen, IdTok} -def test_dragon_book_expression_grammar_follow() -> None: - """Classic Dragon Book expression grammar — textbook FOLLOW sets. - - Grammar (same as above, augmented with S → E): - Expected: - FOLLOW(E) = {PlusTok, RParen, EOS} - FOLLOW(T) = {PlusTok, StarTok, RParen, EOS} - FOLLOW(F) = {PlusTok, StarTok, RParen, EOS} - """ - start = StartVariable("E") - rules: dict[str, Rule[Null]] = { - start: make_rule(start, [["E"]]), - "E": make_rule("E", [["E", PlusTok, "T"], ["T"]]), - "T": make_rule("T", [["T", StarTok, "F"], ["F"]]), - "F": make_rule("F", [[LParen, "E", RParen], [IdTok]]), - } - first = compute_first_sets(rules) - follow = compute_follow_sets(rules, first) - - assert follow["E"] == {PlusTok, RParen, EOS} - assert follow["T"] == {PlusTok, StarTok, RParen, EOS} - assert follow["F"] == {PlusTok, StarTok, RParen, EOS} From d4fdd9f39db4a45663dd8b90c7a2fa689481f681 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:13:14 +0000 Subject: [PATCH 02/11] refactor(parser): rename force_update to resolve_conflict; add expected_tokens Table.force_update renamed to Table.resolve_conflict to make the intent explicit: it is only called after a conflict has been decided, not for arbitrary table overwrites. Table.expected_tokens(state) is a new helper that returns the terminal classes with an action in a given state. It will be used by Parser.parse to populate ParsingError.expected. Co-Authored-By: Claude Sonnet 4.6 --- plare/parser.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/plare/parser.py b/plare/parser.py index 6976467..cd8ff61 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -10,11 +10,10 @@ Construction pipeline (``Parser.__init__``): 1. Augment the grammar with ``StartVariable(X) → X`` entry rules. 2. Compute FIRST sets for every non-terminal. - 3. Compute FOLLOW sets for every non-terminal (requires FIRST sets). - 4. Build the LR(0) canonical collection (states + transitions) via + 3. Build the LR(0) canonical collection (states + transitions) via ``closure`` / ``goto`` BFS. - 5. Compute LALR(1) per-item lookahead sets (ASU §9.6). - 6. Populate the action/goto table; resolve shift/reduce and reduce/reduce + 4. Compute LALR(1) per-item lookahead sets (ASU §9.6). + 5. Populate the action/goto table; resolve shift/reduce and reduce/reduce conflicts using token precedence and associativity. """ @@ -413,13 +412,13 @@ def __getitem__(self, key: tuple[int, Symbol]) -> Action[T] | None: state, symbol = key return self.table[state][symbol] - def force_update(self, state: int, symbol: Symbol, action: Action[T]) -> None: - """Overwrite a table entry without conflict checking. + def expected_tokens(self, state: int) -> list[type[Token]]: + """Return terminal classes that have an action in *state*.""" + return [sym for sym in self.table[state] if isinstance(sym, type) and issubclass(sym, Token)] - Used exclusively by ``Parser.__init__`` after it has decided which - action wins a conflict. Must not be called for any other purpose. - """ - self.table[state][symbol] = action + def resolve_conflict(self, state: int, symbol: Symbol, winner: Action[T]) -> None: + """Overwrite a table entry with the winning action from a resolved conflict.""" + self.table[state][symbol] = winner class Rule[T]: @@ -1000,7 +999,7 @@ def __init__( item.precedence == symbol.precedence and symbol.associative == "left" ): - self.table.force_update( + self.table.resolve_conflict( state.id, symbol, reduce_action ) except ReduceReduceConflict as e: @@ -1011,12 +1010,12 @@ def __init__( item.left, ) if item.precedence > e.precedence: - self.table.force_update( + self.table.resolve_conflict( state.id, symbol, reduce_action ) elif item.precedence == e.precedence: if item.definition_index < e.definition_index: - self.table.force_update( + self.table.resolve_conflict( state.id, symbol, reduce_action ) logger.info("Parser created") From f898d4eca212057fddff24171b48ac1674200c9f Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:13:35 +0000 Subject: [PATCH 03/11] feat(exception): add structured fields and __str__ to LexingError and ParsingError LexingError gains __init__(message, lineno, offset) and a __str__ that formats as "Line X, col Y: ". ParsingError gains __init__(message, token, lineno, offset, expected) and a matching __str__. Both errors use the same "Line X, col Y:" prefix so callers can parse error output uniformly. Co-Authored-By: Claude Sonnet 4.6 --- plare/exception.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/plare/exception.py b/plare/exception.py index 3916e09..9b5a292 100644 --- a/plare/exception.py +++ b/plare/exception.py @@ -5,6 +5,13 @@ they do not need to distinguish between construction errors and runtime errors. """ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from plare.token import Token + class PlareException(Exception): """Base class for all Plare errors.""" @@ -13,12 +20,21 @@ class PlareException(Exception): class LexingError(PlareException): """Raised by ``Lexer.lex`` when no pattern matches the remaining input. - Args: + Attributes: message: Human-readable description including the offending character. lineno: 1-based line number of the failure position. offset: 0-based column offset of the failure position. """ + def __init__(self, message: str, lineno: int, offset: int) -> None: + super().__init__(message) + self.message = message + self.lineno = lineno + self.offset = offset + + def __str__(self) -> str: + return f"Line {self.lineno}, col {self.offset}: {self.message}" + class ParserError(PlareException): """Raised during ``Parser.__init__`` when the grammar is invalid. @@ -31,5 +47,28 @@ class ParserError(PlareException): class ParsingError(PlareException): """Raised by ``Parser.parse`` when the token stream does not match the grammar. - Typical causes: an unexpected token class, or premature end of input. + Attributes: + token: The offending token, or ``None`` when end-of-input is unexpected. + lineno: 1-based line number of the failure position. + offset: 0-based column offset of the failure position. + expected: Terminal token classes that would have been valid at this point. """ + + def __init__( + self, + message: str, + token: Token | None, + lineno: int, + offset: int, + expected: list[type[Token]], + ) -> None: + super().__init__(message) + self.token = token + self.lineno = lineno + self.offset = offset + self.expected = expected + + def __str__(self) -> str: + expected_names = ", ".join(cls.__name__ for cls in self.expected) + loc = f"Line {self.lineno}, col {self.offset}" + return f"{loc}: {self.args[0]}; expected: {expected_names}" From da406b08e114b443c23c3a841e8a71775d2fd5a1 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:14:03 +0000 Subject: [PATCH 04/11] feat(parser): populate ParsingError with token, location, and expected tokens Parser.parse now tracks last_token to carry position context into the end-of-input error. All three ParsingError raise sites pass the offending token (or None for EOF), lineno, offset, and the list of expected terminal classes for the current state. Co-Authored-By: Claude Sonnet 4.6 --- plare/parser.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/plare/parser.py b/plare/parser.py index cd8ff61..b49fe82 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -1053,18 +1053,31 @@ def parse(self, var: str, lexbuf: Iterable[Token]) -> T | Token: key: type[Token] | str | None = None token: Token | None = None + last_token: Token | None = None while True: if token is None: token = next(lexbuf, None) if token is None: - raise ParsingError("Unexpected end of input") + raise ParsingError( + "Unexpected end of input", + token=None, + lineno=last_token.lineno if last_token else 0, + offset=last_token.offset if last_token else 0, + expected=self.table.expected_tokens(state), + ) if key is None: key = type(token) try: action = self.table[state, key] except KeyError: - raise ParsingError(f"Unexpected symbol: {key}") from None + raise ParsingError( + f"Unexpected token: {type(token).__name__}", + token=token, + lineno=token.lineno, + offset=token.offset, + expected=self.table.expected_tokens(state), + ) from None logger.debug("State: %d, Symbol: %s, Action: %s", state, key, action) key = None match action: @@ -1072,6 +1085,7 @@ def parse(self, var: str, lexbuf: Iterable[Token]) -> T | Token: state = n stack.append(state) symbols.append(token) + last_token = token token = None case Reduce(left, n, maker): @@ -1099,10 +1113,22 @@ def parse(self, var: str, lexbuf: Iterable[Token]) -> T | Token: case Accept(symbol): if symbol != var: - raise ParsingError(f"Unexpected symbol parsed: {symbol}") + raise ParsingError( + f"Unexpected symbol parsed: {symbol}", + token=token, + lineno=token.lineno, + offset=token.offset, + expected=self.table.expected_tokens(state), + ) break case _: - raise ParsingError(f"No action for state {state} and symbol {key}") + raise ParsingError( + f"No action for state {state} and symbol {key}", + token=token, + lineno=token.lineno if isinstance(token, Token) else 0, + offset=token.offset if isinstance(token, Token) else 0, + expected=self.table.expected_tokens(state), + ) return symbols[-1] From bf48becf9f045a436176783b01c888a51f64bd8c Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:14:10 +0000 Subject: [PATCH 05/11] docs(__init__): update SLR(1) reference to LALR(1) Co-Authored-By: Claude Sonnet 4.6 --- plare/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plare/__init__.py b/plare/__init__.py index 7d2264e..4e1520e 100644 --- a/plare/__init__.py +++ b/plare/__init__.py @@ -3,7 +3,7 @@ Public API ---------- * ``Lexer`` (:mod:`plare.lexer`) — regex-driven stateful tokeniser. -* ``Parser`` (:mod:`plare.parser`) — SLR(1) parser with operator-precedence +* ``Parser`` (:mod:`plare.parser`) — LALR(1) parser with operator-precedence conflict resolution. * ``Token`` (:mod:`plare.token`) — base class for all terminal symbols. * ``PlareException``, ``LexingError``, ``ParserError``, ``ParsingError`` From 798e16fe763522877ef0c83ec5f39a2b0cfdf574 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:15:22 +0000 Subject: [PATCH 06/11] test(parser): assert structured error fields and message format Six tests cover: - ParsingError.token, .lineno, .offset, .expected on unexpected token - str(ParsingError) contains "Line X, col Y:" prefix and expected class name - ParsingError.expected is non-empty when input is truncated - LexingError.lineno, .offset, .message fields - str(LexingError) "Line X, col Y:" format Co-Authored-By: Claude Sonnet 4.6 --- tests/test_error_reporting.py | 134 ++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 tests/test_error_reporting.py diff --git a/tests/test_error_reporting.py b/tests/test_error_reporting.py new file mode 100644 index 0000000..49b6a98 --- /dev/null +++ b/tests/test_error_reporting.py @@ -0,0 +1,134 @@ +"""Tests for structured LexingError and ParsingError fields.""" + +from __future__ import annotations + +import pytest + +from plare.exception import LexingError, ParsingError +from plare.lexer import Lexer +from plare.parser import Parser +from plare.token import Token + + +# --------------------------------------------------------------------------- +# Token classes for the test grammar: Num "+" Num +# --------------------------------------------------------------------------- + + +class Num(Token): + pass + + +class Plus(Token): + pass + + +class Expr: + def __init__(self, left: Num, right: Num) -> None: + self.left = left + self.right = right + + +# Grammar: expr → Num Plus Num +GRAMMAR: dict[str, list] = { + "expr": [ + ([Num, Plus, Num], Expr, [0, 2]), + ], +} + + +def make_tok(cls: type[Token], *, lineno: int = 1, offset: int = 0) -> Token: + return cls("x", lineno=lineno, offset=offset) + + +# --------------------------------------------------------------------------- +# ParsingError — unexpected token +# --------------------------------------------------------------------------- + + +def test_parsing_error_unexpected_token_fields() -> None: + """ParsingError carries the offending token, its position, and expected classes.""" + p: Parser[Expr] = Parser(GRAMMAR) + wrong_tok = make_tok(Plus, lineno=3, offset=7) + + with pytest.raises(ParsingError) as exc_info: + p.parse("expr", [wrong_tok]) + + e = exc_info.value + assert e.token is wrong_tok + assert e.lineno == 3 + assert e.offset == 7 + assert Num in e.expected + + +def test_parsing_error_unexpected_token_str() -> None: + """str(ParsingError) starts with the 'Line X, col Y:' prefix.""" + p: Parser[Expr] = Parser(GRAMMAR) + wrong_tok = make_tok(Plus, lineno=2, offset=5) + + with pytest.raises(ParsingError) as exc_info: + p.parse("expr", [wrong_tok]) + + msg = str(exc_info.value) + assert msg.startswith("Line 2, col 5:") + assert "Num" in msg + + +# --------------------------------------------------------------------------- +# ParsingError — unexpected end of input +# --------------------------------------------------------------------------- + + +def test_parsing_error_truncated_input_expected_nonempty() -> None: + """When input is too short, ParsingError.expected is non-empty.""" + p: Parser[Expr] = Parser(GRAMMAR) + num_tok = make_tok(Num, lineno=1, offset=0) + + with pytest.raises(ParsingError) as exc_info: + p.parse("expr", [num_tok]) + + e = exc_info.value + assert len(e.expected) > 0 + assert Plus in e.expected + + +def test_parsing_error_truncated_input_str() -> None: + """str(ParsingError) for truncated input contains the expected class name.""" + p: Parser[Expr] = Parser(GRAMMAR) + num_tok = make_tok(Num, lineno=1, offset=0) + + with pytest.raises(ParsingError) as exc_info: + p.parse("expr", [num_tok]) + + msg = str(exc_info.value) + assert "Line " in msg + assert "Plus" in msg + + +# --------------------------------------------------------------------------- +# LexingError — unrecognised character +# --------------------------------------------------------------------------- + + +def test_lexing_error_fields() -> None: + """LexingError carries lineno, offset, and a readable message.""" + lexer: Lexer[None] = Lexer({"start": [("[0-9]+", Num)]}) + + with pytest.raises(LexingError) as exc_info: + list(lexer.lex("start", "123!456")) + + e = exc_info.value + assert e.lineno >= 1 + assert e.offset >= 0 + assert "!" in e.message + + +def test_lexing_error_str_format() -> None: + """str(LexingError) starts with 'Line X, col Y:'.""" + lexer: Lexer[None] = Lexer({"start": [("[0-9]+", Num)]}) + + with pytest.raises(LexingError) as exc_info: + list(lexer.lex("start", "!")) + + msg = str(exc_info.value) + assert msg.startswith("Line 1, col 0:") From c6d0611a9fe14bf637d9c1903fbdc0c8c5b56b8c Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:16:49 +0000 Subject: [PATCH 07/11] style: fix pyright errors and apply black/isort formatting Remove redundant issubclass in Table.expected_tokens (isinstance(sym, type) already narrows Symbol to type[Token]). Remove redundant isinstance in the case _ fallthrough (token is already Token at that point). Add GrammarEntry type alias to test_error_reporting.py so the GRAMMAR dict is fully typed. Co-Authored-By: Claude Sonnet 4.6 --- plare/parser.py | 7 +++---- tests/test_error_reporting.py | 7 ++++++- tests/test_first_follow.py | 2 -- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/plare/parser.py b/plare/parser.py index b49fe82..125cb5a 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -414,7 +414,7 @@ def __getitem__(self, key: tuple[int, Symbol]) -> Action[T] | None: def expected_tokens(self, state: int) -> list[type[Token]]: """Return terminal classes that have an action in *state*.""" - return [sym for sym in self.table[state] if isinstance(sym, type) and issubclass(sym, Token)] + return [sym for sym in self.table[state] if isinstance(sym, type)] def resolve_conflict(self, state: int, symbol: Symbol, winner: Action[T]) -> None: """Overwrite a table entry with the winning action from a resolved conflict.""" @@ -521,7 +521,6 @@ def compute_first_sets[T](rules: dict[str, Rule[T]]) -> dict[str, set[type[Token return first - def symbol_sort_key(s: Symbol) -> tuple[int, str]: """Return a sort key that gives a stable total order over grammar symbols. @@ -1126,8 +1125,8 @@ def parse(self, var: str, lexbuf: Iterable[Token]) -> T | Token: raise ParsingError( f"No action for state {state} and symbol {key}", token=token, - lineno=token.lineno if isinstance(token, Token) else 0, - offset=token.offset if isinstance(token, Token) else 0, + lineno=token.lineno, + offset=token.offset, expected=self.table.expected_tokens(state), ) diff --git a/tests/test_error_reporting.py b/tests/test_error_reporting.py index 49b6a98..04a48d8 100644 --- a/tests/test_error_reporting.py +++ b/tests/test_error_reporting.py @@ -9,6 +9,11 @@ from plare.parser import Parser from plare.token import Token +type GrammarEntry[T] = ( + tuple[list[type[Token] | str], type[T] | None, list[int]] + | tuple[list[type[Token] | str], type[T] | None, list[int], type[Token]] +) + # --------------------------------------------------------------------------- # Token classes for the test grammar: Num "+" Num @@ -30,7 +35,7 @@ def __init__(self, left: Num, right: Num) -> None: # Grammar: expr → Num Plus Num -GRAMMAR: dict[str, list] = { +GRAMMAR: dict[str, list[GrammarEntry[Expr]]] = { "expr": [ ([Num, Plus, Num], Expr, [0, 2]), ], diff --git a/tests/test_first_follow.py b/tests/test_first_follow.py index 51ac9ce..0c53eb4 100644 --- a/tests/test_first_follow.py +++ b/tests/test_first_follow.py @@ -180,5 +180,3 @@ def test_dragon_book_expression_grammar_first() -> None: assert first["E"] == {LParen, IdTok} assert first["T"] == {LParen, IdTok} assert first["F"] == {LParen, IdTok} - - From 7a171c69a0d88ad1bfa2229cf83ba173994a9dba Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:27:53 +0000 Subject: [PATCH 08/11] docs(parser): remove FOLLOW references from docstrings and comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The LALR(1) implementation no longer uses FOLLOW sets; reduce actions fire on per-item lookaheads computed by ASU §9.6. Remove the SLR(1)-vs-LALR(1) comparison phrasing that mentioned FOLLOW. Co-Authored-By: Claude Sonnet 4.6 --- plare/parser.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plare/parser.py b/plare/parser.py index 125cb5a..fa7a69e 100644 --- a/plare/parser.py +++ b/plare/parser.py @@ -2,10 +2,7 @@ This module implements an **LALR(1)** (Look-Ahead LR, 1 token of lookahead) parser. Reduce actions fire on per-item lookahead sets computed via the -spontaneous-generation and propagation algorithm (Aho-Sethi-Ullman §9.6), -rather than on the global FOLLOW set used by SLR(1). This eliminates -spurious conflicts that arise when FOLLOW(A) contains tokens that cannot -actually follow A in a specific state. +spontaneous-generation and propagation algorithm (Aho-Sethi-Ullman §9.6). Construction pipeline (``Parser.__init__``): 1. Augment the grammar with ``StartVariable(X) → X`` entry rules. @@ -958,8 +955,7 @@ def __init__( # Reduce and Accept actions come from complete items (dot at end). # LALR(1): state.lookaheads[item] holds the per-item lookahead set # computed in Phase 4. A reduce for A → α fires only on the tokens - # in that set, which is a subset of FOLLOW(A) and avoids spurious - # conflicts caused by FOLLOW-set inflation. + # in that set. # Conflicts are resolved by precedence and associativity: # Shift/Reduce: prefer shift unless the production has higher # precedence than the lookahead token, or equal precedence with From 0ece0297d459e678ab512280076de38650bfa842 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:31:47 +0000 Subject: [PATCH 09/11] docs(tests): remove FOLLOW set references from safety-net test comments Replace FOLLOW set computations and SLR(1)-vs-LALR(1) comparisons in the section header and two test docstrings with plain descriptions of the per-item lookahead behaviour. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_parser_safety_net.py | 48 +++++++++------------------------ 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/tests/test_parser_safety_net.py b/tests/test_parser_safety_net.py index d1006a2..dd12f15 100644 --- a/tests/test_parser_safety_net.py +++ b/tests/test_parser_safety_net.py @@ -562,25 +562,15 @@ def test_shift_reduce_resolution_prefers_shift() -> None: # --------------------------------------------------------------------------- -# LALR(1)-resolvable grammars — FOLLOW-set inflation creates a spurious R/R -# conflict in SLR(1) that LALR(1) per-item lookaheads cleanly resolve. +# LALR(1)-resolvable grammars — two non-terminals share the same single-token +# body and land in the same LR(0) reduce state, creating an apparent R/R +# conflict. LALR(1) per-item lookaheads in that state are disjoint, so the +# conflict is resolved without ambiguity. # -# Two non-terminals (A_nt/B_nt, X_nt/Y_nt) share the same single-token body -# and land in the same LR(0) reduce state. One of them also appears in an -# *unrelated* production that inflates its FOLLOW set so that it overlaps the -# other's FOLLOW set — but only in states where the two items do NOT coexist. -# -# Key distinction from the Aho/Sethi/Ullman (LR(1)-only) grammar: -# In the Aho grammar both non-terminals appear symmetrically in four -# productions, so every LR(1) state that holds {A_nt → C •, B_nt → C •} -# comes in two mirror images whose lookaheads cross-contaminate when merged -# in LALR(1), creating a new R/R conflict not present in LR(1). -# -# Here, the extra production that inflates FOLLOW(A_nt) produces a *separate* -# LR(0) state (core {A_nt → E •} only, no B_nt) that is never merged with -# the conflict state (core {A_nt → E •, B_nt → E •}). LALR(1) lookaheads -# in the conflict state remain {B8x} vs {C8x, D8x} — disjoint — so no -# new conflict is introduced by merging. +# The grammars are constructed so that an extra production creates a *separate* +# LR(0) state (core with only one of the two non-terminals) that is never +# merged with the conflict state. This keeps the LALR(1) lookaheads in the +# conflict state disjoint, making the grammar LALR(1) but not SLR(1). # --------------------------------------------------------------------------- @@ -610,7 +600,7 @@ def __init__(self, *args: object) -> None: def test_lalr1_rr_conflict_follow_inflation_variant_1() -> None: - """FOLLOW-set inflation creates a spurious SLR(1) R/R conflict. + """LALR(1) resolves an R/R conflict that SLR(1) cannot. Grammar: start → A8x A_nt B8x | A8x B_nt C8x | A8x B_nt D8x | A_nt C8x @@ -618,17 +608,9 @@ def test_lalr1_rr_conflict_follow_inflation_variant_1() -> None: B_nt → E8x The state reached after [A8x, E8x] contains {A_nt → E8x •, B_nt → E8x •}. - FOLLOW(A_nt) = {B8x, C8x} (from "A8x A_nt B8x" and "A_nt C8x") - FOLLOW(B_nt) = {C8x, D8x} (from "A8x B_nt C8x" and "A8x B_nt D8x") - Intersection {C8x} creates a spurious SLR(1) R/R conflict; definition order - picks A_nt. Parsing [A8x, E8x, C8x] therefore fails: SLR(1) reduces E8x - to A_nt, then rejects C8x (expected B8x). - LALR(1) per-item lookaheads in that state are {B8x} for A_nt and - {C8x, D8x} for B_nt — disjoint — so C8x correctly triggers B_nt and the - parse succeeds. The inflation token C8x only appears as a lookahead in a - separate LR(0) state (core {A_nt → E8x •} alone), which is never merged - with the conflict state, so LALR(1) is sufficient. + {C8x, D8x} for B_nt — disjoint — so C8x correctly triggers B_nt and + [A8x, E8x, C8x] parses successfully. """ p = Parser( { @@ -686,13 +668,9 @@ def test_lalr1_rr_conflict_follow_inflation_variant_2() -> None: X_nt → T8y Y_nt → T8y - FOLLOW(X_nt) = {Q8y, R8y} (from "P8y X_nt Q8y" and "X_nt R8y") - FOLLOW(Y_nt) = {R8y, S8y} (from "P8y Y_nt R8y" and "P8y Y_nt S8y") - Intersection {R8y}: spurious SLR(1) R/R; X_nt wins by definition order. - Parsing [P8y, T8y, R8y] fails: SLR(1) reduces T8y to X_nt, then rejects - R8y (expected Q8y). LALR(1) per-item lookaheads after [P8y, T8y]: X_nt → {Q8y}, Y_nt → - {R8y, S8y} — disjoint — so R8y correctly triggers Y_nt. + {R8y, S8y} — disjoint — so R8y correctly triggers Y_nt and + [P8y, T8y, R8y] parses successfully. """ p = Parser( { From 33309743211a4ccb1bd66dba18e1b1351b6ae69c Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:34:37 +0000 Subject: [PATCH 10/11] refactor(tests): rename follow_inflation test functions to resolves_rr_conflict Co-Authored-By: Claude Sonnet 4.6 --- tests/test_parser_safety_net.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_parser_safety_net.py b/tests/test_parser_safety_net.py index dd12f15..5e45754 100644 --- a/tests/test_parser_safety_net.py +++ b/tests/test_parser_safety_net.py @@ -599,7 +599,7 @@ def __init__(self, *args: object) -> None: pass -def test_lalr1_rr_conflict_follow_inflation_variant_1() -> None: +def test_lalr1_resolves_rr_conflict_variant_1() -> None: """LALR(1) resolves an R/R conflict that SLR(1) cannot. Grammar: @@ -660,7 +660,7 @@ def __init__(self, *args: object) -> None: pass -def test_lalr1_rr_conflict_follow_inflation_variant_2() -> None: +def test_lalr1_resolves_rr_conflict_variant_2() -> None: """Isomorphic variant confirming the result is independent of token naming. Grammar: From 2e19f0ca176106c8c304ac4690a9d15807a79276 Mon Sep 17 00:00:00 2001 From: Henry Lee Date: Tue, 12 May 2026 14:39:49 +0000 Subject: [PATCH 11/11] refactor(exception): replace TYPE_CHECKING guard with direct Token import No circular dependency exists between exception.py and token.py. Co-Authored-By: Claude Sonnet 4.6 --- plare/exception.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plare/exception.py b/plare/exception.py index 9b5a292..9398baa 100644 --- a/plare/exception.py +++ b/plare/exception.py @@ -7,10 +7,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from plare.token import Token +from plare.token import Token class PlareException(Exception):