diff --git a/loomgen/README.md b/loomgen/README.md index aa4cda8c..b949bd97 100644 --- a/loomgen/README.md +++ b/loomgen/README.md @@ -25,6 +25,29 @@ string escapes), so write engine-valid MoonBit `re`-dialect regex — e.g. a lit hyphen in a class is `\-`, not a bare `-`. A pattern that can match the empty string is rejected at generation time. + +## Line-mode lexer generation (`--line-lexer`, #561) + +`#loom.line_pattern("regex")` on token variants — emits `line_lexer.g.mbt` with +per-mode lexer functions for each `#loom.line_mode` `#loom.lexmode`. Each function +matches `#loom.line_pattern` regexes against the current line (pos → newline) in +declaration order. Patterns with a declared `#loom.lexmode("ModeName")` are only +emitted in that mode's function; patterns without one appear in all line_mode +functions. Nullary tokens are produced directly; payload-carrying tokens require +`#loom.custom_lex` for extraction. Generated function names follow the same +`lex_` convention as `lexer_skeleton.g.mbt` stubs, so they can +replace the skeleton's `abort("not implemented")` stubs. + +Reuses the same supported regex subset, nullability checks, and structural +validation as `#loom.pattern`. Alternation `|` is rejected (leftmost-match, not +longest-match). Patterns are wrapped in `^(?:...)` so they should not include +their own `^` anchor. + +Usage: +```bash +moon run loomgen --target native -- --line-lexer /line_lexer.g.mbt \ + --term +``` ### Supported `#loom.pattern` subset Literals, *literal-meta* escapes (`\.`, `\-`, `\+`, `\xHH`, …), character classes `[..]` diff --git a/loomgen/emit_lexer.mbt b/loomgen/emit_lexer.mbt index caccf944..7b8f1b8d 100644 --- a/loomgen/emit_lexer.mbt +++ b/loomgen/emit_lexer.mbt @@ -64,11 +64,18 @@ fn collect_lexer_inputs(token_enum : EnumDecl) -> Result[LexerInputs, String] { Some(Keyword(kw)) => keywords.push((kw, v.name)) _ => () } - if v.custom_lex is Some(fn_name) { + // custom_lex functions for #loom.line_pattern variants belong to the + // line-mode lexer, not the character-level lexer — skip them here. + if v.custom_lex is Some(fn_name) && v.line_pattern is None { custom_lex_fns.push(fn_name) } } for v in variants { + // #loom.line_pattern variants are produced by the line-mode lexer, not the + // character-level lexer — skip them entirely. + if v.line_pattern is Some(_) { + continue + } let is_candidate = v.pattern is Some(_) || v.custom_lex is Some(_) || v.role is Some(Punct(_)) || @@ -124,13 +131,13 @@ fn collect_lexer_inputs(token_enum : EnumDecl) -> Result[LexerInputs, String] { "#loom.token enum has #loom.keyword variants and more than one pattern-backed #loom.ident variant; keyword post-classification is ambiguous (it keys off a single identifier match). Keep exactly one #loom.ident, or handle the others via #loom.custom_lex", ) } - // A #loom.punct whose text equals a keyword shadows it: the punct wins - // longest-match at equal length, so the keyword variant would never be - // produced. (General #loom.pattern/keyword overlap is the author's - // responsibility — it cannot be decided statically.) + // Filter out #loom.line_pattern variants — they're produced by the line-mode + // lexer, not the character-level lexer. + let char_variants = variants.filter(fn(v) { v.line_pattern is None }) + // Skip line_pattern variants in keyword/punct collision check for kw in keywords { let (kw_text, kw_name) = kw - for v in variants { + for v in char_variants { if v.role is Some(Punct(lit)) && lit == kw_text { return Err( "#loom.punct variant " + @@ -146,7 +153,12 @@ fn collect_lexer_inputs(token_enum : EnumDecl) -> Result[LexerInputs, String] { } } } - Ok({ variants, keywords, custom_lex_fns, has_keywords: keywords.length() > 0 }) + Ok({ + variants: char_variants, + keywords, + custom_lex_fns, + has_keywords: keywords.length() > 0, + }) } ///| diff --git a/loomgen/emit_lexer_wbtest.mbt b/loomgen/emit_lexer_wbtest.mbt index 4127e31f..eee24718 100644 --- a/loomgen/emit_lexer_wbtest.mbt +++ b/loomgen/emit_lexer_wbtest.mbt @@ -694,3 +694,277 @@ test "emit_lexer golden matches fixtures/default_lex_fixture.g.mbt" { } assert_eq(generated, golden) } + +// ── #561 line_pattern / line_mode tests ────────────────────────────── + +///| +/// Helper: parse a fixture file that has both #loom.token and #loom.term enums. +fn full_enums_of(src : String) -> (EnumDecl, EnumDecl) { + match parse_annotations(src) { + Ok(a) => + ( + match a.token_enum { + Some(te) => te + None => abort("no token enum parsed") + }, + match a.term_enum { + Some(te) => te + None => abort("no term enum parsed") + }, + ) + Err(m) => abort("parse failed: " + m) + } +} + +///| +test "emit_line_lexer generates line-mode lexer functions for line_pattern tokens" { + let src = read_fixture("fixtures/line_pattern_fixture.mbt") + let (te, term_enum) = full_enums_of(src) + let generated = match emit_line_lexer(te, Some(term_enum), "@core") { + Some(s) => s + None => abort("emit_line_lexer returned None") + } + // Should contain the lex_line_start function + // Should contain custom_lex call for HeadingMarker + inspect( + generated.contains("lex_heading_marker(source, pos, len)"), + content="true", + ) + // Should contain pattern checks for each line_pattern token + inspect(generated.contains("ListMarkerDash"), content="true") + inspect(generated.contains("ListMarkerStar"), content="true") + inspect(generated.contains("ListMarkerPlus"), content="true") + inspect(generated.contains("ThematicBreak"), content="true") + inspect(generated.contains("BlockQuoteMarker"), content="true") + // Should contain the regex patterns + inspect(generated.contains("re\"^"), content="true") + inspect(generated.contains("#{1,6} "), content="true") + // Check ListMarkerDash uses escaped hyphen in regex + inspect(generated.contains("\\- "), content="true") + // Should NOT contain Inline mode function (only LineStart has #loom.line_mode) + inspect(generated.contains("fn lex_inline"), content="false") + // Should return Done when pos >= length + inspect(generated.contains("return (Done, LineStart)"), content="true") + // Should have Invalid fallthrough + inspect( + generated.contains("Not a block-level token (line_pattern)"), + content="true", + ) +} + +///| +test "emit_line_lexer returns None when no line_pattern variants exist" { + let src = read_fixture("fixtures/pattern_lexer_fixture.mbt") + match parse_annotations(src) { + Ok(a) => + match a.token_enum { + Some(te) => { + let result = emit_line_lexer(te, a.term_enum, "@core") + guard result is None else { + fail("expected None for pattern-only fixture") + } + } + None => abort("no token enum") + } + Err(m) => abort("parse failed: " + m) + } +} + +///| +test "emit_line_lexer returns None when no term enum has line_mode" { + let src = + #|#loom.token + #|pub(all) enum T { + #| #loom.line_pattern("^test") + #| TestToken + #| #loom.error + #| Error(String) + #| #loom.eof + #| E + #|} derive(Eq) + #|#loom.term + #|pub(all) enum Term { + #| #loom.leaf + #| #loom.lexmode("Inline") + #| SomeNode + #| #loom.root + #| Root + #| #loom.errornode + #| ErrorNode + #|} derive(Eq) + let (te, term_enum) = full_enums_of(src) + guard emit_line_lexer(te, Some(term_enum), "@core") is None else { + fail("expected None when no mode has line_mode") + } +} + +///| +test "parse_annotations rejects #loom.line_pattern with #loom.keyword" { + let src = + #|#loom.token + #|pub(all) enum T { + #| #loom.keyword("let") + #| #loom.line_pattern("^let") + #| Let + #| #loom.eof + #| E + #|} derive(Eq) + match parse_annotations(src) { + Err(msg) => inspect(msg.contains("#loom.keyword"), content="true") + Ok(_) => abort("expected rejection") + } +} + +///| +test "parse_annotations rejects nullable #loom.line_pattern" { + let src = + #|#loom.token + #|pub(all) enum T { + #| #loom.line_pattern("a*") + #| A + #| #loom.eof + #| E + #|} derive(Eq) + match parse_annotations(src) { + Err(msg) => inspect(msg.contains("nullable"), content="true") + Ok(_) => abort("expected rejection") + } +} + +///| +test "parse_annotations rejects #loom.line_pattern on #loom.eof" { + let src = + #|#loom.token + #|pub(all) enum T { + #| #loom.line_pattern("^test") + #| Test + #| #loom.eof + #| #loom.line_pattern("^$") + #| E + #|} derive(Eq) + match parse_annotations(src) { + Err(msg) => inspect(msg.contains("#loom.eof"), content="true") + Ok(_) => abort("expected rejection") + } +} + +///| +test "parse_annotations rejects #loom.line_pattern with payload but no #loom.custom_lex" { + let src = + #|#loom.token + #|pub(all) enum T { + #| #loom.line_pattern("#{1,6} ") + #| HeadingMarker(Int) + #| #loom.error + #| Error(String) + #| #loom.eof + #| E + #|} derive(Eq) + match parse_annotations(src) { + Err(msg) => inspect(msg.contains("#loom.custom_lex"), content="true") + Ok(_) => + abort( + "expected rejection for payload-carrying line_pattern without custom_lex", + ) + } +} + +///| +test "emit_line_lexer golden matches fixtures/line_pattern_fixture.g.mbt" { + let src = read_fixture("fixtures/line_pattern_fixture.mbt") + let golden = read_fixture("fixtures/line_pattern_fixture.g.mbt") + let (te, term_enum) = full_enums_of(src) + let generated = match emit_line_lexer(te, Some(term_enum), "@core") { + Some(s) => s + None => abort("emit_line_lexer returned None") + } + assert_eq(generated, golden) +} + +///| +test "emit_line_lexer accepts #loom.line_pattern with #loom.custom_lex" { + let src = + #|#loom.token + #|pub(all) enum T { + #| #loom.line_pattern("#{1,6} ") + #| #loom.custom_lex("lex_heading") + #| HeadingMarker(Int) + #| #loom.line_pattern(">") + #| BlockQuoteMarker + #| #loom.error + #| Error(String) + #| #loom.eof + #| E + #|} derive(Eq) + #|#loom.term + #|pub(all) enum Term { + #| #loom.leaf + #| #loom.lexmode("LineStart") + #| #loom.line_mode + #| Heading + #| #loom.root + #| Root + #| #loom.errornode + #| ErrorNode + #|} derive(Eq) + let (te, term_enum) = full_enums_of(src) + guard emit_line_lexer(te, Some(term_enum), "@core") is Some(_) else { + fail("expected Some for valid line_pattern with custom_lex") + } +} + +///| +test "emit_line_lexer filters candidates by mode via lexmode_name" { + let src = + #|#loom.token + #|pub(all) enum T { + #| #loom.line_pattern("#{1,6} ") + #| HeadingMarker + #| #loom.line_pattern(">") + #| #loom.lexmode("BlockQuoteLineStart") + #| BlockQuoteMarker + #| #loom.line_pattern("--- ") + #| ThematicBreak + #| #loom.error + #| Error(String) + #| #loom.eof + #| E + #|} derive(Eq) + #|#loom.term + #|pub(all) enum Term { + #| #loom.leaf + #| #loom.lexmode("LineStart") + #| #loom.line_mode + #| Heading + #| #loom.leaf + #| #loom.lexmode("BlockQuoteLineStart") + #| #loom.line_mode + #| BlockQuote + #| #loom.root + #| Root + #| #loom.errornode + #| ErrorNode + #|} derive(Eq) + let (te, term_enum) = full_enums_of(src) + let generated = match emit_line_lexer(te, Some(term_enum), "@core") { + Some(s) => s + None => abort("emit_line_lexer returned None") + } + // lex_line_start should include HeadingMarker and ThematicBreak (no lexmode constraint) + inspect(generated.contains("HeadingMarker"), content="true") + inspect(generated.contains("ThematicBreak"), content="true") + // lex_line_start should NOT include BlockQuoteMarker (assigned to BlockQuoteLineStart) + // Check that lex_line_start function body contains neither the name nor its pattern + inspect(generated.contains("fn lex_line_start"), content="true") + inspect(generated.contains("fn lex_block_quote_line_start"), content="true") + // BlockQuoteMarker should only appear in the block_quote_line_start function. + let line_start_idx = generated.find("fn lex_line_start") + let bq_start_idx = generated.find("fn lex_block_quote_line_start") + guard line_start_idx is Some(ls) && bq_start_idx is Some(bq) else { + fail("expected both lex functions in output") + } + let line_start_section = generated[ls:bq] + inspect(line_start_section.contains("BlockQuoteMarker"), content="false") + let bq_section = generated[bq:] + inspect(bq_section.contains("BlockQuoteMarker"), content="true") +} diff --git a/loomgen/emit_line_lexer.mbt b/loomgen/emit_line_lexer.mbt new file mode 100644 index 00000000..e190cf57 --- /dev/null +++ b/loomgen/emit_line_lexer.mbt @@ -0,0 +1,191 @@ +///| +/// Emit `line_lexer.g.mbt` — line-mode lexer functions driven by +/// `#loom.line_pattern` annotations (#561). +/// +/// For each `#loom.line_mode` lexmode, generates a per-mode function that: +/// 1. Reads the current line (pos → newline) +/// 2. Tries each `#loom.line_pattern` regex in declaration order +/// 3. Produces the matching token (via `#loom.custom_lex` when specified) +/// 4. Falls through to `Invalid` when no pattern matches (caller handles retry) +/// +/// Generated functions follow the same signature as `emit_lexer_skeleton` stubs: +/// `(source: String, pos: Int) -> (@core.LexStep[Token], LexMode)`. + +///| +/// Collect line_pattern variants from a token enum. +/// Returns `None` when no token variant has `#loom.line_pattern`. +fn collect_line_patterns(token_enum : EnumDecl) -> Array[VariantDecl]? { + let candidates : Array[VariantDecl] = [] + for v in token_enum.variants { + if v.line_pattern is Some(_) { + candidates.push(v) + } + } + if candidates.length() == 0 { + return None + } + Some(candidates) +} + +///| +/// Generate the line-mode lexer function for a single mode. +fn emit_mode_line_lexer( + b : StringBuilder, + mode_name : String, + candidates : Array[VariantDecl], + token_type : String, + core_qual : String, + fn_name : String, +) -> Unit { + b.write_string( + "fn " + + fn_name + + "(source : String, pos : Int) -> (" + + core_qual + + ".LexStep[" + + token_type + + "], LexMode) {\n", + ) + b.write_string( + " if pos >= source.length() {\n return (Done, " + mode_name + ")\n }\n", + ) + // Find end of current line + b.write_string(" let mut line_end = pos\n") + b.write_string(" while line_end < source.length() {\n") + b.write_string(" match source[line_end:line_end + 1].to_owned() {\n") + b.write_string(" \"\\r\" | \"\\n\" => break\n") + b.write_string(" _ => line_end = line_end + 1\n") + b.write_string(" }\n }\n") + b.write_string(" let rest = source[pos:line_end].view()\n\n") + // Emit each line_pattern check in declaration order. + // Only emit candidates whose lexmode_name matches the enclosing mode (or + // candidates with no lexmode_name, which apply to all line_mode modes). + // Nullary tokens use the candidate's declared lexmode_name as the transition + // target when present, falling back to the enclosing mode_name. + for v in candidates { + // Skip candidates explicitly assigned to a different mode + match v.lexmode_name { + Some(name) if name != mode_name => continue + _ => () + } + let pattern = v.line_pattern.unwrap() + // Escape double quotes for re"..." literal + let safe_pattern = pattern.replace(old="\"", new="\\\"") + // Determine transition target: candidate's lexmode_name, or enclosing mode + let transition = match v.lexmode_name { + Some(name) => name + None => mode_name + } + b.write_string(" // " + v.name + "\n") + b.write_string( + " if rest =~ (re\"^(?:" + safe_pattern + ")\", after=after) {\n", + ) + b.write_string(" let len = rest.length() - after.length()\n") + match v.custom_lex { + Some(lex_fn) => + // custom_lex handles full token + mode construction + b.write_string(" return " + lex_fn + "(source, pos, len)\n") + None => + if v.arg_count == 0 { + // Nullary token — produce directly with transition target + b.write_string( + " return (Produced(" + + core_qual + + ".TokenInfo::new(" + + v.name + + ", len), next_offset=pos + len), " + + transition + + ")\n", + ) + } else { + // Payload token without custom_lex — should not happen (validation rejects) + b.write_string( + " return (" + + core_qual + + ".LexStep::Invalid(at=pos, width=len, " + + "message=\"line_pattern token " + + v.name + + " needs custom_lex\"), " + + mode_name + + ")\n", + ) + } + } + b.write_string(" }\n") + } + // No pattern matched — advance one character position + b.write_string( + " let next = " + core_qual + ".next_char_offset(source, pos)\n", + ) + b.write_string( + " (" + + core_qual + + ".LexStep::Invalid(at=pos, width=next - pos, " + + "message=\"Not a block-level token (line_pattern)\"), " + + mode_name + + ")\n", + ) + b.write_string("}\n") +} + +///| +/// Emit the full `line_lexer.g.mbt` content for all line_mode modes. +/// Returns `None` when no mode has line_pattern candidates. +fn emit_line_lexer( + token_enum : EnumDecl, + term_enum_decl : EnumDecl?, + core_qual : String, +) -> String? { + let candidates = match collect_line_patterns(token_enum) { + None => return None + Some(c) => c + } + // Collect which modes are line_mode from term enum + let line_modes : @hashset.HashSet[String] = @hashset.HashSet([]) + match term_enum_decl { + Some(term_enum) => + for v in term_enum.variants { + if v.line_mode { + match v.lexmode_name { + Some(name) => line_modes.add(name) + None => () + } + } + } + None => () + } + if line_modes.length() == 0 { + return None + } + let token_type = token_enum.name + let b = StringBuilder::new() + b.write_string("// Generated by loomgen (#561) — do not edit.\n\n") + b.write_string("///|\n") + b.write_string("/// Line-mode lexer functions for #loom.line_mode modes.\n") + b.write_string( + "/// Each function matches #loom.line_pattern tokens against whole lines.\n\n", + ) + // Sort modes for deterministic output + let sorted_modes : Array[String] = line_modes.to_array() + sorted_modes.sort_by(fn(a, b) { + if a < b { + -1 + } else if a > b { + 1 + } else { + 0 + } + }) + // Emit one function per line_mode using mode_to_fn_name naming (same as lexer_skeleton stubs) + for mode in sorted_modes { + emit_mode_line_lexer( + b, + mode, + candidates, + token_type, + core_qual, + mode_to_fn_name(mode), + ) + } + Some(b.to_string()) +} diff --git a/loomgen/emit_syntax_kind.mbt b/loomgen/emit_syntax_kind.mbt index ab1c7b49..02fe2862 100644 --- a/loomgen/emit_syntax_kind.mbt +++ b/loomgen/emit_syntax_kind.mbt @@ -46,11 +46,14 @@ fn build_kind_entries( let seen : @hashset.HashSet[String] = @hashset.HashSet([]) let first_role : Map[String, SyntaxRole] = Map([]) for variant in enum_decl.variants { - let role = match variant.role { - Some(r) => r + // #loom.line_pattern tokens may not have a role — synthesize a kind name + // directly using variant name + "Token" suffix. + let (kind_name, role) = match variant.role { + Some(r) => (derive_kind_name(variant.name, r, variant.kind_override), r) + None if variant.line_pattern is Some(_) => + (variant.name + "Token", Punct(variant.name)) None => continue } - let kind_name = derive_kind_name(variant.name, role, variant.kind_override) if seen.contains(kind_name) { match first_role.get(kind_name) { Some(existing) => { diff --git a/loomgen/emit_token_impls.mbt b/loomgen/emit_token_impls.mbt index 7866ee77..a04a79cc 100644 --- a/loomgen/emit_token_impls.mbt +++ b/loomgen/emit_token_impls.mbt @@ -104,7 +104,7 @@ fn show_literal_for(variant : VariantDecl) -> String { Some(ErrorNode) => abort("term variant in token enum: " + variant.name) Some(Root) => abort("term variant in token enum: " + variant.name) Some(View(_, _, _)) => abort("term variant in token enum: " + variant.name) - None => abort("unannotated variant: " + variant.name) + None => to_lowercase_first(variant.name) // #loom.line_pattern tokens without role } } @@ -114,8 +114,10 @@ fn emit_token_impls( token_kinds : Array[(String, Int)], ) -> String { // Validate every variant has a role before generating. + // #loom.line_pattern tokens without a role are exempt — they're produced by + // the line-mode lexer and don't need a scanning role. for v in token_enum.variants { - if v.role is None { + if v.role is None && v.line_pattern is None { abort("variant " + v.name + " has no #loom.* annotation") } } @@ -192,7 +194,12 @@ fn emit_token_impls( b.write_string(" let n : Int = match self {\n") for v in annotated { let pat = pattern_for(v, false) - let kind_name = derive_kind_name(v.name, v.role.unwrap(), v.kind_override) + // #loom.line_pattern tokens may not have a role — synthesize kind name + // using variant name + "Token" to match build_kind_entries. + let kind_name = match v.role { + Some(r) => derive_kind_name(v.name, r, v.kind_override) + None => v.name + "Token" + } let raw = kind_name_to_raw(token_kinds, kind_name) b.write_string(" " + pat + " => " + raw.to_string() + "\n") } diff --git a/loomgen/fixtures/line_pattern_fixture.g.mbt b/loomgen/fixtures/line_pattern_fixture.g.mbt new file mode 100644 index 00000000..8c0fe63d --- /dev/null +++ b/loomgen/fixtures/line_pattern_fixture.g.mbt @@ -0,0 +1,52 @@ +// Generated by loomgen (#561) — do not edit. + +///| +/// Line-mode lexer functions for #loom.line_mode modes. +/// Each function matches #loom.line_pattern tokens against whole lines. + +fn lex_line_start(source : String, pos : Int) -> (@core.LexStep[Token], LexMode) { + if pos >= source.length() { + return (Done, LineStart) + } + let mut line_end = pos + while line_end < source.length() { + match source[line_end:line_end + 1].to_owned() { + "\r" | "\n" => break + _ => line_end = line_end + 1 + } + } + let rest = source[pos:line_end].view() + + // HeadingMarker + if rest =~ (re"^(?:#{1,6} )", after=after) { + let len = rest.length() - after.length() + return lex_heading_marker(source, pos, len) + } + // ListMarkerDash + if rest =~ (re"^(?:\\- )", after=after) { + let len = rest.length() - after.length() + return (Produced(@core.TokenInfo::new(ListMarkerDash, len), next_offset=pos + len), LineStart) + } + // ListMarkerStar + if rest =~ (re"^(?:\\* )", after=after) { + let len = rest.length() - after.length() + return (Produced(@core.TokenInfo::new(ListMarkerStar, len), next_offset=pos + len), LineStart) + } + // ListMarkerPlus + if rest =~ (re"^(?:\\+ )", after=after) { + let len = rest.length() - after.length() + return (Produced(@core.TokenInfo::new(ListMarkerPlus, len), next_offset=pos + len), LineStart) + } + // ThematicBreak + if rest =~ (re"^(?:--- )", after=after) { + let len = rest.length() - after.length() + return (Produced(@core.TokenInfo::new(ThematicBreak, len), next_offset=pos + len), LineStart) + } + // BlockQuoteMarker + if rest =~ (re"^(?:>)", after=after) { + let len = rest.length() - after.length() + return (Produced(@core.TokenInfo::new(BlockQuoteMarker, len), next_offset=pos + len), LineStart) + } + let next = @core.next_char_offset(source, pos) + (@core.LexStep::Invalid(at=pos, width=next - pos, message="Not a block-level token (line_pattern)"), LineStart) +} diff --git a/loomgen/fixtures/line_pattern_fixture.mbt b/loomgen/fixtures/line_pattern_fixture.mbt new file mode 100644 index 00000000..a93e3eed --- /dev/null +++ b/loomgen/fixtures/line_pattern_fixture.mbt @@ -0,0 +1,47 @@ +/// Fixture for #561 line lexer generation. +/// Exercises #loom.line_pattern on block-level token variants and +/// #loom.line_mode on term variants with #loom.lexmode. +/// Note: regex alternation is not supported in line_pattern (leftmost-match +/// vs longest-match conflict), so each pattern must be unambiguous. +/// The generated code wraps patterns in `^(?:...)` so patterns should not +/// include their own `^` anchor. + +#loom.token +pub(all) enum Token { + #loom.line_pattern("#{1,6} ") + #loom.custom_lex("lex_heading_marker") + HeadingMarker(Int) + #loom.line_pattern("\\- ") + ListMarkerDash + #loom.line_pattern("\\* ") + ListMarkerStar + #loom.line_pattern("\\+ ") + ListMarkerPlus + #loom.line_pattern("--- ") + ThematicBreak + #loom.line_pattern(">") + BlockQuoteMarker + #loom.error + Error(String) + #loom.eof + EOF +} derive(Eq, Debug) + +#loom.term +pub(all) enum Term { + #loom.leaf + #loom.lexmode("LineStart") + #loom.line_mode + Heading + #loom.leaf + #loom.lexmode("LineStart") + #loom.line_mode + ListItem + #loom.leaf + #loom.lexmode("Inline") + Text + #loom.root + Root + #loom.errornode + ErrorNode +} derive(Eq, Debug) diff --git a/loomgen/main.mbt b/loomgen/main.mbt index 821cfe31..2d98dd6c 100644 --- a/loomgen/main.mbt +++ b/loomgen/main.mbt @@ -99,6 +99,10 @@ fn parse_args(argv : Array[String]) -> Result[@argparse.Matches, String] { "lexer", about="Output path for generated step lexer (.g.mbt)", ), + @argparse.OptionArg( + "line-lexer", + about="Output path for generated line-mode lexer functions (.g.mbt) (#561)", + ), @argparse.OptionArg( "grammar-ir", about="Output path for generated GrammarIr from #loom.rule annotations (.g.mbt)", @@ -702,6 +706,9 @@ fn main { matches.values.get_or_default("syntax-type", []), ) let lexer_path = first_value(matches.values.get_or_default("lexer", [])) + let line_lexer_path = first_value( + matches.values.get_or_default("line-lexer", []), + ) let grammar_ir_path = first_value( matches.values.get_or_default("grammar-ir", []), ) @@ -827,6 +834,22 @@ fn main { None => None } + // Preflight the line-mode lexer (#561) BEFORE writing any output. + let line_lexer_job : (String, String)? = match line_lexer_path { + Some(path) => + match emit_line_lexer(te, term_enum_decl, core_qual) { + Some(content) => Some((path, content)) + None => { + println( + "error: --line-lexer requires #loom.line_pattern annotations on token variants and #loom.line_mode annotations on #loom.lexmode term variants", + ) + @sys.exit(1) + return + } + } + None => None + } + // Preflight the GrammarIr emission BEFORE writing any output, so an invalid // --grammar-ir request (unknown symbol, left recursion, malformed rule string) // fails fast instead of leaving other outputs half-written. #522 @@ -937,6 +960,21 @@ fn main { None => () } + // Write the preflighted line-mode lexer (already built + validated above). #561 + match line_lexer_job { + Some((path, content)) => { + @fs.write_string_to_file(path, content) catch { + _ => { + println("Error writing " + path) + @sys.exit(1) + return + } + } + println(" " + path) + } + None => () + } + // Write the preflighted GrammarIr (already built + validated above). #522 match grammar_ir_job { Some((path, content)) => { diff --git a/loomgen/parse_annotations.mbt b/loomgen/parse_annotations.mbt index 21bb4317..36a2363a 100644 --- a/loomgen/parse_annotations.mbt +++ b/loomgen/parse_annotations.mbt @@ -44,7 +44,14 @@ struct VariantDecl { pattern : String? custom_lex : String? lexmode_name : String? - // #522. `rule` carries the raw EBNF-subset string from `#loom.rule("...")` on a + // #561. `line_pattern` carries a regex matched against a whole line (rest-of-line + // from current position to newline). Only valid in token variants consumed by a + // `#loom.line_mode` lexmode's generated lexer function. + line_pattern : String? + // #561. `line_mode` marks that the lexmode referenced by `#loom.lexmode(...)` on + // this term variant is line-oriented (dispatches on full lines, not individual + // characters). Only valid on term variants alongside `#loom.lexmode("ModeName")`. + line_mode : Bool // #627. `is_sync` carries the bool from `#loom.recovery("sync")` on a // #loom.token variant — marks the variant as an error-recovery synchronization point. is_sync : Bool @@ -136,6 +143,11 @@ fn parse_annotations(source : String) -> Result[AnnotatedEnums, String] { } if is_token { for v in classified { + // #loom.line_pattern tokens are produced by the line-mode lexer; + // they don't need a scanning role annotation. + if v.line_pattern is Some(_) { + continue + } if v.role is None { return Err( "variant " + @@ -157,11 +169,13 @@ fn parse_annotations(source : String) -> Result[AnnotatedEnums, String] { " cannot have #loom.rawtext annotation", ) } - if v.lexmode_name is Some(_) { + // #loom.line_pattern tokens may carry #loom.lexmode to associate + // them with a line_mode lexmode. + if v.lexmode_name is Some(_) && v.line_pattern is None { return Err( "token variant " + v.name + - " cannot have #loom.lexmode annotation", + " cannot have #loom.lexmode annotation without #loom.line_pattern", ) } if v.rule is Some(_) { @@ -270,6 +284,99 @@ fn parse_annotations(source : String) -> Result[AnnotatedEnums, String] { None => () } } + // #561: #loom.line_pattern validation — similar to #loom.pattern but + // for line-level patterns. line_pattern tokens may carry constructor + // arguments (payload extracted by #loom.custom_lex). + for v in classified { + match v.line_pattern { + Some(p) => { + match v.role { + Some(Keyword(_)) => + return Err( + "token variant " + + v.name + + " has #loom.line_pattern together with #loom.keyword; keywords are post-classified and must not carry a line pattern", + ) + Some(Eof) => + return Err( + "#loom.eof variant " + + v.name + + " cannot have #loom.line_pattern", + ) + Some(ErrorToken) => + return Err( + "#loom.error variant " + + v.name + + " cannot have #loom.line_pattern; the error token is the no-match fallback", + ) + Some(Ident) => + return Err( + "#loom.ident variant " + + v.name + + " cannot have #loom.line_pattern; line patterns are for block-level tokens, not identifiers", + ) + _ => () + } + if malformed_re(p) is Some(reason) { + return Err( + "token variant " + + v.name + + " has a malformed #loom.line_pattern \"" + + p + + "\": " + + reason + + ". This would emit a regex that fails to compile in the generated line lexer.", + ) + } + if unsupported_re_construct(p) is Some(reason) { + return Err( + "token variant " + + v.name + + " has an unsupported #loom.line_pattern \"" + + p + + "\": " + + reason + + ". loomgen's #loom.line_pattern supports literals, escapes, character classes [..], plain groups (..), anchors ^ $, and quantifiers * + ? {m,n}.", + ) + } + if is_nullable_pattern(p) { + return Err( + "token variant " + + v.name + + " has a nullable #loom.line_pattern \"" + + p + + "\" that can match the empty string; this would produce a zero-width token. Rewrite it with a required atom (e.g. + instead of *).", + ) + } + } + None => () + } + } + // #561: #loom.line_mode is only valid on term variants alongside + // #loom.lexmode. Validate on token enums. + for v in classified { + if v.line_mode { + return Err( + "token variant " + + v.name + + " cannot have #loom.line_mode; #loom.line_mode is only valid on #loom.term variants", + ) + } + } + // #561: #loom.line_pattern must be paired with #loom.custom_lex when the + // token variant has constructor arguments. line_pattern tokens with args + // need a custom_lex to extract the payload. + for v in classified { + match v.line_pattern { + Some(_) if v.arg_count > 0 && v.custom_lex is None => + return Err( + "token variant " + + v.name + + " has constructor arguments and #loom.line_pattern but no #loom.custom_lex; the generated line lexer needs a custom_lex function to extract the payload from the match text", + ) + _ => () + } + } } // All term variants MUST have a #loom.* annotation if is_term { @@ -321,6 +428,18 @@ fn parse_annotations(source : String) -> Result[AnnotatedEnums, String] { } } } + // #561: #loom.line_mode is only valid alongside #loom.lexmode on term variants. + if is_term { + for v in classified { + if v.line_mode && v.lexmode_name is None { + return Err( + "term variant " + + v.name + + " has #loom.line_mode without #loom.lexmode; #loom.line_mode must accompany a #loom.lexmode(\"ModeName\") annotation", + ) + } + } + } // Reject #loom.eof variants with payload arguments if is_token { for v in classified { @@ -496,9 +615,11 @@ fn extract_variants(comp : @syntax.TypeDesc) -> Array[VariantDecl] { // Detect #loom.pattern and #loom.custom_lex lexer-generation modifiers. let mut pattern : String? = None let mut custom_lex : String? = None + let mut line_pattern : String? = None let mut lexmode_name : String? = None let mut rule : String? = None let mut is_sync = false + let mut line_mode = false for attr in cd.attrs { let ko = find_literal_kind_override(attr.parsed) if ko is Some(_) { @@ -523,6 +644,10 @@ fn extract_variants(comp : @syntax.TypeDesc) -> Array[VariantDecl] { rule = first_string_from_apply(attr.parsed) } else if id.name == "recovery" { is_sync = true + } else if id.name == "line_pattern" { + line_pattern = first_string_from_apply(attr.parsed) + } else if id.name == "line_mode" { + line_mode = true } _ => () } @@ -542,6 +667,8 @@ fn extract_variants(comp : @syntax.TypeDesc) -> Array[VariantDecl] { pattern, custom_lex, lexmode_name, + line_pattern, + line_mode, is_sync, rule, } @@ -1298,6 +1425,33 @@ fn classify_role(variant : VariantDecl) -> Result[SyntaxRole?, String] { } continue } + "line_pattern" => { + // Modifier only — skip role assignment. + if count_apply_args(attr.parsed) != 1 { + return Err( + "#loom.line_pattern on " + + variant.name + + " requires exactly 1 string argument (regex pattern)", + ) + } + if first_string_from_apply(attr.parsed) is None { + return Err( + "#loom.line_pattern on " + + variant.name + + " argument must be a string literal", + ) + } + continue + } + "line_mode" => { + // Modifier only — skip role assignment. + if count_apply_args(attr.parsed) > 0 { + return Err( + "#loom.line_mode on " + variant.name + " does not accept arguments", + ) + } + continue + } _ => match nullary_role_map.get(matched.name) { Some(role) => { diff --git a/loomgen/regenerate_fixtures.mbt b/loomgen/regenerate_fixtures.mbt index 27811e29..31f472fe 100644 --- a/loomgen/regenerate_fixtures.mbt +++ b/loomgen/regenerate_fixtures.mbt @@ -110,4 +110,34 @@ pub fn regenerate_fixtures() -> Unit { } } } + // ------ line_pattern_fixture.g.mbt (line-lexer fixture, #561) ------ + let src = @fs.read_file_to_string("loomgen/fixtures/line_pattern_fixture.mbt") catch { + _ => { + println("ERROR: failed to read line_pattern_fixture.mbt") + return + } + } + match parse_annotations(src) { + Err(msg) => println("ERROR: line_pattern_fixture.mbt — " + msg) + Ok(ann) => { + guard ann.token_enum is Some(token_enum) else { + println("ERROR: line_pattern_fixture.mbt has no #loom.token enum") + return + } + guard ann.term_enum is Some(term_enum) else { + println("ERROR: line_pattern_fixture.mbt has no #loom.term enum") + return + } + match emit_line_lexer(token_enum, Some(term_enum), "@core") { + None => println("ERROR: emit_line_lexer returned None") + Some(content) => { + let path = "loomgen/fixtures/line_pattern_fixture.g.mbt" + @fs.write_string_to_file(path, content) catch { + _ => println("ERROR: failed to write " + path) + } + println(" " + path) + } + } + } + } }