test: add generated line-mode lexer Markdown fixture

fixtures/line_lexer_regression/lexmode_def.mbt

@@ -0,0 +1,5 @@
+///|
+pub(all) enum LexMode {
+  Inline
+  LineStart
+} derive(Eq, Debug)

fixtures/line_lexer_regression/line_lexer.g.mbt

@@ -0,0 +1,47 @@
+// Generated by loomgen — do not edit.
+
+///|
+/// Line-mode lexer functions for #loom.line_mode modes.
+/// Each function matches #loom.line_pattern tokens against whole lines.
+fn lex_line_start(
+  source : String,
+  pos : Int,
+) -> (@core.LexStep[Token], LexMode) {
+  if pos >= source.length() {
+    return (Done, LineStart)
+  }
+  let mut line_end = pos
+  while line_end < source.length() {
+    match source[line_end:line_end + 1].to_owned() {
+      "\r" | "\n" => break
+      _ => line_end = line_end + 1
+    }
+  }
+  let rest = source[pos:line_end].view()
+
+  // HeadingMarker
+  if rest =~ (re"^(?:#{1,6} )", after=after) {
+    let len = rest.length() - after.length()
+    return lex_heading_marker(source, pos, len)
+  }
+  // ListMarker
+  if rest =~ (re"^(?:- )", after=after) {
+    let len = rest.length() - after.length()
+    return (
+      Produced(@core.TokenInfo::new(ListMarker, len), next_offset=pos + len),
+      LineStart,
+    )
+  }
+  // BlockQuoteMarker
+  if rest =~ (re"^(?:>)", after=after) {
+    let len = rest.length() - after.length()
+    return (
+      Produced(
+        @core.TokenInfo::new(BlockQuoteMarker, len),
+        next_offset=pos + len,
+      ),
+      LineStart,
+    )
+  }
+  return lex_inline(source, pos)
+}

fixtures/line_lexer_regression/line_lexer_support.mbt

@@ -0,0 +1,72 @@
+///|
+/// Consume the rest of the current line, including its line ending. This is
+/// the fallback used after a generated line marker has consumed its prefix.
+fn lex_inline(source : String, pos : Int) -> (@core.LexStep[Token], LexMode) {
+  if pos >= source.length() {
+    return (Done, LineStart)
+  }
+  let mut line_end = pos
+  while line_end < source.length() {
+    match source[line_end:line_end + 1].to_owned() {
+      "\r" | "\n" => break
+      _ => line_end = line_end + 1
+    }
+  }
+  let newline_width = if line_end >= source.length() {
+    0
+  } else if line_end + 1 < source.length() &&
+    source[line_end:line_end + 2].to_owned() == "\r\n" {
+    2
+  } else {
+    1
+  }
+  let consumed_end = line_end + newline_width
+  if line_end == pos {
+    return (
+      Produced(
+        @core.TokenInfo::new(BlankLine, newline_width),
+        next_offset=consumed_end,
+      ),
+      LineStart,
+    )
+  }
+  (
+    Produced(
+      @core.TokenInfo::new(
+        Text(source[pos:consumed_end].to_owned()),
+        consumed_end - pos,
+      ),
+      next_offset=consumed_end,
+    ),
+    LineStart,
+  )
+}
+
+///|
+/// Extract the number of leading `#` characters from the generated match.
+fn lex_heading_marker(
+  _source : String,
+  pos : Int,
+  len : Int,
+) -> (@core.LexStep[Token], LexMode) {
+  (
+    Produced(
+      @core.TokenInfo::new(HeadingMarker(len - 1), len),
+      next_offset=pos + len,
+    ),
+    LineStart,
+  )
+}
+
+///|
+/// Fixture-local dispatch mirroring the generated lexer skeleton contract.
+pub fn lex(
+  source : String,
+  pos : Int,
+  mode : LexMode,
+) -> (@core.LexStep[Token], LexMode) {
+  match mode {
+    LineStart => lex_line_start(source, pos)
+    Inline => lex_inline(source, pos)
+  }
+}

fixtures/line_lexer_regression/line_lexer_wbtest.mbt

@@ -0,0 +1,67 @@
+///|
+
+///|
+fn fixture_lexer() -> @core.ModeLexer[Token, LexMode] {
+  { lex_step: lex, initial_mode: LineStart }
+}
+
+///|
+test "generated line-mode lexer tokenizes Markdown lines" {
+  let source = "# Heading\n\n- item\n> quote\n"
+  let lexer = fixture_lexer()
+  let (tokens, modes) = @core.tokenize_with_modes(lexer, source, EOF)
+  guard tokens ==
+    [
+      @core.TokenInfo::new(HeadingMarker(1), 2),
+      @core.TokenInfo::new(Text("Heading\n"), 8),
+      @core.TokenInfo::new(BlankLine, 1),
+      @core.TokenInfo::new(ListMarker, 2),
+      @core.TokenInfo::new(Text("item\n"), 5),
+      @core.TokenInfo::new(BlockQuoteMarker, 1),
+      @core.TokenInfo::new(Text(" quote\n"), 7),
+      @core.TokenInfo::new(EOF, 0),
+    ] else {
+    fail("generated lexer token sequence mismatch")
+  }
+  guard @core.build_starts(tokens) == [0, 2, 10, 11, 13, 18, 19, 26] else {
+    fail("generated lexer token positions mismatch")
+  }
+  guard modes ==
+    [
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+      LineStart,
+    ] else {
+    fail("generated lexer mode sequence mismatch")
+  }
+  match lex_line_start(source, source.length()) {
+    (Done, LineStart) => ()
+    _ => fail("generated line lexer must return Done at EOF")
+  }
+}
+
+///|
+test "generated line-mode lexer preserves CRLF line lengths" {
+  let source = "# H\r\n\r\n- x\r\n"
+  let lexer = fixture_lexer()
+  let (tokens, _modes) = @core.tokenize_with_modes(lexer, source, EOF)
+  guard tokens ==
+    [
+      @core.TokenInfo::new(HeadingMarker(1), 2),
+      @core.TokenInfo::new(Text("H\r\n"), 3),
+      @core.TokenInfo::new(BlankLine, 2),
+      @core.TokenInfo::new(ListMarker, 2),
+      @core.TokenInfo::new(Text("x\r\n"), 3),
+      @core.TokenInfo::new(EOF, 0),
+    ] else {
+    fail("generated lexer CRLF sequence mismatch")
+  }
+  guard @core.build_starts(tokens) == [0, 2, 5, 7, 9, 12] else {
+    fail("generated lexer CRLF positions mismatch")
+  }
+}

fixtures/line_lexer_regression/moon.mod

@@ -0,0 +1,15 @@
+name = "dowdiness/loom_tests/line_lexer_regression"
+
+version = "0.1.0"
+
+import {
+  "dowdiness/loom@0.1.0",
+}
+
+readme = "README.md"
+
+repository = "https://github.com/dowdiness/loom"
+
+license = "Apache-2.0"
+
+description = "End-to-end fixture for generated line-mode lexer output"

fixtures/line_lexer_regression/moon.pkg

@@ -0,0 +1,3 @@
+import {
+  "dowdiness/loom/core",
+}

fixtures/line_lexer_regression/pkg.generated.mbti

@@ -0,0 +1,33 @@
+// Generated using `moon info`, DON'T EDIT IT
+package "dowdiness/loom_tests/line_lexer_regression"
+
+import {
+  "dowdiness/loom/core",
+  "moonbitlang/core/debug",
+}
+
+// Values
+pub fn lex(String, Int, LexMode) -> (@core.LexStep[Token], LexMode)
+
+// Errors
+
+// Types and methods
+pub(all) enum LexMode {
+  Inline
+  LineStart
+} derive(Eq, @debug.Debug)
+
+pub(all) enum Token {
+  HeadingMarker(Int)
+  ListMarker
+  BlockQuoteMarker
+  Text(String)
+  BlankLine
+  Error(String)
+  EOF
+} derive(Eq, @debug.Debug)
+
+// Type aliases
+
+// Traits
+

fixtures/line_lexer_regression/regenerate.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Regenerate line_lexer.g.mbt from the annotated source.
+# Run from the loom repo root.
+set -euo pipefail
+
+FIXTURE=fixtures/line_lexer_regression
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+mkdir -p "$TMP/token" "$TMP/syntax"
+
+moon run loomgen --target native -- \
+  "$FIXTURE/src/line_lexer_src.mbt" \
+  "$TMP/token" "$TMP/syntax" \
+  --token-qual "" \
+  --syntax-qual "" \
+  --core-qual "@core" \
+  --line-lexer "$FIXTURE/line_lexer.g.mbt"
+
+moon fmt "$FIXTURE/line_lexer.g.mbt"

fixtures/line_lexer_regression/src/line_lexer_src.mbt

@@ -0,0 +1,33 @@
+///|
+#loom.token
+pub(all) enum Token {
+  #loom.line_pattern("#{1,6} ")
+  #loom.custom_lex("lex_heading_marker")
+  HeadingMarker(Int)
+  #loom.line_pattern("- ")
+  ListMarker
+  #loom.line_pattern(">")
+  BlockQuoteMarker
+  #loom.ident
+  Text(String)
+  #loom.error
+  Error(String)
+  #loom.eof
+  EOF
+} derive(Eq, Debug)
+
+#loom.term
+pub(all) enum Term {
+  #loom.leaf
+  #loom.lexmode("LineStart")
+  #loom.line_mode
+  #loom.fallback_lex("lex_inline")
+  Block
+  #loom.leaf
+  #loom.lexmode("Inline")
+  Inline
+  #loom.root
+  Root
+  #loom.errornode
+  ErrorNode
+} derive(Eq, Debug)

fixtures/line_lexer_regression/token_def.mbt

@@ -0,0 +1,10 @@
+///|
+pub(all) enum Token {
+  HeadingMarker(Int)
+  ListMarker
+  BlockQuoteMarker
+  Text(String)
+  BlankLine
+  Error(String)
+  EOF
+} derive(Eq, Debug)

moon.work

@@ -18,4 +18,5 @@ members = [
   "./egraph",
   "./egglog",
   "./fixtures/grammar_ir_regression",
+  "./fixtures/line_lexer_regression",
 ]