perf(codegen): encode generated data tables as compact blobs

README.md

@@ -170,7 +170,20 @@ Using the same package release for `antlr4-rust-gen` and
 `antlr-rust-runtime` remains the recommended workflow, but matching the
 generated-code API is the compile-time requirement.
 
-The bundled generator currently emits revision 14. Generated parsers now embed
+The bundled generator currently emits revision 15. Generated recognizers now
+embed their static data tables — the ahead-of-time compiled lexer DFA, the
+packed parser ATN, and the serialized lexer ATN inside `GrammarMetadata` — as
+versioned encoded blobs defined by the runtime's `encoded` module: LEB128
+varints (zigzag-mapped for signed values) armored as canonical unpadded base64
+string literals. Each blob carries a magic, format version, element kind, and
+checked element count, and decoding rejects corrupt, truncated, overflowing,
+and unsupported data with targeted diagnostics. The decoded integer streams
+are byte-identical to the previous decimal arrays, so recognizer behavior and
+the inner serialized ATN/DFA format versions are unchanged, while generated
+source shrinks materially and rustc no longer parses hundreds of thousands of
+integer expressions per large grammar.
+
+Revision 14 generated parsers embed
 packed parser ATN format 3, whose rule-transition tags carry validated
 grammar-agnostic tail-call markers. Parser and lexer prediction reuse the
 existing caller context when every continuation from a rule call's follow state
@@ -179,9 +192,11 @@ also omits the same redundant frames. SLL accuracy is preserved by default, and
 the reduced-accuracy parser mode is available only through an explicit
 simulator constructor.
 
-Revision 12 and 13 generated recognizers remain accepted because the runtime
-still provides their source API and reads packed parser ATN formats 1 and 2.
-Regenerate them with revision 14 to emit tail-call metadata.
+Revision 12 to 14 generated recognizers remain accepted because the runtime
+still provides their source API — integer-array `GrammarMetadata` ATN data,
+`CompiledLexerDfa::from_serialized`, and `ParserAtn::from_static` — and reads
+packed parser ATN formats 1 through 3. Regenerate them with revision 15 to
+emit the compact encoded representation.
 
 Revision 13 moved the iterative generated listener tree-walk engine into
 `antlr4_runtime::generated::walk_generated`. Generated parsers retain their

crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__generated_module_file_header.snap

@@ -4,7 +4,7 @@ expression: "generated_module_header.replace(env!(\"CARGO_PKG_VERSION\"),\n\"<ge
 ---
 // @generated by antlr-rust-codegen v<generator-version> - do not edit
 // project: https://github.com/ophi-dev/antlr-rust-runtime
-antlr4_runtime::__antlr4_rust_require_codegen_api!(14, "<generator-version>");
+antlr4_runtime::__antlr4_rust_require_codegen_api!(15, "<generator-version>");
 #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]
 #[rustfmt::skip]
 mod __antlr4_rust_generated {

crates/antlr-rust-codegen/src/generator/tests.rs

@@ -325,9 +325,9 @@ fn generated_parser_rustdoc_is_attached_to_parser_type() {
 fn generated_parser_embeds_only_versioned_packed_atn_data() {
     let rendered = render_parser("TParser", &minimal_parser_data()).expect("parser should render");
 
-    assert!(rendered.contains("static PARSER_ATN_DATA: &[u32]"));
+    assert!(rendered.contains("static PARSER_ATN_DATA: &str"));
     assert!(rendered.contains("static ATN_CELL: OnceLock<ParserAtn>"));
-    assert!(rendered.contains("ParserAtn::from_static(PARSER_ATN_DATA)"));
+    assert!(rendered.contains("ParserAtn::from_encoded(PARSER_ATN_DATA)"));
     assert!(rendered.contains("generated parser ATN is incompatible with this runtime"));
     assert!(rendered.contains("pub fn parser_atn() -> &'static ParserAtn"));
     assert!(rendered.contains("fn parser_atn() -> &'static ParserAtn"));

crates/antlr-rust-codegen/src/grammar/atn/interp_test.rs

@@ -350,7 +350,6 @@ fn usize_to_i32(value: usize) -> i32 {
 #[cfg(test)]
 mod tests {
     use std::collections::{BTreeSet, VecDeque};
-    use std::fmt::Write as _;
     use std::path::{Path, PathBuf};
     use std::rc::Rc;
 

crates/antlr-rust-codegen/src/grammar/atn/lexer.rs

@@ -1599,7 +1599,9 @@ fn parse_property_escape(tail: &str, inverted: bool) -> Result<(CharSetAtom, usi
         .filter(|ranges| !ranges.is_empty())
         .ok_or_else(|| format!("unknown or empty Unicode property {name}"))?;
     let ranges = raw_ranges
-        .chunks_exact(2)
+        .as_chunks::<2>()
+        .0
+        .iter()
         .map(|range| (range[0], range[1]))
         .collect::<Vec<_>>();
     let ranges = if inverted {

crates/antlr-rust-codegen/src/grammar/escape_sequence.rs

@@ -123,7 +123,7 @@ fn simple_escape(escaped: char) -> Option<i32> {
 
 fn interval_set(ranges: &[i32]) -> IntervalSet {
     let mut result = IntervalSet::new();
-    for range in ranges.chunks_exact(2) {
+    for range in ranges.as_chunks::<2>().0 {
         result.add_range(range[0], range[1]);
     }
     result
@@ -132,7 +132,7 @@ fn interval_set(ranges: &[i32]) -> IntervalSet {
 fn complement(ranges: &[i32]) -> IntervalSet {
     let mut result = IntervalSet::new();
     let mut next = 0;
-    for range in ranges.chunks_exact(2) {
+    for range in ranges.as_chunks::<2>().0 {
         if next < range[0] {
             result.add_range(next, range[0] - 1);
         }

crates/antlr-rust-codegen/src/grammar/unicode.rs

@@ -865,8 +865,9 @@ fn decode_compatibility_decomposition_ranges() -> RangesByU8 {
     );
 
     let mut ranges = RangesByU8::new();
-    for record in
-        data[DECOMPOSITION_TYPE_HEADER_LENGTH..].chunks_exact(DECOMPOSITION_TYPE_RECORD_LENGTH)
+    for record in data[DECOMPOSITION_TYPE_HEADER_LENGTH..]
+        .as_chunks::<DECOMPOSITION_TYPE_RECORD_LENGTH>()
+        .0
     {
         let decomposition_type = record[0];
         assert!(
@@ -1002,7 +1003,7 @@ fn ranges_from_iter(ranges: impl Iterator<Item = std::ops::RangeInclusive<u32>>)
 fn subtract_ranges(include: &[i32], exclude: &[i32]) -> Vec<i32> {
     let mut result = Vec::new();
     let mut exclude_index = 0;
-    for included in include.chunks_exact(2) {
+    for included in include.as_chunks::<2>().0 {
         let mut next = included[0];
         let stop = included[1];
         while exclude_index < exclude.len() && exclude[exclude_index + 1] < next {
@@ -1028,8 +1029,10 @@ fn subtract_ranges(include: &[i32], exclude: &[i32]) -> Vec<i32> {
 
 fn union_ranges(left: &[i32], right: &[i32]) -> Vec<i32> {
     let mut pairs = left
-        .chunks_exact(2)
-        .chain(right.chunks_exact(2))
+        .as_chunks::<2>()
+        .0
+        .iter()
+        .chain(right.as_chunks::<2>().0)
         .map(|range| (range[0], range[1]))
         .collect::<Vec<_>>();
     pairs.sort_unstable();
@@ -1075,7 +1078,9 @@ mod tests {
     fn contains(property: &str, code_point: i32) -> bool {
         property_ranges(property)
             .expect("known Unicode property")
-            .chunks_exact(2)
+            .as_chunks::<2>()
+            .0
+            .iter()
             .any(|range| (range[0]..=range[1]).contains(&code_point))
     }
 

crates/antlr-rust-codegen/src/grammar/unicode_icu_tests.rs

@@ -14,7 +14,9 @@ const JAVA_PROPERTY_ORACLE: &str = include_str!(concat!(
 fn contains(property: &str, code_point: i32) -> bool {
     property_ranges(property)
         .expect("known Unicode property")
-        .chunks_exact(2)
+        .as_chunks::<2>()
+        .0
+        .iter()
         .any(|range| (range[0]..=range[1]).contains(&code_point))
 }
 
@@ -142,7 +144,7 @@ fn every_unicode_property_and_alias_matches_java() {
 
 fn interval_digest(ranges: &[i32]) -> String {
     let mut digest = Hash::new();
-    for range in ranges.chunks_exact(2) {
+    for range in ranges.as_chunks::<2>().0 {
         digest.update(range[0].to_be_bytes());
         digest.update(range[1].to_be_bytes());
     }

crates/antlr-rust-codegen/src/lexer/render.rs

@@ -284,16 +284,16 @@ fn atn() -> &'static LexerAtn {{
     }})
 }}
 
-static LEXER_DFA_DATA: &[u32] = &[{lexer_dfa_data}];
+static LEXER_DFA_DATA: &str = {lexer_dfa_data};
 
 static LEXER_DFA_CELL: OnceLock<CompiledLexerDfa> = OnceLock::new();
 
-/// Ahead-of-time lexer DFA tables compiled by antlr4-rust-gen, embedded so
-/// runtime startup only deserializes them. Rebuilt from the ATN instead when
-/// the embedded stream comes from a different runtime version.
+/// Ahead-of-time lexer DFA tables compiled by antlr4-rust-gen, embedded as an
+/// encoded blob so runtime startup only decodes them. Rebuilt from the ATN
+/// instead when the embedded data comes from a different runtime version.
 fn lexer_dfa() -> &'static CompiledLexerDfa {{
     LEXER_DFA_CELL.get_or_init(|| {{
-        CompiledLexerDfa::from_serialized(LEXER_DFA_DATA)
+        CompiledLexerDfa::from_encoded(LEXER_DFA_DATA)
             .unwrap_or_else(|| CompiledLexerDfa::compile(atn()))
     }})
 }}
@@ -382,23 +382,20 @@ pub(crate) fn lexer_actions_require_semantic_hooks(
     })
 }
 
-/// Compiles the lexer DFA at generation time and flattens it for embedding.
+/// Compiles the lexer DFA at generation time and embeds it as an encoded
+/// blob string literal (see `antlr4_runtime::encoded`).
 ///
-/// An empty stream makes the generated lexer fall back to compiling the DFA
-/// from its ATN at first use, so generation never fails on this step.
+/// The generated lexer decodes the blob at first use and falls back to
+/// compiling the DFA from its ATN when the embedded data comes from a
+/// different runtime version, so generation never fails on this step.
 fn compiled_lexer_dfa_words(data: &LexerCodegenData<'_>) -> String {
-    if !data.lexer_dfa_words.is_empty() {
-        return data
-            .lexer_dfa_words
-            .iter()
-            .map(u32::to_string)
-            .collect::<Vec<_>>()
-            .join(",");
-    }
-    let atn = data.lexer_atn();
-    let words = CompiledLexerDfa::compile(atn).serialize();
-    let rendered: Vec<String> = words.iter().map(u32::to_string).collect();
-    rendered.join(",")
+    let encoded = if data.lexer_dfa_words.is_empty() {
+        let atn = data.lexer_atn();
+        antlr4_runtime::encoded::encode_u32_values(&CompiledLexerDfa::compile(atn).serialize())
+    } else {
+        antlr4_runtime::encoded::encode_u32_values(&data.lexer_dfa_words)
+    };
+    rust_encoded_blob_literal(&encoded)
 }
 /// Translates the lexer recognizer surface used by rendered test bodies onto
 /// the `BaseLexer` hooks: `self.text()` / column accessors become position

crates/antlr-rust-codegen/src/lexer/render_model.rs

@@ -46,17 +46,22 @@ pub(crate) fn render_lexer_lex_convenience() -> String {
 }
 
 /// Renders the lexer-owned grammar metadata table.
+///
+/// The serialized ATN travels as an encoded blob string literal (see
+/// `antlr4_runtime::encoded`) instead of a decimal integer array.
 pub(crate) fn render_lexer_metadata(grammar_name: &str, data: &LexerCodegenData<'_>) -> String {
     format!(
-        "pub static METADATA: GrammarMetadata = GrammarMetadata::new(\n    \"{}\",\n    &{},\n    &{},\n    &{},\n    &{},\n    &{},\n    &{},\n    &{},\n);\n\npub fn metadata() -> &'static GrammarMetadata {{\n    &METADATA\n}}\n\npub fn rule_names() -> &'static [&'static str] {{\n    METADATA.rule_names()\n}}\n",
+        "pub static METADATA: GrammarMetadata = GrammarMetadata::new_with_encoded_atn(\n    \"{}\",\n    &{},\n    &{},\n    &{},\n    &{},\n    &{},\n    &{},\n    {},\n);\n\npub fn metadata() -> &'static GrammarMetadata {{\n    &METADATA\n}}\n\npub fn rule_names() -> &'static [&'static str] {{\n    METADATA.rule_names()\n}}\n",
         rust_string(grammar_name),
         render_lexer_str_slice(&data.rule_names),
         render_lexer_option_str_slice(&data.literal_names),
         render_lexer_option_str_slice(&data.symbolic_names),
         render_lexer_empty_option_str_slice(max_len(&data.literal_names, &data.symbolic_names)),
         render_lexer_str_slice(&data.channel_names),
         render_lexer_str_slice(&data.mode_names),
-        render_lexer_i32_slice(&data.lexer_atn_words)
+        rust_encoded_blob_literal(&antlr4_runtime::encoded::encode_i32_values(
+            &data.lexer_atn_words
+        ))
     )
 }
 
@@ -132,12 +137,3 @@ fn render_lexer_str_slice(values: &[String]) -> String {
         .join(", ");
     format!("[{items}]")
 }
-
-fn render_lexer_i32_slice(values: &[i32]) -> String {
-    let items = values
-        .iter()
-        .map(i32::to_string)
-        .collect::<Vec<_>>()
-        .join(", ");
-    format!("[{items}]")
-}

crates/antlr-rust-codegen/src/parser/decision.rs

@@ -561,7 +561,7 @@ fn rectangles_overlap(left: &LookaheadRectangle, right: &LookaheadRectangle) ->
 }
 
 /// Whether two sorted disjoint interval sets share any symbol.
-fn interval_sets_intersect(left: &[(i32, i32)], right: &[(i32, i32)]) -> bool {
+const fn interval_sets_intersect(left: &[(i32, i32)], right: &[(i32, i32)]) -> bool {
     let (mut left_index, mut right_index) = (0, 0);
     while left_index < left.len() && right_index < right.len() {
         let (left_start, left_stop) = left[left_index];

crates/antlr-rust-codegen/src/parser/render/mod.rs

@@ -25,7 +25,10 @@ pub(crate) fn render_parser_with_decision_report(
     let type_name = rust_type_name(grammar_name);
     let metadata = render_parser_metadata(grammar_name, data);
     let parser_atn = data.parser_atn();
-    let parser_atn_data = render_u32_slice(parser_atn.packed_words());
+    let parser_atn_data =
+        rust_encoded_blob_literal(&antlr4_runtime::encoded::encode_u32_values(
+            parser_atn.packed_words(),
+        ));
     // Every constant in the generated module shares one Rust value
     // namespace; allocation order matches emission order (tokens first).
     let mut const_names = BTreeSet::new();
@@ -321,13 +324,13 @@ use std::sync::OnceLock;
 {embedded_attrs_structs}
 {embedded_module_items}
 
-static PARSER_ATN_DATA: &[u32] = &{parser_atn_data};
+static PARSER_ATN_DATA: &str = {parser_atn_data};
 static ATN_CELL: OnceLock<ParserAtn> = OnceLock::new();
 
-/// Validates and caches the packed grammar ATN for all parser instances.
+/// Decodes, validates, and caches the packed grammar ATN for all parser instances.
 fn atn() -> &'static ParserAtn {{
     ATN_CELL.get_or_init(|| {{
-        ParserAtn::from_static(PARSER_ATN_DATA)
+        ParserAtn::from_encoded(PARSER_ATN_DATA)
             .unwrap_or_else(|error| panic!("generated parser ATN is incompatible with this runtime: {{error}}"))
     }})
 }}

crates/antlr-rust-codegen/src/parser/surface/support_abi.rs

@@ -925,16 +925,6 @@ fn render_i32_slice(values: &[i32]) -> String {
     format!("[{items}]")
 }
 
-/// Renders a versioned packed parser ATN word stream.
-pub(crate) fn render_u32_slice(values: &[u32]) -> String {
-    let items = values
-        .iter()
-        .map(u32::to_string)
-        .collect::<Vec<_>>()
-        .join(", ");
-    format!("[{items}]")
-}
-
 /// Renders an inline `[(i32, i32); N]` expression for generated token-set
 /// matches.
 pub(crate) fn render_i32_ranges(values: &[(i32, i32)]) -> String {

crates/antlr-rust-codegen/src/pipeline.rs

@@ -43,8 +43,8 @@ pub(crate) mod prelude {
     #[cfg(test)]
     pub(crate) use crate::rust_output::is_rust_keyword;
     pub(crate) use crate::rust_output::{
-        module_name, replace_all, rust_function_name, rust_identifier, rust_string, rust_type_name,
-        sanitize_identifier, split_identifier_words,
+        module_name, replace_all, rust_encoded_blob_literal, rust_function_name, rust_identifier,
+        rust_string, rust_type_name, sanitize_identifier, split_identifier_words,
     };
 }
 

crates/antlr-rust-codegen/src/rust_output.rs

@@ -62,6 +62,31 @@ pub(crate) fn rust_string(value: &str) -> String {
     value.escape_default().to_string()
 }
 
+/// Renders an encoded-blob payload (see `antlr4_runtime::encoded`) as one
+/// segmented Rust string literal.
+///
+/// Base64 text is pure ASCII with no escapes, so long payloads split into
+/// fixed-width lines joined by `\`-newline continuations: the literal stays a
+/// single token for rustc while no generated source line grows to the
+/// megabyte widths the previous decimal integer arrays produced. rustc strips
+/// each continuation together with the following line's leading whitespace.
+pub(crate) fn rust_encoded_blob_literal(encoded: &str) -> String {
+    const LINE_WIDTH: usize = 120;
+    debug_assert!(encoded.is_ascii(), "encoded blobs are base64 text");
+    let mut out = String::with_capacity(encoded.len() + (encoded.len() / LINE_WIDTH + 1) * 6 + 2);
+    out.push('"');
+    let mut rest = encoded;
+    while rest.len() > LINE_WIDTH {
+        let (line, tail) = rest.split_at(LINE_WIDTH);
+        out.push_str(line);
+        out.push_str("\\\n    ");
+        rest = tail;
+    }
+    out.push_str(rest);
+    out.push('"');
+    out
+}
+
 /// Replaces every non-overlapping occurrence without relying on the
 /// allocation-hiding `str::replace` helper prohibited by the workspace lints.
 pub(crate) fn replace_all(text: &str, needle: &str, replacement: &str) -> String {

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/cli.rs

@@ -99,7 +99,7 @@ fn generated_modules_enforce_codegen_api_compatibility() {
         "__antlr4_rust_require_codegen_api!({},",
         antlr4_runtime::__ANTLR4_RUST_CODEGEN_API
     );
-    let previous = "__antlr4_rust_require_codegen_api!(13,";
+    let previous = "__antlr4_rust_require_codegen_api!(14,";
     let oldest_supported = "__antlr4_rust_require_codegen_api!(12,";
     let unsupported = "__antlr4_rust_require_codegen_api!(11,";
     let mut previous_parser = parser;
@@ -141,7 +141,7 @@ fn generated_modules_enforce_codegen_api_compatibility() {
         .collect::<Vec<_>>()
         .join("\n");
     assert!(
-        diagnostic.contains("supports revisions 12, 13, and 14"),
+        diagnostic.contains("supports revisions 12, 13, 14, and 15"),
         "diagnostic should name the supported revisions: {diagnostic}"
     );
     insta::assert_snapshot!(

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/snapshots/antlr4_rust_gen_cli__cli__generated_codegen_api_checks.snap

@@ -2,5 +2,5 @@
 source: crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/cli.rs
 expression: normalize_current_package_version(&checks)
 ---
-lexer: antlr4_runtime::__antlr4_rust_require_codegen_api!(14, "<generator-version>");
-parser: antlr4_runtime::__antlr4_rust_require_codegen_api!(14, "<generator-version>");
+lexer: antlr4_runtime::__antlr4_rust_require_codegen_api!(15, "<generator-version>");
+parser: antlr4_runtime::__antlr4_rust_require_codegen_api!(15, "<generator-version>");

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/snapshots/antlr4_rust_gen_cli__cli__generated_codegen_api_mismatch_diagnostic.snap

@@ -2,5 +2,5 @@
 source: crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/cli.rs
 expression: normalize_current_package_version(&diagnostic)
 ---
-error: antlr4-rust generated-code API mismatch: antlr4-rust-gen v<generator-version> emitted generated-code API revision 11, but the selected antlr-rust-runtime supports revisions 12, 13, and 14; regenerate this recognizer with a compatible antlr4-rust-gen or select a compatible antlr-rust-runtime dependency
+error: antlr4-rust generated-code API mismatch: antlr4-rust-gen v<generator-version> emitted generated-code API revision 11, but the selected antlr-rust-runtime supports revisions 12, 13, 14, and 15; regenerate this recognizer with a compatible antlr4-rust-gen or select a compatible antlr-rust-runtime dependency
  --> src/codegen_api_parser.rs:3:1