diff --git a/Cargo.lock b/Cargo.lock index 40914acc2..e26880f4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,16 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -414,6 +424,7 @@ name = "cribo" version = "0.9.2" dependencies = [ "anyhow", + "base64-simd", "clap", "cow-utils", "criterion", @@ -423,6 +434,7 @@ dependencies = [ "insta", "log", "once_cell", + "oxc_sourcemap", "pep508_rs", "petgraph", "pretty_assertions", @@ -1173,6 +1185,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-escape-simd" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c22a2041e3874a055a4eb03ea2395aaccdefa84ce75b31d542d72a741c3c6ad3" + [[package]] name = "libc" version = "0.2.186" @@ -1326,6 +1344,25 @@ dependencies = [ "indexmap", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "oxc_sourcemap" +version = "8.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b415102a94b483bbd76d13e850fe87961197e5795c79a8523ebec5d82025f94f" +dependencies = [ + "base64-simd", + "json-escape-simd", + "rustc-hash", + "serde", + "serde_json", +] + [[package]] name = "page_size" version = "0.6.0" @@ -2789,6 +2826,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vte" version = "0.14.1" diff --git a/Cargo.toml b/Cargo.toml index 19496faf7..953b53a0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,10 +25,13 @@ ruff_python_semantic = { git = "https://github.com/astral-sh/ruff/", tag = "0.16 ruff_python_stdlib = { git = "https://github.com/astral-sh/ruff/", tag = "0.16.3" } ruff_text_size = { git = "https://github.com/astral-sh/ruff/", tag = "0.16.3" } +# Source map generation (Source Map v3, pinned exact version) +oxc_sourcemap = "=8.1.2" + # Serialization and configuration -serde = { version = "1.0", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -toml = "0.9" +toml = "0.9" # Package name normalization (PEP 503) pep508_rs = "0.9" @@ -43,8 +46,9 @@ petgraph = "0.8" rustc-hash = "2.1" # Utilities -cow-utils = "0.1.3" -etcetera = "0.11" +base64-simd = "0.8" +cow-utils = "0.1.3" +etcetera = "0.11" # Testing insta = { version = "1.43.1", features = ["filters", "glob", "yaml"] } @@ -62,17 +66,17 @@ unsafe_code = "warn" # Add these for better AI-generated code catching: exported_private_dependencies = { level = "allow", priority = 10 } missing_debug_implementations = "warn" -rust_2018_compatibility = { level = "warn", priority = -2 } -rust_2018_idioms = { level = "warn", priority = -2 } -rust_2021_compatibility = { level = "warn", priority = -2 } -rust_2024_compatibility = { level = "warn", priority = -2 } # Warns about 2015 idioms in 2018 edition +missing_docs = "allow" # Or "allow" if too noisy +rust_2018_compatibility = { level = "warn", priority = -2 } +rust_2018_idioms = { level = "warn", priority = -2 } +rust_2021_compatibility = { level = "warn", priority = -2 } +rust_2024_compatibility = { level = "warn", priority = -2 } # Warns about 2015 idioms in 2018 edition trivial_casts = "warn" trivial_numeric_casts = "warn" unused_extern_crates = "warn" unused_import_braces = "warn" unused_qualifications = "warn" variant_size_differences = "warn" -missing_docs = "allow" # Or "allow" if too noisy [workspace.lints.clippy] # Categories @@ -103,7 +107,7 @@ similar_names = "allow" single_match_else = "allow" too_many_lines = "deny" unnecessary_debug_formatting = "allow" -unused_self = "allow" +unused_self = "allow" # Additional allows for practical development cast_possible_truncation = "allow" @@ -149,29 +153,29 @@ path_buf_push_overwrite = "warn" ptr_as_ptr = "warn" # redundant_pub_crate disabled due to conflict with unreachable-pub lint # When a module is pub but items inside are pub(crate), both lints fire incorrectly -redundant_pub_crate = "allow" -ref_binding_to_reference = "warn" -ref_option_ref = "warn" -semicolon_if_nothing_returned = "warn" -str_to_string = "warn" -string_add = "warn" -string_add_assign = "warn" -string_lit_as_bytes = "warn" +redundant_pub_crate = "allow" +ref_binding_to_reference = "warn" +ref_option_ref = "warn" +semicolon_if_nothing_returned = "warn" +str_to_string = "warn" +string_add = "warn" +string_add_assign = "warn" +string_lit_as_bytes = "warn" # string_to_string has been removed - implicit_clone covers those cases -type_repetition_in_bounds = "warn" -unnecessary_self_imports = "warn" -unnecessary_wraps = "warn" -unneeded_field_pattern = "warn" -unnested_or_patterns = "warn" -unseparated_literal_suffix = "warn" -unused_async = "warn" -use_self = "warn" -useless_let_if_seq = "warn" -verbose_bit_mask = "warn" -wildcard_imports = "warn" +type_repetition_in_bounds = "warn" +unnecessary_self_imports = "warn" +unnecessary_wraps = "warn" +unneeded_field_pattern = "warn" +unnested_or_patterns = "warn" +unseparated_literal_suffix = "warn" +unused_async = "warn" +use_self = "warn" +useless_let_if_seq = "warn" +verbose_bit_mask = "warn" +wildcard_imports = "warn" # Add these specific lints to catch .ok() misuse -match_result_ok = "warn" # Catches if let Some(x) = result.ok() (renamed from if_let_some_result) +match_result_ok = "warn" # Catches if let Some(x) = result.ok() (renamed from if_let_some_result) option_map_unit_fn = "warn" # Catches .map(|_| ()) question_mark = "warn" # Encourages using ? operator result_map_unit_fn = "warn" # Catches .map(|_| ()) diff --git a/README.md b/README.md index 73970d68e..9fcf13450 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,9 @@ cribo --entry src/main.py --output bundle.py -vvv # trace level # Custom config file cribo --entry src/main.py --output bundle.py --config my-cribo.toml + +# Generate a Source Map v3 and remap runtime tracebacks to original sources +cribo --entry src/main.py --output bundle.py --sourcemap ``` ### CLI Options @@ -155,6 +158,8 @@ cribo --entry src/main.py --output bundle.py --config my-cribo.toml - `--python `: Python interpreter whose installed distribution metadata is used for requirements - `--no-tree-shake`: Disable tree-shaking optimization (tree-shaking is enabled by default) - `--target-version `: Target Python version (e.g., py38, py39, py310, py311, py312, py313) +- `--sourcemap[=]`: Generate a Source Map v3 for the bundle and inject a traceback-remapping runtime. Modes: `linked` (default; writes `.map` plus a `# sourceMappingURL=` comment), `inline` (embeds the map as a base64 comment; the default with `--stdout`), `external` (writes `.map` with no comment). See [Source Maps](#source-maps) +- `--sources-content=`: Force embedding original sources in the map (default: omitted for `inline`, included for `linked`/`external`) - `-h, --help`: Print help information - `-V, --version`: Print version information @@ -212,6 +217,61 @@ cribo --entry main.py --output bundle.py --no-tree-shake - When you need to preserve all code for dynamic imports or reflection - For debugging purposes to see the complete bundled output +### Source Maps + +Cribo can emit a [Source Map v3](https://tc39.es/ecma426/) (the language-agnostic +format used across the JavaScript ecosystem) mapping every statement in the bundle +back to its original file and line, plus an injected runtime that remaps uncaught +exception tracebacks to the original sources — analogous to +`node --enable-source-maps`: + +```bash +# linked (default): writes bundle.py.map + a trailing sourceMappingURL comment +cribo --entry src/main.py --output bundle.py --sourcemap + +# inline: the map travels inside the bundle as a base64 comment +cribo --entry src/main.py --output bundle.py --sourcemap=inline + +# external: writes bundle.py.map, no comment in the bundle +cribo --entry src/main.py --output bundle.py --sourcemap=external +``` + +With source maps enabled, a crash inside the bundle prints the original locations: + +```text +Traceback (most recent call last): + File "/app/src/main.py", line 3, in + boom() + File "/app/src/helper.py", line 5, in inner + raise ValueError("kaboom") +ValueError: kaboom +``` + +Runtime activation follows the delivery mode: + +- `inline`: active by default; `CRIBO_SOURCE_MAPS=0` disables it +- `linked`: active exactly when `.map` exists next to the bundle at run + time — delete the map to ship without remapping, drop it back to re-enable +- `external`: dormant unless `CRIBO_SOURCE_MAPS=1` is set (or the variable holds a + path to the map file) +- In every mode, `CRIBO_SOURCE_MAPS=` points the runtime at an explicit map + file — the only way to remap a bundle executed via `python -` (stdin), whose + inline map cannot be re-read at run time + +The runtime is lazy (zero file access, parsing, or decoding until the first +uncaught exception), streams the map in constant memory so it works under resource +pressure, covers `sys.excepthook`, `threading.excepthook`, and +`sys.unraisablehook`, chains to any pre-installed custom hooks, preserves the +default silent handling of `SystemExit` in worker threads, and falls back to the +standard traceback on any failure. Known limitations: user code that formats +tracebacks itself (e.g. `traceback.format_exc()`) is not remapped; +`ExceptionGroup` chains defer to the standard (unremapped but complete) rendering; +and under a hard out-of-memory condition no pure-Python hook can run. +Configuration-file equivalents: `sourcemap = "linked" | "inline" | "external"` and +`sources-content = true|false` in `cribo.toml`; environment equivalents: +`CRIBO_SOURCEMAP` and `CRIBO_SOURCES_CONTENT`. Full design: +[docs/source-maps.md](docs/source-maps.md). + ### Dependency Detection (`cribo deps`) The `deps` subcommand analyzes a Python file or directory and reports the third-party diff --git a/crates/cribo/Cargo.toml b/crates/cribo/Cargo.toml index 61b4af790..7c0587c7c 100644 --- a/crates/cribo/Cargo.toml +++ b/crates/cribo/Cargo.toml @@ -22,12 +22,14 @@ required-features = ["bench"] [dependencies] anyhow = { workspace = true } +base64-simd = { workspace = true } clap = { workspace = true } cow-utils = { workspace = true } env_logger = { workspace = true } etcetera = { workspace = true } indexmap = { workspace = true } log = { workspace = true } +oxc_sourcemap = { workspace = true } pep508_rs = { workspace = true } petgraph = { workspace = true } ruff_python_ast = { workspace = true } diff --git a/crates/cribo/src/code_generator/inliner.rs b/crates/cribo/src/code_generator/inliner.rs index 245af355c..f8e8d2ce3 100644 --- a/crates/cribo/src/code_generator/inliner.rs +++ b/crates/cribo/src/code_generator/inliner.rs @@ -265,9 +265,12 @@ impl Bundler<'_> { module_renames.insert(func_name.clone(), renamed_name.clone()); ctx.global_symbols.insert(renamed_name.clone()); - // Clone and rename the function + // Clone and rename the function, preserving the original + // identifier's range for source-map provenance (see the + // class rename below for the rationale). + let original_name_range = func_def.name.range; let mut func_def_clone = func_def.clone(); - func_def_clone.name = Identifier::new(renamed_name, TextRange::default()); + func_def_clone.name = Identifier::new(renamed_name, original_name_range); // Apply renames to function annotations (parameters and return type) if let Some(ref mut returns) = func_def_clone.returns { @@ -626,9 +629,14 @@ impl Bundler<'_> { module_renames.insert(class_name.clone(), renamed_name.clone()); ctx.global_symbols.insert(renamed_name.clone()); - // Clone and rename the class + // Clone and rename the class. The original identifier's range is + // preserved on the renamed name: these ranges point into the original + // module source and feed source-map provenance (the code generator + // never reads them), and the name is the only header token that + // exists on every class form (`class C:` has no argument list). + let original_name_range = class_def.name.range; let mut class_def_clone = class_def.clone(); - class_def_clone.name = Identifier::new(renamed_name.clone(), TextRange::default()); + class_def_clone.name = Identifier::new(renamed_name.clone(), original_name_range); // Apply renames to base classes and keyword arguments // CRITICAL: For cross-module inheritance, we need to apply renames from the diff --git a/crates/cribo/src/config.rs b/crates/cribo/src/config.rs index c1817c4ef..3a1222445 100644 --- a/crates/cribo/src/config.rs +++ b/crates/cribo/src/config.rs @@ -47,10 +47,47 @@ pub struct Config { #[serde(rename = "bundle-third-party", alias = "bundle_third_party")] pub bundle_third_party: Option, + /// Source map delivery mode. `None` disables source map generation (default). + /// See `docs/source-maps.md`. + pub sourcemap: Option, + + /// Whether to embed original source text in the map as `sourcesContent`. + /// `None` selects the mode-dependent default (omitted for `inline`, + /// included for `linked`/`external`). + #[serde(rename = "sources-content", alias = "sources_content")] + pub sources_content: Option, + /// Configuration for mapping imports to installable requirements pub requirements: RequirementsConfig, } +/// Source map delivery mode, mirroring esbuild's `--sourcemap` values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum SourceMapMode { + /// Write `.map` next to the bundle and append a + /// `# sourceMappingURL=.map` comment (the default for a bare flag). + Linked, + /// Embed the map as a base64 data-URL comment at the end of the bundle. + Inline, + /// Write `.map` with no comment in the bundle. + External, +} + +impl Config { + /// Effective `sourcesContent` policy for source map generation. + /// + /// An explicit `sources-content` setting wins; otherwise the mode default + /// applies — omitted for `inline` (keeps the bundle small), included for + /// `linked`/`external` (self-contained maps). + pub fn include_sources_content(&self) -> bool { + self.sources_content.unwrap_or(match self.sourcemap { + Some(SourceMapMode::Linked | SourceMapMode::External) => true, + Some(SourceMapMode::Inline) | None => false, + }) + } +} + #[derive(Clone, Default, Serialize, Deserialize)] #[serde(default)] pub struct RequirementsConfig { @@ -91,6 +128,8 @@ impl Default for Config { target_version: "py310".to_owned(), tree_shake: true, // Tree-shaking enabled by default bundle_third_party: None, // Opt-in: third-party deps stay external by default + sourcemap: None, // Opt-in: no source map by default + sources_content: None, // Mode-dependent default; see docs/source-maps.md requirements: RequirementsConfig::default(), } } @@ -123,6 +162,8 @@ impl Combine for Config { tree_shake: self.tree_shake, // Option scalar: absent keys in higher-precedence layers preserve lower layers bundle_third_party: self.bundle_third_party.or(other.bundle_third_party), + sourcemap: self.sourcemap.or(other.sourcemap), + sources_content: self.sources_content.or(other.sources_content), requirements: RequirementsConfig { python: self.requirements.python.or(other.requirements.python), module_map: if self.requirements.module_map.is_empty() { @@ -146,6 +187,8 @@ pub(crate) struct EnvConfig { pub target_version: Option, pub tree_shake: Option, pub bundle_third_party: Option, + pub sourcemap: Option, + pub sources_content: Option, pub python: Option, } @@ -218,6 +261,16 @@ impl EnvConfig { config.bundle_third_party = parse_bool(&bundle_third_party_str); } + // CRIBO_SOURCEMAP - source map delivery mode (linked|inline|external) + if let Ok(sourcemap_str) = env::var("CRIBO_SOURCEMAP") { + config.sourcemap = parse_sourcemap_mode(&sourcemap_str); + } + + // CRIBO_SOURCES_CONTENT - boolean flag overriding the sourcesContent default + if let Ok(sources_content_str) = env::var("CRIBO_SOURCES_CONTENT") { + config.sources_content = parse_bool(&sources_content_str); + } + if let Ok(python) = env::var("CRIBO_PYTHON") { config.python = parse_env_path(&python); } @@ -251,6 +304,12 @@ impl EnvConfig { if let Some(bundle_third_party) = self.bundle_third_party { config.bundle_third_party = Some(bundle_third_party); } + if let Some(sourcemap) = self.sourcemap { + config.sourcemap = Some(sourcemap); + } + if let Some(sources_content) = self.sources_content { + config.sources_content = Some(sources_content); + } if let Some(python) = self.python { config.requirements.python = Some(python); } @@ -268,6 +327,17 @@ fn parse_bool(value: &str) -> Option { } } +/// Parse a source map delivery mode from an environment value. +fn parse_sourcemap_mode(value: &str) -> Option { + use cow_utils::CowUtils; + match value.cow_to_lowercase().as_ref() { + "linked" => Some(SourceMapMode::Linked), + "inline" => Some(SourceMapMode::Inline), + "external" => Some(SourceMapMode::External), + _ => None, + } +} + /// Parse a non-empty environment value as a filesystem path. fn parse_env_path(value: &str) -> Option { if value.is_empty() { diff --git a/crates/cribo/src/lib.rs b/crates/cribo/src/lib.rs index 0e237ebd9..ae9c21eac 100644 --- a/crates/cribo/src/lib.rs +++ b/crates/cribo/src/lib.rs @@ -28,6 +28,7 @@ pub(crate) mod module_facts; pub(crate) mod python; pub(crate) mod requirement_resolver; pub(crate) mod side_effects; +pub(crate) mod source_map; pub(crate) mod symbol_conflict_resolver; pub(crate) mod transformation_context; pub(crate) mod tree_shaking; diff --git a/crates/cribo/src/main.rs b/crates/cribo/src/main.rs index 050742818..717f72a41 100644 --- a/crates/cribo/src/main.rs +++ b/crates/cribo/src/main.rs @@ -24,6 +24,7 @@ mod python; mod requirement_resolver; mod resolver; mod side_effects; +mod source_map; mod symbol_conflict_resolver; mod transformation_context; mod tree_shaking; @@ -31,7 +32,7 @@ mod types; mod util; mod visitors; -use config::Config; +use config::{Config, SourceMapMode}; use orchestrator::BundleOrchestrator; #[derive(Parser)] @@ -81,6 +82,20 @@ struct Cli { /// and are emitted into requirements.txt #[arg(long)] bundle_third_party: bool, + + /// Generate a Source Map v3 for the bundle. A bare `--sourcemap` selects + /// `linked` (`inline` with --stdout); or choose explicitly with + /// `--sourcemap=linked|inline|external` + // Option> is clap's idiom for a flag with an optional value: + // the outer Option is flag presence, the inner one the explicit value. + #[expect(clippy::option_option)] + #[arg(long, value_enum, num_args = 0..=1, require_equals = true)] + sourcemap: Option>, + + /// Force embedding original sources in the map as `sourcesContent` + /// (default: omitted for inline, included for linked/external) + #[arg(long, require_equals = true, value_name = "BOOL")] + sources_content: Option, } #[derive(Subcommand)] @@ -219,6 +234,31 @@ fn run_bundle(mut config: Config, cli: &Cli) -> anyhow::Result<()> { config.bundle_third_party = Some(true); } + // Resolve the source map mode: CLI takes precedence over the config file. + // A bare `--sourcemap` selects the esbuild-style default: linked for file + // output, inline for stdout (a linked map has nowhere to live next to stdout). + if let Some(cli_mode) = cli.sourcemap { + config.sourcemap = Some(cli_mode.unwrap_or(if cli.stdout { + SourceMapMode::Inline + } else { + SourceMapMode::Linked + })); + } + if cli.stdout + && matches!( + config.sourcemap, + Some(SourceMapMode::Linked | SourceMapMode::External) + ) + { + return Err(anyhow!( + "linked and external source maps require an output file; use --sourcemap=inline with \ + --stdout" + )); + } + if let Some(sources_content) = cli.sources_content { + config.sources_content = Some(sources_content); + } + debug!("Configuration: {config:?}"); // Display target version for troubleshooting diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 525b461c7..33968a0d4 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -16,12 +16,13 @@ use crate::{ ResolutionStrategy, }, code_generator::{Bundler, phases::orchestrator::PhaseOrchestrator}, - config::Config, + config::{Config, SourceMapMode}, dependency_graph::DependencyGraph, import_rewriter::{ImportDeduplicationStrategy, ImportRewriter}, module_facts::ModuleFacts, requirement_resolver::RequirementResolver, resolver::{ImportOrigin, ModuleId, ModuleResolver}, + source_map::{ProvenanceResolver, SourceMapOptions, build_source_map}, symbol_conflict_resolver::SymbolConflictResolver, tree_shaking::TreeShaker, types::FxIndexMap, @@ -38,6 +39,107 @@ fn get_empty_parsed_module() -> &'static ruff_python_parser::Parsed { .get_or_init(|| ruff_python_parser::parse_module("").expect("Failed to parse empty module")) } +/// Path of the source map file for a bundle output path (`bundle.py` → `bundle.py.map`). +fn source_map_path_for(output_path: &Path) -> PathBuf { + let mut file_name = output_path.file_name().map_or_else( + || std::ffi::OsString::from("bundle.py"), + std::ffi::OsStr::to_os_string, + ); + file_name.push(".map"); + output_path.with_file_name(file_name) +} + +/// Collision-resistant, unpredictable suffix for a staged map file. +/// +/// In a shared writable directory a predictable temp name (e.g. PID-derived) +/// could be pre-created by another local user, turning every `create_new` +/// attempt into a denial of service. The suffix hashes process identity, an +/// ASLR-randomized stack address, wall-clock nanoseconds, and the attempt +/// counter — not guessable ahead of time, and fresh entropy per retry. This +/// names a transient file only; deterministic-output rules are unaffected. +fn staging_suffix(attempt: u32) -> String { + use sha2::{Digest as _, Sha256}; + + let stack_probe = 0_u8; + let mut hasher = Sha256::new(); + hasher.update(std::process::id().to_le_bytes()); + hasher.update(attempt.to_le_bytes()); + hasher.update((&raw const stack_probe as usize).to_le_bytes()); + if let Ok(elapsed) = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) { + hasher.update(elapsed.as_secs().to_le_bytes()); + hasher.update(elapsed.subsec_nanos().to_le_bytes()); + } + let digest = hasher.finalize(); + let mut hex = String::with_capacity(16); + for byte in &digest[..8] { + write!(hex, "{byte:02x}").expect("Writing to String never fails"); + } + hex +} + +/// Stage the source map into a collision-resistant temp file next to `map_path`. +/// +/// The file is opened with `create_new` (`O_EXCL` semantics), which neither +/// follows a pre-planted symlink nor truncates an existing file — a requirement +/// for outputs in shared sticky directories, where a temp name could otherwise +/// be predicted and pointed elsewhere. Names are unpredictable (see +/// [`staging_suffix`]) and collisions retry with fresh entropy, which also +/// keeps concurrent builds from stomping each other's staged map. +/// +/// The staged file is born `0600` on Unix (mode set atomically at `open(2)` +/// time), so no other user can grab a readable handle between creation and a +/// later `chmod`. When a previous map exists, its permissions are then copied +/// onto the staged file before any content is written, so a restricted map +/// (e.g. 0600 protecting `sourcesContent`) stays restricted across rebuilds. +/// With no previous map the file keeps `0600`; after the rename the map is +/// owner-only by default, and users who want it world-readable can `chmod` it +/// once — the permissions persist across subsequent rebuilds. +fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { + use std::io::Write as _; + + let base_name = map_path.file_name().map_or_else( + || std::ffi::OsString::from("bundle.py.map"), + std::ffi::OsStr::to_os_string, + ); + let mut attempt = 0_u32; + loop { + let mut tmp_name = base_name.clone(); + tmp_name.push(format!(".{}.tmp", staging_suffix(attempt))); + let tmp_path = map_path.with_file_name(tmp_name); + let mut open_options = fs::OpenOptions::new(); + open_options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + open_options.mode(0o600); + } + match open_options.open(&tmp_path) { + Ok(mut file) => { + // symlink_metadata: an attacker-planted symlink at the map + // path must not decide the staged file's permissions. + let write_result = fs::symlink_metadata(map_path) + .ok() + .filter(fs::Metadata::is_file) + .map_or(Ok(()), |metadata| { + file.set_permissions(metadata.permissions()) + }) + .and_then(|()| file.write_all(map_json.as_bytes())) + .and_then(|()| file.flush()); + if let Err(err) = write_result { + drop(file); + let _ = fs::remove_file(&tmp_path); + return Err(err); + } + return Ok(tmp_path); + } + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists && attempt < 1024 => { + attempt += 1; + } + Err(err) => return Err(err), + } + } +} + /// Type alias for module processing queue type ModuleQueue = Vec<(ModuleId, PathBuf)>; /// Type alias for processed modules set @@ -70,6 +172,16 @@ struct StaticBundleParams<'a> { graph: &'a DependencyGraph, circular_dep_analysis: Option<&'a CircularDependencyAnalysis>, tree_shaker: Option<&'a TreeShaker<'a>>, + /// Output file path when writing to disk; `None` for stdout output. + /// Used for the source map `file` field and source path relativization. + output_path: Option<&'a Path>, +} + +/// Result of static bundle emission: the code plus an optional source map JSON. +struct EmittedBundle { + code: String, + /// Source Map v3 JSON, present when `Config::sourcemap` is enabled. + source_map: Option, } /// Context for dependency building operations @@ -601,14 +713,32 @@ impl BundleOrchestrator { // Generate bundled code info!("Using hybrid static bundler"); - let bundled_code = self.emit_static_bundle(&StaticBundleParams { + let emitted = self.emit_static_bundle(&StaticBundleParams { sorted_module_ids: &sorted_module_ids, parsed_modules: Some(&parsed_modules), resolver: &resolver, graph: &graph, circular_dep_analysis: circular_dep_analysis.as_ref(), tree_shaker: tree_shaker.as_ref(), + output_path: None, })?; + let mut bundled_code = emitted.code; + + // Bake the map digest into the runtime prologue (or blank the + // placeholder when no map was produced). + if self.config.sourcemap.is_some() { + bundled_code = + crate::source_map::apply_map_digest(&bundled_code, emitted.source_map.as_deref()); + } + + // Stdout output can only carry an inline map; other modes are rejected + // at CLI validation time. + if self.config.sourcemap == Some(SourceMapMode::Inline) + && let Some(map_json) = emitted.source_map.as_deref() + { + bundled_code.push('\n'); + bundled_code.push_str(&crate::source_map::inline_source_mapping_comment(map_json)); + } // Generate requirements.txt if requested if emit_requirements { @@ -660,26 +790,105 @@ impl BundleOrchestrator { // Generate bundled code info!("Using hybrid static bundler"); - let bundled_code = self.emit_static_bundle(&StaticBundleParams { + let emitted = self.emit_static_bundle(&StaticBundleParams { sorted_module_ids: &sorted_module_ids, parsed_modules: Some(&parsed_modules), // Use pre-parsed modules to avoid double parsing resolver: &resolver, graph: &graph, circular_dep_analysis: circular_dep_analysis.as_ref(), tree_shaker: tree_shaker.as_ref(), + output_path: Some(output_path), })?; + let mut bundled_code = emitted.code; + + // Bake the map digest into the runtime prologue (or blank the + // placeholder when no map was produced): the digest lives inside the + // executing code itself, immune to on-disk replacement. + if self.config.sourcemap.is_some() { + bundled_code = + crate::source_map::apply_map_digest(&bundled_code, emitted.source_map.as_deref()); + } + + // Apply the configured source map delivery mode. The map file itself is + // written only after the bundle write succeeds, so a failed run never + // leaves an orphaned (and potentially stale) map next to an old bundle. + let mut pending_map: Option<(PathBuf, &str)> = None; + if let (Some(mode), Some(map_json)) = (self.config.sourcemap, emitted.source_map.as_deref()) + { + match mode { + SourceMapMode::Linked | SourceMapMode::External => { + let map_path = source_map_path_for(output_path); + if mode == SourceMapMode::Linked { + let map_file_name = + map_path.file_name().unwrap_or_else(|| map_path.as_os_str()); + bundled_code.push('\n'); + bundled_code.push_str(&crate::source_map::linked_source_mapping_comment( + map_file_name, + )); + } + pending_map = Some((map_path, map_json)); + } + SourceMapMode::Inline => { + bundled_code.push('\n'); + bundled_code + .push_str(&crate::source_map::inline_source_mapping_comment(map_json)); + } + } + } // Generate requirements.txt if requested if emit_requirements { self.write_requirements_file(&sorted_module_ids, &resolver, &graph, output_path)?; } + // Publish the bundle and map as close to atomically as possible: the + // map content is staged to a temp file *before* the bundle is written + // (a staging failure aborts with the old pair intact) and renamed over + // the final map path *after* (rename is atomic and replaces a stale + // map even when the file itself is read-only). The only remaining + // mismatch window is a rename failure, which requires directory-level + // problems that would have failed the bundle write too. + let staged_map = if let Some((map_path, map_json)) = pending_map { + let tmp_path = stage_map_file(&map_path, map_json).with_context(|| { + format!( + "Failed to stage source map file next to: {}", + map_path.display() + ) + })?; + Some((tmp_path, map_path)) + } else { + None + }; + // Write output file - fs::write(output_path, bundled_code) - .with_context(|| format!("Failed to write output file: {}", output_path.display()))?; + let bundle_write = fs::write(output_path, bundled_code) + .with_context(|| format!("Failed to write output file: {}", output_path.display())); + if let Err(err) = bundle_write { + if let Some((tmp_path, _)) = staged_map { + let _ = fs::remove_file(tmp_path); + } + return Err(err); + } info!("Bundle written to: {}", output_path.display()); + if let Some((tmp_path, map_path)) = staged_map { + // std's rename replaces an existing destination on every platform + // (MoveFileExW with MOVEFILE_REPLACE_EXISTING on Windows). The one + // residual case is a read-only destination on Windows, so fall + // back to remove-then-rename before giving up. + let publish = fs::rename(&tmp_path, &map_path).or_else(|_| { + fs::remove_file(&map_path).and_then(|()| fs::rename(&tmp_path, &map_path)) + }); + if let Err(err) = publish { + let _ = fs::remove_file(&tmp_path); + return Err(err).with_context(|| { + format!("Failed to publish source map file: {}", map_path.display()) + }); + } + info!("Source map written to: {}", map_path.display()); + } + Ok(()) } @@ -1955,7 +2164,7 @@ impl BundleOrchestrator { } /// Emit bundle using static bundler (no exec calls) - fn emit_static_bundle(&mut self, params: &StaticBundleParams<'_>) -> Result { + fn emit_static_bundle(&mut self, params: &StaticBundleParams<'_>) -> Result { // First, detect and resolve conflicts after all modules have been analyzed let conflicts = self.conflict_resolver.detect_and_resolve_conflicts(); if !conflicts.is_empty() { @@ -2034,7 +2243,7 @@ impl BundleOrchestrator { } // Bundle all modules using the phase-based orchestrator - let bundled_ast = PhaseOrchestrator::bundle( + let mut bundled_ast = PhaseOrchestrator::bundle( &mut static_bundler, &crate::code_generator::BundleParams { modules: &module_asts, @@ -2048,6 +2257,14 @@ impl BundleOrchestrator { }, ); + // Inject the traceback-remapping runtime before code generation so the + // emitted text and the bundled AST stay structurally aligned for the + // source map extraction walk. + if let Some(mode) = self.config.sourcemap { + crate::source_map::inject_runtime_prologue(&mut bundled_ast, mode); + } + let bundled_ast = bundled_ast; + // Generate Python code from AST let empty_parsed = get_empty_parsed_module(); let stylist = ruff_python_codegen::Stylist::from_tokens(empty_parsed.tokens(), ""); @@ -2082,8 +2299,72 @@ impl BundleOrchestrator { String::new(), // Empty line ]; final_output.extend(code_parts); + let code = final_output.join("\n"); + + // Extract source map when enabled: re-parse the emitted code and walk it + // in parallel with the bundled AST (which carries node provenance). + let source_map = if self.config.sourcemap.is_some() { + self.extract_source_map(&code, &bundled_ast, params) + } else { + None + }; + + Ok(EmittedBundle { code, source_map }) + } - Ok(final_output.join("\n")) + /// Build the Source Map v3 JSON for an emitted bundle. + /// + /// Never fails the bundle: extraction errors are logged and yield `None`. + fn extract_source_map( + &self, + code: &str, + bundled_ast: &ModModule, + params: &StaticBundleParams<'_>, + ) -> Option { + let parsed_modules = params.parsed_modules?; + + // Module ordinals were assigned by the bundler's AST indexing pass in + // the order of `parsed_modules`; register provenance in the same order. + let mut provenance = ProvenanceResolver::default(); + for (module_id, _imports, _ast, source) in parsed_modules { + let path = params + .resolver + .get_module_path(*module_id) + .unwrap_or_else(|| { + let name = params + .resolver + .get_module_name(*module_id) + .unwrap_or_else(|| format!("module_{}", module_id.as_u32())); + PathBuf::from(&name) + }); + let path = std::path::absolute(&path).unwrap_or(path); + provenance.push_module(path, source.clone()); + } + + let file_name = params.output_path.and_then(Path::file_name).map_or_else( + || "".to_owned(), + |name| name.to_string_lossy().into_owned(), + ); + // Source paths are relative to the directory the map lives in (the + // output directory). For stdout output the bundle's eventual location + // is unknown, so paths stay absolute. + let base_dir = params + .output_path + .and_then(Path::parent) + .map(|dir| std::path::absolute(dir).unwrap_or_else(|_| dir.to_path_buf())); + + let options = SourceMapOptions { + file: &file_name, + include_contents: self.config.include_sources_content(), + base_dir: base_dir.as_deref(), + }; + match build_source_map(code, bundled_ast, &provenance, &options) { + Ok(json) => Some(json), + Err(err) => { + warn!("Source map generation failed; bundling continues without a map: {err:#}"); + None + } + } } /// Generate requirements.txt content from third-party imports @@ -2312,6 +2593,92 @@ mod tests { use super::*; + /// End-to-end source map extraction: bundle a three-module project (inlined + /// module, wrapper module with side effects, entry) with an inline map and + /// verify statement mappings point back at the original files and lines. + #[test] + fn test_source_map_extraction_end_to_end() -> Result<()> { + let temp_dir = TempDir::new()?; + fs::write( + temp_dir.path().join("main.py"), + "from utils import add\nimport effects\n\nresult = add(1, 2)\nprint(result, \ + effects.X)\n", + )?; + fs::write( + temp_dir.path().join("utils.py"), + "def add(a, b):\n total = a + b\n return total\n", + )?; + // The print side effect forces this module onto the wrapper path. + fs::write( + temp_dir.path().join("effects.py"), + "print(\"side effect\")\nX = 42\n", + )?; + + let config = Config { + sourcemap: Some(SourceMapMode::Inline), + ..Config::default() + }; + let mut orchestrator = BundleOrchestrator::new(config); + let code = orchestrator.bundle_to_string(&temp_dir.path().join("main.py"), false)?; + + // The bundle must end with an inline sourceMappingURL comment. + let marker = "# sourceMappingURL=data:application/json;base64,"; + let marker_pos = code + .rfind(marker) + .expect("inline source map comment present"); + let payload = code[marker_pos + marker.len()..].trim_end(); + let map_bytes = base64_simd::STANDARD + .decode_to_vec(payload.as_bytes()) + .expect("valid base64 payload"); + let map_json = String::from_utf8(map_bytes).expect("valid UTF-8 source map"); + + let map = oxc_sourcemap::SourceMap::from_json_string(&map_json) + .expect("valid Source Map v3 JSON"); + assert_eq!(map.get_file(), Some("")); + // Inline mode omits sourcesContent by default. + assert!(!map_json.contains("sourcesContent")); + + let lookup = map.generate_lookup_table(); + let find_generated_line = |needle: &str| -> u32 { + code.lines() + .position(|line| line.trim() == needle) + .unwrap_or_else(|| panic!("bundle must contain a line matching `{needle}`")) + as u32 + }; + let assert_maps_to = |needle: &str, source_suffix: &str, original_line: u32| { + let generated_line = find_generated_line(needle); + let token = map + .lookup_token(&lookup, generated_line, 0) + .unwrap_or_else(|| panic!("mapping for `{needle}` on line {generated_line}")); + assert_eq!( + token.get_dst_line(), + generated_line, + "`{needle}` must have a mapping on its own line, not inherit an earlier one" + ); + let source = map + .get_source(token.get_source_id().expect("source id")) + .expect("source path"); + assert!( + source.ends_with(source_suffix), + "`{needle}` should map into {source_suffix}, got {source}" + ); + assert_eq!( + token.get_src_line(), + original_line, + "`{needle}` should map to 0-based line {original_line} of {source_suffix}" + ); + }; + + // Inlined module: statement nested in a function body. + assert_maps_to("return total", "utils.py", 2); + // Wrapper module: statement inside the synthesized init function. + assert_maps_to("X = 42", "effects.py", 1); + // Entry module statement. + assert_maps_to("result = add(1, 2)", "main.py", 3); + + Ok(()) + } + /// External importlib targets recorded during one bundle run must not leak into a /// later run on the same orchestrator instance. #[test] diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py new file mode 100644 index 000000000..605017e02 --- /dev/null +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -0,0 +1,1265 @@ +"""Cribo source map runtime (injected prologue). + +Remaps tracebacks of uncaught exceptions back to the original source files +using the Source Map v3 emitted at bundle time. Lazy by design: no file I/O, +parsing, or decoding happens at import time; everything is deferred to the +first uncaught exception. Under resource pressure the decoder streams the map +in constant memory and falls back to the default traceback on any failure. +See docs/source-maps.md in the cribo repository. + +Note: this leading docstring is stripped at injection time so the bundle's +``__doc__`` is not affected. +""" + +import sys as _cribo_sys + + +def _cribo_sm_import( + name, + script_path=None, + *, + _sys=_cribo_sys, + _list=list, + _import=__import__, + _isinstance=isinstance, + _str=str, + _bex=BaseException, +): + """Import a stdlib module immune to script-directory shadowing. + + The bundle's directory can reach `sys.path` several ways: as `sys.path[0]` + (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where + `sys.path[0]` is the *driver's* directory) — so the bundle's own path is + passed in explicitly and every equivalent entry is removed from a private + *snapshot* of the search path. Resolution goes through the frozen importer + machinery (`PathFinder.find_spec` with that explicit snapshot), so global + `sys.path` is never mutated and concurrent bootstraps cannot race each + other's save/restore. Lexical de-duplication works even before `os` is + importable (`python -S`); once `os` is available, entries are also + compared by absolute path. Non-string entries are kept untouched. + (`sys` and the frozen importlib modules are builtins and can never be + shadowed.) + """ + module = _sys.modules.get(name) + if module is not None: + return module + saved_path = _list(_sys.path) + candidates = [] + if saved_path and _isinstance(saved_path[0], _str): + candidates.append(saved_path[0]) + if script_path and _isinstance(script_path, _str): + cut = script_path.rfind("/") + cut_windows = script_path.rfind("\\") + if cut_windows > cut: + cut = cut_windows + candidates.append(script_path[:cut] if cut >= 0 else ".") + trimmed = [] + for candidate in candidates: + trimmed.append(candidate.rstrip("/\\") or candidate) + filtered = [] + for entry in saved_path[1:]: + if _isinstance(entry, _str): + if entry in ("", ".", "./"): + continue + stripped = entry.rstrip("/\\") or entry + if stripped in trimmed: + continue + filtered.append(entry) + os_mod = _sys.modules.get("os") + if os_mod is not None and trimmed: + resolved = [] + for candidate in trimmed: + try: + resolved.append(os_mod.path.normcase(os_mod.path.abspath(candidate or "."))) + except _bex: + pass + kept = [] + for entry in filtered: + if _isinstance(entry, _str): + try: + if os_mod.path.normcase(os_mod.path.abspath(entry or ".")) in resolved: + continue + except _bex: + pass + kept.append(entry) + filtered = kept + # Primary path: resolve against the private snapshot via the frozen + # importer machinery — no global state is touched. Builtin and frozen + # modules (which sys.path shadowing can never affect, and which PathFinder + # cannot see — e.g. `binascii` is compiled into some interpreters) are + # consulted first, mirroring the interpreter's own finder order. + frozen_external = _sys.modules.get("_frozen_importlib_external") + frozen_bootstrap = _sys.modules.get("_frozen_importlib") + if frozen_external is not None and frozen_bootstrap is not None: + spec = ( + frozen_bootstrap.BuiltinImporter.find_spec(name) + or frozen_bootstrap.FrozenImporter.find_spec(name) + or frozen_external.PathFinder.find_spec(name, filtered) + ) + if spec is None: + raise ImportError("cribo runtime could not resolve " + name) + module = frozen_bootstrap.module_from_spec(spec) + _sys.modules[name] = module + try: + spec.loader.exec_module(module) + except _bex: + _sys.modules.pop(name, None) + raise + return module + # Fallback for exotic hosts without the frozen modules: brief global + # swap (the historical behavior). + _sys.path = filtered + try: + return _import(name) + finally: + _sys.path = saved_path + + +class _CriboSmStream(object): + """Byte-at-a-time reader over an iterator of byte chunks. + + Keyword-only defaults snapshot the builtins at definition time (before any + bundled user code runs), so later shadowing of e.g. `len` cannot break the + reader — the same idiom cribo's generated module proxies use. + """ + + __slots__ = ("_chunks", "_buf", "_pos") + + def __init__(self, chunks, *, _iter=iter): + self._chunks = _iter(chunks) + self._buf = b"" + self._pos = 0 + + def read_byte(self, *, _len=len, _next=next, _stop=StopIteration): + while self._pos >= _len(self._buf): + try: + self._buf = _next(self._chunks) + except _stop: + return -1 + self._pos = 0 + value = self._buf[self._pos] + self._pos += 1 + return value + + +class _CriboSourceMapRuntime(object): + """Traceback-remapping runtime. + + All collaborators (modules, the stream class, previous hooks) are bound to + the instance at construction time, and every method snapshots the builtins + it needs via keyword-only defaults evaluated at class-definition time — so + the installed hooks keep working even if bundled user code later rebinds + any module-level name this template introduced, or common builtins such as + `open`, `len`, or `max`. + """ + + _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + _CHUNK = 8192 + + def __init__( + self, + mode, + bundle_file, + expected_digest, + os_mod, + binascii_mod, + threading_mod, + traceback_mod, + hashlib_mod, + *, + # Class-definition-time snapshots: the module-level helper names are + # deleted at the end of the prologue, so they must never be looked up + # globally at construction time (test harnesses construct instances + # long after that cleanup). + _sys=_cribo_sys, + _stream_cls=_CriboSmStream, + _sm_import=_cribo_sm_import, + ): + self._mode = mode + # As-given path for frame matching (co_filename uses the invocation + # spelling), plus a startup-anchored absolute path for file I/O so a + # later os.chdir() in bundled code cannot orphan a relative path. + self._bundle = bundle_file + if bundle_file == "": + self._bundle_anchor = bundle_file + else: + self._bundle_anchor = os_mod.path.abspath(bundle_file) + # SHA-256 of this build's map, baked into the executing code itself at + # build time — immune to any on-disk replacement of the bundle, so a + # redeployed bundle's map can never be applied to old in-memory code. + self._expected_digest = None + try: + if isinstance(expected_digest, str) and len(expected_digest) == 64: + int(expected_digest, 16) + self._expected_digest = expected_digest.lower() + except ValueError: + self._expected_digest = None + # Marks this instance so another cribo runtime chained behind it can + # recognize it (see _notify_custom_hook). + self._cribo_sm_runtime_marker = True + # Startup working directory, for anchoring a relative + # CRIBO_SOURCE_MAPS override before user code can chdir away. + try: + self._startup_cwd = os_mod.getcwd() + except OSError: + self._startup_cwd = "." + self._os = os_mod + self._sys = _sys + self._binascii = binascii_mod + self._threading = threading_mod + self._stream_cls = _stream_cls + self._import = _sm_import + # Captured at construction (before any bundled user code runs) so a + # first-party module registering sys.modules["traceback"] later cannot + # degrade exception formatting. + self._traceback = traceback_mod + self._hashlib = hashlib_mod + # Re-entrancy guard; thread-local so a hook firing on one thread never + # disables remapping on another. + self._local = threading_mod.local() + self._prev_excepthook = _sys.excepthook + self._prev_unraisablehook = _sys.unraisablehook + self._prev_threading_hook = threading_mod.excepthook + # Interpreter defaults, snapshotted now so user code rebinding e.g. + # sys.__excepthook__ later cannot make the captured previous hook look + # custom (which would double-print) or raise from the hook. + self._default_excepthook = _sys.__excepthook__ + self._default_unraisablehook = _sys.__unraisablehook__ + self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) + try: + self._group_type = BaseExceptionGroup + except NameError: # Python < 3.11 + self._group_type = None + + def install(self): + """Install the three hooks; the previous hooks stay chained. + + Also joins the interpreter-wide runtime registry (stored on the + unshadowable `sys` module) so stacked bundles can merge each other's + mappings when a traceback crosses bundle boundaries. + """ + registry = getattr(self._sys, "_cribo_sm_runtimes", None) + if not isinstance(registry, list): + registry = [] + self._sys._cribo_sm_runtimes = registry + registry.append(self) + self._sys.excepthook = self.excepthook + self._sys.unraisablehook = self.unraisablehook + self._threading.excepthook = self.threading_hook + + @classmethod + def _bootstrap(cls, mode, bundle_file, expected_digest, *, _sm_import=_cribo_sm_import): + """Import dependencies safely, construct, and install — fail-open. + + Any failure (however exotic the host environment) leaves the program + running without remapping instead of aborting it at startup. + """ + try: + script = None if bundle_file == "" else bundle_file + runtime = cls( + mode, + bundle_file, + expected_digest, + _sm_import("os", script), + _sm_import("binascii", script), + _sm_import("threading", script), + _sm_import("traceback", script), + _sm_import("hashlib", script), + ) + runtime.install() + except BaseException: + pass + + # -- map location and raw chunk access --------------------------------- + + def _map_location(self): + """Resolve the map location per delivery mode, or None when inactive. + + Returns (map_path, map_dir, verify); map_path is None for inline mode + (the map lives inside the bundle file itself) and `verify` says whether + the sibling map must match the digest recorded in the bundle. Called + lazily at hook-fire time so the happy path never touches the + environment or the filesystem. + """ + env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + # An explicit path wins for every mode (and skips digest verification — + # it is the user's explicit choice). This is also the only way to + # supply a map to a bundle executed via `python -` (stdin), whose + # source cannot be re-read at hook time. A relative override is + # anchored to the startup working directory, immune to later chdir. + if env not in ("", "1", "true", "yes", "on"): + path = env + if not self._os.path.isabs(path): + path = self._os.path.join(self._startup_cwd, path) + return (path, self._os.path.dirname(self._os.path.abspath(path)), False) + bundle = self._bundle_anchor + if self._mode == "inline": + if bundle == "": + return None # stdin cannot be re-opened; use CRIBO_SOURCE_MAPS= + return (None, self._os.path.dirname(bundle), True) + sibling = bundle + ".map" + if self._mode == "linked": + if self._os.path.exists(sibling): + return (sibling, self._os.path.dirname(sibling), True) + return None + # external: opt in via CRIBO_SOURCE_MAPS=1 (a path was handled above) + if env in ("1", "true", "yes", "on"): + return (sibling, self._os.path.dirname(sibling), True) + return None + + def _file_chunks(self, path, *, _open=open): + """Yield fixed-size chunks of a file (constant memory).""" + handle = _open(path, "rb") + try: + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() + + def _handle_chunks(self, handle): + """Yield fixed-size chunks from an already-open handle, from offset 0. + + Keeping one open handle across the verify/decode passes pins the inode: + a concurrent build renaming a new map into place cannot swap the bytes + between passes. + """ + handle.seek(0) + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + + def _find_marker_tail(self, handle, marker, *, _len=len): + """Backward-scan a file for the last occurrence of `marker`. + + Returns the offset of the payload following the marker, or -1. Only the + tail of the file is examined; the body is never read. + """ + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + while position > 0: + step = self._CHUNK if position >= self._CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + return position + index + _len(marker) + overlap = data[: _len(marker) - 1] + return -1 + + def _find_inline_payload(self, handle): + """Offset of the inline map's base64 payload in the bundle, or -1.""" + found = self._find_marker_tail(handle, b"# sourceMappingURL=data:") + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + 7 + + def _map_matches_digest(self, chunks_factory, *, _bex=BaseException): + """False only when a build digest is known and the map bytes disagree. + + The SHA-256 of this build's map is baked into the executing code at + build time, so it is immune to any on-disk replacement of the bundle — + no interleaving of concurrent builds, manual file shuffling, or + redeploy-while-running can pair these code objects with another + build's mappings. Hashing streams the same chunk source later used for + decoding. Verification errors fail open (the subsequent map read would + surface them anyway). + """ + try: + expected = self._expected_digest + if expected is None: + return True + hasher = self._hashlib.sha256() + for chunk in chunks_factory(): + hasher.update(chunk) + return hasher.hexdigest() == expected + except _bex: + return True + + def _inline_chunks(self, handle, *, _len=len): + """Yield decoded chunks of an inline (base64 data URL) source map. + + Reads from an already-open bundle handle so all passes see one inode. + """ + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = _len(data) - (_len(data) % 4) + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) + + # -- streaming JSON field scanner --------------------------------------- + + def _skip_ws(self, stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte + + def _read_hex4(self, stream, *, _int=int, _chr=chr, _range=range, _error=ValueError): + """Read the four hex digits of a \\uXXXX escape; return the code unit.""" + code = 0 + for _ in _range(4): + digit = stream.read_byte() + if digit < 0: + raise _error("unterminated unicode escape") + code = code * 16 + _int(_chr(digit), 16) + return code + + def _read_string( + self, + stream, + collect, + *, + _bytearray=bytearray, + _chr=chr, + _error=ValueError, + ): + """Consume a JSON string whose opening quote was already read. + + Returns the decoded text when collect is true, else None (contents are + discarded byte-by-byte). Escaped quotes, \\uXXXX sequences, and UTF-16 + surrogate pairs (non-BMP characters) are handled, so string *values* + containing text like '"mappings":' cannot confuse the key scanner and + emoji-bearing paths survive intact. + """ + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf = _bytearray() if collect else None + pending = None # one byte of lookahead pushed back by surrogate handling + while True: + if pending is None: + byte = stream.read_byte() + else: + byte, pending = pending, None + if byte < 0: + raise _error("unterminated JSON string") + if byte == 34: # '"' + return buf.decode("utf-8", "replace") if collect else None + if not byte == 92: # '\\' + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise _error("unterminated JSON escape") + if not escape == 117: # 'u' + if buf is not None: + buf.append(table.get(escape, escape)) + continue + code = self._read_hex4(stream) + if buf is None: + continue + if 0xD800 <= code <= 0xDBFF: + # High surrogate: a following \uXXXX low surrogate combines + # into one non-BMP code point. + nxt = stream.read_byte() + if nxt == 92: + escape2 = stream.read_byte() + if escape2 == 117: + low = self._read_hex4(stream) + if 0xDC00 <= low <= 0xDFFF: + code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00) + buf.extend(_chr(code).encode("utf-8")) + else: + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(low).encode("utf-8", "surrogatepass")) + continue + if escape2 < 0: + raise _error("unterminated JSON escape") + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.append(table.get(escape2, escape2)) + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + pending = nxt # includes EOF/quote; the main loop handles both + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + + def _skip_value(self, stream, byte, *, _error=ValueError): + """Skip one JSON value; return the first byte after it (or -1).""" + if byte == 34: # string + self._read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): # object / array + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise _error("unterminated JSON container") + if byte == 34: + self._read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + # number / true / false / null: consume until a delimiter + while byte >= 0 and byte not in (44, 125, 93): # ',' '}' ']' + byte = stream.read_byte() + return byte + + def _decode_vlq( + self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError + ): + """Streaming VLQ state machine over the raw bytes of the mappings string. + + Constant state: line/segment counters plus running deltas. Records the + first segment per needed generated line; exits as soon as every needed + line is resolved or the max needed line is passed. Returns + ``(table, terminated)`` where ``terminated`` says whether the closing + quote was consumed (early exits leave the stream inside the string). + """ + lut = {} + for index in _range(64): + lut[self._B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 + vlq_value = 0 + vlq_shift = 0 + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = (src_idx, src_line) + + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: # EOF or closing '"' + end_segment() + return result, True + if byte == 59: # ';' + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or _len(result) == _len(needed): + return result, False + continue + if byte == 44: # ',' + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise _error("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 + + def _scan(self, chunks_factory, needed, max_needed, *, _set=set, _error=ValueError): + """Two-pass scan of the map; return (sources dict, line table). + + Pass 1 decodes only `mappings` (skipping the sources array entirely); + pass 2 re-streams the map and collects only the source paths actually + referenced by the decoded lines. Memory therefore scales with the + traceback's needed frames, not with the bundle's full source table — + the property the whole streaming decoder exists for. `chunks_factory` + is a zero-argument callable producing a fresh chunk iterator per pass. + """ + table = self._scan_mappings(self._stream_cls(chunks_factory()), needed, max_needed) + wanted = _set(src_idx for (src_idx, _line) in table.values()) + sources = {} + if wanted: + sources = self._scan_sources(self._stream_cls(chunks_factory()), wanted) + return sources, table + + def _scan_mappings(self, stream, needed, max_needed, *, _error=ValueError): + """Pass 1: decode the `mappings` field; every other value is skipped.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: # '{' + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: # '"' starting a key + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: # ':' + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "mappings": + if not byte == 34: + raise _error("mappings is not a string") + table, _terminated = self._decode_vlq(stream, needed, max_needed) + return table + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: # ',' + byte = self._skip_ws(stream, stream.read_byte()) + raise _error("no mappings field") + + def _scan_sources(self, stream, wanted, *, _error=ValueError): + """Pass 2: collect only the `sources` entries whose index is wanted.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: # '{' + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: # '"' starting a key + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: # ':' + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + if not byte == 91: # '[' + raise _error("sources is not an array") + return self._read_wanted_array_items(stream, wanted) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: # ',' + byte = self._skip_ws(stream, stream.read_byte()) + return {} + + def _read_wanted_array_items(self, stream, wanted, *, _len=len, _error=ValueError): + """Read a JSON array, collecting only string items at wanted indices.""" + items = {} + index = 0 + byte = self._skip_ws(stream, stream.read_byte()) + if byte == 93: # ']' + return items + while True: + if byte == 34 and index in wanted: + items[index] = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + else: + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + index += 1 + if byte == 93 or _len(items) == _len(wanted): + return items + if not byte == 44: # ',' + raise _error("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) + + # -- loading ------------------------------------------------------------- + + def _load(self, needed_lines, *, _open=open, _set=set, _max=max): + """Load (table, sources, map_dir) for 1-based bundle line numbers. + + Returns None when the runtime is inactive for the current mode. The + returned table is keyed by 1-based bundle lines mapping to + (source_index, 1-based original line). The map is opened exactly once: + digest verification and both decode passes read from the same pinned + handle, so the verified bytes are the decoded bytes even if a + concurrent build renames a new map into place mid-decode. + """ + location = self._map_location() + if location is None: + return None + map_path, map_dir, verify = location + needed0 = _set(line - 1 for line in needed_lines) + max_needed = _max(needed0) + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is None: + + def chunks_factory(): + return self._inline_chunks(handle) + + else: + + def chunks_factory(): + return self._handle_chunks(handle) + + if verify and not self._map_matches_digest(chunks_factory): + return None + sources, table0 = self._scan(chunks_factory, needed0, max_needed) + finally: + handle.close() + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = (src_idx, src_line0 + 1) + return (table, sources, map_dir) + + def _load_json_fallback( + self, needed_lines, *, _open=open, _set=set, _max=max, _enumerate=enumerate + ): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = self._map_location() + if location is None: + return None + map_path, map_dir, verify = location + json = self._import("json") + + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is None: + + def fallback_chunks(): + return self._inline_chunks(handle) + + else: + + def fallback_chunks(): + return self._handle_chunks(handle) + + if verify and not self._map_matches_digest(fallback_chunks): + return None + raw = b"".join(fallback_chunks()) + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = {} + for index, source in _enumerate(data.get("sources") or []): + sources[index] = source + mappings = data.get("mappings") or "" + needed0 = _set(line - 1 for line in needed_lines) + stream = self._stream_cls([mappings.encode("ascii"), b'"']) + table0, _terminated = self._decode_vlq(stream, needed0, _max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = (src_idx, src_line0 + 1) + return (table, sources, map_dir) + + # -- traceback collection and rendering ---------------------------------- + + def _collect_needed( + self, exc_value, traceback_obj, bundle_file, *, _set=set, _id=id, _getattr=getattr + ): + """1-based lines of `bundle_file` referenced by the traceback chain.""" + needed = _set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == bundle_file: + needed.add(tb.tb_lineno) + tb = tb.tb_next + + add(traceback_obj) + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + add(_getattr(exc, "__traceback__", None)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed + + def _chain_has_group( + self, exc_value, *, _set=set, _id=id, _getattr=getattr, _isinstance=isinstance + ): + """Whether the exception chain contains a BaseExceptionGroup. + + CPython renders groups with a dedicated nested layout; rather than + losing the nested tracebacks, the runtime defers group rendering + entirely to the previous hook (unremapped but complete). + """ + if self._group_type is None: + return False + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + if _isinstance(exc, self._group_type): + return True + cause = _getattr(exc, "__cause__", None) + if cause is not None: + exc = cause + continue + # A suppressed context (`raise ... from None`) is never rendered, + # so a group hidden there must not force the unremapped fallback. + if _getattr(exc, "__suppress_context__", False): + return False + exc = _getattr(exc, "__context__", None) + return False + + def _percent_decode(self, text, *, _int=int, _chr=chr, _len=len, _bex=BaseException): + """Inverse of the build-time percent-encoding of `sources` entries.""" + if "%" not in text: + return text + out = [] + position = 0 + length = _len(text) + while position < length: + character = text[position] + if character == "%" and position + 2 < length: + try: + out.append(_chr(_int(text[position + 1 : position + 3], 16))) + position += 3 + continue + except _bex: + pass + out.append(character) + position += 1 + return "".join(out) + + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): + """Read a single 1-based line from a file without caching it.""" + try: + handle = _open(path, "rb") + except _os_error: + return None + try: + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except _os_error: + return None + finally: + handle.close() + return None + + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int=int): + """The application's `sys.tracebacklimit`, or None when unset/invalid.""" + limit = _getattr(self._sys, "tracebacklimit", None) + return limit if _isinstance(limit, _int) else None + + def _write_frames(self, traceback_obj, maps_by_file, write, limit, summaries, *, _bex=BaseException, _len=len): + """Write remapped frame lines, collapsing repeated frames like CPython. + + `maps_by_file` holds one (table, sources, map_dir) triple per known + bundle file, so a traceback crossing several stacked cribo bundles + remaps every frame. Frames that stay unmapped render through the + standard traceback machinery (`summaries`, when available) to preserve + PEP 657 position indicators. Consecutive identical frames (recursion) + print at most 3 times followed by a "[Previous line repeated N more + times]" marker; source line text is cached per (file, line) within one + rendering. A positive `limit` keeps only the last `limit` frames, + matching the interpreter's `sys.tracebacklimit` handling. + """ + index = 0 + if limit is not None: + total = 0 + probe = traceback_obj + while probe is not None: + total += 1 + probe = probe.tb_next + skip = total - limit + while skip > 0 and traceback_obj is not None: + traceback_obj = traceback_obj.tb_next + skip -= 1 + index += 1 + + cache = {} + last = None + repeats = 0 + + def emit(entry, frame_index, mapped_frame): + if not mapped_frame and summaries is not None and frame_index < _len(summaries): + # Standard rendering for untouched frames keeps PEP 657 + # carets and anchors intact. + try: + rendered = self._traceback.StackSummary.from_list( + [summaries[frame_index]] + ).format() + except _bex: + rendered = None + if rendered: + for line in rendered: + write(line) + return + write(' File "%s", line %d, in %s\n' % entry) + key = (entry[0], entry[1]) + if key not in cache: + cache[key] = self._source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + mapped_frame = False + bundle_map = maps_by_file.get(filename) + if bundle_map is not None: + table, sources, map_dir = bundle_map + mapped = table.get(lineno) + if mapped is not None: + source = sources.get(mapped[0]) + if source: + source = self._percent_decode(source) + if not self._os.path.isabs(source): + source = self._os.path.normpath( + self._os.path.join(map_dir, source) + ) + filename, lineno = source, mapped[1] + mapped_frame = True + entry = (filename, lineno, name) + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry, index, mapped_frame) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 + emit(entry, index, mapped_frame) + traceback_obj = traceback_obj.tb_next + index += 1 + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + + def _exception_line( + self, exc_value, *, _type=type, _getattr=getattr, _str=str, _bex=BaseException + ): + """Minimal `Type: message` line, the fallback formatter.""" + exc_type = _type(exc_value) + name = _getattr(exc_type, "__qualname__", exc_type.__name__) + module = _getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = _str(exc_value) + except _bex: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name + + def _write_exception_only( + self, exc, write, *, _type=type, _getattr=getattr, _bex=BaseException + ): + """Write the exception line(s) with full standard-library fidelity. + + `traceback.format_exception_only` supplies the interpreter's + specialized rendering — SyntaxError source line and caret, NameError / + AttributeError "Did you mean" suggestions, and `__notes__` — so the + remapped output matches the default hook. Falls back to the minimal + line (plus notes) when the traceback module is unavailable. + """ + try: + te = self._traceback.TracebackException( + _type(exc), + exc, + _getattr(exc, "__traceback__", None), + lookup_lines=False, + ) + for line in te.format_exception_only(): + write(line) + return + except _bex: + pass + write(self._exception_line(exc)) + notes = _getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except _bex: + pass + + def _render( + self, + exc_value, + maps_by_file, + write, + *, + _getattr=getattr, + _set=set, + _id=id, + _list=list, + _reversed=reversed, + _enumerate=enumerate, + _type=type, + _bex=BaseException, + ): + """Render the exception (with its cause/context chain) like CPython.""" + chain = [] + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + suppress = _getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context + else: + chain.append((exc, None)) + exc = None + # Print innermost first, like CPython. The link stored on an exception + # describes its relation to its own inner exception — which is exactly + # the one printed immediately before it. + ordered = _list(_reversed(chain)) + for index, (exc, link) in _enumerate(ordered): + if index > 0: + if link == "cause": + write( + "\nThe above exception was the direct cause of the following " + "exception:\n\n" + ) + else: + write( + "\nDuring handling of the above exception, another exception " + "occurred:\n\n" + ) + tb = _getattr(exc, "__traceback__", None) + limit = self._effective_tb_limit() + if tb is not None and (limit is None or limit > 0): + # Standard per-frame summaries (with PEP 657 positions on + # 3.11+) render the frames this runtime does not remap. The + # extraction passes an explicit limit spanning the whole + # traceback: left implicit, the traceback module would honor a + # positive sys.tracebacklimit by keeping the FIRST n frames, + # while this renderer (like the interpreter's C printer) keeps + # the LAST n — the truncated list would misalign or fall out + # of range against raw traceback indices in _write_frames, + # which applies the real limit itself. + total_frames = 0 + probe = tb + while probe is not None: + total_frames += 1 + probe = probe.tb_next + try: + summaries = self._traceback.TracebackException( + _type(exc), exc, tb, limit=total_frames, lookup_lines=False + ).stack + except _bex: + summaries = None + write("Traceback (most recent call last):\n") + self._write_frames(tb, maps_by_file, write, limit, summaries) + self._write_exception_only(exc, write) + + def _try_render( + self, + exc_value, + traceback_obj, + prefix, + *, + _getattr=getattr, + _bex=BaseException, + _set=set, + _len=len, + ): + """Attempt a remapped rendering to stderr; True on success. + + Never raises and never masks the original exception: any failure in + this runtime returns False so callers can delegate to the previous + hook. + """ + if _getattr(self._local, "in_hook", False) or exc_value is None: + return False + self._local.in_hook = True + try: + if self._chain_has_group(exc_value): + return False + # Every stacked cribo runtime contributes mappings for its own + # bundle, so a traceback crossing several bundles remaps fully. + # When two runtimes share one filename spelling (e.g. both loaded + # as a relative "bundle.py" from different directories), frames + # carrying that spelling are ambiguous — decline rather than remap + # through the wrong project's sources. + anchors_by_name = {} + for runtime in self._registered_runtimes(): + bundle_file = _getattr(runtime, "_bundle", None) + anchor = _getattr(runtime, "_bundle_anchor", None) + anchors_by_name.setdefault(bundle_file, _set()).add(anchor) + maps_by_file = {} + for runtime in self._registered_runtimes(): + try: + bundle_file = runtime._bundle + if bundle_file in maps_by_file: + continue + if _len(anchors_by_name.get(bundle_file, ())) > 1: + continue # ambiguous spelling: skip these frames + needed = self._collect_needed(exc_value, traceback_obj, bundle_file) + if not needed: + continue + loaded = None + try: + loaded = runtime._load(needed) + except _bex: + try: + loaded = runtime._load_json_fallback(needed) + except _bex: + loaded = None + if loaded and loaded[0]: + maps_by_file[bundle_file] = loaded + except _bex: + continue + if not maps_by_file: + return False + # Buffer the rendering so a mid-render failure produces no partial + # output before the previous hook prints the standard traceback. + parts = [] + if prefix: + parts.append(prefix) + self._render(exc_value, maps_by_file, parts.append) + stderr = self._sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except _bex: + pass + return True + except _bex: + return False + finally: + self._local.in_hook = False + + def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): + """All installed cribo runtimes in this interpreter, self included. + + The registry lives on the (unshadowable) `sys` module so stacked + bundles can find each other's mappings. + """ + registry = _getattr(self._sys, "_cribo_sm_runtimes", None) + runtimes = [] + if _isinstance(registry, _list): + for runtime in registry: + if _getattr(runtime, "_cribo_sm_runtime_marker", False): + runtimes.append(runtime) + if self not in runtimes: + runtimes.append(self) + return runtimes + + def _notify_custom_hook( + self, + prev, + default, + call, + prev_attr, + default_attr, + *, + _bex=BaseException, + _getattr=getattr, + _set=set, + _id=id, + ): + """Invoke a chained hook after a successful remap when it is custom. + + A successful remap replaces the *default* printer, but preinstalled + custom hooks (error reporters, sitecustomize) must still observe the + exception; their own output is theirs to manage. Earlier cribo + runtimes in the chain are traversed (via their own captured + predecessor, named by `prev_attr`) rather than invoked, so a custom + hook installed before any bundle still gets notified while the default + printer is never reached (which would duplicate the traceback). Each + traversed runtime's own captured default (named by `default_attr`) is + collected along the way: an earlier runtime may have snapshotted a + different default object than this one (e.g. after someone swapped + `sys.__excepthook__` between bundle imports), and a chain ending on + *any* of those defaults must invoke nothing — every default is a + traceback printer, and the remap already printed. Cycle detection + guards the traversal, and a chain that still ends on a cribo hook + likewise invokes nothing (its registry-aware renderer would print the + traceback a second time). When the interpreter default is unavailable + for comparison (e.g. `threading.__excepthook__` before Python 3.10) no + notification happens — better to skip a custom hook than to + double-print via the default one. + """ + defaults = [default] + seen = _set() + while prev is not None and _id(prev) not in seen: + seen.add(_id(prev)) + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + other_default = _getattr(bound_to, default_attr, None) + if other_default is not None: + defaults.append(other_default) + prev = _getattr(bound_to, prev_attr, None) + continue + break + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return + if default is None or prev is None: + return + for known_default in defaults: + if prev is known_default: + return + try: + call(prev) + except _bex: + pass + + # -- installed hooks ------------------------------------------------------ + + def excepthook(self, exc_type, exc_value, traceback_obj): + if self._try_render(exc_value, traceback_obj, None): + self._notify_custom_hook( + self._prev_excepthook, + self._default_excepthook, + lambda hook: hook(exc_type, exc_value, traceback_obj), + "_prev_excepthook", + "_default_excepthook", + ) + return + self._prev_excepthook(exc_type, exc_value, traceback_obj) + + def threading_hook( + self, args, *, _getattr=getattr, _issubclass=issubclass, _system_exit=SystemExit + ): + # The default threading hook deliberately ignores SystemExit (normal + # sys.exit() in a worker thread); preserve that by delegating. + if args.exc_type is not None and _issubclass(args.exc_type, _system_exit): + self._prev_threading_hook(args) + return + thread = _getattr(args, "thread", None) + name = _getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if self._try_render(args.exc_value, args.exc_traceback, prefix): + self._notify_custom_hook( + self._prev_threading_hook, + self._default_threading_hook, + lambda hook: hook(args), + "_prev_threading_hook", + "_default_threading_hook", + ) + return + self._prev_threading_hook(args) + + def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): + message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except _bex: + prefix = "%s\n" % message + if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + self._notify_custom_hook( + self._prev_unraisablehook, + self._default_unraisablehook, + lambda hook: hook(unraisable), + "_prev_unraisablehook", + "_default_unraisablehook", + ) + return + self._prev_unraisablehook(unraisable) + + +_CriboSourceMapRuntime._bootstrap( + "__CRIBO_SOURCEMAP_MODE__", + globals().get("__file__", ""), + "__CRIBO_SOURCEMAP_DIGEST__", +) +# Leave no trace in the bundle's namespace: user code must not see (or +# collide with) runtime helpers, and `from bundle import *` must not export +# them. The installed instance holds every reference it needs (captured in +# __init__/keyword defaults), so nothing here is looked up globally after +# bootstrap. +del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs new file mode 100644 index 000000000..8f3265ac0 --- /dev/null +++ b/crates/cribo/src/source_map.rs @@ -0,0 +1,1017 @@ +//! Source Map v3 generation for the bundled output. +//! +//! Cribo emits statement/line-level mappings (column 0): each mapped statement in +//! the bundle contributes one token `generated line → (original file, original line)`. +//! That granularity matches Python's line-oriented tracebacks, which are the primary +//! consumer via the injected runtime (see `docs/source-maps.md`). +//! +//! Serialization is delegated to `oxc_sourcemap`, the encoder used by rolldown and +//! rspack, which guarantees spec-compliant VLQ `mappings` output. + +use std::borrow::Cow; + +use anyhow::Context as _; +use oxc_sourcemap::{SourceMap, Token}; +use ruff_python_ast::{HasNodeIndex as _, ModModule, NodeIndex, Stmt}; +use ruff_text_size::{Ranged as _, TextSize}; + +use crate::{ast_indexer::MODULE_INDEX_RANGE, types::FxIndexMap}; + +/// Identifier of a registered original source file within a [`SourceMapGenerator`]. +/// +/// Indexes into the emitted `sources` array of the Source Map v3 JSON. +pub(crate) type SourceId = u32; + +/// Byte offsets of line starts, for converting a `TextSize` offset to a line number. +#[derive(Debug)] +pub(crate) struct LineIndex { + /// Byte offset of the start of each line; `line_starts[0] == 0`. + line_starts: Vec, +} + +impl LineIndex { + /// Build the index from source text (lines separated by `\n`; sources are + /// normalized to `\n` line endings when read). + pub(crate) fn new(source: &str) -> Self { + let mut line_starts = vec![0_u32]; + for (offset, byte) in source.bytes().enumerate() { + if byte == b'\n' { + line_starts.push(offset as u32 + 1); + } + } + Self { line_starts } + } + + /// 0-based line containing byte `offset`. + pub(crate) fn line_of(&self, offset: TextSize) -> u32 { + let offset = offset.to_u32(); + // partition_point returns the count of line starts <= offset; the line + // containing the offset is the last such line. + (self.line_starts.partition_point(|&start| start <= offset) - 1) as u32 + } +} + +/// Provenance data for one bundled module, in module-ordinal order. +#[derive(Debug)] +pub(crate) struct ModuleSourceInfo { + /// Original filesystem path of the module (as recorded in the `sources` array). + pub(crate) path: std::path::PathBuf, + /// The module's original source text (for `sourcesContent` and line lookup). + pub(crate) source: String, + /// Line index over `source`. + pub(crate) line_index: LineIndex, +} + +/// Resolves node provenance: which original module and line an AST node came from. +/// +/// Module ordinals are assigned by the bundler's AST indexing pass +/// (`Bundler::index_module_asts`): the n-th module receives node indices in +/// `[n * MODULE_INDEX_RANGE, (n + 1) * MODULE_INDEX_RANGE)`. Entries here MUST be +/// registered in that same order. Synthesized nodes carry `AtomicNodeIndex::NONE` +/// (or an index past all module ranges) and resolve to `None`. +#[derive(Debug, Default)] +pub(crate) struct ProvenanceResolver { + modules: Vec, +} + +impl ProvenanceResolver { + /// Register the next module (ordinal = number of previously registered modules). + pub(crate) fn push_module(&mut self, path: std::path::PathBuf, source: String) { + let line_index = LineIndex::new(&source); + self.modules.push(ModuleSourceInfo { + path, + source, + line_index, + }); + } + + /// Registered modules, in ordinal order. + pub(crate) fn modules(&self) -> &[ModuleSourceInfo] { + &self.modules + } + + /// Resolve a node to (module ordinal, 0-based original line). + /// + /// Returns `None` for synthesized nodes: those with the `NONE` placeholder + /// index or with an index beyond all module ranges (allocated by + /// `TransformationContext` after indexing). + pub(crate) fn resolve( + &self, + node_index: NodeIndex, + range_start: TextSize, + ) -> Option<(usize, u32)> { + let index = node_index.as_u32()?; + let ordinal = (index / MODULE_INDEX_RANGE) as usize; + let module = self.modules.get(ordinal)?; + Some((ordinal, module.line_index.line_of(range_start))) + } +} + +/// A single line-level mapping record. +#[derive(Debug, Clone, Copy)] +struct Mapping { + /// 0-based line in the generated bundle. + generated_line: u32, + /// Which original source file this line came from. + source_id: SourceId, + /// 0-based line in the original source file. + original_line: u32, +} + +/// Collects line-level mapping records and serializes them as Source Map v3 JSON. +/// +/// Lines are 0-based on both the generated and original side, matching the +/// Source Map v3 token encoding (callers converting from 1-based line numbers +/// must subtract one). +#[derive(Debug, Default)] +pub(crate) struct SourceMapGenerator { + /// Name of the generated file (the bundle), emitted as the `file` field. + file: String, + /// Original source path → optional embedded content. Insertion order defines + /// the `sources` array order; the map index is the [`SourceId`]. + sources: FxIndexMap>, + /// Collected mappings, in insertion order (sorted at serialization time). + mappings: Vec, +} + +impl SourceMapGenerator { + /// Create a generator for a bundle named `file` (emitted as the `file` field). + pub(crate) fn new(file: impl Into) -> Self { + Self { + file: file.into(), + sources: FxIndexMap::default(), + mappings: Vec::new(), + } + } + + /// Register an original source file, returning its stable [`SourceId`]. + /// + /// Paths are deduplicated: registering the same path twice returns the same + /// id, and the first non-`None` `content` wins. + pub(crate) fn add_source(&mut self, path: &str, content: Option) -> SourceId { + if let Some(index) = self.sources.get_index_of(path) { + let id = index as SourceId; + if let Some(existing) = self.sources.get_index_mut(index) + && existing.1.is_none() + { + *existing.1 = content; + } + return id; + } + let id = self.sources.len() as SourceId; + self.sources.insert(path.to_owned(), content); + id + } + + /// Record that 0-based `generated_line` in the bundle originates from + /// 0-based `original_line` of `source_id`. + /// + /// Multiple records for the same generated line are allowed; the first one + /// added wins at serialization time (statement granularity means the first + /// statement starting on a line is the authoritative origin). + pub(crate) fn add_mapping( + &mut self, + generated_line: u32, + source_id: SourceId, + original_line: u32, + ) { + debug_assert!( + (source_id as usize) < self.sources.len(), + "add_mapping called with unregistered source_id {source_id}" + ); + self.mappings.push(Mapping { + generated_line, + source_id, + original_line, + }); + } + + /// Whether any mappings have been recorded. + #[cfg(test)] + pub(crate) const fn is_empty(&self) -> bool { + self.mappings.is_empty() + } + + /// Serialize to a Source Map v3 JSON string. + /// + /// Tokens are emitted sorted by generated line, one token per line (the + /// first mapping recorded for a line wins). `sourcesContent` is included iff + /// at least one source has content (the field is omitted entirely otherwise). + pub(crate) fn into_json(self) -> String { + let mut mappings = self.mappings; + // Stable sort: for duplicate generated lines the earliest insertion stays first. + mappings.sort_by_key(|m| m.generated_line); + mappings.dedup_by_key(|m| m.generated_line); + + let tokens: Vec = mappings + .iter() + .map(|m| { + Token::new( + m.generated_line, + 0, + m.original_line, + 0, + Some(m.source_id), + None, + ) + }) + .collect(); + + let sources: Vec> = self + .sources + .keys() + .map(|path| Cow::Borrowed(path.as_str())) + .collect(); + let source_contents: Vec>> = self + .sources + .values() + .map(|content| content.as_deref().map(Cow::Borrowed)) + .collect(); + + let map = SourceMap::new( + Some(Cow::Borrowed(self.file.as_str())), + Vec::new(), + None, + sources, + source_contents, + tokens.into_boxed_slice(), + None, + ); + map.to_json_string() + } +} + +/// One record produced by the parallel statement walk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct MappingRecord { + /// 0-based line in the generated bundle text. + pub(crate) generated_line: u32, + /// Ordinal of the originating module in the [`ProvenanceResolver`]. + pub(crate) module_ordinal: usize, + /// 0-based line in the original module source. + pub(crate) original_line: u32, +} + +/// Extract statement-level mappings by re-parsing the emitted bundle text and +/// walking it in parallel with the bundled AST (which carries node provenance). +/// +/// The re-parse doubles as a validity check of the emitted bundle: a parse +/// failure is returned as an error. Structural divergence between the two ASTs +/// (possible where post-generation string patching altered the code shape) is +/// handled defensively: the divergent subtree is skipped with a debug log. +pub(crate) fn extract_statement_mappings( + bundle_text: &str, + bundled_ast: &ModModule, + provenance: &ProvenanceResolver, +) -> anyhow::Result> { + let reparsed = ruff_python_parser::parse_module(bundle_text) + .context("bundled output failed to re-parse during source map extraction")? + .into_syntax(); + + let mut walker = ParallelWalker { + line_index: LineIndex::new(bundle_text), + provenance, + records: Vec::new(), + }; + walker.walk_body(&reparsed.body, &bundled_ast.body); + Ok(walker.records) +} + +/// Statement-only parallel traversal of the re-parsed bundle and the bundled AST. +struct ParallelWalker<'a> { + /// Line index over the emitted bundle text. + line_index: LineIndex, + provenance: &'a ProvenanceResolver, + records: Vec, +} + +impl ParallelWalker<'_> { + /// Walk two statement lists in lockstep; skip entirely on length divergence. + fn walk_body(&mut self, generated: &[Stmt], original: &[Stmt]) { + if generated.len() != original.len() { + log::debug!( + "source map: skipping diverged body (generated {} statements, bundled AST {})", + generated.len(), + original.len() + ); + return; + } + for (generated_stmt, original_stmt) in generated.iter().zip(original) { + self.walk_stmt(generated_stmt, original_stmt); + } + } + + /// Record a mapping for one aligned statement pair and recurse into nested bodies. + fn walk_stmt(&mut self, generated: &Stmt, original: &Stmt) { + if std::mem::discriminant(generated) != std::mem::discriminant(original) { + log::debug!("source map: skipping diverged statement pair"); + return; + } + + let stmt_provenance = self + .provenance + .resolve(original.node_index().load(), original.range().start()); + if let Some((module_ordinal, original_line)) = stmt_provenance { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(generated.range().start()), + module_ordinal, + original_line, + }); + } + // Note on multiline statements: ruff's generator emits every statement + // on a single physical line — multiline string/f-string literals are + // rendered with `\n` escapes, and docstrings likewise. There are + // therefore no interior physical lines to map; a raising expression + // inside such a literal is attributed to the statement's single + // generated line, which the record above already covers. + + // Evaluating a decorator can raise on its own `@...` line; give every + // decorator its own mapping (the statement mapping above only covers + // the `def`/`class` header). + let decorator_pairs = match (generated, original) { + (Stmt::FunctionDef(g), Stmt::FunctionDef(o)) => { + Some((&g.decorator_list, &o.decorator_list)) + } + (Stmt::ClassDef(g), Stmt::ClassDef(o)) => Some((&g.decorator_list, &o.decorator_list)), + _ => None, + }; + if let Some((gen_decorators, orig_decorators)) = decorator_pairs + && gen_decorators.len() == orig_decorators.len() + { + for (gen_decorator, orig_decorator) in gen_decorators.iter().zip(orig_decorators) { + if let Some((module_ordinal, original_line)) = self.provenance.resolve( + orig_decorator.node_index.load(), + orig_decorator.range().start(), + ) { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_decorator.range().start()), + module_ordinal, + original_line, + }); + } + } + // With decorators present, the statement range starts at the first + // `@...` line, so the mapping recorded above covers that decorator + // — the actual `def`/`class` header still needs its own record. + // The name identifier sits on the header line on both sides, but + // symbol renaming can regenerate the original identifier with a + // synthetic (default) range; only ranges inside the statement are + // trusted, with the parameter list as a fallback anchor. + let header_anchor = match (generated, original) { + (Stmt::FunctionDef(g), Stmt::FunctionDef(o)) => { + let orig_anchor = Some(o.name.range()) + .filter(|range| o.range().contains(range.start())) + .or_else(|| { + Some(o.parameters.range()) + .filter(|range| o.range().contains(range.start())) + }); + orig_anchor.map(|range| (g.name.range(), range, o.node_index().load())) + } + (Stmt::ClassDef(g), Stmt::ClassDef(o)) => { + // The inliner preserves the original identifier range on + // renamed class names, so the name anchor holds even for + // `class C:` headers with no argument list. The + // base-class / metaclass argument list stays as a + // fallback for any other transformation that regenerates + // the identifier with a synthetic (default) range. + let orig_anchor = Some(o.name.range()) + .filter(|range| o.range().contains(range.start())) + .or_else(|| { + o.arguments + .as_deref() + .map(ruff_text_size::Ranged::range) + .filter(|range| o.range().contains(range.start())) + }); + orig_anchor.map(|range| (g.name.range(), range, o.node_index().load())) + } + _ => None, + }; + if !gen_decorators.is_empty() + && let Some((gen_anchor, orig_anchor, orig_index)) = header_anchor + && let Some((module_ordinal, original_line)) = + self.provenance.resolve(orig_index, orig_anchor.start()) + { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_anchor.start()), + module_ordinal, + original_line, + }); + } + } + + // Recurse into nested statement bodies even when the statement itself is + // synthesized: wrapper-module init functions are synthesized `def`s whose + // bodies contain original module statements. + match (generated, original) { + (Stmt::FunctionDef(g), Stmt::FunctionDef(o)) => self.walk_body(&g.body, &o.body), + (Stmt::ClassDef(g), Stmt::ClassDef(o)) => self.walk_body(&g.body, &o.body), + (Stmt::If(g), Stmt::If(o)) => { + self.walk_body(&g.body, &o.body); + if g.elif_else_clauses.len() == o.elif_else_clauses.len() { + for (gen_clause, orig_clause) in + g.elif_else_clauses.iter().zip(&o.elif_else_clauses) + { + // An exception raised while evaluating an `elif` + // condition reports the clause header line, so the + // header needs its own mapping (provenance comes from + // the condition expression, which carries a node index). + if let (Some(_gen_test), Some(orig_test)) = + (&gen_clause.test, &orig_clause.test) + && let Some((module_ordinal, original_line)) = self + .provenance + .resolve(orig_test.node_index().load(), orig_clause.range().start()) + { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_clause.range().start()), + module_ordinal, + original_line, + }); + } + self.walk_body(&gen_clause.body, &orig_clause.body); + } + } + } + (Stmt::While(g), Stmt::While(o)) => { + self.walk_body(&g.body, &o.body); + self.walk_body(&g.orelse, &o.orelse); + } + (Stmt::For(g), Stmt::For(o)) => { + self.walk_body(&g.body, &o.body); + self.walk_body(&g.orelse, &o.orelse); + } + (Stmt::With(g), Stmt::With(o)) => self.walk_body(&g.body, &o.body), + (Stmt::Try(g), Stmt::Try(o)) => { + self.walk_body(&g.body, &o.body); + if g.handlers.len() == o.handlers.len() { + for (gen_handler, orig_handler) in g.handlers.iter().zip(&o.handlers) { + let ruff_python_ast::ExceptHandler::ExceptHandler(gen_handler) = + gen_handler; + let ruff_python_ast::ExceptHandler::ExceptHandler(orig_handler) = + orig_handler; + // Evaluating an exception matcher can itself raise, and + // Python reports the `except` header line; give the + // header its own mapping via the matcher expression's + // provenance. + if let (Some(_), Some(orig_type)) = + (&gen_handler.type_, &orig_handler.type_) + && let Some((module_ordinal, original_line)) = self.provenance.resolve( + orig_type.node_index().load(), + orig_handler.range().start(), + ) + { + self.records.push(MappingRecord { + generated_line: self + .line_index + .line_of(gen_handler.range().start()), + module_ordinal, + original_line, + }); + } + self.walk_body(&gen_handler.body, &orig_handler.body); + } + } + self.walk_body(&g.orelse, &o.orelse); + self.walk_body(&g.finalbody, &o.finalbody); + } + (Stmt::Match(g), Stmt::Match(o)) if g.cases.len() == o.cases.len() => { + for (gen_case, orig_case) in g.cases.iter().zip(&o.cases) { + // A raising pattern operation or guard is attributed to the + // `case` header line; map it via the case's provenance. + if let Some((module_ordinal, original_line)) = self + .provenance + .resolve(orig_case.node_index.load(), orig_case.range().start()) + { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_case.range().start()), + module_ordinal, + original_line, + }); + } + self.walk_body(&gen_case.body, &orig_case.body); + } + } + _ => {} + } + } +} + +/// Options for assembling a complete source map for a bundle. +#[derive(Debug)] +pub(crate) struct SourceMapOptions<'a> { + /// Value of the `file` field (the bundle's file name, or ``). + pub(crate) file: &'a str, + /// Whether to embed original source text as `sourcesContent`. + pub(crate) include_contents: bool, + /// Directory the map will live in; source paths are recorded relative to it. + /// When `None`, paths are recorded as given. + pub(crate) base_dir: Option<&'a std::path::Path>, +} + +/// Compute a relative path from `base` to `target` using lexical components +/// (no filesystem access). Falls back to `target` as-is when the two share no +/// common prefix that allows a relative form (e.g., different roots). +fn relative_path(base: &std::path::Path, target: &std::path::Path) -> std::path::PathBuf { + use std::path::Component; + + let mut base_components = base.components().peekable(); + let mut target_components = target.components().peekable(); + + // Drop the shared prefix. + while let (Some(b), Some(t)) = (base_components.peek(), target_components.peek()) { + if b == t { + base_components.next(); + target_components.next(); + } else { + break; + } + } + + let mut result = std::path::PathBuf::new(); + for component in base_components { + match component { + Component::Normal(_) => result.push(".."), + // A remaining root/prefix component means the paths have no common + // ancestor expressible relatively; a `..` component cannot be + // inverted lexically. In both cases keep the target as-is. + Component::RootDir | Component::Prefix(_) | Component::ParentDir => { + return target.to_path_buf(); + } + Component::CurDir => {} + } + } + result.extend(target_components); + result +} + +/// Comment linking the bundle to an adjacent source map file. +/// +/// The `sourceMappingURL` convention is borrowed from the JS ecosystem; Python +/// treats the line as a plain comment. The file name is percent-encoded so a +/// hostile or accidental control character (e.g. a newline in a Unix filename) +/// cannot terminate the comment and inject executable text into the bundle. +/// +/// Takes the name as `OsStr` because Unix filenames need not be UTF-8: invalid +/// bytes are percent-encoded verbatim (`%XX` of the raw byte), which is both +/// injection-safe and the faithful URL form of the on-disk name — a lossy +/// U+FFFD substitution would point consumers at a file that does not exist. +pub(crate) fn linked_source_mapping_comment(map_file_name: &std::ffi::OsStr) -> String { + let mut encoded = String::with_capacity(map_file_name.len()); + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt as _; + let mut bytes = map_file_name.as_bytes(); + while !bytes.is_empty() { + match std::str::from_utf8(bytes) { + Ok(valid) => { + encode_map_url_into(valid, &mut encoded); + break; + } + Err(err) => { + let (valid, rest) = bytes.split_at(err.valid_up_to()); + encode_map_url_into( + std::str::from_utf8(valid).expect("valid_up_to prefix is UTF-8"), + &mut encoded, + ); + // error_len is None only at end-of-input (truncated + // sequence); encode every remaining byte in that case. + let invalid_len = err.error_len().unwrap_or(rest.len()); + for byte in &rest[..invalid_len] { + let _ = + std::fmt::Write::write_fmt(&mut encoded, format_args!("%{byte:02X}")); + } + bytes = &rest[invalid_len..]; + } + } + } + } + #[cfg(not(unix))] + { + // Windows filenames are UTF-16; unpaired surrogates cannot be + // expressed in a UTF-8 comment at all, so lossy substitution is the + // only option there. + encode_map_url_into(&map_file_name.to_string_lossy(), &mut encoded); + } + format!("# sourceMappingURL={encoded}\n") +} + +/// Percent-encode `text` for a `sourceMappingURL` comment, appending to `out`. +fn encode_map_url_into(text: &str, out: &mut String) { + for character in text.chars() { + // Control characters would break out of the comment; '#', '?', and + // other URL delimiters would truncate the reference for + // standards-compliant URL consumers. Non-ASCII Unicode passes through + // verbatim. + if character < '\u{21}' + || character == '\u{7F}' + || matches!( + character, + '%' | '#' | '?' | '"' | '<' | '>' | '\\' | '^' | '`' | '|' + ) + { + let _ = std::fmt::Write::write_fmt(out, format_args!("%{:02X}", character as u32)); + } else { + out.push(character); + } + } +} + +/// Placeholder in the runtime template replaced with this build's map digest. +const RUNTIME_DIGEST_PLACEHOLDER: &str = "__CRIBO_SOURCEMAP_DIGEST__"; + +/// Bake the map's SHA-256 into the emitted bundle code. +/// +/// The digest lives inside the executing code itself (not in a trailer read +/// back from disk), so it is immune to any on-disk replacement of the bundle: +/// no interleaving of concurrent builds, manual file shuffling, or +/// redeploy-while-running can pair in-memory code objects with another build's +/// mappings. The placeholder sits inside a single-line string literal, so the +/// substitution never shifts line numbers and the extracted mappings stay +/// valid. With no map, the placeholder becomes an empty string, which the +/// runtime treats as "no digest known" (verification is skipped). +/// +/// Only the first occurrence is replaced: the prologue is emitted before any +/// user code, so its placeholder is always the first one in the bundle, and +/// user code that happens to contain the same spelling (in a string literal, +/// comment, or identifier) is left untouched. +pub(crate) fn apply_map_digest(code: &str, map_json: Option<&str>) -> String { + use cow_utils::CowUtils as _; + use sha2::{Digest as _, Sha256}; + + let digest_hex = map_json.map_or_else(String::new, |json| { + let digest = Sha256::digest(json.as_bytes()); + let mut hex = String::with_capacity(64); + for byte in digest { + let _ = std::fmt::Write::write_fmt(&mut hex, format_args!("{byte:02x}")); + } + hex + }); + code.cow_replacen(RUNTIME_DIGEST_PLACEHOLDER, &digest_hex, 1) + .into_owned() +} + +/// Comment embedding the source map as a base64 data URL. +pub(crate) fn inline_source_mapping_comment(map_json: &str) -> String { + let encoded = base64_simd::STANDARD.encode_to_string(map_json.as_bytes()); + format!("# sourceMappingURL=data:application/json;base64,{encoded}\n") +} + +/// The traceback-remapping runtime injected into bundles built with source maps. +/// +/// Lazy and duress-tolerant by design; see `docs/source-maps.md`. +const RUNTIME_TEMPLATE: &str = include_str!("python/sourcemap_runtime.py"); + +/// Placeholder in the runtime template replaced with the delivery mode. +const RUNTIME_MODE_PLACEHOLDER: &str = "__CRIBO_SOURCEMAP_MODE__"; + +/// Inject the traceback-remapping runtime prologue into the bundled AST. +/// +/// Statements are inserted after any leading `from __future__` imports (which +/// must stay first) and before all other code, so the hooks install before any +/// user code runs. The parsed statements carry no node provenance, so the +/// mapping walk transparently skips them while staying structurally aligned. +/// +/// A template parse failure is a cribo bug; it degrades gracefully (warning, +/// no runtime) rather than failing the bundle. +pub(crate) fn inject_runtime_prologue( + bundled_ast: &mut ModModule, + mode: crate::config::SourceMapMode, +) { + use cow_utils::CowUtils as _; + + let mode_str = match mode { + crate::config::SourceMapMode::Linked => "linked", + crate::config::SourceMapMode::Inline => "inline", + crate::config::SourceMapMode::External => "external", + }; + let source = RUNTIME_TEMPLATE.cow_replace(RUNTIME_MODE_PLACEHOLDER, mode_str); + match ruff_python_parser::parse_module(&source) { + Ok(parsed) => { + let mut statements = parsed.into_syntax().body; + // Drop the template's leading docstring: injected near position + // zero it could otherwise become the bundle's module docstring and + // change the program's observable `__doc__`. + if statements.first().is_some_and(is_docstring) { + statements.remove(0); + } + // Insert after the bundle's own docstring (which must stay first to + // remain `__doc__`) and after any `from __future__` imports (which + // must precede all other code). + let mut insert_at = usize::from(bundled_ast.body.first().is_some_and(is_docstring)); + insert_at += bundled_ast.body[insert_at..] + .iter() + .take_while(|stmt| is_future_import(stmt)) + .count(); + bundled_ast.body.splice(insert_at..insert_at, statements); + } + Err(err) => log::warn!( + "source map runtime template failed to parse; traceback remapping disabled: {err}" + ), + } +} + +/// Whether a statement is a `from __future__ import ...`. +fn is_future_import(stmt: &Stmt) -> bool { + matches!( + stmt, + Stmt::ImportFrom(import) + if import.module.as_ref().is_some_and(|module| module.as_str() == "__future__") + ) +} + +/// Whether a statement is a bare string-literal expression (a docstring when leading). +fn is_docstring(stmt: &Stmt) -> bool { + matches!(stmt, Stmt::Expr(expr) if expr.value.is_string_literal_expr()) +} + +/// Percent-encode a source path for the `sources` array. +/// +/// Source Map v3 consumers resolve `sources` entries as URL references, so a +/// raw `#` or `?` in a filesystem name would be read as fragment/query +/// delimiters and point at the wrong resource. Only URL-breaking characters +/// are escaped; everything else (including non-ASCII) passes through. The +/// injected runtime applies the inverse decoding before filesystem access. +fn percent_encode_source(path: &str) -> String { + let mut encoded = String::with_capacity(path.len()); + for character in path.chars() { + // On Windows the native separator is '\'; URL consumers only treat + // '/' as a path separator, so normalize. (On Unix a backslash is a + // legal filename character and passes through untouched.) + #[cfg(windows)] + if character == '\\' { + encoded.push('/'); + continue; + } + if character < '\u{20}' || matches!(character, '\u{7F}' | '%' | '#' | '?') { + let _ = + std::fmt::Write::write_fmt(&mut encoded, format_args!("%{:02X}", character as u32)); + } else { + encoded.push(character); + } + } + encoded +} + +/// Build the complete Source Map v3 JSON for an emitted bundle. +/// +/// Re-parses `bundle_text`, extracts statement mappings against `bundled_ast`, +/// and serializes them. Only modules that actually contributed mappings appear +/// in the `sources` array. +pub(crate) fn build_source_map( + bundle_text: &str, + bundled_ast: &ModModule, + provenance: &ProvenanceResolver, + options: &SourceMapOptions<'_>, +) -> anyhow::Result { + let records = extract_statement_mappings(bundle_text, bundled_ast, provenance)?; + + let mut generator = SourceMapGenerator::new(options.file); + // None = not yet seen; Some(None) = seen but unmappable (non-UTF-8 path). + let mut ordinal_to_source: Vec>> = + vec![None; provenance.modules().len()]; + + for record in records { + let module = &provenance.modules()[record.module_ordinal]; + let source_id = ordinal_to_source[record.module_ordinal].unwrap_or_else(|| { + let display_path = options.base_dir.map_or_else( + || module.path.clone(), + |base| relative_path(base, &module.path), + ); + // Source Map v3 is JSON: a path that is not valid UTF-8 has no + // lossless representation. A lossy rendering would display a + // nonexistent replacement-character path and could collide with a + // *different* non-UTF-8 path, mispairing mappings and + // sourcesContent — skip such modules instead (their frames stay on + // bundle coordinates). + let resolved = display_path.to_str().map_or_else( + || { + log::debug!( + "source map: skipping module with non-UTF-8 path: {}", + display_path.display() + ); + None + }, + |display| { + let content = options.include_contents.then(|| module.source.clone()); + Some(generator.add_source(&percent_encode_source(display), content)) + }, + ); + ordinal_to_source[record.module_ordinal] = Some(resolved); + resolved + }); + if let Some(source_id) = source_id { + generator.add_mapping(record.generated_line, source_id, record.original_line); + } + } + + Ok(generator.into_json()) +} + +#[cfg(test)] +mod tests { + use oxc_sourcemap::SourceMap; + use ruff_python_ast::Stmt; + + use super::*; + + #[test] + fn empty_map_is_valid_v3() { + let generator = SourceMapGenerator::new("bundle.py"); + assert!(generator.is_empty()); + let json = generator.into_json(); + + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_file(), Some("bundle.py")); + assert_eq!(parsed.get_tokens().count(), 0); + assert!(json.contains("\"version\":3")); + assert!(!json.contains("sourcesContent")); + } + + #[test] + fn mappings_round_trip_through_vlq() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let main = generator.add_source("main.py", None); + let utils = generator.add_source("utils.py", None); + // Insert out of order to exercise the sort. + generator.add_mapping(10, utils, 3); + generator.add_mapping(4, main, 0); + generator.add_mapping(7, utils, 1); + let json = generator.into_json(); + + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + let lookup = parsed.generate_lookup_table(); + + let token = parsed + .lookup_token(&lookup, 4, 0) + .expect("mapping for line 4"); + assert_eq!( + parsed.get_source(token.get_source_id().expect("source id")), + Some("main.py") + ); + assert_eq!(token.get_src_line(), 0); + + let token = parsed + .lookup_token(&lookup, 10, 0) + .expect("mapping for line 10"); + assert_eq!( + parsed.get_source(token.get_source_id().expect("source id")), + Some("utils.py") + ); + assert_eq!(token.get_src_line(), 3); + + // An unmapped line before the first token has no mapping. + assert!(parsed.lookup_token(&lookup, 0, 0).is_none()); + } + + #[test] + fn source_dedup_returns_same_id_and_first_content_wins() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let first = generator.add_source("pkg/mod.py", Some("x = 1\n".to_owned())); + let second = generator.add_source("pkg/mod.py", Some("ignored".to_owned())); + assert_eq!(first, second); + + generator.add_mapping(0, first, 0); + let json = generator.into_json(); + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_sources().count(), 1); + assert_eq!(parsed.get_source_content(first), Some("x = 1\n")); + } + + #[test] + fn content_backfills_when_first_registration_had_none() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let id = generator.add_source("mod.py", None); + let same = generator.add_source("mod.py", Some("y = 2\n".to_owned())); + assert_eq!(id, same); + + generator.add_mapping(0, id, 0); + let json = generator.into_json(); + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_source_content(id), Some("y = 2\n")); + } + + #[test] + fn sources_content_present_iff_any_content() { + let mut with_content = SourceMapGenerator::new("bundle.py"); + let id = with_content.add_source("a.py", Some("a = 1\n".to_owned())); + with_content.add_source("b.py", None); + with_content.add_mapping(0, id, 0); + let json = with_content.into_json(); + assert!(json.contains("sourcesContent")); + assert!(json.contains("null"), "missing content must encode as null"); + + let mut without_content = SourceMapGenerator::new("bundle.py"); + let id = without_content.add_source("a.py", None); + without_content.add_mapping(0, id, 0); + assert!(!without_content.into_json().contains("sourcesContent")); + } + + #[test] + fn first_mapping_per_generated_line_wins() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let first = generator.add_source("first.py", None); + let second = generator.add_source("second.py", None); + generator.add_mapping(5, first, 11); + generator.add_mapping(5, second, 99); + let json = generator.into_json(); + + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_tokens().count(), 1); + let lookup = parsed.generate_lookup_table(); + let token = parsed + .lookup_token(&lookup, 5, 0) + .expect("mapping for line 5"); + assert_eq!( + parsed.get_source(token.get_source_id().expect("source id")), + Some("first.py") + ); + assert_eq!(token.get_src_line(), 11); + } + + #[test] + fn line_index_maps_offsets_to_lines() { + let index = LineIndex::new("a = 1\nb = 2\n\nc = 3\n"); + assert_eq!(index.line_of(TextSize::from(0)), 0); // 'a' + assert_eq!(index.line_of(TextSize::from(5)), 0); // the '\n' itself + assert_eq!(index.line_of(TextSize::from(6)), 1); // 'b' + assert_eq!(index.line_of(TextSize::from(12)), 2); // empty line + assert_eq!(index.line_of(TextSize::from(13)), 3); // 'c' + } + + #[test] + fn line_index_handles_source_without_trailing_newline() { + let index = LineIndex::new("x = 1"); + assert_eq!(index.line_of(TextSize::from(0)), 0); + assert_eq!(index.line_of(TextSize::from(4)), 0); + } + + #[test] + fn provenance_resolves_nodes_across_modules() { + use ruff_text_size::Ranged as _; + + let source_a = "a1 = 1\na2 = 2\n"; + let source_b = "def f():\n return 3\n\nb2 = f()\n"; + let mut ast_a = ruff_python_parser::parse_module(source_a) + .expect("parse module a") + .into_syntax(); + let mut ast_b = ruff_python_parser::parse_module(source_b) + .expect("parse module b") + .into_syntax(); + crate::ast_indexer::index_module_with_id(&mut ast_a, 0); + crate::ast_indexer::index_module_with_id(&mut ast_b, 1); + + let mut resolver = ProvenanceResolver::default(); + resolver.push_module(std::path::PathBuf::from("a.py"), source_a.to_owned()); + resolver.push_module(std::path::PathBuf::from("b.py"), source_b.to_owned()); + + // Second statement of module a: `a2 = 2` on line 1. + let stmt = &ast_a.body[1]; + let Stmt::Assign(assign) = stmt else { + panic!("expected assign statement"); + }; + let resolved = resolver.resolve(assign.node_index.load(), stmt.range().start()); + assert_eq!(resolved, Some((0, 1))); + + // The `return 3` statement nested inside `def f()` of module b: line 1. + let Stmt::FunctionDef(func) = &ast_b.body[0] else { + panic!("expected function def"); + }; + let ret = &func.body[0]; + let Stmt::Return(ret_stmt) = ret else { + panic!("expected return statement"); + }; + let resolved = resolver.resolve(ret_stmt.node_index.load(), ret.range().start()); + assert_eq!(resolved, Some((1, 1))); + + // Last statement of module b: `b2 = f()` on line 3. + let Stmt::Assign(assign) = &ast_b.body[1] else { + panic!("expected assign statement"); + }; + let resolved = resolver.resolve(assign.node_index.load(), ast_b.body[1].range().start()); + assert_eq!(resolved, Some((1, 3))); + } + + #[test] + fn provenance_rejects_synthesized_nodes() { + use ruff_python_ast::AtomicNodeIndex; + use ruff_text_size::TextRange; + + let mut resolver = ProvenanceResolver::default(); + resolver.push_module(std::path::PathBuf::from("a.py"), "x = 1\n".to_owned()); + + // A node built by ast_builder carries the NONE placeholder index. + let synthesized = crate::ast_builder::statements::pass(); + let Stmt::Pass(pass) = &synthesized else { + panic!("expected pass statement"); + }; + assert_eq!( + resolver.resolve(pass.node_index.load(), TextRange::default().start()), + None + ); + + // A node index past all module ranges (TransformationContext territory) + // also resolves to None. + let index = AtomicNodeIndex::default(); + index.set(NodeIndex::from(MODULE_INDEX_RANGE)); + assert_eq!(resolver.resolve(index.load(), TextSize::from(0)), None); + } +} diff --git a/crates/cribo/tests/fixtures/sourcemap_basic/calculator.py b/crates/cribo/tests/fixtures/sourcemap_basic/calculator.py new file mode 100644 index 000000000..0665c86b1 --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_basic/calculator.py @@ -0,0 +1,8 @@ +def add(a, b): + result = a + b + return result + + +def multiply(a, b): + result = a * b + return result diff --git a/crates/cribo/tests/fixtures/sourcemap_basic/main.py b/crates/cribo/tests/fixtures/sourcemap_basic/main.py new file mode 100644 index 000000000..59e4de30e --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_basic/main.py @@ -0,0 +1,7 @@ +from calculator import add, multiply +from utils import describe + +total = add(2, 3) +product = multiply(total, 4) +print(describe("total", total)) +print(describe("product", product)) diff --git a/crates/cribo/tests/fixtures/sourcemap_basic/utils.py b/crates/cribo/tests/fixtures/sourcemap_basic/utils.py new file mode 100644 index 000000000..37121d48a --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_basic/utils.py @@ -0,0 +1,2 @@ +def describe(name, value): + return f"{name} = {value}" diff --git a/crates/cribo/tests/fixtures/sourcemap_wrapper/effects.py b/crates/cribo/tests/fixtures/sourcemap_wrapper/effects.py new file mode 100644 index 000000000..a7fefa262 --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_wrapper/effects.py @@ -0,0 +1,8 @@ +print("effects module loading") + +COUNTER = 1 + + +def boost(value): + boosted = value * 2 + COUNTER + return boosted diff --git a/crates/cribo/tests/fixtures/sourcemap_wrapper/main.py b/crates/cribo/tests/fixtures/sourcemap_wrapper/main.py new file mode 100644 index 000000000..1da29be74 --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_wrapper/main.py @@ -0,0 +1,4 @@ +import effects + +print("counter:", effects.COUNTER) +print("boosted:", effects.boost(10)) diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py new file mode 100644 index 000000000..504b957d4 --- /dev/null +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -0,0 +1,286 @@ +"""Unit tests for the cribo source map runtime internals. + +Driven by the Rust integration test `python_runtime_unit_tests` (plain asserts, +no pytest dependency). Usage: python test_sourcemap_runtime.py +where is the template with the mode placeholder substituted. + +Tests are discovered automatically: every module-level callable whose name +starts with ``test_`` runs once, receiving the runtime instance. +""" + +import importlib.util +import os +import sys +import tempfile +import threading + + +def load_runtime(path): + """Import the runtime module and return a runtime instance for testing. + + The import installs the hooks; they are restored immediately so failures + in this harness surface as normal tracebacks. The runtime class is + recovered from the installed hook's bound instance — the template's last + statement deletes every module-level helper name, so the module namespace + is intentionally empty after import. + """ + prev_hooks = (sys.excepthook, sys.unraisablehook, threading.excepthook) + spec = importlib.util.spec_from_file_location("cribo_sm_runtime", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + installed_hook = sys.excepthook + sys.excepthook, sys.unraisablehook, threading.excepthook = prev_hooks + assert not hasattr(module, "_CriboSourceMapRuntime"), ( + "template must scrub its helper names from the bundle namespace" + ) + runtime_cls = type(installed_hook.__self__) + return runtime_cls( + "external", + "", + "", # no build digest: verification is skipped + os, + __import__("binascii"), + threading, + __import__("traceback"), + __import__("hashlib"), + ) + + +def test_stream_reads_across_chunk_boundaries(rt): + stream = rt._stream_cls([b"ab", b"", b"c", b"de"]) + got = [] + while True: + byte = stream.read_byte() + if byte < 0: + break + got.append(chr(byte)) + assert got == list("abcde"), got + assert stream.read_byte() == -1 # stays exhausted + + +def _scan(rt, json_text, needed, max_needed): + return rt._scan(lambda: [json_text.encode("utf-8")], needed, max_needed) + + +def test_scan_extracts_sources_and_mappings(rt): + # AAAA;AACA;AACA: one segment per line, source line advancing by one. + json_text = '{"version":3,"sources":["a.py","b.py"],"mappings":"AAAA;AACA;AACA"}' + sources, table = _scan(rt, json_text, {0, 2}, 2) + assert sources == {0: "a.py"}, sources # only referenced indices collected + assert table == {0: (0, 0), 2: (0, 2)}, table + + +def test_scan_negative_delta(rt): + # Line 0 advances src_line by +1 (C), line 1 rewinds it by -1 (D). + json_text = '{"sources":["a.py"],"mappings":"AACA;AADA"}' + _sources, table = _scan(rt, json_text, {0, 1}, 1) + assert table == {0: (0, 1), 1: (0, 0)}, table + + +def test_scan_source_index_delta(rt): + # Second line switches to source 1 (C in field 1). + json_text = '{"sources":["a.py","b.py"],"mappings":"AAAA;ACAA"}' + _sources, table = _scan(rt, json_text, {0, 1}, 1) + assert table == {0: (0, 0), 1: (1, 0)}, table + + +def test_scan_ignores_adversarial_sources_content(rt): + # sourcesContent value contains fake keys and escaped quotes; the string + # lexer must skip it without being fooled. + evil = '\\"mappings\\": \\"ZZZZ\\", \\\\' + json_text = ( + '{"sources":["a.py"],"sourcesContent":["' + evil + '"],"mappings":"AAAA"}' + ) + sources, table = _scan(rt, json_text, {0}, 0) + assert sources == {0: "a.py"}, sources + assert table == {0: (0, 0)}, table + + +def test_scan_handles_unicode_escapes_in_sources(rt): + json_text = '{"sources":["\\u00e9t\\u00e9.py"],"mappings":"AAAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources == {0: "\u00e9t\u00e9.py"}, sources + + +def test_scan_null_in_sources_array(rt): + json_text = '{"sources":["a.py",null,"c.py"],"mappings":"ACAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources == {}, sources # a null entry is simply not collected + + +def test_vlq_early_exit_stops_reading(rt): + # The decoder must stop pulling chunks once past max_needed: the second + # chunk raises if consumed. + class Boom(Exception): + pass + + def chunks_factory(): + yield b'{"sources":["a.py"],"mappings":"AAAA;AACA;' + raise Boom("decoder read past its early-exit point") + + _sources, table = rt._scan(chunks_factory, {0}, 0) + assert table == {0: (0, 0)}, table + + +def test_vlq_rejects_escapes_in_mappings(rt): + json_text = '{"sources":["a.py"],"mappings":"AA\\\\AA"}' + try: + _scan(rt, json_text, {0}, 0) + except ValueError: + pass + else: + raise AssertionError("escape inside mappings must raise") + + +def test_scan_handles_mappings_before_sources(rt): + # A spec-valid map may order keys arbitrarily. Early exit inside the + # mappings string must not derail parsing of a later sources field. + json_text = '{"mappings":"AAAA;AACA;AACA","sources":["a.py","b.py"]}' + sources, table = _scan(rt, json_text, {0}, 0) # early exit after line 0 + assert sources == {0: "a.py"}, sources + assert table == {0: (0, 0)}, table + + +def test_scan_combines_surrogate_pairs(rt): + # Non-BMP characters arrive as JSON surrogate pairs; they must decode to + # one code point, not two replacement characters. + json_text = '{"sources":["\\ud83d\\ude00.py"],"mappings":"AAAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources == {0: "\U0001f600.py"}, sources + # A lone high surrogate followed by a plain character stays recoverable + # (replacement character), and the rest of the string is intact. + json_text = '{"sources":["\\ud83dx.py"],"mappings":"AAAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources[0].endswith("x.py"), sources + + +def make_inline_bundle(payload_json): + """Create a temp file shaped like an inline-mode bundle; return its path.""" + import base64 + + encoded = base64.b64encode(payload_json.encode("utf-8")).decode("ascii") + handle = tempfile.NamedTemporaryFile( + "w", suffix=".py", delete=False, encoding="utf-8" + ) + with handle as f: + f.write("print('hello')\n" * 300) # push the marker past one chunk + f.write("# sourceMappingURL=data:application/json;base64," + encoded + "\n") + return handle.name + + +def test_inline_payload_scan_and_chunked_base64(rt): + # Payload much larger than one 8 KiB chunk exercises 4-byte alignment + # handling across chunk boundaries. + filler = "x" * 40000 + json_text = '{"filler":"' + filler + '","sources":["a.py"],"mappings":"AAAA"}' + path = make_inline_bundle(json_text) + try: + with open(path, "rb") as handle: + decoded = b"".join(rt._inline_chunks(handle)) + assert decoded.decode("utf-8") == json_text + # And end-to-end through the scanner (one pinned handle, two passes): + with open(path, "rb") as handle: + sources, table = rt._scan(lambda: rt._inline_chunks(handle), {0}, 0) + assert sources == {0: "a.py"}, sources + assert table == {0: (0, 0)}, table + finally: + os.unlink(path) + + +def test_inline_scan_without_marker_yields_nothing(rt): + handle = tempfile.NamedTemporaryFile( + "w", suffix=".py", delete=False, encoding="utf-8" + ) + with handle as f: + f.write("print('no map here')\n" * 50) + try: + with open(handle.name, "rb") as bundle: + assert b"".join(rt._inline_chunks(bundle)) == b"" + finally: + os.unlink(handle.name) + + +def test_json_fallback_matches_streaming(rt): + json_text = '{"sources":["a.py","b.py"],"mappings":"AAAA;ACCA"}' + streaming = _scan(rt, json_text, {0, 1}, 1) + + handle = tempfile.NamedTemporaryFile( + "w", suffix=".map", delete=False, encoding="utf-8" + ) + with handle as f: + f.write(json_text) + path = handle.name + previous = os.environ.get("CRIBO_SOURCE_MAPS") + try: + os.environ["CRIBO_SOURCE_MAPS"] = path + loaded = rt._load_json_fallback({1, 2}) + assert loaded is not None + table, sources, _map_dir = loaded + # Fallback tables are 1-based. + expected = { + line0 + 1: (idx, src_line0 + 1) + for line0, (idx, src_line0) in streaming[1].items() + } + assert table == expected, (table, expected) + assert sources == streaming[0] + finally: + if previous is None: + os.environ.pop("CRIBO_SOURCE_MAPS", None) + else: + os.environ["CRIBO_SOURCE_MAPS"] = previous + os.unlink(path) + + +def test_env_path_wins_for_every_mode(rt): + # A CRIBO_SOURCE_MAPS path activates the runtime even for a bundle + # (the stdin piping workflow cannot re-read its own inline map). + handle = tempfile.NamedTemporaryFile( + "w", suffix=".map", delete=False, encoding="utf-8" + ) + with handle as f: + f.write('{"sources":["a.py"],"mappings":"AAAA"}') + path = handle.name + previous = os.environ.get("CRIBO_SOURCE_MAPS") + try: + os.environ["CRIBO_SOURCE_MAPS"] = path + inline_stdin = type(rt)( + "inline", + "", + "", + os, + __import__("binascii"), + threading, + __import__("traceback"), + __import__("hashlib"), + ) + loaded = inline_stdin._load({1}) + assert loaded is not None, "env path must activate a inline bundle" + table, sources, _map_dir = loaded + assert sources == {0: "a.py"}, sources + assert table == {1: (0, 1)}, table + # Without the env override, a inline bundle stays inactive. + os.environ.pop("CRIBO_SOURCE_MAPS", None) + assert inline_stdin._map_location() is None + finally: + if previous is None: + os.environ.pop("CRIBO_SOURCE_MAPS", None) + else: + os.environ["CRIBO_SOURCE_MAPS"] = previous + os.unlink(path) + + +def main(): + runtime_path = sys.argv[1] + rt = load_runtime(runtime_path) + tests = sorted( + (obj for name, obj in globals().items() if name.startswith("test_") and callable(obj)), + key=lambda obj: obj.__name__, + ) + for test in tests: + test(rt) + print("PASS %s" % test.__name__) + print("ALL %d RUNTIME TESTS PASSED" % len(tests)) + + +if __name__ == "__main__": + main() diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap new file mode 100644 index 000000000..38f2fb2c8 --- /dev/null +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -0,0 +1,1029 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +#!/usr/bin/env python3 +# Generated by Cribo - Python Source Bundler +# https://github.com/ophidiarium/cribo + +import sys as _cribo_sys +def _cribo_sm_import(name, script_path=None, *, _sys=_cribo_sys, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed from a private\n *snapshot* of the search path. Resolution goes through the frozen importer\n machinery (`PathFinder.find_spec` with that explicit snapshot), so global\n `sys.path` is never mutated and concurrent bootstraps cannot race each\n other's save/restore. Lexical de-duplication works even before `os` is\n importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. Non-string entries are kept untouched.\n (`sys` and the frozen importlib modules are builtins and can never be\n shadowed.)\n """ + module = _sys.modules.get(name) + if module is not None: + return module + saved_path = _list(_sys.path) + candidates = [] + if saved_path and _isinstance(saved_path[0], _str): + candidates.append(saved_path[0]) + if script_path and _isinstance(script_path, _str): + cut = script_path.rfind("/") + cut_windows = script_path.rfind("\\") + if cut_windows > cut: + cut = cut_windows + candidates.append(script_path[:cut] if cut >= 0 else ".") + trimmed = [] + for candidate in candidates: + trimmed.append(candidate.rstrip("/\\") or candidate) + filtered = [] + for entry in saved_path[1:]: + if _isinstance(entry, _str): + if entry in ("", ".", "./"): + continue + stripped = entry.rstrip("/\\") or entry + if stripped in trimmed: + continue + filtered.append(entry) + os_mod = _sys.modules.get("os") + if os_mod is not None and trimmed: + resolved = [] + for candidate in trimmed: + try: + resolved.append(os_mod.path.normcase(os_mod.path.abspath(candidate or "."))) + except _bex: + pass + kept = [] + for entry in filtered: + if _isinstance(entry, _str): + try: + if os_mod.path.normcase(os_mod.path.abspath(entry or ".")) in resolved: + continue + except _bex: + pass + kept.append(entry) + filtered = kept + frozen_external = _sys.modules.get("_frozen_importlib_external") + frozen_bootstrap = _sys.modules.get("_frozen_importlib") + if frozen_external is not None and frozen_bootstrap is not None: + spec = frozen_bootstrap.BuiltinImporter.find_spec(name) or frozen_bootstrap.FrozenImporter.find_spec(name) or frozen_external.PathFinder.find_spec(name, filtered) + if spec is None: + raise ImportError("cribo runtime could not resolve " + name) + module = frozen_bootstrap.module_from_spec(spec) + _sys.modules[name] = module + try: + spec.loader.exec_module(module) + except _bex: + _sys.modules.pop(name, None) + raise + return module + _sys.path = filtered + try: + return _import(name) + finally: + _sys.path = saved_path +class _CriboSmStream(object): + """Byte-at-a-time reader over an iterator of byte chunks.\n\n Keyword-only defaults snapshot the builtins at definition time (before any\n bundled user code runs), so later shadowing of e.g. `len` cannot break the\n reader — the same idiom cribo's generated module proxies use.\n """ + __slots__ = "_chunks", "_buf", "_pos" + + def __init__(self, chunks, *, _iter=iter): + self._chunks = _iter(chunks) + self._buf = b"" + self._pos = 0 + + def read_byte(self, *, _len=len, _next=next, _stop=StopIteration): + while self._pos >= _len(self._buf): + try: + self._buf = _next(self._chunks) + except _stop: + return -1 + self._pos = 0 + value = self._buf[self._pos] + self._pos += 1 + return value +class _CriboSourceMapRuntime(object): + """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, and every method snapshots the builtins\n it needs via keyword-only defaults evaluated at class-definition time — so\n the installed hooks keep working even if bundled user code later rebinds\n any module-level name this template introduced, or common builtins such as\n `open`, `len`, or `max`.\n """ + _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + _CHUNK = 8192 + + def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod, *, _sys=_cribo_sys, _stream_cls=_CriboSmStream, _sm_import=_cribo_sm_import): + self._mode = mode + self._bundle = bundle_file + if bundle_file == "": + self._bundle_anchor = bundle_file + else: + self._bundle_anchor = os_mod.path.abspath(bundle_file) + self._expected_digest = None + try: + if isinstance(expected_digest, str) and len(expected_digest) == 64: + int(expected_digest, 16) + self._expected_digest = expected_digest.lower() + except ValueError: + self._expected_digest = None + self._cribo_sm_runtime_marker = True + try: + self._startup_cwd = os_mod.getcwd() + except OSError: + self._startup_cwd = "." + self._os = os_mod + self._sys = _sys + self._binascii = binascii_mod + self._threading = threading_mod + self._stream_cls = _stream_cls + self._import = _sm_import + self._traceback = traceback_mod + self._hashlib = hashlib_mod + self._local = threading_mod.local() + self._prev_excepthook = _sys.excepthook + self._prev_unraisablehook = _sys.unraisablehook + self._prev_threading_hook = threading_mod.excepthook + self._default_excepthook = _sys.__excepthook__ + self._default_unraisablehook = _sys.__unraisablehook__ + self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) + try: + self._group_type = BaseExceptionGroup + except NameError: + self._group_type = None + + def install(self): + """Install the three hooks; the previous hooks stay chained.\n\n Also joins the interpreter-wide runtime registry (stored on the\n unshadowable `sys` module) so stacked bundles can merge each other's\n mappings when a traceback crosses bundle boundaries.\n """ + registry = getattr(self._sys, "_cribo_sm_runtimes", None) + if not isinstance(registry, list): + registry = [] + self._sys._cribo_sm_runtimes = registry + registry.append(self) + self._sys.excepthook = self.excepthook + self._sys.unraisablehook = self.unraisablehook + self._threading.excepthook = self.threading_hook + + @classmethod + def _bootstrap(cls, mode, bundle_file, expected_digest, *, _sm_import=_cribo_sm_import): + """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ + try: + script = None if bundle_file == "" else bundle_file + runtime = cls(mode, bundle_file, expected_digest, _sm_import("os", script), _sm_import("binascii", script), _sm_import("threading", script), _sm_import("traceback", script), _sm_import("hashlib", script)) + runtime.install() + except BaseException: + pass + + def _map_location(self): + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir, verify); map_path is None for inline mode\n (the map lives inside the bundle file itself) and `verify` says whether\n the sibling map must match the digest recorded in the bundle. Called\n lazily at hook-fire time so the happy path never touches the\n environment or the filesystem.\n """ + env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + if env not in ("", "1", "true", "yes", "on"): + path = env + if not self._os.path.isabs(path): + path = self._os.path.join(self._startup_cwd, path) + return path, self._os.path.dirname(self._os.path.abspath(path)), False + bundle = self._bundle_anchor + if self._mode == "inline": + if bundle == "": + return None + return None, self._os.path.dirname(bundle), True + sibling = bundle + ".map" + if self._mode == "linked": + if self._os.path.exists(sibling): + return sibling, self._os.path.dirname(sibling), True + return None + if env in ("1", "true", "yes", "on"): + return sibling, self._os.path.dirname(sibling), True + return None + + def _file_chunks(self, path, *, _open=open): + """Yield fixed-size chunks of a file (constant memory).""" + handle = _open(path, "rb") + try: + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() + + def _handle_chunks(self, handle): + """Yield fixed-size chunks from an already-open handle, from offset 0.\n\n Keeping one open handle across the verify/decode passes pins the inode:\n a concurrent build renaming a new map into place cannot swap the bytes\n between passes.\n """ + handle.seek(0) + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + + def _find_marker_tail(self, handle, marker, *, _len=len): + """Backward-scan a file for the last occurrence of `marker`.\n\n Returns the offset of the payload following the marker, or -1. Only the\n tail of the file is examined; the body is never read.\n """ + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + while position > 0: + step = self._CHUNK if position >= self._CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + return position + index + _len(marker) + overlap = data[:_len(marker) - 1] + return -1 + + def _find_inline_payload(self, handle): + """Offset of the inline map's base64 payload in the bundle, or -1.""" + found = self._find_marker_tail(handle, b"# sourceMappingURL=data:") + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + 7 + + def _map_matches_digest(self, chunks_factory, *, _bex=BaseException): + """False only when a build digest is known and the map bytes disagree.\n\n The SHA-256 of this build's map is baked into the executing code at\n build time, so it is immune to any on-disk replacement of the bundle —\n no interleaving of concurrent builds, manual file shuffling, or\n redeploy-while-running can pair these code objects with another\n build's mappings. Hashing streams the same chunk source later used for\n decoding. Verification errors fail open (the subsequent map read would\n surface them anyway).\n """ + try: + expected = self._expected_digest + if expected is None: + return True + hasher = self._hashlib.sha256() + for chunk in chunks_factory(): + hasher.update(chunk) + return hasher.hexdigest() == expected + except _bex: + return True + + def _inline_chunks(self, handle, *, _len=len): + """Yield decoded chunks of an inline (base64 data URL) source map.\n\n Reads from an already-open bundle handle so all passes see one inode.\n """ + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = _len(data) - _len(data) % 4 + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) + + def _skip_ws(self, stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte + + def _read_hex4(self, stream, *, _int=int, _chr=chr, _range=range, _error=ValueError): + """Read the four hex digits of a \\uXXXX escape; return the code unit.""" + code = 0 + for _ in _range(4): + digit = stream.read_byte() + if digit < 0: + raise _error("unterminated unicode escape") + code = code * 16 + _int(_chr(digit), 16) + return code + + def _read_string(self, stream, collect, *, _bytearray=bytearray, _chr=chr, _error=ValueError): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes, \\uXXXX sequences, and UTF-16\n surrogate pairs (non-BMP characters) are handled, so string *values*\n containing text like '\"mappings\":' cannot confuse the key scanner and\n emoji-bearing paths survive intact.\n """ + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf = _bytearray() if collect else None + pending = None + while True: + if pending is None: + byte = stream.read_byte() + else: + byte, pending = pending, None + if byte < 0: + raise _error("unterminated JSON string") + if byte == 34: + return buf.decode("utf-8", "replace") if collect else None + if not byte == 92: + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise _error("unterminated JSON escape") + if not escape == 117: + if buf is not None: + buf.append(table.get(escape, escape)) + continue + code = self._read_hex4(stream) + if buf is None: + continue + if 55296 <= code <= 56319: + nxt = stream.read_byte() + if nxt == 92: + escape2 = stream.read_byte() + if escape2 == 117: + low = self._read_hex4(stream) + if 56320 <= low <= 57343: + code = 65536 + (code - 55296 << 10) + (low - 56320) + buf.extend(_chr(code).encode("utf-8")) + else: + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(low).encode("utf-8", "surrogatepass")) + continue + if escape2 < 0: + raise _error("unterminated JSON escape") + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.append(table.get(escape2, escape2)) + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + pending = nxt + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + + def _skip_value(self, stream, byte, *, _error=ValueError): + """Skip one JSON value; return the first byte after it (or -1).""" + if byte == 34: + self._read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise _error("unterminated JSON container") + if byte == 34: + self._read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + while byte >= 0 and byte not in (44, 125, 93): + byte = stream.read_byte() + return byte + + def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError): + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ + lut = {} + for index in _range(64): + lut[self._B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 + vlq_value = 0 + vlq_shift = 0 + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = src_idx, src_line + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: + end_segment() + return result, True + if byte == 59: + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or _len(result) == _len(needed): + return result, False + continue + if byte == 44: + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise _error("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 + + def _scan(self, chunks_factory, needed, max_needed, *, _set=set, _error=ValueError): + """Two-pass scan of the map; return (sources dict, line table).\n\n Pass 1 decodes only `mappings` (skipping the sources array entirely);\n pass 2 re-streams the map and collects only the source paths actually\n referenced by the decoded lines. Memory therefore scales with the\n traceback's needed frames, not with the bundle's full source table —\n the property the whole streaming decoder exists for. `chunks_factory`\n is a zero-argument callable producing a fresh chunk iterator per pass.\n """ + table = self._scan_mappings(self._stream_cls(chunks_factory()), needed, max_needed) + wanted = _set(src_idx for src_idx, _line in table.values()) + sources = {} + if wanted: + sources = self._scan_sources(self._stream_cls(chunks_factory()), wanted) + return sources, table + + def _scan_mappings(self, stream, needed, max_needed, *, _error=ValueError): + """Pass 1: decode the `mappings` field; every other value is skipped.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "mappings": + if not byte == 34: + raise _error("mappings is not a string") + table, _terminated = self._decode_vlq(stream, needed, max_needed) + return table + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + raise _error("no mappings field") + + def _scan_sources(self, stream, wanted, *, _error=ValueError): + """Pass 2: collect only the `sources` entries whose index is wanted.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + if not byte == 91: + raise _error("sources is not an array") + return self._read_wanted_array_items(stream, wanted) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + return {} + + def _read_wanted_array_items(self, stream, wanted, *, _len=len, _error=ValueError): + """Read a JSON array, collecting only string items at wanted indices.""" + items = {} + index = 0 + byte = self._skip_ws(stream, stream.read_byte()) + if byte == 93: + return items + while True: + if byte == 34 and index in wanted: + items[index] = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + else: + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + index += 1 + if byte == 93 or _len(items) == _len(wanted): + return items + if not byte == 44: + raise _error("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) + + def _load(self, needed_lines, *, _open=open, _set=set, _max=max): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line). The map is opened exactly once:\n digest verification and both decode passes read from the same pinned\n handle, so the verified bytes are the decoded bytes even if a\n concurrent build renames a new map into place mid-decode.\n """ + location = self._map_location() + if location is None: + return None + map_path, map_dir, verify = location + needed0 = _set(line - 1 for line in needed_lines) + max_needed = _max(needed0) + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is None: + + def chunks_factory(): + return self._inline_chunks(handle) + else: + + def chunks_factory(): + return self._handle_chunks(handle) + if verify and not self._map_matches_digest(chunks_factory): + return None + sources, table0 = self._scan(chunks_factory, needed0, max_needed) + finally: + handle.close() + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max, _enumerate=enumerate): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = self._map_location() + if location is None: + return None + map_path, map_dir, verify = location + json = self._import("json") + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is None: + + def fallback_chunks(): + return self._inline_chunks(handle) + else: + + def fallback_chunks(): + return self._handle_chunks(handle) + if verify and not self._map_matches_digest(fallback_chunks): + return None + raw = b"".join(fallback_chunks()) + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = {} + for index, source in _enumerate(data.get("sources") or []): + sources[index] = source + mappings = data.get("mappings") or "" + needed0 = _set(line - 1 for line in needed_lines) + stream = self._stream_cls([mappings.encode("ascii"), b'"']) + table0, _terminated = self._decode_vlq(stream, needed0, _max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _collect_needed(self, exc_value, traceback_obj, bundle_file, *, _set=set, _id=id, _getattr=getattr): + """1-based lines of `bundle_file` referenced by the traceback chain.""" + needed = _set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == bundle_file: + needed.add(tb.tb_lineno) + tb = tb.tb_next + add(traceback_obj) + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + add(_getattr(exc, "__traceback__", None)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed + + def _chain_has_group(self, exc_value, *, _set=set, _id=id, _getattr=getattr, _isinstance=isinstance): + """Whether the exception chain contains a BaseExceptionGroup.\n\n CPython renders groups with a dedicated nested layout; rather than\n losing the nested tracebacks, the runtime defers group rendering\n entirely to the previous hook (unremapped but complete).\n """ + if self._group_type is None: + return False + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + if _isinstance(exc, self._group_type): + return True + cause = _getattr(exc, "__cause__", None) + if cause is not None: + exc = cause + continue + if _getattr(exc, "__suppress_context__", False): + return False + exc = _getattr(exc, "__context__", None) + return False + + def _percent_decode(self, text, *, _int=int, _chr=chr, _len=len, _bex=BaseException): + """Inverse of the build-time percent-encoding of `sources` entries.""" + if "%" not in text: + return text + out = [] + position = 0 + length = _len(text) + while position < length: + character = text[position] + if character == "%" and position + 2 < length: + try: + out.append(_chr(_int(text[position + 1:position + 3], 16))) + position += 3 + continue + except _bex: + pass + out.append(character) + position += 1 + return "".join(out) + + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): + """Read a single 1-based line from a file without caching it.""" + try: + handle = _open(path, "rb") + except _os_error: + return None + try: + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except _os_error: + return None + finally: + handle.close() + return None + + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int=int): + """The application's `sys.tracebacklimit`, or None when unset/invalid.""" + limit = _getattr(self._sys, "tracebacklimit", None) + return limit if _isinstance(limit, _int) else None + + def _write_frames(self, traceback_obj, maps_by_file, write, limit, summaries, *, _bex=BaseException, _len=len): + """Write remapped frame lines, collapsing repeated frames like CPython.\n\n `maps_by_file` holds one (table, sources, map_dir) triple per known\n bundle file, so a traceback crossing several stacked cribo bundles\n remaps every frame. Frames that stay unmapped render through the\n standard traceback machinery (`summaries`, when available) to preserve\n PEP 657 position indicators. Consecutive identical frames (recursion)\n print at most 3 times followed by a \"[Previous line repeated N more\n times]\" marker; source line text is cached per (file, line) within one\n rendering. A positive `limit` keeps only the last `limit` frames,\n matching the interpreter's `sys.tracebacklimit` handling.\n """ + index = 0 + if limit is not None: + total = 0 + probe = traceback_obj + while probe is not None: + total += 1 + probe = probe.tb_next + skip = total - limit + while skip > 0 and traceback_obj is not None: + traceback_obj = traceback_obj.tb_next + skip -= 1 + index += 1 + cache = {} + last = None + repeats = 0 + + def emit(entry, frame_index, mapped_frame): + if not mapped_frame and summaries is not None and frame_index < _len(summaries): + try: + rendered = self._traceback.StackSummary.from_list([summaries[frame_index]]).format() + except _bex: + rendered = None + if rendered: + for line in rendered: + write(line) + return + write(' File "%s", line %d, in %s\n' % entry) + key = entry[0], entry[1] + if key not in cache: + cache[key] = self._source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + mapped_frame = False + bundle_map = maps_by_file.get(filename) + if bundle_map is not None: + table, sources, map_dir = bundle_map + mapped = table.get(lineno) + if mapped is not None: + source = sources.get(mapped[0]) + if source: + source = self._percent_decode(source) + if not self._os.path.isabs(source): + source = self._os.path.normpath(self._os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] + mapped_frame = True + entry = filename, lineno, name + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry, index, mapped_frame) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 + emit(entry, index, mapped_frame) + traceback_obj = traceback_obj.tb_next + index += 1 + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + + def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str, _bex=BaseException): + """Minimal `Type: message` line, the fallback formatter.""" + exc_type = _type(exc_value) + name = _getattr(exc_type, "__qualname__", exc_type.__name__) + module = _getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = _str(exc_value) + except _bex: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name + + def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr, _bex=BaseException): + """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ + try: + te = self._traceback.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) + for line in te.format_exception_only(): + write(line) + return + except _bex: + pass + write(self._exception_line(exc)) + notes = _getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except _bex: + pass + + def _render(self, exc_value, maps_by_file, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate, _type=type, _bex=BaseException): + """Render the exception (with its cause/context chain) like CPython.""" + chain = [] + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + suppress = _getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context + else: + chain.append((exc, None)) + exc = None + ordered = _list(_reversed(chain)) + for index, (exc, link) in _enumerate(ordered): + if index > 0: + if link == "cause": + write("\nThe above exception was the direct cause of the following " "exception:\n\n") + else: + write("\nDuring handling of the above exception, another exception " "occurred:\n\n") + tb = _getattr(exc, "__traceback__", None) + limit = self._effective_tb_limit() + if tb is not None and (limit is None or limit > 0): + total_frames = 0 + probe = tb + while probe is not None: + total_frames += 1 + probe = probe.tb_next + try: + summaries = self._traceback.TracebackException(_type(exc), exc, tb, limit=total_frames, lookup_lines=False).stack + except _bex: + summaries = None + write("Traceback (most recent call last):\n") + self._write_frames(tb, maps_by_file, write, limit, summaries) + self._write_exception_only(exc, write) + + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException, _set=set, _len=len): + """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ + if _getattr(self._local, "in_hook", False) or exc_value is None: + return False + self._local.in_hook = True + try: + if self._chain_has_group(exc_value): + return False + anchors_by_name = {} + for runtime in self._registered_runtimes(): + bundle_file = _getattr(runtime, "_bundle", None) + anchor = _getattr(runtime, "_bundle_anchor", None) + anchors_by_name.setdefault(bundle_file, _set()).add(anchor) + maps_by_file = {} + for runtime in self._registered_runtimes(): + try: + bundle_file = runtime._bundle + if bundle_file in maps_by_file: + continue + if _len(anchors_by_name.get(bundle_file, ())) > 1: + continue + needed = self._collect_needed(exc_value, traceback_obj, bundle_file) + if not needed: + continue + loaded = None + try: + loaded = runtime._load(needed) + except _bex: + try: + loaded = runtime._load_json_fallback(needed) + except _bex: + loaded = None + if loaded and loaded[0]: + maps_by_file[bundle_file] = loaded + except _bex: + continue + if not maps_by_file: + return False + parts = [] + if prefix: + parts.append(prefix) + self._render(exc_value, maps_by_file, parts.append) + stderr = self._sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except _bex: + pass + return True + except _bex: + return False + finally: + self._local.in_hook = False + + def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): + """All installed cribo runtimes in this interpreter, self included.\n\n The registry lives on the (unshadowable) `sys` module so stacked\n bundles can find each other's mappings.\n """ + registry = _getattr(self._sys, "_cribo_sm_runtimes", None) + runtimes = [] + if _isinstance(registry, _list): + for runtime in registry: + if _getattr(runtime, "_cribo_sm_runtime_marker", False): + runtimes.append(runtime) + if self not in runtimes: + runtimes.append(self) + return runtimes + + def _notify_custom_hook(self, prev, default, call, prev_attr, default_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Each\n traversed runtime's own captured default (named by `default_attr`) is\n collected along the way: an earlier runtime may have snapshotted a\n different default object than this one (e.g. after someone swapped\n `sys.__excepthook__` between bundle imports), and a chain ending on\n *any* of those defaults must invoke nothing — every default is a\n traceback printer, and the remap already printed. Cycle detection\n guards the traversal, and a chain that still ends on a cribo hook\n likewise invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + defaults = [default] + seen = _set() + while prev is not None and _id(prev) not in seen: + seen.add(_id(prev)) + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + other_default = _getattr(bound_to, default_attr, None) + if other_default is not None: + defaults.append(other_default) + prev = _getattr(bound_to, prev_attr, None) + continue + break + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return + if default is None or prev is None: + return + for known_default in defaults: + if prev is known_default: + return + try: + call(prev) + except _bex: + pass + + def excepthook(self, exc_type, exc_value, traceback_obj): + if self._try_render(exc_value, traceback_obj, None): + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook", "_default_excepthook") + return + self._prev_excepthook(exc_type, exc_value, traceback_obj) + + def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass, _system_exit=SystemExit): + if args.exc_type is not None and _issubclass(args.exc_type, _system_exit): + self._prev_threading_hook(args) + return + thread = _getattr(args, "thread", None) + name = _getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if self._try_render(args.exc_value, args.exc_traceback, prefix): + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook", "_default_threading_hook") + return + self._prev_threading_hook(args) + + def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): + message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except _bex: + prefix = "%s\n" % message + if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook", "_default_unraisablehook") + return + self._prev_unraisablehook(unraisable) +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "c36dd7f51595374bafef64709ded727675b756b31a07e0e1dd59bfdc790bf228") +del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime +import sys as _sys +import importlib as _importlib +class _CriboModule(): + + def __init__(_cribo_self, m, p): + _cribo_self._m, _cribo_self._p = m, p + + def __getattr__(_cribo_self, n, *, _getattr=getattr): + f = _cribo_self._p + '.' + n + try: + return _CriboModule(_importlib.import_module(f), f) + except ImportError: + return _getattr(_cribo_self._m, n) + + def __getattribute__(_cribo_self, n, *, _getattr=getattr, _object=object): + return _object.__getattribute__(_cribo_self, n) if n in ('_m', '_p', '__getattr__', '__class__', '__dict__', '__dir__', '__module__', '__qualname__') else _getattr(_object.__getattribute__(_cribo_self, '_m'), n) +class _Cribo(): + + def __getattr__(_cribo_self, n): + m = _sys.modules.get(n) or _importlib.import_module(n) + return _CriboModule(m, n) +_cribo = _Cribo() +_cribo_captured = {} +class _CriboPreservedLoader: + _pristine = {} + + def __init__(self, entry, namespace=None, reload_target=None): + self._entry = entry + self._namespace = namespace + self._reload_target = reload_target + + def create_module(self, spec, *, _getattr=getattr, _globals=globals, _id=id, _setattr=setattr, _type=type, _ModuleType=_cribo.types.ModuleType, _captured=_cribo_captured): + module = None + if self._namespace is not None: + module = self._namespace + elif self._entry[1] is not None: + module = _globals()[self._entry[1]] + if module is not None: + if _getattr(module, '_cribo_machinery_loaded', False) or _getattr(module, '_cribo_failed_life', False) or _getattr(module, '__initialized__', False) and _getattr(module, '_cribo_registered', False): + saved = _type(self)._pristine.get(_id(module)) + fresh = _ModuleType(spec.name) + fresh.__dict__.update(saved if saved is not None else {'__name__': spec.name}) + fresh._cribo_fresh_life = True + return fresh + return module + module = _ModuleType(spec.name) + for export, binding in self._entry[3].items(): + _setattr(module, export, _captured[binding] if binding in _captured else _globals()[binding]) + return module + + def exec_module(self, module, *, _BaseException=BaseException, _dict=dict, _getattr=getattr, _globals=globals, _id=id, _type=type): + init = self._entry[0] + pristine = _type(self)._pristine + key = _id(module) + record_snapshot = not _getattr(module, '_cribo_fresh_life', False) and not _getattr(module, '__initialized__', False) and (self._namespace is not None or self._entry[1] is not None) + if init is None: + if record_snapshot and key not in pristine: + pristine[key] = _dict(module.__dict__) + module._cribo_machinery_loaded = True + return + is_reload = self._reload_target is module + if is_reload: + module.__initialized__ = False + module.__initializing__ = False + state = _dict(module.__dict__) + if record_snapshot and key not in pristine: + pristine[key] = _dict(state) + try: + _globals()[init](module) + module._cribo_machinery_loaded = True + except _BaseException: + if is_reload: + module.__initializing__ = False + else: + module.__dict__.clear() + module.__dict__.update(state) + module.__initializing__ = False + module._cribo_failed_life = True + raise +class _CriboPreservedFinder: + + def __init__(self): + self._targets = {} + self._namespaces = {} + self._registry = {} + self._loader = _CriboPreservedLoader + + def register(self, name, init, namespace, is_package, exports=None): + entry = init, namespace, is_package, exports or {} + self._targets[name] = entry + self._registry[name] = entry + + def bind(self, name, namespace): + self._namespaces[name] = namespace + + def find_spec(self, name, path=None, target=None, *, _getattr=getattr, _globals=globals): + entry = self._targets.get(name) + if entry is None: + return None + parent_name = name.rpartition('.')[0] + if parent_name: + parent = _sys.modules.get(parent_name) + expected = self._namespaces.get(parent_name) + if expected is None: + parent_entry = self._registry.get(parent_name) + if parent_entry is not None and parent_entry[1] is not None: + expected = _globals().get(parent_entry[1]) + if parent is not None and expected is not None and parent is not expected and not _getattr(parent, '_cribo_fresh_life', False) and not _getattr(parent, '_cribo_machinery_loaded', False): + return None + from importlib.machinery import ModuleSpec + return ModuleSpec(name, self._loader(entry, self._namespaces.get(name), target), is_package=entry[2]) +_cribo_finder = _CriboPreservedFinder() +_sys.meta_path.append(_cribo_finder) +_cribo_finder_local = _CriboPreservedFinder() +_cribo_finder_local._namespaces = _cribo_finder._namespaces +_cribo_finder_local._registry = _cribo_finder._registry +for _cribo_index, _cribo_meta_finder in enumerate(_sys.meta_path): + if getattr(_cribo_meta_finder, '__name__', '') == 'PathFinder': + _sys.meta_path.insert(_cribo_index, _cribo_finder_local) + break +else: + _sys.meta_path.insert(0, _cribo_finder_local) +_cribo_finder_local.register('calculator', None, None, False, {'add': 'add', 'multiply': 'multiply'}) +_cribo_finder_local.register('utils', None, None, False, {'describe': 'describe'}) +calculator = _cribo.types.SimpleNamespace(__name__='calculator') +def add(a, b): + result = a + b + return result +add.__module__ = 'calculator' +def multiply(a, b): + result = a * b + return result +multiply.__module__ = 'calculator' +calculator.add = add +calculator.multiply = multiply +utils = _cribo.types.SimpleNamespace(__name__='utils') +def describe(name, value): + return f"{name} = {value}" +describe.__module__ = 'utils' +utils.describe = describe +_cribo_captured['add'] = add +_cribo_captured['multiply'] = multiply +_cribo_captured['describe'] = describe +total = add(2, 3) +product = multiply(total, 4) +print(describe("total", total)) +print(describe("product", product)) +# sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap new file mode 100644 index 000000000..05751ee45 --- /dev/null +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -0,0 +1,1036 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +#!/usr/bin/env python3 +# Generated by Cribo - Python Source Bundler +# https://github.com/ophidiarium/cribo + +import sys as _cribo_sys +def _cribo_sm_import(name, script_path=None, *, _sys=_cribo_sys, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed from a private\n *snapshot* of the search path. Resolution goes through the frozen importer\n machinery (`PathFinder.find_spec` with that explicit snapshot), so global\n `sys.path` is never mutated and concurrent bootstraps cannot race each\n other's save/restore. Lexical de-duplication works even before `os` is\n importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. Non-string entries are kept untouched.\n (`sys` and the frozen importlib modules are builtins and can never be\n shadowed.)\n """ + module = _sys.modules.get(name) + if module is not None: + return module + saved_path = _list(_sys.path) + candidates = [] + if saved_path and _isinstance(saved_path[0], _str): + candidates.append(saved_path[0]) + if script_path and _isinstance(script_path, _str): + cut = script_path.rfind("/") + cut_windows = script_path.rfind("\\") + if cut_windows > cut: + cut = cut_windows + candidates.append(script_path[:cut] if cut >= 0 else ".") + trimmed = [] + for candidate in candidates: + trimmed.append(candidate.rstrip("/\\") or candidate) + filtered = [] + for entry in saved_path[1:]: + if _isinstance(entry, _str): + if entry in ("", ".", "./"): + continue + stripped = entry.rstrip("/\\") or entry + if stripped in trimmed: + continue + filtered.append(entry) + os_mod = _sys.modules.get("os") + if os_mod is not None and trimmed: + resolved = [] + for candidate in trimmed: + try: + resolved.append(os_mod.path.normcase(os_mod.path.abspath(candidate or "."))) + except _bex: + pass + kept = [] + for entry in filtered: + if _isinstance(entry, _str): + try: + if os_mod.path.normcase(os_mod.path.abspath(entry or ".")) in resolved: + continue + except _bex: + pass + kept.append(entry) + filtered = kept + frozen_external = _sys.modules.get("_frozen_importlib_external") + frozen_bootstrap = _sys.modules.get("_frozen_importlib") + if frozen_external is not None and frozen_bootstrap is not None: + spec = frozen_bootstrap.BuiltinImporter.find_spec(name) or frozen_bootstrap.FrozenImporter.find_spec(name) or frozen_external.PathFinder.find_spec(name, filtered) + if spec is None: + raise ImportError("cribo runtime could not resolve " + name) + module = frozen_bootstrap.module_from_spec(spec) + _sys.modules[name] = module + try: + spec.loader.exec_module(module) + except _bex: + _sys.modules.pop(name, None) + raise + return module + _sys.path = filtered + try: + return _import(name) + finally: + _sys.path = saved_path +class _CriboSmStream(object): + """Byte-at-a-time reader over an iterator of byte chunks.\n\n Keyword-only defaults snapshot the builtins at definition time (before any\n bundled user code runs), so later shadowing of e.g. `len` cannot break the\n reader — the same idiom cribo's generated module proxies use.\n """ + __slots__ = "_chunks", "_buf", "_pos" + + def __init__(self, chunks, *, _iter=iter): + self._chunks = _iter(chunks) + self._buf = b"" + self._pos = 0 + + def read_byte(self, *, _len=len, _next=next, _stop=StopIteration): + while self._pos >= _len(self._buf): + try: + self._buf = _next(self._chunks) + except _stop: + return -1 + self._pos = 0 + value = self._buf[self._pos] + self._pos += 1 + return value +class _CriboSourceMapRuntime(object): + """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, and every method snapshots the builtins\n it needs via keyword-only defaults evaluated at class-definition time — so\n the installed hooks keep working even if bundled user code later rebinds\n any module-level name this template introduced, or common builtins such as\n `open`, `len`, or `max`.\n """ + _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + _CHUNK = 8192 + + def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod, *, _sys=_cribo_sys, _stream_cls=_CriboSmStream, _sm_import=_cribo_sm_import): + self._mode = mode + self._bundle = bundle_file + if bundle_file == "": + self._bundle_anchor = bundle_file + else: + self._bundle_anchor = os_mod.path.abspath(bundle_file) + self._expected_digest = None + try: + if isinstance(expected_digest, str) and len(expected_digest) == 64: + int(expected_digest, 16) + self._expected_digest = expected_digest.lower() + except ValueError: + self._expected_digest = None + self._cribo_sm_runtime_marker = True + try: + self._startup_cwd = os_mod.getcwd() + except OSError: + self._startup_cwd = "." + self._os = os_mod + self._sys = _sys + self._binascii = binascii_mod + self._threading = threading_mod + self._stream_cls = _stream_cls + self._import = _sm_import + self._traceback = traceback_mod + self._hashlib = hashlib_mod + self._local = threading_mod.local() + self._prev_excepthook = _sys.excepthook + self._prev_unraisablehook = _sys.unraisablehook + self._prev_threading_hook = threading_mod.excepthook + self._default_excepthook = _sys.__excepthook__ + self._default_unraisablehook = _sys.__unraisablehook__ + self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) + try: + self._group_type = BaseExceptionGroup + except NameError: + self._group_type = None + + def install(self): + """Install the three hooks; the previous hooks stay chained.\n\n Also joins the interpreter-wide runtime registry (stored on the\n unshadowable `sys` module) so stacked bundles can merge each other's\n mappings when a traceback crosses bundle boundaries.\n """ + registry = getattr(self._sys, "_cribo_sm_runtimes", None) + if not isinstance(registry, list): + registry = [] + self._sys._cribo_sm_runtimes = registry + registry.append(self) + self._sys.excepthook = self.excepthook + self._sys.unraisablehook = self.unraisablehook + self._threading.excepthook = self.threading_hook + + @classmethod + def _bootstrap(cls, mode, bundle_file, expected_digest, *, _sm_import=_cribo_sm_import): + """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ + try: + script = None if bundle_file == "" else bundle_file + runtime = cls(mode, bundle_file, expected_digest, _sm_import("os", script), _sm_import("binascii", script), _sm_import("threading", script), _sm_import("traceback", script), _sm_import("hashlib", script)) + runtime.install() + except BaseException: + pass + + def _map_location(self): + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir, verify); map_path is None for inline mode\n (the map lives inside the bundle file itself) and `verify` says whether\n the sibling map must match the digest recorded in the bundle. Called\n lazily at hook-fire time so the happy path never touches the\n environment or the filesystem.\n """ + env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + if env not in ("", "1", "true", "yes", "on"): + path = env + if not self._os.path.isabs(path): + path = self._os.path.join(self._startup_cwd, path) + return path, self._os.path.dirname(self._os.path.abspath(path)), False + bundle = self._bundle_anchor + if self._mode == "inline": + if bundle == "": + return None + return None, self._os.path.dirname(bundle), True + sibling = bundle + ".map" + if self._mode == "linked": + if self._os.path.exists(sibling): + return sibling, self._os.path.dirname(sibling), True + return None + if env in ("1", "true", "yes", "on"): + return sibling, self._os.path.dirname(sibling), True + return None + + def _file_chunks(self, path, *, _open=open): + """Yield fixed-size chunks of a file (constant memory).""" + handle = _open(path, "rb") + try: + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() + + def _handle_chunks(self, handle): + """Yield fixed-size chunks from an already-open handle, from offset 0.\n\n Keeping one open handle across the verify/decode passes pins the inode:\n a concurrent build renaming a new map into place cannot swap the bytes\n between passes.\n """ + handle.seek(0) + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + + def _find_marker_tail(self, handle, marker, *, _len=len): + """Backward-scan a file for the last occurrence of `marker`.\n\n Returns the offset of the payload following the marker, or -1. Only the\n tail of the file is examined; the body is never read.\n """ + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + while position > 0: + step = self._CHUNK if position >= self._CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + return position + index + _len(marker) + overlap = data[:_len(marker) - 1] + return -1 + + def _find_inline_payload(self, handle): + """Offset of the inline map's base64 payload in the bundle, or -1.""" + found = self._find_marker_tail(handle, b"# sourceMappingURL=data:") + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + 7 + + def _map_matches_digest(self, chunks_factory, *, _bex=BaseException): + """False only when a build digest is known and the map bytes disagree.\n\n The SHA-256 of this build's map is baked into the executing code at\n build time, so it is immune to any on-disk replacement of the bundle —\n no interleaving of concurrent builds, manual file shuffling, or\n redeploy-while-running can pair these code objects with another\n build's mappings. Hashing streams the same chunk source later used for\n decoding. Verification errors fail open (the subsequent map read would\n surface them anyway).\n """ + try: + expected = self._expected_digest + if expected is None: + return True + hasher = self._hashlib.sha256() + for chunk in chunks_factory(): + hasher.update(chunk) + return hasher.hexdigest() == expected + except _bex: + return True + + def _inline_chunks(self, handle, *, _len=len): + """Yield decoded chunks of an inline (base64 data URL) source map.\n\n Reads from an already-open bundle handle so all passes see one inode.\n """ + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = _len(data) - _len(data) % 4 + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) + + def _skip_ws(self, stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte + + def _read_hex4(self, stream, *, _int=int, _chr=chr, _range=range, _error=ValueError): + """Read the four hex digits of a \\uXXXX escape; return the code unit.""" + code = 0 + for _ in _range(4): + digit = stream.read_byte() + if digit < 0: + raise _error("unterminated unicode escape") + code = code * 16 + _int(_chr(digit), 16) + return code + + def _read_string(self, stream, collect, *, _bytearray=bytearray, _chr=chr, _error=ValueError): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes, \\uXXXX sequences, and UTF-16\n surrogate pairs (non-BMP characters) are handled, so string *values*\n containing text like '\"mappings\":' cannot confuse the key scanner and\n emoji-bearing paths survive intact.\n """ + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf = _bytearray() if collect else None + pending = None + while True: + if pending is None: + byte = stream.read_byte() + else: + byte, pending = pending, None + if byte < 0: + raise _error("unterminated JSON string") + if byte == 34: + return buf.decode("utf-8", "replace") if collect else None + if not byte == 92: + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise _error("unterminated JSON escape") + if not escape == 117: + if buf is not None: + buf.append(table.get(escape, escape)) + continue + code = self._read_hex4(stream) + if buf is None: + continue + if 55296 <= code <= 56319: + nxt = stream.read_byte() + if nxt == 92: + escape2 = stream.read_byte() + if escape2 == 117: + low = self._read_hex4(stream) + if 56320 <= low <= 57343: + code = 65536 + (code - 55296 << 10) + (low - 56320) + buf.extend(_chr(code).encode("utf-8")) + else: + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(low).encode("utf-8", "surrogatepass")) + continue + if escape2 < 0: + raise _error("unterminated JSON escape") + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.append(table.get(escape2, escape2)) + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + pending = nxt + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + + def _skip_value(self, stream, byte, *, _error=ValueError): + """Skip one JSON value; return the first byte after it (or -1).""" + if byte == 34: + self._read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise _error("unterminated JSON container") + if byte == 34: + self._read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + while byte >= 0 and byte not in (44, 125, 93): + byte = stream.read_byte() + return byte + + def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError): + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ + lut = {} + for index in _range(64): + lut[self._B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 + vlq_value = 0 + vlq_shift = 0 + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = src_idx, src_line + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: + end_segment() + return result, True + if byte == 59: + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or _len(result) == _len(needed): + return result, False + continue + if byte == 44: + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise _error("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 + + def _scan(self, chunks_factory, needed, max_needed, *, _set=set, _error=ValueError): + """Two-pass scan of the map; return (sources dict, line table).\n\n Pass 1 decodes only `mappings` (skipping the sources array entirely);\n pass 2 re-streams the map and collects only the source paths actually\n referenced by the decoded lines. Memory therefore scales with the\n traceback's needed frames, not with the bundle's full source table —\n the property the whole streaming decoder exists for. `chunks_factory`\n is a zero-argument callable producing a fresh chunk iterator per pass.\n """ + table = self._scan_mappings(self._stream_cls(chunks_factory()), needed, max_needed) + wanted = _set(src_idx for src_idx, _line in table.values()) + sources = {} + if wanted: + sources = self._scan_sources(self._stream_cls(chunks_factory()), wanted) + return sources, table + + def _scan_mappings(self, stream, needed, max_needed, *, _error=ValueError): + """Pass 1: decode the `mappings` field; every other value is skipped.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "mappings": + if not byte == 34: + raise _error("mappings is not a string") + table, _terminated = self._decode_vlq(stream, needed, max_needed) + return table + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + raise _error("no mappings field") + + def _scan_sources(self, stream, wanted, *, _error=ValueError): + """Pass 2: collect only the `sources` entries whose index is wanted.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + if not byte == 91: + raise _error("sources is not an array") + return self._read_wanted_array_items(stream, wanted) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + return {} + + def _read_wanted_array_items(self, stream, wanted, *, _len=len, _error=ValueError): + """Read a JSON array, collecting only string items at wanted indices.""" + items = {} + index = 0 + byte = self._skip_ws(stream, stream.read_byte()) + if byte == 93: + return items + while True: + if byte == 34 and index in wanted: + items[index] = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + else: + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + index += 1 + if byte == 93 or _len(items) == _len(wanted): + return items + if not byte == 44: + raise _error("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) + + def _load(self, needed_lines, *, _open=open, _set=set, _max=max): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line). The map is opened exactly once:\n digest verification and both decode passes read from the same pinned\n handle, so the verified bytes are the decoded bytes even if a\n concurrent build renames a new map into place mid-decode.\n """ + location = self._map_location() + if location is None: + return None + map_path, map_dir, verify = location + needed0 = _set(line - 1 for line in needed_lines) + max_needed = _max(needed0) + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is None: + + def chunks_factory(): + return self._inline_chunks(handle) + else: + + def chunks_factory(): + return self._handle_chunks(handle) + if verify and not self._map_matches_digest(chunks_factory): + return None + sources, table0 = self._scan(chunks_factory, needed0, max_needed) + finally: + handle.close() + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max, _enumerate=enumerate): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = self._map_location() + if location is None: + return None + map_path, map_dir, verify = location + json = self._import("json") + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is None: + + def fallback_chunks(): + return self._inline_chunks(handle) + else: + + def fallback_chunks(): + return self._handle_chunks(handle) + if verify and not self._map_matches_digest(fallback_chunks): + return None + raw = b"".join(fallback_chunks()) + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = {} + for index, source in _enumerate(data.get("sources") or []): + sources[index] = source + mappings = data.get("mappings") or "" + needed0 = _set(line - 1 for line in needed_lines) + stream = self._stream_cls([mappings.encode("ascii"), b'"']) + table0, _terminated = self._decode_vlq(stream, needed0, _max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _collect_needed(self, exc_value, traceback_obj, bundle_file, *, _set=set, _id=id, _getattr=getattr): + """1-based lines of `bundle_file` referenced by the traceback chain.""" + needed = _set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == bundle_file: + needed.add(tb.tb_lineno) + tb = tb.tb_next + add(traceback_obj) + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + add(_getattr(exc, "__traceback__", None)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed + + def _chain_has_group(self, exc_value, *, _set=set, _id=id, _getattr=getattr, _isinstance=isinstance): + """Whether the exception chain contains a BaseExceptionGroup.\n\n CPython renders groups with a dedicated nested layout; rather than\n losing the nested tracebacks, the runtime defers group rendering\n entirely to the previous hook (unremapped but complete).\n """ + if self._group_type is None: + return False + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + if _isinstance(exc, self._group_type): + return True + cause = _getattr(exc, "__cause__", None) + if cause is not None: + exc = cause + continue + if _getattr(exc, "__suppress_context__", False): + return False + exc = _getattr(exc, "__context__", None) + return False + + def _percent_decode(self, text, *, _int=int, _chr=chr, _len=len, _bex=BaseException): + """Inverse of the build-time percent-encoding of `sources` entries.""" + if "%" not in text: + return text + out = [] + position = 0 + length = _len(text) + while position < length: + character = text[position] + if character == "%" and position + 2 < length: + try: + out.append(_chr(_int(text[position + 1:position + 3], 16))) + position += 3 + continue + except _bex: + pass + out.append(character) + position += 1 + return "".join(out) + + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): + """Read a single 1-based line from a file without caching it.""" + try: + handle = _open(path, "rb") + except _os_error: + return None + try: + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except _os_error: + return None + finally: + handle.close() + return None + + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int=int): + """The application's `sys.tracebacklimit`, or None when unset/invalid.""" + limit = _getattr(self._sys, "tracebacklimit", None) + return limit if _isinstance(limit, _int) else None + + def _write_frames(self, traceback_obj, maps_by_file, write, limit, summaries, *, _bex=BaseException, _len=len): + """Write remapped frame lines, collapsing repeated frames like CPython.\n\n `maps_by_file` holds one (table, sources, map_dir) triple per known\n bundle file, so a traceback crossing several stacked cribo bundles\n remaps every frame. Frames that stay unmapped render through the\n standard traceback machinery (`summaries`, when available) to preserve\n PEP 657 position indicators. Consecutive identical frames (recursion)\n print at most 3 times followed by a \"[Previous line repeated N more\n times]\" marker; source line text is cached per (file, line) within one\n rendering. A positive `limit` keeps only the last `limit` frames,\n matching the interpreter's `sys.tracebacklimit` handling.\n """ + index = 0 + if limit is not None: + total = 0 + probe = traceback_obj + while probe is not None: + total += 1 + probe = probe.tb_next + skip = total - limit + while skip > 0 and traceback_obj is not None: + traceback_obj = traceback_obj.tb_next + skip -= 1 + index += 1 + cache = {} + last = None + repeats = 0 + + def emit(entry, frame_index, mapped_frame): + if not mapped_frame and summaries is not None and frame_index < _len(summaries): + try: + rendered = self._traceback.StackSummary.from_list([summaries[frame_index]]).format() + except _bex: + rendered = None + if rendered: + for line in rendered: + write(line) + return + write(' File "%s", line %d, in %s\n' % entry) + key = entry[0], entry[1] + if key not in cache: + cache[key] = self._source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + mapped_frame = False + bundle_map = maps_by_file.get(filename) + if bundle_map is not None: + table, sources, map_dir = bundle_map + mapped = table.get(lineno) + if mapped is not None: + source = sources.get(mapped[0]) + if source: + source = self._percent_decode(source) + if not self._os.path.isabs(source): + source = self._os.path.normpath(self._os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] + mapped_frame = True + entry = filename, lineno, name + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry, index, mapped_frame) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 + emit(entry, index, mapped_frame) + traceback_obj = traceback_obj.tb_next + index += 1 + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + + def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str, _bex=BaseException): + """Minimal `Type: message` line, the fallback formatter.""" + exc_type = _type(exc_value) + name = _getattr(exc_type, "__qualname__", exc_type.__name__) + module = _getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = _str(exc_value) + except _bex: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name + + def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr, _bex=BaseException): + """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ + try: + te = self._traceback.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) + for line in te.format_exception_only(): + write(line) + return + except _bex: + pass + write(self._exception_line(exc)) + notes = _getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except _bex: + pass + + def _render(self, exc_value, maps_by_file, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate, _type=type, _bex=BaseException): + """Render the exception (with its cause/context chain) like CPython.""" + chain = [] + exc = exc_value + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + suppress = _getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context + else: + chain.append((exc, None)) + exc = None + ordered = _list(_reversed(chain)) + for index, (exc, link) in _enumerate(ordered): + if index > 0: + if link == "cause": + write("\nThe above exception was the direct cause of the following " "exception:\n\n") + else: + write("\nDuring handling of the above exception, another exception " "occurred:\n\n") + tb = _getattr(exc, "__traceback__", None) + limit = self._effective_tb_limit() + if tb is not None and (limit is None or limit > 0): + total_frames = 0 + probe = tb + while probe is not None: + total_frames += 1 + probe = probe.tb_next + try: + summaries = self._traceback.TracebackException(_type(exc), exc, tb, limit=total_frames, lookup_lines=False).stack + except _bex: + summaries = None + write("Traceback (most recent call last):\n") + self._write_frames(tb, maps_by_file, write, limit, summaries) + self._write_exception_only(exc, write) + + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException, _set=set, _len=len): + """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ + if _getattr(self._local, "in_hook", False) or exc_value is None: + return False + self._local.in_hook = True + try: + if self._chain_has_group(exc_value): + return False + anchors_by_name = {} + for runtime in self._registered_runtimes(): + bundle_file = _getattr(runtime, "_bundle", None) + anchor = _getattr(runtime, "_bundle_anchor", None) + anchors_by_name.setdefault(bundle_file, _set()).add(anchor) + maps_by_file = {} + for runtime in self._registered_runtimes(): + try: + bundle_file = runtime._bundle + if bundle_file in maps_by_file: + continue + if _len(anchors_by_name.get(bundle_file, ())) > 1: + continue + needed = self._collect_needed(exc_value, traceback_obj, bundle_file) + if not needed: + continue + loaded = None + try: + loaded = runtime._load(needed) + except _bex: + try: + loaded = runtime._load_json_fallback(needed) + except _bex: + loaded = None + if loaded and loaded[0]: + maps_by_file[bundle_file] = loaded + except _bex: + continue + if not maps_by_file: + return False + parts = [] + if prefix: + parts.append(prefix) + self._render(exc_value, maps_by_file, parts.append) + stderr = self._sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except _bex: + pass + return True + except _bex: + return False + finally: + self._local.in_hook = False + + def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): + """All installed cribo runtimes in this interpreter, self included.\n\n The registry lives on the (unshadowable) `sys` module so stacked\n bundles can find each other's mappings.\n """ + registry = _getattr(self._sys, "_cribo_sm_runtimes", None) + runtimes = [] + if _isinstance(registry, _list): + for runtime in registry: + if _getattr(runtime, "_cribo_sm_runtime_marker", False): + runtimes.append(runtime) + if self not in runtimes: + runtimes.append(self) + return runtimes + + def _notify_custom_hook(self, prev, default, call, prev_attr, default_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Each\n traversed runtime's own captured default (named by `default_attr`) is\n collected along the way: an earlier runtime may have snapshotted a\n different default object than this one (e.g. after someone swapped\n `sys.__excepthook__` between bundle imports), and a chain ending on\n *any* of those defaults must invoke nothing — every default is a\n traceback printer, and the remap already printed. Cycle detection\n guards the traversal, and a chain that still ends on a cribo hook\n likewise invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + defaults = [default] + seen = _set() + while prev is not None and _id(prev) not in seen: + seen.add(_id(prev)) + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + other_default = _getattr(bound_to, default_attr, None) + if other_default is not None: + defaults.append(other_default) + prev = _getattr(bound_to, prev_attr, None) + continue + break + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return + if default is None or prev is None: + return + for known_default in defaults: + if prev is known_default: + return + try: + call(prev) + except _bex: + pass + + def excepthook(self, exc_type, exc_value, traceback_obj): + if self._try_render(exc_value, traceback_obj, None): + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook", "_default_excepthook") + return + self._prev_excepthook(exc_type, exc_value, traceback_obj) + + def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass, _system_exit=SystemExit): + if args.exc_type is not None and _issubclass(args.exc_type, _system_exit): + self._prev_threading_hook(args) + return + thread = _getattr(args, "thread", None) + name = _getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if self._try_render(args.exc_value, args.exc_traceback, prefix): + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook", "_default_threading_hook") + return + self._prev_threading_hook(args) + + def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): + message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except _bex: + prefix = "%s\n" % message + if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook", "_default_unraisablehook") + return + self._prev_unraisablehook(unraisable) +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "d1929b8451a8662aa287cb9620ed4718dbb5db55abe122d15d965adbc4bdc823") +del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime +import sys as _sys +import importlib as _importlib +class _CriboModule(): + + def __init__(_cribo_self, m, p): + _cribo_self._m, _cribo_self._p = m, p + + def __getattr__(_cribo_self, n, *, _getattr=getattr): + f = _cribo_self._p + '.' + n + try: + return _CriboModule(_importlib.import_module(f), f) + except ImportError: + return _getattr(_cribo_self._m, n) + + def __getattribute__(_cribo_self, n, *, _getattr=getattr, _object=object): + return _object.__getattribute__(_cribo_self, n) if n in ('_m', '_p', '__getattr__', '__class__', '__dict__', '__dir__', '__module__', '__qualname__') else _getattr(_object.__getattribute__(_cribo_self, '_m'), n) +class _Cribo(): + + def __getattr__(_cribo_self, n): + m = _sys.modules.get(n) or _importlib.import_module(n) + return _CriboModule(m, n) +_cribo = _Cribo() +_cribo_captured = {} +class _CriboPreservedLoader: + _pristine = {} + + def __init__(self, entry, namespace=None, reload_target=None): + self._entry = entry + self._namespace = namespace + self._reload_target = reload_target + + def create_module(self, spec, *, _getattr=getattr, _globals=globals, _id=id, _setattr=setattr, _type=type, _ModuleType=_cribo.types.ModuleType, _captured=_cribo_captured): + module = None + if self._namespace is not None: + module = self._namespace + elif self._entry[1] is not None: + module = _globals()[self._entry[1]] + if module is not None: + if _getattr(module, '_cribo_machinery_loaded', False) or _getattr(module, '_cribo_failed_life', False) or _getattr(module, '__initialized__', False) and _getattr(module, '_cribo_registered', False): + saved = _type(self)._pristine.get(_id(module)) + fresh = _ModuleType(spec.name) + fresh.__dict__.update(saved if saved is not None else {'__name__': spec.name}) + fresh._cribo_fresh_life = True + return fresh + return module + module = _ModuleType(spec.name) + for export, binding in self._entry[3].items(): + _setattr(module, export, _captured[binding] if binding in _captured else _globals()[binding]) + return module + + def exec_module(self, module, *, _BaseException=BaseException, _dict=dict, _getattr=getattr, _globals=globals, _id=id, _type=type): + init = self._entry[0] + pristine = _type(self)._pristine + key = _id(module) + record_snapshot = not _getattr(module, '_cribo_fresh_life', False) and not _getattr(module, '__initialized__', False) and (self._namespace is not None or self._entry[1] is not None) + if init is None: + if record_snapshot and key not in pristine: + pristine[key] = _dict(module.__dict__) + module._cribo_machinery_loaded = True + return + is_reload = self._reload_target is module + if is_reload: + module.__initialized__ = False + module.__initializing__ = False + state = _dict(module.__dict__) + if record_snapshot and key not in pristine: + pristine[key] = _dict(state) + try: + _globals()[init](module) + module._cribo_machinery_loaded = True + except _BaseException: + if is_reload: + module.__initializing__ = False + else: + module.__dict__.clear() + module.__dict__.update(state) + module.__initializing__ = False + module._cribo_failed_life = True + raise +class _CriboPreservedFinder: + + def __init__(self): + self._targets = {} + self._namespaces = {} + self._registry = {} + self._loader = _CriboPreservedLoader + + def register(self, name, init, namespace, is_package, exports=None): + entry = init, namespace, is_package, exports or {} + self._targets[name] = entry + self._registry[name] = entry + + def bind(self, name, namespace): + self._namespaces[name] = namespace + + def find_spec(self, name, path=None, target=None, *, _getattr=getattr, _globals=globals): + entry = self._targets.get(name) + if entry is None: + return None + parent_name = name.rpartition('.')[0] + if parent_name: + parent = _sys.modules.get(parent_name) + expected = self._namespaces.get(parent_name) + if expected is None: + parent_entry = self._registry.get(parent_name) + if parent_entry is not None and parent_entry[1] is not None: + expected = _globals().get(parent_entry[1]) + if parent is not None and expected is not None and parent is not expected and not _getattr(parent, '_cribo_fresh_life', False) and not _getattr(parent, '_cribo_machinery_loaded', False): + return None + from importlib.machinery import ModuleSpec + return ModuleSpec(name, self._loader(entry, self._namespaces.get(name), target), is_package=entry[2]) +_cribo_finder = _CriboPreservedFinder() +_sys.meta_path.append(_cribo_finder) +_cribo_finder_local = _CriboPreservedFinder() +_cribo_finder_local._namespaces = _cribo_finder._namespaces +_cribo_finder_local._registry = _cribo_finder._registry +for _cribo_index, _cribo_meta_finder in enumerate(_sys.meta_path): + if getattr(_cribo_meta_finder, '__name__', '') == 'PathFinder': + _sys.meta_path.insert(_cribo_index, _cribo_finder_local) + break +else: + _sys.meta_path.insert(0, _cribo_finder_local) +_cribo_finder_local.register('effects', '_cribo_init___cribo_503b17_effects', 'effects', False) +effects = _cribo.types.SimpleNamespace(__name__='effects', __initializing__=False, __initialized__=False) +_cribo_finder.bind('effects', effects) +def _cribo_init___cribo_503b17_effects(_cribo_self, *, _cribo_getattr=getattr, _cribo_base_exception=BaseException, _cribo=_cribo, _sys=_sys): + if _cribo_getattr(_cribo_self, '__initialized__', False): + return _cribo_self + if _cribo_getattr(_cribo_self, '__initializing__', False): + return _cribo_self + _cribo_self.__initializing__ = True + try: + _cribo_self.__package__ = '' + _cribo_self.__doc__ = None + print("effects module loading") + COUNTER = 1 + _cribo_self.COUNTER = COUNTER + + def boost(value): + boosted = value * 2 + COUNTER + return boosted + boost.__module__ = 'effects' + boost.__qualname__ = 'boost' + _cribo_self.boost = boost + _cribo_self.__initialized__ = True + _cribo_self.__initializing__ = False + return _cribo_self + except _cribo_base_exception: + _cribo_self.__initializing__ = False + raise +effects.__init__ = _cribo_init___cribo_503b17_effects +effects = _cribo_init___cribo_503b17_effects(effects) +print("counter:", effects.COUNTER) +print("boosted:", effects.boost(10)) +# sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/execution_results@sourcemap_basic.snap b/crates/cribo/tests/snapshots/execution_results@sourcemap_basic.snap new file mode 100644 index 000000000..cd9792f9c --- /dev/null +++ b/crates/cribo/tests/snapshots/execution_results@sourcemap_basic.snap @@ -0,0 +1,9 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +ExecutionResults { + status: Success, + stdout: "total = 5\nproduct = 20", + stderr: "", +} diff --git a/crates/cribo/tests/snapshots/execution_results@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/execution_results@sourcemap_wrapper.snap new file mode 100644 index 000000000..45ef213d3 --- /dev/null +++ b/crates/cribo/tests/snapshots/execution_results@sourcemap_wrapper.snap @@ -0,0 +1,9 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +ExecutionResults { + status: Success, + stdout: "effects module loading\ncounter: 1\nboosted: 21", + stderr: "", +} diff --git a/crates/cribo/tests/snapshots/requirements@sourcemap_basic.snap b/crates/cribo/tests/snapshots/requirements@sourcemap_basic.snap new file mode 100644 index 000000000..d6b88d9ea --- /dev/null +++ b/crates/cribo/tests/snapshots/requirements@sourcemap_basic.snap @@ -0,0 +1,6 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +packages: [] +count: 0 diff --git a/crates/cribo/tests/snapshots/requirements@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/requirements@sourcemap_wrapper.snap new file mode 100644 index 000000000..984c8102e --- /dev/null +++ b/crates/cribo/tests/snapshots/requirements@sourcemap_wrapper.snap @@ -0,0 +1,6 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +packages: [] +count: 0 diff --git a/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_basic.snap b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_basic.snap new file mode 100644 index 000000000..e44913377 --- /dev/null +++ b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_basic.snap @@ -0,0 +1,10 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +RuffLintResults { + f401: [], + f404: [], + other: [], + total: 0, +} diff --git a/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_wrapper.snap new file mode 100644 index 000000000..7b6c053ee --- /dev/null +++ b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_wrapper.snap @@ -0,0 +1,10 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +RuffLintResults { + f401: [], + f404: [], + other: [], + total: 0, +} diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap new file mode 100644 index 000000000..91f143d0b --- /dev/null +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -0,0 +1,16 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +bundle:1003 `def add(a, b):` -> calculator.py:1 +bundle:1004 `result = a + b` -> calculator.py:2 +bundle:1005 `return result` -> calculator.py:3 +bundle:1007 `def multiply(a, b):` -> calculator.py:6 +bundle:1008 `result = a * b` -> calculator.py:7 +bundle:1009 `return result` -> calculator.py:8 +bundle:1014 `def describe(name, value):` -> utils.py:1 +bundle:1015 `return f"{name} = {value}"` -> utils.py:2 +bundle:1021 `total = add(2, 3)` -> main.py:4 +bundle:1022 `product = multiply(total, 4)` -> main.py:5 +bundle:1023 `print(describe("total", total))` -> main.py:6 +bundle:1024 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap new file mode 100644 index 000000000..9b0147233 --- /dev/null +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -0,0 +1,11 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +bundle:1012 `print("effects module loading")` -> effects.py:1 +bundle:1013 `COUNTER = 1` -> effects.py:3 +bundle:1016 `def boost(value):` -> effects.py:6 +bundle:1017 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:1018 `return boosted` -> effects.py:8 +bundle:1030 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:1031 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_bundling_snapshots.rs b/crates/cribo/tests/test_bundling_snapshots.rs index 8ec359920..b75900f88 100644 --- a/crates/cribo/tests/test_bundling_snapshots.rs +++ b/crates/cribo/tests/test_bundling_snapshots.rs @@ -144,6 +144,43 @@ struct RequirementsData { count: usize, } +/// Render a Source Map v3 JSON as a deterministic, path-free text dump. +/// +/// One line per mapping: `bundle: `` -> :` with 1-based line numbers. Basenames keep the snapshot +/// free of machine-specific absolute paths. +fn render_source_map_dump(map_json: &str, bundled_code: &str) -> String { + use std::fmt::Write; + + let map = oxc_sourcemap::SourceMap::from_json_string(map_json) + .expect("emitted source map must be valid Source Map v3 JSON"); + let bundle_lines: Vec<&str> = bundled_code.lines().collect(); + let mut dump = String::new(); + for token in map.get_tokens() { + let source = token + .get_source_id() + .and_then(|id| map.get_source(id)) + .unwrap_or(""); + let source_name = Path::new(source).file_name().map_or_else( + || source.to_owned(), + |name| name.to_string_lossy().into_owned(), + ); + let generated_text = bundle_lines + .get(token.get_dst_line() as usize) + .map(|line| line.trim()) + .unwrap_or_default(); + let _ = writeln!( + dump, + "bundle:{:<4} `{}` -> {}:{}", + token.get_dst_line() + 1, + generated_text, + source_name, + token.get_src_line() + 1, + ); + } + dump +} + /// Run ruff linting on bundled code to cross-validate import handling fn run_ruff_lint_on_bundle(bundled_code: &str) -> RuffLintResults { // Create settings for multiple import-related rules with both F401 and F404 enabled @@ -230,6 +267,9 @@ fn test_bundling_fixtures() { // Check fixture type based on prefix let expects_bundling_failure = fixture_name.starts_with("xfail_"); let expects_python_failure = fixture_name.starts_with("pyfail_"); + // sourcemap_ fixtures opt into --sourcemap=linked and get an extra + // normalized mapping snapshot (see render_source_map_dump) + let enables_sourcemap = fixture_name.starts_with("sourcemap_"); // Get Python executable once for the entire test let python_cmd = common::get_python_executable(); @@ -332,6 +372,9 @@ fn test_bundling_fixtures() { if fake_venv.is_some() { cribo_args.push("--bundle-third-party"); } + if enables_sourcemap { + cribo_args.push("--sourcemap=linked"); + } // A fixture-level cribo.toml supplies configuration (e.g. module-map entries) let fixture_config = fixture_dir.join("cribo.toml"); let fixture_config_str = fixture_config.to_str().map(ToOwned::to_owned); @@ -376,6 +419,15 @@ fn test_bundling_fixtures() { // Read the bundled code let bundled_code = fs::read_to_string(&bundle_path).unwrap(); + // Sourcemap fixtures must produce a sibling map; snapshot it in a + // normalized, path-free form so mapping regressions are visible. + let source_map_dump = enables_sourcemap.then(|| { + let map_path = temp_dir.path().join("bundled.py.map"); + let map_json = fs::read_to_string(&map_path) + .expect("sourcemap fixture must produce bundled.py.map"); + render_source_map_dump(&map_json, &bundled_code) + }); + // Read and parse the requirements.txt if it was generated let requirements_path = temp_dir.path().join("requirements.txt"); let requirements_data = if requirements_path.exists() { @@ -610,6 +662,11 @@ fn test_bundling_fixtures() { // Snapshot ruff linting results insta::assert_debug_snapshot!("ruff_lint_results", ruff_results); + // Snapshot the normalized source map for sourcemap_ fixtures + if let Some(dump) = &source_map_dump { + insta::assert_snapshot!("source_map", dump); + } + // Snapshot requirements data as YAML insta::assert_yaml_snapshot!("requirements", requirements_data); }); diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs new file mode 100644 index 000000000..28c14c073 --- /dev/null +++ b/crates/cribo/tests/test_source_maps.rs @@ -0,0 +1,1867 @@ +//! Integration tests for `--sourcemap` CLI delivery modes. +//! +//! Each test drives the cribo binary end-to-end and inspects the emitted +//! bundle and/or `.map` file. See `docs/source-maps.md` for the design. + +mod common; + +use std::{fs, path::Path, process::Command}; + +use tempfile::TempDir; + +/// Marker prefix of an inline source map comment. +const INLINE_MARKER: &str = "# sourceMappingURL=data:application/json;base64,"; + +/// Create a project from (file name, content) pairs and return its directory. +fn make_project(files: &[(&str, &str)]) -> TempDir { + let dir = TempDir::new().expect("create temp dir"); + for (name, content) in files { + fs::write(dir.path().join(name), content).expect("write fixture file"); + } + dir +} + +/// Create a two-module fixture project and return its directory. +fn fixture_project() -> TempDir { + make_project(&[ + ( + "main.py", + "from helper import greet\n\nprint(greet(\"world\"))\n", + ), + ( + "helper.py", + "def greet(name):\n message = f\"hello {name}\"\n return message\n", + ), + ]) +} + +/// Run the cribo binary with `args`, returning (status success, stdout, stderr). +fn run_cribo(args: &[&str]) -> (bool, String, String) { + let output = Command::new(env!("CARGO_BIN_EXE_cribo")) + .args(args) + .output() + .expect("run cribo binary"); + ( + output.status.success(), + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) +} + +/// Parse a source map JSON string and assert it is valid Source Map v3. +fn parse_map(json: &str) -> oxc_sourcemap::SourceMap<'_> { + oxc_sourcemap::SourceMap::from_json_string(json).expect("valid Source Map v3 JSON") +} + +/// Return the fixture entry path (`main.py`) as a CLI argument string. +fn entry_arg(dir: &TempDir) -> String { + dir.path().join("main.py").to_string_lossy().into_owned() +} + +/// Assert the map targets `bundle_file`, lists `helper.py`, and has mappings. +fn assert_map_covers_helper(map_json: &str, bundle_file: &str) { + let map = parse_map(map_json); + assert_eq!(map.get_file(), Some(bundle_file)); + assert!( + map.get_sources() + .any(|source| source.ends_with("helper.py")), + "map sources must include helper.py: {:?}", + map.get_sources().collect::>() + ); + assert!(map.get_tokens().count() > 0, "map must contain mappings"); +} + +#[test] +fn linked_mode_writes_map_and_comment() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle + .trim_end() + .ends_with("# sourceMappingURL=bundle.py.map"), + "linked mode must append the sourceMappingURL comment" + ); + + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map file"); + assert_map_covers_helper(&map_json, "bundle.py"); + // Linked mode embeds sourcesContent by default. + assert!(map_json.contains("sourcesContent")); +} + +#[test] +fn bare_sourcemap_flag_defaults_to_linked() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!(dir.path().join("bundle.py.map").exists()); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!(bundle.contains("# sourceMappingURL=bundle.py.map")); +} + +#[test] +fn external_mode_writes_map_without_comment() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=external", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + !bundle + .lines() + .any(|line| line.starts_with("# sourceMappingURL=")), + "external mode must not reference the map from the bundle" + ); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map file"); + assert_map_covers_helper(&map_json, "bundle.py"); +} + +#[test] +fn inline_mode_embeds_map_and_writes_no_file() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!( + !dir.path().join("bundle.py.map").exists(), + "inline mode must not write a map file" + ); + + let bundle = fs::read_to_string(&out).expect("read bundle"); + let map_json = decode_inline_map(&bundle); + assert_map_covers_helper(&map_json, "bundle.py"); + // Inline mode omits sourcesContent by default. + assert!(!map_json.contains("sourcesContent")); +} + +#[test] +fn stdout_with_bare_sourcemap_selects_inline() { + let dir = fixture_project(); + let (ok, stdout, stderr) = run_cribo(&["--entry", &entry_arg(&dir), "--stdout", "--sourcemap"]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = decode_inline_map(&stdout); + assert_map_covers_helper(&map_json, ""); + // A stdout bundle can be redirected anywhere, so its map has no anchor + // directory: source paths must stay absolute. + let map = parse_map(&map_json); + assert!( + map.get_sources() + .all(|source| Path::new(source).is_absolute()), + "stdout maps must carry absolute source paths: {:?}", + map.get_sources().collect::>() + ); +} + +#[test] +fn stdout_with_linked_sourcemap_errors() { + let dir = fixture_project(); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--stdout", + "--sourcemap=linked", + ]); + assert!(!ok, "linked + stdout must be rejected"); + assert!( + stderr.contains("--sourcemap=inline"), + "error must suggest inline mode: {stderr}" + ); +} + +#[test] +fn stdout_with_external_sourcemap_errors() { + let dir = fixture_project(); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--stdout", + "--sourcemap=external", + ]); + assert!(!ok, "external + stdout must be rejected"); + assert!(stderr.contains("--sourcemap=inline")); +} + +#[test] +fn no_sourcemap_by_default() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!(!dir.path().join("bundle.py.map").exists()); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!(!bundle.contains("sourceMappingURL")); +} + +#[test] +fn config_file_sourcemap_key_is_honored() { + let dir = fixture_project(); + fs::write(dir.path().join("cribo.toml"), "sourcemap = \"external\"\n").expect("write config"); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--config", + &dir.path().join("cribo.toml").to_string_lossy(), + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!( + dir.path().join("bundle.py.map").exists(), + "config-file sourcemap key must enable map emission" + ); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + !bundle + .lines() + .any(|line| line.starts_with("# sourceMappingURL=")), + "external mode: no comment" + ); +} + +/// Extract and decode the inline source map data URL from bundle text. +fn decode_inline_map(bundle: &str) -> String { + let marker_pos = bundle + .rfind(INLINE_MARKER) + .expect("inline source map comment present"); + let payload = bundle[marker_pos + INLINE_MARKER.len()..].trim_end(); + let bytes = base64_simd::STANDARD + .decode_to_vec(payload.as_bytes()) + .expect("valid base64 payload"); + String::from_utf8(bytes).expect("valid UTF-8 source map") +} + +/// The map file sits next to the bundle even when the output path is nested. +#[test] +fn linked_map_lands_next_to_nested_output() { + let dir = fixture_project(); + let nested = dir.path().join("dist").join("app.py"); + fs::create_dir_all(nested.parent().expect("parent")).expect("mkdir dist"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &nested.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_path = dir.path().join("dist").join("app.py.map"); + assert!(map_path.exists(), "map must sit next to the nested output"); + let bundle = fs::read_to_string(&nested).expect("read bundle"); + assert!(bundle.contains("# sourceMappingURL=app.py.map")); + + // Source paths must be relative to the map's directory. + let map_json = fs::read_to_string(&map_path).expect("read map"); + let map = parse_map(&map_json); + assert!( + map.get_sources() + .all(|source| Path::new(source).is_relative()), + "sources must be relative paths: {:?}", + map.get_sources().collect::>() + ); +} + +#[test] +fn sources_content_can_be_forced_on_for_inline() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + "--sources-content=true", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + let map_json = decode_inline_map(&bundle); + assert!( + map_json.contains("sourcesContent"), + "--sources-content=true must force embedding for inline maps" + ); + let map = parse_map(&map_json); + assert!( + map.get_source_contents() + .flatten() + .any(|content| content.contains("def greet")), + "embedded content must carry the original helper source" + ); +} + +#[test] +fn sources_content_can_be_forced_off_for_linked() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + "--sources-content=false", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + assert!( + !map_json.contains("sourcesContent"), + "--sources-content=false must strip embedding for linked maps" + ); +} + +#[test] +fn config_file_sources_content_key_is_honored() { + let dir = fixture_project(); + fs::write( + dir.path().join("cribo.toml"), + "sourcemap = \"external\"\nsources-content = false\n", + ) + .expect("write config"); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--config", + &dir.path().join("cribo.toml").to_string_lossy(), + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + assert!(!map_json.contains("sourcesContent")); +} + +// --------------------------------------------------------------------------- +// Runtime traceback remapping (injected prologue) +// --------------------------------------------------------------------------- + +/// Create a fixture whose entry crashes two calls deep inside helper.py. +fn crash_project() -> TempDir { + make_project(&[ + ("main.py", "from helper import boom\n\nboom()\n"), + ( + "helper.py", + "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"kaboom\")\n", + ), + ]) +} + +/// Bundle the crash project with the given sourcemap argument; return bundle path. +fn bundle_crash_project(dir: &TempDir, sourcemap_arg: &str) -> std::path::PathBuf { + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(dir), + "--output", + &out.to_string_lossy(), + sourcemap_arg, + ]); + assert!(ok, "bundling must succeed: {stderr}"); + out +} + +/// Run a Python file; returns (status success, stdout, stderr). +fn run_python(bundle: &Path, envs: &[(&str, &str)]) -> (bool, String, String) { + let mut command = Command::new(common::get_python_executable()); + command.arg(bundle); + // The env-gating tests require a clean slate; an explicit entry below wins. + command.env_remove("CRIBO_SOURCE_MAPS"); + for (key, value) in envs { + command.env(key, value); + } + let output = command.output().expect("run python"); + ( + output.status.success(), + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) +} + +/// Assert stderr shows the remapped traceback pointing at original files. +fn assert_remapped(stderr: &str) { + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "traceback must point at helper.py:5: {stderr}" + ); + assert!( + stderr.contains("raise ValueError(\"kaboom\")"), + "traceback must show the original source line: {stderr}" + ); + assert!( + stderr.contains("main.py\", line 3, in "), + "traceback must point at main.py:3: {stderr}" + ); + assert!( + !stderr.contains("bundle.py\", line"), + "no frame should remain on bundle coordinates: {stderr}" + ); +} + +/// Assert stderr is one single standard (non-remapped) traceback with no +/// runtime noise. +fn assert_standard_traceback(stderr: &str) { + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "exactly one traceback expected: {stderr}" + ); + assert!( + stderr.contains("bundle.py\", line"), + "standard traceback must show bundle coordinates: {stderr}" + ); + assert!( + !stderr.contains("helper.py\", line 5"), + "no remapping must happen: {stderr}" + ); +} + +#[test] +fn runtime_remaps_linked_crash() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok, "the crashing bundle must exit non-zero"); + assert_remapped(&stderr); +} + +#[test] +fn runtime_disabled_when_linked_map_missing() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + fs::remove_file(dir.path().join("bundle.py.map")).expect("delete map"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn runtime_remaps_inline_crash() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=inline"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_remapped(&stderr); +} + +#[test] +fn runtime_kill_switch_disables_remapping() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=inline"); + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "0")]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn runtime_external_mode_is_env_gated() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=external"); + + // Without the env var the runtime stays dormant. + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); + + // CRIBO_SOURCE_MAPS=1 activates it against the sibling map. + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "1")]); + assert!(!ok); + assert_remapped(&stderr); + + // The env var can also point directly at a relocated map file. + let moved = dir.path().join("elsewhere.map"); + fs::rename(dir.path().join("bundle.py.map"), &moved).expect("move map"); + let (ok, _, stderr) = run_python( + &bundle, + &[("CRIBO_SOURCE_MAPS", moved.to_string_lossy().as_ref())], + ); + assert!(!ok); + assert_remapped(&stderr); + + // With the map moved away, =1 finds nothing and stays silent. + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "1")]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +/// Drive the pure-Python unit tests for the runtime internals (VLQ machine, +/// JSON scanner, backward EOF scan, base64 chunk alignment, json fallback). +#[test] +fn python_runtime_unit_tests() { + use cow_utils::CowUtils as _; + + const TEMPLATE: &str = include_str!("../src/python/sourcemap_runtime.py"); + let dir = TempDir::new().expect("create temp dir"); + let runtime_path = dir.path().join("runtime.py"); + fs::write( + &runtime_path, + TEMPLATE + .cow_replace("__CRIBO_SOURCEMAP_MODE__", "external") + .as_ref(), + ) + .expect("write substituted runtime"); + + let script = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("python") + .join("test_sourcemap_runtime.py"); + let output = Command::new(common::get_python_executable()) + .arg(&script) + .arg(&runtime_path) + .output() + .expect("run python unit tests"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "python runtime unit tests failed:\n{stdout}\n{stderr}" + ); + // The harness discovers its tests and reports the count itself; assert the + // sentinel plus a sanity floor instead of duplicating the exact count here. + assert!(stdout.contains("RUNTIME TESTS PASSED"), "{stdout}"); + assert!( + stdout.matches("PASS test_").count() >= 10, + "expected a healthy number of runtime unit tests: {stdout}" + ); +} + +// --------------------------------------------------------------------------- +// Full hook coverage, duress conditions, and laziness +// --------------------------------------------------------------------------- + +#[test] +fn runtime_remaps_thread_crash() { + let dir = make_project(&[ + ( + "main.py", + "import threading\nfrom helper import boom\n\nworker = \ + threading.Thread(target=boom)\nworker.start()\nworker.join()\nprint(\"done\")\n", + ), + ( + "helper.py", + "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"thread kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + // An uncaught exception in a non-main thread does not fail the process. + let (ok, stdout, stderr) = run_python(&bundle, &[]); + assert!(ok, "main thread must finish normally: {stderr}"); + assert!(stdout.contains("done")); + assert!( + stderr.contains("Exception in thread"), + "threading hook must announce the thread: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "thread traceback must be remapped: {stderr}" + ); + assert!(stderr.contains("thread kaboom")); +} + +#[test] +fn runtime_remaps_unraisable_error() { + let dir = make_project(&[ + ( + "main.py", + "from helper import make\n\nobj = make()\ndel obj\nprint(\"done\")\n", + ), + ( + "helper.py", + "class Cursed:\n def __del__(self):\n raise RuntimeError(\"del \ + failed\")\n\ndef make():\n return Cursed()\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, stdout, stderr) = run_python(&bundle, &[]); + assert!(ok, "unraisable errors must not fail the process: {stderr}"); + assert!(stdout.contains("done")); + assert!( + stderr.contains("Exception ignored in"), + "unraisable hook must keep the standard preamble: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 3, in __del__"), + "unraisable traceback must be remapped: {stderr}" + ); +} + +#[test] +fn runtime_survives_recursion_error() { + let dir = make_project(&[ + ("main.py", "from helper import spiral\n\nspiral()\n"), + ("helper.py", "def spiral():\n spiral()\n"), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("RecursionError"), "{stderr}"); + assert!( + stderr.contains("helper.py\", line 2, in spiral"), + "recursion frames must be remapped: {stderr}" + ); + assert!( + stderr.contains("[Previous line repeated"), + "repeated recursion frames must be collapsed: {stderr}" + ); + // Collapsing keeps the output small even for ~1000 recorded frames. + assert!( + stderr.lines().count() < 60, + "collapsed traceback expected, got {} lines", + stderr.lines().count() + ); +} + +#[cfg(unix)] +#[test] +fn runtime_survives_memory_pressure() { + let dir = make_project(&[ + ( + "main.py", + "import resource\nfrom helper import hoard\n\n_soft, hard = \ + resource.getrlimit(resource.RLIMIT_AS)\ntarget = 512 * 1024 * 1024\nif hard != \ + resource.RLIM_INFINITY:\n target = min(target, hard)\n\ + resource.setrlimit(resource.RLIMIT_AS, (target, hard))\nhoard()\n", + ), + ( + "helper.py", + "def hoard():\n blocks = []\n try:\n while True:\n \ + blocks.append(bytearray(16 * 1024 * 1024))\n except MemoryError:\n \ + blocks.clear()\n raise MemoryError(\"exhausted\") from None\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("MemoryError"), "{stderr}"); + // The runtime must never produce a secondary error, whichever path it took. + assert!( + !stderr.contains("Error in sys.excepthook"), + "the hook must not raise: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 8, in hoard"), + "MemoryError under an address-space limit should still remap: {stderr}" + ); +} + +#[cfg(unix)] +#[test] +fn runtime_falls_back_cleanly_on_fd_exhaustion() { + let dir = make_project(&[ + ( + "main.py", + "from helper import consume_fds_and_boom\n\nconsume_fds_and_boom()\n", + ), + ( + "helper.py", + "import resource\n\ndef consume_fds_and_boom():\n _soft, hard = \ + resource.getrlimit(resource.RLIMIT_NOFILE)\n \ + resource.setrlimit(resource.RLIMIT_NOFILE, (16, hard))\n holders = []\n \ + try:\n while True:\n holders.append(open(\"/dev/null\", \ + \"rb\"))\n except OSError:\n pass\n raise ValueError(\"fd exhausted \ + kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + // The map cannot be opened, so the runtime must fall back to the default + // traceback without any secondary noise. + assert!(stderr.contains("fd exhausted kaboom"), "{stderr}"); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "exactly one traceback expected: {stderr}" + ); + assert!( + !stderr.contains("Error in sys.excepthook"), + "the hook must not raise: {stderr}" + ); + assert!( + !stderr.contains("helper.py\", line"), + "with the map unreadable, frames stay on bundle coordinates: {stderr}" + ); +} + +#[test] +fn runtime_tolerates_broken_map_on_happy_path() { + let dir = fixture_project(); // non-throwing project + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + // Replace the map with garbage (and drop read permission on Unix, though + // that is a no-op when running as root). The runtime is fail-open, so this + // cannot *prove* the map is never touched — laziness itself is enforced by + // the runtime design (all map access lives behind the hook path). What it + // proves is that a broken or unreadable map never disturbs a successful run. + let map_path = dir.path().join("bundle.py.map"); + fs::write(&map_path, "NOT JSON {{{").expect("overwrite map"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(&map_path, fs::Permissions::from_mode(0o000)) + .expect("make map unreadable"); + } + + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "happy-path run must succeed: {stderr}"); + assert!(stdout.contains("hello world")); + assert!( + stderr.is_empty(), + "a broken map must not disturb a successful run: {stderr}" + ); +} + +// --------------------------------------------------------------------------- +// Precedence, environment configuration, and hook-chaining behavior +// --------------------------------------------------------------------------- + +#[test] +fn cli_sourcemap_flag_overrides_config_file() { + let dir = fixture_project(); + fs::write(dir.path().join("cribo.toml"), "sourcemap = \"external\"\n").expect("write config"); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--config", + &dir.path().join("cribo.toml").to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bundle.py.map"), + "CLI --sourcemap=linked must override the config file's external mode" + ); +} + +#[test] +fn env_var_enables_sourcemap_generation() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let output = Command::new(env!("CARGO_BIN_EXE_cribo")) + .args([ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + ]) + .env("CRIBO_SOURCEMAP", "external") + .env("CRIBO_SOURCES_CONTENT", "false") + .output() + .expect("run cribo binary"); + assert!( + output.status.success(), + "bundling must succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")) + .expect("CRIBO_SOURCEMAP env var must enable map emission"); + assert_map_covers_helper(&map_json, "bundle.py"); + assert!( + !map_json.contains("sourcesContent"), + "CRIBO_SOURCES_CONTENT=false must strip embedding" + ); +} + +#[test] +fn runtime_keeps_thread_sys_exit_silent() { + let dir = make_project(&[ + ( + "main.py", + "import sys\nimport threading\n\nworker = threading.Thread(target=lambda: \ + sys.exit(3))\nworker.start()\nworker.join()\nprint(\"done\")\n", + ), + ("helper.py", "unused = True\n"), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!( + ok, + "sys.exit in a worker thread must not fail the process: {stderr}" + ); + assert!(stdout.contains("done")); + assert!( + stderr.is_empty(), + "SystemExit in a thread must stay silent, as with the default hook: {stderr}" + ); +} + +#[test] +fn runtime_notifies_preinstalled_custom_excepthook() { + // A custom excepthook installed before the bundle's prologue (via + // sitecustomize) must still observe the exception after a successful remap. + let dir = crash_project(); + fs::write( + dir.path().join("sitecustomize.py"), + "import sys\n\n_original = sys.excepthook\n\n\ndef reporting_hook(exc_type, exc_value, \ + tb):\n print(\"REPORTER SAW:\", exc_type.__name__, file=sys.stderr)\n\n\nsys.excepthook \ + = reporting_hook\n", + ) + .expect("write sitecustomize"); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + + let mut command = Command::new(common::get_python_executable()); + command.arg(&bundle); + command.env_remove("CRIBO_SOURCE_MAPS"); + // Make usercustomize importable so the custom hook installs before the bundle. + command.env("PYTHONPATH", dir.path()); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert_remapped(&stderr); + assert!( + stderr.contains("REPORTER SAW: ValueError"), + "the preinstalled custom hook must still be notified after a remap: {stderr}" + ); +} + +// --------------------------------------------------------------------------- +// Second review round: shadowing, docstring, tracebacklimit, notes, handlers +// --------------------------------------------------------------------------- + +#[test] +fn runtime_survives_shadowing_threading_module() { + // A project file named threading.py next to the bundle must not be able + // to break (or be imported by) the runtime's own stdlib imports. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + fs::write( + dir.path().join("threading.py"), + "raise RuntimeError(\"shadow module imported\")\n", + ) + .expect("write shadowing module"); + + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok, "the crash must still surface"); + assert!( + !stderr.contains("shadow module imported"), + "the runtime must not import the adjacent threading.py: {stderr}" + ); + assert_remapped(&stderr); +} + +#[test] +fn bundle_docstring_survives_runtime_injection() { + let dir = make_project(&[ + ("main.py", "\"\"\"Entry doc.\"\"\"\n\nprint(__doc__)\n"), + ("helper.py", "unused = True\n"), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!( + stdout.contains("Entry doc."), + "the injected runtime must not displace the bundle docstring: {stdout}" + ); +} + +#[test] +fn runtime_honors_tracebacklimit() { + let dir = make_project(&[ + ( + "main.py", + // Set the limit on the real sys module: the bundler's stdlib + // import proxy forwards attribute reads but not writes, so a + // plain `sys.tracebacklimit = 0` would only decorate the proxy + // (true for bundles with or without source maps). + "from helper import boom\n\n__import__(\"sys\").tracebacklimit = 0\nboom()\n", + ), + ( + "helper.py", + "def boom():\n raise ValueError(\"limited kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("ValueError: limited kaboom"), "{stderr}"); + assert!( + !stderr.contains("Traceback (most recent call last):"), + "tracebacklimit = 0 must suppress the frame listing: {stderr}" + ); + assert!( + !stderr.contains("File \""), + "tracebacklimit = 0 must suppress all frames: {stderr}" + ); +} + +#[test] +fn runtime_renders_exception_notes() { + let dir = make_project(&[ + ("main.py", "from helper import boom\n\nboom()\n"), + ( + "helper.py", + "def boom():\n error = ValueError(\"kaboom\")\n error.add_note(\"NOTE: check \ + the flux capacitor\")\n raise error\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("helper.py\", line 4, in boom"), + "traceback must be remapped: {stderr}" + ); + assert!( + stderr.contains("NOTE: check the flux capacitor"), + "__notes__ must survive remapped rendering: {stderr}" + ); +} + +#[test] +fn map_covers_elif_and_except_headers() { + let dir = make_project(&[ + ( + "main.py", + "from helper import classify\n\nprint(classify(2))\n", + ), + ( + "helper.py", + "def classify(value):\n if value == 0:\n return \"zero\"\n elif value == \ + 1:\n return \"one\"\n elif value == 2:\n return \"two\"\n \ + try:\n return int(value)\n except ValueError:\n return \"other\"\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + let map = parse_map(&map_json); + + let helper_id = (0..map.get_sources().count() as u32) + .find(|id| { + map.get_source(*id) + .is_some_and(|s| s.ends_with("helper.py")) + }) + .expect("helper.py in sources"); + let mapped_helper_lines: Vec = map + .get_tokens() + .filter(|token| token.get_source_id() == Some(helper_id)) + .map(|token| token.get_src_line()) + .collect(); + // 0-based original lines: 3 and 5 are the `elif` headers, 9 is `except ValueError:`. + for header_line in [3, 5, 9] { + assert!( + mapped_helper_lines.contains(&header_line), + "helper.py 0-based line {header_line} (a clause header) must be mapped; mapped \ + lines: {mapped_helper_lines:?}" + ); + } +} + +// --------------------------------------------------------------------------- +// Third review round: specialized formatting, long chains, header coverage +// --------------------------------------------------------------------------- + +#[test] +fn runtime_keeps_name_error_suggestions() { + let dir = make_project(&[ + ("main.py", "from helper import go\n\ngo()\n"), + ( + "helper.py", + "def go():\n valuable = 1\n return valuabl\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("helper.py\", line 3, in go"), + "traceback must be remapped: {stderr}" + ); + assert!(stderr.contains("NameError"), "{stderr}"); + assert!( + stderr.contains("Did you mean"), + "interpreter suggestions must survive remapped rendering: {stderr}" + ); +} + +#[test] +fn runtime_renders_long_exception_chains_fully() { + // 20 chained causes exceed the previous traversal cap; the innermost + // (root) exception and its traceback must still be rendered. + let dir = make_project(&[ + ("main.py", "from helper import cascade\n\ncascade()\n"), + ( + "helper.py", + "def cascade():\n try:\n raise ValueError(\"root kaboom\")\n except \ + ValueError as error:\n current = error\n for depth in range(20):\n \ + try:\n raise RuntimeError(\"layer %d\" % depth) from \ + current\n except RuntimeError as next_error:\n current = \ + next_error\n raise current\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("ValueError: root kaboom"), + "the root cause of a 20-deep chain must be rendered: {stderr}" + ); + assert!(stderr.contains("layer 19"), "{stderr}"); + assert!( + stderr.contains("helper.py\", line 3"), + "the root cause frame must be remapped: {stderr}" + ); +} + +#[test] +fn map_covers_match_case_headers_and_decorators() { + let dir = make_project(&[ + ( + "main.py", + "from helper import Widget, run\n\nprint(run(1))\nprint(Widget())\n", + ), + ( + "helper.py", + "def trace(func):\n return func\n\n\n@trace\n@trace\ndef run(value):\n match \ + value:\n case 0:\n return \"zero\"\n case _ if value > \ + 0:\n return \"positive\"\n case _:\n return \ + \"negative\"\n\n\n@trace\nclass Widget(dict):\n pass\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + let map = parse_map(&map_json); + + let helper_id = (0..map.get_sources().count() as u32) + .find(|id| { + map.get_source(*id) + .is_some_and(|s| s.ends_with("helper.py")) + }) + .expect("helper.py in sources"); + let mapped_helper_lines: Vec = map + .get_tokens() + .filter(|token| token.get_source_id() == Some(helper_id)) + .map(|token| token.get_src_line()) + .collect(); + // 0-based original lines: 4 and 5 are the def decorators; 6 is the + // decorated `def run(value):` header itself; 8, 10, and 12 are the `case` + // headers; 16 and 17 are the decorated class's decorator and header (the + // inliner preserves the original name range on renamed identifiers, and + // the base-class list serves as a fallback anchor). + for header_line in [4, 5, 6, 8, 10, 12, 16, 17] { + assert!( + mapped_helper_lines.contains(&header_line), + "helper.py 0-based line {header_line} (decorator or case header) must be mapped; \ + mapped lines: {mapped_helper_lines:?}" + ); + } +} + +#[test] +fn runtime_survives_shadowed_builtins() { + // Entry code that rebinds common builtins at module level must not break + // the runtime: every method snapshots its builtins at definition time. + let dir = make_project(&[ + ( + "main.py", + "from helper import boom\n\nopen = None\nlen = None\nmax = None\nset = \ + None\ngetattr = None\nboom()\n", + ), + ( + "helper.py", + "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "traceback must be remapped despite shadowed builtins: {stderr}" + ); + assert!( + stderr.contains("main.py\", line 8, in "), + "entry frame must be remapped despite shadowed builtins: {stderr}" + ); + assert!(!stderr.contains("bundle.py\", line"), "{stderr}"); +} + +#[test] +fn runtime_survives_chdir_before_crash() { + // A bundle launched via a relative path that chdirs away before crashing + // must still locate itself and its sibling map (paths are anchored at + // startup). + let dir = make_project(&[ + ( + "main.py", + "import os\nfrom helper import boom\n\nos.chdir(\"/\")\nboom()\n", + ), + ( + "helper.py", + "def boom():\n raise ValueError(\"kaboom after chdir\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + + let mut command = Command::new(common::get_python_executable()); + // Invoke through the relative file name, from the bundle's directory. + command.arg(bundle.file_name().expect("bundle name")); + command.current_dir(dir.path()); + command.env_remove("CRIBO_SOURCE_MAPS"); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert!( + stderr.contains("helper.py\", line 2, in boom"), + "remapping must survive os.chdir before the crash: {stderr}" + ); + assert!(stderr.contains("kaboom after chdir")); +} + +#[test] +fn stacked_runtimes_do_not_duplicate_tracebacks() { + // Two source-mapped bundles executed in one interpreter stack their hooks; + // the outer runtime must recognize the inner one and not chain into it + // (which would fall through to the default printer and duplicate output). + let quiet = fixture_project(); + let quiet_bundle = quiet.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&quiet), + "--output", + &quiet_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let crash = crash_project(); + let crash_bundle = bundle_crash_project(&crash, "--sourcemap=linked"); + + let driver = crash.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({quiet:?})\nrunpy.run_path({crash:?})\n", + quiet = quiet_bundle.to_string_lossy(), + crash = crash_bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let (ok, _, stderr) = run_python(&driver, &[]); + assert!(!ok); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "stacked runtimes must render exactly one traceback: {stderr}" + ); + assert_eq!( + stderr.matches("ValueError: kaboom").count(), + 1, + "the exception line must print exactly once: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "the crashing bundle's frames must be remapped: {stderr}" + ); +} + +#[test] +fn runtime_remaps_when_group_is_suppressed() { + // A caught ExceptionGroup replaced via `raise ... from None` never + // renders; its hidden presence in __context__ must not force the + // unremapped fallback for the visible ordinary exception. + let dir = make_project(&[ + ("main.py", "from helper import convert\n\nconvert()\n"), + ( + "helper.py", + "def convert():\n try:\n raise ExceptionGroup(\"grp\", \ + [ValueError(\"inner\")])\n except ExceptionGroup:\n raise \ + RuntimeError(\"converted kaboom\") from None\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("RuntimeError: converted kaboom"), + "{stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in convert"), + "a suppressed group in __context__ must not disable remapping: {stderr}" + ); + assert!( + !stderr.contains("ExceptionGroup"), + "the suppressed group must not be rendered: {stderr}" + ); +} + +#[test] +fn runtime_rejects_mismatched_sibling_map() { + // Linked bundles embed a SHA-256 of their map; a sibling map from a + // different build (concurrent-build interleaving, manual copying) must be + // ignored rather than silently applying wrong mappings. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + + // Build a different project and steal its map. + let other = make_project(&[ + ("main.py", "from helper import other\n\nprint(other())\n"), + ("helper.py", "def other():\n return 42\n"), + ]); + let other_bundle = other.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&other), + "--output", + &other_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + fs::copy( + other.path().join("bundle.py.map"), + dir.path().join("bundle.py.map"), + ) + .expect("swap in a foreign map"); + + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[cfg(unix)] +#[test] +fn linked_comment_escapes_control_characters_in_names() { + // A newline inside the output file name must not break out of the + // sourceMappingURL comment (which would inject executable text). + let dir = fixture_project(); + let out = dir.path().join("bun\ndle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bun%0Adle.py.map"), + "control characters in the map name must be percent-encoded" + ); + // The bundle must remain valid, runnable Python. + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!(stdout.contains("hello world")); +} + +#[test] +fn linked_comment_preserves_unicode_names() { + // Non-ASCII characters in the output name must pass through verbatim + // (percent-encoding is reserved for control characters), so external map + // consumers following the comment can locate the sibling file. + let dir = fixture_project(); + let out = dir.path().join("bündle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bündle.py.map"), + "unicode map names must not be mangled" + ); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!(stdout.contains("hello world")); +} + +#[test] +fn external_mode_rejects_mismatched_sibling_map() { + // External mode embeds the same digest as linked mode; CRIBO_SOURCE_MAPS=1 + // against a foreign sibling map must fall back to the standard traceback. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=external"); + + let other = make_project(&[ + ("main.py", "from helper import other\n\nprint(other())\n"), + ("helper.py", "def other():\n return 42\n"), + ]); + let other_bundle = other.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&other), + "--output", + &other_bundle.to_string_lossy(), + "--sourcemap=external", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + fs::copy( + other.path().join("bundle.py.map"), + dir.path().join("bundle.py.map"), + ) + .expect("swap in a foreign map"); + + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "1")]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn stacked_runtimes_merge_maps_across_bundles() { + // A traceback crossing two source-mapped bundles (the crashing bundle + // calls a function retained from an earlier one) must remap the frames of + // BOTH bundles; unmapped driver frames keep standard rendering. + let provider = make_project(&[ + ( + "main.py", + // The real builtins module (bypassing the bundler's stdlib proxy, + // whose speculative import_module would add chained-context noise). + "from helper import provider_boom\n\n__import__(\"builtins\").provider_boom = \ + provider_boom\nprint(\"provider ready\")\n", + ), + ( + "helper.py", + "def provider_boom():\n raise ValueError(\"cross-bundle kaboom\")\n", + ), + ]); + let provider_bundle = provider.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&provider), + "--output", + &provider_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let caller = make_project(&[ + ( + "main.py", + "print(\"calling provider\")\n__import__(\"builtins\").provider_boom()\n", + ), + ("helper.py", "unused = True\n"), + ]); + let caller_bundle = caller.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&caller), + "--output", + &caller_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let driver = caller.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({provider:?})\nrunpy.run_path({caller:?})\n", + provider = provider_bundle.to_string_lossy(), + caller = caller_bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let (ok, _, stderr) = run_python(&driver, &[]); + assert!(!ok); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "exactly one traceback expected: {stderr}" + ); + // The caller bundle's frame (installed second, renders) is remapped... + assert!( + stderr.contains("main.py\", line 2, in "), + "the caller frame must be remapped: {stderr}" + ); + // ...and so is the provider bundle's frame, via the runtime registry. + assert!( + stderr.contains("helper.py\", line 2, in provider_boom"), + "the earlier bundle's frame must be remapped through the shared registry: {stderr}" + ); + assert!(stderr.contains("cross-bundle kaboom")); +} + +#[test] +fn runtime_refuses_map_after_bundle_replacement() { + // A bundle rebuilt at the same path while an old instance is running must + // not have its (new) map applied to the old in-memory code. The digest of + // this build's map is baked into the executing code, so the old process + // rejects the replacement map. Simulated here by rebuilding with shifted + // lines and running the ORIGINAL bundle against the REBUILT map. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let original_bundle = fs::read(&bundle).expect("read original bundle"); + + // Rebuild with different line numbers → different map (and digest). + fs::write( + dir.path().join("helper.py"), + "# shifted\n# shifted again\ndef boom():\n inner()\n\ndef inner():\n raise \ + ValueError(\"kaboom\")\n", + ) + .expect("shift helper lines"); + bundle_crash_project(&dir, "--sourcemap=linked"); + + // Old bundle + new map: the replacement pairing must be refused. + fs::write(&bundle, original_bundle).expect("restore original bundle"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn runtime_survives_shadowed_threading_under_runpy() { + // Under runpy, sys.path[0] is the DRIVER's directory; the bundle's own + // directory (reached via PYTHONPATH here) must still be filtered when the + // runtime imports its stdlib dependencies. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + fs::write( + dir.path().join("threading.py"), + "raise RuntimeError(\"shadow module imported\")\n", + ) + .expect("write shadowing module"); + + let driver_dir = TempDir::new().expect("create driver dir"); + let driver = driver_dir.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({bundle:?}, run_name=\"__main__\")\n", + bundle = bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let mut command = Command::new(common::get_python_executable()); + command.arg(&driver); + command.env_remove("CRIBO_SOURCE_MAPS"); + command.env("PYTHONPATH", dir.path()); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert!( + !stderr.contains("shadow module imported"), + "the runtime must not import the bundle-adjacent threading.py: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "remapping must work under runpy: {stderr}" + ); +} + +#[test] +fn stacked_runtimes_still_notify_preinstalled_custom_hook() { + // A custom hook installed before ANY bundle must still observe exceptions + // when several cribo runtimes have stacked on top of it: the notifier + // traverses through earlier cribo hooks to the original custom one. + let quiet = fixture_project(); + let quiet_bundle = quiet.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&quiet), + "--output", + &quiet_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let crash = crash_project(); + let crash_bundle = bundle_crash_project(&crash, "--sourcemap=linked"); + + let driver_dir = TempDir::new().expect("create driver dir"); + fs::write( + driver_dir.path().join("sitecustomize.py"), + "import sys\n\n\ndef reporting_hook(exc_type, exc_value, tb):\n print(\"REPORTER \ + SAW:\", exc_type.__name__, file=sys.stderr)\n\n\nsys.excepthook = reporting_hook\n", + ) + .expect("write sitecustomize"); + let driver = driver_dir.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({quiet:?})\nrunpy.run_path({crash:?})\n", + quiet = quiet_bundle.to_string_lossy(), + crash = crash_bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let mut command = Command::new(common::get_python_executable()); + command.arg(&driver); + command.env_remove("CRIBO_SOURCE_MAPS"); + command.env("PYTHONPATH", driver_dir.path()); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "remapping must work: {stderr}" + ); + assert!( + stderr.contains("REPORTER SAW: ValueError"), + "the custom hook below two stacked runtimes must still be notified: {stderr}" + ); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "no duplicate rendering through the default printer: {stderr}" + ); +} + +#[test] +fn runtime_remaps_raise_inside_multiline_fstring() { + // ruff's generator emits multiline f-strings on a single physical line + // (`\n` escapes), so a replacement expression raising on what was an + // interior line in the ORIGINAL source is attributed to the statement's + // single generated line — which must remap to the statement's original + // starting line. + let dir = make_project(&[ + ("main.py", "from helper import render\n\nprint(render(0))\n"), + ( + "helper.py", + "def render(value):\n banner = f\"\"\"first {value}\nsecond {1 // value}\nthird \ + {value}\"\"\"\n return banner\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("ZeroDivisionError"), "{stderr}"); + assert!( + stderr.contains("helper.py\", line 2, in render"), + "a raise inside a multiline f-string must map to the statement's original start line: \ + {stderr}" + ); +} + +#[test] +fn sources_with_url_delimiters_are_encoded_and_decoded() { + // '#' in a directory name would be read as a URL fragment by map + // consumers; the map must percent-encode it and the runtime must decode + // it back before touching the filesystem. + let dir = TempDir::new().expect("create temp dir"); + let src_dir = dir.path().join("we#ird"); + fs::create_dir_all(&src_dir).expect("create source dir"); + fs::write( + src_dir.join("main.py"), + "from helper import boom\n\nboom()\n", + ) + .expect("write main.py"); + fs::write( + src_dir.join("helper.py"), + "def boom():\n raise ValueError(\"kaboom\")\n", + ) + .expect("write helper.py"); + + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &src_dir.join("main.py").to_string_lossy(), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + assert!( + map_json.contains("we%23ird"), + "URL delimiters in source paths must be percent-encoded: {map_json}" + ); + + let (ok, _, stderr) = run_python(&out, &[]); + assert!(!ok); + assert!( + stderr.contains("we#ird/helper.py\", line 2, in boom") + || stderr.contains("we#ird\\helper.py\", line 2, in boom"), + "the runtime must decode the path before display and file access: {stderr}" + ); + assert!( + stderr.contains("raise ValueError(\"kaboom\")"), + "the original source line must load from the decoded path: {stderr}" + ); +} + +#[test] +fn digest_placeholder_in_user_code_is_left_untouched() { + // Only the prologue's own placeholder receives the digest; a user string + // that happens to share the spelling must survive verbatim, and the + // runtime must still verify (i.e. the prologue occurrence was the one + // substituted). + let dir = make_project(&[ + ( + "main.py", + "from helper import boom\n\nprint(\"__CRIBO_SOURCEMAP_DIGEST__\")\nboom()\n", + ), + ( + "helper.py", + "def boom():\n raise ValueError(\"kaboom\")\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert_eq!( + bundle.matches("__CRIBO_SOURCEMAP_DIGEST__").count(), + 1, + "exactly the user occurrence must remain (prologue one substituted)" + ); + + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(!ok); + assert!( + stdout.contains("__CRIBO_SOURCEMAP_DIGEST__"), + "user string must print unchanged: {stdout}" + ); + // Remapping proves the prologue digest matched the sibling map. + assert!( + stderr.contains("helper.py\", line 2, in boom"), + "traceback must remap, proving digest verification passed: {stderr}" + ); + assert!(stderr.contains("raise ValueError(\"kaboom\")"), "{stderr}"); +} + +#[cfg(unix)] +#[test] +fn linked_comment_encodes_non_utf8_names() { + // Unix filenames need not be UTF-8. A lossy conversion would bake U+FFFD + // into the sourceMappingURL comment — a name that does not exist on disk. + // The raw invalid byte must be percent-encoded instead. + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt as _; + + let dir = fixture_project(); + let mut name = b"bun".to_vec(); + name.push(0xFF); + name.extend_from_slice(b"dle.py"); + let out = dir.path().join(OsString::from_vec(name)); + let output = Command::new(env!("CARGO_BIN_EXE_cribo")) + .arg("--entry") + .arg(dir.path().join("main.py")) + .arg("--output") + .arg(&out) + .arg("--sourcemap=linked") + .output() + .expect("run cribo"); + assert!( + output.status.success(), + "bundling must succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bun%FFdle.py.map"), + "invalid bytes must be percent-encoded, not replaced with U+FFFD" + ); + assert!( + !bundle.contains('\u{FFFD}'), + "no lossy replacement character may reach the comment" + ); + // The bundle must remain valid, runnable Python (with its sibling map). + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!(stdout.contains("hello world")); +} + +#[test] +fn bundle_namespace_is_clean_of_runtime_helpers() { + // The prologue must leave no helper names behind: user code and + // `from bundle import *` consumers must not see (or collide with) them. + let dir = make_project(&[( + "main.py", + "prologue_names = (\"_cribo_sys\", \"_cribo_sm_import\", \"_CriboSmStream\", \"_CriboSourceMapRuntime\")\nleaked = sorted(name for name in prologue_names if name in globals())\nprint(\"LEAKED:\", leaked)\n", + )]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!( + stdout.contains("LEAKED: []"), + "runtime helper names must not leak into the bundle namespace: {stdout}" + ); +} + +#[test] +fn map_covers_decorated_class_without_argument_list() { + // `class Widget:` has no base-class list to fall back on, and the inliner + // rewrites every inlined class name — the header anchor must survive via + // the preserved identifier range, or exceptions CPython attributes to the + // class header during construction (e.g. a descriptor raising from + // `__set_name__`) stay on bundle coordinates. + let dir = make_project(&[ + ( + "main.py", + "from helper import Widget\n\nprint(Widget().value)\n", + ), + ( + "helper.py", + "def trace(cls):\n return cls\n\n\n@trace\nclass Widget:\n def \ + __init__(self):\n self.value = \"widget value\"\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + let map = parse_map(&map_json); + + let helper_id = (0..map.get_sources().count() as u32) + .find(|id| { + map.get_source(*id) + .is_some_and(|s| s.ends_with("helper.py")) + }) + .expect("helper.py in sources"); + let mapped_helper_lines: Vec = map + .get_tokens() + .filter(|token| token.get_source_id() == Some(helper_id)) + .map(|token| token.get_src_line()) + .collect(); + // 0-based: 4 is the `@trace` decorator, 5 the bare `class Widget:` header. + for header_line in [4, 5] { + assert!( + mapped_helper_lines.contains(&header_line), + "helper.py 0-based line {header_line} (decorator or bare class header) must be \ + mapped; mapped lines: {mapped_helper_lines:?}" + ); + } +} + +#[test] +fn truncated_traceback_keeps_pep657_on_unmapped_frames() { + // With a positive sys.tracebacklimit the runtime keeps the LAST n frames + // (like the interpreter's C printer), while the traceback module keeps + // the FIRST n. The PEP 657 summaries must therefore be extracted + // untruncated, or a retained unmapped frame would index past (or into the + // wrong slot of) the shortened list and lose its caret anchors. + let dir = make_project(&[ + ( + "main.py", + "import importlib.util\nimport os\n\nfrom helper import boom\n\n__import__(\"sys\")\ + .tracebacklimit = 2\nspec = importlib.util.spec_from_file_location(\"ext\", \ + os.path.join(os.path.dirname(os.path.abspath(__file__)), \"ext.py\"))\next = \ + importlib.util.module_from_spec(spec)\nspec.loader.exec_module(ext)\nboom(ext)\n", + ), + ("helper.py", "def boom(ext):\n ext.go()\n"), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + // Placed next to the bundle and loaded by explicit path at run time, so + // cribo never bundles it: its frames stay unmapped by design. + fs::write( + dir.path().join("ext.py"), + "def go():\n left = 1\n return left + \"boom\"\n", + ) + .expect("write ext.py"); + + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + // Retained frames: helper.boom (remapped) and ext.go (unmapped). + assert!( + stderr.contains("helper.py\", line 2, in boom"), + "the retained bundle frame must remap: {stderr}" + ); + assert!( + stderr.contains("ext.py\", line 3, in go"), + "the retained external frame must render: {stderr}" + ); + assert!( + !stderr.contains("main.py"), + "tracebacklimit = 2 must drop the module frame: {stderr}" + ); + // The caret line proves the frame came from the standard summaries (the + // plain fallback prints only the file line and source text). + assert!( + stderr.contains('^'), + "the unmapped frame must keep its PEP 657 carets: {stderr}" + ); +} diff --git a/docs/source-maps.md b/docs/source-maps.md new file mode 100644 index 000000000..c2dde216b --- /dev/null +++ b/docs/source-maps.md @@ -0,0 +1,267 @@ +# Source Map v3 Support + +Status: implemented. This document records the agreed design plus the +implementation notes at the end. + +## Problem Statement + +Cribo produces a single bundled `.py` file; when that bundle raises an exception, +tracebacks point at bundle line numbers instead of the original source files, making +debugging painful. This feature adds opt-in Source Map v3 generation (the +language-agnostic JS-ecosystem format) plus an injected Python runtime that remaps +tracebacks to original sources at run time, analogous to `node --enable-source-maps`. + +## Requirements + +1. **Granularity:** statement/line-level mappings (column 0) — sufficient for Python's + line-oriented tracebacks. +2. **Delivery:** esbuild-style `--sourcemap[=linked|inline|external]` (bare flag = + `linked`). + - `linked`: write `.map` next to the output and append a + `# sourceMappingURL=.map` comment as the last line. + A newly created map file is owner-only (`0600` on Unix, set atomically at + creation) since it may embed `sourcesContent`; rebuilds preserve whatever + permissions the existing map carries, so a one-time `chmod` sticks. + - `inline`: append a `# sourceMappingURL=data:application/json;base64,...` comment. + - `external`: write the `.map` file with no comment. +3. **Runtime traceback injection** is bundled into the output whenever `--sourcemap` + is used; activation depends on the mode: + - `inline` → active by default (`CRIBO_SOURCE_MAPS=0` acts as a kill-switch). + - `external` → gated on the env var `CRIBO_SOURCE_MAPS=1` (a value that is a path + is treated as the map file location). + - `linked` → active iff the sibling `.map` file exists at run time; silently + disabled otherwise. +4. **Crate:** `oxc_sourcemap` v8.x (pinned exact version), BSD-3-Clause, maintained at + `oxc-project/oxc-sourcemap`, used by rolldown/rspack. Its dependencies + (`rustc-hash 2`, `serde`, `serde_json`, `base64-simd`, `json-escape-simd`) align + with the workspace; MSRV 1.95 is below the repo toolchain 1.97.1. +5. **`sourcesContent`:** the default depends on the mode — omitted for `inline`, + included for `linked`/`external`; forcible override via `--sources-content=true|false` + (and the matching config key). +6. **Hook scope:** full set — `sys.excepthook`, `threading.excepthook`, + `sys.unraisablehook`; chain to previously installed hooks; never crash (fail open + to default behavior). +7. **Laziness:** zero-to-negligible happy-path cost. The prologue does NO I/O, no + parsing, no decoding at startup — it only defines functions, stores `__file__` and + the bundle-time mode constant, and installs the 3 hooks. Activation gating itself + (map existence check, env var read, data-URL location) is deferred to the first + exception. +8. **Duress tolerance:** the decoder must work under resource-constrained conditions + (`MemoryError`, `RecursionError`, FD exhaustion) — streaming, constant-memory + implementation; performance is secondary on the error path. + +## Design + +### Mapping extraction (Rust side) + +Cribo emits the bundle by unparsing the merged AST statement-by-statement with ruff's +`Generator` (`orchestrator.rs::bundle_to_string` → +`code_generator/python_codegen.rs::generate_statement`). Ruff codegen emits no +position info, so mappings cannot be captured during emission. + +Instead, cribo re-parses the final bundle text once with ruff's parser and performs a +defensive parallel statement-only walk against the bundled AST (which carries node +provenance via node indices and original `TextRange`s). Each aligned statement with +provenance yields one mapping: generated line → (original file, original line). On +structural divergence (e.g., the class-pattern rewriter's string patching), the walk +logs at debug level and skips the subtree rather than failing. The re-parse doubles +as a bundle-validity check. + +Provenance groundwork already exists: + +- `ast_indexer.rs` gives each module a 1,000,000-wide node-index range + (`node_index / MODULE_INDEX_RANGE` = module ordinal); statements copied from module + ASTs retain their original `TextRange`. +- Synthesized nodes get indices from `transformation_context.rs` and are + distinguishable (no original range) — they produce no mapping. + +Key components: + +- `crates/cribo/src/source_map.rs`: mapping record builder wrapping + `oxc_sourcemap::SourceMapBuilder`, provenance resolution, and the parallel walk. +- Each parsed module's source text plus a line-offset index is retained (keyed by + `ModuleId`) through `bundle_core` — needed both for offset→line conversion and for + `sourcesContent`. +- `sources` paths are recorded relative to the map location (esbuild convention), + with an empty `sourceRoot`. +- `--stdout` interplay: bare `--sourcemap` with `--stdout` defaults to `inline`; + explicit `linked`/`external` with `--stdout` is an error suggesting `inline`. +- Config-file (`cribo.toml`) equivalents: `sourcemap = "linked" | "inline" | "external"`, + `sources-content = true | false`. + +### Runtime prologue (Python side) — lazy, streaming, duress-tolerant + +The template lives in `crates/cribo/src/python/` and is embedded via `include_str!`, +injected as a prologue when source maps are enabled, with the delivery mode baked in +as a constant. + +**Happy path:** define functions + 3 hook assignments (capturing previous hooks for +chaining). Nothing else — no file I/O, no parsing, no decoding. + +**Exception path pipeline:** + +1. **Collect needed lines.** Walk the `tb_next` chain (plus `__cause__`/`__context__`) + collecting `tb_lineno` for frames whose `co_filename` matches the bundle path. + Result: a small set of ints and `max_needed` for early exit. +2. **Locate the mappings bytes without loading the map.** + - `linked`/`external`: stream the `.map` file in fixed 8 KiB chunks. + - `inline`: open the bundle file, `seek()` to EOF, scan backward in chunks for the + last newline to find the `# sourceMappingURL=data:...;base64,` line (the bundle + body is never read), then decode the base64 payload incrementally with + `binascii.a2b_base64` on 4-byte-aligned chunk boundaries. +3. **Targeted streaming JSON field extraction.** The map is NOT parsed with + `json.loads`. A minimal JSON string-lexer scans the chunk stream for the top-level + `"sources"` key (a small array, parsed into a list) and the `"mappings"` key + (consumed by the VLQ state machine directly from the stream, never held whole in + memory). The lexer handles escaped quotes so `sourcesContent` values containing + fake `"mappings":` keys cannot fool it. +4. **Streaming VLQ state machine, constant memory.** State is six integers: + `gen_line`, running deltas `src_idx` and `src_line`, VLQ accumulators `vlq_value` + and `vlq_shift`, and a `field` counter. `;` increments `gen_line`, `,` ends a + segment; `(gen_line → (src_idx, src_line))` is recorded only for lines in the + needed set (first segment per line); the machine early-exits once + `gen_line > max_needed`. Total heap: chunk buffer + six ints + k result entries. +5. **Re-render, best-effort in layers.** The file:line remap is always attempted; + the original source-line *text* is decoration in a separate `try` — stream-read + just the needed line from the original file on disk (iterate, never slurp; NO + `linecache` — it caches whole files), or a second targeted pass over + `sourcesContent`; skipped silently on failure. + +**Failure containment:** + +- Fallback ladder, never mask the real error: (1) streaming targeted path → (2) one + attempt at plain `json.loads` → (3) delegate to the captured previous hook with the + original exception. The entire hook body is wrapped catching `BaseException` raised + by our own code. +- No imports inside the hook: `sys`, `os`, etc. are bound at prologue time. The + renderer formats frames directly from `tb_frame.f_code` attributes mirroring + CPython's format (self-contained; no dependency on `traceback.TracebackException`). +- Iterative code only (no recursion); the recursion limit is bumped by a small margin + (`sys.setrecursionlimit(cur + 64)`) inside a `try` before rendering and restored + after. +- Re-entrancy guard: a module-level flag prevents the hook recursing into itself. + +**Documented caveats:** + +- Under hard OOM where the interpreter cannot allocate at all, no pure-Python hook + can run; the target is constrained-but-alive conditions with guaranteed + non-interference at the floor. +- User code calling `traceback.format_exc()` (or otherwise formatting tracebacks + itself) is not remapped — only the installed hooks re-render. + +## Task Breakdown + +- **Task 0:** this document, cross-referenced from `docs/static-bundling.md`. +- **Task 1:** `oxc_sourcemap` workspace dependency (pinned) + `source_map.rs` with a + `SourceMapGenerator` accepting `(generated_line, source_file, original_line)` + records and optional per-source content, serializing valid Source Map v3 JSON. + Unit tests: mapping order, VLQ round-trip via the crate's consumer API, + `sourcesContent` on/off, empty map. +- **Task 2:** retain module source text + line-offset index keyed by `ModuleId`; + provenance resolver `node_index` → module ordinal → original file path and + `TextRange.start()` → original line; `None` for synthesized nodes. Unit tests with + multi-module inputs including a synthesized-node case. +- **Task 3:** mapping extraction via re-parse + parallel statement walk (including + statements nested in function/class bodies and wrapper-module init functions), + integrating Tasks 1+2 into a complete `SourceMap` per bundle. Tests assert selected + known mappings for inlined-module, wrapper-module, and class-pattern bundles. +- **Task 4:** CLI `--sourcemap[=linked|inline|external]` (clap `ValueEnum`, bare = + `linked`) + `cribo.toml` key; the three delivery modes including the + `# sourceMappingURL=` trailer; `--stdout` interplay. Integration tests per mode. +- **Task 5:** `sourcesContent` mode-dependent default with `--sources-content` + override (+ config key). Tests cover the default matrix and both overrides. +- **Task 6:** Python runtime prologue per the design above, with a pytest suite for + the unit-testable pieces (VLQ state machine against maps generated by + `oxc_sourcemap`, backward EOF scan, base64 chunk alignment, JSON scanner against + adversarial `sourcesContent`) and an integration test asserting a remapped + traceback on stderr. +- **Task 7:** `threading.excepthook` + `sys.unraisablehook`; duress tests + (`RecursionError` at depth, `MemoryError`, FD exhaustion via + `resource.setrlimit(RLIMIT_NOFILE, ...)`), activation-matrix tests, and a laziness + test asserting a non-throwing run performs no map access. +- **Task 8:** snapshot-framework integration (fixtures opt into source maps, + snapshotting remapped-traceback output), at least two fixtures, README/CLI docs, + and caveat documentation. + + +## Implementation Notes + +Decisions made (or refined) during implementation: + +- **Prologue injection is AST-level, not text-level.** The runtime template + (`crates/cribo/src/python/sourcemap_runtime.py`, embedded via `include_str!`) + is parsed with ruff and its statements are spliced into the bundled AST after + any leading `from __future__` imports, *before* code generation. The extraction + walk therefore stays structurally aligned automatically (prologue statements + carry no provenance and simply produce no mappings), with no line-offset + bookkeeping. +- **Source line text comes from disk only.** The runtime resolves relative + source paths against the map's directory and stream-reads the single needed + line. A second streaming pass over `sourcesContent` for line text was + deliberately skipped to bound runtime complexity — `sourcesContent` is still + embedded per the configured policy for external tooling (IDEs, error + trackers). When original files are absent, tracebacks still remap `file:line` + and simply omit the source-line text. +- **Repeated frames are collapsed like CPython** (at most 3 identical + consecutive frames, then `[Previous line repeated N more times]`), with a + per-render line-text cache; a `RecursionError` traceback stays small and does + not trigger thousands of file reads. +- **`threading` and `traceback` are imported at bundle startup** (through a + shadow-proof importer that skips the script directory) so + `threading.excepthook` can be installed and exception formatting cannot be + degraded by a first-party module registering `sys.modules["traceback"]`; + these are the only non-trivial startup costs and are negligible in practice. + Map location checks, environment reads, file access, and decoding all remain + deferred to the first exception. +- **Snapshot integration:** fixtures under `crates/cribo/tests/fixtures/` whose + name starts with `sourcemap_` are bundled with `--sourcemap=linked` and gain a + `source_map@.snap` snapshot: a normalized, path-free dump of every + mapping (`bundle: `` -> :`). Remapped + *traceback output* is asserted exactly (not snapshotted) in + `crates/cribo/tests/test_source_maps.rs`, which also covers the activation + matrix, thread/unraisable hooks, and the duress suite (RecursionError, + MemoryError under `RLIMIT_AS`, FD exhaustion under `RLIMIT_NOFILE`, and + happy-path laziness with an unreadable map). + +Known limitations (documented in the README as well): + +- User code that formats tracebacks itself (`traceback.format_exc()`, + `traceback.print_exc()`, custom formatters) is not remapped; only the + installed hooks re-render. +- Under a hard out-of-memory condition where the interpreter cannot allocate at + all, no pure-Python hook can run; the guarantee is non-interference (the + default traceback still prints) rather than remapping. +- Mappings are statement/line-level with column 0 by design; Python tracebacks + are line-oriented, so finer columns would add cost without changing the + rendered output. + + +## Review-Driven Refinements (PR #570) + +- **The runtime is a single class** (`_CriboSourceMapRuntime`); all collaborators + (modules, previous hooks, state) live on the instance and the installed hooks + are bound methods, so bundled user code rebinding any injected module-level + name cannot break remapping. +- **Custom pre-installed hooks stay notified.** After a successful remap, a + previous `sys.excepthook`/`threading.excepthook`/`sys.unraisablehook` that + differs from the interpreter default is still invoked (error reporters and + `sitecustomize` integrations keep working); the default printer is the only + thing the remap replaces. +- **`SystemExit` in worker threads stays silent**, matching the default + `threading.excepthook`. +- **`ExceptionGroup` chains defer to the previous hook** (complete, unremapped + rendering) rather than losing nested tracebacks. +- **The re-entrancy guard is thread-local.** +- **`CRIBO_SOURCE_MAPS=` works in every mode** and is the supported way to + remap a bundle executed via `python -` (stdin), whose own file cannot be + re-read; without it, inline mode deactivates gracefully for `` bundles. +- **Environment configuration:** `CRIBO_SOURCEMAP=linked|inline|external` and + `CRIBO_SOURCES_CONTENT=` participate in the standard `CRIBO_*` layering. +- **The template's docstring is stripped at injection** so enabling source maps + does not change the bundle's `__doc__`. +- **`elif` headers get their own mappings** (Python reports the header line when + a condition raises), using the condition expression's provenance. +- **Linked/external maps are written only after the bundle write succeeds**, so + a failed run cannot leave a stale map next to an older bundle. +- `relative_path` refuses lexically non-invertible bases (`..` components) and + falls back to the absolute source path. diff --git a/docs/static-bundling.md b/docs/static-bundling.md index eabf81312..0e44faf5c 100644 --- a/docs/static-bundling.md +++ b/docs/static-bundling.md @@ -563,6 +563,10 @@ The simple renaming approach is tempting for its simplicity, but Python's dynami ## Source Map Support +> **Note:** the agreed, implemented design for source maps lives in +> [`docs/source-maps.md`](./source-maps.md). The sections below are the original +> exploratory sketch and are kept for historical context. + ### Overview Source maps enable debugging of bundled code by mapping locations in the bundle back to original source files. Cribo should adopt the JavaScript Source Map v3 specification for Python bundles.