From cc557b291ae428d08489de166ea2a0145c8a8334 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 14:58:41 +0200 Subject: [PATCH 01/17] feat: Source Map v3 generation with runtime traceback remapping Adds opt-in --sourcemap[=linked|inline|external] (esbuild-style) producing Source Map v3 JSON via oxc_sourcemap, mapping every bundled statement back to its original file and line. Bundles built with source maps carry an injected runtime that remaps uncaught-exception tracebacks to original sources (analogous to node --enable-source-maps): lazy (zero I/O until the first exception), constant-memory streaming VLQ decoder resilient to MemoryError / RecursionError / FD exhaustion, covering sys.excepthook, threading.excepthook and sys.unraisablehook with fail-open fallback to the standard traceback. Runtime activation per mode: inline always (CRIBO_SOURCE_MAPS=0 kill switch), linked gated on sibling .map presence, external gated on CRIBO_SOURCE_MAPS. sourcesContent defaults per mode (omitted inline, embedded otherwise) with a --sources-content override. Design doc at docs/source-maps.md. --- Cargo.lock | 43 + Cargo.toml | 64 +- README.md | 53 ++ crates/cribo/Cargo.toml | 2 + crates/cribo/src/config.rs | 41 + crates/cribo/src/lib.rs | 1 + crates/cribo/src/main.rs | 42 +- crates/cribo/src/orchestrator.rs | 233 +++++- crates/cribo/src/python/sourcemap_runtime.py | 603 ++++++++++++++ crates/cribo/src/source_map.rs | 727 +++++++++++++++++ .../fixtures/sourcemap_basic/calculator.py | 8 + .../tests/fixtures/sourcemap_basic/main.py | 7 + .../tests/fixtures/sourcemap_basic/utils.py | 2 + .../fixtures/sourcemap_wrapper/effects.py | 8 + .../tests/fixtures/sourcemap_wrapper/main.py | 4 + .../tests/python/test_sourcemap_runtime.py | 203 +++++ .../bundled_code@sourcemap_basic.snap | 651 ++++++++++++++++ .../bundled_code@sourcemap_wrapper.snap | 658 ++++++++++++++++ .../execution_results@sourcemap_basic.snap | 9 + .../execution_results@sourcemap_wrapper.snap | 9 + .../requirements@sourcemap_basic.snap | 6 + .../requirements@sourcemap_wrapper.snap | 6 + .../ruff_lint_results@sourcemap_basic.snap | 10 + .../ruff_lint_results@sourcemap_wrapper.snap | 10 + .../snapshots/source_map@sourcemap_basic.snap | 16 + .../source_map@sourcemap_wrapper.snap | 11 + crates/cribo/tests/test_bundling_snapshots.rs | 57 ++ crates/cribo/tests/test_source_maps.rs | 737 ++++++++++++++++++ docs/source-maps.md | 230 ++++++ docs/static-bundling.md | 4 + 30 files changed, 4418 insertions(+), 37 deletions(-) create mode 100644 crates/cribo/src/python/sourcemap_runtime.py create mode 100644 crates/cribo/src/source_map.rs create mode 100644 crates/cribo/tests/fixtures/sourcemap_basic/calculator.py create mode 100644 crates/cribo/tests/fixtures/sourcemap_basic/main.py create mode 100644 crates/cribo/tests/fixtures/sourcemap_basic/utils.py create mode 100644 crates/cribo/tests/fixtures/sourcemap_wrapper/effects.py create mode 100644 crates/cribo/tests/fixtures/sourcemap_wrapper/main.py create mode 100644 crates/cribo/tests/python/test_sourcemap_runtime.py create mode 100644 crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap create mode 100644 crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap create mode 100644 crates/cribo/tests/snapshots/execution_results@sourcemap_basic.snap create mode 100644 crates/cribo/tests/snapshots/execution_results@sourcemap_wrapper.snap create mode 100644 crates/cribo/tests/snapshots/requirements@sourcemap_basic.snap create mode 100644 crates/cribo/tests/snapshots/requirements@sourcemap_wrapper.snap create mode 100644 crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_basic.snap create mode 100644 crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_wrapper.snap create mode 100644 crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap create mode 100644 crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap create mode 100644 crates/cribo/tests/test_source_maps.rs create mode 100644 docs/source-maps.md diff --git a/Cargo.lock b/Cargo.lock index 40914acc2..e26880f4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,6 +149,16 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -414,6 +424,7 @@ name = "cribo" version = "0.9.2" dependencies = [ "anyhow", + "base64-simd", "clap", "cow-utils", "criterion", @@ -423,6 +434,7 @@ dependencies = [ "insta", "log", "once_cell", + "oxc_sourcemap", "pep508_rs", "petgraph", "pretty_assertions", @@ -1173,6 +1185,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-escape-simd" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c22a2041e3874a055a4eb03ea2395aaccdefa84ce75b31d542d72a741c3c6ad3" + [[package]] name = "libc" version = "0.2.186" @@ -1326,6 +1344,25 @@ dependencies = [ "indexmap", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "oxc_sourcemap" +version = "8.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b415102a94b483bbd76d13e850fe87961197e5795c79a8523ebec5d82025f94f" +dependencies = [ + "base64-simd", + "json-escape-simd", + "rustc-hash", + "serde", + "serde_json", +] + [[package]] name = "page_size" version = "0.6.0" @@ -2789,6 +2826,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vte" version = "0.14.1" diff --git a/Cargo.toml b/Cargo.toml index 19496faf7..953b53a0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,10 +25,13 @@ ruff_python_semantic = { git = "https://github.com/astral-sh/ruff/", tag = "0.16 ruff_python_stdlib = { git = "https://github.com/astral-sh/ruff/", tag = "0.16.3" } ruff_text_size = { git = "https://github.com/astral-sh/ruff/", tag = "0.16.3" } +# Source map generation (Source Map v3, pinned exact version) +oxc_sourcemap = "=8.1.2" + # Serialization and configuration -serde = { version = "1.0", features = ["derive"] } +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -toml = "0.9" +toml = "0.9" # Package name normalization (PEP 503) pep508_rs = "0.9" @@ -43,8 +46,9 @@ petgraph = "0.8" rustc-hash = "2.1" # Utilities -cow-utils = "0.1.3" -etcetera = "0.11" +base64-simd = "0.8" +cow-utils = "0.1.3" +etcetera = "0.11" # Testing insta = { version = "1.43.1", features = ["filters", "glob", "yaml"] } @@ -62,17 +66,17 @@ unsafe_code = "warn" # Add these for better AI-generated code catching: exported_private_dependencies = { level = "allow", priority = 10 } missing_debug_implementations = "warn" -rust_2018_compatibility = { level = "warn", priority = -2 } -rust_2018_idioms = { level = "warn", priority = -2 } -rust_2021_compatibility = { level = "warn", priority = -2 } -rust_2024_compatibility = { level = "warn", priority = -2 } # Warns about 2015 idioms in 2018 edition +missing_docs = "allow" # Or "allow" if too noisy +rust_2018_compatibility = { level = "warn", priority = -2 } +rust_2018_idioms = { level = "warn", priority = -2 } +rust_2021_compatibility = { level = "warn", priority = -2 } +rust_2024_compatibility = { level = "warn", priority = -2 } # Warns about 2015 idioms in 2018 edition trivial_casts = "warn" trivial_numeric_casts = "warn" unused_extern_crates = "warn" unused_import_braces = "warn" unused_qualifications = "warn" variant_size_differences = "warn" -missing_docs = "allow" # Or "allow" if too noisy [workspace.lints.clippy] # Categories @@ -103,7 +107,7 @@ similar_names = "allow" single_match_else = "allow" too_many_lines = "deny" unnecessary_debug_formatting = "allow" -unused_self = "allow" +unused_self = "allow" # Additional allows for practical development cast_possible_truncation = "allow" @@ -149,29 +153,29 @@ path_buf_push_overwrite = "warn" ptr_as_ptr = "warn" # redundant_pub_crate disabled due to conflict with unreachable-pub lint # When a module is pub but items inside are pub(crate), both lints fire incorrectly -redundant_pub_crate = "allow" -ref_binding_to_reference = "warn" -ref_option_ref = "warn" -semicolon_if_nothing_returned = "warn" -str_to_string = "warn" -string_add = "warn" -string_add_assign = "warn" -string_lit_as_bytes = "warn" +redundant_pub_crate = "allow" +ref_binding_to_reference = "warn" +ref_option_ref = "warn" +semicolon_if_nothing_returned = "warn" +str_to_string = "warn" +string_add = "warn" +string_add_assign = "warn" +string_lit_as_bytes = "warn" # string_to_string has been removed - implicit_clone covers those cases -type_repetition_in_bounds = "warn" -unnecessary_self_imports = "warn" -unnecessary_wraps = "warn" -unneeded_field_pattern = "warn" -unnested_or_patterns = "warn" -unseparated_literal_suffix = "warn" -unused_async = "warn" -use_self = "warn" -useless_let_if_seq = "warn" -verbose_bit_mask = "warn" -wildcard_imports = "warn" +type_repetition_in_bounds = "warn" +unnecessary_self_imports = "warn" +unnecessary_wraps = "warn" +unneeded_field_pattern = "warn" +unnested_or_patterns = "warn" +unseparated_literal_suffix = "warn" +unused_async = "warn" +use_self = "warn" +useless_let_if_seq = "warn" +verbose_bit_mask = "warn" +wildcard_imports = "warn" # Add these specific lints to catch .ok() misuse -match_result_ok = "warn" # Catches if let Some(x) = result.ok() (renamed from if_let_some_result) +match_result_ok = "warn" # Catches if let Some(x) = result.ok() (renamed from if_let_some_result) option_map_unit_fn = "warn" # Catches .map(|_| ()) question_mark = "warn" # Encourages using ? operator result_map_unit_fn = "warn" # Catches .map(|_| ()) diff --git a/README.md b/README.md index 73970d68e..014574b99 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,9 @@ cribo --entry src/main.py --output bundle.py -vvv # trace level # Custom config file cribo --entry src/main.py --output bundle.py --config my-cribo.toml + +# Generate a Source Map v3 and remap runtime tracebacks to original sources +cribo --entry src/main.py --output bundle.py --sourcemap ``` ### CLI Options @@ -155,6 +158,8 @@ cribo --entry src/main.py --output bundle.py --config my-cribo.toml - `--python `: Python interpreter whose installed distribution metadata is used for requirements - `--no-tree-shake`: Disable tree-shaking optimization (tree-shaking is enabled by default) - `--target-version `: Target Python version (e.g., py38, py39, py310, py311, py312, py313) +- `--sourcemap[=]`: Generate a Source Map v3 for the bundle and inject a traceback-remapping runtime. Modes: `linked` (default; writes `.map` plus a `# sourceMappingURL=` comment), `inline` (embeds the map as a base64 comment; the default with `--stdout`), `external` (writes `.map` with no comment). See [Source Maps](#source-maps) +- `--sources-content=`: Force embedding original sources in the map (default: omitted for `inline`, included for `linked`/`external`) - `-h, --help`: Print help information - `-V, --version`: Print version information @@ -212,6 +217,54 @@ cribo --entry main.py --output bundle.py --no-tree-shake - When you need to preserve all code for dynamic imports or reflection - For debugging purposes to see the complete bundled output +### Source Maps + +Cribo can emit a [Source Map v3](https://tc39.es/ecma426/) (the language-agnostic +format used across the JavaScript ecosystem) mapping every statement in the bundle +back to its original file and line, plus an injected runtime that remaps uncaught +exception tracebacks to the original sources — analogous to +`node --enable-source-maps`: + +```bash +# linked (default): writes bundle.py.map + a trailing sourceMappingURL comment +cribo --entry src/main.py --output bundle.py --sourcemap + +# inline: the map travels inside the bundle as a base64 comment +cribo --entry src/main.py --output bundle.py --sourcemap=inline + +# external: writes bundle.py.map, no comment in the bundle +cribo --entry src/main.py --output bundle.py --sourcemap=external +``` + +With source maps enabled, a crash inside the bundle prints the original locations: + +```text +Traceback (most recent call last): + File "/app/src/main.py", line 3, in + boom() + File "/app/src/helper.py", line 5, in inner + raise ValueError("kaboom") +ValueError: kaboom +``` + +Runtime activation follows the delivery mode: + +- `inline`: active by default; `CRIBO_SOURCE_MAPS=0` disables it +- `linked`: active exactly when `.map` exists next to the bundle at run + time — delete the map to ship without remapping, drop it back to re-enable +- `external`: dormant unless `CRIBO_SOURCE_MAPS=1` is set (or the variable holds a + path to the map file) + +The runtime is lazy (zero file access, parsing, or decoding until the first +uncaught exception), streams the map in constant memory so it works under resource +pressure, covers `sys.excepthook`, `threading.excepthook`, and +`sys.unraisablehook`, and falls back to the standard traceback on any failure. +Known limitations: user code that formats tracebacks itself (e.g. +`traceback.format_exc()`) is not remapped, and under a hard out-of-memory +condition no pure-Python hook can run. Configuration-file equivalents: +`sourcemap = "linked" | "inline" | "external"` and `sources-content = true|false` +in `cribo.toml`. Full design: [docs/source-maps.md](docs/source-maps.md). + ### Dependency Detection (`cribo deps`) The `deps` subcommand analyzes a Python file or directory and reports the third-party diff --git a/crates/cribo/Cargo.toml b/crates/cribo/Cargo.toml index 61b4af790..7c0587c7c 100644 --- a/crates/cribo/Cargo.toml +++ b/crates/cribo/Cargo.toml @@ -22,12 +22,14 @@ required-features = ["bench"] [dependencies] anyhow = { workspace = true } +base64-simd = { workspace = true } clap = { workspace = true } cow-utils = { workspace = true } env_logger = { workspace = true } etcetera = { workspace = true } indexmap = { workspace = true } log = { workspace = true } +oxc_sourcemap = { workspace = true } pep508_rs = { workspace = true } petgraph = { workspace = true } ruff_python_ast = { workspace = true } diff --git a/crates/cribo/src/config.rs b/crates/cribo/src/config.rs index c1817c4ef..ea636b815 100644 --- a/crates/cribo/src/config.rs +++ b/crates/cribo/src/config.rs @@ -47,10 +47,47 @@ pub struct Config { #[serde(rename = "bundle-third-party", alias = "bundle_third_party")] pub bundle_third_party: Option, + /// Source map delivery mode. `None` disables source map generation (default). + /// See `docs/source-maps.md`. + pub sourcemap: Option, + + /// Whether to embed original source text in the map as `sourcesContent`. + /// `None` selects the mode-dependent default (omitted for `inline`, + /// included for `linked`/`external`). + #[serde(rename = "sources-content", alias = "sources_content")] + pub sources_content: Option, + /// Configuration for mapping imports to installable requirements pub requirements: RequirementsConfig, } +/// Source map delivery mode, mirroring esbuild's `--sourcemap` values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum SourceMapMode { + /// Write `.map` next to the bundle and append a + /// `# sourceMappingURL=.map` comment (the default for a bare flag). + Linked, + /// Embed the map as a base64 data-URL comment at the end of the bundle. + Inline, + /// Write `.map` with no comment in the bundle. + External, +} + +impl Config { + /// Effective `sourcesContent` policy for source map generation. + /// + /// An explicit `sources-content` setting wins; otherwise the mode default + /// applies — omitted for `inline` (keeps the bundle small), included for + /// `linked`/`external` (self-contained maps). + pub fn include_sources_content(&self) -> bool { + self.sources_content.unwrap_or(match self.sourcemap { + Some(SourceMapMode::Linked | SourceMapMode::External) => true, + Some(SourceMapMode::Inline) | None => false, + }) + } +} + #[derive(Clone, Default, Serialize, Deserialize)] #[serde(default)] pub struct RequirementsConfig { @@ -91,6 +128,8 @@ impl Default for Config { target_version: "py310".to_owned(), tree_shake: true, // Tree-shaking enabled by default bundle_third_party: None, // Opt-in: third-party deps stay external by default + sourcemap: None, // Opt-in: no source map by default + sources_content: None, // Mode-dependent default; see docs/source-maps.md requirements: RequirementsConfig::default(), } } @@ -123,6 +162,8 @@ impl Combine for Config { tree_shake: self.tree_shake, // Option scalar: absent keys in higher-precedence layers preserve lower layers bundle_third_party: self.bundle_third_party.or(other.bundle_third_party), + sourcemap: self.sourcemap.or(other.sourcemap), + sources_content: self.sources_content.or(other.sources_content), requirements: RequirementsConfig { python: self.requirements.python.or(other.requirements.python), module_map: if self.requirements.module_map.is_empty() { diff --git a/crates/cribo/src/lib.rs b/crates/cribo/src/lib.rs index 0e237ebd9..ae9c21eac 100644 --- a/crates/cribo/src/lib.rs +++ b/crates/cribo/src/lib.rs @@ -28,6 +28,7 @@ pub(crate) mod module_facts; pub(crate) mod python; pub(crate) mod requirement_resolver; pub(crate) mod side_effects; +pub(crate) mod source_map; pub(crate) mod symbol_conflict_resolver; pub(crate) mod transformation_context; pub(crate) mod tree_shaking; diff --git a/crates/cribo/src/main.rs b/crates/cribo/src/main.rs index 050742818..717f72a41 100644 --- a/crates/cribo/src/main.rs +++ b/crates/cribo/src/main.rs @@ -24,6 +24,7 @@ mod python; mod requirement_resolver; mod resolver; mod side_effects; +mod source_map; mod symbol_conflict_resolver; mod transformation_context; mod tree_shaking; @@ -31,7 +32,7 @@ mod types; mod util; mod visitors; -use config::Config; +use config::{Config, SourceMapMode}; use orchestrator::BundleOrchestrator; #[derive(Parser)] @@ -81,6 +82,20 @@ struct Cli { /// and are emitted into requirements.txt #[arg(long)] bundle_third_party: bool, + + /// Generate a Source Map v3 for the bundle. A bare `--sourcemap` selects + /// `linked` (`inline` with --stdout); or choose explicitly with + /// `--sourcemap=linked|inline|external` + // Option> is clap's idiom for a flag with an optional value: + // the outer Option is flag presence, the inner one the explicit value. + #[expect(clippy::option_option)] + #[arg(long, value_enum, num_args = 0..=1, require_equals = true)] + sourcemap: Option>, + + /// Force embedding original sources in the map as `sourcesContent` + /// (default: omitted for inline, included for linked/external) + #[arg(long, require_equals = true, value_name = "BOOL")] + sources_content: Option, } #[derive(Subcommand)] @@ -219,6 +234,31 @@ fn run_bundle(mut config: Config, cli: &Cli) -> anyhow::Result<()> { config.bundle_third_party = Some(true); } + // Resolve the source map mode: CLI takes precedence over the config file. + // A bare `--sourcemap` selects the esbuild-style default: linked for file + // output, inline for stdout (a linked map has nowhere to live next to stdout). + if let Some(cli_mode) = cli.sourcemap { + config.sourcemap = Some(cli_mode.unwrap_or(if cli.stdout { + SourceMapMode::Inline + } else { + SourceMapMode::Linked + })); + } + if cli.stdout + && matches!( + config.sourcemap, + Some(SourceMapMode::Linked | SourceMapMode::External) + ) + { + return Err(anyhow!( + "linked and external source maps require an output file; use --sourcemap=inline with \ + --stdout" + )); + } + if let Some(sources_content) = cli.sources_content { + config.sources_content = Some(sources_content); + } + debug!("Configuration: {config:?}"); // Display target version for troubleshooting diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 525b461c7..35ac18d2f 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -16,12 +16,13 @@ use crate::{ ResolutionStrategy, }, code_generator::{Bundler, phases::orchestrator::PhaseOrchestrator}, - config::Config, + config::{Config, SourceMapMode}, dependency_graph::DependencyGraph, import_rewriter::{ImportDeduplicationStrategy, ImportRewriter}, module_facts::ModuleFacts, requirement_resolver::RequirementResolver, resolver::{ImportOrigin, ModuleId, ModuleResolver}, + source_map::{ProvenanceResolver, SourceMapOptions, build_source_map}, symbol_conflict_resolver::SymbolConflictResolver, tree_shaking::TreeShaker, types::FxIndexMap, @@ -38,6 +39,16 @@ fn get_empty_parsed_module() -> &'static ruff_python_parser::Parsed { .get_or_init(|| ruff_python_parser::parse_module("").expect("Failed to parse empty module")) } +/// Path of the source map file for a bundle output path (`bundle.py` → `bundle.py.map`). +fn source_map_path_for(output_path: &Path) -> PathBuf { + let mut file_name = output_path.file_name().map_or_else( + || std::ffi::OsString::from("bundle.py"), + std::ffi::OsStr::to_os_string, + ); + file_name.push(".map"); + output_path.with_file_name(file_name) +} + /// Type alias for module processing queue type ModuleQueue = Vec<(ModuleId, PathBuf)>; /// Type alias for processed modules set @@ -70,6 +81,16 @@ struct StaticBundleParams<'a> { graph: &'a DependencyGraph, circular_dep_analysis: Option<&'a CircularDependencyAnalysis>, tree_shaker: Option<&'a TreeShaker<'a>>, + /// Output file path when writing to disk; `None` for stdout output. + /// Used for the source map `file` field and source path relativization. + output_path: Option<&'a Path>, +} + +/// Result of static bundle emission: the code plus an optional source map JSON. +struct EmittedBundle { + code: String, + /// Source Map v3 JSON, present when `Config::sourcemap` is enabled. + source_map: Option, } /// Context for dependency building operations @@ -601,14 +622,25 @@ impl BundleOrchestrator { // Generate bundled code info!("Using hybrid static bundler"); - let bundled_code = self.emit_static_bundle(&StaticBundleParams { + let emitted = self.emit_static_bundle(&StaticBundleParams { sorted_module_ids: &sorted_module_ids, parsed_modules: Some(&parsed_modules), resolver: &resolver, graph: &graph, circular_dep_analysis: circular_dep_analysis.as_ref(), tree_shaker: tree_shaker.as_ref(), + output_path: None, })?; + let mut bundled_code = emitted.code; + + // Stdout output can only carry an inline map; other modes are rejected + // at CLI validation time. + if self.config.sourcemap == Some(SourceMapMode::Inline) + && let Some(map_json) = emitted.source_map.as_deref() + { + bundled_code.push('\n'); + bundled_code.push_str(&crate::source_map::inline_source_mapping_comment(map_json)); + } // Generate requirements.txt if requested if emit_requirements { @@ -660,14 +692,45 @@ impl BundleOrchestrator { // Generate bundled code info!("Using hybrid static bundler"); - let bundled_code = self.emit_static_bundle(&StaticBundleParams { + let emitted = self.emit_static_bundle(&StaticBundleParams { sorted_module_ids: &sorted_module_ids, parsed_modules: Some(&parsed_modules), // Use pre-parsed modules to avoid double parsing resolver: &resolver, graph: &graph, circular_dep_analysis: circular_dep_analysis.as_ref(), tree_shaker: tree_shaker.as_ref(), + output_path: Some(output_path), })?; + let mut bundled_code = emitted.code; + + // Apply the configured source map delivery mode. + if let (Some(mode), Some(map_json)) = (self.config.sourcemap, emitted.source_map.as_deref()) + { + match mode { + SourceMapMode::Linked | SourceMapMode::External => { + let map_path = source_map_path_for(output_path); + fs::write(&map_path, map_json).with_context(|| { + format!("Failed to write source map file: {}", map_path.display()) + })?; + info!("Source map written to: {}", map_path.display()); + if mode == SourceMapMode::Linked { + let map_file_name = map_path.file_name().map_or_else( + || map_path.to_string_lossy().into_owned(), + |name| name.to_string_lossy().into_owned(), + ); + bundled_code.push('\n'); + bundled_code.push_str(&crate::source_map::linked_source_mapping_comment( + &map_file_name, + )); + } + } + SourceMapMode::Inline => { + bundled_code.push('\n'); + bundled_code + .push_str(&crate::source_map::inline_source_mapping_comment(map_json)); + } + } + } // Generate requirements.txt if requested if emit_requirements { @@ -1955,7 +2018,7 @@ impl BundleOrchestrator { } /// Emit bundle using static bundler (no exec calls) - fn emit_static_bundle(&mut self, params: &StaticBundleParams<'_>) -> Result { + fn emit_static_bundle(&mut self, params: &StaticBundleParams<'_>) -> Result { // First, detect and resolve conflicts after all modules have been analyzed let conflicts = self.conflict_resolver.detect_and_resolve_conflicts(); if !conflicts.is_empty() { @@ -2034,7 +2097,7 @@ impl BundleOrchestrator { } // Bundle all modules using the phase-based orchestrator - let bundled_ast = PhaseOrchestrator::bundle( + let mut bundled_ast = PhaseOrchestrator::bundle( &mut static_bundler, &crate::code_generator::BundleParams { modules: &module_asts, @@ -2048,6 +2111,14 @@ impl BundleOrchestrator { }, ); + // Inject the traceback-remapping runtime before code generation so the + // emitted text and the bundled AST stay structurally aligned for the + // source map extraction walk. + if let Some(mode) = self.config.sourcemap { + crate::source_map::inject_runtime_prologue(&mut bundled_ast, mode); + } + let bundled_ast = bundled_ast; + // Generate Python code from AST let empty_parsed = get_empty_parsed_module(); let stylist = ruff_python_codegen::Stylist::from_tokens(empty_parsed.tokens(), ""); @@ -2082,8 +2153,72 @@ impl BundleOrchestrator { String::new(), // Empty line ]; final_output.extend(code_parts); + let code = final_output.join("\n"); + + // Extract source map when enabled: re-parse the emitted code and walk it + // in parallel with the bundled AST (which carries node provenance). + let source_map = if self.config.sourcemap.is_some() { + self.extract_source_map(&code, &bundled_ast, params) + } else { + None + }; + + Ok(EmittedBundle { code, source_map }) + } + + /// Build the Source Map v3 JSON for an emitted bundle. + /// + /// Never fails the bundle: extraction errors are logged and yield `None`. + fn extract_source_map( + &self, + code: &str, + bundled_ast: &ModModule, + params: &StaticBundleParams<'_>, + ) -> Option { + let parsed_modules = params.parsed_modules?; + + // Module ordinals were assigned by the bundler's AST indexing pass in + // the order of `parsed_modules`; register provenance in the same order. + let mut provenance = ProvenanceResolver::default(); + for (module_id, _imports, _ast, source) in parsed_modules { + let path = params + .resolver + .get_module_path(*module_id) + .unwrap_or_else(|| { + let name = params + .resolver + .get_module_name(*module_id) + .unwrap_or_else(|| format!("module_{}", module_id.as_u32())); + PathBuf::from(&name) + }); + let path = std::path::absolute(&path).unwrap_or(path); + provenance.push_module(path, source.clone()); + } - Ok(final_output.join("\n")) + let file_name = params.output_path.and_then(Path::file_name).map_or_else( + || "".to_owned(), + |name| name.to_string_lossy().into_owned(), + ); + // Source paths are relative to the directory the map lives in (the + // output directory), or the current directory for stdout output. + let base_dir = params + .output_path + .and_then(Path::parent) + .map(|dir| std::path::absolute(dir).unwrap_or_else(|_| dir.to_path_buf())) + .or_else(|| std::env::current_dir().ok()); + + let options = SourceMapOptions { + file: &file_name, + include_contents: self.config.include_sources_content(), + base_dir: base_dir.as_deref(), + }; + match build_source_map(code, bundled_ast, &provenance, &options) { + Ok(json) => Some(json), + Err(err) => { + warn!("Source map generation failed; bundling continues without a map: {err:#}"); + None + } + } } /// Generate requirements.txt content from third-party imports @@ -2312,6 +2447,92 @@ mod tests { use super::*; + /// End-to-end source map extraction: bundle a three-module project (inlined + /// module, wrapper module with side effects, entry) with an inline map and + /// verify statement mappings point back at the original files and lines. + #[test] + fn test_source_map_extraction_end_to_end() -> Result<()> { + let temp_dir = TempDir::new()?; + fs::write( + temp_dir.path().join("main.py"), + "from utils import add\nimport effects\n\nresult = add(1, 2)\nprint(result, \ + effects.X)\n", + )?; + fs::write( + temp_dir.path().join("utils.py"), + "def add(a, b):\n total = a + b\n return total\n", + )?; + // The print side effect forces this module onto the wrapper path. + fs::write( + temp_dir.path().join("effects.py"), + "print(\"side effect\")\nX = 42\n", + )?; + + let config = Config { + sourcemap: Some(SourceMapMode::Inline), + ..Config::default() + }; + let mut orchestrator = BundleOrchestrator::new(config); + let code = orchestrator.bundle_to_string(&temp_dir.path().join("main.py"), false)?; + + // The bundle must end with an inline sourceMappingURL comment. + let marker = "# sourceMappingURL=data:application/json;base64,"; + let marker_pos = code + .rfind(marker) + .expect("inline source map comment present"); + let payload = code[marker_pos + marker.len()..].trim_end(); + let map_bytes = base64_simd::STANDARD + .decode_to_vec(payload.as_bytes()) + .expect("valid base64 payload"); + let map_json = String::from_utf8(map_bytes).expect("valid UTF-8 source map"); + + let map = oxc_sourcemap::SourceMap::from_json_string(&map_json) + .expect("valid Source Map v3 JSON"); + assert_eq!(map.get_file(), Some("")); + // Inline mode omits sourcesContent by default. + assert!(!map_json.contains("sourcesContent")); + + let lookup = map.generate_lookup_table(); + let find_generated_line = |needle: &str| -> u32 { + code.lines() + .position(|line| line.trim() == needle) + .unwrap_or_else(|| panic!("bundle must contain a line matching `{needle}`")) + as u32 + }; + let assert_maps_to = |needle: &str, source_suffix: &str, original_line: u32| { + let generated_line = find_generated_line(needle); + let token = map + .lookup_token(&lookup, generated_line, 0) + .unwrap_or_else(|| panic!("mapping for `{needle}` on line {generated_line}")); + assert_eq!( + token.get_dst_line(), + generated_line, + "`{needle}` must have a mapping on its own line, not inherit an earlier one" + ); + let source = map + .get_source(token.get_source_id().expect("source id")) + .expect("source path"); + assert!( + source.ends_with(source_suffix), + "`{needle}` should map into {source_suffix}, got {source}" + ); + assert_eq!( + token.get_src_line(), + original_line, + "`{needle}` should map to 0-based line {original_line} of {source_suffix}" + ); + }; + + // Inlined module: statement nested in a function body. + assert_maps_to("return total", "utils.py", 2); + // Wrapper module: statement inside the synthesized init function. + assert_maps_to("X = 42", "effects.py", 1); + // Entry module statement. + assert_maps_to("result = add(1, 2)", "main.py", 3); + + Ok(()) + } + /// External importlib targets recorded during one bundle run must not leak into a /// later run on the same orchestrator instance. #[test] diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py new file mode 100644 index 000000000..cc1882f7a --- /dev/null +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -0,0 +1,603 @@ +"""Cribo source map runtime (injected prologue). + +Remaps tracebacks of uncaught exceptions back to the original source files +using the Source Map v3 emitted at bundle time. Lazy by design: no file I/O, +parsing, or decoding happens at import time; everything is deferred to the +first uncaught exception. Under resource pressure the decoder streams the map +in constant memory and falls back to the default traceback on any failure. +See docs/source-maps.md in the cribo repository. +""" + +import binascii as _cribo_binascii +import os as _cribo_os +import sys as _cribo_sys +import threading as _cribo_threading + +_CRIBO_SM_MODE = "__CRIBO_SOURCEMAP_MODE__" +_CRIBO_SM_BUNDLE = globals().get("__file__", "") +_CRIBO_SM_CHUNK = 8192 +_CRIBO_SM_STATE = {"in_hook": False} +_CRIBO_SM_B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +_cribo_sm_prev_excepthook = _cribo_sys.excepthook +_cribo_sm_prev_unraisablehook = _cribo_sys.unraisablehook +_cribo_sm_prev_threading_hook = _cribo_threading.excepthook + + +def _cribo_sm_map_location(): + """Resolve the map location per delivery mode, or None when inactive. + + Returns (map_path, map_dir); map_path is None for inline mode (the map + lives inside the bundle file itself). Called lazily at hook-fire time so + the happy path never touches the environment or the filesystem. + """ + env = _cribo_os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + bundle = _CRIBO_SM_BUNDLE + if _CRIBO_SM_MODE == "inline": + return (None, _cribo_os.path.dirname(_cribo_os.path.abspath(bundle))) + sibling = bundle + ".map" + if _CRIBO_SM_MODE == "linked": + if _cribo_os.path.exists(sibling): + return (sibling, _cribo_os.path.dirname(_cribo_os.path.abspath(sibling))) + return None + # external: opt in via CRIBO_SOURCE_MAPS=1 (or a path to the map file) + if env in ("1", "true", "yes", "on"): + path = sibling + elif env: + path = env + else: + return None + return (path, _cribo_os.path.dirname(_cribo_os.path.abspath(path))) + + +def _cribo_sm_file_chunks(path): + """Yield fixed-size chunks of a file (constant memory).""" + handle = open(path, "rb") + try: + while True: + chunk = handle.read(_CRIBO_SM_CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() + + +def _cribo_sm_find_inline_payload(handle): + """Backward-scan the bundle for the last inline map marker. + + Returns the byte offset of the base64 payload, or -1. Only the tail of the + file is examined; the bundle body is never read. + """ + marker = b"# sourceMappingURL=data:" + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + found = -1 + while position > 0: + step = _CRIBO_SM_CHUNK if position >= _CRIBO_SM_CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + found = position + index + break + overlap = data[: len(marker) - 1] + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + len(b"base64,") + + +def _cribo_sm_inline_chunks(path): + """Yield decoded chunks of an inline (base64 data URL) source map.""" + handle = open(path, "rb") + try: + start = _cribo_sm_find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(_CRIBO_SM_CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = len(data) - (len(data) % 4) + pending = data[usable:] + if usable: + yield _cribo_binascii.a2b_base64(data[:usable]) + if pending: + yield _cribo_binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + finally: + handle.close() + + +class _CriboSmStream(object): + """Byte-at-a-time reader over an iterator of byte chunks.""" + + __slots__ = ("_chunks", "_buf", "_pos") + + def __init__(self, chunks): + self._chunks = iter(chunks) + self._buf = b"" + self._pos = 0 + + def read_byte(self): + while self._pos >= len(self._buf): + try: + self._buf = next(self._chunks) + except StopIteration: + return -1 + self._pos = 0 + value = self._buf[self._pos] + self._pos += 1 + return value + + +def _cribo_sm_skip_ws(stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte + + +def _cribo_sm_read_string(stream, collect): + """Consume a JSON string whose opening quote was already read. + + Returns the decoded text when collect is true, else None (contents are + discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are handled, + so string *values* containing text like '"mappings":' cannot confuse the + key scanner. + """ + buf = bytearray() if collect else None + while True: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON string") + if byte == 34: # '"' + return buf.decode("utf-8", "replace") if collect else None + if byte != 92: # '\\' + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise ValueError("unterminated JSON escape") + if escape == 117: # 'u' + code = 0 + for _ in range(4): + digit = stream.read_byte() + if digit < 0: + raise ValueError("unterminated unicode escape") + code = code * 16 + int(chr(digit), 16) + if buf is not None: + buf.extend(chr(code).encode("utf-8", "surrogatepass")) + elif buf is not None: + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf.append(table.get(escape, escape)) + + +def _cribo_sm_skip_value(stream, byte): + """Skip one JSON value; return the first byte after it (or -1).""" + if byte == 34: # string + _cribo_sm_read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): # object / array + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON container") + if byte == 34: + _cribo_sm_read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + # number / true / false / null: consume until a delimiter + while byte >= 0 and byte not in (44, 125, 93): # ',' '}' ']' + byte = stream.read_byte() + return byte + + +def _cribo_sm_read_string_array(stream, byte): + """Read a JSON array of strings/nulls; return (list, byte after array).""" + if byte != 91: # '[' + raise ValueError("expected array") + items = [] + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte == 93: # ']' + return items, stream.read_byte() + while True: + if byte == 34: + items.append(_cribo_sm_read_string(stream, True)) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + else: + items.append(None) + byte = _cribo_sm_skip_ws(stream, _cribo_sm_skip_value(stream, byte)) + if byte == 93: + return items, stream.read_byte() + if byte != 44: # ',' + raise ValueError("malformed array") + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + + +def _cribo_sm_decode_vlq(stream, needed, max_needed): + """Streaming VLQ state machine over the raw bytes of the mappings string. + + Constant state: line/segment counters plus running deltas. Records the + first segment per needed generated line; exits as soon as every needed + line is resolved or the max needed line is passed. Consumes up to and + including the closing quote (or stops early). + """ + lut = {} + for index in range(64): + lut[_CRIBO_SM_B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 + vlq_value = 0 + vlq_shift = 0 + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = (src_idx, src_line) + + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: # EOF or closing '"' + end_segment() + return result + if byte == 59: # ';' + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or len(result) == len(needed): + return result + continue + if byte == 44: # ',' + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise ValueError("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 + + +def _cribo_sm_scan(chunks, needed, max_needed): + """Scan the map's top-level object; return (sources, line table).""" + stream = _CriboSmStream(chunks) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte != 123: # '{' + raise ValueError("not a JSON object") + sources = [] + table = {} + saw_mappings = False + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + while byte == 34: # '"' starting a key + key = _cribo_sm_read_string(stream, True) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte != 58: # ':' + raise ValueError("malformed object") + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if key == "sources": + sources, byte = _cribo_sm_read_string_array(stream, byte) + elif key == "mappings": + if byte != 34: + raise ValueError("mappings is not a string") + table = _cribo_sm_decode_vlq(stream, needed, max_needed) + saw_mappings = True + if sources: + break # both fields consumed; ignore the rest of the map + byte = stream.read_byte() + else: + byte = _cribo_sm_skip_value(stream, byte) + byte = _cribo_sm_skip_ws(stream, byte) + if byte == 44: # ',' + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if not saw_mappings: + raise ValueError("no mappings field") + return sources, table + + +def _cribo_sm_load(needed_lines): + """Load (table, sources, map_dir) for 1-based bundle line numbers. + + Returns None when the runtime is inactive for the current mode. The + returned table is keyed by 1-based bundle lines mapping to + (source_index, 1-based original line). + """ + location = _cribo_sm_map_location() + if location is None: + return None + map_path, map_dir = location + needed0 = set(line - 1 for line in needed_lines) + max_needed = max(needed0) + if map_path is None: + chunks = _cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE) + else: + chunks = _cribo_sm_file_chunks(map_path) + sources, table0 = _cribo_sm_scan(chunks, needed0, max_needed) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = (src_idx, src_line0 + 1) + return (table, sources, map_dir) + + +def _cribo_sm_load_json_fallback(needed_lines): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = _cribo_sm_map_location() + if location is None: + return None + map_path, map_dir = location + import json + + if map_path is None: + raw = b"".join(_cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE)) + else: + handle = open(map_path, "rb") + try: + raw = handle.read() + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = data.get("sources") or [] + mappings = data.get("mappings") or "" + needed0 = set(line - 1 for line in needed_lines) + stream = _CriboSmStream([mappings.encode("ascii"), b'"']) + table0 = _cribo_sm_decode_vlq(stream, needed0, max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = (src_idx, src_line0 + 1) + return (table, sources, map_dir) + + +def _cribo_sm_collect_needed(exc_value, traceback_obj): + """1-based bundle line numbers referenced by the traceback (and its chain).""" + needed = set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == _CRIBO_SM_BUNDLE: + needed.add(tb.tb_lineno) + tb = tb.tb_next + + add(traceback_obj) + exc = exc_value + seen = set() + depth = 0 + while exc is not None and id(exc) not in seen and depth < 16: + seen.add(id(exc)) + depth += 1 + add(getattr(exc, "__traceback__", None)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed + + +def _cribo_sm_source_line(path, lineno): + """Read a single 1-based line from a file without caching it.""" + try: + handle = open(path, "rb") + except OSError: + return None + try: + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except OSError: + return None + finally: + handle.close() + return None + + +def _cribo_sm_write_frames(traceback_obj, table, sources, map_dir, write): + """Write remapped frame lines, collapsing repeated frames like CPython. + + Consecutive identical frames (recursion) print at most 3 times followed by + a "[Previous line repeated N more times]" marker; source line text is + cached per (file, line) within one rendering to avoid re-reading files. + """ + cache = {} + last = None + repeats = 0 + + def emit(entry): + write(' File "%s", line %d, in %s\n' % entry) + key = (entry[0], entry[1]) + if key not in cache: + cache[key] = _cribo_sm_source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + if filename == _CRIBO_SM_BUNDLE: + mapped = table.get(lineno) + if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + source = sources[mapped[0]] + if not _cribo_os.path.isabs(source): + source = _cribo_os.path.normpath(_cribo_os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] + entry = (filename, lineno, name) + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 + emit(entry) + traceback_obj = traceback_obj.tb_next + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + + +def _cribo_sm_exception_line(exc_value): + exc_type = type(exc_value) + name = getattr(exc_type, "__qualname__", exc_type.__name__) + module = getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = str(exc_value) + except BaseException: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name + + +def _cribo_sm_render(exc_value, table, sources, map_dir, write): + """Render the exception (with its cause/context chain) like CPython does.""" + chain = [] + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(chain) < 16: + seen.add(id(exc)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + suppress = getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context + else: + chain.append((exc, None)) + exc = None + # Print innermost first, like CPython. The link stored on an exception + # describes its relation to its own inner exception — which is exactly + # the one printed immediately before it. + ordered = list(reversed(chain)) + for index, (exc, link) in enumerate(ordered): + if index > 0: + if link == "cause": + write( + "\nThe above exception was the direct cause of the following exception:\n\n" + ) + else: + write( + "\nDuring handling of the above exception, another exception occurred:\n\n" + ) + tb = getattr(exc, "__traceback__", None) + if tb is not None: + write("Traceback (most recent call last):\n") + _cribo_sm_write_frames(tb, table, sources, map_dir, write) + write(_cribo_sm_exception_line(exc)) + + +def _cribo_sm_try_render(exc_value, traceback_obj, prefix): + """Attempt a remapped rendering to stderr; True on success. + + Never raises and never masks the original exception: any failure in this + runtime returns False so callers can delegate to the previous hook. + """ + if _CRIBO_SM_STATE["in_hook"] or exc_value is None: + return False + _CRIBO_SM_STATE["in_hook"] = True + old_limit = None + try: + needed = _cribo_sm_collect_needed(exc_value, traceback_obj) + if not needed: + return False + loaded = None + try: + loaded = _cribo_sm_load(needed) + except BaseException: + try: + loaded = _cribo_sm_load_json_fallback(needed) + except BaseException: + loaded = None + if not loaded: + return False + table, sources, map_dir = loaded + if not table: + return False + try: + old_limit = _cribo_sys.getrecursionlimit() + _cribo_sys.setrecursionlimit(old_limit + 64) + except BaseException: + old_limit = None + # Buffer the rendering so a mid-render failure produces no partial + # output before the previous hook prints the standard traceback. + parts = [] + if prefix: + parts.append(prefix) + _cribo_sm_render(exc_value, table, sources, map_dir, parts.append) + stderr = _cribo_sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except BaseException: + pass + return True + except BaseException: + return False + finally: + if old_limit is not None: + try: + _cribo_sys.setrecursionlimit(old_limit) + except BaseException: + pass + _CRIBO_SM_STATE["in_hook"] = False + + +def _cribo_sm_excepthook(exc_type, exc_value, traceback_obj): + if not _cribo_sm_try_render(exc_value, traceback_obj, None): + _cribo_sm_prev_excepthook(exc_type, exc_value, traceback_obj) + + +def _cribo_sm_threading_hook(args): + thread = getattr(args, "thread", None) + name = getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if not _cribo_sm_try_render(args.exc_value, args.exc_traceback, prefix): + _cribo_sm_prev_threading_hook(args) + + +def _cribo_sm_unraisablehook(unraisable): + message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except BaseException: + prefix = "%s\n" % message + if not _cribo_sm_try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + _cribo_sm_prev_unraisablehook(unraisable) + + +_cribo_sys.excepthook = _cribo_sm_excepthook +_cribo_sys.unraisablehook = _cribo_sm_unraisablehook +_cribo_threading.excepthook = _cribo_sm_threading_hook diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs new file mode 100644 index 000000000..cc74b5ae3 --- /dev/null +++ b/crates/cribo/src/source_map.rs @@ -0,0 +1,727 @@ +//! Source Map v3 generation for the bundled output. +//! +//! Cribo emits statement/line-level mappings (column 0): each mapped statement in +//! the bundle contributes one token `generated line → (original file, original line)`. +//! That granularity matches Python's line-oriented tracebacks, which are the primary +//! consumer via the injected runtime (see `docs/source-maps.md`). +//! +//! Serialization is delegated to `oxc_sourcemap`, the encoder used by rolldown and +//! rspack, which guarantees spec-compliant VLQ `mappings` output. + +use std::borrow::Cow; + +use anyhow::Context as _; +use oxc_sourcemap::{SourceMap, Token}; +use ruff_python_ast::{HasNodeIndex as _, ModModule, NodeIndex, Stmt}; +use ruff_text_size::{Ranged as _, TextSize}; + +use crate::{ast_indexer::MODULE_INDEX_RANGE, types::FxIndexMap}; + +/// Identifier of a registered original source file within a [`SourceMapGenerator`]. +/// +/// Indexes into the emitted `sources` array of the Source Map v3 JSON. +pub(crate) type SourceId = u32; + +/// Byte offsets of line starts, for converting a `TextSize` offset to a line number. +#[derive(Debug)] +pub(crate) struct LineIndex { + /// Byte offset of the start of each line; `line_starts[0] == 0`. + line_starts: Vec, +} + +impl LineIndex { + /// Build the index from source text (lines separated by `\n`; sources are + /// normalized to `\n` line endings when read). + pub(crate) fn new(source: &str) -> Self { + let mut line_starts = vec![0_u32]; + for (offset, byte) in source.bytes().enumerate() { + if byte == b'\n' { + line_starts.push(offset as u32 + 1); + } + } + Self { line_starts } + } + + /// 0-based line containing byte `offset`. + pub(crate) fn line_of(&self, offset: TextSize) -> u32 { + let offset = offset.to_u32(); + // partition_point returns the count of line starts <= offset; the line + // containing the offset is the last such line. + (self.line_starts.partition_point(|&start| start <= offset) - 1) as u32 + } +} + +/// Provenance data for one bundled module, in module-ordinal order. +#[derive(Debug)] +pub(crate) struct ModuleSourceInfo { + /// Original filesystem path of the module (as recorded in the `sources` array). + pub(crate) path: std::path::PathBuf, + /// The module's original source text (for `sourcesContent` and line lookup). + pub(crate) source: String, + /// Line index over `source`. + pub(crate) line_index: LineIndex, +} + +/// Resolves node provenance: which original module and line an AST node came from. +/// +/// Module ordinals are assigned by the bundler's AST indexing pass +/// (`Bundler::index_module_asts`): the n-th module receives node indices in +/// `[n * MODULE_INDEX_RANGE, (n + 1) * MODULE_INDEX_RANGE)`. Entries here MUST be +/// registered in that same order. Synthesized nodes carry `AtomicNodeIndex::NONE` +/// (or an index past all module ranges) and resolve to `None`. +#[derive(Debug, Default)] +pub(crate) struct ProvenanceResolver { + modules: Vec, +} + +impl ProvenanceResolver { + /// Register the next module (ordinal = number of previously registered modules). + pub(crate) fn push_module(&mut self, path: std::path::PathBuf, source: String) { + let line_index = LineIndex::new(&source); + self.modules.push(ModuleSourceInfo { + path, + source, + line_index, + }); + } + + /// Registered modules, in ordinal order. + pub(crate) fn modules(&self) -> &[ModuleSourceInfo] { + &self.modules + } + + /// Resolve a node to (module ordinal, 0-based original line). + /// + /// Returns `None` for synthesized nodes: those with the `NONE` placeholder + /// index or with an index beyond all module ranges (allocated by + /// `TransformationContext` after indexing). + pub(crate) fn resolve( + &self, + node_index: NodeIndex, + range_start: TextSize, + ) -> Option<(usize, u32)> { + let index = node_index.as_u32()?; + let ordinal = (index / MODULE_INDEX_RANGE) as usize; + let module = self.modules.get(ordinal)?; + Some((ordinal, module.line_index.line_of(range_start))) + } +} + +/// A single line-level mapping record. +#[derive(Debug, Clone, Copy)] +struct Mapping { + /// 0-based line in the generated bundle. + generated_line: u32, + /// Which original source file this line came from. + source_id: SourceId, + /// 0-based line in the original source file. + original_line: u32, +} + +/// Collects line-level mapping records and serializes them as Source Map v3 JSON. +/// +/// Lines are 0-based on both the generated and original side, matching the +/// Source Map v3 token encoding (callers converting from 1-based line numbers +/// must subtract one). +#[derive(Debug, Default)] +pub(crate) struct SourceMapGenerator { + /// Name of the generated file (the bundle), emitted as the `file` field. + file: String, + /// Original source path → optional embedded content. Insertion order defines + /// the `sources` array order; the map index is the [`SourceId`]. + sources: FxIndexMap>, + /// Collected mappings, in insertion order (sorted at serialization time). + mappings: Vec, +} + +impl SourceMapGenerator { + /// Create a generator for a bundle named `file` (emitted as the `file` field). + pub(crate) fn new(file: impl Into) -> Self { + Self { + file: file.into(), + sources: FxIndexMap::default(), + mappings: Vec::new(), + } + } + + /// Register an original source file, returning its stable [`SourceId`]. + /// + /// Paths are deduplicated: registering the same path twice returns the same + /// id, and the first non-`None` `content` wins. + pub(crate) fn add_source(&mut self, path: &str, content: Option) -> SourceId { + if let Some(index) = self.sources.get_index_of(path) { + let id = index as SourceId; + if let Some(existing) = self.sources.get_index_mut(index) + && existing.1.is_none() + { + *existing.1 = content; + } + return id; + } + let id = self.sources.len() as SourceId; + self.sources.insert(path.to_owned(), content); + id + } + + /// Record that 0-based `generated_line` in the bundle originates from + /// 0-based `original_line` of `source_id`. + /// + /// Multiple records for the same generated line are allowed; the first one + /// added wins at serialization time (statement granularity means the first + /// statement starting on a line is the authoritative origin). + pub(crate) fn add_mapping( + &mut self, + generated_line: u32, + source_id: SourceId, + original_line: u32, + ) { + debug_assert!( + (source_id as usize) < self.sources.len(), + "add_mapping called with unregistered source_id {source_id}" + ); + self.mappings.push(Mapping { + generated_line, + source_id, + original_line, + }); + } + + /// Whether any mappings have been recorded. + #[cfg(test)] + pub(crate) const fn is_empty(&self) -> bool { + self.mappings.is_empty() + } + + /// Serialize to a Source Map v3 JSON string. + /// + /// Tokens are emitted sorted by generated line, one token per line (the + /// first mapping recorded for a line wins). `sourcesContent` is included iff + /// at least one source has content (the field is omitted entirely otherwise). + pub(crate) fn into_json(self) -> String { + let mut mappings = self.mappings; + // Stable sort: for duplicate generated lines the earliest insertion stays first. + mappings.sort_by_key(|m| m.generated_line); + mappings.dedup_by_key(|m| m.generated_line); + + let tokens: Vec = mappings + .iter() + .map(|m| { + Token::new( + m.generated_line, + 0, + m.original_line, + 0, + Some(m.source_id), + None, + ) + }) + .collect(); + + let sources: Vec> = self + .sources + .keys() + .map(|path| Cow::Borrowed(path.as_str())) + .collect(); + let source_contents: Vec>> = self + .sources + .values() + .map(|content| content.as_deref().map(Cow::Borrowed)) + .collect(); + + let map = SourceMap::new( + Some(Cow::Borrowed(self.file.as_str())), + Vec::new(), + None, + sources, + source_contents, + tokens.into_boxed_slice(), + None, + ); + map.to_json_string() + } +} + +/// One record produced by the parallel statement walk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct MappingRecord { + /// 0-based line in the generated bundle text. + pub(crate) generated_line: u32, + /// Ordinal of the originating module in the [`ProvenanceResolver`]. + pub(crate) module_ordinal: usize, + /// 0-based line in the original module source. + pub(crate) original_line: u32, +} + +/// Extract statement-level mappings by re-parsing the emitted bundle text and +/// walking it in parallel with the bundled AST (which carries node provenance). +/// +/// The re-parse doubles as a validity check of the emitted bundle: a parse +/// failure is returned as an error. Structural divergence between the two ASTs +/// (possible where post-generation string patching altered the code shape) is +/// handled defensively: the divergent subtree is skipped with a debug log. +pub(crate) fn extract_statement_mappings( + bundle_text: &str, + bundled_ast: &ModModule, + provenance: &ProvenanceResolver, +) -> anyhow::Result> { + let reparsed = ruff_python_parser::parse_module(bundle_text) + .context("bundled output failed to re-parse during source map extraction")? + .into_syntax(); + + let mut walker = ParallelWalker { + line_index: LineIndex::new(bundle_text), + provenance, + records: Vec::new(), + }; + walker.walk_body(&reparsed.body, &bundled_ast.body); + Ok(walker.records) +} + +/// Statement-only parallel traversal of the re-parsed bundle and the bundled AST. +struct ParallelWalker<'a> { + /// Line index over the emitted bundle text. + line_index: LineIndex, + provenance: &'a ProvenanceResolver, + records: Vec, +} + +impl ParallelWalker<'_> { + /// Walk two statement lists in lockstep; skip entirely on length divergence. + fn walk_body(&mut self, generated: &[Stmt], original: &[Stmt]) { + if generated.len() != original.len() { + log::debug!( + "source map: skipping diverged body (generated {} statements, bundled AST {})", + generated.len(), + original.len() + ); + return; + } + for (generated_stmt, original_stmt) in generated.iter().zip(original) { + self.walk_stmt(generated_stmt, original_stmt); + } + } + + /// Record a mapping for one aligned statement pair and recurse into nested bodies. + fn walk_stmt(&mut self, generated: &Stmt, original: &Stmt) { + if std::mem::discriminant(generated) != std::mem::discriminant(original) { + log::debug!("source map: skipping diverged statement pair"); + return; + } + + if let Some((module_ordinal, original_line)) = self + .provenance + .resolve(original.node_index().load(), original.range().start()) + { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(generated.range().start()), + module_ordinal, + original_line, + }); + } + + // Recurse into nested statement bodies even when the statement itself is + // synthesized: wrapper-module init functions are synthesized `def`s whose + // bodies contain original module statements. + match (generated, original) { + (Stmt::FunctionDef(g), Stmt::FunctionDef(o)) => self.walk_body(&g.body, &o.body), + (Stmt::ClassDef(g), Stmt::ClassDef(o)) => self.walk_body(&g.body, &o.body), + (Stmt::If(g), Stmt::If(o)) => { + self.walk_body(&g.body, &o.body); + if g.elif_else_clauses.len() == o.elif_else_clauses.len() { + for (gen_clause, orig_clause) in + g.elif_else_clauses.iter().zip(&o.elif_else_clauses) + { + self.walk_body(&gen_clause.body, &orig_clause.body); + } + } + } + (Stmt::While(g), Stmt::While(o)) => { + self.walk_body(&g.body, &o.body); + self.walk_body(&g.orelse, &o.orelse); + } + (Stmt::For(g), Stmt::For(o)) => { + self.walk_body(&g.body, &o.body); + self.walk_body(&g.orelse, &o.orelse); + } + (Stmt::With(g), Stmt::With(o)) => self.walk_body(&g.body, &o.body), + (Stmt::Try(g), Stmt::Try(o)) => { + self.walk_body(&g.body, &o.body); + if g.handlers.len() == o.handlers.len() { + for (gen_handler, orig_handler) in g.handlers.iter().zip(&o.handlers) { + let ruff_python_ast::ExceptHandler::ExceptHandler(gen_handler) = + gen_handler; + let ruff_python_ast::ExceptHandler::ExceptHandler(orig_handler) = + orig_handler; + self.walk_body(&gen_handler.body, &orig_handler.body); + } + } + self.walk_body(&g.orelse, &o.orelse); + self.walk_body(&g.finalbody, &o.finalbody); + } + (Stmt::Match(g), Stmt::Match(o)) if g.cases.len() == o.cases.len() => { + for (gen_case, orig_case) in g.cases.iter().zip(&o.cases) { + self.walk_body(&gen_case.body, &orig_case.body); + } + } + _ => {} + } + } +} + +/// Options for assembling a complete source map for a bundle. +#[derive(Debug)] +pub(crate) struct SourceMapOptions<'a> { + /// Value of the `file` field (the bundle's file name, or ``). + pub(crate) file: &'a str, + /// Whether to embed original source text as `sourcesContent`. + pub(crate) include_contents: bool, + /// Directory the map will live in; source paths are recorded relative to it. + /// When `None`, paths are recorded as given. + pub(crate) base_dir: Option<&'a std::path::Path>, +} + +/// Compute a relative path from `base` to `target` using lexical components +/// (no filesystem access). Falls back to `target` as-is when the two share no +/// common prefix that allows a relative form (e.g., different roots). +fn relative_path(base: &std::path::Path, target: &std::path::Path) -> std::path::PathBuf { + use std::path::Component; + + let mut base_components = base.components().peekable(); + let mut target_components = target.components().peekable(); + + // Drop the shared prefix. + while let (Some(b), Some(t)) = (base_components.peek(), target_components.peek()) { + if b == t { + base_components.next(); + target_components.next(); + } else { + break; + } + } + + let mut result = std::path::PathBuf::new(); + for component in base_components { + match component { + Component::Normal(_) => result.push(".."), + // A remaining root/prefix component means the paths have no common + // ancestor expressible relatively; keep the target as-is. + Component::RootDir | Component::Prefix(_) => return target.to_path_buf(), + _ => {} + } + } + result.extend(target_components); + result +} + +/// Comment linking the bundle to an adjacent source map file. +/// +/// The `sourceMappingURL` convention is borrowed from the JS ecosystem; Python +/// treats the line as a plain comment. +pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { + format!("# sourceMappingURL={map_file_name}\n") +} + +/// Comment embedding the source map as a base64 data URL. +pub(crate) fn inline_source_mapping_comment(map_json: &str) -> String { + let encoded = base64_simd::STANDARD.encode_to_string(map_json.as_bytes()); + format!("# sourceMappingURL=data:application/json;base64,{encoded}\n") +} + +/// The traceback-remapping runtime injected into bundles built with source maps. +/// +/// Lazy and duress-tolerant by design; see `docs/source-maps.md`. +const RUNTIME_TEMPLATE: &str = include_str!("python/sourcemap_runtime.py"); + +/// Placeholder in the runtime template replaced with the delivery mode. +const RUNTIME_MODE_PLACEHOLDER: &str = "__CRIBO_SOURCEMAP_MODE__"; + +/// Inject the traceback-remapping runtime prologue into the bundled AST. +/// +/// Statements are inserted after any leading `from __future__` imports (which +/// must stay first) and before all other code, so the hooks install before any +/// user code runs. The parsed statements carry no node provenance, so the +/// mapping walk transparently skips them while staying structurally aligned. +/// +/// A template parse failure is a cribo bug; it degrades gracefully (warning, +/// no runtime) rather than failing the bundle. +pub(crate) fn inject_runtime_prologue( + bundled_ast: &mut ModModule, + mode: crate::config::SourceMapMode, +) { + use cow_utils::CowUtils as _; + + let mode_str = match mode { + crate::config::SourceMapMode::Linked => "linked", + crate::config::SourceMapMode::Inline => "inline", + crate::config::SourceMapMode::External => "external", + }; + let source = RUNTIME_TEMPLATE.cow_replace(RUNTIME_MODE_PLACEHOLDER, mode_str); + match ruff_python_parser::parse_module(&source) { + Ok(parsed) => { + let insert_at = bundled_ast + .body + .iter() + .take_while(|stmt| is_future_import(stmt)) + .count(); + bundled_ast + .body + .splice(insert_at..insert_at, parsed.into_syntax().body); + } + Err(err) => log::warn!( + "source map runtime template failed to parse; traceback remapping disabled: {err}" + ), + } +} + +/// Whether a statement is a `from __future__ import ...`. +fn is_future_import(stmt: &Stmt) -> bool { + matches!( + stmt, + Stmt::ImportFrom(import) + if import.module.as_ref().is_some_and(|module| module.as_str() == "__future__") + ) +} + +/// Build the complete Source Map v3 JSON for an emitted bundle. +/// +/// Re-parses `bundle_text`, extracts statement mappings against `bundled_ast`, +/// and serializes them. Only modules that actually contributed mappings appear +/// in the `sources` array. +pub(crate) fn build_source_map( + bundle_text: &str, + bundled_ast: &ModModule, + provenance: &ProvenanceResolver, + options: &SourceMapOptions<'_>, +) -> anyhow::Result { + let records = extract_statement_mappings(bundle_text, bundled_ast, provenance)?; + + let mut generator = SourceMapGenerator::new(options.file); + let mut ordinal_to_source: Vec> = vec![None; provenance.modules().len()]; + + for record in records { + let module = &provenance.modules()[record.module_ordinal]; + let source_id = ordinal_to_source[record.module_ordinal].unwrap_or_else(|| { + let display_path = options.base_dir.map_or_else( + || module.path.clone(), + |base| relative_path(base, &module.path), + ); + let content = options.include_contents.then(|| module.source.clone()); + let id = generator.add_source(&display_path.to_string_lossy(), content); + ordinal_to_source[record.module_ordinal] = Some(id); + id + }); + generator.add_mapping(record.generated_line, source_id, record.original_line); + } + + Ok(generator.into_json()) +} + +#[cfg(test)] +mod tests { + use oxc_sourcemap::SourceMap; + use ruff_python_ast::Stmt; + + use super::*; + + #[test] + fn empty_map_is_valid_v3() { + let generator = SourceMapGenerator::new("bundle.py"); + assert!(generator.is_empty()); + let json = generator.into_json(); + + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_file(), Some("bundle.py")); + assert_eq!(parsed.get_tokens().count(), 0); + assert!(json.contains("\"version\":3")); + assert!(!json.contains("sourcesContent")); + } + + #[test] + fn mappings_round_trip_through_vlq() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let main = generator.add_source("main.py", None); + let utils = generator.add_source("utils.py", None); + // Insert out of order to exercise the sort. + generator.add_mapping(10, utils, 3); + generator.add_mapping(4, main, 0); + generator.add_mapping(7, utils, 1); + let json = generator.into_json(); + + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + let lookup = parsed.generate_lookup_table(); + + let token = parsed + .lookup_token(&lookup, 4, 0) + .expect("mapping for line 4"); + assert_eq!( + parsed.get_source(token.get_source_id().expect("source id")), + Some("main.py") + ); + assert_eq!(token.get_src_line(), 0); + + let token = parsed + .lookup_token(&lookup, 10, 0) + .expect("mapping for line 10"); + assert_eq!( + parsed.get_source(token.get_source_id().expect("source id")), + Some("utils.py") + ); + assert_eq!(token.get_src_line(), 3); + + // An unmapped line before the first token has no mapping. + assert!(parsed.lookup_token(&lookup, 0, 0).is_none()); + } + + #[test] + fn source_dedup_returns_same_id_and_first_content_wins() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let first = generator.add_source("pkg/mod.py", Some("x = 1\n".to_owned())); + let second = generator.add_source("pkg/mod.py", Some("ignored".to_owned())); + assert_eq!(first, second); + + generator.add_mapping(0, first, 0); + let json = generator.into_json(); + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_sources().count(), 1); + assert_eq!(parsed.get_source_content(first), Some("x = 1\n")); + } + + #[test] + fn content_backfills_when_first_registration_had_none() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let id = generator.add_source("mod.py", None); + let same = generator.add_source("mod.py", Some("y = 2\n".to_owned())); + assert_eq!(id, same); + + generator.add_mapping(0, id, 0); + let json = generator.into_json(); + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_source_content(id), Some("y = 2\n")); + } + + #[test] + fn sources_content_present_iff_any_content() { + let mut with_content = SourceMapGenerator::new("bundle.py"); + let id = with_content.add_source("a.py", Some("a = 1\n".to_owned())); + with_content.add_source("b.py", None); + with_content.add_mapping(0, id, 0); + let json = with_content.into_json(); + assert!(json.contains("sourcesContent")); + assert!(json.contains("null"), "missing content must encode as null"); + + let mut without_content = SourceMapGenerator::new("bundle.py"); + let id = without_content.add_source("a.py", None); + without_content.add_mapping(0, id, 0); + assert!(!without_content.into_json().contains("sourcesContent")); + } + + #[test] + fn first_mapping_per_generated_line_wins() { + let mut generator = SourceMapGenerator::new("bundle.py"); + let first = generator.add_source("first.py", None); + let second = generator.add_source("second.py", None); + generator.add_mapping(5, first, 11); + generator.add_mapping(5, second, 99); + let json = generator.into_json(); + + let parsed = SourceMap::from_json_string(&json).expect("valid source map JSON"); + assert_eq!(parsed.get_tokens().count(), 1); + let lookup = parsed.generate_lookup_table(); + let token = parsed + .lookup_token(&lookup, 5, 0) + .expect("mapping for line 5"); + assert_eq!( + parsed.get_source(token.get_source_id().expect("source id")), + Some("first.py") + ); + assert_eq!(token.get_src_line(), 11); + } + + #[test] + fn line_index_maps_offsets_to_lines() { + let index = LineIndex::new("a = 1\nb = 2\n\nc = 3\n"); + assert_eq!(index.line_of(TextSize::from(0)), 0); // 'a' + assert_eq!(index.line_of(TextSize::from(5)), 0); // the '\n' itself + assert_eq!(index.line_of(TextSize::from(6)), 1); // 'b' + assert_eq!(index.line_of(TextSize::from(12)), 2); // empty line + assert_eq!(index.line_of(TextSize::from(13)), 3); // 'c' + } + + #[test] + fn line_index_handles_source_without_trailing_newline() { + let index = LineIndex::new("x = 1"); + assert_eq!(index.line_of(TextSize::from(0)), 0); + assert_eq!(index.line_of(TextSize::from(4)), 0); + } + + #[test] + fn provenance_resolves_nodes_across_modules() { + use ruff_text_size::Ranged as _; + + let source_a = "a1 = 1\na2 = 2\n"; + let source_b = "def f():\n return 3\n\nb2 = f()\n"; + let mut ast_a = ruff_python_parser::parse_module(source_a) + .expect("parse module a") + .into_syntax(); + let mut ast_b = ruff_python_parser::parse_module(source_b) + .expect("parse module b") + .into_syntax(); + crate::ast_indexer::index_module_with_id(&mut ast_a, 0); + crate::ast_indexer::index_module_with_id(&mut ast_b, 1); + + let mut resolver = ProvenanceResolver::default(); + resolver.push_module(std::path::PathBuf::from("a.py"), source_a.to_owned()); + resolver.push_module(std::path::PathBuf::from("b.py"), source_b.to_owned()); + + // Second statement of module a: `a2 = 2` on line 1. + let stmt = &ast_a.body[1]; + let Stmt::Assign(assign) = stmt else { + panic!("expected assign statement"); + }; + let resolved = resolver.resolve(assign.node_index.load(), stmt.range().start()); + assert_eq!(resolved, Some((0, 1))); + + // The `return 3` statement nested inside `def f()` of module b: line 1. + let Stmt::FunctionDef(func) = &ast_b.body[0] else { + panic!("expected function def"); + }; + let ret = &func.body[0]; + let Stmt::Return(ret_stmt) = ret else { + panic!("expected return statement"); + }; + let resolved = resolver.resolve(ret_stmt.node_index.load(), ret.range().start()); + assert_eq!(resolved, Some((1, 1))); + + // Last statement of module b: `b2 = f()` on line 3. + let Stmt::Assign(assign) = &ast_b.body[1] else { + panic!("expected assign statement"); + }; + let resolved = resolver.resolve(assign.node_index.load(), ast_b.body[1].range().start()); + assert_eq!(resolved, Some((1, 3))); + } + + #[test] + fn provenance_rejects_synthesized_nodes() { + use ruff_python_ast::AtomicNodeIndex; + use ruff_text_size::TextRange; + + let mut resolver = ProvenanceResolver::default(); + resolver.push_module(std::path::PathBuf::from("a.py"), "x = 1\n".to_owned()); + + // A node built by ast_builder carries the NONE placeholder index. + let synthesized = crate::ast_builder::statements::pass(); + let Stmt::Pass(pass) = &synthesized else { + panic!("expected pass statement"); + }; + assert_eq!( + resolver.resolve(pass.node_index.load(), TextRange::default().start()), + None + ); + + // A node index past all module ranges (TransformationContext territory) + // also resolves to None. + let index = AtomicNodeIndex::default(); + index.set(NodeIndex::from(MODULE_INDEX_RANGE)); + assert_eq!(resolver.resolve(index.load(), TextSize::from(0)), None); + } +} diff --git a/crates/cribo/tests/fixtures/sourcemap_basic/calculator.py b/crates/cribo/tests/fixtures/sourcemap_basic/calculator.py new file mode 100644 index 000000000..0665c86b1 --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_basic/calculator.py @@ -0,0 +1,8 @@ +def add(a, b): + result = a + b + return result + + +def multiply(a, b): + result = a * b + return result diff --git a/crates/cribo/tests/fixtures/sourcemap_basic/main.py b/crates/cribo/tests/fixtures/sourcemap_basic/main.py new file mode 100644 index 000000000..59e4de30e --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_basic/main.py @@ -0,0 +1,7 @@ +from calculator import add, multiply +from utils import describe + +total = add(2, 3) +product = multiply(total, 4) +print(describe("total", total)) +print(describe("product", product)) diff --git a/crates/cribo/tests/fixtures/sourcemap_basic/utils.py b/crates/cribo/tests/fixtures/sourcemap_basic/utils.py new file mode 100644 index 000000000..37121d48a --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_basic/utils.py @@ -0,0 +1,2 @@ +def describe(name, value): + return f"{name} = {value}" diff --git a/crates/cribo/tests/fixtures/sourcemap_wrapper/effects.py b/crates/cribo/tests/fixtures/sourcemap_wrapper/effects.py new file mode 100644 index 000000000..a7fefa262 --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_wrapper/effects.py @@ -0,0 +1,8 @@ +print("effects module loading") + +COUNTER = 1 + + +def boost(value): + boosted = value * 2 + COUNTER + return boosted diff --git a/crates/cribo/tests/fixtures/sourcemap_wrapper/main.py b/crates/cribo/tests/fixtures/sourcemap_wrapper/main.py new file mode 100644 index 000000000..1da29be74 --- /dev/null +++ b/crates/cribo/tests/fixtures/sourcemap_wrapper/main.py @@ -0,0 +1,4 @@ +import effects + +print("counter:", effects.COUNTER) +print("boosted:", effects.boost(10)) diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py new file mode 100644 index 000000000..0fe4fe783 --- /dev/null +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -0,0 +1,203 @@ +"""Unit tests for the cribo source map runtime internals. + +Driven by the Rust integration test `python_runtime_unit_tests` (plain asserts, +no pytest dependency). Usage: python test_sourcemap_runtime.py +where is the template with the mode placeholder substituted. +""" + +import importlib.util +import os +import sys +import tempfile +import threading + + +def load_runtime(path): + """Import the runtime module, then restore the hooks it installs.""" + prev_hooks = (sys.excepthook, sys.unraisablehook, threading.excepthook) + spec = importlib.util.spec_from_file_location("cribo_sm_runtime", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.excepthook, sys.unraisablehook, threading.excepthook = prev_hooks + return module + + +def test_stream_reads_across_chunk_boundaries(rt): + stream = rt._CriboSmStream([b"ab", b"", b"c", b"de"]) + got = [] + while True: + byte = stream.read_byte() + if byte < 0: + break + got.append(chr(byte)) + assert got == list("abcde"), got + assert stream.read_byte() == -1 # stays exhausted + + +def _scan(rt, json_text, needed, max_needed): + return rt._cribo_sm_scan([json_text.encode("utf-8")], needed, max_needed) + + +def test_scan_extracts_sources_and_mappings(rt): + # AAAA;AACA;AACA: one segment per line, source line advancing by one. + json_text = '{"version":3,"sources":["a.py","b.py"],"mappings":"AAAA;AACA;AACA"}' + sources, table = _scan(rt, json_text, {0, 2}, 2) + assert sources == ["a.py", "b.py"], sources + assert table == {0: (0, 0), 2: (0, 2)}, table + + +def test_scan_negative_delta(rt): + # Line 0 advances src_line by +1 (C), line 1 rewinds it by -1 (D). + json_text = '{"sources":["a.py"],"mappings":"AACA;AADA"}' + _sources, table = _scan(rt, json_text, {0, 1}, 1) + assert table == {0: (0, 1), 1: (0, 0)}, table + + +def test_scan_source_index_delta(rt): + # Second line switches to source 1 (C in field 1). + json_text = '{"sources":["a.py","b.py"],"mappings":"AAAA;ACAA"}' + _sources, table = _scan(rt, json_text, {0, 1}, 1) + assert table == {0: (0, 0), 1: (1, 0)}, table + + +def test_scan_ignores_adversarial_sources_content(rt): + # sourcesContent value contains fake keys and escaped quotes; the string + # lexer must skip it without being fooled. + evil = '\\"mappings\\": \\"ZZZZ\\", \\\\' + json_text = ( + '{"sources":["a.py"],"sourcesContent":["' + evil + '"],"mappings":"AAAA"}' + ) + sources, table = _scan(rt, json_text, {0}, 0) + assert sources == ["a.py"], sources + assert table == {0: (0, 0)}, table + + +def test_scan_handles_unicode_escapes_in_sources(rt): + json_text = '{"sources":["\\u00e9t\\u00e9.py"],"mappings":"AAAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources == ["\u00e9t\u00e9.py"], sources + + +def test_scan_null_in_sources_array(rt): + json_text = '{"sources":["a.py",null,"c.py"],"mappings":"AAAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources == ["a.py", None, "c.py"], sources + + +def test_vlq_early_exit_stops_reading(rt): + # The decoder must stop pulling chunks once past max_needed: the second + # chunk raises if consumed. + class Boom(Exception): + pass + + def chunks(): + yield b'{"sources":["a.py"],"mappings":"AAAA;AACA;' + raise Boom("decoder read past its early-exit point") + + sources, table = rt._cribo_sm_scan(chunks(), {0}, 0) + assert table == {0: (0, 0)}, table + + +def test_vlq_rejects_escapes_in_mappings(rt): + json_text = '{"sources":["a.py"],"mappings":"AA\\\\AA"}' + try: + _scan(rt, json_text, {0}, 0) + except ValueError: + pass + else: + raise AssertionError("escape inside mappings must raise") + + +def make_inline_bundle(payload_json, line_length=None): + """Create a temp file shaped like an inline-mode bundle; return its path.""" + import base64 + + encoded = base64.b64encode(payload_json.encode("utf-8")).decode("ascii") + handle = tempfile.NamedTemporaryFile( + "w", suffix=".py", delete=False, encoding="utf-8" + ) + with handle as f: + f.write("print('hello')\n" * 300) # push the marker past one chunk + f.write("# sourceMappingURL=data:application/json;base64," + encoded + "\n") + return handle.name + + +def test_inline_payload_scan_and_chunked_base64(rt): + # Payload much larger than one 8 KiB chunk exercises 4-byte alignment + # handling across chunk boundaries. + filler = "x" * 40000 + json_text = '{"filler":"' + filler + '","sources":["a.py"],"mappings":"AAAA"}' + path = make_inline_bundle(json_text) + try: + decoded = b"".join(rt._cribo_sm_inline_chunks(path)) + assert decoded.decode("utf-8") == json_text + # And end-to-end through the scanner: + sources, table = rt._cribo_sm_scan( + rt._cribo_sm_inline_chunks(path), {0}, 0 + ) + assert sources == ["a.py"], sources + assert table == {0: (0, 0)}, table + finally: + os.unlink(path) + + +def test_inline_scan_without_marker_yields_nothing(rt): + handle = tempfile.NamedTemporaryFile( + "w", suffix=".py", delete=False, encoding="utf-8" + ) + with handle as f: + f.write("print('no map here')\n" * 50) + try: + assert b"".join(rt._cribo_sm_inline_chunks(handle.name)) == b"" + finally: + os.unlink(handle.name) + + +def test_json_fallback_matches_streaming(rt): + json_text = '{"sources":["a.py","b.py"],"mappings":"AAAA;ACCA"}' + streaming = _scan(rt, json_text, {0, 1}, 1) + + path = None + handle = tempfile.NamedTemporaryFile("w", suffix=".map", delete=False) + with handle as f: + f.write(json_text) + path = handle.name + try: + os.environ["CRIBO_SOURCE_MAPS"] = path + loaded = rt._cribo_sm_load_json_fallback({1, 2}) + assert loaded is not None + table, sources, _map_dir = loaded + # Fallback tables are 1-based. + expected = {line0 + 1: (idx, line0src + 1) for line0, (idx, line0src) in streaming[1].items()} + assert table == expected, (table, expected) + assert sources == streaming[0] + finally: + del os.environ["CRIBO_SOURCE_MAPS"] + os.unlink(path) + + +def main(): + runtime_path = sys.argv[1] + rt = load_runtime(runtime_path) + tests = [ + test_stream_reads_across_chunk_boundaries, + test_scan_extracts_sources_and_mappings, + test_scan_negative_delta, + test_scan_source_index_delta, + test_scan_ignores_adversarial_sources_content, + test_scan_handles_unicode_escapes_in_sources, + test_scan_null_in_sources_array, + test_vlq_early_exit_stops_reading, + test_vlq_rejects_escapes_in_mappings, + test_inline_payload_scan_and_chunked_base64, + test_inline_scan_without_marker_yields_nothing, + test_json_fallback_matches_streaming, + ] + for test in tests: + test(rt) + print("PASS %s" % test.__name__) + print("ALL %d RUNTIME TESTS PASSED" % len(tests)) + + +if __name__ == "__main__": + main() diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap new file mode 100644 index 000000000..d717b2542 --- /dev/null +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -0,0 +1,651 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +#!/usr/bin/env python3 +# Generated by Cribo - Python Source Bundler +# https://github.com/ophidiarium/cribo + +"""Cribo source map runtime (injected prologue).\n\nRemaps tracebacks of uncaught exceptions back to the original source files\nusing the Source Map v3 emitted at bundle time. Lazy by design: no file I/O,\nparsing, or decoding happens at import time; everything is deferred to the\nfirst uncaught exception. Under resource pressure the decoder streams the map\nin constant memory and falls back to the default traceback on any failure.\nSee docs/source-maps.md in the cribo repository.\n""" +import binascii as _cribo_binascii +import os as _cribo_os +import sys as _cribo_sys +import threading as _cribo_threading +_CRIBO_SM_MODE = "linked" +_CRIBO_SM_BUNDLE = globals().get("__file__", "") +_CRIBO_SM_CHUNK = 8192 +_CRIBO_SM_STATE = {"in_hook": False} +_CRIBO_SM_B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" +_cribo_sm_prev_excepthook = _cribo_sys.excepthook +_cribo_sm_prev_unraisablehook = _cribo_sys.unraisablehook +_cribo_sm_prev_threading_hook = _cribo_threading.excepthook +def _cribo_sm_map_location(): + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time so\n the happy path never touches the environment or the filesystem.\n """ + env = _cribo_os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + bundle = _CRIBO_SM_BUNDLE + if _CRIBO_SM_MODE == "inline": + return None, _cribo_os.path.dirname(_cribo_os.path.abspath(bundle)) + sibling = bundle + ".map" + if _CRIBO_SM_MODE == "linked": + if _cribo_os.path.exists(sibling): + return sibling, _cribo_os.path.dirname(_cribo_os.path.abspath(sibling)) + return None + if env in ("1", "true", "yes", "on"): + path = sibling + elif env: + path = env + else: + return None + return path, _cribo_os.path.dirname(_cribo_os.path.abspath(path)) +def _cribo_sm_file_chunks(path): + """Yield fixed-size chunks of a file (constant memory).""" + handle = open(path, "rb") + try: + while True: + chunk = handle.read(_CRIBO_SM_CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() +def _cribo_sm_find_inline_payload(handle): + """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of the\n file is examined; the bundle body is never read.\n """ + marker = b"# sourceMappingURL=data:" + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + found = -1 + while position > 0: + step = _CRIBO_SM_CHUNK if position >= _CRIBO_SM_CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + found = position + index + break + overlap = data[:len(marker) - 1] + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + len(b"base64,") +def _cribo_sm_inline_chunks(path): + """Yield decoded chunks of an inline (base64 data URL) source map.""" + handle = open(path, "rb") + try: + start = _cribo_sm_find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(_CRIBO_SM_CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = len(data) - len(data) % 4 + pending = data[usable:] + if usable: + yield _cribo_binascii.a2b_base64(data[:usable]) + if pending: + yield _cribo_binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + finally: + handle.close() +class _CriboSmStream(object): + """Byte-at-a-time reader over an iterator of byte chunks.""" + __slots__ = "_chunks", "_buf", "_pos" + + def __init__(self, chunks): + self._chunks = iter(chunks) + self._buf = b"" + self._pos = 0 + + def read_byte(self): + while self._pos >= len(self._buf): + try: + self._buf = next(self._chunks) + except StopIteration: + return -1 + self._pos = 0 + value = self._buf[self._pos] + self._pos += 1 + return value +def _cribo_sm_skip_ws(stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte +def _cribo_sm_read_string(stream, collect): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are handled,\n so string *values* containing text like '\"mappings\":' cannot confuse the\n key scanner.\n """ + buf = bytearray() if collect else None + while True: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON string") + if byte == 34: + return buf.decode("utf-8", "replace") if collect else None + if byte != 92: + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise ValueError("unterminated JSON escape") + if escape == 117: + code = 0 + for _ in range(4): + digit = stream.read_byte() + if digit < 0: + raise ValueError("unterminated unicode escape") + code = code * 16 + int(chr(digit), 16) + if buf is not None: + buf.extend(chr(code).encode("utf-8", "surrogatepass")) + elif buf is not None: + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf.append(table.get(escape, escape)) +def _cribo_sm_skip_value(stream, byte): + """Skip one JSON value; return the first byte after it (or -1).""" + if byte == 34: + _cribo_sm_read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON container") + if byte == 34: + _cribo_sm_read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + while byte >= 0 and byte not in (44, 125, 93): + byte = stream.read_byte() + return byte +def _cribo_sm_read_string_array(stream, byte): + """Read a JSON array of strings/nulls; return (list, byte after array).""" + if byte != 91: + raise ValueError("expected array") + items = [] + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte == 93: + return items, stream.read_byte() + while True: + if byte == 34: + items.append(_cribo_sm_read_string(stream, True)) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + else: + items.append(None) + byte = _cribo_sm_skip_ws(stream, _cribo_sm_skip_value(stream, byte)) + if byte == 93: + return items, stream.read_byte() + if byte != 44: + raise ValueError("malformed array") + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) +def _cribo_sm_decode_vlq(stream, needed, max_needed): + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ + lut = {} + for index in range(64): + lut[_CRIBO_SM_B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 + vlq_value = 0 + vlq_shift = 0 + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = src_idx, src_line + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: + end_segment() + return result + if byte == 59: + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or len(result) == len(needed): + return result + continue + if byte == 44: + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise ValueError("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 +def _cribo_sm_scan(chunks, needed, max_needed): + """Scan the map's top-level object; return (sources, line table).""" + stream = _CriboSmStream(chunks) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte != 123: + raise ValueError("not a JSON object") + sources = [] + table = {} + saw_mappings = False + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + while byte == 34: + key = _cribo_sm_read_string(stream, True) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte != 58: + raise ValueError("malformed object") + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if key == "sources": + sources, byte = _cribo_sm_read_string_array(stream, byte) + elif key == "mappings": + if byte != 34: + raise ValueError("mappings is not a string") + table = _cribo_sm_decode_vlq(stream, needed, max_needed) + saw_mappings = True + if sources: + break + byte = stream.read_byte() + else: + byte = _cribo_sm_skip_value(stream, byte) + byte = _cribo_sm_skip_ws(stream, byte) + if byte == 44: + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if not saw_mappings: + raise ValueError("no mappings field") + return sources, table +def _cribo_sm_load(needed_lines): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ + location = _cribo_sm_map_location() + if location is None: + return None + map_path, map_dir = location + needed0 = set(line - 1 for line in needed_lines) + max_needed = max(needed0) + if map_path is None: + chunks = _cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE) + else: + chunks = _cribo_sm_file_chunks(map_path) + sources, table0 = _cribo_sm_scan(chunks, needed0, max_needed) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir +def _cribo_sm_load_json_fallback(needed_lines): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = _cribo_sm_map_location() + if location is None: + return None + map_path, map_dir = location + import json + if map_path is None: + raw = b"".join(_cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE)) + else: + handle = open(map_path, "rb") + try: + raw = handle.read() + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = data.get("sources") or [] + mappings = data.get("mappings") or "" + needed0 = set(line - 1 for line in needed_lines) + stream = _CriboSmStream([mappings.encode("ascii"), b'"']) + table0 = _cribo_sm_decode_vlq(stream, needed0, max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir +def _cribo_sm_collect_needed(exc_value, traceback_obj): + """1-based bundle line numbers referenced by the traceback (and its chain).""" + needed = set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == _CRIBO_SM_BUNDLE: + needed.add(tb.tb_lineno) + tb = tb.tb_next + add(traceback_obj) + exc = exc_value + seen = set() + depth = 0 + while exc is not None and id(exc) not in seen and depth < 16: + seen.add(id(exc)) + depth += 1 + add(getattr(exc, "__traceback__", None)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed +def _cribo_sm_source_line(path, lineno): + """Read a single 1-based line from a file without caching it.""" + try: + handle = open(path, "rb") + except OSError: + return None + try: + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except OSError: + return None + finally: + handle.close() + return None +def _cribo_sm_write_frames(traceback_obj, table, sources, map_dir, write): + '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed by\n a "[Previous line repeated N more times]" marker; source line text is\n cached per (file, line) within one rendering to avoid re-reading files.\n ''' + cache = {} + last = None + repeats = 0 + + def emit(entry): + write(' File "%s", line %d, in %s\n' % entry) + key = entry[0], entry[1] + if key not in cache: + cache[key] = _cribo_sm_source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + if filename == _CRIBO_SM_BUNDLE: + mapped = table.get(lineno) + if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + source = sources[mapped[0]] + if not _cribo_os.path.isabs(source): + source = _cribo_os.path.normpath(_cribo_os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] + entry = filename, lineno, name + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 + emit(entry) + traceback_obj = traceback_obj.tb_next + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) +def _cribo_sm_exception_line(exc_value): + exc_type = type(exc_value) + name = getattr(exc_type, "__qualname__", exc_type.__name__) + module = getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = str(exc_value) + except BaseException: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name +def _cribo_sm_render(exc_value, table, sources, map_dir, write): + """Render the exception (with its cause/context chain) like CPython does.""" + chain = [] + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(chain) < 16: + seen.add(id(exc)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + suppress = getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context + else: + chain.append((exc, None)) + exc = None + ordered = list(reversed(chain)) + for index, (exc, link) in enumerate(ordered): + if index > 0: + if link == "cause": + write("\nThe above exception was the direct cause of the following exception:\n\n") + else: + write("\nDuring handling of the above exception, another exception occurred:\n\n") + tb = getattr(exc, "__traceback__", None) + if tb is not None: + write("Traceback (most recent call last):\n") + _cribo_sm_write_frames(tb, table, sources, map_dir, write) + write(_cribo_sm_exception_line(exc)) +def _cribo_sm_try_render(exc_value, traceback_obj, prefix): + """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in this\n runtime returns False so callers can delegate to the previous hook.\n """ + if _CRIBO_SM_STATE["in_hook"] or exc_value is None: + return False + _CRIBO_SM_STATE["in_hook"] = True + old_limit = None + try: + needed = _cribo_sm_collect_needed(exc_value, traceback_obj) + if not needed: + return False + loaded = None + try: + loaded = _cribo_sm_load(needed) + except BaseException: + try: + loaded = _cribo_sm_load_json_fallback(needed) + except BaseException: + loaded = None + if not loaded: + return False + table, sources, map_dir = loaded + if not table: + return False + try: + old_limit = _cribo_sys.getrecursionlimit() + _cribo_sys.setrecursionlimit(old_limit + 64) + except BaseException: + old_limit = None + parts = [] + if prefix: + parts.append(prefix) + _cribo_sm_render(exc_value, table, sources, map_dir, parts.append) + stderr = _cribo_sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except BaseException: + pass + return True + except BaseException: + return False + finally: + if old_limit is not None: + try: + _cribo_sys.setrecursionlimit(old_limit) + except BaseException: + pass + _CRIBO_SM_STATE["in_hook"] = False +def _cribo_sm_excepthook(exc_type, exc_value, traceback_obj): + if not _cribo_sm_try_render(exc_value, traceback_obj, None): + _cribo_sm_prev_excepthook(exc_type, exc_value, traceback_obj) +def _cribo_sm_threading_hook(args): + thread = getattr(args, "thread", None) + name = getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if not _cribo_sm_try_render(args.exc_value, args.exc_traceback, prefix): + _cribo_sm_prev_threading_hook(args) +def _cribo_sm_unraisablehook(unraisable): + message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except BaseException: + prefix = "%s\n" % message + if not _cribo_sm_try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + _cribo_sm_prev_unraisablehook(unraisable) +_cribo_sys.excepthook = _cribo_sm_excepthook +_cribo_sys.unraisablehook = _cribo_sm_unraisablehook +_cribo_threading.excepthook = _cribo_sm_threading_hook +import sys as _sys +import importlib as _importlib +class _CriboModule(): + + def __init__(_cribo_self, m, p): + _cribo_self._m, _cribo_self._p = m, p + + def __getattr__(_cribo_self, n, *, _getattr=getattr): + f = _cribo_self._p + '.' + n + try: + return _CriboModule(_importlib.import_module(f), f) + except ImportError: + return _getattr(_cribo_self._m, n) + + def __getattribute__(_cribo_self, n, *, _getattr=getattr, _object=object): + return _object.__getattribute__(_cribo_self, n) if n in ('_m', '_p', '__getattr__', '__class__', '__dict__', '__dir__', '__module__', '__qualname__') else _getattr(_object.__getattribute__(_cribo_self, '_m'), n) +class _Cribo(): + + def __getattr__(_cribo_self, n): + m = _sys.modules.get(n) or _importlib.import_module(n) + return _CriboModule(m, n) +_cribo = _Cribo() +_cribo_captured = {} +class _CriboPreservedLoader: + _pristine = {} + + def __init__(self, entry, namespace=None, reload_target=None): + self._entry = entry + self._namespace = namespace + self._reload_target = reload_target + + def create_module(self, spec, *, _getattr=getattr, _globals=globals, _id=id, _setattr=setattr, _type=type, _ModuleType=_cribo.types.ModuleType, _captured=_cribo_captured): + module = None + if self._namespace is not None: + module = self._namespace + elif self._entry[1] is not None: + module = _globals()[self._entry[1]] + if module is not None: + if _getattr(module, '_cribo_machinery_loaded', False) or _getattr(module, '_cribo_failed_life', False) or _getattr(module, '__initialized__', False) and _getattr(module, '_cribo_registered', False): + saved = _type(self)._pristine.get(_id(module)) + fresh = _ModuleType(spec.name) + fresh.__dict__.update(saved if saved is not None else {'__name__': spec.name}) + fresh._cribo_fresh_life = True + return fresh + return module + module = _ModuleType(spec.name) + for export, binding in self._entry[3].items(): + _setattr(module, export, _captured[binding] if binding in _captured else _globals()[binding]) + return module + + def exec_module(self, module, *, _BaseException=BaseException, _dict=dict, _getattr=getattr, _globals=globals, _id=id, _type=type): + init = self._entry[0] + pristine = _type(self)._pristine + key = _id(module) + record_snapshot = not _getattr(module, '_cribo_fresh_life', False) and not _getattr(module, '__initialized__', False) and (self._namespace is not None or self._entry[1] is not None) + if init is None: + if record_snapshot and key not in pristine: + pristine[key] = _dict(module.__dict__) + module._cribo_machinery_loaded = True + return + is_reload = self._reload_target is module + if is_reload: + module.__initialized__ = False + module.__initializing__ = False + state = _dict(module.__dict__) + if record_snapshot and key not in pristine: + pristine[key] = _dict(state) + try: + _globals()[init](module) + module._cribo_machinery_loaded = True + except _BaseException: + if is_reload: + module.__initializing__ = False + else: + module.__dict__.clear() + module.__dict__.update(state) + module.__initializing__ = False + module._cribo_failed_life = True + raise +class _CriboPreservedFinder: + + def __init__(self): + self._targets = {} + self._namespaces = {} + self._registry = {} + self._loader = _CriboPreservedLoader + + def register(self, name, init, namespace, is_package, exports=None): + entry = init, namespace, is_package, exports or {} + self._targets[name] = entry + self._registry[name] = entry + + def bind(self, name, namespace): + self._namespaces[name] = namespace + + def find_spec(self, name, path=None, target=None, *, _getattr=getattr, _globals=globals): + entry = self._targets.get(name) + if entry is None: + return None + parent_name = name.rpartition('.')[0] + if parent_name: + parent = _sys.modules.get(parent_name) + expected = self._namespaces.get(parent_name) + if expected is None: + parent_entry = self._registry.get(parent_name) + if parent_entry is not None and parent_entry[1] is not None: + expected = _globals().get(parent_entry[1]) + if parent is not None and expected is not None and parent is not expected and not _getattr(parent, '_cribo_fresh_life', False) and not _getattr(parent, '_cribo_machinery_loaded', False): + return None + from importlib.machinery import ModuleSpec + return ModuleSpec(name, self._loader(entry, self._namespaces.get(name), target), is_package=entry[2]) +_cribo_finder = _CriboPreservedFinder() +_sys.meta_path.append(_cribo_finder) +_cribo_finder_local = _CriboPreservedFinder() +_cribo_finder_local._namespaces = _cribo_finder._namespaces +_cribo_finder_local._registry = _cribo_finder._registry +for _cribo_index, _cribo_meta_finder in enumerate(_sys.meta_path): + if getattr(_cribo_meta_finder, '__name__', '') == 'PathFinder': + _sys.meta_path.insert(_cribo_index, _cribo_finder_local) + break +else: + _sys.meta_path.insert(0, _cribo_finder_local) +_cribo_finder_local.register('calculator', None, None, False, {'add': 'add', 'multiply': 'multiply'}) +_cribo_finder_local.register('utils', None, None, False, {'describe': 'describe'}) +calculator = _cribo.types.SimpleNamespace(__name__='calculator') +def add(a, b): + result = a + b + return result +add.__module__ = 'calculator' +def multiply(a, b): + result = a * b + return result +multiply.__module__ = 'calculator' +calculator.add = add +calculator.multiply = multiply +utils = _cribo.types.SimpleNamespace(__name__='utils') +def describe(name, value): + return f"{name} = {value}" +describe.__module__ = 'utils' +utils.describe = describe +_cribo_captured['add'] = add +_cribo_captured['multiply'] = multiply +_cribo_captured['describe'] = describe +total = add(2, 3) +product = multiply(total, 4) +print(describe("total", total)) +print(describe("product", product)) +# sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap new file mode 100644 index 000000000..363b32253 --- /dev/null +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -0,0 +1,658 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +#!/usr/bin/env python3 +# Generated by Cribo - Python Source Bundler +# https://github.com/ophidiarium/cribo + +"""Cribo source map runtime (injected prologue).\n\nRemaps tracebacks of uncaught exceptions back to the original source files\nusing the Source Map v3 emitted at bundle time. Lazy by design: no file I/O,\nparsing, or decoding happens at import time; everything is deferred to the\nfirst uncaught exception. Under resource pressure the decoder streams the map\nin constant memory and falls back to the default traceback on any failure.\nSee docs/source-maps.md in the cribo repository.\n""" +import binascii as _cribo_binascii +import os as _cribo_os +import sys as _cribo_sys +import threading as _cribo_threading +_CRIBO_SM_MODE = "linked" +_CRIBO_SM_BUNDLE = globals().get("__file__", "") +_CRIBO_SM_CHUNK = 8192 +_CRIBO_SM_STATE = {"in_hook": False} +_CRIBO_SM_B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" +_cribo_sm_prev_excepthook = _cribo_sys.excepthook +_cribo_sm_prev_unraisablehook = _cribo_sys.unraisablehook +_cribo_sm_prev_threading_hook = _cribo_threading.excepthook +def _cribo_sm_map_location(): + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time so\n the happy path never touches the environment or the filesystem.\n """ + env = _cribo_os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + bundle = _CRIBO_SM_BUNDLE + if _CRIBO_SM_MODE == "inline": + return None, _cribo_os.path.dirname(_cribo_os.path.abspath(bundle)) + sibling = bundle + ".map" + if _CRIBO_SM_MODE == "linked": + if _cribo_os.path.exists(sibling): + return sibling, _cribo_os.path.dirname(_cribo_os.path.abspath(sibling)) + return None + if env in ("1", "true", "yes", "on"): + path = sibling + elif env: + path = env + else: + return None + return path, _cribo_os.path.dirname(_cribo_os.path.abspath(path)) +def _cribo_sm_file_chunks(path): + """Yield fixed-size chunks of a file (constant memory).""" + handle = open(path, "rb") + try: + while True: + chunk = handle.read(_CRIBO_SM_CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() +def _cribo_sm_find_inline_payload(handle): + """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of the\n file is examined; the bundle body is never read.\n """ + marker = b"# sourceMappingURL=data:" + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + found = -1 + while position > 0: + step = _CRIBO_SM_CHUNK if position >= _CRIBO_SM_CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + found = position + index + break + overlap = data[:len(marker) - 1] + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + len(b"base64,") +def _cribo_sm_inline_chunks(path): + """Yield decoded chunks of an inline (base64 data URL) source map.""" + handle = open(path, "rb") + try: + start = _cribo_sm_find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(_CRIBO_SM_CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = len(data) - len(data) % 4 + pending = data[usable:] + if usable: + yield _cribo_binascii.a2b_base64(data[:usable]) + if pending: + yield _cribo_binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + finally: + handle.close() +class _CriboSmStream(object): + """Byte-at-a-time reader over an iterator of byte chunks.""" + __slots__ = "_chunks", "_buf", "_pos" + + def __init__(self, chunks): + self._chunks = iter(chunks) + self._buf = b"" + self._pos = 0 + + def read_byte(self): + while self._pos >= len(self._buf): + try: + self._buf = next(self._chunks) + except StopIteration: + return -1 + self._pos = 0 + value = self._buf[self._pos] + self._pos += 1 + return value +def _cribo_sm_skip_ws(stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte +def _cribo_sm_read_string(stream, collect): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are handled,\n so string *values* containing text like '\"mappings\":' cannot confuse the\n key scanner.\n """ + buf = bytearray() if collect else None + while True: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON string") + if byte == 34: + return buf.decode("utf-8", "replace") if collect else None + if byte != 92: + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise ValueError("unterminated JSON escape") + if escape == 117: + code = 0 + for _ in range(4): + digit = stream.read_byte() + if digit < 0: + raise ValueError("unterminated unicode escape") + code = code * 16 + int(chr(digit), 16) + if buf is not None: + buf.extend(chr(code).encode("utf-8", "surrogatepass")) + elif buf is not None: + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf.append(table.get(escape, escape)) +def _cribo_sm_skip_value(stream, byte): + """Skip one JSON value; return the first byte after it (or -1).""" + if byte == 34: + _cribo_sm_read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON container") + if byte == 34: + _cribo_sm_read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + while byte >= 0 and byte not in (44, 125, 93): + byte = stream.read_byte() + return byte +def _cribo_sm_read_string_array(stream, byte): + """Read a JSON array of strings/nulls; return (list, byte after array).""" + if byte != 91: + raise ValueError("expected array") + items = [] + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte == 93: + return items, stream.read_byte() + while True: + if byte == 34: + items.append(_cribo_sm_read_string(stream, True)) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + else: + items.append(None) + byte = _cribo_sm_skip_ws(stream, _cribo_sm_skip_value(stream, byte)) + if byte == 93: + return items, stream.read_byte() + if byte != 44: + raise ValueError("malformed array") + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) +def _cribo_sm_decode_vlq(stream, needed, max_needed): + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ + lut = {} + for index in range(64): + lut[_CRIBO_SM_B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 + vlq_value = 0 + vlq_shift = 0 + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = src_idx, src_line + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: + end_segment() + return result + if byte == 59: + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or len(result) == len(needed): + return result + continue + if byte == 44: + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise ValueError("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 +def _cribo_sm_scan(chunks, needed, max_needed): + """Scan the map's top-level object; return (sources, line table).""" + stream = _CriboSmStream(chunks) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte != 123: + raise ValueError("not a JSON object") + sources = [] + table = {} + saw_mappings = False + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + while byte == 34: + key = _cribo_sm_read_string(stream, True) + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if byte != 58: + raise ValueError("malformed object") + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if key == "sources": + sources, byte = _cribo_sm_read_string_array(stream, byte) + elif key == "mappings": + if byte != 34: + raise ValueError("mappings is not a string") + table = _cribo_sm_decode_vlq(stream, needed, max_needed) + saw_mappings = True + if sources: + break + byte = stream.read_byte() + else: + byte = _cribo_sm_skip_value(stream, byte) + byte = _cribo_sm_skip_ws(stream, byte) + if byte == 44: + byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + if not saw_mappings: + raise ValueError("no mappings field") + return sources, table +def _cribo_sm_load(needed_lines): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ + location = _cribo_sm_map_location() + if location is None: + return None + map_path, map_dir = location + needed0 = set(line - 1 for line in needed_lines) + max_needed = max(needed0) + if map_path is None: + chunks = _cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE) + else: + chunks = _cribo_sm_file_chunks(map_path) + sources, table0 = _cribo_sm_scan(chunks, needed0, max_needed) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir +def _cribo_sm_load_json_fallback(needed_lines): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = _cribo_sm_map_location() + if location is None: + return None + map_path, map_dir = location + import json + if map_path is None: + raw = b"".join(_cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE)) + else: + handle = open(map_path, "rb") + try: + raw = handle.read() + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = data.get("sources") or [] + mappings = data.get("mappings") or "" + needed0 = set(line - 1 for line in needed_lines) + stream = _CriboSmStream([mappings.encode("ascii"), b'"']) + table0 = _cribo_sm_decode_vlq(stream, needed0, max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir +def _cribo_sm_collect_needed(exc_value, traceback_obj): + """1-based bundle line numbers referenced by the traceback (and its chain).""" + needed = set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == _CRIBO_SM_BUNDLE: + needed.add(tb.tb_lineno) + tb = tb.tb_next + add(traceback_obj) + exc = exc_value + seen = set() + depth = 0 + while exc is not None and id(exc) not in seen and depth < 16: + seen.add(id(exc)) + depth += 1 + add(getattr(exc, "__traceback__", None)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed +def _cribo_sm_source_line(path, lineno): + """Read a single 1-based line from a file without caching it.""" + try: + handle = open(path, "rb") + except OSError: + return None + try: + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except OSError: + return None + finally: + handle.close() + return None +def _cribo_sm_write_frames(traceback_obj, table, sources, map_dir, write): + '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed by\n a "[Previous line repeated N more times]" marker; source line text is\n cached per (file, line) within one rendering to avoid re-reading files.\n ''' + cache = {} + last = None + repeats = 0 + + def emit(entry): + write(' File "%s", line %d, in %s\n' % entry) + key = entry[0], entry[1] + if key not in cache: + cache[key] = _cribo_sm_source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + if filename == _CRIBO_SM_BUNDLE: + mapped = table.get(lineno) + if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + source = sources[mapped[0]] + if not _cribo_os.path.isabs(source): + source = _cribo_os.path.normpath(_cribo_os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] + entry = filename, lineno, name + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 + emit(entry) + traceback_obj = traceback_obj.tb_next + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) +def _cribo_sm_exception_line(exc_value): + exc_type = type(exc_value) + name = getattr(exc_type, "__qualname__", exc_type.__name__) + module = getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = str(exc_value) + except BaseException: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name +def _cribo_sm_render(exc_value, table, sources, map_dir, write): + """Render the exception (with its cause/context chain) like CPython does.""" + chain = [] + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(chain) < 16: + seen.add(id(exc)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + suppress = getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context + else: + chain.append((exc, None)) + exc = None + ordered = list(reversed(chain)) + for index, (exc, link) in enumerate(ordered): + if index > 0: + if link == "cause": + write("\nThe above exception was the direct cause of the following exception:\n\n") + else: + write("\nDuring handling of the above exception, another exception occurred:\n\n") + tb = getattr(exc, "__traceback__", None) + if tb is not None: + write("Traceback (most recent call last):\n") + _cribo_sm_write_frames(tb, table, sources, map_dir, write) + write(_cribo_sm_exception_line(exc)) +def _cribo_sm_try_render(exc_value, traceback_obj, prefix): + """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in this\n runtime returns False so callers can delegate to the previous hook.\n """ + if _CRIBO_SM_STATE["in_hook"] or exc_value is None: + return False + _CRIBO_SM_STATE["in_hook"] = True + old_limit = None + try: + needed = _cribo_sm_collect_needed(exc_value, traceback_obj) + if not needed: + return False + loaded = None + try: + loaded = _cribo_sm_load(needed) + except BaseException: + try: + loaded = _cribo_sm_load_json_fallback(needed) + except BaseException: + loaded = None + if not loaded: + return False + table, sources, map_dir = loaded + if not table: + return False + try: + old_limit = _cribo_sys.getrecursionlimit() + _cribo_sys.setrecursionlimit(old_limit + 64) + except BaseException: + old_limit = None + parts = [] + if prefix: + parts.append(prefix) + _cribo_sm_render(exc_value, table, sources, map_dir, parts.append) + stderr = _cribo_sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except BaseException: + pass + return True + except BaseException: + return False + finally: + if old_limit is not None: + try: + _cribo_sys.setrecursionlimit(old_limit) + except BaseException: + pass + _CRIBO_SM_STATE["in_hook"] = False +def _cribo_sm_excepthook(exc_type, exc_value, traceback_obj): + if not _cribo_sm_try_render(exc_value, traceback_obj, None): + _cribo_sm_prev_excepthook(exc_type, exc_value, traceback_obj) +def _cribo_sm_threading_hook(args): + thread = getattr(args, "thread", None) + name = getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if not _cribo_sm_try_render(args.exc_value, args.exc_traceback, prefix): + _cribo_sm_prev_threading_hook(args) +def _cribo_sm_unraisablehook(unraisable): + message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except BaseException: + prefix = "%s\n" % message + if not _cribo_sm_try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + _cribo_sm_prev_unraisablehook(unraisable) +_cribo_sys.excepthook = _cribo_sm_excepthook +_cribo_sys.unraisablehook = _cribo_sm_unraisablehook +_cribo_threading.excepthook = _cribo_sm_threading_hook +import sys as _sys +import importlib as _importlib +class _CriboModule(): + + def __init__(_cribo_self, m, p): + _cribo_self._m, _cribo_self._p = m, p + + def __getattr__(_cribo_self, n, *, _getattr=getattr): + f = _cribo_self._p + '.' + n + try: + return _CriboModule(_importlib.import_module(f), f) + except ImportError: + return _getattr(_cribo_self._m, n) + + def __getattribute__(_cribo_self, n, *, _getattr=getattr, _object=object): + return _object.__getattribute__(_cribo_self, n) if n in ('_m', '_p', '__getattr__', '__class__', '__dict__', '__dir__', '__module__', '__qualname__') else _getattr(_object.__getattribute__(_cribo_self, '_m'), n) +class _Cribo(): + + def __getattr__(_cribo_self, n): + m = _sys.modules.get(n) or _importlib.import_module(n) + return _CriboModule(m, n) +_cribo = _Cribo() +_cribo_captured = {} +class _CriboPreservedLoader: + _pristine = {} + + def __init__(self, entry, namespace=None, reload_target=None): + self._entry = entry + self._namespace = namespace + self._reload_target = reload_target + + def create_module(self, spec, *, _getattr=getattr, _globals=globals, _id=id, _setattr=setattr, _type=type, _ModuleType=_cribo.types.ModuleType, _captured=_cribo_captured): + module = None + if self._namespace is not None: + module = self._namespace + elif self._entry[1] is not None: + module = _globals()[self._entry[1]] + if module is not None: + if _getattr(module, '_cribo_machinery_loaded', False) or _getattr(module, '_cribo_failed_life', False) or _getattr(module, '__initialized__', False) and _getattr(module, '_cribo_registered', False): + saved = _type(self)._pristine.get(_id(module)) + fresh = _ModuleType(spec.name) + fresh.__dict__.update(saved if saved is not None else {'__name__': spec.name}) + fresh._cribo_fresh_life = True + return fresh + return module + module = _ModuleType(spec.name) + for export, binding in self._entry[3].items(): + _setattr(module, export, _captured[binding] if binding in _captured else _globals()[binding]) + return module + + def exec_module(self, module, *, _BaseException=BaseException, _dict=dict, _getattr=getattr, _globals=globals, _id=id, _type=type): + init = self._entry[0] + pristine = _type(self)._pristine + key = _id(module) + record_snapshot = not _getattr(module, '_cribo_fresh_life', False) and not _getattr(module, '__initialized__', False) and (self._namespace is not None or self._entry[1] is not None) + if init is None: + if record_snapshot and key not in pristine: + pristine[key] = _dict(module.__dict__) + module._cribo_machinery_loaded = True + return + is_reload = self._reload_target is module + if is_reload: + module.__initialized__ = False + module.__initializing__ = False + state = _dict(module.__dict__) + if record_snapshot and key not in pristine: + pristine[key] = _dict(state) + try: + _globals()[init](module) + module._cribo_machinery_loaded = True + except _BaseException: + if is_reload: + module.__initializing__ = False + else: + module.__dict__.clear() + module.__dict__.update(state) + module.__initializing__ = False + module._cribo_failed_life = True + raise +class _CriboPreservedFinder: + + def __init__(self): + self._targets = {} + self._namespaces = {} + self._registry = {} + self._loader = _CriboPreservedLoader + + def register(self, name, init, namespace, is_package, exports=None): + entry = init, namespace, is_package, exports or {} + self._targets[name] = entry + self._registry[name] = entry + + def bind(self, name, namespace): + self._namespaces[name] = namespace + + def find_spec(self, name, path=None, target=None, *, _getattr=getattr, _globals=globals): + entry = self._targets.get(name) + if entry is None: + return None + parent_name = name.rpartition('.')[0] + if parent_name: + parent = _sys.modules.get(parent_name) + expected = self._namespaces.get(parent_name) + if expected is None: + parent_entry = self._registry.get(parent_name) + if parent_entry is not None and parent_entry[1] is not None: + expected = _globals().get(parent_entry[1]) + if parent is not None and expected is not None and parent is not expected and not _getattr(parent, '_cribo_fresh_life', False) and not _getattr(parent, '_cribo_machinery_loaded', False): + return None + from importlib.machinery import ModuleSpec + return ModuleSpec(name, self._loader(entry, self._namespaces.get(name), target), is_package=entry[2]) +_cribo_finder = _CriboPreservedFinder() +_sys.meta_path.append(_cribo_finder) +_cribo_finder_local = _CriboPreservedFinder() +_cribo_finder_local._namespaces = _cribo_finder._namespaces +_cribo_finder_local._registry = _cribo_finder._registry +for _cribo_index, _cribo_meta_finder in enumerate(_sys.meta_path): + if getattr(_cribo_meta_finder, '__name__', '') == 'PathFinder': + _sys.meta_path.insert(_cribo_index, _cribo_finder_local) + break +else: + _sys.meta_path.insert(0, _cribo_finder_local) +_cribo_finder_local.register('effects', '_cribo_init___cribo_503b17_effects', 'effects', False) +effects = _cribo.types.SimpleNamespace(__name__='effects', __initializing__=False, __initialized__=False) +_cribo_finder.bind('effects', effects) +def _cribo_init___cribo_503b17_effects(_cribo_self, *, _cribo_getattr=getattr, _cribo_base_exception=BaseException, _cribo=_cribo, _sys=_sys): + if _cribo_getattr(_cribo_self, '__initialized__', False): + return _cribo_self + if _cribo_getattr(_cribo_self, '__initializing__', False): + return _cribo_self + _cribo_self.__initializing__ = True + try: + _cribo_self.__package__ = '' + _cribo_self.__doc__ = None + print("effects module loading") + COUNTER = 1 + _cribo_self.COUNTER = COUNTER + + def boost(value): + boosted = value * 2 + COUNTER + return boosted + boost.__module__ = 'effects' + boost.__qualname__ = 'boost' + _cribo_self.boost = boost + _cribo_self.__initialized__ = True + _cribo_self.__initializing__ = False + return _cribo_self + except _cribo_base_exception: + _cribo_self.__initializing__ = False + raise +effects.__init__ = _cribo_init___cribo_503b17_effects +effects = _cribo_init___cribo_503b17_effects(effects) +print("counter:", effects.COUNTER) +print("boosted:", effects.boost(10)) +# sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/execution_results@sourcemap_basic.snap b/crates/cribo/tests/snapshots/execution_results@sourcemap_basic.snap new file mode 100644 index 000000000..cd9792f9c --- /dev/null +++ b/crates/cribo/tests/snapshots/execution_results@sourcemap_basic.snap @@ -0,0 +1,9 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +ExecutionResults { + status: Success, + stdout: "total = 5\nproduct = 20", + stderr: "", +} diff --git a/crates/cribo/tests/snapshots/execution_results@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/execution_results@sourcemap_wrapper.snap new file mode 100644 index 000000000..45ef213d3 --- /dev/null +++ b/crates/cribo/tests/snapshots/execution_results@sourcemap_wrapper.snap @@ -0,0 +1,9 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +ExecutionResults { + status: Success, + stdout: "effects module loading\ncounter: 1\nboosted: 21", + stderr: "", +} diff --git a/crates/cribo/tests/snapshots/requirements@sourcemap_basic.snap b/crates/cribo/tests/snapshots/requirements@sourcemap_basic.snap new file mode 100644 index 000000000..d6b88d9ea --- /dev/null +++ b/crates/cribo/tests/snapshots/requirements@sourcemap_basic.snap @@ -0,0 +1,6 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +packages: [] +count: 0 diff --git a/crates/cribo/tests/snapshots/requirements@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/requirements@sourcemap_wrapper.snap new file mode 100644 index 000000000..984c8102e --- /dev/null +++ b/crates/cribo/tests/snapshots/requirements@sourcemap_wrapper.snap @@ -0,0 +1,6 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +packages: [] +count: 0 diff --git a/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_basic.snap b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_basic.snap new file mode 100644 index 000000000..e44913377 --- /dev/null +++ b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_basic.snap @@ -0,0 +1,10 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +RuffLintResults { + f401: [], + f404: [], + other: [], + total: 0, +} diff --git a/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_wrapper.snap new file mode 100644 index 000000000..7b6c053ee --- /dev/null +++ b/crates/cribo/tests/snapshots/ruff_lint_results@sourcemap_wrapper.snap @@ -0,0 +1,10 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +RuffLintResults { + f401: [], + f404: [], + other: [], + total: 0, +} diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap new file mode 100644 index 000000000..368595ddd --- /dev/null +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -0,0 +1,16 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py +--- +bundle:625 `def add(a, b):` -> calculator.py:1 +bundle:626 `result = a + b` -> calculator.py:2 +bundle:627 `return result` -> calculator.py:3 +bundle:629 `def multiply(a, b):` -> calculator.py:6 +bundle:630 `result = a * b` -> calculator.py:7 +bundle:631 `return result` -> calculator.py:8 +bundle:636 `def describe(name, value):` -> utils.py:1 +bundle:637 `return f"{name} = {value}"` -> utils.py:2 +bundle:643 `total = add(2, 3)` -> main.py:4 +bundle:644 `product = multiply(total, 4)` -> main.py:5 +bundle:645 `print(describe("total", total))` -> main.py:6 +bundle:646 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap new file mode 100644 index 000000000..aa025a0b2 --- /dev/null +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -0,0 +1,11 @@ +--- +source: crates/cribo/tests/test_bundling_snapshots.rs +input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py +--- +bundle:634 `print("effects module loading")` -> effects.py:1 +bundle:635 `COUNTER = 1` -> effects.py:3 +bundle:638 `def boost(value):` -> effects.py:6 +bundle:639 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:640 `return boosted` -> effects.py:8 +bundle:652 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:653 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_bundling_snapshots.rs b/crates/cribo/tests/test_bundling_snapshots.rs index 8ec359920..b75900f88 100644 --- a/crates/cribo/tests/test_bundling_snapshots.rs +++ b/crates/cribo/tests/test_bundling_snapshots.rs @@ -144,6 +144,43 @@ struct RequirementsData { count: usize, } +/// Render a Source Map v3 JSON as a deterministic, path-free text dump. +/// +/// One line per mapping: `bundle: `` -> :` with 1-based line numbers. Basenames keep the snapshot +/// free of machine-specific absolute paths. +fn render_source_map_dump(map_json: &str, bundled_code: &str) -> String { + use std::fmt::Write; + + let map = oxc_sourcemap::SourceMap::from_json_string(map_json) + .expect("emitted source map must be valid Source Map v3 JSON"); + let bundle_lines: Vec<&str> = bundled_code.lines().collect(); + let mut dump = String::new(); + for token in map.get_tokens() { + let source = token + .get_source_id() + .and_then(|id| map.get_source(id)) + .unwrap_or(""); + let source_name = Path::new(source).file_name().map_or_else( + || source.to_owned(), + |name| name.to_string_lossy().into_owned(), + ); + let generated_text = bundle_lines + .get(token.get_dst_line() as usize) + .map(|line| line.trim()) + .unwrap_or_default(); + let _ = writeln!( + dump, + "bundle:{:<4} `{}` -> {}:{}", + token.get_dst_line() + 1, + generated_text, + source_name, + token.get_src_line() + 1, + ); + } + dump +} + /// Run ruff linting on bundled code to cross-validate import handling fn run_ruff_lint_on_bundle(bundled_code: &str) -> RuffLintResults { // Create settings for multiple import-related rules with both F401 and F404 enabled @@ -230,6 +267,9 @@ fn test_bundling_fixtures() { // Check fixture type based on prefix let expects_bundling_failure = fixture_name.starts_with("xfail_"); let expects_python_failure = fixture_name.starts_with("pyfail_"); + // sourcemap_ fixtures opt into --sourcemap=linked and get an extra + // normalized mapping snapshot (see render_source_map_dump) + let enables_sourcemap = fixture_name.starts_with("sourcemap_"); // Get Python executable once for the entire test let python_cmd = common::get_python_executable(); @@ -332,6 +372,9 @@ fn test_bundling_fixtures() { if fake_venv.is_some() { cribo_args.push("--bundle-third-party"); } + if enables_sourcemap { + cribo_args.push("--sourcemap=linked"); + } // A fixture-level cribo.toml supplies configuration (e.g. module-map entries) let fixture_config = fixture_dir.join("cribo.toml"); let fixture_config_str = fixture_config.to_str().map(ToOwned::to_owned); @@ -376,6 +419,15 @@ fn test_bundling_fixtures() { // Read the bundled code let bundled_code = fs::read_to_string(&bundle_path).unwrap(); + // Sourcemap fixtures must produce a sibling map; snapshot it in a + // normalized, path-free form so mapping regressions are visible. + let source_map_dump = enables_sourcemap.then(|| { + let map_path = temp_dir.path().join("bundled.py.map"); + let map_json = fs::read_to_string(&map_path) + .expect("sourcemap fixture must produce bundled.py.map"); + render_source_map_dump(&map_json, &bundled_code) + }); + // Read and parse the requirements.txt if it was generated let requirements_path = temp_dir.path().join("requirements.txt"); let requirements_data = if requirements_path.exists() { @@ -610,6 +662,11 @@ fn test_bundling_fixtures() { // Snapshot ruff linting results insta::assert_debug_snapshot!("ruff_lint_results", ruff_results); + // Snapshot the normalized source map for sourcemap_ fixtures + if let Some(dump) = &source_map_dump { + insta::assert_snapshot!("source_map", dump); + } + // Snapshot requirements data as YAML insta::assert_yaml_snapshot!("requirements", requirements_data); }); diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs new file mode 100644 index 000000000..496c0fe4d --- /dev/null +++ b/crates/cribo/tests/test_source_maps.rs @@ -0,0 +1,737 @@ +//! Integration tests for `--sourcemap` CLI delivery modes. +//! +//! Each test drives the cribo binary end-to-end and inspects the emitted +//! bundle and/or `.map` file. See `docs/source-maps.md` for the design. + +mod common; + +use std::{fs, path::Path, process::Command}; + +use tempfile::TempDir; + +/// Marker prefix of an inline source map comment. +const INLINE_MARKER: &str = "# sourceMappingURL=data:application/json;base64,"; + +/// Create a two-module fixture project and return its directory. +fn fixture_project() -> TempDir { + let dir = TempDir::new().expect("create temp dir"); + fs::write( + dir.path().join("main.py"), + "from helper import greet\n\nprint(greet(\"world\"))\n", + ) + .expect("write main.py"); + fs::write( + dir.path().join("helper.py"), + "def greet(name):\n message = f\"hello {name}\"\n return message\n", + ) + .expect("write helper.py"); + dir +} + +/// Run the cribo binary with `args`, returning (status success, stdout, stderr). +fn run_cribo(args: &[&str]) -> (bool, String, String) { + let output = Command::new(env!("CARGO_BIN_EXE_cribo")) + .args(args) + .output() + .expect("run cribo binary"); + ( + output.status.success(), + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) +} + +/// Parse a source map JSON string and assert it is valid Source Map v3. +fn parse_map(json: &str) -> oxc_sourcemap::SourceMap<'_> { + oxc_sourcemap::SourceMap::from_json_string(json).expect("valid Source Map v3 JSON") +} + +fn entry_arg(dir: &TempDir) -> String { + dir.path().join("main.py").to_string_lossy().into_owned() +} + +fn assert_map_covers_helper(map_json: &str, bundle_file: &str) { + let map = parse_map(map_json); + assert_eq!(map.get_file(), Some(bundle_file)); + assert!( + map.get_sources() + .any(|source| source.ends_with("helper.py")), + "map sources must include helper.py: {:?}", + map.get_sources().collect::>() + ); + assert!(map.get_tokens().count() > 0, "map must contain mappings"); +} + +#[test] +fn linked_mode_writes_map_and_comment() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle + .trim_end() + .ends_with("# sourceMappingURL=bundle.py.map"), + "linked mode must append the sourceMappingURL comment" + ); + + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map file"); + assert_map_covers_helper(&map_json, "bundle.py"); + // Linked mode embeds sourcesContent by default. + assert!(map_json.contains("sourcesContent")); +} + +#[test] +fn bare_sourcemap_flag_defaults_to_linked() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!(dir.path().join("bundle.py.map").exists()); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!(bundle.contains("# sourceMappingURL=bundle.py.map")); +} + +#[test] +fn external_mode_writes_map_without_comment() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=external", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + !bundle + .lines() + .any(|line| line.starts_with("# sourceMappingURL=")), + "external mode must not reference the map from the bundle" + ); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map file"); + assert_map_covers_helper(&map_json, "bundle.py"); +} + +#[test] +fn inline_mode_embeds_map_and_writes_no_file() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!( + !dir.path().join("bundle.py.map").exists(), + "inline mode must not write a map file" + ); + + let bundle = fs::read_to_string(&out).expect("read bundle"); + let map_json = decode_inline_map(&bundle); + assert_map_covers_helper(&map_json, "bundle.py"); + // Inline mode omits sourcesContent by default. + assert!(!map_json.contains("sourcesContent")); +} + +#[test] +fn stdout_with_bare_sourcemap_selects_inline() { + let dir = fixture_project(); + let (ok, stdout, stderr) = run_cribo(&["--entry", &entry_arg(&dir), "--stdout", "--sourcemap"]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = decode_inline_map(&stdout); + assert_map_covers_helper(&map_json, ""); +} + +#[test] +fn stdout_with_linked_sourcemap_errors() { + let dir = fixture_project(); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--stdout", + "--sourcemap=linked", + ]); + assert!(!ok, "linked + stdout must be rejected"); + assert!( + stderr.contains("--sourcemap=inline"), + "error must suggest inline mode: {stderr}" + ); +} + +#[test] +fn stdout_with_external_sourcemap_errors() { + let dir = fixture_project(); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--stdout", + "--sourcemap=external", + ]); + assert!(!ok, "external + stdout must be rejected"); + assert!(stderr.contains("--sourcemap=inline")); +} + +#[test] +fn no_sourcemap_by_default() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!(!dir.path().join("bundle.py.map").exists()); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!(!bundle.contains("sourceMappingURL")); +} + +#[test] +fn config_file_sourcemap_key_is_honored() { + let dir = fixture_project(); + fs::write(dir.path().join("cribo.toml"), "sourcemap = \"external\"\n").expect("write config"); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--config", + &dir.path().join("cribo.toml").to_string_lossy(), + ]); + assert!(ok, "bundling must succeed: {stderr}"); + assert!( + dir.path().join("bundle.py.map").exists(), + "config-file sourcemap key must enable map emission" + ); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + !bundle + .lines() + .any(|line| line.starts_with("# sourceMappingURL=")), + "external mode: no comment" + ); +} + +/// Extract and decode the inline source map data URL from bundle text. +fn decode_inline_map(bundle: &str) -> String { + let marker_pos = bundle + .rfind(INLINE_MARKER) + .expect("inline source map comment present"); + let payload = bundle[marker_pos + INLINE_MARKER.len()..].trim_end(); + let bytes = base64_simd::STANDARD + .decode_to_vec(payload.as_bytes()) + .expect("valid base64 payload"); + String::from_utf8(bytes).expect("valid UTF-8 source map") +} + +/// The map file sits next to the bundle even when the output path is nested. +#[test] +fn linked_map_lands_next_to_nested_output() { + let dir = fixture_project(); + let nested = dir.path().join("dist").join("app.py"); + fs::create_dir_all(nested.parent().expect("parent")).expect("mkdir dist"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &nested.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_path = dir.path().join("dist").join("app.py.map"); + assert!(map_path.exists(), "map must sit next to the nested output"); + let bundle = fs::read_to_string(&nested).expect("read bundle"); + assert!(bundle.contains("# sourceMappingURL=app.py.map")); + + // Source paths must be relative to the map's directory. + let map_json = fs::read_to_string(&map_path).expect("read map"); + let map = parse_map(&map_json); + assert!( + map.get_sources() + .all(|source| Path::new(source).is_relative()), + "sources must be relative paths: {:?}", + map.get_sources().collect::>() + ); +} + +#[test] +fn sources_content_can_be_forced_on_for_inline() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + "--sources-content=true", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + let map_json = decode_inline_map(&bundle); + assert!( + map_json.contains("sourcesContent"), + "--sources-content=true must force embedding for inline maps" + ); + let map = parse_map(&map_json); + assert!( + map.get_source_contents() + .flatten() + .any(|content| content.contains("def greet")), + "embedded content must carry the original helper source" + ); +} + +#[test] +fn sources_content_can_be_forced_off_for_linked() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + "--sources-content=false", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + assert!( + !map_json.contains("sourcesContent"), + "--sources-content=false must strip embedding for linked maps" + ); +} + +#[test] +fn config_file_sources_content_key_is_honored() { + let dir = fixture_project(); + fs::write( + dir.path().join("cribo.toml"), + "sourcemap = \"external\"\nsources-content = false\n", + ) + .expect("write config"); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--config", + &dir.path().join("cribo.toml").to_string_lossy(), + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + assert!(!map_json.contains("sourcesContent")); +} + +// --------------------------------------------------------------------------- +// Runtime traceback remapping (injected prologue) +// --------------------------------------------------------------------------- + +/// Create a fixture whose entry crashes two calls deep inside helper.py. +fn crash_project() -> TempDir { + let dir = TempDir::new().expect("create temp dir"); + fs::write( + dir.path().join("main.py"), + "from helper import boom\n\nboom()\n", + ) + .expect("write main.py"); + fs::write( + dir.path().join("helper.py"), + "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"kaboom\")\n", + ) + .expect("write helper.py"); + dir +} + +/// Bundle the crash project with the given sourcemap argument; return bundle path. +fn bundle_crash_project(dir: &TempDir, sourcemap_arg: &str) -> std::path::PathBuf { + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(dir), + "--output", + &out.to_string_lossy(), + sourcemap_arg, + ]); + assert!(ok, "bundling must succeed: {stderr}"); + out +} + +/// Run a Python file; returns (status success, stdout, stderr). +fn run_python(bundle: &Path, envs: &[(&str, &str)]) -> (bool, String, String) { + let mut command = Command::new(common::get_python_executable()); + command.arg(bundle); + for (key, value) in envs { + command.env(key, value); + } + let output = command.output().expect("run python"); + ( + output.status.success(), + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) +} + +/// Assert stderr shows the remapped traceback pointing at original files. +fn assert_remapped(stderr: &str) { + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "traceback must point at helper.py:5: {stderr}" + ); + assert!( + stderr.contains("raise ValueError(\"kaboom\")"), + "traceback must show the original source line: {stderr}" + ); + assert!( + stderr.contains("main.py\", line 3, in "), + "traceback must point at main.py:3: {stderr}" + ); + assert!( + !stderr.contains("bundle.py\", line"), + "no frame should remain on bundle coordinates: {stderr}" + ); +} + +/// Assert stderr is one single standard (non-remapped) traceback with no +/// runtime noise. +fn assert_standard_traceback(stderr: &str) { + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "exactly one traceback expected: {stderr}" + ); + assert!( + stderr.contains("bundle.py\", line"), + "standard traceback must show bundle coordinates: {stderr}" + ); + assert!( + !stderr.contains("helper.py\", line 5"), + "no remapping must happen: {stderr}" + ); +} + +#[test] +fn runtime_remaps_linked_crash() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok, "the crashing bundle must exit non-zero"); + assert_remapped(&stderr); +} + +#[test] +fn runtime_disabled_when_linked_map_missing() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + fs::remove_file(dir.path().join("bundle.py.map")).expect("delete map"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn runtime_remaps_inline_crash() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=inline"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_remapped(&stderr); +} + +#[test] +fn runtime_kill_switch_disables_remapping() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=inline"); + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "0")]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn runtime_external_mode_is_env_gated() { + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=external"); + + // Without the env var the runtime stays dormant. + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); + + // CRIBO_SOURCE_MAPS=1 activates it against the sibling map. + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "1")]); + assert!(!ok); + assert_remapped(&stderr); + + // The env var can also point directly at a relocated map file. + let moved = dir.path().join("elsewhere.map"); + fs::rename(dir.path().join("bundle.py.map"), &moved).expect("move map"); + let (ok, _, stderr) = run_python( + &bundle, + &[("CRIBO_SOURCE_MAPS", moved.to_string_lossy().as_ref())], + ); + assert!(!ok); + assert_remapped(&stderr); + + // With the map moved away, =1 finds nothing and stays silent. + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "1")]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +/// Drive the pure-Python unit tests for the runtime internals (VLQ machine, +/// JSON scanner, backward EOF scan, base64 chunk alignment, json fallback). +#[test] +fn python_runtime_unit_tests() { + use cow_utils::CowUtils as _; + + const TEMPLATE: &str = include_str!("../src/python/sourcemap_runtime.py"); + let dir = TempDir::new().expect("create temp dir"); + let runtime_path = dir.path().join("runtime.py"); + fs::write( + &runtime_path, + TEMPLATE + .cow_replace("__CRIBO_SOURCEMAP_MODE__", "external") + .as_ref(), + ) + .expect("write substituted runtime"); + + let script = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("python") + .join("test_sourcemap_runtime.py"); + let output = Command::new(common::get_python_executable()) + .arg(&script) + .arg(&runtime_path) + .output() + .expect("run python unit tests"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "python runtime unit tests failed:\n{stdout}\n{stderr}" + ); + assert!(stdout.contains("ALL 12 RUNTIME TESTS PASSED"), "{stdout}"); +} + +// --------------------------------------------------------------------------- +// Full hook coverage, duress conditions, and laziness +// --------------------------------------------------------------------------- + +/// Create a project from (file name, content) pairs and return its directory. +fn make_project(files: &[(&str, &str)]) -> TempDir { + let dir = TempDir::new().expect("create temp dir"); + for (name, content) in files { + fs::write(dir.path().join(name), content).expect("write fixture file"); + } + dir +} + +#[test] +fn runtime_remaps_thread_crash() { + let dir = make_project(&[ + ( + "main.py", + "import threading\nfrom helper import boom\n\nworker = \ + threading.Thread(target=boom)\nworker.start()\nworker.join()\nprint(\"done\")\n", + ), + ( + "helper.py", + "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"thread kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + // An uncaught exception in a non-main thread does not fail the process. + let (ok, stdout, stderr) = run_python(&bundle, &[]); + assert!(ok, "main thread must finish normally: {stderr}"); + assert!(stdout.contains("done")); + assert!( + stderr.contains("Exception in thread"), + "threading hook must announce the thread: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "thread traceback must be remapped: {stderr}" + ); + assert!(stderr.contains("thread kaboom")); +} + +#[test] +fn runtime_remaps_unraisable_error() { + let dir = make_project(&[ + ( + "main.py", + "from helper import make\n\nobj = make()\ndel obj\nprint(\"done\")\n", + ), + ( + "helper.py", + "class Cursed:\n def __del__(self):\n raise RuntimeError(\"del \ + failed\")\n\ndef make():\n return Cursed()\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, stdout, stderr) = run_python(&bundle, &[]); + assert!(ok, "unraisable errors must not fail the process: {stderr}"); + assert!(stdout.contains("done")); + assert!( + stderr.contains("Exception ignored in"), + "unraisable hook must keep the standard preamble: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 3, in __del__"), + "unraisable traceback must be remapped: {stderr}" + ); +} + +#[test] +fn runtime_survives_recursion_error() { + let dir = make_project(&[ + ("main.py", "from helper import spiral\n\nspiral()\n"), + ("helper.py", "def spiral():\n spiral()\n"), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("RecursionError"), "{stderr}"); + assert!( + stderr.contains("helper.py\", line 2, in spiral"), + "recursion frames must be remapped: {stderr}" + ); + assert!( + stderr.contains("[Previous line repeated"), + "repeated recursion frames must be collapsed: {stderr}" + ); + // Collapsing keeps the output small even for ~1000 recorded frames. + assert!( + stderr.lines().count() < 60, + "collapsed traceback expected, got {} lines", + stderr.lines().count() + ); +} + +#[cfg(unix)] +#[test] +fn runtime_survives_memory_pressure() { + let dir = make_project(&[ + ( + "main.py", + "import resource\nfrom helper import hoard\n\n_soft, hard = \ + resource.getrlimit(resource.RLIMIT_AS)\n\ + resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, hard))\nhoard()\n", + ), + ( + "helper.py", + "def hoard():\n blocks = []\n try:\n while True:\n \ + blocks.append(bytearray(16 * 1024 * 1024))\n except MemoryError:\n \ + blocks.clear()\n raise MemoryError(\"exhausted\") from None\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("MemoryError"), "{stderr}"); + // The runtime must never produce a secondary error, whichever path it took. + assert!( + !stderr.contains("Error in sys.excepthook"), + "the hook must not raise: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 8, in hoard"), + "MemoryError under an address-space limit should still remap: {stderr}" + ); +} + +#[cfg(unix)] +#[test] +fn runtime_falls_back_cleanly_on_fd_exhaustion() { + let dir = make_project(&[ + ( + "main.py", + "from helper import consume_fds_and_boom\n\nconsume_fds_and_boom()\n", + ), + ( + "helper.py", + "import resource\n\ndef consume_fds_and_boom():\n _soft, hard = \ + resource.getrlimit(resource.RLIMIT_NOFILE)\n \ + resource.setrlimit(resource.RLIMIT_NOFILE, (16, hard))\n holders = []\n \ + try:\n while True:\n holders.append(open(\"/dev/null\", \ + \"rb\"))\n except OSError:\n pass\n raise ValueError(\"fd exhausted \ + kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + // The map cannot be opened, so the runtime must fall back to the default + // traceback without any secondary noise. + assert!(stderr.contains("fd exhausted kaboom"), "{stderr}"); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "exactly one traceback expected: {stderr}" + ); + assert!( + !stderr.contains("Error in sys.excepthook"), + "the hook must not raise: {stderr}" + ); + assert!( + !stderr.contains("helper.py\", line"), + "with the map unreadable, frames stay on bundle coordinates: {stderr}" + ); +} + +#[test] +fn runtime_is_lazy_on_happy_path() { + let dir = fixture_project(); // non-throwing project + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + // Replace the map with something that would fail loudly on ANY access: + // garbage content and, on Unix, no read permission at all. + let map_path = dir.path().join("bundle.py.map"); + fs::write(&map_path, "NOT JSON {{{").expect("overwrite map"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(&map_path, fs::Permissions::from_mode(0o000)) + .expect("make map unreadable"); + } + + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "happy-path run must succeed: {stderr}"); + assert!(stdout.contains("hello world")); + assert!( + stderr.is_empty(), + "no map access may happen on the happy path: {stderr}" + ); +} diff --git a/docs/source-maps.md b/docs/source-maps.md new file mode 100644 index 000000000..a83c5d05b --- /dev/null +++ b/docs/source-maps.md @@ -0,0 +1,230 @@ +# Source Map v3 Support + +Status: implemented. This document records the agreed design plus the +implementation notes at the end. + +## Problem Statement + +Cribo produces a single bundled `.py` file; when that bundle raises an exception, +tracebacks point at bundle line numbers instead of the original source files, making +debugging painful. This feature adds opt-in Source Map v3 generation (the +language-agnostic JS-ecosystem format) plus an injected Python runtime that remaps +tracebacks to original sources at run time, analogous to `node --enable-source-maps`. + +## Requirements + +1. **Granularity:** statement/line-level mappings (column 0) — sufficient for Python's + line-oriented tracebacks. +2. **Delivery:** esbuild-style `--sourcemap[=linked|inline|external]` (bare flag = + `linked`). + - `linked`: write `.map` next to the output and append a + `# sourceMappingURL=.map` comment as the last line. + - `inline`: append a `# sourceMappingURL=data:application/json;base64,...` comment. + - `external`: write the `.map` file with no comment. +3. **Runtime traceback injection** is bundled into the output whenever `--sourcemap` + is used; activation depends on the mode: + - `inline` → active by default (`CRIBO_SOURCE_MAPS=0` acts as a kill-switch). + - `external` → gated on the env var `CRIBO_SOURCE_MAPS=1` (a value that is a path + is treated as the map file location). + - `linked` → active iff the sibling `.map` file exists at run time; silently + disabled otherwise. +4. **Crate:** `oxc_sourcemap` v8.x (pinned exact version), BSD-3-Clause, maintained at + `oxc-project/oxc-sourcemap`, used by rolldown/rspack. Its dependencies + (`rustc-hash 2`, `serde`, `serde_json`, `base64-simd`, `json-escape-simd`) align + with the workspace; MSRV 1.95 is below the repo toolchain 1.97.1. +5. **`sourcesContent`:** the default depends on the mode — omitted for `inline`, + included for `linked`/`external`; forcible override via `--sources-content=true|false` + (and the matching config key). +6. **Hook scope:** full set — `sys.excepthook`, `threading.excepthook`, + `sys.unraisablehook`; chain to previously installed hooks; never crash (fail open + to default behavior). +7. **Laziness:** zero-to-negligible happy-path cost. The prologue does NO I/O, no + parsing, no decoding at startup — it only defines functions, stores `__file__` and + the bundle-time mode constant, and installs the 3 hooks. Activation gating itself + (map existence check, env var read, data-URL location) is deferred to the first + exception. +8. **Duress tolerance:** the decoder must work under resource-constrained conditions + (`MemoryError`, `RecursionError`, FD exhaustion) — streaming, constant-memory + implementation; performance is secondary on the error path. + +## Design + +### Mapping extraction (Rust side) + +Cribo emits the bundle by unparsing the merged AST statement-by-statement with ruff's +`Generator` (`orchestrator.rs::bundle_to_string` → +`code_generator/python_codegen.rs::generate_statement`). Ruff codegen emits no +position info, so mappings cannot be captured during emission. + +Instead, cribo re-parses the final bundle text once with ruff's parser and performs a +defensive parallel statement-only walk against the bundled AST (which carries node +provenance via node indices and original `TextRange`s). Each aligned statement with +provenance yields one mapping: generated line → (original file, original line). On +structural divergence (e.g., the class-pattern rewriter's string patching), the walk +logs at debug level and skips the subtree rather than failing. The re-parse doubles +as a bundle-validity check. + +Provenance groundwork already exists: + +- `ast_indexer.rs` gives each module a 1,000,000-wide node-index range + (`node_index / MODULE_INDEX_RANGE` = module ordinal); statements copied from module + ASTs retain their original `TextRange`. +- Synthesized nodes get indices from `transformation_context.rs` and are + distinguishable (no original range) — they produce no mapping. + +Key components: + +- `crates/cribo/src/source_map.rs`: mapping record builder wrapping + `oxc_sourcemap::SourceMapBuilder`, provenance resolution, and the parallel walk. +- Each parsed module's source text plus a line-offset index is retained (keyed by + `ModuleId`) through `bundle_core` — needed both for offset→line conversion and for + `sourcesContent`. +- `sources` paths are recorded relative to the map location (esbuild convention), + with an empty `sourceRoot`. +- `--stdout` interplay: bare `--sourcemap` with `--stdout` defaults to `inline`; + explicit `linked`/`external` with `--stdout` is an error suggesting `inline`. +- Config-file (`cribo.toml`) equivalents: `sourcemap = "linked" | "inline" | "external"`, + `sources-content = true | false`. + +### Runtime prologue (Python side) — lazy, streaming, duress-tolerant + +The template lives in `crates/cribo/src/python/` and is embedded via `include_str!`, +injected as a prologue when source maps are enabled, with the delivery mode baked in +as a constant. + +**Happy path:** define functions + 3 hook assignments (capturing previous hooks for +chaining). Nothing else — no file I/O, no parsing, no decoding. + +**Exception path pipeline:** + +1. **Collect needed lines.** Walk the `tb_next` chain (plus `__cause__`/`__context__`) + collecting `tb_lineno` for frames whose `co_filename` matches the bundle path. + Result: a small set of ints and `max_needed` for early exit. +2. **Locate the mappings bytes without loading the map.** + - `linked`/`external`: stream the `.map` file in fixed 8 KiB chunks. + - `inline`: open the bundle file, `seek()` to EOF, scan backward in chunks for the + last newline to find the `# sourceMappingURL=data:...;base64,` line (the bundle + body is never read), then decode the base64 payload incrementally with + `binascii.a2b_base64` on 4-byte-aligned chunk boundaries. +3. **Targeted streaming JSON field extraction.** The map is NOT parsed with + `json.loads`. A minimal JSON string-lexer scans the chunk stream for the top-level + `"sources"` key (a small array, parsed into a list) and the `"mappings"` key + (consumed by the VLQ state machine directly from the stream, never held whole in + memory). The lexer handles escaped quotes so `sourcesContent` values containing + fake `"mappings":` keys cannot fool it. +4. **Streaming VLQ state machine, constant memory.** State is six integers: + `gen_line`, running deltas `src_idx` and `src_line`, VLQ accumulators `vlq_value` + and `vlq_shift`, and a `field` counter. `;` increments `gen_line`, `,` ends a + segment; `(gen_line → (src_idx, src_line))` is recorded only for lines in the + needed set (first segment per line); the machine early-exits once + `gen_line > max_needed`. Total heap: chunk buffer + six ints + k result entries. +5. **Re-render, best-effort in layers.** The file:line remap is always attempted; + the original source-line *text* is decoration in a separate `try` — stream-read + just the needed line from the original file on disk (iterate, never slurp; NO + `linecache` — it caches whole files), or a second targeted pass over + `sourcesContent`; skipped silently on failure. + +**Failure containment:** + +- Fallback ladder, never mask the real error: (1) streaming targeted path → (2) one + attempt at plain `json.loads` → (3) delegate to the captured previous hook with the + original exception. The entire hook body is wrapped catching `BaseException` raised + by our own code. +- No imports inside the hook: `sys`, `os`, etc. are bound at prologue time. The + renderer formats frames directly from `tb_frame.f_code` attributes mirroring + CPython's format (self-contained; no dependency on `traceback.TracebackException`). +- Iterative code only (no recursion); the recursion limit is bumped by a small margin + (`sys.setrecursionlimit(cur + 64)`) inside a `try` before rendering and restored + after. +- Re-entrancy guard: a module-level flag prevents the hook recursing into itself. + +**Documented caveats:** + +- Under hard OOM where the interpreter cannot allocate at all, no pure-Python hook + can run; the target is constrained-but-alive conditions with guaranteed + non-interference at the floor. +- User code calling `traceback.format_exc()` (or otherwise formatting tracebacks + itself) is not remapped — only the installed hooks re-render. + +## Task Breakdown + +- **Task 0:** this document, cross-referenced from `docs/static-bundling.md`. +- **Task 1:** `oxc_sourcemap` workspace dependency (pinned) + `source_map.rs` with a + `SourceMapGenerator` accepting `(generated_line, source_file, original_line)` + records and optional per-source content, serializing valid Source Map v3 JSON. + Unit tests: mapping order, VLQ round-trip via the crate's consumer API, + `sourcesContent` on/off, empty map. +- **Task 2:** retain module source text + line-offset index keyed by `ModuleId`; + provenance resolver `node_index` → module ordinal → original file path and + `TextRange.start()` → original line; `None` for synthesized nodes. Unit tests with + multi-module inputs including a synthesized-node case. +- **Task 3:** mapping extraction via re-parse + parallel statement walk (including + statements nested in function/class bodies and wrapper-module init functions), + integrating Tasks 1+2 into a complete `SourceMap` per bundle. Tests assert selected + known mappings for inlined-module, wrapper-module, and class-pattern bundles. +- **Task 4:** CLI `--sourcemap[=linked|inline|external]` (clap `ValueEnum`, bare = + `linked`) + `cribo.toml` key; the three delivery modes including the + `# sourceMappingURL=` trailer; `--stdout` interplay. Integration tests per mode. +- **Task 5:** `sourcesContent` mode-dependent default with `--sources-content` + override (+ config key). Tests cover the default matrix and both overrides. +- **Task 6:** Python runtime prologue per the design above, with a pytest suite for + the unit-testable pieces (VLQ state machine against maps generated by + `oxc_sourcemap`, backward EOF scan, base64 chunk alignment, JSON scanner against + adversarial `sourcesContent`) and an integration test asserting a remapped + traceback on stderr. +- **Task 7:** `threading.excepthook` + `sys.unraisablehook`; duress tests + (`RecursionError` at depth, `MemoryError`, FD exhaustion via + `resource.setrlimit(RLIMIT_NOFILE, ...)`), activation-matrix tests, and a laziness + test asserting a non-throwing run performs no map access. +- **Task 8:** snapshot-framework integration (fixtures opt into source maps, + snapshotting remapped-traceback output), at least two fixtures, README/CLI docs, + and caveat documentation. + + +## Implementation Notes + +Decisions made (or refined) during implementation: + +- **Prologue injection is AST-level, not text-level.** The runtime template + (`crates/cribo/src/python/sourcemap_runtime.py`, embedded via `include_str!`) + is parsed with ruff and its statements are spliced into the bundled AST after + any leading `from __future__` imports, *before* code generation. The extraction + walk therefore stays structurally aligned automatically (prologue statements + carry no provenance and simply produce no mappings), with no line-offset + bookkeeping. +- **Source line text comes from disk only.** The runtime resolves relative + source paths against the map's directory and stream-reads the single needed + line. A second streaming pass over `sourcesContent` for line text was + deliberately skipped to bound runtime complexity — `sourcesContent` is still + embedded per the configured policy for external tooling (IDEs, error + trackers). When original files are absent, tracebacks still remap `file:line` + and simply omit the source-line text. +- **Repeated frames are collapsed like CPython** (at most 3 identical + consecutive frames, then `[Previous line repeated N more times]`), with a + per-render line-text cache; a `RecursionError` traceback stays small and does + not trigger thousands of file reads. +- **`threading` is imported at bundle startup** (aliased) so + `threading.excepthook` can be installed; this is the only non-trivial startup + cost and is negligible in practice. Map location checks, environment reads, + file access, and decoding all remain deferred to the first exception. +- **Snapshot integration:** fixtures under `crates/cribo/tests/fixtures/` whose + name starts with `sourcemap_` are bundled with `--sourcemap=linked` and gain a + `source_map@.snap` snapshot: a normalized, path-free dump of every + mapping (`bundle: `` -> :`). Remapped + *traceback output* is asserted exactly (not snapshotted) in + `crates/cribo/tests/test_source_maps.rs`, which also covers the activation + matrix, thread/unraisable hooks, and the duress suite (RecursionError, + MemoryError under `RLIMIT_AS`, FD exhaustion under `RLIMIT_NOFILE`, and + happy-path laziness with an unreadable map). + +Known limitations (documented in the README as well): + +- User code that formats tracebacks itself (`traceback.format_exc()`, + `traceback.print_exc()`, custom formatters) is not remapped; only the + installed hooks re-render. +- Under a hard out-of-memory condition where the interpreter cannot allocate at + all, no pure-Python hook can run; the guarantee is non-interference (the + default traceback still prints) rather than remapping. +- Mappings are statement/line-level with column 0 by design; Python tracebacks + are line-oriented, so finer columns would add cost without changing the + rendered output. diff --git a/docs/static-bundling.md b/docs/static-bundling.md index eabf81312..0e44faf5c 100644 --- a/docs/static-bundling.md +++ b/docs/static-bundling.md @@ -563,6 +563,10 @@ The simple renaming approach is tempting for its simplicity, but Python's dynami ## Source Map Support +> **Note:** the agreed, implemented design for source maps lives in +> [`docs/source-maps.md`](./source-maps.md). The sections below are the original +> exploratory sketch and are kept for historical context. + ### Overview Source maps enable debugging of bundled code by mapping locations in the bundle back to original source files. Cribo should adopt the JavaScript Source Map v3 specification for Python bundles. From 7992159660b7dc05a77353a233d5b9ff2471a848 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 16:53:45 +0200 Subject: [PATCH 02/17] fix: address PR review feedback on source map runtime and tests - preserve preinstalled custom sys/threading/unraisable hooks after a successful remap (notified when they differ from the interpreter default) - restructure the runtime into a class with instance-bound state so bundled user globals cannot shadow the hooks; re-entrancy guard is thread-local - keep SystemExit from worker threads silent like the default threading hook - defer ExceptionGroup chains to the previous hook instead of losing nested tracebacks - honor CRIBO_SOURCE_MAPS= in every mode (only way to remap python - stdin bundles); inline mode deactivates gracefully for - strip the template docstring at injection so __doc__ is unchanged - map elif headers to their original lines via the condition's provenance - add CRIBO_SOURCEMAP / CRIBO_SOURCES_CONTENT environment overrides - write linked/external maps only after the bundle write succeeds - relative_path bails out on non-invertible .. base components - tests: clear CRIBO_SOURCE_MAPS from child env, CLI-over-config precedence, env-var config, thread SystemExit silence, sitecustomize hook notification, RLIMIT_AS clamped to the inherited hard limit, laziness test claims aligned with what it proves, harness discovers python tests via globals(), fixture helpers deduplicated, doc comments added Addresses review comments on #570 --- README.md | 19 +- crates/cribo/src/config.rs | 29 + crates/cribo/src/orchestrator.rs | 17 +- crates/cribo/src/python/sourcemap_runtime.py | 1159 +++++++++-------- crates/cribo/src/source_map.rs | 41 +- .../tests/python/test_sourcemap_runtime.py | 92 +- .../bundled_code@sourcemap_basic.snap | 968 +++++++------- .../bundled_code@sourcemap_wrapper.snap | 968 +++++++------- .../snapshots/source_map@sourcemap_basic.snap | 24 +- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 202 ++- docs/source-maps.md | 31 + 12 files changed, 2031 insertions(+), 1533 deletions(-) diff --git a/README.md b/README.md index 014574b99..9fcf13450 100644 --- a/README.md +++ b/README.md @@ -254,16 +254,23 @@ Runtime activation follows the delivery mode: time — delete the map to ship without remapping, drop it back to re-enable - `external`: dormant unless `CRIBO_SOURCE_MAPS=1` is set (or the variable holds a path to the map file) +- In every mode, `CRIBO_SOURCE_MAPS=` points the runtime at an explicit map + file — the only way to remap a bundle executed via `python -` (stdin), whose + inline map cannot be re-read at run time The runtime is lazy (zero file access, parsing, or decoding until the first uncaught exception), streams the map in constant memory so it works under resource pressure, covers `sys.excepthook`, `threading.excepthook`, and -`sys.unraisablehook`, and falls back to the standard traceback on any failure. -Known limitations: user code that formats tracebacks itself (e.g. -`traceback.format_exc()`) is not remapped, and under a hard out-of-memory -condition no pure-Python hook can run. Configuration-file equivalents: -`sourcemap = "linked" | "inline" | "external"` and `sources-content = true|false` -in `cribo.toml`. Full design: [docs/source-maps.md](docs/source-maps.md). +`sys.unraisablehook`, chains to any pre-installed custom hooks, preserves the +default silent handling of `SystemExit` in worker threads, and falls back to the +standard traceback on any failure. Known limitations: user code that formats +tracebacks itself (e.g. `traceback.format_exc()`) is not remapped; +`ExceptionGroup` chains defer to the standard (unremapped but complete) rendering; +and under a hard out-of-memory condition no pure-Python hook can run. +Configuration-file equivalents: `sourcemap = "linked" | "inline" | "external"` and +`sources-content = true|false` in `cribo.toml`; environment equivalents: +`CRIBO_SOURCEMAP` and `CRIBO_SOURCES_CONTENT`. Full design: +[docs/source-maps.md](docs/source-maps.md). ### Dependency Detection (`cribo deps`) diff --git a/crates/cribo/src/config.rs b/crates/cribo/src/config.rs index ea636b815..3a1222445 100644 --- a/crates/cribo/src/config.rs +++ b/crates/cribo/src/config.rs @@ -187,6 +187,8 @@ pub(crate) struct EnvConfig { pub target_version: Option, pub tree_shake: Option, pub bundle_third_party: Option, + pub sourcemap: Option, + pub sources_content: Option, pub python: Option, } @@ -259,6 +261,16 @@ impl EnvConfig { config.bundle_third_party = parse_bool(&bundle_third_party_str); } + // CRIBO_SOURCEMAP - source map delivery mode (linked|inline|external) + if let Ok(sourcemap_str) = env::var("CRIBO_SOURCEMAP") { + config.sourcemap = parse_sourcemap_mode(&sourcemap_str); + } + + // CRIBO_SOURCES_CONTENT - boolean flag overriding the sourcesContent default + if let Ok(sources_content_str) = env::var("CRIBO_SOURCES_CONTENT") { + config.sources_content = parse_bool(&sources_content_str); + } + if let Ok(python) = env::var("CRIBO_PYTHON") { config.python = parse_env_path(&python); } @@ -292,6 +304,12 @@ impl EnvConfig { if let Some(bundle_third_party) = self.bundle_third_party { config.bundle_third_party = Some(bundle_third_party); } + if let Some(sourcemap) = self.sourcemap { + config.sourcemap = Some(sourcemap); + } + if let Some(sources_content) = self.sources_content { + config.sources_content = Some(sources_content); + } if let Some(python) = self.python { config.requirements.python = Some(python); } @@ -309,6 +327,17 @@ fn parse_bool(value: &str) -> Option { } } +/// Parse a source map delivery mode from an environment value. +fn parse_sourcemap_mode(value: &str) -> Option { + use cow_utils::CowUtils; + match value.cow_to_lowercase().as_ref() { + "linked" => Some(SourceMapMode::Linked), + "inline" => Some(SourceMapMode::Inline), + "external" => Some(SourceMapMode::External), + _ => None, + } +} + /// Parse a non-empty environment value as a filesystem path. fn parse_env_path(value: &str) -> Option { if value.is_empty() { diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 35ac18d2f..e3dddb1d5 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -703,16 +703,15 @@ impl BundleOrchestrator { })?; let mut bundled_code = emitted.code; - // Apply the configured source map delivery mode. + // Apply the configured source map delivery mode. The map file itself is + // written only after the bundle write succeeds, so a failed run never + // leaves an orphaned (and potentially stale) map next to an old bundle. + let mut pending_map: Option<(PathBuf, &str)> = None; if let (Some(mode), Some(map_json)) = (self.config.sourcemap, emitted.source_map.as_deref()) { match mode { SourceMapMode::Linked | SourceMapMode::External => { let map_path = source_map_path_for(output_path); - fs::write(&map_path, map_json).with_context(|| { - format!("Failed to write source map file: {}", map_path.display()) - })?; - info!("Source map written to: {}", map_path.display()); if mode == SourceMapMode::Linked { let map_file_name = map_path.file_name().map_or_else( || map_path.to_string_lossy().into_owned(), @@ -723,6 +722,7 @@ impl BundleOrchestrator { &map_file_name, )); } + pending_map = Some((map_path, map_json)); } SourceMapMode::Inline => { bundled_code.push('\n'); @@ -743,6 +743,13 @@ impl BundleOrchestrator { info!("Bundle written to: {}", output_path.display()); + if let Some((map_path, map_json)) = pending_map { + fs::write(&map_path, map_json).with_context(|| { + format!("Failed to write source map file: {}", map_path.display()) + })?; + info!("Source map written to: {}", map_path.display()); + } + Ok(()) } diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index cc1882f7a..c3df15cbe 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -6,6 +6,9 @@ first uncaught exception. Under resource pressure the decoder streams the map in constant memory and falls back to the default traceback on any failure. See docs/source-maps.md in the cribo repository. + +Note: this leading docstring is stripped at injection time so the bundle's +``__doc__`` is not affected. """ import binascii as _cribo_binascii @@ -13,112 +16,6 @@ import sys as _cribo_sys import threading as _cribo_threading -_CRIBO_SM_MODE = "__CRIBO_SOURCEMAP_MODE__" -_CRIBO_SM_BUNDLE = globals().get("__file__", "") -_CRIBO_SM_CHUNK = 8192 -_CRIBO_SM_STATE = {"in_hook": False} -_CRIBO_SM_B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" - -_cribo_sm_prev_excepthook = _cribo_sys.excepthook -_cribo_sm_prev_unraisablehook = _cribo_sys.unraisablehook -_cribo_sm_prev_threading_hook = _cribo_threading.excepthook - - -def _cribo_sm_map_location(): - """Resolve the map location per delivery mode, or None when inactive. - - Returns (map_path, map_dir); map_path is None for inline mode (the map - lives inside the bundle file itself). Called lazily at hook-fire time so - the happy path never touches the environment or the filesystem. - """ - env = _cribo_os.environ.get("CRIBO_SOURCE_MAPS", "") - if env == "0": - return None - bundle = _CRIBO_SM_BUNDLE - if _CRIBO_SM_MODE == "inline": - return (None, _cribo_os.path.dirname(_cribo_os.path.abspath(bundle))) - sibling = bundle + ".map" - if _CRIBO_SM_MODE == "linked": - if _cribo_os.path.exists(sibling): - return (sibling, _cribo_os.path.dirname(_cribo_os.path.abspath(sibling))) - return None - # external: opt in via CRIBO_SOURCE_MAPS=1 (or a path to the map file) - if env in ("1", "true", "yes", "on"): - path = sibling - elif env: - path = env - else: - return None - return (path, _cribo_os.path.dirname(_cribo_os.path.abspath(path))) - - -def _cribo_sm_file_chunks(path): - """Yield fixed-size chunks of a file (constant memory).""" - handle = open(path, "rb") - try: - while True: - chunk = handle.read(_CRIBO_SM_CHUNK) - if not chunk: - break - yield chunk - finally: - handle.close() - - -def _cribo_sm_find_inline_payload(handle): - """Backward-scan the bundle for the last inline map marker. - - Returns the byte offset of the base64 payload, or -1. Only the tail of the - file is examined; the bundle body is never read. - """ - marker = b"# sourceMappingURL=data:" - handle.seek(0, 2) - position = handle.tell() - overlap = b"" - found = -1 - while position > 0: - step = _CRIBO_SM_CHUNK if position >= _CRIBO_SM_CHUNK else position - position -= step - handle.seek(position) - data = handle.read(step) + overlap - index = data.rfind(marker) - if index >= 0: - found = position + index - break - overlap = data[: len(marker) - 1] - if found < 0: - return -1 - handle.seek(found) - head = handle.read(192) - base64_at = head.find(b"base64,") - if base64_at < 0: - return -1 - return found + base64_at + len(b"base64,") - - -def _cribo_sm_inline_chunks(path): - """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = open(path, "rb") - try: - start = _cribo_sm_find_inline_payload(handle) - if start < 0: - return - handle.seek(start) - pending = b"" - while True: - raw = handle.read(_CRIBO_SM_CHUNK) - if not raw: - break - data = pending + raw.translate(None, b"\r\n") - usable = len(data) - (len(data) % 4) - pending = data[usable:] - if usable: - yield _cribo_binascii.a2b_base64(data[:usable]) - if pending: - yield _cribo_binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) - finally: - handle.close() - class _CriboSmStream(object): """Byte-at-a-time reader over an iterator of byte chunks.""" @@ -142,462 +39,662 @@ def read_byte(self): return value -def _cribo_sm_skip_ws(stream, byte): - while byte in (32, 9, 10, 13): - byte = stream.read_byte() - return byte +class _CriboSourceMapRuntime(object): + """Traceback-remapping runtime. + All collaborators (modules, the stream class, previous hooks) are bound to + the instance at construction time, so the installed hooks keep working even + if bundled user code later rebinds any module-level name this template + introduced. + """ -def _cribo_sm_read_string(stream, collect): - """Consume a JSON string whose opening quote was already read. + _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + _CHUNK = 8192 + + def __init__(self, mode, bundle_file): + self._mode = mode + self._bundle = bundle_file + self._os = _cribo_os + self._sys = _cribo_sys + self._binascii = _cribo_binascii + self._threading = _cribo_threading + self._stream_cls = _CriboSmStream + # Re-entrancy guard; thread-local so a hook firing on one thread never + # disables remapping on another. + self._local = _cribo_threading.local() + self._prev_excepthook = _cribo_sys.excepthook + self._prev_unraisablehook = _cribo_sys.unraisablehook + self._prev_threading_hook = _cribo_threading.excepthook + try: + self._group_type = BaseExceptionGroup + except NameError: # Python < 3.11 + self._group_type = None + + def install(self): + """Install the three hooks; the previous hooks stay chained.""" + self._sys.excepthook = self.excepthook + self._sys.unraisablehook = self.unraisablehook + self._threading.excepthook = self.threading_hook + + # -- map location and raw chunk access --------------------------------- + + def _map_location(self): + """Resolve the map location per delivery mode, or None when inactive. + + Returns (map_path, map_dir); map_path is None for inline mode (the map + lives inside the bundle file itself). Called lazily at hook-fire time + so the happy path never touches the environment or the filesystem. + """ + env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + # An explicit path wins for every mode. This is also the only way to + # supply a map to a bundle executed via `python -` (stdin), whose + # source cannot be re-read at hook time. + if env not in ("", "1", "true", "yes", "on"): + path = env + return (path, self._os.path.dirname(self._os.path.abspath(path))) + bundle = self._bundle + if self._mode == "inline": + if bundle == "": + return None # stdin cannot be re-opened; use CRIBO_SOURCE_MAPS= + return (None, self._os.path.dirname(self._os.path.abspath(bundle))) + sibling = bundle + ".map" + if self._mode == "linked": + if self._os.path.exists(sibling): + return ( + sibling, + self._os.path.dirname(self._os.path.abspath(sibling)), + ) + return None + # external: opt in via CRIBO_SOURCE_MAPS=1 (a path was handled above) + if env in ("1", "true", "yes", "on"): + return ( + sibling, + self._os.path.dirname(self._os.path.abspath(sibling)), + ) + return None - Returns the decoded text when collect is true, else None (contents are - discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are handled, - so string *values* containing text like '"mappings":' cannot confuse the - key scanner. - """ - buf = bytearray() if collect else None - while True: - byte = stream.read_byte() - if byte < 0: - raise ValueError("unterminated JSON string") - if byte == 34: # '"' - return buf.decode("utf-8", "replace") if collect else None - if byte != 92: # '\\' - if buf is not None: - buf.append(byte) - continue - escape = stream.read_byte() - if escape < 0: - raise ValueError("unterminated JSON escape") - if escape == 117: # 'u' - code = 0 - for _ in range(4): - digit = stream.read_byte() - if digit < 0: - raise ValueError("unterminated unicode escape") - code = code * 16 + int(chr(digit), 16) - if buf is not None: - buf.extend(chr(code).encode("utf-8", "surrogatepass")) - elif buf is not None: - table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} - buf.append(table.get(escape, escape)) - - -def _cribo_sm_skip_value(stream, byte): - """Skip one JSON value; return the first byte after it (or -1).""" - if byte == 34: # string - _cribo_sm_read_string(stream, False) - return stream.read_byte() - if byte in (123, 91): # object / array - depth = 1 - while depth > 0: - byte = stream.read_byte() - if byte < 0: - raise ValueError("unterminated JSON container") - if byte == 34: - _cribo_sm_read_string(stream, False) - elif byte in (123, 91): - depth += 1 - elif byte in (125, 93): - depth -= 1 - return stream.read_byte() - # number / true / false / null: consume until a delimiter - while byte >= 0 and byte not in (44, 125, 93): # ',' '}' ']' - byte = stream.read_byte() - return byte - - -def _cribo_sm_read_string_array(stream, byte): - """Read a JSON array of strings/nulls; return (list, byte after array).""" - if byte != 91: # '[' - raise ValueError("expected array") - items = [] - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte == 93: # ']' - return items, stream.read_byte() - while True: - if byte == 34: - items.append(_cribo_sm_read_string(stream, True)) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - else: - items.append(None) - byte = _cribo_sm_skip_ws(stream, _cribo_sm_skip_value(stream, byte)) - if byte == 93: - return items, stream.read_byte() - if byte != 44: # ',' - raise ValueError("malformed array") - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) + def _file_chunks(self, path): + """Yield fixed-size chunks of a file (constant memory).""" + handle = open(path, "rb") + try: + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() + def _find_inline_payload(self, handle): + """Backward-scan the bundle for the last inline map marker. + + Returns the byte offset of the base64 payload, or -1. Only the tail of + the file is examined; the bundle body is never read. + """ + marker = b"# sourceMappingURL=data:" + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + found = -1 + while position > 0: + step = self._CHUNK if position >= self._CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + found = position + index + break + overlap = data[: len(marker) - 1] + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + len(b"base64,") + + def _inline_chunks(self, path): + """Yield decoded chunks of an inline (base64 data URL) source map.""" + handle = open(path, "rb") + try: + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = len(data) - (len(data) % 4) + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + finally: + handle.close() -def _cribo_sm_decode_vlq(stream, needed, max_needed): - """Streaming VLQ state machine over the raw bytes of the mappings string. + # -- streaming JSON field scanner --------------------------------------- - Constant state: line/segment counters plus running deltas. Records the - first segment per needed generated line; exits as soon as every needed - line is resolved or the max needed line is passed. Consumes up to and - including the closing quote (or stops early). - """ - lut = {} - for index in range(64): - lut[_CRIBO_SM_B64[index]] = index - result = {} - gen_line = 0 - src_idx = 0 - src_line = 0 - field = 0 - vlq_value = 0 - vlq_shift = 0 - - def end_segment(): - if field >= 4 and gen_line in needed and gen_line not in result: - result[gen_line] = (src_idx, src_line) - - while True: - byte = stream.read_byte() - if byte < 0 or byte == 34: # EOF or closing '"' - end_segment() - return result - if byte == 59: # ';' - end_segment() - gen_line += 1 - field = 0 - if gen_line > max_needed or len(result) == len(needed): - return result - continue - if byte == 44: # ',' - end_segment() - field = 0 - continue - value = lut.get(byte) - if value is None: - raise ValueError("unexpected byte in mappings") - vlq_value += (value & 31) << vlq_shift - if value & 32: - vlq_shift += 5 - continue - signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 - if field == 1: - src_idx += signed - elif field == 2: - src_line += signed - field += 1 + def _skip_ws(self, stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte + + def _read_string(self, stream, collect): + """Consume a JSON string whose opening quote was already read. + + Returns the decoded text when collect is true, else None (contents are + discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are + handled, so string *values* containing text like '"mappings":' cannot + confuse the key scanner. + """ + buf = bytearray() if collect else None + while True: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON string") + if byte == 34: # '"' + return buf.decode("utf-8", "replace") if collect else None + if byte != 92: # '\\' + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise ValueError("unterminated JSON escape") + if escape == 117: # 'u' + code = 0 + for _ in range(4): + digit = stream.read_byte() + if digit < 0: + raise ValueError("unterminated unicode escape") + code = code * 16 + int(chr(digit), 16) + if buf is not None: + buf.extend(chr(code).encode("utf-8", "surrogatepass")) + elif buf is not None: + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf.append(table.get(escape, escape)) + + def _skip_value(self, stream, byte): + """Skip one JSON value; return the first byte after it (or -1).""" + if byte == 34: # string + self._read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): # object / array + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON container") + if byte == 34: + self._read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + # number / true / false / null: consume until a delimiter + while byte >= 0 and byte not in (44, 125, 93): # ',' '}' ']' + byte = stream.read_byte() + return byte + + def _read_string_array(self, stream, byte): + """Read a JSON array of strings/nulls; return (list, byte after array).""" + if byte != 91: # '[' + raise ValueError("expected array") + items = [] + byte = self._skip_ws(stream, stream.read_byte()) + if byte == 93: # ']' + return items, stream.read_byte() + while True: + if byte == 34: + items.append(self._read_string(stream, True)) + byte = self._skip_ws(stream, stream.read_byte()) + else: + items.append(None) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 93: + return items, stream.read_byte() + if byte != 44: # ',' + raise ValueError("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) + + def _decode_vlq(self, stream, needed, max_needed): + """Streaming VLQ state machine over the raw bytes of the mappings string. + + Constant state: line/segment counters plus running deltas. Records the + first segment per needed generated line; exits as soon as every needed + line is resolved or the max needed line is passed. Consumes up to and + including the closing quote (or stops early). + """ + lut = {} + for index in range(64): + lut[self._B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 vlq_value = 0 vlq_shift = 0 + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = (src_idx, src_line) -def _cribo_sm_scan(chunks, needed, max_needed): - """Scan the map's top-level object; return (sources, line table).""" - stream = _CriboSmStream(chunks) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte != 123: # '{' - raise ValueError("not a JSON object") - sources = [] - table = {} - saw_mappings = False - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - while byte == 34: # '"' starting a key - key = _cribo_sm_read_string(stream, True) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte != 58: # ':' - raise ValueError("malformed object") - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if key == "sources": - sources, byte = _cribo_sm_read_string_array(stream, byte) - elif key == "mappings": - if byte != 34: - raise ValueError("mappings is not a string") - table = _cribo_sm_decode_vlq(stream, needed, max_needed) - saw_mappings = True - if sources: - break # both fields consumed; ignore the rest of the map + while True: byte = stream.read_byte() + if byte < 0 or byte == 34: # EOF or closing '"' + end_segment() + return result + if byte == 59: # ';' + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or len(result) == len(needed): + return result + continue + if byte == 44: # ',' + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise ValueError("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 + + def _scan(self, chunks, needed, max_needed): + """Scan the map's top-level object; return (sources, line table).""" + stream = self._stream_cls(chunks) + byte = self._skip_ws(stream, stream.read_byte()) + if byte != 123: # '{' + raise ValueError("not a JSON object") + sources = [] + table = {} + saw_mappings = False + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: # '"' starting a key + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if byte != 58: # ':' + raise ValueError("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + sources, byte = self._read_string_array(stream, byte) + elif key == "mappings": + if byte != 34: + raise ValueError("mappings is not a string") + table = self._decode_vlq(stream, needed, max_needed) + saw_mappings = True + if sources: + break # both fields consumed; ignore the rest of the map + byte = stream.read_byte() + else: + byte = self._skip_value(stream, byte) + byte = self._skip_ws(stream, byte) + if byte == 44: # ',' + byte = self._skip_ws(stream, stream.read_byte()) + if not saw_mappings: + raise ValueError("no mappings field") + return sources, table + + # -- loading ------------------------------------------------------------- + + def _load(self, needed_lines): + """Load (table, sources, map_dir) for 1-based bundle line numbers. + + Returns None when the runtime is inactive for the current mode. The + returned table is keyed by 1-based bundle lines mapping to + (source_index, 1-based original line). + """ + location = self._map_location() + if location is None: + return None + map_path, map_dir = location + needed0 = set(line - 1 for line in needed_lines) + max_needed = max(needed0) + if map_path is None: + chunks = self._inline_chunks(self._bundle) else: - byte = _cribo_sm_skip_value(stream, byte) - byte = _cribo_sm_skip_ws(stream, byte) - if byte == 44: # ',' - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if not saw_mappings: - raise ValueError("no mappings field") - return sources, table - - -def _cribo_sm_load(needed_lines): - """Load (table, sources, map_dir) for 1-based bundle line numbers. - - Returns None when the runtime is inactive for the current mode. The - returned table is keyed by 1-based bundle lines mapping to - (source_index, 1-based original line). - """ - location = _cribo_sm_map_location() - if location is None: - return None - map_path, map_dir = location - needed0 = set(line - 1 for line in needed_lines) - max_needed = max(needed0) - if map_path is None: - chunks = _cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE) - else: - chunks = _cribo_sm_file_chunks(map_path) - sources, table0 = _cribo_sm_scan(chunks, needed0, max_needed) - table = {} - for line0, (src_idx, src_line0) in table0.items(): - table[line0 + 1] = (src_idx, src_line0 + 1) - return (table, sources, map_dir) - - -def _cribo_sm_load_json_fallback(needed_lines): - """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" - location = _cribo_sm_map_location() - if location is None: - return None - map_path, map_dir = location - import json + chunks = self._file_chunks(map_path) + sources, table0 = self._scan(chunks, needed0, max_needed) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = (src_idx, src_line0 + 1) + return (table, sources, map_dir) + + def _load_json_fallback(self, needed_lines): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = self._map_location() + if location is None: + return None + map_path, map_dir = location + import json + + if map_path is None: + raw = b"".join(self._inline_chunks(self._bundle)) + else: + handle = open(map_path, "rb") + try: + raw = handle.read() + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = data.get("sources") or [] + mappings = data.get("mappings") or "" + needed0 = set(line - 1 for line in needed_lines) + stream = self._stream_cls([mappings.encode("ascii"), b'"']) + table0 = self._decode_vlq(stream, needed0, max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = (src_idx, src_line0 + 1) + return (table, sources, map_dir) + + # -- traceback collection and rendering ---------------------------------- + + def _collect_needed(self, exc_value, traceback_obj): + """1-based bundle lines referenced by the traceback (and its chain).""" + needed = set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == self._bundle: + needed.add(tb.tb_lineno) + tb = tb.tb_next + + add(traceback_obj) + exc = exc_value + seen = set() + depth = 0 + while exc is not None and id(exc) not in seen and depth < 16: + seen.add(id(exc)) + depth += 1 + add(getattr(exc, "__traceback__", None)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed + + def _chain_has_group(self, exc_value): + """Whether the exception chain contains a BaseExceptionGroup. + + CPython renders groups with a dedicated nested layout; rather than + losing the nested tracebacks, the runtime defers group rendering + entirely to the previous hook (unremapped but complete). + """ + if self._group_type is None: + return False + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(seen) < 16: + seen.add(id(exc)) + if isinstance(exc, self._group_type): + return True + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return False - if map_path is None: - raw = b"".join(_cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE)) - else: - handle = open(map_path, "rb") + def _source_line(self, path, lineno): + """Read a single 1-based line from a file without caching it.""" + try: + handle = open(path, "rb") + except OSError: + return None try: - raw = handle.read() + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except OSError: + return None finally: handle.close() - data = json.loads(raw.decode("utf-8")) - sources = data.get("sources") or [] - mappings = data.get("mappings") or "" - needed0 = set(line - 1 for line in needed_lines) - stream = _CriboSmStream([mappings.encode("ascii"), b'"']) - table0 = _cribo_sm_decode_vlq(stream, needed0, max(needed0)) - table = {} - for line0, (src_idx, src_line0) in table0.items(): - table[line0 + 1] = (src_idx, src_line0 + 1) - return (table, sources, map_dir) - - -def _cribo_sm_collect_needed(exc_value, traceback_obj): - """1-based bundle line numbers referenced by the traceback (and its chain).""" - needed = set() - - def add(tb): - while tb is not None: - if tb.tb_frame.f_code.co_filename == _CRIBO_SM_BUNDLE: - needed.add(tb.tb_lineno) - tb = tb.tb_next - - add(traceback_obj) - exc = exc_value - seen = set() - depth = 0 - while exc is not None and id(exc) not in seen and depth < 16: - seen.add(id(exc)) - depth += 1 - add(getattr(exc, "__traceback__", None)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - exc = cause if cause is not None else context - return needed - - -def _cribo_sm_source_line(path, lineno): - """Read a single 1-based line from a file without caching it.""" - try: - handle = open(path, "rb") - except OSError: return None - try: - current = 0 - for raw in handle: - current += 1 - if current == lineno: - return raw.decode("utf-8", "replace").strip() - if current > lineno: - break - except OSError: - return None - finally: - handle.close() - return None - - -def _cribo_sm_write_frames(traceback_obj, table, sources, map_dir, write): - """Write remapped frame lines, collapsing repeated frames like CPython. - Consecutive identical frames (recursion) print at most 3 times followed by - a "[Previous line repeated N more times]" marker; source line text is - cached per (file, line) within one rendering to avoid re-reading files. - """ - cache = {} - last = None - repeats = 0 - - def emit(entry): - write(' File "%s", line %d, in %s\n' % entry) - key = (entry[0], entry[1]) - if key not in cache: - cache[key] = _cribo_sm_source_line(entry[0], entry[1]) - if cache[key]: - write(" %s\n" % cache[key]) - - while traceback_obj is not None: - frame = traceback_obj.tb_frame - filename = frame.f_code.co_filename - lineno = traceback_obj.tb_lineno - name = frame.f_code.co_name - if filename == _CRIBO_SM_BUNDLE: - mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: - source = sources[mapped[0]] - if not _cribo_os.path.isabs(source): - source = _cribo_os.path.normpath(_cribo_os.path.join(map_dir, source)) - filename, lineno = source, mapped[1] - entry = (filename, lineno, name) - if entry == last: - repeats += 1 - if repeats <= 3: + def _write_frames(self, traceback_obj, table, sources, map_dir, write): + """Write remapped frame lines, collapsing repeated frames like CPython. + + Consecutive identical frames (recursion) print at most 3 times followed + by a "[Previous line repeated N more times]" marker; source line text + is cached per (file, line) within one rendering to avoid re-reading + files. + """ + cache = {} + last = None + repeats = 0 + + def emit(entry): + write(' File "%s", line %d, in %s\n' % entry) + key = (entry[0], entry[1]) + if key not in cache: + cache[key] = self._source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + if filename == self._bundle: + mapped = table.get(lineno) + if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + source = sources[mapped[0]] + if not self._os.path.isabs(source): + source = self._os.path.normpath( + self._os.path.join(map_dir, source) + ) + filename, lineno = source, mapped[1] + entry = (filename, lineno, name) + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 emit(entry) - else: - if repeats > 3: - write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - last = entry - repeats = 1 - emit(entry) - traceback_obj = traceback_obj.tb_next - if repeats > 3: - write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - - -def _cribo_sm_exception_line(exc_value): - exc_type = type(exc_value) - name = getattr(exc_type, "__qualname__", exc_type.__name__) - module = getattr(exc_type, "__module__", None) - if module not in (None, "builtins", "__main__"): - name = "%s.%s" % (module, name) - try: - text = str(exc_value) - except BaseException: - text = "" - return "%s: %s\n" % (name, text) if text else "%s\n" % name - - -def _cribo_sm_render(exc_value, table, sources, map_dir, write): - """Render the exception (with its cause/context chain) like CPython does.""" - chain = [] - exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen and len(chain) < 16: - seen.add(id(exc)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - suppress = getattr(exc, "__suppress_context__", False) - if cause is not None: - chain.append((exc, "cause")) - exc = cause - elif context is not None and not suppress: - chain.append((exc, "context")) - exc = context - else: - chain.append((exc, None)) - exc = None - # Print innermost first, like CPython. The link stored on an exception - # describes its relation to its own inner exception — which is exactly - # the one printed immediately before it. - ordered = list(reversed(chain)) - for index, (exc, link) in enumerate(ordered): - if index > 0: - if link == "cause": - write( - "\nThe above exception was the direct cause of the following exception:\n\n" - ) + traceback_obj = traceback_obj.tb_next + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + + def _exception_line(self, exc_value): + exc_type = type(exc_value) + name = getattr(exc_type, "__qualname__", exc_type.__name__) + module = getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = str(exc_value) + except BaseException: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name + + def _render(self, exc_value, table, sources, map_dir, write): + """Render the exception (with its cause/context chain) like CPython.""" + chain = [] + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(chain) < 16: + seen.add(id(exc)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + suppress = getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context else: - write( - "\nDuring handling of the above exception, another exception occurred:\n\n" - ) - tb = getattr(exc, "__traceback__", None) - if tb is not None: - write("Traceback (most recent call last):\n") - _cribo_sm_write_frames(tb, table, sources, map_dir, write) - write(_cribo_sm_exception_line(exc)) - - -def _cribo_sm_try_render(exc_value, traceback_obj, prefix): - """Attempt a remapped rendering to stderr; True on success. - - Never raises and never masks the original exception: any failure in this - runtime returns False so callers can delegate to the previous hook. - """ - if _CRIBO_SM_STATE["in_hook"] or exc_value is None: - return False - _CRIBO_SM_STATE["in_hook"] = True - old_limit = None - try: - needed = _cribo_sm_collect_needed(exc_value, traceback_obj) - if not needed: + chain.append((exc, None)) + exc = None + # Print innermost first, like CPython. The link stored on an exception + # describes its relation to its own inner exception — which is exactly + # the one printed immediately before it. + ordered = list(reversed(chain)) + for index, (exc, link) in enumerate(ordered): + if index > 0: + if link == "cause": + write( + "\nThe above exception was the direct cause of the following " + "exception:\n\n" + ) + else: + write( + "\nDuring handling of the above exception, another exception " + "occurred:\n\n" + ) + tb = getattr(exc, "__traceback__", None) + if tb is not None: + write("Traceback (most recent call last):\n") + self._write_frames(tb, table, sources, map_dir, write) + write(self._exception_line(exc)) + + def _try_render(self, exc_value, traceback_obj, prefix): + """Attempt a remapped rendering to stderr; True on success. + + Never raises and never masks the original exception: any failure in + this runtime returns False so callers can delegate to the previous + hook. + """ + if getattr(self._local, "in_hook", False) or exc_value is None: return False - loaded = None + self._local.in_hook = True + old_limit = None try: - loaded = _cribo_sm_load(needed) - except BaseException: + if self._chain_has_group(exc_value): + return False + needed = self._collect_needed(exc_value, traceback_obj) + if not needed: + return False + loaded = None try: - loaded = _cribo_sm_load_json_fallback(needed) + loaded = self._load(needed) except BaseException: - loaded = None - if not loaded: - return False - table, sources, map_dir = loaded - if not table: - return False - try: - old_limit = _cribo_sys.getrecursionlimit() - _cribo_sys.setrecursionlimit(old_limit + 64) - except BaseException: - old_limit = None - # Buffer the rendering so a mid-render failure produces no partial - # output before the previous hook prints the standard traceback. - parts = [] - if prefix: - parts.append(prefix) - _cribo_sm_render(exc_value, table, sources, map_dir, parts.append) - stderr = _cribo_sys.stderr - stderr.write("".join(parts)) - try: - stderr.flush() + try: + loaded = self._load_json_fallback(needed) + except BaseException: + loaded = None + if not loaded: + return False + table, sources, map_dir = loaded + if not table: + return False + try: + old_limit = self._sys.getrecursionlimit() + self._sys.setrecursionlimit(old_limit + 64) + except BaseException: + old_limit = None + # Buffer the rendering so a mid-render failure produces no partial + # output before the previous hook prints the standard traceback. + parts = [] + if prefix: + parts.append(prefix) + self._render(exc_value, table, sources, map_dir, parts.append) + stderr = self._sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except BaseException: + pass + return True except BaseException: - pass - return True - except BaseException: - return False - finally: - if old_limit is not None: + return False + finally: + if old_limit is not None: + try: + self._sys.setrecursionlimit(old_limit) + except BaseException: + pass + self._local.in_hook = False + + def _notify_custom_hook(self, prev, default, call): + """Invoke a chained hook after a successful remap when it is custom. + + A successful remap replaces the *default* printer, but preinstalled + custom hooks (error reporters, sitecustomize) must still observe the + exception; their own output is theirs to manage. When the interpreter + default is unavailable for comparison (e.g. `threading.__excepthook__` + before Python 3.10), no notification happens — better to skip a custom + hook than to double-print via the default one. + """ + if default is not None and prev is not None and prev is not default: try: - _cribo_sys.setrecursionlimit(old_limit) + call(prev) except BaseException: pass - _CRIBO_SM_STATE["in_hook"] = False - - -def _cribo_sm_excepthook(exc_type, exc_value, traceback_obj): - if not _cribo_sm_try_render(exc_value, traceback_obj, None): - _cribo_sm_prev_excepthook(exc_type, exc_value, traceback_obj) + # -- installed hooks ------------------------------------------------------ -def _cribo_sm_threading_hook(args): - thread = getattr(args, "thread", None) - name = getattr(thread, "name", None) or "Thread" - prefix = "Exception in thread %s:\n" % name - if not _cribo_sm_try_render(args.exc_value, args.exc_traceback, prefix): - _cribo_sm_prev_threading_hook(args) + def excepthook(self, exc_type, exc_value, traceback_obj): + if self._try_render(exc_value, traceback_obj, None): + self._notify_custom_hook( + self._prev_excepthook, + self._sys.__excepthook__, + lambda hook: hook(exc_type, exc_value, traceback_obj), + ) + return + self._prev_excepthook(exc_type, exc_value, traceback_obj) + def threading_hook(self, args): + # The default threading hook deliberately ignores SystemExit (normal + # sys.exit() in a worker thread); preserve that by delegating. + if args.exc_type is not None and issubclass(args.exc_type, SystemExit): + self._prev_threading_hook(args) + return + thread = getattr(args, "thread", None) + name = getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if self._try_render(args.exc_value, args.exc_traceback, prefix): + self._notify_custom_hook( + self._prev_threading_hook, + getattr(self._threading, "__excepthook__", None), + lambda hook: hook(args), + ) + return + self._prev_threading_hook(args) -def _cribo_sm_unraisablehook(unraisable): - message = getattr(unraisable, "err_msg", None) or "Exception ignored in" - try: - prefix = "%s: %r\n" % (message, unraisable.object) - except BaseException: - prefix = "%s\n" % message - if not _cribo_sm_try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - _cribo_sm_prev_unraisablehook(unraisable) + def unraisablehook(self, unraisable): + message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except BaseException: + prefix = "%s\n" % message + if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + self._notify_custom_hook( + self._prev_unraisablehook, + self._sys.__unraisablehook__, + lambda hook: hook(unraisable), + ) + return + self._prev_unraisablehook(unraisable) -_cribo_sys.excepthook = _cribo_sm_excepthook -_cribo_sys.unraisablehook = _cribo_sm_unraisablehook -_cribo_threading.excepthook = _cribo_sm_threading_hook +_CriboSourceMapRuntime( + "__CRIBO_SOURCEMAP_MODE__", globals().get("__file__", "") +).install() diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index cc74b5ae3..ad000ea5d 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -331,6 +331,22 @@ impl ParallelWalker<'_> { for (gen_clause, orig_clause) in g.elif_else_clauses.iter().zip(&o.elif_else_clauses) { + // An exception raised while evaluating an `elif` + // condition reports the clause header line, so the + // header needs its own mapping (provenance comes from + // the condition expression, which carries a node index). + if let (Some(_gen_test), Some(orig_test)) = + (&gen_clause.test, &orig_clause.test) + && let Some((module_ordinal, original_line)) = self + .provenance + .resolve(orig_test.node_index().load(), orig_clause.range().start()) + { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_clause.range().start()), + module_ordinal, + original_line, + }); + } self.walk_body(&gen_clause.body, &orig_clause.body); } } @@ -404,9 +420,12 @@ fn relative_path(base: &std::path::Path, target: &std::path::Path) -> std::path: match component { Component::Normal(_) => result.push(".."), // A remaining root/prefix component means the paths have no common - // ancestor expressible relatively; keep the target as-is. - Component::RootDir | Component::Prefix(_) => return target.to_path_buf(), - _ => {} + // ancestor expressible relatively; a `..` component cannot be + // inverted lexically. In both cases keep the target as-is. + Component::RootDir | Component::Prefix(_) | Component::ParentDir => { + return target.to_path_buf(); + } + Component::CurDir => {} } } result.extend(target_components); @@ -458,14 +477,24 @@ pub(crate) fn inject_runtime_prologue( let source = RUNTIME_TEMPLATE.cow_replace(RUNTIME_MODE_PLACEHOLDER, mode_str); match ruff_python_parser::parse_module(&source) { Ok(parsed) => { + let mut statements = parsed.into_syntax().body; + // Drop the template's leading docstring: injected at position zero + // it would otherwise become the bundle's module docstring and + // change the program's observable `__doc__`. + if statements.first().is_some_and(|stmt| { + matches!( + stmt, + Stmt::Expr(expr) if expr.value.is_string_literal_expr() + ) + }) { + statements.remove(0); + } let insert_at = bundled_ast .body .iter() .take_while(|stmt| is_future_import(stmt)) .count(); - bundled_ast - .body - .splice(insert_at..insert_at, parsed.into_syntax().body); + bundled_ast.body.splice(insert_at..insert_at, statements); } Err(err) => log::warn!( "source map runtime template failed to parse; traceback remapping disabled: {err}" diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index 0fe4fe783..fa1572d84 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -3,6 +3,9 @@ Driven by the Rust integration test `python_runtime_unit_tests` (plain asserts, no pytest dependency). Usage: python test_sourcemap_runtime.py where is the template with the mode placeholder substituted. + +Tests are discovered automatically: every module-level callable whose name +starts with ``test_`` runs once, receiving the runtime instance. """ import importlib.util @@ -13,17 +16,21 @@ def load_runtime(path): - """Import the runtime module, then restore the hooks it installs.""" + """Import the runtime module and return a runtime instance for testing. + + The import installs the hooks; they are restored immediately so failures + in this harness surface as normal tracebacks. + """ prev_hooks = (sys.excepthook, sys.unraisablehook, threading.excepthook) spec = importlib.util.spec_from_file_location("cribo_sm_runtime", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) sys.excepthook, sys.unraisablehook, threading.excepthook = prev_hooks - return module + return module._CriboSourceMapRuntime("external", "") def test_stream_reads_across_chunk_boundaries(rt): - stream = rt._CriboSmStream([b"ab", b"", b"c", b"de"]) + stream = rt._stream_cls([b"ab", b"", b"c", b"de"]) got = [] while True: byte = stream.read_byte() @@ -35,7 +42,7 @@ def test_stream_reads_across_chunk_boundaries(rt): def _scan(rt, json_text, needed, max_needed): - return rt._cribo_sm_scan([json_text.encode("utf-8")], needed, max_needed) + return rt._scan([json_text.encode("utf-8")], needed, max_needed) def test_scan_extracts_sources_and_mappings(rt): @@ -94,7 +101,7 @@ def chunks(): yield b'{"sources":["a.py"],"mappings":"AAAA;AACA;' raise Boom("decoder read past its early-exit point") - sources, table = rt._cribo_sm_scan(chunks(), {0}, 0) + _sources, table = rt._scan(chunks(), {0}, 0) assert table == {0: (0, 0)}, table @@ -108,7 +115,7 @@ def test_vlq_rejects_escapes_in_mappings(rt): raise AssertionError("escape inside mappings must raise") -def make_inline_bundle(payload_json, line_length=None): +def make_inline_bundle(payload_json): """Create a temp file shaped like an inline-mode bundle; return its path.""" import base64 @@ -129,12 +136,10 @@ def test_inline_payload_scan_and_chunked_base64(rt): json_text = '{"filler":"' + filler + '","sources":["a.py"],"mappings":"AAAA"}' path = make_inline_bundle(json_text) try: - decoded = b"".join(rt._cribo_sm_inline_chunks(path)) + decoded = b"".join(rt._inline_chunks(path)) assert decoded.decode("utf-8") == json_text # And end-to-end through the scanner: - sources, table = rt._cribo_sm_scan( - rt._cribo_sm_inline_chunks(path), {0}, 0 - ) + sources, table = rt._scan(rt._inline_chunks(path), {0}, 0) assert sources == ["a.py"], sources assert table == {0: (0, 0)}, table finally: @@ -148,7 +153,7 @@ def test_inline_scan_without_marker_yields_nothing(rt): with handle as f: f.write("print('no map here')\n" * 50) try: - assert b"".join(rt._cribo_sm_inline_chunks(handle.name)) == b"" + assert b"".join(rt._inline_chunks(handle.name)) == b"" finally: os.unlink(handle.name) @@ -157,42 +162,69 @@ def test_json_fallback_matches_streaming(rt): json_text = '{"sources":["a.py","b.py"],"mappings":"AAAA;ACCA"}' streaming = _scan(rt, json_text, {0, 1}, 1) - path = None - handle = tempfile.NamedTemporaryFile("w", suffix=".map", delete=False) + handle = tempfile.NamedTemporaryFile( + "w", suffix=".map", delete=False, encoding="utf-8" + ) with handle as f: f.write(json_text) path = handle.name + previous = os.environ.get("CRIBO_SOURCE_MAPS") try: os.environ["CRIBO_SOURCE_MAPS"] = path - loaded = rt._cribo_sm_load_json_fallback({1, 2}) + loaded = rt._load_json_fallback({1, 2}) assert loaded is not None table, sources, _map_dir = loaded # Fallback tables are 1-based. - expected = {line0 + 1: (idx, line0src + 1) for line0, (idx, line0src) in streaming[1].items()} + expected = { + line0 + 1: (idx, src_line0 + 1) + for line0, (idx, src_line0) in streaming[1].items() + } assert table == expected, (table, expected) assert sources == streaming[0] finally: - del os.environ["CRIBO_SOURCE_MAPS"] + if previous is None: + os.environ.pop("CRIBO_SOURCE_MAPS", None) + else: + os.environ["CRIBO_SOURCE_MAPS"] = previous + os.unlink(path) + + +def test_env_path_wins_for_every_mode(rt): + # A CRIBO_SOURCE_MAPS path activates the runtime even for a bundle + # (the stdin piping workflow cannot re-read its own inline map). + handle = tempfile.NamedTemporaryFile( + "w", suffix=".map", delete=False, encoding="utf-8" + ) + with handle as f: + f.write('{"sources":["a.py"],"mappings":"AAAA"}') + path = handle.name + previous = os.environ.get("CRIBO_SOURCE_MAPS") + try: + os.environ["CRIBO_SOURCE_MAPS"] = path + inline_stdin = type(rt)("inline", "") + loaded = inline_stdin._load({1}) + assert loaded is not None, "env path must activate a inline bundle" + table, sources, _map_dir = loaded + assert sources == ["a.py"] + assert table == {1: (0, 1)}, table + # Without the env override, a inline bundle stays inactive. + os.environ.pop("CRIBO_SOURCE_MAPS", None) + assert inline_stdin._map_location() is None + finally: + if previous is None: + os.environ.pop("CRIBO_SOURCE_MAPS", None) + else: + os.environ["CRIBO_SOURCE_MAPS"] = previous os.unlink(path) def main(): runtime_path = sys.argv[1] rt = load_runtime(runtime_path) - tests = [ - test_stream_reads_across_chunk_boundaries, - test_scan_extracts_sources_and_mappings, - test_scan_negative_delta, - test_scan_source_index_delta, - test_scan_ignores_adversarial_sources_content, - test_scan_handles_unicode_escapes_in_sources, - test_scan_null_in_sources_array, - test_vlq_early_exit_stops_reading, - test_vlq_rejects_escapes_in_mappings, - test_inline_payload_scan_and_chunked_base64, - test_inline_scan_without_marker_yields_nothing, - test_json_fallback_matches_streaming, - ] + tests = sorted( + (obj for name, obj in globals().items() if name.startswith("test_") and callable(obj)), + key=lambda obj: obj.__name__, + ) for test in tests: test(rt) print("PASS %s" % test.__name__) diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index d717b2542..782179815 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -6,97 +6,10 @@ input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py # Generated by Cribo - Python Source Bundler # https://github.com/ophidiarium/cribo -"""Cribo source map runtime (injected prologue).\n\nRemaps tracebacks of uncaught exceptions back to the original source files\nusing the Source Map v3 emitted at bundle time. Lazy by design: no file I/O,\nparsing, or decoding happens at import time; everything is deferred to the\nfirst uncaught exception. Under resource pressure the decoder streams the map\nin constant memory and falls back to the default traceback on any failure.\nSee docs/source-maps.md in the cribo repository.\n""" import binascii as _cribo_binascii import os as _cribo_os import sys as _cribo_sys import threading as _cribo_threading -_CRIBO_SM_MODE = "linked" -_CRIBO_SM_BUNDLE = globals().get("__file__", "") -_CRIBO_SM_CHUNK = 8192 -_CRIBO_SM_STATE = {"in_hook": False} -_CRIBO_SM_B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" -_cribo_sm_prev_excepthook = _cribo_sys.excepthook -_cribo_sm_prev_unraisablehook = _cribo_sys.unraisablehook -_cribo_sm_prev_threading_hook = _cribo_threading.excepthook -def _cribo_sm_map_location(): - """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time so\n the happy path never touches the environment or the filesystem.\n """ - env = _cribo_os.environ.get("CRIBO_SOURCE_MAPS", "") - if env == "0": - return None - bundle = _CRIBO_SM_BUNDLE - if _CRIBO_SM_MODE == "inline": - return None, _cribo_os.path.dirname(_cribo_os.path.abspath(bundle)) - sibling = bundle + ".map" - if _CRIBO_SM_MODE == "linked": - if _cribo_os.path.exists(sibling): - return sibling, _cribo_os.path.dirname(_cribo_os.path.abspath(sibling)) - return None - if env in ("1", "true", "yes", "on"): - path = sibling - elif env: - path = env - else: - return None - return path, _cribo_os.path.dirname(_cribo_os.path.abspath(path)) -def _cribo_sm_file_chunks(path): - """Yield fixed-size chunks of a file (constant memory).""" - handle = open(path, "rb") - try: - while True: - chunk = handle.read(_CRIBO_SM_CHUNK) - if not chunk: - break - yield chunk - finally: - handle.close() -def _cribo_sm_find_inline_payload(handle): - """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of the\n file is examined; the bundle body is never read.\n """ - marker = b"# sourceMappingURL=data:" - handle.seek(0, 2) - position = handle.tell() - overlap = b"" - found = -1 - while position > 0: - step = _CRIBO_SM_CHUNK if position >= _CRIBO_SM_CHUNK else position - position -= step - handle.seek(position) - data = handle.read(step) + overlap - index = data.rfind(marker) - if index >= 0: - found = position + index - break - overlap = data[:len(marker) - 1] - if found < 0: - return -1 - handle.seek(found) - head = handle.read(192) - base64_at = head.find(b"base64,") - if base64_at < 0: - return -1 - return found + base64_at + len(b"base64,") -def _cribo_sm_inline_chunks(path): - """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = open(path, "rb") - try: - start = _cribo_sm_find_inline_payload(handle) - if start < 0: - return - handle.seek(start) - pending = b"" - while True: - raw = handle.read(_CRIBO_SM_CHUNK) - if not raw: - break - data = pending + raw.translate(None, b"\r\n") - usable = len(data) - len(data) % 4 - pending = data[usable:] - if usable: - yield _cribo_binascii.a2b_base64(data[:usable]) - if pending: - yield _cribo_binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) - finally: - handle.close() class _CriboSmStream(object): """Byte-at-a-time reader over an iterator of byte chunks.""" __slots__ = "_chunks", "_buf", "_pos" @@ -116,391 +29,550 @@ class _CriboSmStream(object): value = self._buf[self._pos] self._pos += 1 return value -def _cribo_sm_skip_ws(stream, byte): - while byte in (32, 9, 10, 13): - byte = stream.read_byte() - return byte -def _cribo_sm_read_string(stream, collect): - """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are handled,\n so string *values* containing text like '\"mappings\":' cannot confuse the\n key scanner.\n """ - buf = bytearray() if collect else None - while True: - byte = stream.read_byte() - if byte < 0: - raise ValueError("unterminated JSON string") - if byte == 34: - return buf.decode("utf-8", "replace") if collect else None - if byte != 92: - if buf is not None: - buf.append(byte) - continue - escape = stream.read_byte() - if escape < 0: - raise ValueError("unterminated JSON escape") - if escape == 117: - code = 0 - for _ in range(4): - digit = stream.read_byte() - if digit < 0: - raise ValueError("unterminated unicode escape") - code = code * 16 + int(chr(digit), 16) - if buf is not None: - buf.extend(chr(code).encode("utf-8", "surrogatepass")) - elif buf is not None: - table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} - buf.append(table.get(escape, escape)) -def _cribo_sm_skip_value(stream, byte): - """Skip one JSON value; return the first byte after it (or -1).""" - if byte == 34: - _cribo_sm_read_string(stream, False) - return stream.read_byte() - if byte in (123, 91): - depth = 1 - while depth > 0: +class _CriboSourceMapRuntime(object): + """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, so the installed hooks keep working even\n if bundled user code later rebinds any module-level name this template\n introduced.\n """ + _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + _CHUNK = 8192 + + def __init__(self, mode, bundle_file): + self._mode = mode + self._bundle = bundle_file + self._os = _cribo_os + self._sys = _cribo_sys + self._binascii = _cribo_binascii + self._threading = _cribo_threading + self._stream_cls = _CriboSmStream + self._local = _cribo_threading.local() + self._prev_excepthook = _cribo_sys.excepthook + self._prev_unraisablehook = _cribo_sys.unraisablehook + self._prev_threading_hook = _cribo_threading.excepthook + try: + self._group_type = BaseExceptionGroup + except NameError: + self._group_type = None + + def install(self): + """Install the three hooks; the previous hooks stay chained.""" + self._sys.excepthook = self.excepthook + self._sys.unraisablehook = self.unraisablehook + self._threading.excepthook = self.threading_hook + + def _map_location(self): + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time\n so the happy path never touches the environment or the filesystem.\n """ + env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + if env not in ("", "1", "true", "yes", "on"): + path = env + return path, self._os.path.dirname(self._os.path.abspath(path)) + bundle = self._bundle + if self._mode == "inline": + if bundle == "": + return None + return None, self._os.path.dirname(self._os.path.abspath(bundle)) + sibling = bundle + ".map" + if self._mode == "linked": + if self._os.path.exists(sibling): + return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return None + if env in ("1", "true", "yes", "on"): + return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return None + + def _file_chunks(self, path): + """Yield fixed-size chunks of a file (constant memory).""" + handle = open(path, "rb") + try: + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() + + def _find_inline_payload(self, handle): + """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of\n the file is examined; the bundle body is never read.\n """ + marker = b"# sourceMappingURL=data:" + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + found = -1 + while position > 0: + step = self._CHUNK if position >= self._CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + found = position + index + break + overlap = data[:len(marker) - 1] + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + len(b"base64,") + + def _inline_chunks(self, path): + """Yield decoded chunks of an inline (base64 data URL) source map.""" + handle = open(path, "rb") + try: + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = len(data) - len(data) % 4 + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + finally: + handle.close() + + def _skip_ws(self, stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte + + def _read_string(self, stream, collect): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ + buf = bytearray() if collect else None + while True: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON container") + raise ValueError("unterminated JSON string") if byte == 34: - _cribo_sm_read_string(stream, False) - elif byte in (123, 91): - depth += 1 - elif byte in (125, 93): - depth -= 1 - return stream.read_byte() - while byte >= 0 and byte not in (44, 125, 93): - byte = stream.read_byte() - return byte -def _cribo_sm_read_string_array(stream, byte): - """Read a JSON array of strings/nulls; return (list, byte after array).""" - if byte != 91: - raise ValueError("expected array") - items = [] - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte == 93: - return items, stream.read_byte() - while True: + return buf.decode("utf-8", "replace") if collect else None + if byte != 92: + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise ValueError("unterminated JSON escape") + if escape == 117: + code = 0 + for _ in range(4): + digit = stream.read_byte() + if digit < 0: + raise ValueError("unterminated unicode escape") + code = code * 16 + int(chr(digit), 16) + if buf is not None: + buf.extend(chr(code).encode("utf-8", "surrogatepass")) + elif buf is not None: + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf.append(table.get(escape, escape)) + + def _skip_value(self, stream, byte): + """Skip one JSON value; return the first byte after it (or -1).""" if byte == 34: - items.append(_cribo_sm_read_string(stream, True)) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - else: - items.append(None) - byte = _cribo_sm_skip_ws(stream, _cribo_sm_skip_value(stream, byte)) + self._read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON container") + if byte == 34: + self._read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + while byte >= 0 and byte not in (44, 125, 93): + byte = stream.read_byte() + return byte + + def _read_string_array(self, stream, byte): + """Read a JSON array of strings/nulls; return (list, byte after array).""" + if byte != 91: + raise ValueError("expected array") + items = [] + byte = self._skip_ws(stream, stream.read_byte()) if byte == 93: return items, stream.read_byte() - if byte != 44: - raise ValueError("malformed array") - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) -def _cribo_sm_decode_vlq(stream, needed, max_needed): - """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ - lut = {} - for index in range(64): - lut[_CRIBO_SM_B64[index]] = index - result = {} - gen_line = 0 - src_idx = 0 - src_line = 0 - field = 0 - vlq_value = 0 - vlq_shift = 0 - - def end_segment(): - if field >= 4 and gen_line in needed and gen_line not in result: - result[gen_line] = src_idx, src_line - while True: - byte = stream.read_byte() - if byte < 0 or byte == 34: - end_segment() - return result - if byte == 59: - end_segment() - gen_line += 1 - field = 0 - if gen_line > max_needed or len(result) == len(needed): - return result - continue - if byte == 44: - end_segment() - field = 0 - continue - value = lut.get(byte) - if value is None: - raise ValueError("unexpected byte in mappings") - vlq_value += (value & 31) << vlq_shift - if value & 32: - vlq_shift += 5 - continue - signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 - if field == 1: - src_idx += signed - elif field == 2: - src_line += signed - field += 1 + while True: + if byte == 34: + items.append(self._read_string(stream, True)) + byte = self._skip_ws(stream, stream.read_byte()) + else: + items.append(None) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 93: + return items, stream.read_byte() + if byte != 44: + raise ValueError("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) + + def _decode_vlq(self, stream, needed, max_needed): + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ + lut = {} + for index in range(64): + lut[self._B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 vlq_value = 0 vlq_shift = 0 -def _cribo_sm_scan(chunks, needed, max_needed): - """Scan the map's top-level object; return (sources, line table).""" - stream = _CriboSmStream(chunks) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte != 123: - raise ValueError("not a JSON object") - sources = [] - table = {} - saw_mappings = False - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - while byte == 34: - key = _cribo_sm_read_string(stream, True) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte != 58: - raise ValueError("malformed object") - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if key == "sources": - sources, byte = _cribo_sm_read_string_array(stream, byte) - elif key == "mappings": - if byte != 34: - raise ValueError("mappings is not a string") - table = _cribo_sm_decode_vlq(stream, needed, max_needed) - saw_mappings = True - if sources: - break + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = src_idx, src_line + while True: byte = stream.read_byte() + if byte < 0 or byte == 34: + end_segment() + return result + if byte == 59: + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or len(result) == len(needed): + return result + continue + if byte == 44: + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise ValueError("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 + + def _scan(self, chunks, needed, max_needed): + """Scan the map's top-level object; return (sources, line table).""" + stream = self._stream_cls(chunks) + byte = self._skip_ws(stream, stream.read_byte()) + if byte != 123: + raise ValueError("not a JSON object") + sources = [] + table = {} + saw_mappings = False + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if byte != 58: + raise ValueError("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + sources, byte = self._read_string_array(stream, byte) + elif key == "mappings": + if byte != 34: + raise ValueError("mappings is not a string") + table = self._decode_vlq(stream, needed, max_needed) + saw_mappings = True + if sources: + break + byte = stream.read_byte() + else: + byte = self._skip_value(stream, byte) + byte = self._skip_ws(stream, byte) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + if not saw_mappings: + raise ValueError("no mappings field") + return sources, table + + def _load(self, needed_lines): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ + location = self._map_location() + if location is None: + return None + map_path, map_dir = location + needed0 = set(line - 1 for line in needed_lines) + max_needed = max(needed0) + if map_path is None: + chunks = self._inline_chunks(self._bundle) else: - byte = _cribo_sm_skip_value(stream, byte) - byte = _cribo_sm_skip_ws(stream, byte) - if byte == 44: - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if not saw_mappings: - raise ValueError("no mappings field") - return sources, table -def _cribo_sm_load(needed_lines): - """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ - location = _cribo_sm_map_location() - if location is None: - return None - map_path, map_dir = location - needed0 = set(line - 1 for line in needed_lines) - max_needed = max(needed0) - if map_path is None: - chunks = _cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE) - else: - chunks = _cribo_sm_file_chunks(map_path) - sources, table0 = _cribo_sm_scan(chunks, needed0, max_needed) - table = {} - for line0, (src_idx, src_line0) in table0.items(): - table[line0 + 1] = src_idx, src_line0 + 1 - return table, sources, map_dir -def _cribo_sm_load_json_fallback(needed_lines): - """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" - location = _cribo_sm_map_location() - if location is None: - return None - map_path, map_dir = location - import json - if map_path is None: - raw = b"".join(_cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE)) - else: - handle = open(map_path, "rb") + chunks = self._file_chunks(map_path) + sources, table0 = self._scan(chunks, needed0, max_needed) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _load_json_fallback(self, needed_lines): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = self._map_location() + if location is None: + return None + map_path, map_dir = location + import json + if map_path is None: + raw = b"".join(self._inline_chunks(self._bundle)) + else: + handle = open(map_path, "rb") + try: + raw = handle.read() + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = data.get("sources") or [] + mappings = data.get("mappings") or "" + needed0 = set(line - 1 for line in needed_lines) + stream = self._stream_cls([mappings.encode("ascii"), b'"']) + table0 = self._decode_vlq(stream, needed0, max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _collect_needed(self, exc_value, traceback_obj): + """1-based bundle lines referenced by the traceback (and its chain).""" + needed = set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == self._bundle: + needed.add(tb.tb_lineno) + tb = tb.tb_next + add(traceback_obj) + exc = exc_value + seen = set() + depth = 0 + while exc is not None and id(exc) not in seen and depth < 16: + seen.add(id(exc)) + depth += 1 + add(getattr(exc, "__traceback__", None)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed + + def _chain_has_group(self, exc_value): + """Whether the exception chain contains a BaseExceptionGroup.\n\n CPython renders groups with a dedicated nested layout; rather than\n losing the nested tracebacks, the runtime defers group rendering\n entirely to the previous hook (unremapped but complete).\n """ + if self._group_type is None: + return False + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(seen) < 16: + seen.add(id(exc)) + if isinstance(exc, self._group_type): + return True + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return False + + def _source_line(self, path, lineno): + """Read a single 1-based line from a file without caching it.""" try: - raw = handle.read() + handle = open(path, "rb") + except OSError: + return None + try: + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except OSError: + return None finally: handle.close() - data = json.loads(raw.decode("utf-8")) - sources = data.get("sources") or [] - mappings = data.get("mappings") or "" - needed0 = set(line - 1 for line in needed_lines) - stream = _CriboSmStream([mappings.encode("ascii"), b'"']) - table0 = _cribo_sm_decode_vlq(stream, needed0, max(needed0)) - table = {} - for line0, (src_idx, src_line0) in table0.items(): - table[line0 + 1] = src_idx, src_line0 + 1 - return table, sources, map_dir -def _cribo_sm_collect_needed(exc_value, traceback_obj): - """1-based bundle line numbers referenced by the traceback (and its chain).""" - needed = set() - - def add(tb): - while tb is not None: - if tb.tb_frame.f_code.co_filename == _CRIBO_SM_BUNDLE: - needed.add(tb.tb_lineno) - tb = tb.tb_next - add(traceback_obj) - exc = exc_value - seen = set() - depth = 0 - while exc is not None and id(exc) not in seen and depth < 16: - seen.add(id(exc)) - depth += 1 - add(getattr(exc, "__traceback__", None)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - exc = cause if cause is not None else context - return needed -def _cribo_sm_source_line(path, lineno): - """Read a single 1-based line from a file without caching it.""" - try: - handle = open(path, "rb") - except OSError: - return None - try: - current = 0 - for raw in handle: - current += 1 - if current == lineno: - return raw.decode("utf-8", "replace").strip() - if current > lineno: - break - except OSError: return None - finally: - handle.close() - return None -def _cribo_sm_write_frames(traceback_obj, table, sources, map_dir, write): - '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed by\n a "[Previous line repeated N more times]" marker; source line text is\n cached per (file, line) within one rendering to avoid re-reading files.\n ''' - cache = {} - last = None - repeats = 0 - - def emit(entry): - write(' File "%s", line %d, in %s\n' % entry) - key = entry[0], entry[1] - if key not in cache: - cache[key] = _cribo_sm_source_line(entry[0], entry[1]) - if cache[key]: - write(" %s\n" % cache[key]) - while traceback_obj is not None: - frame = traceback_obj.tb_frame - filename = frame.f_code.co_filename - lineno = traceback_obj.tb_lineno - name = frame.f_code.co_name - if filename == _CRIBO_SM_BUNDLE: - mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: - source = sources[mapped[0]] - if not _cribo_os.path.isabs(source): - source = _cribo_os.path.normpath(_cribo_os.path.join(map_dir, source)) - filename, lineno = source, mapped[1] - entry = filename, lineno, name - if entry == last: - repeats += 1 - if repeats <= 3: + + def _write_frames(self, traceback_obj, table, sources, map_dir, write): + '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a "[Previous line repeated N more times]" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files.\n ''' + cache = {} + last = None + repeats = 0 + + def emit(entry): + write(' File "%s", line %d, in %s\n' % entry) + key = entry[0], entry[1] + if key not in cache: + cache[key] = self._source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + if filename == self._bundle: + mapped = table.get(lineno) + if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + source = sources[mapped[0]] + if not self._os.path.isabs(source): + source = self._os.path.normpath(self._os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] + entry = filename, lineno, name + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 emit(entry) - else: - if repeats > 3: - write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - last = entry - repeats = 1 - emit(entry) - traceback_obj = traceback_obj.tb_next - if repeats > 3: - write(" [Previous line repeated %d more times]\n" % (repeats - 3)) -def _cribo_sm_exception_line(exc_value): - exc_type = type(exc_value) - name = getattr(exc_type, "__qualname__", exc_type.__name__) - module = getattr(exc_type, "__module__", None) - if module not in (None, "builtins", "__main__"): - name = "%s.%s" % (module, name) - try: - text = str(exc_value) - except BaseException: - text = "" - return "%s: %s\n" % (name, text) if text else "%s\n" % name -def _cribo_sm_render(exc_value, table, sources, map_dir, write): - """Render the exception (with its cause/context chain) like CPython does.""" - chain = [] - exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen and len(chain) < 16: - seen.add(id(exc)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - suppress = getattr(exc, "__suppress_context__", False) - if cause is not None: - chain.append((exc, "cause")) - exc = cause - elif context is not None and not suppress: - chain.append((exc, "context")) - exc = context - else: - chain.append((exc, None)) - exc = None - ordered = list(reversed(chain)) - for index, (exc, link) in enumerate(ordered): - if index > 0: - if link == "cause": - write("\nThe above exception was the direct cause of the following exception:\n\n") + traceback_obj = traceback_obj.tb_next + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + + def _exception_line(self, exc_value): + exc_type = type(exc_value) + name = getattr(exc_type, "__qualname__", exc_type.__name__) + module = getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = str(exc_value) + except BaseException: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name + + def _render(self, exc_value, table, sources, map_dir, write): + """Render the exception (with its cause/context chain) like CPython.""" + chain = [] + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(chain) < 16: + seen.add(id(exc)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + suppress = getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context else: - write("\nDuring handling of the above exception, another exception occurred:\n\n") - tb = getattr(exc, "__traceback__", None) - if tb is not None: - write("Traceback (most recent call last):\n") - _cribo_sm_write_frames(tb, table, sources, map_dir, write) - write(_cribo_sm_exception_line(exc)) -def _cribo_sm_try_render(exc_value, traceback_obj, prefix): - """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in this\n runtime returns False so callers can delegate to the previous hook.\n """ - if _CRIBO_SM_STATE["in_hook"] or exc_value is None: - return False - _CRIBO_SM_STATE["in_hook"] = True - old_limit = None - try: - needed = _cribo_sm_collect_needed(exc_value, traceback_obj) - if not needed: + chain.append((exc, None)) + exc = None + ordered = list(reversed(chain)) + for index, (exc, link) in enumerate(ordered): + if index > 0: + if link == "cause": + write("\nThe above exception was the direct cause of the following " "exception:\n\n") + else: + write("\nDuring handling of the above exception, another exception " "occurred:\n\n") + tb = getattr(exc, "__traceback__", None) + if tb is not None: + write("Traceback (most recent call last):\n") + self._write_frames(tb, table, sources, map_dir, write) + write(self._exception_line(exc)) + + def _try_render(self, exc_value, traceback_obj, prefix): + """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ + if getattr(self._local, "in_hook", False) or exc_value is None: return False - loaded = None + self._local.in_hook = True + old_limit = None try: - loaded = _cribo_sm_load(needed) - except BaseException: + if self._chain_has_group(exc_value): + return False + needed = self._collect_needed(exc_value, traceback_obj) + if not needed: + return False + loaded = None try: - loaded = _cribo_sm_load_json_fallback(needed) + loaded = self._load(needed) except BaseException: - loaded = None - if not loaded: - return False - table, sources, map_dir = loaded - if not table: - return False - try: - old_limit = _cribo_sys.getrecursionlimit() - _cribo_sys.setrecursionlimit(old_limit + 64) - except BaseException: - old_limit = None - parts = [] - if prefix: - parts.append(prefix) - _cribo_sm_render(exc_value, table, sources, map_dir, parts.append) - stderr = _cribo_sys.stderr - stderr.write("".join(parts)) - try: - stderr.flush() + try: + loaded = self._load_json_fallback(needed) + except BaseException: + loaded = None + if not loaded: + return False + table, sources, map_dir = loaded + if not table: + return False + try: + old_limit = self._sys.getrecursionlimit() + self._sys.setrecursionlimit(old_limit + 64) + except BaseException: + old_limit = None + parts = [] + if prefix: + parts.append(prefix) + self._render(exc_value, table, sources, map_dir, parts.append) + stderr = self._sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except BaseException: + pass + return True except BaseException: - pass - return True - except BaseException: - return False - finally: - if old_limit is not None: + return False + finally: + if old_limit is not None: + try: + self._sys.setrecursionlimit(old_limit) + except BaseException: + pass + self._local.in_hook = False + + def _notify_custom_hook(self, prev, default, call): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. When the interpreter\n default is unavailable for comparison (e.g. `threading.__excepthook__`\n before Python 3.10), no notification happens — better to skip a custom\n hook than to double-print via the default one.\n """ + if default is not None and prev is not None and prev is not default: try: - _cribo_sys.setrecursionlimit(old_limit) + call(prev) except BaseException: pass - _CRIBO_SM_STATE["in_hook"] = False -def _cribo_sm_excepthook(exc_type, exc_value, traceback_obj): - if not _cribo_sm_try_render(exc_value, traceback_obj, None): - _cribo_sm_prev_excepthook(exc_type, exc_value, traceback_obj) -def _cribo_sm_threading_hook(args): - thread = getattr(args, "thread", None) - name = getattr(thread, "name", None) or "Thread" - prefix = "Exception in thread %s:\n" % name - if not _cribo_sm_try_render(args.exc_value, args.exc_traceback, prefix): - _cribo_sm_prev_threading_hook(args) -def _cribo_sm_unraisablehook(unraisable): - message = getattr(unraisable, "err_msg", None) or "Exception ignored in" - try: - prefix = "%s: %r\n" % (message, unraisable.object) - except BaseException: - prefix = "%s\n" % message - if not _cribo_sm_try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - _cribo_sm_prev_unraisablehook(unraisable) -_cribo_sys.excepthook = _cribo_sm_excepthook -_cribo_sys.unraisablehook = _cribo_sm_unraisablehook -_cribo_threading.excepthook = _cribo_sm_threading_hook + + def excepthook(self, exc_type, exc_value, traceback_obj): + if self._try_render(exc_value, traceback_obj, None): + self._notify_custom_hook(self._prev_excepthook, self._sys.__excepthook__, lambda hook: hook(exc_type, exc_value, traceback_obj)) + return + self._prev_excepthook(exc_type, exc_value, traceback_obj) + + def threading_hook(self, args): + if args.exc_type is not None and issubclass(args.exc_type, SystemExit): + self._prev_threading_hook(args) + return + thread = getattr(args, "thread", None) + name = getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if self._try_render(args.exc_value, args.exc_traceback, prefix): + self._notify_custom_hook(self._prev_threading_hook, getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) + return + self._prev_threading_hook(args) + + def unraisablehook(self, unraisable): + message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except BaseException: + prefix = "%s\n" % message + if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) + return + self._prev_unraisablehook(unraisable) +_CriboSourceMapRuntime("linked", globals().get("__file__", "")).install() import sys as _sys import importlib as _importlib class _CriboModule(): diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 363b32253..81aa8902d 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -6,97 +6,10 @@ input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py # Generated by Cribo - Python Source Bundler # https://github.com/ophidiarium/cribo -"""Cribo source map runtime (injected prologue).\n\nRemaps tracebacks of uncaught exceptions back to the original source files\nusing the Source Map v3 emitted at bundle time. Lazy by design: no file I/O,\nparsing, or decoding happens at import time; everything is deferred to the\nfirst uncaught exception. Under resource pressure the decoder streams the map\nin constant memory and falls back to the default traceback on any failure.\nSee docs/source-maps.md in the cribo repository.\n""" import binascii as _cribo_binascii import os as _cribo_os import sys as _cribo_sys import threading as _cribo_threading -_CRIBO_SM_MODE = "linked" -_CRIBO_SM_BUNDLE = globals().get("__file__", "") -_CRIBO_SM_CHUNK = 8192 -_CRIBO_SM_STATE = {"in_hook": False} -_CRIBO_SM_B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" -_cribo_sm_prev_excepthook = _cribo_sys.excepthook -_cribo_sm_prev_unraisablehook = _cribo_sys.unraisablehook -_cribo_sm_prev_threading_hook = _cribo_threading.excepthook -def _cribo_sm_map_location(): - """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time so\n the happy path never touches the environment or the filesystem.\n """ - env = _cribo_os.environ.get("CRIBO_SOURCE_MAPS", "") - if env == "0": - return None - bundle = _CRIBO_SM_BUNDLE - if _CRIBO_SM_MODE == "inline": - return None, _cribo_os.path.dirname(_cribo_os.path.abspath(bundle)) - sibling = bundle + ".map" - if _CRIBO_SM_MODE == "linked": - if _cribo_os.path.exists(sibling): - return sibling, _cribo_os.path.dirname(_cribo_os.path.abspath(sibling)) - return None - if env in ("1", "true", "yes", "on"): - path = sibling - elif env: - path = env - else: - return None - return path, _cribo_os.path.dirname(_cribo_os.path.abspath(path)) -def _cribo_sm_file_chunks(path): - """Yield fixed-size chunks of a file (constant memory).""" - handle = open(path, "rb") - try: - while True: - chunk = handle.read(_CRIBO_SM_CHUNK) - if not chunk: - break - yield chunk - finally: - handle.close() -def _cribo_sm_find_inline_payload(handle): - """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of the\n file is examined; the bundle body is never read.\n """ - marker = b"# sourceMappingURL=data:" - handle.seek(0, 2) - position = handle.tell() - overlap = b"" - found = -1 - while position > 0: - step = _CRIBO_SM_CHUNK if position >= _CRIBO_SM_CHUNK else position - position -= step - handle.seek(position) - data = handle.read(step) + overlap - index = data.rfind(marker) - if index >= 0: - found = position + index - break - overlap = data[:len(marker) - 1] - if found < 0: - return -1 - handle.seek(found) - head = handle.read(192) - base64_at = head.find(b"base64,") - if base64_at < 0: - return -1 - return found + base64_at + len(b"base64,") -def _cribo_sm_inline_chunks(path): - """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = open(path, "rb") - try: - start = _cribo_sm_find_inline_payload(handle) - if start < 0: - return - handle.seek(start) - pending = b"" - while True: - raw = handle.read(_CRIBO_SM_CHUNK) - if not raw: - break - data = pending + raw.translate(None, b"\r\n") - usable = len(data) - len(data) % 4 - pending = data[usable:] - if usable: - yield _cribo_binascii.a2b_base64(data[:usable]) - if pending: - yield _cribo_binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) - finally: - handle.close() class _CriboSmStream(object): """Byte-at-a-time reader over an iterator of byte chunks.""" __slots__ = "_chunks", "_buf", "_pos" @@ -116,391 +29,550 @@ class _CriboSmStream(object): value = self._buf[self._pos] self._pos += 1 return value -def _cribo_sm_skip_ws(stream, byte): - while byte in (32, 9, 10, 13): - byte = stream.read_byte() - return byte -def _cribo_sm_read_string(stream, collect): - """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are handled,\n so string *values* containing text like '\"mappings\":' cannot confuse the\n key scanner.\n """ - buf = bytearray() if collect else None - while True: - byte = stream.read_byte() - if byte < 0: - raise ValueError("unterminated JSON string") - if byte == 34: - return buf.decode("utf-8", "replace") if collect else None - if byte != 92: - if buf is not None: - buf.append(byte) - continue - escape = stream.read_byte() - if escape < 0: - raise ValueError("unterminated JSON escape") - if escape == 117: - code = 0 - for _ in range(4): - digit = stream.read_byte() - if digit < 0: - raise ValueError("unterminated unicode escape") - code = code * 16 + int(chr(digit), 16) - if buf is not None: - buf.extend(chr(code).encode("utf-8", "surrogatepass")) - elif buf is not None: - table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} - buf.append(table.get(escape, escape)) -def _cribo_sm_skip_value(stream, byte): - """Skip one JSON value; return the first byte after it (or -1).""" - if byte == 34: - _cribo_sm_read_string(stream, False) - return stream.read_byte() - if byte in (123, 91): - depth = 1 - while depth > 0: +class _CriboSourceMapRuntime(object): + """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, so the installed hooks keep working even\n if bundled user code later rebinds any module-level name this template\n introduced.\n """ + _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + _CHUNK = 8192 + + def __init__(self, mode, bundle_file): + self._mode = mode + self._bundle = bundle_file + self._os = _cribo_os + self._sys = _cribo_sys + self._binascii = _cribo_binascii + self._threading = _cribo_threading + self._stream_cls = _CriboSmStream + self._local = _cribo_threading.local() + self._prev_excepthook = _cribo_sys.excepthook + self._prev_unraisablehook = _cribo_sys.unraisablehook + self._prev_threading_hook = _cribo_threading.excepthook + try: + self._group_type = BaseExceptionGroup + except NameError: + self._group_type = None + + def install(self): + """Install the three hooks; the previous hooks stay chained.""" + self._sys.excepthook = self.excepthook + self._sys.unraisablehook = self.unraisablehook + self._threading.excepthook = self.threading_hook + + def _map_location(self): + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time\n so the happy path never touches the environment or the filesystem.\n """ + env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") + if env == "0": + return None + if env not in ("", "1", "true", "yes", "on"): + path = env + return path, self._os.path.dirname(self._os.path.abspath(path)) + bundle = self._bundle + if self._mode == "inline": + if bundle == "": + return None + return None, self._os.path.dirname(self._os.path.abspath(bundle)) + sibling = bundle + ".map" + if self._mode == "linked": + if self._os.path.exists(sibling): + return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return None + if env in ("1", "true", "yes", "on"): + return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return None + + def _file_chunks(self, path): + """Yield fixed-size chunks of a file (constant memory).""" + handle = open(path, "rb") + try: + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + finally: + handle.close() + + def _find_inline_payload(self, handle): + """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of\n the file is examined; the bundle body is never read.\n """ + marker = b"# sourceMappingURL=data:" + handle.seek(0, 2) + position = handle.tell() + overlap = b"" + found = -1 + while position > 0: + step = self._CHUNK if position >= self._CHUNK else position + position -= step + handle.seek(position) + data = handle.read(step) + overlap + index = data.rfind(marker) + if index >= 0: + found = position + index + break + overlap = data[:len(marker) - 1] + if found < 0: + return -1 + handle.seek(found) + head = handle.read(192) + base64_at = head.find(b"base64,") + if base64_at < 0: + return -1 + return found + base64_at + len(b"base64,") + + def _inline_chunks(self, path): + """Yield decoded chunks of an inline (base64 data URL) source map.""" + handle = open(path, "rb") + try: + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = len(data) - len(data) % 4 + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + finally: + handle.close() + + def _skip_ws(self, stream, byte): + while byte in (32, 9, 10, 13): + byte = stream.read_byte() + return byte + + def _read_string(self, stream, collect): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ + buf = bytearray() if collect else None + while True: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON container") + raise ValueError("unterminated JSON string") if byte == 34: - _cribo_sm_read_string(stream, False) - elif byte in (123, 91): - depth += 1 - elif byte in (125, 93): - depth -= 1 - return stream.read_byte() - while byte >= 0 and byte not in (44, 125, 93): - byte = stream.read_byte() - return byte -def _cribo_sm_read_string_array(stream, byte): - """Read a JSON array of strings/nulls; return (list, byte after array).""" - if byte != 91: - raise ValueError("expected array") - items = [] - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte == 93: - return items, stream.read_byte() - while True: + return buf.decode("utf-8", "replace") if collect else None + if byte != 92: + if buf is not None: + buf.append(byte) + continue + escape = stream.read_byte() + if escape < 0: + raise ValueError("unterminated JSON escape") + if escape == 117: + code = 0 + for _ in range(4): + digit = stream.read_byte() + if digit < 0: + raise ValueError("unterminated unicode escape") + code = code * 16 + int(chr(digit), 16) + if buf is not None: + buf.extend(chr(code).encode("utf-8", "surrogatepass")) + elif buf is not None: + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} + buf.append(table.get(escape, escape)) + + def _skip_value(self, stream, byte): + """Skip one JSON value; return the first byte after it (or -1).""" if byte == 34: - items.append(_cribo_sm_read_string(stream, True)) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - else: - items.append(None) - byte = _cribo_sm_skip_ws(stream, _cribo_sm_skip_value(stream, byte)) + self._read_string(stream, False) + return stream.read_byte() + if byte in (123, 91): + depth = 1 + while depth > 0: + byte = stream.read_byte() + if byte < 0: + raise ValueError("unterminated JSON container") + if byte == 34: + self._read_string(stream, False) + elif byte in (123, 91): + depth += 1 + elif byte in (125, 93): + depth -= 1 + return stream.read_byte() + while byte >= 0 and byte not in (44, 125, 93): + byte = stream.read_byte() + return byte + + def _read_string_array(self, stream, byte): + """Read a JSON array of strings/nulls; return (list, byte after array).""" + if byte != 91: + raise ValueError("expected array") + items = [] + byte = self._skip_ws(stream, stream.read_byte()) if byte == 93: return items, stream.read_byte() - if byte != 44: - raise ValueError("malformed array") - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) -def _cribo_sm_decode_vlq(stream, needed, max_needed): - """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ - lut = {} - for index in range(64): - lut[_CRIBO_SM_B64[index]] = index - result = {} - gen_line = 0 - src_idx = 0 - src_line = 0 - field = 0 - vlq_value = 0 - vlq_shift = 0 - - def end_segment(): - if field >= 4 and gen_line in needed and gen_line not in result: - result[gen_line] = src_idx, src_line - while True: - byte = stream.read_byte() - if byte < 0 or byte == 34: - end_segment() - return result - if byte == 59: - end_segment() - gen_line += 1 - field = 0 - if gen_line > max_needed or len(result) == len(needed): - return result - continue - if byte == 44: - end_segment() - field = 0 - continue - value = lut.get(byte) - if value is None: - raise ValueError("unexpected byte in mappings") - vlq_value += (value & 31) << vlq_shift - if value & 32: - vlq_shift += 5 - continue - signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 - if field == 1: - src_idx += signed - elif field == 2: - src_line += signed - field += 1 + while True: + if byte == 34: + items.append(self._read_string(stream, True)) + byte = self._skip_ws(stream, stream.read_byte()) + else: + items.append(None) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 93: + return items, stream.read_byte() + if byte != 44: + raise ValueError("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) + + def _decode_vlq(self, stream, needed, max_needed): + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ + lut = {} + for index in range(64): + lut[self._B64[index]] = index + result = {} + gen_line = 0 + src_idx = 0 + src_line = 0 + field = 0 vlq_value = 0 vlq_shift = 0 -def _cribo_sm_scan(chunks, needed, max_needed): - """Scan the map's top-level object; return (sources, line table).""" - stream = _CriboSmStream(chunks) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte != 123: - raise ValueError("not a JSON object") - sources = [] - table = {} - saw_mappings = False - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - while byte == 34: - key = _cribo_sm_read_string(stream, True) - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if byte != 58: - raise ValueError("malformed object") - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if key == "sources": - sources, byte = _cribo_sm_read_string_array(stream, byte) - elif key == "mappings": - if byte != 34: - raise ValueError("mappings is not a string") - table = _cribo_sm_decode_vlq(stream, needed, max_needed) - saw_mappings = True - if sources: - break + + def end_segment(): + if field >= 4 and gen_line in needed and gen_line not in result: + result[gen_line] = src_idx, src_line + while True: byte = stream.read_byte() + if byte < 0 or byte == 34: + end_segment() + return result + if byte == 59: + end_segment() + gen_line += 1 + field = 0 + if gen_line > max_needed or len(result) == len(needed): + return result + continue + if byte == 44: + end_segment() + field = 0 + continue + value = lut.get(byte) + if value is None: + raise ValueError("unexpected byte in mappings") + vlq_value += (value & 31) << vlq_shift + if value & 32: + vlq_shift += 5 + continue + signed = -(vlq_value >> 1) if vlq_value & 1 else vlq_value >> 1 + if field == 1: + src_idx += signed + elif field == 2: + src_line += signed + field += 1 + vlq_value = 0 + vlq_shift = 0 + + def _scan(self, chunks, needed, max_needed): + """Scan the map's top-level object; return (sources, line table).""" + stream = self._stream_cls(chunks) + byte = self._skip_ws(stream, stream.read_byte()) + if byte != 123: + raise ValueError("not a JSON object") + sources = [] + table = {} + saw_mappings = False + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if byte != 58: + raise ValueError("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + sources, byte = self._read_string_array(stream, byte) + elif key == "mappings": + if byte != 34: + raise ValueError("mappings is not a string") + table = self._decode_vlq(stream, needed, max_needed) + saw_mappings = True + if sources: + break + byte = stream.read_byte() + else: + byte = self._skip_value(stream, byte) + byte = self._skip_ws(stream, byte) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + if not saw_mappings: + raise ValueError("no mappings field") + return sources, table + + def _load(self, needed_lines): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ + location = self._map_location() + if location is None: + return None + map_path, map_dir = location + needed0 = set(line - 1 for line in needed_lines) + max_needed = max(needed0) + if map_path is None: + chunks = self._inline_chunks(self._bundle) else: - byte = _cribo_sm_skip_value(stream, byte) - byte = _cribo_sm_skip_ws(stream, byte) - if byte == 44: - byte = _cribo_sm_skip_ws(stream, stream.read_byte()) - if not saw_mappings: - raise ValueError("no mappings field") - return sources, table -def _cribo_sm_load(needed_lines): - """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ - location = _cribo_sm_map_location() - if location is None: - return None - map_path, map_dir = location - needed0 = set(line - 1 for line in needed_lines) - max_needed = max(needed0) - if map_path is None: - chunks = _cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE) - else: - chunks = _cribo_sm_file_chunks(map_path) - sources, table0 = _cribo_sm_scan(chunks, needed0, max_needed) - table = {} - for line0, (src_idx, src_line0) in table0.items(): - table[line0 + 1] = src_idx, src_line0 + 1 - return table, sources, map_dir -def _cribo_sm_load_json_fallback(needed_lines): - """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" - location = _cribo_sm_map_location() - if location is None: - return None - map_path, map_dir = location - import json - if map_path is None: - raw = b"".join(_cribo_sm_inline_chunks(_CRIBO_SM_BUNDLE)) - else: - handle = open(map_path, "rb") + chunks = self._file_chunks(map_path) + sources, table0 = self._scan(chunks, needed0, max_needed) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _load_json_fallback(self, needed_lines): + """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" + location = self._map_location() + if location is None: + return None + map_path, map_dir = location + import json + if map_path is None: + raw = b"".join(self._inline_chunks(self._bundle)) + else: + handle = open(map_path, "rb") + try: + raw = handle.read() + finally: + handle.close() + data = json.loads(raw.decode("utf-8")) + sources = data.get("sources") or [] + mappings = data.get("mappings") or "" + needed0 = set(line - 1 for line in needed_lines) + stream = self._stream_cls([mappings.encode("ascii"), b'"']) + table0 = self._decode_vlq(stream, needed0, max(needed0)) + table = {} + for line0, (src_idx, src_line0) in table0.items(): + table[line0 + 1] = src_idx, src_line0 + 1 + return table, sources, map_dir + + def _collect_needed(self, exc_value, traceback_obj): + """1-based bundle lines referenced by the traceback (and its chain).""" + needed = set() + + def add(tb): + while tb is not None: + if tb.tb_frame.f_code.co_filename == self._bundle: + needed.add(tb.tb_lineno) + tb = tb.tb_next + add(traceback_obj) + exc = exc_value + seen = set() + depth = 0 + while exc is not None and id(exc) not in seen and depth < 16: + seen.add(id(exc)) + depth += 1 + add(getattr(exc, "__traceback__", None)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return needed + + def _chain_has_group(self, exc_value): + """Whether the exception chain contains a BaseExceptionGroup.\n\n CPython renders groups with a dedicated nested layout; rather than\n losing the nested tracebacks, the runtime defers group rendering\n entirely to the previous hook (unremapped but complete).\n """ + if self._group_type is None: + return False + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(seen) < 16: + seen.add(id(exc)) + if isinstance(exc, self._group_type): + return True + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + exc = cause if cause is not None else context + return False + + def _source_line(self, path, lineno): + """Read a single 1-based line from a file without caching it.""" + try: + handle = open(path, "rb") + except OSError: + return None try: - raw = handle.read() + current = 0 + for raw in handle: + current += 1 + if current == lineno: + return raw.decode("utf-8", "replace").strip() + if current > lineno: + break + except OSError: + return None finally: handle.close() - data = json.loads(raw.decode("utf-8")) - sources = data.get("sources") or [] - mappings = data.get("mappings") or "" - needed0 = set(line - 1 for line in needed_lines) - stream = _CriboSmStream([mappings.encode("ascii"), b'"']) - table0 = _cribo_sm_decode_vlq(stream, needed0, max(needed0)) - table = {} - for line0, (src_idx, src_line0) in table0.items(): - table[line0 + 1] = src_idx, src_line0 + 1 - return table, sources, map_dir -def _cribo_sm_collect_needed(exc_value, traceback_obj): - """1-based bundle line numbers referenced by the traceback (and its chain).""" - needed = set() - - def add(tb): - while tb is not None: - if tb.tb_frame.f_code.co_filename == _CRIBO_SM_BUNDLE: - needed.add(tb.tb_lineno) - tb = tb.tb_next - add(traceback_obj) - exc = exc_value - seen = set() - depth = 0 - while exc is not None and id(exc) not in seen and depth < 16: - seen.add(id(exc)) - depth += 1 - add(getattr(exc, "__traceback__", None)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - exc = cause if cause is not None else context - return needed -def _cribo_sm_source_line(path, lineno): - """Read a single 1-based line from a file without caching it.""" - try: - handle = open(path, "rb") - except OSError: return None - try: - current = 0 - for raw in handle: - current += 1 - if current == lineno: - return raw.decode("utf-8", "replace").strip() - if current > lineno: - break - except OSError: - return None - finally: - handle.close() - return None -def _cribo_sm_write_frames(traceback_obj, table, sources, map_dir, write): - '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed by\n a "[Previous line repeated N more times]" marker; source line text is\n cached per (file, line) within one rendering to avoid re-reading files.\n ''' - cache = {} - last = None - repeats = 0 - - def emit(entry): - write(' File "%s", line %d, in %s\n' % entry) - key = entry[0], entry[1] - if key not in cache: - cache[key] = _cribo_sm_source_line(entry[0], entry[1]) - if cache[key]: - write(" %s\n" % cache[key]) - while traceback_obj is not None: - frame = traceback_obj.tb_frame - filename = frame.f_code.co_filename - lineno = traceback_obj.tb_lineno - name = frame.f_code.co_name - if filename == _CRIBO_SM_BUNDLE: - mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: - source = sources[mapped[0]] - if not _cribo_os.path.isabs(source): - source = _cribo_os.path.normpath(_cribo_os.path.join(map_dir, source)) - filename, lineno = source, mapped[1] - entry = filename, lineno, name - if entry == last: - repeats += 1 - if repeats <= 3: + + def _write_frames(self, traceback_obj, table, sources, map_dir, write): + '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a "[Previous line repeated N more times]" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files.\n ''' + cache = {} + last = None + repeats = 0 + + def emit(entry): + write(' File "%s", line %d, in %s\n' % entry) + key = entry[0], entry[1] + if key not in cache: + cache[key] = self._source_line(entry[0], entry[1]) + if cache[key]: + write(" %s\n" % cache[key]) + while traceback_obj is not None: + frame = traceback_obj.tb_frame + filename = frame.f_code.co_filename + lineno = traceback_obj.tb_lineno + name = frame.f_code.co_name + if filename == self._bundle: + mapped = table.get(lineno) + if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + source = sources[mapped[0]] + if not self._os.path.isabs(source): + source = self._os.path.normpath(self._os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] + entry = filename, lineno, name + if entry == last: + repeats += 1 + if repeats <= 3: + emit(entry) + else: + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + last = entry + repeats = 1 emit(entry) - else: - if repeats > 3: - write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - last = entry - repeats = 1 - emit(entry) - traceback_obj = traceback_obj.tb_next - if repeats > 3: - write(" [Previous line repeated %d more times]\n" % (repeats - 3)) -def _cribo_sm_exception_line(exc_value): - exc_type = type(exc_value) - name = getattr(exc_type, "__qualname__", exc_type.__name__) - module = getattr(exc_type, "__module__", None) - if module not in (None, "builtins", "__main__"): - name = "%s.%s" % (module, name) - try: - text = str(exc_value) - except BaseException: - text = "" - return "%s: %s\n" % (name, text) if text else "%s\n" % name -def _cribo_sm_render(exc_value, table, sources, map_dir, write): - """Render the exception (with its cause/context chain) like CPython does.""" - chain = [] - exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen and len(chain) < 16: - seen.add(id(exc)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - suppress = getattr(exc, "__suppress_context__", False) - if cause is not None: - chain.append((exc, "cause")) - exc = cause - elif context is not None and not suppress: - chain.append((exc, "context")) - exc = context - else: - chain.append((exc, None)) - exc = None - ordered = list(reversed(chain)) - for index, (exc, link) in enumerate(ordered): - if index > 0: - if link == "cause": - write("\nThe above exception was the direct cause of the following exception:\n\n") + traceback_obj = traceback_obj.tb_next + if repeats > 3: + write(" [Previous line repeated %d more times]\n" % (repeats - 3)) + + def _exception_line(self, exc_value): + exc_type = type(exc_value) + name = getattr(exc_type, "__qualname__", exc_type.__name__) + module = getattr(exc_type, "__module__", None) + if module not in (None, "builtins", "__main__"): + name = "%s.%s" % (module, name) + try: + text = str(exc_value) + except BaseException: + text = "" + return "%s: %s\n" % (name, text) if text else "%s\n" % name + + def _render(self, exc_value, table, sources, map_dir, write): + """Render the exception (with its cause/context chain) like CPython.""" + chain = [] + exc = exc_value + seen = set() + while exc is not None and id(exc) not in seen and len(chain) < 16: + seen.add(id(exc)) + cause = getattr(exc, "__cause__", None) + context = getattr(exc, "__context__", None) + suppress = getattr(exc, "__suppress_context__", False) + if cause is not None: + chain.append((exc, "cause")) + exc = cause + elif context is not None and not suppress: + chain.append((exc, "context")) + exc = context else: - write("\nDuring handling of the above exception, another exception occurred:\n\n") - tb = getattr(exc, "__traceback__", None) - if tb is not None: - write("Traceback (most recent call last):\n") - _cribo_sm_write_frames(tb, table, sources, map_dir, write) - write(_cribo_sm_exception_line(exc)) -def _cribo_sm_try_render(exc_value, traceback_obj, prefix): - """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in this\n runtime returns False so callers can delegate to the previous hook.\n """ - if _CRIBO_SM_STATE["in_hook"] or exc_value is None: - return False - _CRIBO_SM_STATE["in_hook"] = True - old_limit = None - try: - needed = _cribo_sm_collect_needed(exc_value, traceback_obj) - if not needed: + chain.append((exc, None)) + exc = None + ordered = list(reversed(chain)) + for index, (exc, link) in enumerate(ordered): + if index > 0: + if link == "cause": + write("\nThe above exception was the direct cause of the following " "exception:\n\n") + else: + write("\nDuring handling of the above exception, another exception " "occurred:\n\n") + tb = getattr(exc, "__traceback__", None) + if tb is not None: + write("Traceback (most recent call last):\n") + self._write_frames(tb, table, sources, map_dir, write) + write(self._exception_line(exc)) + + def _try_render(self, exc_value, traceback_obj, prefix): + """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ + if getattr(self._local, "in_hook", False) or exc_value is None: return False - loaded = None + self._local.in_hook = True + old_limit = None try: - loaded = _cribo_sm_load(needed) - except BaseException: + if self._chain_has_group(exc_value): + return False + needed = self._collect_needed(exc_value, traceback_obj) + if not needed: + return False + loaded = None try: - loaded = _cribo_sm_load_json_fallback(needed) + loaded = self._load(needed) except BaseException: - loaded = None - if not loaded: - return False - table, sources, map_dir = loaded - if not table: - return False - try: - old_limit = _cribo_sys.getrecursionlimit() - _cribo_sys.setrecursionlimit(old_limit + 64) - except BaseException: - old_limit = None - parts = [] - if prefix: - parts.append(prefix) - _cribo_sm_render(exc_value, table, sources, map_dir, parts.append) - stderr = _cribo_sys.stderr - stderr.write("".join(parts)) - try: - stderr.flush() + try: + loaded = self._load_json_fallback(needed) + except BaseException: + loaded = None + if not loaded: + return False + table, sources, map_dir = loaded + if not table: + return False + try: + old_limit = self._sys.getrecursionlimit() + self._sys.setrecursionlimit(old_limit + 64) + except BaseException: + old_limit = None + parts = [] + if prefix: + parts.append(prefix) + self._render(exc_value, table, sources, map_dir, parts.append) + stderr = self._sys.stderr + stderr.write("".join(parts)) + try: + stderr.flush() + except BaseException: + pass + return True except BaseException: - pass - return True - except BaseException: - return False - finally: - if old_limit is not None: + return False + finally: + if old_limit is not None: + try: + self._sys.setrecursionlimit(old_limit) + except BaseException: + pass + self._local.in_hook = False + + def _notify_custom_hook(self, prev, default, call): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. When the interpreter\n default is unavailable for comparison (e.g. `threading.__excepthook__`\n before Python 3.10), no notification happens — better to skip a custom\n hook than to double-print via the default one.\n """ + if default is not None and prev is not None and prev is not default: try: - _cribo_sys.setrecursionlimit(old_limit) + call(prev) except BaseException: pass - _CRIBO_SM_STATE["in_hook"] = False -def _cribo_sm_excepthook(exc_type, exc_value, traceback_obj): - if not _cribo_sm_try_render(exc_value, traceback_obj, None): - _cribo_sm_prev_excepthook(exc_type, exc_value, traceback_obj) -def _cribo_sm_threading_hook(args): - thread = getattr(args, "thread", None) - name = getattr(thread, "name", None) or "Thread" - prefix = "Exception in thread %s:\n" % name - if not _cribo_sm_try_render(args.exc_value, args.exc_traceback, prefix): - _cribo_sm_prev_threading_hook(args) -def _cribo_sm_unraisablehook(unraisable): - message = getattr(unraisable, "err_msg", None) or "Exception ignored in" - try: - prefix = "%s: %r\n" % (message, unraisable.object) - except BaseException: - prefix = "%s\n" % message - if not _cribo_sm_try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - _cribo_sm_prev_unraisablehook(unraisable) -_cribo_sys.excepthook = _cribo_sm_excepthook -_cribo_sys.unraisablehook = _cribo_sm_unraisablehook -_cribo_threading.excepthook = _cribo_sm_threading_hook + + def excepthook(self, exc_type, exc_value, traceback_obj): + if self._try_render(exc_value, traceback_obj, None): + self._notify_custom_hook(self._prev_excepthook, self._sys.__excepthook__, lambda hook: hook(exc_type, exc_value, traceback_obj)) + return + self._prev_excepthook(exc_type, exc_value, traceback_obj) + + def threading_hook(self, args): + if args.exc_type is not None and issubclass(args.exc_type, SystemExit): + self._prev_threading_hook(args) + return + thread = getattr(args, "thread", None) + name = getattr(thread, "name", None) or "Thread" + prefix = "Exception in thread %s:\n" % name + if self._try_render(args.exc_value, args.exc_traceback, prefix): + self._notify_custom_hook(self._prev_threading_hook, getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) + return + self._prev_threading_hook(args) + + def unraisablehook(self, unraisable): + message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + try: + prefix = "%s: %r\n" % (message, unraisable.object) + except BaseException: + prefix = "%s\n" % message + if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): + self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) + return + self._prev_unraisablehook(unraisable) +_CriboSourceMapRuntime("linked", globals().get("__file__", "")).install() import sys as _sys import importlib as _importlib class _CriboModule(): diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 368595ddd..3fe98558d 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:625 `def add(a, b):` -> calculator.py:1 -bundle:626 `result = a + b` -> calculator.py:2 -bundle:627 `return result` -> calculator.py:3 -bundle:629 `def multiply(a, b):` -> calculator.py:6 -bundle:630 `result = a * b` -> calculator.py:7 -bundle:631 `return result` -> calculator.py:8 -bundle:636 `def describe(name, value):` -> utils.py:1 -bundle:637 `return f"{name} = {value}"` -> utils.py:2 -bundle:643 `total = add(2, 3)` -> main.py:4 -bundle:644 `product = multiply(total, 4)` -> main.py:5 -bundle:645 `print(describe("total", total))` -> main.py:6 -bundle:646 `print(describe("product", product))` -> main.py:7 +bundle:697 `def add(a, b):` -> calculator.py:1 +bundle:698 `result = a + b` -> calculator.py:2 +bundle:699 `return result` -> calculator.py:3 +bundle:701 `def multiply(a, b):` -> calculator.py:6 +bundle:702 `result = a * b` -> calculator.py:7 +bundle:703 `return result` -> calculator.py:8 +bundle:708 `def describe(name, value):` -> utils.py:1 +bundle:709 `return f"{name} = {value}"` -> utils.py:2 +bundle:715 `total = add(2, 3)` -> main.py:4 +bundle:716 `product = multiply(total, 4)` -> main.py:5 +bundle:717 `print(describe("total", total))` -> main.py:6 +bundle:718 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index aa025a0b2..084075543 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:634 `print("effects module loading")` -> effects.py:1 -bundle:635 `COUNTER = 1` -> effects.py:3 -bundle:638 `def boost(value):` -> effects.py:6 -bundle:639 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:640 `return boosted` -> effects.py:8 -bundle:652 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:653 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:706 `print("effects module loading")` -> effects.py:1 +bundle:707 `COUNTER = 1` -> effects.py:3 +bundle:710 `def boost(value):` -> effects.py:6 +bundle:711 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:712 `return boosted` -> effects.py:8 +bundle:724 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:725 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 496c0fe4d..4e0816ed0 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -12,22 +12,29 @@ use tempfile::TempDir; /// Marker prefix of an inline source map comment. const INLINE_MARKER: &str = "# sourceMappingURL=data:application/json;base64,"; -/// Create a two-module fixture project and return its directory. -fn fixture_project() -> TempDir { +/// Create a project from (file name, content) pairs and return its directory. +fn make_project(files: &[(&str, &str)]) -> TempDir { let dir = TempDir::new().expect("create temp dir"); - fs::write( - dir.path().join("main.py"), - "from helper import greet\n\nprint(greet(\"world\"))\n", - ) - .expect("write main.py"); - fs::write( - dir.path().join("helper.py"), - "def greet(name):\n message = f\"hello {name}\"\n return message\n", - ) - .expect("write helper.py"); + for (name, content) in files { + fs::write(dir.path().join(name), content).expect("write fixture file"); + } dir } +/// Create a two-module fixture project and return its directory. +fn fixture_project() -> TempDir { + make_project(&[ + ( + "main.py", + "from helper import greet\n\nprint(greet(\"world\"))\n", + ), + ( + "helper.py", + "def greet(name):\n message = f\"hello {name}\"\n return message\n", + ), + ]) +} + /// Run the cribo binary with `args`, returning (status success, stdout, stderr). fn run_cribo(args: &[&str]) -> (bool, String, String) { let output = Command::new(env!("CARGO_BIN_EXE_cribo")) @@ -46,10 +53,12 @@ fn parse_map(json: &str) -> oxc_sourcemap::SourceMap<'_> { oxc_sourcemap::SourceMap::from_json_string(json).expect("valid Source Map v3 JSON") } +/// Return the fixture entry path (`main.py`) as a CLI argument string. fn entry_arg(dir: &TempDir) -> String { dir.path().join("main.py").to_string_lossy().into_owned() } +/// Assert the map targets `bundle_file`, lists `helper.py`, and has mappings. fn assert_map_covers_helper(map_json: &str, bundle_file: &str) { let map = parse_map(map_json); assert_eq!(map.get_file(), Some(bundle_file)); @@ -353,18 +362,13 @@ fn config_file_sources_content_key_is_honored() { /// Create a fixture whose entry crashes two calls deep inside helper.py. fn crash_project() -> TempDir { - let dir = TempDir::new().expect("create temp dir"); - fs::write( - dir.path().join("main.py"), - "from helper import boom\n\nboom()\n", - ) - .expect("write main.py"); - fs::write( - dir.path().join("helper.py"), - "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"kaboom\")\n", - ) - .expect("write helper.py"); - dir + make_project(&[ + ("main.py", "from helper import boom\n\nboom()\n"), + ( + "helper.py", + "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"kaboom\")\n", + ), + ]) } /// Bundle the crash project with the given sourcemap argument; return bundle path. @@ -385,6 +389,8 @@ fn bundle_crash_project(dir: &TempDir, sourcemap_arg: &str) -> std::path::PathBu fn run_python(bundle: &Path, envs: &[(&str, &str)]) -> (bool, String, String) { let mut command = Command::new(common::get_python_executable()); command.arg(bundle); + // The env-gating tests require a clean slate; an explicit entry below wins. + command.env_remove("CRIBO_SOURCE_MAPS"); for (key, value) in envs { command.env(key, value); } @@ -534,22 +540,19 @@ fn python_runtime_unit_tests() { output.status.success(), "python runtime unit tests failed:\n{stdout}\n{stderr}" ); - assert!(stdout.contains("ALL 12 RUNTIME TESTS PASSED"), "{stdout}"); + // The harness discovers its tests and reports the count itself; assert the + // sentinel plus a sanity floor instead of duplicating the exact count here. + assert!(stdout.contains("RUNTIME TESTS PASSED"), "{stdout}"); + assert!( + stdout.matches("PASS test_").count() >= 10, + "expected a healthy number of runtime unit tests: {stdout}" + ); } // --------------------------------------------------------------------------- // Full hook coverage, duress conditions, and laziness // --------------------------------------------------------------------------- -/// Create a project from (file name, content) pairs and return its directory. -fn make_project(files: &[(&str, &str)]) -> TempDir { - let dir = TempDir::new().expect("create temp dir"); - for (name, content) in files { - fs::write(dir.path().join(name), content).expect("write fixture file"); - } - dir -} - #[test] fn runtime_remaps_thread_crash() { let dir = make_project(&[ @@ -639,8 +642,9 @@ fn runtime_survives_memory_pressure() { ( "main.py", "import resource\nfrom helper import hoard\n\n_soft, hard = \ - resource.getrlimit(resource.RLIMIT_AS)\n\ - resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, hard))\nhoard()\n", + resource.getrlimit(resource.RLIMIT_AS)\ntarget = 512 * 1024 * 1024\nif hard != \ + resource.RLIM_INFINITY:\n target = min(target, hard)\n\ + resource.setrlimit(resource.RLIMIT_AS, (target, hard))\nhoard()\n", ), ( "helper.py", @@ -704,7 +708,7 @@ fn runtime_falls_back_cleanly_on_fd_exhaustion() { } #[test] -fn runtime_is_lazy_on_happy_path() { +fn runtime_tolerates_broken_map_on_happy_path() { let dir = fixture_project(); // non-throwing project let out = dir.path().join("bundle.py"); let (ok, _, stderr) = run_cribo(&[ @@ -716,8 +720,11 @@ fn runtime_is_lazy_on_happy_path() { ]); assert!(ok, "bundling must succeed: {stderr}"); - // Replace the map with something that would fail loudly on ANY access: - // garbage content and, on Unix, no read permission at all. + // Replace the map with garbage (and drop read permission on Unix, though + // that is a no-op when running as root). The runtime is fail-open, so this + // cannot *prove* the map is never touched — laziness itself is enforced by + // the runtime design (all map access lives behind the hook path). What it + // proves is that a broken or unreadable map never disturbs a successful run. let map_path = dir.path().join("bundle.py.map"); fs::write(&map_path, "NOT JSON {{{").expect("overwrite map"); #[cfg(unix)] @@ -732,6 +739,121 @@ fn runtime_is_lazy_on_happy_path() { assert!(stdout.contains("hello world")); assert!( stderr.is_empty(), - "no map access may happen on the happy path: {stderr}" + "a broken map must not disturb a successful run: {stderr}" + ); +} + +// --------------------------------------------------------------------------- +// Precedence, environment configuration, and hook-chaining behavior +// --------------------------------------------------------------------------- + +#[test] +fn cli_sourcemap_flag_overrides_config_file() { + let dir = fixture_project(); + fs::write(dir.path().join("cribo.toml"), "sourcemap = \"external\"\n").expect("write config"); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--config", + &dir.path().join("cribo.toml").to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bundle.py.map"), + "CLI --sourcemap=linked must override the config file's external mode" + ); +} + +#[test] +fn env_var_enables_sourcemap_generation() { + let dir = fixture_project(); + let out = dir.path().join("bundle.py"); + let output = Command::new(env!("CARGO_BIN_EXE_cribo")) + .args([ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + ]) + .env("CRIBO_SOURCEMAP", "external") + .env("CRIBO_SOURCES_CONTENT", "false") + .output() + .expect("run cribo binary"); + assert!( + output.status.success(), + "bundling must succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")) + .expect("CRIBO_SOURCEMAP env var must enable map emission"); + assert_map_covers_helper(&map_json, "bundle.py"); + assert!( + !map_json.contains("sourcesContent"), + "CRIBO_SOURCES_CONTENT=false must strip embedding" + ); +} + +#[test] +fn runtime_keeps_thread_sys_exit_silent() { + let dir = make_project(&[ + ( + "main.py", + "import sys\nimport threading\n\nworker = threading.Thread(target=lambda: \ + sys.exit(3))\nworker.start()\nworker.join()\nprint(\"done\")\n", + ), + ("helper.py", "unused = True\n"), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!( + ok, + "sys.exit in a worker thread must not fail the process: {stderr}" + ); + assert!(stdout.contains("done")); + assert!( + stderr.is_empty(), + "SystemExit in a thread must stay silent, as with the default hook: {stderr}" + ); +} + +#[test] +fn runtime_notifies_preinstalled_custom_excepthook() { + // A custom excepthook installed before the bundle's prologue (via + // sitecustomize) must still observe the exception after a successful remap. + let dir = crash_project(); + fs::write( + dir.path().join("sitecustomize.py"), + "import sys\n\n_original = sys.excepthook\n\n\ndef reporting_hook(exc_type, exc_value, \ + tb):\n print(\"REPORTER SAW:\", exc_type.__name__, file=sys.stderr)\n\n\nsys.excepthook \ + = reporting_hook\n", + ) + .expect("write sitecustomize"); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + + let mut command = Command::new(common::get_python_executable()); + command.arg(&bundle); + command.env_remove("CRIBO_SOURCE_MAPS"); + // Make usercustomize importable so the custom hook installs before the bundle. + command.env("PYTHONPATH", dir.path()); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert_remapped(&stderr); + assert!( + stderr.contains("REPORTER SAW: ValueError"), + "the preinstalled custom hook must still be notified after a remap: {stderr}" ); } diff --git a/docs/source-maps.md b/docs/source-maps.md index a83c5d05b..0f56b4f71 100644 --- a/docs/source-maps.md +++ b/docs/source-maps.md @@ -228,3 +228,34 @@ Known limitations (documented in the README as well): - Mappings are statement/line-level with column 0 by design; Python tracebacks are line-oriented, so finer columns would add cost without changing the rendered output. + + +## Review-Driven Refinements (PR #570) + +- **The runtime is a single class** (`_CriboSourceMapRuntime`); all collaborators + (modules, previous hooks, state) live on the instance and the installed hooks + are bound methods, so bundled user code rebinding any injected module-level + name cannot break remapping. +- **Custom pre-installed hooks stay notified.** After a successful remap, a + previous `sys.excepthook`/`threading.excepthook`/`sys.unraisablehook` that + differs from the interpreter default is still invoked (error reporters and + `sitecustomize` integrations keep working); the default printer is the only + thing the remap replaces. +- **`SystemExit` in worker threads stays silent**, matching the default + `threading.excepthook`. +- **`ExceptionGroup` chains defer to the previous hook** (complete, unremapped + rendering) rather than losing nested tracebacks. +- **The re-entrancy guard is thread-local.** +- **`CRIBO_SOURCE_MAPS=` works in every mode** and is the supported way to + remap a bundle executed via `python -` (stdin), whose own file cannot be + re-read; without it, inline mode deactivates gracefully for `` bundles. +- **Environment configuration:** `CRIBO_SOURCEMAP=linked|inline|external` and + `CRIBO_SOURCES_CONTENT=` participate in the standard `CRIBO_*` layering. +- **The template's docstring is stripped at injection** so enabling source maps + does not change the bundle's `__doc__`. +- **`elif` headers get their own mappings** (Python reports the header line when + a condition raises), using the condition expression's provenance. +- **Linked/external maps are written only after the bundle write succeeds**, so + a failed run cannot leave a stale map next to an older bundle. +- `relative_path` refuses lexically non-invertible bases (`..` components) and + falls back to the absolute source path. From 73e717396cee31bf5fbdcdfd6bf5b35b6c2b7107 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 17:19:52 +0200 Subject: [PATCH 03/17] fix: address second review round on source map runtime - import os/binascii/threading immune to script-directory shadowing (drop sys.path[0] during the runtime's own imports; fail-open bootstrap wraps construction so no host condition can abort the bundle at startup) - record mappings for except-handler headers via the matcher expression's provenance (CPython reports the header line when a matcher raises) - insert the runtime prologue after the bundle's own leading docstring so __doc__ is preserved - render BaseException.__notes__ after the exception line - honor sys.tracebacklimit: <=0 suppresses the frame listing, a positive value keeps the last N frames like the interpreter's default hook - tests: shadowed threading.py, preserved docstring, tracebacklimit=0, exception notes, and map coverage of elif/except headers Addresses second-round review comments on #570 --- crates/cribo/src/python/sourcemap_runtime.py | 91 +++++++++-- crates/cribo/src/source_map.rs | 42 +++-- .../tests/python/test_sourcemap_runtime.py | 8 +- .../bundled_code@sourcemap_basic.snap | 68 ++++++-- .../bundled_code@sourcemap_wrapper.snap | 68 ++++++-- .../snapshots/source_map@sourcemap_basic.snap | 24 +-- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 148 ++++++++++++++++++ 8 files changed, 389 insertions(+), 74 deletions(-) diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index c3df15cbe..f9d334e37 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -11,10 +11,27 @@ ``__doc__`` is not affected. """ -import binascii as _cribo_binascii -import os as _cribo_os import sys as _cribo_sys -import threading as _cribo_threading + + +def _cribo_sm_import(name): + """Import a stdlib module immune to script-directory shadowing. + + The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a + project file named e.g. `threading.py` sitting next to the bundle would + otherwise shadow the stdlib for this runtime. Already-imported modules are + taken from `sys.modules`; otherwise the import runs with that first path + entry dropped. (`sys` itself is a builtin and can never be shadowed.) + """ + module = _cribo_sys.modules.get(name) + if module is not None: + return module + saved_path = _cribo_sys.path + _cribo_sys.path = list(saved_path[1:]) + try: + return __import__(name) + finally: + _cribo_sys.path = saved_path class _CriboSmStream(object): @@ -51,20 +68,20 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): self._mode = mode self._bundle = bundle_file - self._os = _cribo_os + self._os = os_mod self._sys = _cribo_sys - self._binascii = _cribo_binascii - self._threading = _cribo_threading + self._binascii = binascii_mod + self._threading = threading_mod self._stream_cls = _CriboSmStream # Re-entrancy guard; thread-local so a hook firing on one thread never # disables remapping on another. - self._local = _cribo_threading.local() + self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook - self._prev_threading_hook = _cribo_threading.excepthook + self._prev_threading_hook = threading_mod.excepthook try: self._group_type = BaseExceptionGroup except NameError: # Python < 3.11 @@ -76,6 +93,25 @@ def install(self): self._sys.unraisablehook = self.unraisablehook self._threading.excepthook = self.threading_hook + @classmethod + def _bootstrap(cls, mode, bundle_file): + """Import dependencies safely, construct, and install — fail-open. + + Any failure (however exotic the host environment) leaves the program + running without remapping instead of aborting it at startup. + """ + try: + runtime = cls( + mode, + bundle_file, + _cribo_sm_import("os"), + _cribo_sm_import("binascii"), + _cribo_sm_import("threading"), + ) + runtime.install() + except BaseException: + pass + # -- map location and raw chunk access --------------------------------- def _map_location(self): @@ -473,14 +509,31 @@ def _source_line(self, path, lineno): handle.close() return None - def _write_frames(self, traceback_obj, table, sources, map_dir, write): + def _effective_tb_limit(self): + """The application's `sys.tracebacklimit`, or None when unset/invalid.""" + limit = getattr(self._sys, "tracebacklimit", None) + return limit if isinstance(limit, int) else None + + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): """Write remapped frame lines, collapsing repeated frames like CPython. Consecutive identical frames (recursion) print at most 3 times followed by a "[Previous line repeated N more times]" marker; source line text is cached per (file, line) within one rendering to avoid re-reading - files. + files. A positive `limit` keeps only the last `limit` frames, matching + the interpreter's `sys.tracebacklimit` handling in the default hook. """ + if limit is not None: + total = 0 + probe = traceback_obj + while probe is not None: + total += 1 + probe = probe.tb_next + skip = total - limit + while skip > 0 and traceback_obj is not None: + traceback_obj = traceback_obj.tb_next + skip -= 1 + cache = {} last = None repeats = 0 @@ -570,10 +623,18 @@ def _render(self, exc_value, table, sources, map_dir, write): "occurred:\n\n" ) tb = getattr(exc, "__traceback__", None) - if tb is not None: + limit = self._effective_tb_limit() + if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") - self._write_frames(tb, table, sources, map_dir, write) + self._write_frames(tb, table, sources, map_dir, write, limit) write(self._exception_line(exc)) + notes = getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except BaseException: + pass def _try_render(self, exc_value, traceback_obj, prefix): """Attempt a remapped rendering to stderr; True on success. @@ -695,6 +756,6 @@ def unraisablehook(self, unraisable): self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime( +_CriboSourceMapRuntime._bootstrap( "__CRIBO_SOURCEMAP_MODE__", globals().get("__file__", "") -).install() +) diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index ad000ea5d..c9600c26e 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -368,6 +368,25 @@ impl ParallelWalker<'_> { gen_handler; let ruff_python_ast::ExceptHandler::ExceptHandler(orig_handler) = orig_handler; + // Evaluating an exception matcher can itself raise, and + // Python reports the `except` header line; give the + // header its own mapping via the matcher expression's + // provenance. + if let (Some(_), Some(orig_type)) = + (&gen_handler.type_, &orig_handler.type_) + && let Some((module_ordinal, original_line)) = self.provenance.resolve( + orig_type.node_index().load(), + orig_handler.range().start(), + ) + { + self.records.push(MappingRecord { + generated_line: self + .line_index + .line_of(gen_handler.range().start()), + module_ordinal, + original_line, + }); + } self.walk_body(&gen_handler.body, &orig_handler.body); } } @@ -478,19 +497,17 @@ pub(crate) fn inject_runtime_prologue( match ruff_python_parser::parse_module(&source) { Ok(parsed) => { let mut statements = parsed.into_syntax().body; - // Drop the template's leading docstring: injected at position zero - // it would otherwise become the bundle's module docstring and + // Drop the template's leading docstring: injected near position + // zero it could otherwise become the bundle's module docstring and // change the program's observable `__doc__`. - if statements.first().is_some_and(|stmt| { - matches!( - stmt, - Stmt::Expr(expr) if expr.value.is_string_literal_expr() - ) - }) { + if statements.first().is_some_and(is_docstring) { statements.remove(0); } - let insert_at = bundled_ast - .body + // Insert after the bundle's own docstring (which must stay first to + // remain `__doc__`) and after any `from __future__` imports (which + // must precede all other code). + let mut insert_at = usize::from(bundled_ast.body.first().is_some_and(is_docstring)); + insert_at += bundled_ast.body[insert_at..] .iter() .take_while(|stmt| is_future_import(stmt)) .count(); @@ -511,6 +528,11 @@ fn is_future_import(stmt: &Stmt) -> bool { ) } +/// Whether a statement is a bare string-literal expression (a docstring when leading). +fn is_docstring(stmt: &Stmt) -> bool { + matches!(stmt, Stmt::Expr(expr) if expr.value.is_string_literal_expr()) +} + /// Build the complete Source Map v3 JSON for an emitted bundle. /// /// Re-parses `bundle_text`, extracts statement mappings against `bundled_ast`, diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index fa1572d84..5a45bd2b0 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -26,7 +26,9 @@ def load_runtime(path): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) sys.excepthook, sys.unraisablehook, threading.excepthook = prev_hooks - return module._CriboSourceMapRuntime("external", "") + return module._CriboSourceMapRuntime( + "external", "", os, __import__("binascii"), threading + ) def test_stream_reads_across_chunk_boundaries(rt): @@ -201,7 +203,9 @@ def test_env_path_wins_for_every_mode(rt): previous = os.environ.get("CRIBO_SOURCE_MAPS") try: os.environ["CRIBO_SOURCE_MAPS"] = path - inline_stdin = type(rt)("inline", "") + inline_stdin = type(rt)( + "inline", "", os, __import__("binascii"), threading + ) loaded = inline_stdin._load({1}) assert loaded is not None, "env path must activate a inline bundle" table, sources, _map_dir = loaded diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 782179815..97d5c167b 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -6,10 +6,18 @@ input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py # Generated by Cribo - Python Source Bundler # https://github.com/ophidiarium/cribo -import binascii as _cribo_binascii -import os as _cribo_os import sys as _cribo_sys -import threading as _cribo_threading +def _cribo_sm_import(name): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with that first path\n entry dropped. (`sys` itself is a builtin and can never be shadowed.)\n """ + module = _cribo_sys.modules.get(name) + if module is not None: + return module + saved_path = _cribo_sys.path + _cribo_sys.path = list(saved_path[1:]) + try: + return __import__(name) + finally: + _cribo_sys.path = saved_path class _CriboSmStream(object): """Byte-at-a-time reader over an iterator of byte chunks.""" __slots__ = "_chunks", "_buf", "_pos" @@ -34,18 +42,18 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): self._mode = mode self._bundle = bundle_file - self._os = _cribo_os + self._os = os_mod self._sys = _cribo_sys - self._binascii = _cribo_binascii - self._threading = _cribo_threading + self._binascii = binascii_mod + self._threading = threading_mod self._stream_cls = _CriboSmStream - self._local = _cribo_threading.local() + self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook - self._prev_threading_hook = _cribo_threading.excepthook + self._prev_threading_hook = threading_mod.excepthook try: self._group_type = BaseExceptionGroup except NameError: @@ -57,6 +65,15 @@ class _CriboSourceMapRuntime(object): self._sys.unraisablehook = self.unraisablehook self._threading.excepthook = self.threading_hook + @classmethod + def _bootstrap(cls, mode, bundle_file): + """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ + try: + runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading")) + runtime.install() + except BaseException: + pass + def _map_location(self): """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time\n so the happy path never touches the environment or the filesystem.\n """ env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") @@ -401,8 +418,23 @@ class _CriboSourceMapRuntime(object): handle.close() return None - def _write_frames(self, traceback_obj, table, sources, map_dir, write): - '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a "[Previous line repeated N more times]" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files.\n ''' + def _effective_tb_limit(self): + """The application's `sys.tracebacklimit`, or None when unset/invalid.""" + limit = getattr(self._sys, "tracebacklimit", None) + return limit if isinstance(limit, int) else None + + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): + """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ + if limit is not None: + total = 0 + probe = traceback_obj + while probe is not None: + total += 1 + probe = probe.tb_next + skip = total - limit + while skip > 0 and traceback_obj is not None: + traceback_obj = traceback_obj.tb_next + skip -= 1 cache = {} last = None repeats = 0 @@ -480,10 +512,18 @@ class _CriboSourceMapRuntime(object): else: write("\nDuring handling of the above exception, another exception " "occurred:\n\n") tb = getattr(exc, "__traceback__", None) - if tb is not None: + limit = self._effective_tb_limit() + if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") - self._write_frames(tb, table, sources, map_dir, write) + self._write_frames(tb, table, sources, map_dir, write, limit) write(self._exception_line(exc)) + notes = getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except BaseException: + pass def _try_render(self, exc_value, traceback_obj, prefix): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ @@ -572,7 +612,7 @@ class _CriboSourceMapRuntime(object): self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime("linked", globals().get("__file__", "")).install() +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) import sys as _sys import importlib as _importlib class _CriboModule(): diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 81aa8902d..72c0e2ec2 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -6,10 +6,18 @@ input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py # Generated by Cribo - Python Source Bundler # https://github.com/ophidiarium/cribo -import binascii as _cribo_binascii -import os as _cribo_os import sys as _cribo_sys -import threading as _cribo_threading +def _cribo_sm_import(name): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with that first path\n entry dropped. (`sys` itself is a builtin and can never be shadowed.)\n """ + module = _cribo_sys.modules.get(name) + if module is not None: + return module + saved_path = _cribo_sys.path + _cribo_sys.path = list(saved_path[1:]) + try: + return __import__(name) + finally: + _cribo_sys.path = saved_path class _CriboSmStream(object): """Byte-at-a-time reader over an iterator of byte chunks.""" __slots__ = "_chunks", "_buf", "_pos" @@ -34,18 +42,18 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): self._mode = mode self._bundle = bundle_file - self._os = _cribo_os + self._os = os_mod self._sys = _cribo_sys - self._binascii = _cribo_binascii - self._threading = _cribo_threading + self._binascii = binascii_mod + self._threading = threading_mod self._stream_cls = _CriboSmStream - self._local = _cribo_threading.local() + self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook - self._prev_threading_hook = _cribo_threading.excepthook + self._prev_threading_hook = threading_mod.excepthook try: self._group_type = BaseExceptionGroup except NameError: @@ -57,6 +65,15 @@ class _CriboSourceMapRuntime(object): self._sys.unraisablehook = self.unraisablehook self._threading.excepthook = self.threading_hook + @classmethod + def _bootstrap(cls, mode, bundle_file): + """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ + try: + runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading")) + runtime.install() + except BaseException: + pass + def _map_location(self): """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time\n so the happy path never touches the environment or the filesystem.\n """ env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") @@ -401,8 +418,23 @@ class _CriboSourceMapRuntime(object): handle.close() return None - def _write_frames(self, traceback_obj, table, sources, map_dir, write): - '''Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a "[Previous line repeated N more times]" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files.\n ''' + def _effective_tb_limit(self): + """The application's `sys.tracebacklimit`, or None when unset/invalid.""" + limit = getattr(self._sys, "tracebacklimit", None) + return limit if isinstance(limit, int) else None + + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): + """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ + if limit is not None: + total = 0 + probe = traceback_obj + while probe is not None: + total += 1 + probe = probe.tb_next + skip = total - limit + while skip > 0 and traceback_obj is not None: + traceback_obj = traceback_obj.tb_next + skip -= 1 cache = {} last = None repeats = 0 @@ -480,10 +512,18 @@ class _CriboSourceMapRuntime(object): else: write("\nDuring handling of the above exception, another exception " "occurred:\n\n") tb = getattr(exc, "__traceback__", None) - if tb is not None: + limit = self._effective_tb_limit() + if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") - self._write_frames(tb, table, sources, map_dir, write) + self._write_frames(tb, table, sources, map_dir, write, limit) write(self._exception_line(exc)) + notes = getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except BaseException: + pass def _try_render(self, exc_value, traceback_obj, prefix): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ @@ -572,7 +612,7 @@ class _CriboSourceMapRuntime(object): self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime("linked", globals().get("__file__", "")).install() +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) import sys as _sys import importlib as _importlib class _CriboModule(): diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 3fe98558d..cafcd7017 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:697 `def add(a, b):` -> calculator.py:1 -bundle:698 `result = a + b` -> calculator.py:2 -bundle:699 `return result` -> calculator.py:3 -bundle:701 `def multiply(a, b):` -> calculator.py:6 -bundle:702 `result = a * b` -> calculator.py:7 -bundle:703 `return result` -> calculator.py:8 -bundle:708 `def describe(name, value):` -> utils.py:1 -bundle:709 `return f"{name} = {value}"` -> utils.py:2 -bundle:715 `total = add(2, 3)` -> main.py:4 -bundle:716 `product = multiply(total, 4)` -> main.py:5 -bundle:717 `print(describe("total", total))` -> main.py:6 -bundle:718 `print(describe("product", product))` -> main.py:7 +bundle:737 `def add(a, b):` -> calculator.py:1 +bundle:738 `result = a + b` -> calculator.py:2 +bundle:739 `return result` -> calculator.py:3 +bundle:741 `def multiply(a, b):` -> calculator.py:6 +bundle:742 `result = a * b` -> calculator.py:7 +bundle:743 `return result` -> calculator.py:8 +bundle:748 `def describe(name, value):` -> utils.py:1 +bundle:749 `return f"{name} = {value}"` -> utils.py:2 +bundle:755 `total = add(2, 3)` -> main.py:4 +bundle:756 `product = multiply(total, 4)` -> main.py:5 +bundle:757 `print(describe("total", total))` -> main.py:6 +bundle:758 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 084075543..8b36cfd12 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:706 `print("effects module loading")` -> effects.py:1 -bundle:707 `COUNTER = 1` -> effects.py:3 -bundle:710 `def boost(value):` -> effects.py:6 -bundle:711 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:712 `return boosted` -> effects.py:8 -bundle:724 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:725 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:746 `print("effects module loading")` -> effects.py:1 +bundle:747 `COUNTER = 1` -> effects.py:3 +bundle:750 `def boost(value):` -> effects.py:6 +bundle:751 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:752 `return boosted` -> effects.py:8 +bundle:764 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:765 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 4e0816ed0..cfc988a31 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -857,3 +857,151 @@ fn runtime_notifies_preinstalled_custom_excepthook() { "the preinstalled custom hook must still be notified after a remap: {stderr}" ); } + +// --------------------------------------------------------------------------- +// Second review round: shadowing, docstring, tracebacklimit, notes, handlers +// --------------------------------------------------------------------------- + +#[test] +fn runtime_survives_shadowing_threading_module() { + // A project file named threading.py next to the bundle must not be able + // to break (or be imported by) the runtime's own stdlib imports. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + fs::write( + dir.path().join("threading.py"), + "raise RuntimeError(\"shadow module imported\")\n", + ) + .expect("write shadowing module"); + + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok, "the crash must still surface"); + assert!( + !stderr.contains("shadow module imported"), + "the runtime must not import the adjacent threading.py: {stderr}" + ); + assert_remapped(&stderr); +} + +#[test] +fn bundle_docstring_survives_runtime_injection() { + let dir = make_project(&[ + ("main.py", "\"\"\"Entry doc.\"\"\"\n\nprint(__doc__)\n"), + ("helper.py", "unused = True\n"), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!( + stdout.contains("Entry doc."), + "the injected runtime must not displace the bundle docstring: {stdout}" + ); +} + +#[test] +fn runtime_honors_tracebacklimit() { + let dir = make_project(&[ + ( + "main.py", + // Set the limit on the real sys module: the bundler's stdlib + // import proxy forwards attribute reads but not writes, so a + // plain `sys.tracebacklimit = 0` would only decorate the proxy + // (true for bundles with or without source maps). + "from helper import boom\n\n__import__(\"sys\").tracebacklimit = 0\nboom()\n", + ), + ( + "helper.py", + "def boom():\n raise ValueError(\"limited kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("ValueError: limited kaboom"), "{stderr}"); + assert!( + !stderr.contains("Traceback (most recent call last):"), + "tracebacklimit = 0 must suppress the frame listing: {stderr}" + ); + assert!( + !stderr.contains("File \""), + "tracebacklimit = 0 must suppress all frames: {stderr}" + ); +} + +#[test] +fn runtime_renders_exception_notes() { + let dir = make_project(&[ + ("main.py", "from helper import boom\n\nboom()\n"), + ( + "helper.py", + "def boom():\n error = ValueError(\"kaboom\")\n error.add_note(\"NOTE: check \ + the flux capacitor\")\n raise error\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("helper.py\", line 4, in boom"), + "traceback must be remapped: {stderr}" + ); + assert!( + stderr.contains("NOTE: check the flux capacitor"), + "__notes__ must survive remapped rendering: {stderr}" + ); +} + +#[test] +fn map_covers_elif_and_except_headers() { + let dir = make_project(&[ + ( + "main.py", + "from helper import classify\n\nprint(classify(2))\n", + ), + ( + "helper.py", + "def classify(value):\n if value == 0:\n return \"zero\"\n elif value == \ + 1:\n return \"one\"\n elif value == 2:\n return \"two\"\n \ + try:\n return int(value)\n except ValueError:\n return \"other\"\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + let map = parse_map(&map_json); + + let helper_id = (0..map.get_sources().count() as u32) + .find(|id| { + map.get_source(*id) + .is_some_and(|s| s.ends_with("helper.py")) + }) + .expect("helper.py in sources"); + let mapped_helper_lines: Vec = map + .get_tokens() + .filter(|token| token.get_source_id() == Some(helper_id)) + .map(|token| token.get_src_line()) + .collect(); + // 0-based original lines: 3 and 5 are the `elif` headers, 9 is `except ValueError:`. + for header_line in [3, 5, 9] { + assert!( + mapped_helper_lines.contains(&header_line), + "helper.py 0-based line {header_line} (a clause header) must be mapped; mapped \ + lines: {mapped_helper_lines:?}" + ); + } +} From d25ccdeb5cd8cbb4a99405d0381f6b6fcbd32882 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 17:44:36 +0200 Subject: [PATCH 04/17] fix: address third review round on source map runtime - map match-case headers (case provenance) and every decorator line - render exception lines via traceback.TracebackException.format_exception_only (SyntaxError caret, NameError/AttributeError suggestions, __notes__) with the minimal formatter as fail-open fallback - publish bundle + map near-atomically: map staged to a temp file before the bundle write, renamed over the final path after it, so no failure mode leaves a fresh bundle beside a stale map - remove the arbitrary 16-deep exception-chain cap (cycle guard remains) - make the map scanner order-independent: an early VLQ exit skims to the end of the mappings string when sources still needs parsing, and stops reading entirely when sources was already consumed Addresses third-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 38 ++++++- crates/cribo/src/python/sourcemap_runtime.py | 75 +++++++++---- crates/cribo/src/source_map.rs | 39 +++++++ .../tests/python/test_sourcemap_runtime.py | 9 ++ .../bundled_code@sourcemap_basic.snap | 52 +++++---- .../bundled_code@sourcemap_wrapper.snap | 52 +++++---- .../snapshots/source_map@sourcemap_basic.snap | 24 ++--- .../source_map@sourcemap_wrapper.snap | 14 +-- crates/cribo/tests/test_source_maps.rs | 102 ++++++++++++++++++ 9 files changed, 325 insertions(+), 80 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index e3dddb1d5..8b2216bd9 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -737,15 +737,43 @@ impl BundleOrchestrator { self.write_requirements_file(&sorted_module_ids, &resolver, &graph, output_path)?; } + // Publish the bundle and map as close to atomically as possible: the + // map content is staged to a temp file *before* the bundle is written + // (a staging failure aborts with the old pair intact) and renamed over + // the final map path *after* (rename is atomic and replaces a stale + // map even when the file itself is read-only). The only remaining + // mismatch window is a rename failure, which requires directory-level + // problems that would have failed the bundle write too. + let staged_map = if let Some((map_path, map_json)) = pending_map { + let mut tmp_name = map_path.file_name().map_or_else( + || std::ffi::OsString::from("bundle.py.map"), + std::ffi::OsStr::to_os_string, + ); + tmp_name.push(".tmp"); + let tmp_path = map_path.with_file_name(tmp_name); + fs::write(&tmp_path, map_json).with_context(|| { + format!("Failed to stage source map file: {}", tmp_path.display()) + })?; + Some((tmp_path, map_path)) + } else { + None + }; + // Write output file - fs::write(output_path, bundled_code) - .with_context(|| format!("Failed to write output file: {}", output_path.display()))?; + let bundle_write = fs::write(output_path, bundled_code) + .with_context(|| format!("Failed to write output file: {}", output_path.display())); + if let Err(err) = bundle_write { + if let Some((tmp_path, _)) = staged_map { + let _ = fs::remove_file(tmp_path); + } + return Err(err); + } info!("Bundle written to: {}", output_path.display()); - if let Some((map_path, map_json)) = pending_map { - fs::write(&map_path, map_json).with_context(|| { - format!("Failed to write source map file: {}", map_path.display()) + if let Some((tmp_path, map_path)) = staged_map { + fs::rename(&tmp_path, &map_path).with_context(|| { + format!("Failed to publish source map file: {}", map_path.display()) })?; info!("Source map written to: {}", map_path.display()); } diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index f9d334e37..6bf57362f 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -307,8 +307,9 @@ def _decode_vlq(self, stream, needed, max_needed): Constant state: line/segment counters plus running deltas. Records the first segment per needed generated line; exits as soon as every needed - line is resolved or the max needed line is passed. Consumes up to and - including the closing quote (or stops early). + line is resolved or the max needed line is passed. Returns + ``(table, terminated)`` where ``terminated`` says whether the closing + quote was consumed (early exits leave the stream inside the string). """ lut = {} for index in range(64): @@ -329,13 +330,13 @@ def end_segment(): byte = stream.read_byte() if byte < 0 or byte == 34: # EOF or closing '"' end_segment() - return result + return result, True if byte == 59: # ';' end_segment() gen_line += 1 field = 0 if gen_line > max_needed or len(result) == len(needed): - return result + return result, False continue if byte == 44: # ',' end_segment() @@ -378,10 +379,21 @@ def _scan(self, chunks, needed, max_needed): elif key == "mappings": if byte != 34: raise ValueError("mappings is not a string") - table = self._decode_vlq(stream, needed, max_needed) + table, terminated = self._decode_vlq(stream, needed, max_needed) saw_mappings = True if sources: - break # both fields consumed; ignore the rest of the map + # Both fields consumed; stop without reading further (the + # early exit inside the VLQ machine is preserved). + break + if not terminated: + # Early exit left the stream inside the mappings string; + # skim to its closing quote so a `sources` field that + # follows `mappings` still parses correctly. VLQ data + # contains no escapes, so a bare '"' terminates. + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: + break byte = stream.read_byte() else: byte = self._skip_value(stream, byte) @@ -438,7 +450,7 @@ def _load_json_fallback(self, needed_lines): mappings = data.get("mappings") or "" needed0 = set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) - table0 = self._decode_vlq(stream, needed0, max(needed0)) + table0, _terminated = self._decode_vlq(stream, needed0, max(needed0)) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = (src_idx, src_line0 + 1) @@ -459,10 +471,8 @@ def add(tb): add(traceback_obj) exc = exc_value seen = set() - depth = 0 - while exc is not None and id(exc) not in seen and depth < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) - depth += 1 add(getattr(exc, "__traceback__", None)) cause = getattr(exc, "__cause__", None) context = getattr(exc, "__context__", None) @@ -480,7 +490,7 @@ def _chain_has_group(self, exc_value): return False exc = exc_value seen = set() - while exc is not None and id(exc) not in seen and len(seen) < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) if isinstance(exc, self._group_type): return True @@ -576,6 +586,7 @@ def emit(entry): write(" [Previous line repeated %d more times]\n" % (repeats - 3)) def _exception_line(self, exc_value): + """Minimal `Type: message` line, the fallback formatter.""" exc_type = type(exc_value) name = getattr(exc_type, "__qualname__", exc_type.__name__) module = getattr(exc_type, "__module__", None) @@ -587,12 +598,43 @@ def _exception_line(self, exc_value): text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name + def _write_exception_only(self, exc, write): + """Write the exception line(s) with full standard-library fidelity. + + `traceback.format_exception_only` supplies the interpreter's + specialized rendering — SyntaxError source line and caret, NameError / + AttributeError "Did you mean" suggestions, and `__notes__` — so the + remapped output matches the default hook. Falls back to the minimal + line (plus notes) when the traceback module is unavailable. + """ + try: + traceback_mod = _cribo_sm_import("traceback") + te = traceback_mod.TracebackException( + type(exc), + exc, + getattr(exc, "__traceback__", None), + lookup_lines=False, + ) + for line in te.format_exception_only(): + write(line) + return + except BaseException: + pass + write(self._exception_line(exc)) + notes = getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except BaseException: + pass + def _render(self, exc_value, table, sources, map_dir, write): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value seen = set() - while exc is not None and id(exc) not in seen and len(chain) < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) cause = getattr(exc, "__cause__", None) context = getattr(exc, "__context__", None) @@ -627,14 +669,7 @@ def _render(self, exc_value, table, sources, map_dir, write): if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") self._write_frames(tb, table, sources, map_dir, write, limit) - write(self._exception_line(exc)) - notes = getattr(exc, "__notes__", None) - if notes: - try: - for note in notes: - write("%s\n" % (note,)) - except BaseException: - pass + self._write_exception_only(exc, write) def _try_render(self, exc_value, traceback_obj, prefix): """Attempt a remapped rendering to stderr; True on success. diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index c9600c26e..47a69920c 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -319,6 +319,33 @@ impl ParallelWalker<'_> { }); } + // Evaluating a decorator can raise on its own `@...` line; give every + // decorator its own mapping (the statement mapping above only covers + // the `def`/`class` header). + let decorator_pairs = match (generated, original) { + (Stmt::FunctionDef(g), Stmt::FunctionDef(o)) => { + Some((&g.decorator_list, &o.decorator_list)) + } + (Stmt::ClassDef(g), Stmt::ClassDef(o)) => Some((&g.decorator_list, &o.decorator_list)), + _ => None, + }; + if let Some((gen_decorators, orig_decorators)) = decorator_pairs + && gen_decorators.len() == orig_decorators.len() + { + for (gen_decorator, orig_decorator) in gen_decorators.iter().zip(orig_decorators) { + if let Some((module_ordinal, original_line)) = self.provenance.resolve( + orig_decorator.node_index.load(), + orig_decorator.range().start(), + ) { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_decorator.range().start()), + module_ordinal, + original_line, + }); + } + } + } + // Recurse into nested statement bodies even when the statement itself is // synthesized: wrapper-module init functions are synthesized `def`s whose // bodies contain original module statements. @@ -395,6 +422,18 @@ impl ParallelWalker<'_> { } (Stmt::Match(g), Stmt::Match(o)) if g.cases.len() == o.cases.len() => { for (gen_case, orig_case) in g.cases.iter().zip(&o.cases) { + // A raising pattern operation or guard is attributed to the + // `case` header line; map it via the case's provenance. + if let Some((module_ordinal, original_line)) = self + .provenance + .resolve(orig_case.node_index.load(), orig_case.range().start()) + { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_case.range().start()), + module_ordinal, + original_line, + }); + } self.walk_body(&gen_case.body, &orig_case.body); } } diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index 5a45bd2b0..39306d5c0 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -117,6 +117,15 @@ def test_vlq_rejects_escapes_in_mappings(rt): raise AssertionError("escape inside mappings must raise") +def test_scan_handles_mappings_before_sources(rt): + # A spec-valid map may order keys arbitrarily. Early exit inside the + # mappings string must not derail parsing of a later sources field. + json_text = '{"mappings":"AAAA;AACA;AACA","sources":["a.py","b.py"]}' + sources, table = _scan(rt, json_text, {0}, 0) # early exit after line 0 + assert sources == ["a.py", "b.py"], sources + assert table == {0: (0, 0)}, table + + def make_inline_bundle(payload_json): """Create a temp file shaped like an inline-mode bundle; return its path.""" import base64 diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 97d5c167b..e55c470c8 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -235,7 +235,7 @@ class _CriboSourceMapRuntime(object): byte = self._skip_ws(stream, stream.read_byte()) def _decode_vlq(self, stream, needed, max_needed): - """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} for index in range(64): lut[self._B64[index]] = index @@ -254,13 +254,13 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() if byte < 0 or byte == 34: end_segment() - return result + return result, True if byte == 59: end_segment() gen_line += 1 field = 0 if gen_line > max_needed or len(result) == len(needed): - return result + return result, False continue if byte == 44: end_segment() @@ -303,10 +303,15 @@ class _CriboSourceMapRuntime(object): elif key == "mappings": if byte != 34: raise ValueError("mappings is not a string") - table = self._decode_vlq(stream, needed, max_needed) + table, terminated = self._decode_vlq(stream, needed, max_needed) saw_mappings = True if sources: break + if not terminated: + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: + break byte = stream.read_byte() else: byte = self._skip_value(stream, byte) @@ -355,7 +360,7 @@ class _CriboSourceMapRuntime(object): mappings = data.get("mappings") or "" needed0 = set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) - table0 = self._decode_vlq(stream, needed0, max(needed0)) + table0, _terminated = self._decode_vlq(stream, needed0, max(needed0)) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 @@ -373,10 +378,8 @@ class _CriboSourceMapRuntime(object): add(traceback_obj) exc = exc_value seen = set() - depth = 0 - while exc is not None and id(exc) not in seen and depth < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) - depth += 1 add(getattr(exc, "__traceback__", None)) cause = getattr(exc, "__cause__", None) context = getattr(exc, "__context__", None) @@ -389,7 +392,7 @@ class _CriboSourceMapRuntime(object): return False exc = exc_value seen = set() - while exc is not None and id(exc) not in seen and len(seen) < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) if isinstance(exc, self._group_type): return True @@ -474,6 +477,7 @@ class _CriboSourceMapRuntime(object): write(" [Previous line repeated %d more times]\n" % (repeats - 3)) def _exception_line(self, exc_value): + """Minimal `Type: message` line, the fallback formatter.""" exc_type = type(exc_value) name = getattr(exc_type, "__qualname__", exc_type.__name__) module = getattr(exc_type, "__module__", None) @@ -485,12 +489,31 @@ class _CriboSourceMapRuntime(object): text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name + def _write_exception_only(self, exc, write): + """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ + try: + traceback_mod = _cribo_sm_import("traceback") + te = traceback_mod.TracebackException(type(exc), exc, getattr(exc, "__traceback__", None), lookup_lines=False) + for line in te.format_exception_only(): + write(line) + return + except BaseException: + pass + write(self._exception_line(exc)) + notes = getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except BaseException: + pass + def _render(self, exc_value, table, sources, map_dir, write): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value seen = set() - while exc is not None and id(exc) not in seen and len(chain) < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) cause = getattr(exc, "__cause__", None) context = getattr(exc, "__context__", None) @@ -516,14 +539,7 @@ class _CriboSourceMapRuntime(object): if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") self._write_frames(tb, table, sources, map_dir, write, limit) - write(self._exception_line(exc)) - notes = getattr(exc, "__notes__", None) - if notes: - try: - for note in notes: - write("%s\n" % (note,)) - except BaseException: - pass + self._write_exception_only(exc, write) def _try_render(self, exc_value, traceback_obj, prefix): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 72c0e2ec2..5f54a0bc3 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -235,7 +235,7 @@ class _CriboSourceMapRuntime(object): byte = self._skip_ws(stream, stream.read_byte()) def _decode_vlq(self, stream, needed, max_needed): - """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Consumes up to and\n including the closing quote (or stops early).\n """ + """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} for index in range(64): lut[self._B64[index]] = index @@ -254,13 +254,13 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() if byte < 0 or byte == 34: end_segment() - return result + return result, True if byte == 59: end_segment() gen_line += 1 field = 0 if gen_line > max_needed or len(result) == len(needed): - return result + return result, False continue if byte == 44: end_segment() @@ -303,10 +303,15 @@ class _CriboSourceMapRuntime(object): elif key == "mappings": if byte != 34: raise ValueError("mappings is not a string") - table = self._decode_vlq(stream, needed, max_needed) + table, terminated = self._decode_vlq(stream, needed, max_needed) saw_mappings = True if sources: break + if not terminated: + while True: + byte = stream.read_byte() + if byte < 0 or byte == 34: + break byte = stream.read_byte() else: byte = self._skip_value(stream, byte) @@ -355,7 +360,7 @@ class _CriboSourceMapRuntime(object): mappings = data.get("mappings") or "" needed0 = set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) - table0 = self._decode_vlq(stream, needed0, max(needed0)) + table0, _terminated = self._decode_vlq(stream, needed0, max(needed0)) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 @@ -373,10 +378,8 @@ class _CriboSourceMapRuntime(object): add(traceback_obj) exc = exc_value seen = set() - depth = 0 - while exc is not None and id(exc) not in seen and depth < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) - depth += 1 add(getattr(exc, "__traceback__", None)) cause = getattr(exc, "__cause__", None) context = getattr(exc, "__context__", None) @@ -389,7 +392,7 @@ class _CriboSourceMapRuntime(object): return False exc = exc_value seen = set() - while exc is not None and id(exc) not in seen and len(seen) < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) if isinstance(exc, self._group_type): return True @@ -474,6 +477,7 @@ class _CriboSourceMapRuntime(object): write(" [Previous line repeated %d more times]\n" % (repeats - 3)) def _exception_line(self, exc_value): + """Minimal `Type: message` line, the fallback formatter.""" exc_type = type(exc_value) name = getattr(exc_type, "__qualname__", exc_type.__name__) module = getattr(exc_type, "__module__", None) @@ -485,12 +489,31 @@ class _CriboSourceMapRuntime(object): text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name + def _write_exception_only(self, exc, write): + """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ + try: + traceback_mod = _cribo_sm_import("traceback") + te = traceback_mod.TracebackException(type(exc), exc, getattr(exc, "__traceback__", None), lookup_lines=False) + for line in te.format_exception_only(): + write(line) + return + except BaseException: + pass + write(self._exception_line(exc)) + notes = getattr(exc, "__notes__", None) + if notes: + try: + for note in notes: + write("%s\n" % (note,)) + except BaseException: + pass + def _render(self, exc_value, table, sources, map_dir, write): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value seen = set() - while exc is not None and id(exc) not in seen and len(chain) < 16: + while exc is not None and id(exc) not in seen: seen.add(id(exc)) cause = getattr(exc, "__cause__", None) context = getattr(exc, "__context__", None) @@ -516,14 +539,7 @@ class _CriboSourceMapRuntime(object): if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") self._write_frames(tb, table, sources, map_dir, write, limit) - write(self._exception_line(exc)) - notes = getattr(exc, "__notes__", None) - if notes: - try: - for note in notes: - write("%s\n" % (note,)) - except BaseException: - pass + self._write_exception_only(exc, write) def _try_render(self, exc_value, traceback_obj, prefix): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index cafcd7017..9c591a629 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:737 `def add(a, b):` -> calculator.py:1 -bundle:738 `result = a + b` -> calculator.py:2 -bundle:739 `return result` -> calculator.py:3 -bundle:741 `def multiply(a, b):` -> calculator.py:6 -bundle:742 `result = a * b` -> calculator.py:7 -bundle:743 `return result` -> calculator.py:8 -bundle:748 `def describe(name, value):` -> utils.py:1 -bundle:749 `return f"{name} = {value}"` -> utils.py:2 -bundle:755 `total = add(2, 3)` -> main.py:4 -bundle:756 `product = multiply(total, 4)` -> main.py:5 -bundle:757 `print(describe("total", total))` -> main.py:6 -bundle:758 `print(describe("product", product))` -> main.py:7 +bundle:753 `def add(a, b):` -> calculator.py:1 +bundle:754 `result = a + b` -> calculator.py:2 +bundle:755 `return result` -> calculator.py:3 +bundle:757 `def multiply(a, b):` -> calculator.py:6 +bundle:758 `result = a * b` -> calculator.py:7 +bundle:759 `return result` -> calculator.py:8 +bundle:764 `def describe(name, value):` -> utils.py:1 +bundle:765 `return f"{name} = {value}"` -> utils.py:2 +bundle:771 `total = add(2, 3)` -> main.py:4 +bundle:772 `product = multiply(total, 4)` -> main.py:5 +bundle:773 `print(describe("total", total))` -> main.py:6 +bundle:774 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 8b36cfd12..038c5091b 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:746 `print("effects module loading")` -> effects.py:1 -bundle:747 `COUNTER = 1` -> effects.py:3 -bundle:750 `def boost(value):` -> effects.py:6 -bundle:751 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:752 `return boosted` -> effects.py:8 -bundle:764 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:765 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:762 `print("effects module loading")` -> effects.py:1 +bundle:763 `COUNTER = 1` -> effects.py:3 +bundle:766 `def boost(value):` -> effects.py:6 +bundle:767 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:768 `return boosted` -> effects.py:8 +bundle:780 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:781 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index cfc988a31..52533706f 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1005,3 +1005,105 @@ fn map_covers_elif_and_except_headers() { ); } } + +// --------------------------------------------------------------------------- +// Third review round: specialized formatting, long chains, header coverage +// --------------------------------------------------------------------------- + +#[test] +fn runtime_keeps_name_error_suggestions() { + let dir = make_project(&[ + ("main.py", "from helper import go\n\ngo()\n"), + ( + "helper.py", + "def go():\n valuable = 1\n return valuabl\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("helper.py\", line 3, in go"), + "traceback must be remapped: {stderr}" + ); + assert!(stderr.contains("NameError"), "{stderr}"); + assert!( + stderr.contains("Did you mean"), + "interpreter suggestions must survive remapped rendering: {stderr}" + ); +} + +#[test] +fn runtime_renders_long_exception_chains_fully() { + // 20 chained causes exceed the previous traversal cap; the innermost + // (root) exception and its traceback must still be rendered. + let dir = make_project(&[ + ("main.py", "from helper import cascade\n\ncascade()\n"), + ( + "helper.py", + "def cascade():\n try:\n raise ValueError(\"root kaboom\")\n except \ + ValueError as error:\n current = error\n for depth in range(20):\n \ + try:\n raise RuntimeError(\"layer %d\" % depth) from \ + current\n except RuntimeError as next_error:\n current = \ + next_error\n raise current\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("ValueError: root kaboom"), + "the root cause of a 20-deep chain must be rendered: {stderr}" + ); + assert!(stderr.contains("layer 19"), "{stderr}"); + assert!( + stderr.contains("helper.py\", line 3"), + "the root cause frame must be remapped: {stderr}" + ); +} + +#[test] +fn map_covers_match_case_headers_and_decorators() { + let dir = make_project(&[ + ("main.py", "from helper import run\n\nprint(run(1))\n"), + ( + "helper.py", + "def trace(func):\n return func\n\n\n@trace\n@trace\ndef run(value):\n match \ + value:\n case 0:\n return \"zero\"\n case _ if value > \ + 0:\n return \"positive\"\n case _:\n return \ + \"negative\"\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + let map = parse_map(&map_json); + + let helper_id = (0..map.get_sources().count() as u32) + .find(|id| { + map.get_source(*id) + .is_some_and(|s| s.ends_with("helper.py")) + }) + .expect("helper.py in sources"); + let mapped_helper_lines: Vec = map + .get_tokens() + .filter(|token| token.get_source_id() == Some(helper_id)) + .map(|token| token.get_src_line()) + .collect(); + // 0-based original lines: 4 and 5 are the two decorators; 8, 10, and 12 + // are the `case` headers. + for header_line in [4, 5, 8, 10, 12] { + assert!( + mapped_helper_lines.contains(&header_line), + "helper.py 0-based line {header_line} (decorator or case header) must be mapped; \ + mapped lines: {mapped_helper_lines:?}" + ); + } +} From a9661a8760c9ac5d145be5734f98863719484200 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 18:02:10 +0200 Subject: [PATCH 05/17] fix: harden runtime against shadowed builtins and read-only maps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - snapshot shadowable builtins (open, len, max, set, getattr, ...) via keyword-only defaults evaluated at class-definition time — before any bundled user code runs — matching the idiom of cribo's generated proxies; entry code rebinding common builtins can no longer disable remapping - lazy imports inside the hook path (json, traceback) go through the instance-bound shadow-proof importer - map publish: fall back to remove-then-rename for the Windows read-only destination case (std rename already replaces via MOVEFILE_REPLACE_EXISTING) Addresses fourth-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 16 +- crates/cribo/src/python/sourcemap_runtime.py | 206 ++++++++++-------- .../bundled_code@sourcemap_basic.snap | 171 +++++++-------- .../bundled_code@sourcemap_wrapper.snap | 171 +++++++-------- .../snapshots/source_map@sourcemap_basic.snap | 24 +- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 29 +++ 7 files changed, 351 insertions(+), 280 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 8b2216bd9..5682f3f39 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -772,9 +772,19 @@ impl BundleOrchestrator { info!("Bundle written to: {}", output_path.display()); if let Some((tmp_path, map_path)) = staged_map { - fs::rename(&tmp_path, &map_path).with_context(|| { - format!("Failed to publish source map file: {}", map_path.display()) - })?; + // std's rename replaces an existing destination on every platform + // (MoveFileExW with MOVEFILE_REPLACE_EXISTING on Windows). The one + // residual case is a read-only destination on Windows, so fall + // back to remove-then-rename before giving up. + let publish = fs::rename(&tmp_path, &map_path).or_else(|_| { + fs::remove_file(&map_path).and_then(|()| fs::rename(&tmp_path, &map_path)) + }); + if let Err(err) = publish { + let _ = fs::remove_file(&tmp_path); + return Err(err).with_context(|| { + format!("Failed to publish source map file: {}", map_path.display()) + }); + } info!("Source map written to: {}", map_path.display()); } diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 6bf57362f..58f8af981 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -14,7 +14,7 @@ import sys as _cribo_sys -def _cribo_sm_import(name): +def _cribo_sm_import(name, *, _list=list, _import=__import__): """Import a stdlib module immune to script-directory shadowing. The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a @@ -27,27 +27,32 @@ def _cribo_sm_import(name): if module is not None: return module saved_path = _cribo_sys.path - _cribo_sys.path = list(saved_path[1:]) + _cribo_sys.path = _list(saved_path[1:]) try: - return __import__(name) + return _import(name) finally: _cribo_sys.path = saved_path class _CriboSmStream(object): - """Byte-at-a-time reader over an iterator of byte chunks.""" + """Byte-at-a-time reader over an iterator of byte chunks. + + Keyword-only defaults snapshot the builtins at definition time (before any + bundled user code runs), so later shadowing of e.g. `len` cannot break the + reader — the same idiom cribo's generated module proxies use. + """ __slots__ = ("_chunks", "_buf", "_pos") - def __init__(self, chunks): - self._chunks = iter(chunks) + def __init__(self, chunks, *, _iter=iter): + self._chunks = _iter(chunks) self._buf = b"" self._pos = 0 - def read_byte(self): - while self._pos >= len(self._buf): + def read_byte(self, *, _len=len, _next=next): + while self._pos >= _len(self._buf): try: - self._buf = next(self._chunks) + self._buf = _next(self._chunks) except StopIteration: return -1 self._pos = 0 @@ -60,9 +65,11 @@ class _CriboSourceMapRuntime(object): """Traceback-remapping runtime. All collaborators (modules, the stream class, previous hooks) are bound to - the instance at construction time, so the installed hooks keep working even - if bundled user code later rebinds any module-level name this template - introduced. + the instance at construction time, and every method snapshots the builtins + it needs via keyword-only defaults evaluated at class-definition time — so + the installed hooks keep working even if bundled user code later rebinds + any module-level name this template introduced, or common builtins such as + `open`, `len`, or `max`. """ _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" @@ -76,6 +83,7 @@ def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): self._binascii = binascii_mod self._threading = threading_mod self._stream_cls = _CriboSmStream + self._import = _cribo_sm_import # Re-entrancy guard; thread-local so a hook firing on one thread never # disables remapping on another. self._local = threading_mod.local() @@ -151,9 +159,9 @@ def _map_location(self): ) return None - def _file_chunks(self, path): + def _file_chunks(self, path, *, _open=open): """Yield fixed-size chunks of a file (constant memory).""" - handle = open(path, "rb") + handle = _open(path, "rb") try: while True: chunk = handle.read(self._CHUNK) @@ -163,7 +171,7 @@ def _file_chunks(self, path): finally: handle.close() - def _find_inline_payload(self, handle): + def _find_inline_payload(self, handle, *, _len=len): """Backward-scan the bundle for the last inline map marker. Returns the byte offset of the base64 payload, or -1. Only the tail of @@ -183,7 +191,7 @@ def _find_inline_payload(self, handle): if index >= 0: found = position + index break - overlap = data[: len(marker) - 1] + overlap = data[: _len(marker) - 1] if found < 0: return -1 handle.seek(found) @@ -191,11 +199,11 @@ def _find_inline_payload(self, handle): base64_at = head.find(b"base64,") if base64_at < 0: return -1 - return found + base64_at + len(b"base64,") + return found + base64_at + _len(b"base64,") - def _inline_chunks(self, path): + def _inline_chunks(self, path, *, _open=open, _len=len): """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = open(path, "rb") + handle = _open(path, "rb") try: start = self._find_inline_payload(handle) if start < 0: @@ -207,12 +215,12 @@ def _inline_chunks(self, path): if not raw: break data = pending + raw.translate(None, b"\r\n") - usable = len(data) - (len(data) % 4) + usable = _len(data) - (_len(data) % 4) pending = data[usable:] if usable: yield self._binascii.a2b_base64(data[:usable]) if pending: - yield self._binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) finally: handle.close() @@ -223,7 +231,9 @@ def _skip_ws(self, stream, byte): byte = stream.read_byte() return byte - def _read_string(self, stream, collect): + def _read_string( + self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range + ): """Consume a JSON string whose opening quote was already read. Returns the decoded text when collect is true, else None (contents are @@ -231,7 +241,7 @@ def _read_string(self, stream, collect): handled, so string *values* containing text like '"mappings":' cannot confuse the key scanner. """ - buf = bytearray() if collect else None + buf = _bytearray() if collect else None while True: byte = stream.read_byte() if byte < 0: @@ -247,13 +257,13 @@ def _read_string(self, stream, collect): raise ValueError("unterminated JSON escape") if escape == 117: # 'u' code = 0 - for _ in range(4): + for _ in _range(4): digit = stream.read_byte() if digit < 0: raise ValueError("unterminated unicode escape") - code = code * 16 + int(chr(digit), 16) + code = code * 16 + _int(_chr(digit), 16) if buf is not None: - buf.extend(chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) elif buf is not None: table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf.append(table.get(escape, escape)) @@ -302,7 +312,7 @@ def _read_string_array(self, stream, byte): raise ValueError("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed): + def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len): """Streaming VLQ state machine over the raw bytes of the mappings string. Constant state: line/segment counters plus running deltas. Records the @@ -312,7 +322,7 @@ def _decode_vlq(self, stream, needed, max_needed): quote was consumed (early exits leave the stream inside the string). """ lut = {} - for index in range(64): + for index in _range(64): lut[self._B64[index]] = index result = {} gen_line = 0 @@ -335,7 +345,7 @@ def end_segment(): end_segment() gen_line += 1 field = 0 - if gen_line > max_needed or len(result) == len(needed): + if gen_line > max_needed or _len(result) == _len(needed): return result, False continue if byte == 44: # ',' @@ -406,7 +416,7 @@ def _scan(self, chunks, needed, max_needed): # -- loading ------------------------------------------------------------- - def _load(self, needed_lines): + def _load(self, needed_lines, *, _set=set, _max=max): """Load (table, sources, map_dir) for 1-based bundle line numbers. Returns None when the runtime is inactive for the current mode. The @@ -417,8 +427,8 @@ def _load(self, needed_lines): if location is None: return None map_path, map_dir = location - needed0 = set(line - 1 for line in needed_lines) - max_needed = max(needed0) + needed0 = _set(line - 1 for line in needed_lines) + max_needed = _max(needed0) if map_path is None: chunks = self._inline_chunks(self._bundle) else: @@ -429,18 +439,18 @@ def _load(self, needed_lines): table[line0 + 1] = (src_idx, src_line0 + 1) return (table, sources, map_dir) - def _load_json_fallback(self, needed_lines): + def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" location = self._map_location() if location is None: return None map_path, map_dir = location - import json + json = self._import("json") if map_path is None: raw = b"".join(self._inline_chunks(self._bundle)) else: - handle = open(map_path, "rb") + handle = _open(map_path, "rb") try: raw = handle.read() finally: @@ -448,9 +458,9 @@ def _load_json_fallback(self, needed_lines): data = json.loads(raw.decode("utf-8")) sources = data.get("sources") or [] mappings = data.get("mappings") or "" - needed0 = set(line - 1 for line in needed_lines) + needed0 = _set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) - table0, _terminated = self._decode_vlq(stream, needed0, max(needed0)) + table0, _terminated = self._decode_vlq(stream, needed0, _max(needed0)) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = (src_idx, src_line0 + 1) @@ -458,9 +468,11 @@ def _load_json_fallback(self, needed_lines): # -- traceback collection and rendering ---------------------------------- - def _collect_needed(self, exc_value, traceback_obj): + def _collect_needed( + self, exc_value, traceback_obj, *, _set=set, _id=id, _getattr=getattr + ): """1-based bundle lines referenced by the traceback (and its chain).""" - needed = set() + needed = _set() def add(tb): while tb is not None: @@ -470,16 +482,18 @@ def add(tb): add(traceback_obj) exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - add(getattr(exc, "__traceback__", None)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + add(_getattr(exc, "__traceback__", None)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) exc = cause if cause is not None else context return needed - def _chain_has_group(self, exc_value): + def _chain_has_group( + self, exc_value, *, _set=set, _id=id, _getattr=getattr, _isinstance=isinstance + ): """Whether the exception chain contains a BaseExceptionGroup. CPython renders groups with a dedicated nested layout; rather than @@ -489,20 +503,20 @@ def _chain_has_group(self, exc_value): if self._group_type is None: return False exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - if isinstance(exc, self._group_type): + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + if _isinstance(exc, self._group_type): return True - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) exc = cause if cause is not None else context return False - def _source_line(self, path, lineno): + def _source_line(self, path, lineno, *, _open=open): """Read a single 1-based line from a file without caching it.""" try: - handle = open(path, "rb") + handle = _open(path, "rb") except OSError: return None try: @@ -519,12 +533,14 @@ def _source_line(self, path, lineno): handle.close() return None - def _effective_tb_limit(self): + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance): """The application's `sys.tracebacklimit`, or None when unset/invalid.""" - limit = getattr(self._sys, "tracebacklimit", None) - return limit if isinstance(limit, int) else None + limit = _getattr(self._sys, "tracebacklimit", None) + return limit if _isinstance(limit, int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): + def _write_frames( + self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len + ): """Write remapped frame lines, collapsing repeated frames like CPython. Consecutive identical frames (recursion) print at most 3 times followed @@ -563,7 +579,7 @@ def emit(entry): name = frame.f_code.co_name if filename == self._bundle: mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + if mapped is not None and 0 <= mapped[0] < _len(sources) and sources[mapped[0]]: source = sources[mapped[0]] if not self._os.path.isabs(source): source = self._os.path.normpath( @@ -585,20 +601,20 @@ def emit(entry): if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - def _exception_line(self, exc_value): + def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str): """Minimal `Type: message` line, the fallback formatter.""" - exc_type = type(exc_value) - name = getattr(exc_type, "__qualname__", exc_type.__name__) - module = getattr(exc_type, "__module__", None) + exc_type = _type(exc_value) + name = _getattr(exc_type, "__qualname__", exc_type.__name__) + module = _getattr(exc_type, "__module__", None) if module not in (None, "builtins", "__main__"): name = "%s.%s" % (module, name) try: - text = str(exc_value) + text = _str(exc_value) except BaseException: text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name - def _write_exception_only(self, exc, write): + def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): """Write the exception line(s) with full standard-library fidelity. `traceback.format_exception_only` supplies the interpreter's @@ -608,11 +624,11 @@ def _write_exception_only(self, exc, write): line (plus notes) when the traceback module is unavailable. """ try: - traceback_mod = _cribo_sm_import("traceback") + traceback_mod = self._import("traceback") te = traceback_mod.TracebackException( - type(exc), + _type(exc), exc, - getattr(exc, "__traceback__", None), + _getattr(exc, "__traceback__", None), lookup_lines=False, ) for line in te.format_exception_only(): @@ -621,7 +637,7 @@ def _write_exception_only(self, exc, write): except BaseException: pass write(self._exception_line(exc)) - notes = getattr(exc, "__notes__", None) + notes = _getattr(exc, "__notes__", None) if notes: try: for note in notes: @@ -629,16 +645,30 @@ def _write_exception_only(self, exc, write): except BaseException: pass - def _render(self, exc_value, table, sources, map_dir, write): + def _render( + self, + exc_value, + table, + sources, + map_dir, + write, + *, + _getattr=getattr, + _set=set, + _id=id, + _list=list, + _reversed=reversed, + _enumerate=enumerate, + ): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - suppress = getattr(exc, "__suppress_context__", False) + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + suppress = _getattr(exc, "__suppress_context__", False) if cause is not None: chain.append((exc, "cause")) exc = cause @@ -651,8 +681,8 @@ def _render(self, exc_value, table, sources, map_dir, write): # Print innermost first, like CPython. The link stored on an exception # describes its relation to its own inner exception — which is exactly # the one printed immediately before it. - ordered = list(reversed(chain)) - for index, (exc, link) in enumerate(ordered): + ordered = _list(_reversed(chain)) + for index, (exc, link) in _enumerate(ordered): if index > 0: if link == "cause": write( @@ -664,21 +694,21 @@ def _render(self, exc_value, table, sources, map_dir, write): "\nDuring handling of the above exception, another exception " "occurred:\n\n" ) - tb = getattr(exc, "__traceback__", None) + tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") self._write_frames(tb, table, sources, map_dir, write, limit) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix): + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): """Attempt a remapped rendering to stderr; True on success. Never raises and never masks the original exception: any failure in this runtime returns False so callers can delegate to the previous hook. """ - if getattr(self._local, "in_hook", False) or exc_value is None: + if _getattr(self._local, "in_hook", False) or exc_value is None: return False self._local.in_hook = True old_limit = None @@ -757,26 +787,26 @@ def excepthook(self, exc_type, exc_value, traceback_obj): return self._prev_excepthook(exc_type, exc_value, traceback_obj) - def threading_hook(self, args): + def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass): # The default threading hook deliberately ignores SystemExit (normal # sys.exit() in a worker thread); preserve that by delegating. - if args.exc_type is not None and issubclass(args.exc_type, SystemExit): + if args.exc_type is not None and _issubclass(args.exc_type, SystemExit): self._prev_threading_hook(args) return - thread = getattr(args, "thread", None) - name = getattr(thread, "name", None) or "Thread" + thread = _getattr(args, "thread", None) + name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): self._notify_custom_hook( self._prev_threading_hook, - getattr(self._threading, "__excepthook__", None), + _getattr(self._threading, "__excepthook__", None), lambda hook: hook(args), ) return self._prev_threading_hook(args) - def unraisablehook(self, unraisable): - message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + def unraisablehook(self, unraisable, *, _getattr=getattr): + message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" try: prefix = "%s: %r\n" % (message, unraisable.object) except BaseException: diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index e55c470c8..23563b030 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -7,30 +7,30 @@ input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py # https://github.com/ophidiarium/cribo import sys as _cribo_sys -def _cribo_sm_import(name): +def _cribo_sm_import(name, *, _list=list, _import=__import__): """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with that first path\n entry dropped. (`sys` itself is a builtin and can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - _cribo_sys.path = list(saved_path[1:]) + _cribo_sys.path = _list(saved_path[1:]) try: - return __import__(name) + return _import(name) finally: _cribo_sys.path = saved_path class _CriboSmStream(object): - """Byte-at-a-time reader over an iterator of byte chunks.""" + """Byte-at-a-time reader over an iterator of byte chunks.\n\n Keyword-only defaults snapshot the builtins at definition time (before any\n bundled user code runs), so later shadowing of e.g. `len` cannot break the\n reader — the same idiom cribo's generated module proxies use.\n """ __slots__ = "_chunks", "_buf", "_pos" - def __init__(self, chunks): - self._chunks = iter(chunks) + def __init__(self, chunks, *, _iter=iter): + self._chunks = _iter(chunks) self._buf = b"" self._pos = 0 - def read_byte(self): - while self._pos >= len(self._buf): + def read_byte(self, *, _len=len, _next=next): + while self._pos >= _len(self._buf): try: - self._buf = next(self._chunks) + self._buf = _next(self._chunks) except StopIteration: return -1 self._pos = 0 @@ -38,7 +38,7 @@ class _CriboSmStream(object): self._pos += 1 return value class _CriboSourceMapRuntime(object): - """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, so the installed hooks keep working even\n if bundled user code later rebinds any module-level name this template\n introduced.\n """ + """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, and every method snapshots the builtins\n it needs via keyword-only defaults evaluated at class-definition time — so\n the installed hooks keep working even if bundled user code later rebinds\n any module-level name this template introduced, or common builtins such as\n `open`, `len`, or `max`.\n """ _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 @@ -50,6 +50,7 @@ class _CriboSourceMapRuntime(object): self._binascii = binascii_mod self._threading = threading_mod self._stream_cls = _CriboSmStream + self._import = _cribo_sm_import self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook @@ -96,9 +97,9 @@ class _CriboSourceMapRuntime(object): return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) return None - def _file_chunks(self, path): + def _file_chunks(self, path, *, _open=open): """Yield fixed-size chunks of a file (constant memory).""" - handle = open(path, "rb") + handle = _open(path, "rb") try: while True: chunk = handle.read(self._CHUNK) @@ -108,7 +109,7 @@ class _CriboSourceMapRuntime(object): finally: handle.close() - def _find_inline_payload(self, handle): + def _find_inline_payload(self, handle, *, _len=len): """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of\n the file is examined; the bundle body is never read.\n """ marker = b"# sourceMappingURL=data:" handle.seek(0, 2) @@ -124,7 +125,7 @@ class _CriboSourceMapRuntime(object): if index >= 0: found = position + index break - overlap = data[:len(marker) - 1] + overlap = data[:_len(marker) - 1] if found < 0: return -1 handle.seek(found) @@ -132,11 +133,11 @@ class _CriboSourceMapRuntime(object): base64_at = head.find(b"base64,") if base64_at < 0: return -1 - return found + base64_at + len(b"base64,") + return found + base64_at + _len(b"base64,") - def _inline_chunks(self, path): + def _inline_chunks(self, path, *, _open=open, _len=len): """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = open(path, "rb") + handle = _open(path, "rb") try: start = self._find_inline_payload(handle) if start < 0: @@ -148,12 +149,12 @@ class _CriboSourceMapRuntime(object): if not raw: break data = pending + raw.translate(None, b"\r\n") - usable = len(data) - len(data) % 4 + usable = _len(data) - _len(data) % 4 pending = data[usable:] if usable: yield self._binascii.a2b_base64(data[:usable]) if pending: - yield self._binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) finally: handle.close() @@ -162,9 +163,9 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string(self, stream, collect): + def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range): """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ - buf = bytearray() if collect else None + buf = _bytearray() if collect else None while True: byte = stream.read_byte() if byte < 0: @@ -180,13 +181,13 @@ class _CriboSourceMapRuntime(object): raise ValueError("unterminated JSON escape") if escape == 117: code = 0 - for _ in range(4): + for _ in _range(4): digit = stream.read_byte() if digit < 0: raise ValueError("unterminated unicode escape") - code = code * 16 + int(chr(digit), 16) + code = code * 16 + _int(_chr(digit), 16) if buf is not None: - buf.extend(chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) elif buf is not None: table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf.append(table.get(escape, escape)) @@ -234,10 +235,10 @@ class _CriboSourceMapRuntime(object): raise ValueError("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed): + def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len): """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} - for index in range(64): + for index in _range(64): lut[self._B64[index]] = index result = {} gen_line = 0 @@ -259,7 +260,7 @@ class _CriboSourceMapRuntime(object): end_segment() gen_line += 1 field = 0 - if gen_line > max_needed or len(result) == len(needed): + if gen_line > max_needed or _len(result) == _len(needed): return result, False continue if byte == 44: @@ -322,14 +323,14 @@ class _CriboSourceMapRuntime(object): raise ValueError("no mappings field") return sources, table - def _load(self, needed_lines): + def _load(self, needed_lines, *, _set=set, _max=max): """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ location = self._map_location() if location is None: return None map_path, map_dir = location - needed0 = set(line - 1 for line in needed_lines) - max_needed = max(needed0) + needed0 = _set(line - 1 for line in needed_lines) + max_needed = _max(needed0) if map_path is None: chunks = self._inline_chunks(self._bundle) else: @@ -340,17 +341,17 @@ class _CriboSourceMapRuntime(object): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _load_json_fallback(self, needed_lines): + def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" location = self._map_location() if location is None: return None map_path, map_dir = location - import json + json = self._import("json") if map_path is None: raw = b"".join(self._inline_chunks(self._bundle)) else: - handle = open(map_path, "rb") + handle = _open(map_path, "rb") try: raw = handle.read() finally: @@ -358,17 +359,17 @@ class _CriboSourceMapRuntime(object): data = json.loads(raw.decode("utf-8")) sources = data.get("sources") or [] mappings = data.get("mappings") or "" - needed0 = set(line - 1 for line in needed_lines) + needed0 = _set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) - table0, _terminated = self._decode_vlq(stream, needed0, max(needed0)) + table0, _terminated = self._decode_vlq(stream, needed0, _max(needed0)) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _collect_needed(self, exc_value, traceback_obj): + def _collect_needed(self, exc_value, traceback_obj, *, _set=set, _id=id, _getattr=getattr): """1-based bundle lines referenced by the traceback (and its chain).""" - needed = set() + needed = _set() def add(tb): while tb is not None: @@ -377,34 +378,34 @@ class _CriboSourceMapRuntime(object): tb = tb.tb_next add(traceback_obj) exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - add(getattr(exc, "__traceback__", None)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + add(_getattr(exc, "__traceback__", None)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) exc = cause if cause is not None else context return needed - def _chain_has_group(self, exc_value): + def _chain_has_group(self, exc_value, *, _set=set, _id=id, _getattr=getattr, _isinstance=isinstance): """Whether the exception chain contains a BaseExceptionGroup.\n\n CPython renders groups with a dedicated nested layout; rather than\n losing the nested tracebacks, the runtime defers group rendering\n entirely to the previous hook (unremapped but complete).\n """ if self._group_type is None: return False exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - if isinstance(exc, self._group_type): + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + if _isinstance(exc, self._group_type): return True - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) exc = cause if cause is not None else context return False - def _source_line(self, path, lineno): + def _source_line(self, path, lineno, *, _open=open): """Read a single 1-based line from a file without caching it.""" try: - handle = open(path, "rb") + handle = _open(path, "rb") except OSError: return None try: @@ -421,12 +422,12 @@ class _CriboSourceMapRuntime(object): handle.close() return None - def _effective_tb_limit(self): + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance): """The application's `sys.tracebacklimit`, or None when unset/invalid.""" - limit = getattr(self._sys, "tracebacklimit", None) - return limit if isinstance(limit, int) else None + limit = _getattr(self._sys, "tracebacklimit", None) + return limit if _isinstance(limit, int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len): """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ if limit is not None: total = 0 @@ -456,7 +457,7 @@ class _CriboSourceMapRuntime(object): name = frame.f_code.co_name if filename == self._bundle: mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + if mapped is not None and 0 <= mapped[0] < _len(sources) and sources[mapped[0]]: source = sources[mapped[0]] if not self._os.path.isabs(source): source = self._os.path.normpath(self._os.path.join(map_dir, source)) @@ -476,31 +477,31 @@ class _CriboSourceMapRuntime(object): if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - def _exception_line(self, exc_value): + def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str): """Minimal `Type: message` line, the fallback formatter.""" - exc_type = type(exc_value) - name = getattr(exc_type, "__qualname__", exc_type.__name__) - module = getattr(exc_type, "__module__", None) + exc_type = _type(exc_value) + name = _getattr(exc_type, "__qualname__", exc_type.__name__) + module = _getattr(exc_type, "__module__", None) if module not in (None, "builtins", "__main__"): name = "%s.%s" % (module, name) try: - text = str(exc_value) + text = _str(exc_value) except BaseException: text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name - def _write_exception_only(self, exc, write): + def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ try: - traceback_mod = _cribo_sm_import("traceback") - te = traceback_mod.TracebackException(type(exc), exc, getattr(exc, "__traceback__", None), lookup_lines=False) + traceback_mod = self._import("traceback") + te = traceback_mod.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) for line in te.format_exception_only(): write(line) return except BaseException: pass write(self._exception_line(exc)) - notes = getattr(exc, "__notes__", None) + notes = _getattr(exc, "__notes__", None) if notes: try: for note in notes: @@ -508,16 +509,16 @@ class _CriboSourceMapRuntime(object): except BaseException: pass - def _render(self, exc_value, table, sources, map_dir, write): + def _render(self, exc_value, table, sources, map_dir, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - suppress = getattr(exc, "__suppress_context__", False) + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + suppress = _getattr(exc, "__suppress_context__", False) if cause is not None: chain.append((exc, "cause")) exc = cause @@ -527,23 +528,23 @@ class _CriboSourceMapRuntime(object): else: chain.append((exc, None)) exc = None - ordered = list(reversed(chain)) - for index, (exc, link) in enumerate(ordered): + ordered = _list(_reversed(chain)) + for index, (exc, link) in _enumerate(ordered): if index > 0: if link == "cause": write("\nThe above exception was the direct cause of the following " "exception:\n\n") else: write("\nDuring handling of the above exception, another exception " "occurred:\n\n") - tb = getattr(exc, "__traceback__", None) + tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") self._write_frames(tb, table, sources, map_dir, write, limit) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix): + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ - if getattr(self._local, "in_hook", False) or exc_value is None: + if _getattr(self._local, "in_hook", False) or exc_value is None: return False self._local.in_hook = True old_limit = None @@ -606,20 +607,20 @@ class _CriboSourceMapRuntime(object): return self._prev_excepthook(exc_type, exc_value, traceback_obj) - def threading_hook(self, args): - if args.exc_type is not None and issubclass(args.exc_type, SystemExit): + def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass): + if args.exc_type is not None and _issubclass(args.exc_type, SystemExit): self._prev_threading_hook(args) return - thread = getattr(args, "thread", None) - name = getattr(thread, "name", None) or "Thread" + thread = _getattr(args, "thread", None) + name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) + self._notify_custom_hook(self._prev_threading_hook, _getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) return self._prev_threading_hook(args) - def unraisablehook(self, unraisable): - message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + def unraisablehook(self, unraisable, *, _getattr=getattr): + message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" try: prefix = "%s: %r\n" % (message, unraisable.object) except BaseException: diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 5f54a0bc3..2fe928215 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -7,30 +7,30 @@ input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py # https://github.com/ophidiarium/cribo import sys as _cribo_sys -def _cribo_sm_import(name): +def _cribo_sm_import(name, *, _list=list, _import=__import__): """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with that first path\n entry dropped. (`sys` itself is a builtin and can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - _cribo_sys.path = list(saved_path[1:]) + _cribo_sys.path = _list(saved_path[1:]) try: - return __import__(name) + return _import(name) finally: _cribo_sys.path = saved_path class _CriboSmStream(object): - """Byte-at-a-time reader over an iterator of byte chunks.""" + """Byte-at-a-time reader over an iterator of byte chunks.\n\n Keyword-only defaults snapshot the builtins at definition time (before any\n bundled user code runs), so later shadowing of e.g. `len` cannot break the\n reader — the same idiom cribo's generated module proxies use.\n """ __slots__ = "_chunks", "_buf", "_pos" - def __init__(self, chunks): - self._chunks = iter(chunks) + def __init__(self, chunks, *, _iter=iter): + self._chunks = _iter(chunks) self._buf = b"" self._pos = 0 - def read_byte(self): - while self._pos >= len(self._buf): + def read_byte(self, *, _len=len, _next=next): + while self._pos >= _len(self._buf): try: - self._buf = next(self._chunks) + self._buf = _next(self._chunks) except StopIteration: return -1 self._pos = 0 @@ -38,7 +38,7 @@ class _CriboSmStream(object): self._pos += 1 return value class _CriboSourceMapRuntime(object): - """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, so the installed hooks keep working even\n if bundled user code later rebinds any module-level name this template\n introduced.\n """ + """Traceback-remapping runtime.\n\n All collaborators (modules, the stream class, previous hooks) are bound to\n the instance at construction time, and every method snapshots the builtins\n it needs via keyword-only defaults evaluated at class-definition time — so\n the installed hooks keep working even if bundled user code later rebinds\n any module-level name this template introduced, or common builtins such as\n `open`, `len`, or `max`.\n """ _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 @@ -50,6 +50,7 @@ class _CriboSourceMapRuntime(object): self._binascii = binascii_mod self._threading = threading_mod self._stream_cls = _CriboSmStream + self._import = _cribo_sm_import self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook @@ -96,9 +97,9 @@ class _CriboSourceMapRuntime(object): return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) return None - def _file_chunks(self, path): + def _file_chunks(self, path, *, _open=open): """Yield fixed-size chunks of a file (constant memory).""" - handle = open(path, "rb") + handle = _open(path, "rb") try: while True: chunk = handle.read(self._CHUNK) @@ -108,7 +109,7 @@ class _CriboSourceMapRuntime(object): finally: handle.close() - def _find_inline_payload(self, handle): + def _find_inline_payload(self, handle, *, _len=len): """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of\n the file is examined; the bundle body is never read.\n """ marker = b"# sourceMappingURL=data:" handle.seek(0, 2) @@ -124,7 +125,7 @@ class _CriboSourceMapRuntime(object): if index >= 0: found = position + index break - overlap = data[:len(marker) - 1] + overlap = data[:_len(marker) - 1] if found < 0: return -1 handle.seek(found) @@ -132,11 +133,11 @@ class _CriboSourceMapRuntime(object): base64_at = head.find(b"base64,") if base64_at < 0: return -1 - return found + base64_at + len(b"base64,") + return found + base64_at + _len(b"base64,") - def _inline_chunks(self, path): + def _inline_chunks(self, path, *, _open=open, _len=len): """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = open(path, "rb") + handle = _open(path, "rb") try: start = self._find_inline_payload(handle) if start < 0: @@ -148,12 +149,12 @@ class _CriboSourceMapRuntime(object): if not raw: break data = pending + raw.translate(None, b"\r\n") - usable = len(data) - len(data) % 4 + usable = _len(data) - _len(data) % 4 pending = data[usable:] if usable: yield self._binascii.a2b_base64(data[:usable]) if pending: - yield self._binascii.a2b_base64(pending + b"=" * (-len(pending) % 4)) + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) finally: handle.close() @@ -162,9 +163,9 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string(self, stream, collect): + def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range): """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ - buf = bytearray() if collect else None + buf = _bytearray() if collect else None while True: byte = stream.read_byte() if byte < 0: @@ -180,13 +181,13 @@ class _CriboSourceMapRuntime(object): raise ValueError("unterminated JSON escape") if escape == 117: code = 0 - for _ in range(4): + for _ in _range(4): digit = stream.read_byte() if digit < 0: raise ValueError("unterminated unicode escape") - code = code * 16 + int(chr(digit), 16) + code = code * 16 + _int(_chr(digit), 16) if buf is not None: - buf.extend(chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) elif buf is not None: table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf.append(table.get(escape, escape)) @@ -234,10 +235,10 @@ class _CriboSourceMapRuntime(object): raise ValueError("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed): + def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len): """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} - for index in range(64): + for index in _range(64): lut[self._B64[index]] = index result = {} gen_line = 0 @@ -259,7 +260,7 @@ class _CriboSourceMapRuntime(object): end_segment() gen_line += 1 field = 0 - if gen_line > max_needed or len(result) == len(needed): + if gen_line > max_needed or _len(result) == _len(needed): return result, False continue if byte == 44: @@ -322,14 +323,14 @@ class _CriboSourceMapRuntime(object): raise ValueError("no mappings field") return sources, table - def _load(self, needed_lines): + def _load(self, needed_lines, *, _set=set, _max=max): """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ location = self._map_location() if location is None: return None map_path, map_dir = location - needed0 = set(line - 1 for line in needed_lines) - max_needed = max(needed0) + needed0 = _set(line - 1 for line in needed_lines) + max_needed = _max(needed0) if map_path is None: chunks = self._inline_chunks(self._bundle) else: @@ -340,17 +341,17 @@ class _CriboSourceMapRuntime(object): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _load_json_fallback(self, needed_lines): + def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" location = self._map_location() if location is None: return None map_path, map_dir = location - import json + json = self._import("json") if map_path is None: raw = b"".join(self._inline_chunks(self._bundle)) else: - handle = open(map_path, "rb") + handle = _open(map_path, "rb") try: raw = handle.read() finally: @@ -358,17 +359,17 @@ class _CriboSourceMapRuntime(object): data = json.loads(raw.decode("utf-8")) sources = data.get("sources") or [] mappings = data.get("mappings") or "" - needed0 = set(line - 1 for line in needed_lines) + needed0 = _set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) - table0, _terminated = self._decode_vlq(stream, needed0, max(needed0)) + table0, _terminated = self._decode_vlq(stream, needed0, _max(needed0)) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _collect_needed(self, exc_value, traceback_obj): + def _collect_needed(self, exc_value, traceback_obj, *, _set=set, _id=id, _getattr=getattr): """1-based bundle lines referenced by the traceback (and its chain).""" - needed = set() + needed = _set() def add(tb): while tb is not None: @@ -377,34 +378,34 @@ class _CriboSourceMapRuntime(object): tb = tb.tb_next add(traceback_obj) exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - add(getattr(exc, "__traceback__", None)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + add(_getattr(exc, "__traceback__", None)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) exc = cause if cause is not None else context return needed - def _chain_has_group(self, exc_value): + def _chain_has_group(self, exc_value, *, _set=set, _id=id, _getattr=getattr, _isinstance=isinstance): """Whether the exception chain contains a BaseExceptionGroup.\n\n CPython renders groups with a dedicated nested layout; rather than\n losing the nested tracebacks, the runtime defers group rendering\n entirely to the previous hook (unremapped but complete).\n """ if self._group_type is None: return False exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - if isinstance(exc, self._group_type): + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + if _isinstance(exc, self._group_type): return True - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) exc = cause if cause is not None else context return False - def _source_line(self, path, lineno): + def _source_line(self, path, lineno, *, _open=open): """Read a single 1-based line from a file without caching it.""" try: - handle = open(path, "rb") + handle = _open(path, "rb") except OSError: return None try: @@ -421,12 +422,12 @@ class _CriboSourceMapRuntime(object): handle.close() return None - def _effective_tb_limit(self): + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance): """The application's `sys.tracebacklimit`, or None when unset/invalid.""" - limit = getattr(self._sys, "tracebacklimit", None) - return limit if isinstance(limit, int) else None + limit = _getattr(self._sys, "tracebacklimit", None) + return limit if _isinstance(limit, int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len): """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ if limit is not None: total = 0 @@ -456,7 +457,7 @@ class _CriboSourceMapRuntime(object): name = frame.f_code.co_name if filename == self._bundle: mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < len(sources) and sources[mapped[0]]: + if mapped is not None and 0 <= mapped[0] < _len(sources) and sources[mapped[0]]: source = sources[mapped[0]] if not self._os.path.isabs(source): source = self._os.path.normpath(self._os.path.join(map_dir, source)) @@ -476,31 +477,31 @@ class _CriboSourceMapRuntime(object): if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - def _exception_line(self, exc_value): + def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str): """Minimal `Type: message` line, the fallback formatter.""" - exc_type = type(exc_value) - name = getattr(exc_type, "__qualname__", exc_type.__name__) - module = getattr(exc_type, "__module__", None) + exc_type = _type(exc_value) + name = _getattr(exc_type, "__qualname__", exc_type.__name__) + module = _getattr(exc_type, "__module__", None) if module not in (None, "builtins", "__main__"): name = "%s.%s" % (module, name) try: - text = str(exc_value) + text = _str(exc_value) except BaseException: text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name - def _write_exception_only(self, exc, write): + def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ try: - traceback_mod = _cribo_sm_import("traceback") - te = traceback_mod.TracebackException(type(exc), exc, getattr(exc, "__traceback__", None), lookup_lines=False) + traceback_mod = self._import("traceback") + te = traceback_mod.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) for line in te.format_exception_only(): write(line) return except BaseException: pass write(self._exception_line(exc)) - notes = getattr(exc, "__notes__", None) + notes = _getattr(exc, "__notes__", None) if notes: try: for note in notes: @@ -508,16 +509,16 @@ class _CriboSourceMapRuntime(object): except BaseException: pass - def _render(self, exc_value, table, sources, map_dir, write): + def _render(self, exc_value, table, sources, map_dir, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value - seen = set() - while exc is not None and id(exc) not in seen: - seen.add(id(exc)) - cause = getattr(exc, "__cause__", None) - context = getattr(exc, "__context__", None) - suppress = getattr(exc, "__suppress_context__", False) + seen = _set() + while exc is not None and _id(exc) not in seen: + seen.add(_id(exc)) + cause = _getattr(exc, "__cause__", None) + context = _getattr(exc, "__context__", None) + suppress = _getattr(exc, "__suppress_context__", False) if cause is not None: chain.append((exc, "cause")) exc = cause @@ -527,23 +528,23 @@ class _CriboSourceMapRuntime(object): else: chain.append((exc, None)) exc = None - ordered = list(reversed(chain)) - for index, (exc, link) in enumerate(ordered): + ordered = _list(_reversed(chain)) + for index, (exc, link) in _enumerate(ordered): if index > 0: if link == "cause": write("\nThe above exception was the direct cause of the following " "exception:\n\n") else: write("\nDuring handling of the above exception, another exception " "occurred:\n\n") - tb = getattr(exc, "__traceback__", None) + tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): write("Traceback (most recent call last):\n") self._write_frames(tb, table, sources, map_dir, write, limit) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix): + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ - if getattr(self._local, "in_hook", False) or exc_value is None: + if _getattr(self._local, "in_hook", False) or exc_value is None: return False self._local.in_hook = True old_limit = None @@ -606,20 +607,20 @@ class _CriboSourceMapRuntime(object): return self._prev_excepthook(exc_type, exc_value, traceback_obj) - def threading_hook(self, args): - if args.exc_type is not None and issubclass(args.exc_type, SystemExit): + def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass): + if args.exc_type is not None and _issubclass(args.exc_type, SystemExit): self._prev_threading_hook(args) return - thread = getattr(args, "thread", None) - name = getattr(thread, "name", None) or "Thread" + thread = _getattr(args, "thread", None) + name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) + self._notify_custom_hook(self._prev_threading_hook, _getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) return self._prev_threading_hook(args) - def unraisablehook(self, unraisable): - message = getattr(unraisable, "err_msg", None) or "Exception ignored in" + def unraisablehook(self, unraisable, *, _getattr=getattr): + message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" try: prefix = "%s: %r\n" % (message, unraisable.object) except BaseException: diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 9c591a629..742b93a60 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:753 `def add(a, b):` -> calculator.py:1 -bundle:754 `result = a + b` -> calculator.py:2 -bundle:755 `return result` -> calculator.py:3 -bundle:757 `def multiply(a, b):` -> calculator.py:6 -bundle:758 `result = a * b` -> calculator.py:7 -bundle:759 `return result` -> calculator.py:8 -bundle:764 `def describe(name, value):` -> utils.py:1 -bundle:765 `return f"{name} = {value}"` -> utils.py:2 -bundle:771 `total = add(2, 3)` -> main.py:4 -bundle:772 `product = multiply(total, 4)` -> main.py:5 -bundle:773 `print(describe("total", total))` -> main.py:6 -bundle:774 `print(describe("product", product))` -> main.py:7 +bundle:754 `def add(a, b):` -> calculator.py:1 +bundle:755 `result = a + b` -> calculator.py:2 +bundle:756 `return result` -> calculator.py:3 +bundle:758 `def multiply(a, b):` -> calculator.py:6 +bundle:759 `result = a * b` -> calculator.py:7 +bundle:760 `return result` -> calculator.py:8 +bundle:765 `def describe(name, value):` -> utils.py:1 +bundle:766 `return f"{name} = {value}"` -> utils.py:2 +bundle:772 `total = add(2, 3)` -> main.py:4 +bundle:773 `product = multiply(total, 4)` -> main.py:5 +bundle:774 `print(describe("total", total))` -> main.py:6 +bundle:775 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 038c5091b..a7bd65f69 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:762 `print("effects module loading")` -> effects.py:1 -bundle:763 `COUNTER = 1` -> effects.py:3 -bundle:766 `def boost(value):` -> effects.py:6 -bundle:767 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:768 `return boosted` -> effects.py:8 -bundle:780 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:781 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:763 `print("effects module loading")` -> effects.py:1 +bundle:764 `COUNTER = 1` -> effects.py:3 +bundle:767 `def boost(value):` -> effects.py:6 +bundle:768 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:769 `return boosted` -> effects.py:8 +bundle:781 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:782 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 52533706f..ad917db5e 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1107,3 +1107,32 @@ fn map_covers_match_case_headers_and_decorators() { ); } } + +#[test] +fn runtime_survives_shadowed_builtins() { + // Entry code that rebinds common builtins at module level must not break + // the runtime: every method snapshots its builtins at definition time. + let dir = make_project(&[ + ( + "main.py", + "from helper import boom\n\nopen = None\nlen = None\nmax = None\nset = \ + None\ngetattr = None\nboom()\n", + ), + ( + "helper.py", + "def boom():\n inner()\n\ndef inner():\n raise ValueError(\"kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "traceback must be remapped despite shadowed builtins: {stderr}" + ); + assert!( + stderr.contains("main.py\", line 8, in "), + "entry frame must be remapped despite shadowed builtins: {stderr}" + ); + assert!(!stderr.contains("bundle.py\", line"), "{stderr}"); +} From ccfcddea8a28842f4377547ac805ec5da9035667 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 18:22:04 +0200 Subject: [PATCH 06/17] fix: address fifth review round on source map runtime - snapshot exception classes (BaseException, ValueError, OSError, StopIteration, SystemExit) and int at definition time, so bundled code rebinding them cannot make the hook itself raise - capture the genuine stdlib traceback module at bootstrap, before bundled first-party modules can register a shadowing sys.modules entry - stage the map under a process-unique temp name so concurrent builds to the same output cannot stomp each other's staged file - avoid != in the template: it is spliced after hoisted future imports and must stay valid under barry_as_FLUFL Addresses fifth-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 6 +- crates/cribo/src/python/sourcemap_runtime.py | 122 +++++++++++------- .../tests/python/test_sourcemap_runtime.py | 14 +- .../bundled_code@sourcemap_basic.snap | 102 +++++++-------- .../bundled_code@sourcemap_wrapper.snap | 102 +++++++-------- docs/source-maps.md | 11 +- 6 files changed, 198 insertions(+), 159 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 5682f3f39..49408c038 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -749,7 +749,11 @@ impl BundleOrchestrator { || std::ffi::OsString::from("bundle.py.map"), std::ffi::OsStr::to_os_string, ); - tmp_name.push(".tmp"); + // Process-unique staging name: concurrent builds targeting the same + // output must not stomp each other's staged map (concurrent writers + // to one output path remain externally undefined, as for the bundle + // file itself, but each publish stays internally consistent). + tmp_name.push(format!(".{}.tmp", std::process::id())); let tmp_path = map_path.with_file_name(tmp_name); fs::write(&tmp_path, map_json).with_context(|| { format!("Failed to stage source map file: {}", tmp_path.display()) diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 58f8af981..12d806b18 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -49,11 +49,11 @@ def __init__(self, chunks, *, _iter=iter): self._buf = b"" self._pos = 0 - def read_byte(self, *, _len=len, _next=next): + def read_byte(self, *, _len=len, _next=next, _stop=StopIteration): while self._pos >= _len(self._buf): try: self._buf = _next(self._chunks) - except StopIteration: + except _stop: return -1 self._pos = 0 value = self._buf[self._pos] @@ -75,7 +75,7 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): self._mode = mode self._bundle = bundle_file self._os = os_mod @@ -84,6 +84,10 @@ def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): self._threading = threading_mod self._stream_cls = _CriboSmStream self._import = _cribo_sm_import + # Captured at construction (before any bundled user code runs) so a + # first-party module registering sys.modules["traceback"] later cannot + # degrade exception formatting. + self._traceback = traceback_mod # Re-entrancy guard; thread-local so a hook firing on one thread never # disables remapping on another. self._local = threading_mod.local() @@ -115,6 +119,7 @@ def _bootstrap(cls, mode, bundle_file): _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), + _cribo_sm_import("traceback"), ) runtime.install() except BaseException: @@ -232,7 +237,15 @@ def _skip_ws(self, stream, byte): return byte def _read_string( - self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range + self, + stream, + collect, + *, + _bytearray=bytearray, + _int=int, + _chr=chr, + _range=range, + _error=ValueError, ): """Consume a JSON string whose opening quote was already read. @@ -245,22 +258,22 @@ def _read_string( while True: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON string") + raise _error("unterminated JSON string") if byte == 34: # '"' return buf.decode("utf-8", "replace") if collect else None - if byte != 92: # '\\' + if not byte == 92: # '\\' if buf is not None: buf.append(byte) continue escape = stream.read_byte() if escape < 0: - raise ValueError("unterminated JSON escape") + raise _error("unterminated JSON escape") if escape == 117: # 'u' code = 0 for _ in _range(4): digit = stream.read_byte() if digit < 0: - raise ValueError("unterminated unicode escape") + raise _error("unterminated unicode escape") code = code * 16 + _int(_chr(digit), 16) if buf is not None: buf.extend(_chr(code).encode("utf-8", "surrogatepass")) @@ -268,7 +281,7 @@ def _read_string( table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf.append(table.get(escape, escape)) - def _skip_value(self, stream, byte): + def _skip_value(self, stream, byte, *, _error=ValueError): """Skip one JSON value; return the first byte after it (or -1).""" if byte == 34: # string self._read_string(stream, False) @@ -278,7 +291,7 @@ def _skip_value(self, stream, byte): while depth > 0: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON container") + raise _error("unterminated JSON container") if byte == 34: self._read_string(stream, False) elif byte in (123, 91): @@ -291,10 +304,10 @@ def _skip_value(self, stream, byte): byte = stream.read_byte() return byte - def _read_string_array(self, stream, byte): + def _read_string_array(self, stream, byte, *, _error=ValueError): """Read a JSON array of strings/nulls; return (list, byte after array).""" - if byte != 91: # '[' - raise ValueError("expected array") + if not byte == 91: # '[' + raise _error("expected array") items = [] byte = self._skip_ws(stream, stream.read_byte()) if byte == 93: # ']' @@ -308,11 +321,13 @@ def _read_string_array(self, stream, byte): byte = self._skip_ws(stream, self._skip_value(stream, byte)) if byte == 93: return items, stream.read_byte() - if byte != 44: # ',' - raise ValueError("malformed array") + if not byte == 44: # ',' + raise _error("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len): + def _decode_vlq( + self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError + ): """Streaming VLQ state machine over the raw bytes of the mappings string. Constant state: line/segment counters plus running deltas. Records the @@ -354,7 +369,7 @@ def end_segment(): continue value = lut.get(byte) if value is None: - raise ValueError("unexpected byte in mappings") + raise _error("unexpected byte in mappings") vlq_value += (value & 31) << vlq_shift if value & 32: vlq_shift += 5 @@ -368,12 +383,12 @@ def end_segment(): vlq_value = 0 vlq_shift = 0 - def _scan(self, chunks, needed, max_needed): + def _scan(self, chunks, needed, max_needed, *, _error=ValueError): """Scan the map's top-level object; return (sources, line table).""" stream = self._stream_cls(chunks) byte = self._skip_ws(stream, stream.read_byte()) - if byte != 123: # '{' - raise ValueError("not a JSON object") + if not byte == 123: # '{' + raise _error("not a JSON object") sources = [] table = {} saw_mappings = False @@ -381,14 +396,14 @@ def _scan(self, chunks, needed, max_needed): while byte == 34: # '"' starting a key key = self._read_string(stream, True) byte = self._skip_ws(stream, stream.read_byte()) - if byte != 58: # ':' - raise ValueError("malformed object") + if not byte == 58: # ':' + raise _error("malformed object") byte = self._skip_ws(stream, stream.read_byte()) if key == "sources": sources, byte = self._read_string_array(stream, byte) elif key == "mappings": - if byte != 34: - raise ValueError("mappings is not a string") + if not byte == 34: + raise _error("mappings is not a string") table, terminated = self._decode_vlq(stream, needed, max_needed) saw_mappings = True if sources: @@ -411,7 +426,7 @@ def _scan(self, chunks, needed, max_needed): if byte == 44: # ',' byte = self._skip_ws(stream, stream.read_byte()) if not saw_mappings: - raise ValueError("no mappings field") + raise _error("no mappings field") return sources, table # -- loading ------------------------------------------------------------- @@ -513,11 +528,11 @@ def _chain_has_group( exc = cause if cause is not None else context return False - def _source_line(self, path, lineno, *, _open=open): + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): """Read a single 1-based line from a file without caching it.""" try: handle = _open(path, "rb") - except OSError: + except _os_error: return None try: current = 0 @@ -527,16 +542,16 @@ def _source_line(self, path, lineno, *, _open=open): return raw.decode("utf-8", "replace").strip() if current > lineno: break - except OSError: + except _os_error: return None finally: handle.close() return None - def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance): + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int=int): """The application's `sys.tracebacklimit`, or None when unset/invalid.""" limit = _getattr(self._sys, "tracebacklimit", None) - return limit if _isinstance(limit, int) else None + return limit if _isinstance(limit, _int) else None def _write_frames( self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len @@ -601,7 +616,9 @@ def emit(entry): if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str): + def _exception_line( + self, exc_value, *, _type=type, _getattr=getattr, _str=str, _bex=BaseException + ): """Minimal `Type: message` line, the fallback formatter.""" exc_type = _type(exc_value) name = _getattr(exc_type, "__qualname__", exc_type.__name__) @@ -610,11 +627,13 @@ def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str): name = "%s.%s" % (module, name) try: text = _str(exc_value) - except BaseException: + except _bex: text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name - def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): + def _write_exception_only( + self, exc, write, *, _type=type, _getattr=getattr, _bex=BaseException + ): """Write the exception line(s) with full standard-library fidelity. `traceback.format_exception_only` supplies the interpreter's @@ -624,8 +643,7 @@ def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): line (plus notes) when the traceback module is unavailable. """ try: - traceback_mod = self._import("traceback") - te = traceback_mod.TracebackException( + te = self._traceback.TracebackException( _type(exc), exc, _getattr(exc, "__traceback__", None), @@ -634,7 +652,7 @@ def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): for line in te.format_exception_only(): write(line) return - except BaseException: + except _bex: pass write(self._exception_line(exc)) notes = _getattr(exc, "__notes__", None) @@ -642,7 +660,7 @@ def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): try: for note in notes: write("%s\n" % (note,)) - except BaseException: + except _bex: pass def _render( @@ -701,7 +719,9 @@ def _render( self._write_frames(tb, table, sources, map_dir, write, limit) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): + def _try_render( + self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException + ): """Attempt a remapped rendering to stderr; True on success. Never raises and never masks the original exception: any failure in @@ -721,10 +741,10 @@ def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): loaded = None try: loaded = self._load(needed) - except BaseException: + except _bex: try: loaded = self._load_json_fallback(needed) - except BaseException: + except _bex: loaded = None if not loaded: return False @@ -734,7 +754,7 @@ def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): try: old_limit = self._sys.getrecursionlimit() self._sys.setrecursionlimit(old_limit + 64) - except BaseException: + except _bex: old_limit = None # Buffer the rendering so a mid-render failure produces no partial # output before the previous hook prints the standard traceback. @@ -746,20 +766,20 @@ def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): stderr.write("".join(parts)) try: stderr.flush() - except BaseException: + except _bex: pass return True - except BaseException: + except _bex: return False finally: if old_limit is not None: try: self._sys.setrecursionlimit(old_limit) - except BaseException: + except _bex: pass self._local.in_hook = False - def _notify_custom_hook(self, prev, default, call): + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException): """Invoke a chained hook after a successful remap when it is custom. A successful remap replaces the *default* printer, but preinstalled @@ -772,7 +792,7 @@ def _notify_custom_hook(self, prev, default, call): if default is not None and prev is not None and prev is not default: try: call(prev) - except BaseException: + except _bex: pass # -- installed hooks ------------------------------------------------------ @@ -787,10 +807,12 @@ def excepthook(self, exc_type, exc_value, traceback_obj): return self._prev_excepthook(exc_type, exc_value, traceback_obj) - def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass): + def threading_hook( + self, args, *, _getattr=getattr, _issubclass=issubclass, _system_exit=SystemExit + ): # The default threading hook deliberately ignores SystemExit (normal # sys.exit() in a worker thread); preserve that by delegating. - if args.exc_type is not None and _issubclass(args.exc_type, SystemExit): + if args.exc_type is not None and _issubclass(args.exc_type, _system_exit): self._prev_threading_hook(args) return thread = _getattr(args, "thread", None) @@ -805,11 +827,11 @@ def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass): return self._prev_threading_hook(args) - def unraisablehook(self, unraisable, *, _getattr=getattr): + def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" try: prefix = "%s: %r\n" % (message, unraisable.object) - except BaseException: + except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): self._notify_custom_hook( diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index 39306d5c0..717acaf23 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -27,7 +27,12 @@ def load_runtime(path): spec.loader.exec_module(module) sys.excepthook, sys.unraisablehook, threading.excepthook = prev_hooks return module._CriboSourceMapRuntime( - "external", "", os, __import__("binascii"), threading + "external", + "", + os, + __import__("binascii"), + threading, + __import__("traceback"), ) @@ -213,7 +218,12 @@ def test_env_path_wins_for_every_mode(rt): try: os.environ["CRIBO_SOURCE_MAPS"] = path inline_stdin = type(rt)( - "inline", "", os, __import__("binascii"), threading + "inline", + "", + os, + __import__("binascii"), + threading, + __import__("traceback"), ) loaded = inline_stdin._load({1}) assert loaded is not None, "env path must activate a inline bundle" diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 23563b030..0cc57aa13 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -27,11 +27,11 @@ class _CriboSmStream(object): self._buf = b"" self._pos = 0 - def read_byte(self, *, _len=len, _next=next): + def read_byte(self, *, _len=len, _next=next, _stop=StopIteration): while self._pos >= _len(self._buf): try: self._buf = _next(self._chunks) - except StopIteration: + except _stop: return -1 self._pos = 0 value = self._buf[self._pos] @@ -42,7 +42,7 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): self._mode = mode self._bundle = bundle_file self._os = os_mod @@ -51,6 +51,7 @@ class _CriboSourceMapRuntime(object): self._threading = threading_mod self._stream_cls = _CriboSmStream self._import = _cribo_sm_import + self._traceback = traceback_mod self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook @@ -70,7 +71,7 @@ class _CriboSourceMapRuntime(object): def _bootstrap(cls, mode, bundle_file): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: - runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading")) + runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback")) runtime.install() except BaseException: pass @@ -163,28 +164,28 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range): + def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range, _error=ValueError): """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ buf = _bytearray() if collect else None while True: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON string") + raise _error("unterminated JSON string") if byte == 34: return buf.decode("utf-8", "replace") if collect else None - if byte != 92: + if not byte == 92: if buf is not None: buf.append(byte) continue escape = stream.read_byte() if escape < 0: - raise ValueError("unterminated JSON escape") + raise _error("unterminated JSON escape") if escape == 117: code = 0 for _ in _range(4): digit = stream.read_byte() if digit < 0: - raise ValueError("unterminated unicode escape") + raise _error("unterminated unicode escape") code = code * 16 + _int(_chr(digit), 16) if buf is not None: buf.extend(_chr(code).encode("utf-8", "surrogatepass")) @@ -192,7 +193,7 @@ class _CriboSourceMapRuntime(object): table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf.append(table.get(escape, escape)) - def _skip_value(self, stream, byte): + def _skip_value(self, stream, byte, *, _error=ValueError): """Skip one JSON value; return the first byte after it (or -1).""" if byte == 34: self._read_string(stream, False) @@ -202,7 +203,7 @@ class _CriboSourceMapRuntime(object): while depth > 0: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON container") + raise _error("unterminated JSON container") if byte == 34: self._read_string(stream, False) elif byte in (123, 91): @@ -214,10 +215,10 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string_array(self, stream, byte): + def _read_string_array(self, stream, byte, *, _error=ValueError): """Read a JSON array of strings/nulls; return (list, byte after array).""" - if byte != 91: - raise ValueError("expected array") + if not byte == 91: + raise _error("expected array") items = [] byte = self._skip_ws(stream, stream.read_byte()) if byte == 93: @@ -231,11 +232,11 @@ class _CriboSourceMapRuntime(object): byte = self._skip_ws(stream, self._skip_value(stream, byte)) if byte == 93: return items, stream.read_byte() - if byte != 44: - raise ValueError("malformed array") + if not byte == 44: + raise _error("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len): + def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError): """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} for index in _range(64): @@ -269,7 +270,7 @@ class _CriboSourceMapRuntime(object): continue value = lut.get(byte) if value is None: - raise ValueError("unexpected byte in mappings") + raise _error("unexpected byte in mappings") vlq_value += (value & 31) << vlq_shift if value & 32: vlq_shift += 5 @@ -283,12 +284,12 @@ class _CriboSourceMapRuntime(object): vlq_value = 0 vlq_shift = 0 - def _scan(self, chunks, needed, max_needed): + def _scan(self, chunks, needed, max_needed, *, _error=ValueError): """Scan the map's top-level object; return (sources, line table).""" stream = self._stream_cls(chunks) byte = self._skip_ws(stream, stream.read_byte()) - if byte != 123: - raise ValueError("not a JSON object") + if not byte == 123: + raise _error("not a JSON object") sources = [] table = {} saw_mappings = False @@ -296,14 +297,14 @@ class _CriboSourceMapRuntime(object): while byte == 34: key = self._read_string(stream, True) byte = self._skip_ws(stream, stream.read_byte()) - if byte != 58: - raise ValueError("malformed object") + if not byte == 58: + raise _error("malformed object") byte = self._skip_ws(stream, stream.read_byte()) if key == "sources": sources, byte = self._read_string_array(stream, byte) elif key == "mappings": - if byte != 34: - raise ValueError("mappings is not a string") + if not byte == 34: + raise _error("mappings is not a string") table, terminated = self._decode_vlq(stream, needed, max_needed) saw_mappings = True if sources: @@ -320,7 +321,7 @@ class _CriboSourceMapRuntime(object): if byte == 44: byte = self._skip_ws(stream, stream.read_byte()) if not saw_mappings: - raise ValueError("no mappings field") + raise _error("no mappings field") return sources, table def _load(self, needed_lines, *, _set=set, _max=max): @@ -402,11 +403,11 @@ class _CriboSourceMapRuntime(object): exc = cause if cause is not None else context return False - def _source_line(self, path, lineno, *, _open=open): + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): """Read a single 1-based line from a file without caching it.""" try: handle = _open(path, "rb") - except OSError: + except _os_error: return None try: current = 0 @@ -416,16 +417,16 @@ class _CriboSourceMapRuntime(object): return raw.decode("utf-8", "replace").strip() if current > lineno: break - except OSError: + except _os_error: return None finally: handle.close() return None - def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance): + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int=int): """The application's `sys.tracebacklimit`, or None when unset/invalid.""" limit = _getattr(self._sys, "tracebacklimit", None) - return limit if _isinstance(limit, int) else None + return limit if _isinstance(limit, _int) else None def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len): """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ @@ -477,7 +478,7 @@ class _CriboSourceMapRuntime(object): if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str): + def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str, _bex=BaseException): """Minimal `Type: message` line, the fallback formatter.""" exc_type = _type(exc_value) name = _getattr(exc_type, "__qualname__", exc_type.__name__) @@ -486,19 +487,18 @@ class _CriboSourceMapRuntime(object): name = "%s.%s" % (module, name) try: text = _str(exc_value) - except BaseException: + except _bex: text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name - def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): + def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr, _bex=BaseException): """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ try: - traceback_mod = self._import("traceback") - te = traceback_mod.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) + te = self._traceback.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) for line in te.format_exception_only(): write(line) return - except BaseException: + except _bex: pass write(self._exception_line(exc)) notes = _getattr(exc, "__notes__", None) @@ -506,7 +506,7 @@ class _CriboSourceMapRuntime(object): try: for note in notes: write("%s\n" % (note,)) - except BaseException: + except _bex: pass def _render(self, exc_value, table, sources, map_dir, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate): @@ -542,7 +542,7 @@ class _CriboSourceMapRuntime(object): self._write_frames(tb, table, sources, map_dir, write, limit) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ if _getattr(self._local, "in_hook", False) or exc_value is None: return False @@ -557,10 +557,10 @@ class _CriboSourceMapRuntime(object): loaded = None try: loaded = self._load(needed) - except BaseException: + except _bex: try: loaded = self._load_json_fallback(needed) - except BaseException: + except _bex: loaded = None if not loaded: return False @@ -570,7 +570,7 @@ class _CriboSourceMapRuntime(object): try: old_limit = self._sys.getrecursionlimit() self._sys.setrecursionlimit(old_limit + 64) - except BaseException: + except _bex: old_limit = None parts = [] if prefix: @@ -580,25 +580,25 @@ class _CriboSourceMapRuntime(object): stderr.write("".join(parts)) try: stderr.flush() - except BaseException: + except _bex: pass return True - except BaseException: + except _bex: return False finally: if old_limit is not None: try: self._sys.setrecursionlimit(old_limit) - except BaseException: + except _bex: pass self._local.in_hook = False - def _notify_custom_hook(self, prev, default, call): + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException): """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. When the interpreter\n default is unavailable for comparison (e.g. `threading.__excepthook__`\n before Python 3.10), no notification happens — better to skip a custom\n hook than to double-print via the default one.\n """ if default is not None and prev is not None and prev is not default: try: call(prev) - except BaseException: + except _bex: pass def excepthook(self, exc_type, exc_value, traceback_obj): @@ -607,8 +607,8 @@ class _CriboSourceMapRuntime(object): return self._prev_excepthook(exc_type, exc_value, traceback_obj) - def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass): - if args.exc_type is not None and _issubclass(args.exc_type, SystemExit): + def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass, _system_exit=SystemExit): + if args.exc_type is not None and _issubclass(args.exc_type, _system_exit): self._prev_threading_hook(args) return thread = _getattr(args, "thread", None) @@ -619,11 +619,11 @@ class _CriboSourceMapRuntime(object): return self._prev_threading_hook(args) - def unraisablehook(self, unraisable, *, _getattr=getattr): + def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" try: prefix = "%s: %r\n" % (message, unraisable.object) - except BaseException: + except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 2fe928215..05d7276bb 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -27,11 +27,11 @@ class _CriboSmStream(object): self._buf = b"" self._pos = 0 - def read_byte(self, *, _len=len, _next=next): + def read_byte(self, *, _len=len, _next=next, _stop=StopIteration): while self._pos >= _len(self._buf): try: self._buf = _next(self._chunks) - except StopIteration: + except _stop: return -1 self._pos = 0 value = self._buf[self._pos] @@ -42,7 +42,7 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): self._mode = mode self._bundle = bundle_file self._os = os_mod @@ -51,6 +51,7 @@ class _CriboSourceMapRuntime(object): self._threading = threading_mod self._stream_cls = _CriboSmStream self._import = _cribo_sm_import + self._traceback = traceback_mod self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook @@ -70,7 +71,7 @@ class _CriboSourceMapRuntime(object): def _bootstrap(cls, mode, bundle_file): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: - runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading")) + runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback")) runtime.install() except BaseException: pass @@ -163,28 +164,28 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range): + def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range, _error=ValueError): """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ buf = _bytearray() if collect else None while True: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON string") + raise _error("unterminated JSON string") if byte == 34: return buf.decode("utf-8", "replace") if collect else None - if byte != 92: + if not byte == 92: if buf is not None: buf.append(byte) continue escape = stream.read_byte() if escape < 0: - raise ValueError("unterminated JSON escape") + raise _error("unterminated JSON escape") if escape == 117: code = 0 for _ in _range(4): digit = stream.read_byte() if digit < 0: - raise ValueError("unterminated unicode escape") + raise _error("unterminated unicode escape") code = code * 16 + _int(_chr(digit), 16) if buf is not None: buf.extend(_chr(code).encode("utf-8", "surrogatepass")) @@ -192,7 +193,7 @@ class _CriboSourceMapRuntime(object): table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf.append(table.get(escape, escape)) - def _skip_value(self, stream, byte): + def _skip_value(self, stream, byte, *, _error=ValueError): """Skip one JSON value; return the first byte after it (or -1).""" if byte == 34: self._read_string(stream, False) @@ -202,7 +203,7 @@ class _CriboSourceMapRuntime(object): while depth > 0: byte = stream.read_byte() if byte < 0: - raise ValueError("unterminated JSON container") + raise _error("unterminated JSON container") if byte == 34: self._read_string(stream, False) elif byte in (123, 91): @@ -214,10 +215,10 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string_array(self, stream, byte): + def _read_string_array(self, stream, byte, *, _error=ValueError): """Read a JSON array of strings/nulls; return (list, byte after array).""" - if byte != 91: - raise ValueError("expected array") + if not byte == 91: + raise _error("expected array") items = [] byte = self._skip_ws(stream, stream.read_byte()) if byte == 93: @@ -231,11 +232,11 @@ class _CriboSourceMapRuntime(object): byte = self._skip_ws(stream, self._skip_value(stream, byte)) if byte == 93: return items, stream.read_byte() - if byte != 44: - raise ValueError("malformed array") + if not byte == 44: + raise _error("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len): + def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError): """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} for index in _range(64): @@ -269,7 +270,7 @@ class _CriboSourceMapRuntime(object): continue value = lut.get(byte) if value is None: - raise ValueError("unexpected byte in mappings") + raise _error("unexpected byte in mappings") vlq_value += (value & 31) << vlq_shift if value & 32: vlq_shift += 5 @@ -283,12 +284,12 @@ class _CriboSourceMapRuntime(object): vlq_value = 0 vlq_shift = 0 - def _scan(self, chunks, needed, max_needed): + def _scan(self, chunks, needed, max_needed, *, _error=ValueError): """Scan the map's top-level object; return (sources, line table).""" stream = self._stream_cls(chunks) byte = self._skip_ws(stream, stream.read_byte()) - if byte != 123: - raise ValueError("not a JSON object") + if not byte == 123: + raise _error("not a JSON object") sources = [] table = {} saw_mappings = False @@ -296,14 +297,14 @@ class _CriboSourceMapRuntime(object): while byte == 34: key = self._read_string(stream, True) byte = self._skip_ws(stream, stream.read_byte()) - if byte != 58: - raise ValueError("malformed object") + if not byte == 58: + raise _error("malformed object") byte = self._skip_ws(stream, stream.read_byte()) if key == "sources": sources, byte = self._read_string_array(stream, byte) elif key == "mappings": - if byte != 34: - raise ValueError("mappings is not a string") + if not byte == 34: + raise _error("mappings is not a string") table, terminated = self._decode_vlq(stream, needed, max_needed) saw_mappings = True if sources: @@ -320,7 +321,7 @@ class _CriboSourceMapRuntime(object): if byte == 44: byte = self._skip_ws(stream, stream.read_byte()) if not saw_mappings: - raise ValueError("no mappings field") + raise _error("no mappings field") return sources, table def _load(self, needed_lines, *, _set=set, _max=max): @@ -402,11 +403,11 @@ class _CriboSourceMapRuntime(object): exc = cause if cause is not None else context return False - def _source_line(self, path, lineno, *, _open=open): + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): """Read a single 1-based line from a file without caching it.""" try: handle = _open(path, "rb") - except OSError: + except _os_error: return None try: current = 0 @@ -416,16 +417,16 @@ class _CriboSourceMapRuntime(object): return raw.decode("utf-8", "replace").strip() if current > lineno: break - except OSError: + except _os_error: return None finally: handle.close() return None - def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance): + def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int=int): """The application's `sys.tracebacklimit`, or None when unset/invalid.""" limit = _getattr(self._sys, "tracebacklimit", None) - return limit if _isinstance(limit, int) else None + return limit if _isinstance(limit, _int) else None def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len): """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ @@ -477,7 +478,7 @@ class _CriboSourceMapRuntime(object): if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) - def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str): + def _exception_line(self, exc_value, *, _type=type, _getattr=getattr, _str=str, _bex=BaseException): """Minimal `Type: message` line, the fallback formatter.""" exc_type = _type(exc_value) name = _getattr(exc_type, "__qualname__", exc_type.__name__) @@ -486,19 +487,18 @@ class _CriboSourceMapRuntime(object): name = "%s.%s" % (module, name) try: text = _str(exc_value) - except BaseException: + except _bex: text = "" return "%s: %s\n" % (name, text) if text else "%s\n" % name - def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr): + def _write_exception_only(self, exc, write, *, _type=type, _getattr=getattr, _bex=BaseException): """Write the exception line(s) with full standard-library fidelity.\n\n `traceback.format_exception_only` supplies the interpreter's\n specialized rendering — SyntaxError source line and caret, NameError /\n AttributeError \"Did you mean\" suggestions, and `__notes__` — so the\n remapped output matches the default hook. Falls back to the minimal\n line (plus notes) when the traceback module is unavailable.\n """ try: - traceback_mod = self._import("traceback") - te = traceback_mod.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) + te = self._traceback.TracebackException(_type(exc), exc, _getattr(exc, "__traceback__", None), lookup_lines=False) for line in te.format_exception_only(): write(line) return - except BaseException: + except _bex: pass write(self._exception_line(exc)) notes = _getattr(exc, "__notes__", None) @@ -506,7 +506,7 @@ class _CriboSourceMapRuntime(object): try: for note in notes: write("%s\n" % (note,)) - except BaseException: + except _bex: pass def _render(self, exc_value, table, sources, map_dir, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate): @@ -542,7 +542,7 @@ class _CriboSourceMapRuntime(object): self._write_frames(tb, table, sources, map_dir, write, limit) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr): + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ if _getattr(self._local, "in_hook", False) or exc_value is None: return False @@ -557,10 +557,10 @@ class _CriboSourceMapRuntime(object): loaded = None try: loaded = self._load(needed) - except BaseException: + except _bex: try: loaded = self._load_json_fallback(needed) - except BaseException: + except _bex: loaded = None if not loaded: return False @@ -570,7 +570,7 @@ class _CriboSourceMapRuntime(object): try: old_limit = self._sys.getrecursionlimit() self._sys.setrecursionlimit(old_limit + 64) - except BaseException: + except _bex: old_limit = None parts = [] if prefix: @@ -580,25 +580,25 @@ class _CriboSourceMapRuntime(object): stderr.write("".join(parts)) try: stderr.flush() - except BaseException: + except _bex: pass return True - except BaseException: + except _bex: return False finally: if old_limit is not None: try: self._sys.setrecursionlimit(old_limit) - except BaseException: + except _bex: pass self._local.in_hook = False - def _notify_custom_hook(self, prev, default, call): + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException): """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. When the interpreter\n default is unavailable for comparison (e.g. `threading.__excepthook__`\n before Python 3.10), no notification happens — better to skip a custom\n hook than to double-print via the default one.\n """ if default is not None and prev is not None and prev is not default: try: call(prev) - except BaseException: + except _bex: pass def excepthook(self, exc_type, exc_value, traceback_obj): @@ -607,8 +607,8 @@ class _CriboSourceMapRuntime(object): return self._prev_excepthook(exc_type, exc_value, traceback_obj) - def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass): - if args.exc_type is not None and _issubclass(args.exc_type, SystemExit): + def threading_hook(self, args, *, _getattr=getattr, _issubclass=issubclass, _system_exit=SystemExit): + if args.exc_type is not None and _issubclass(args.exc_type, _system_exit): self._prev_threading_hook(args) return thread = _getattr(args, "thread", None) @@ -619,11 +619,11 @@ class _CriboSourceMapRuntime(object): return self._prev_threading_hook(args) - def unraisablehook(self, unraisable, *, _getattr=getattr): + def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): message = _getattr(unraisable, "err_msg", None) or "Exception ignored in" try: prefix = "%s: %r\n" % (message, unraisable.object) - except BaseException: + except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) diff --git a/docs/source-maps.md b/docs/source-maps.md index 0f56b4f71..180ac05d5 100644 --- a/docs/source-maps.md +++ b/docs/source-maps.md @@ -203,10 +203,13 @@ Decisions made (or refined) during implementation: consecutive frames, then `[Previous line repeated N more times]`), with a per-render line-text cache; a `RecursionError` traceback stays small and does not trigger thousands of file reads. -- **`threading` is imported at bundle startup** (aliased) so - `threading.excepthook` can be installed; this is the only non-trivial startup - cost and is negligible in practice. Map location checks, environment reads, - file access, and decoding all remain deferred to the first exception. +- **`threading` and `traceback` are imported at bundle startup** (through a + shadow-proof importer that skips the script directory) so + `threading.excepthook` can be installed and exception formatting cannot be + degraded by a first-party module registering `sys.modules["traceback"]`; + these are the only non-trivial startup costs and are negligible in practice. + Map location checks, environment reads, file access, and decoding all remain + deferred to the first exception. - **Snapshot integration:** fixtures under `crates/cribo/tests/fixtures/` whose name starts with `sourcemap_` are bundled with `--sourcemap=linked` and gain a `source_map@.snap` snapshot: a normalized, path-free dump of every From e615e7072096982db87fa81bdbfc896538d4758b Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 22:25:06 +0200 Subject: [PATCH 07/17] fix: address sixth review round on source map runtime - anchor the bundle path at startup so os.chdir before a crash cannot orphan relative linked/inline map lookups (frame matching keeps the as-given spelling) - mark runtime instances and skip an earlier cribo runtime when chaining hooks, so stacked source-mapped bundles render exactly one traceback - combine UTF-16 surrogate pairs when decoding map strings so non-BMP source paths survive intact Addresses sixth-round review comments on #570 --- crates/cribo/src/python/sourcemap_runtime.py | 120 ++++++++++++------ .../tests/python/test_sourcemap_runtime.py | 13 ++ .../bundled_code@sourcemap_basic.snap | 93 ++++++++++---- .../bundled_code@sourcemap_wrapper.snap | 93 ++++++++++---- .../snapshots/source_map@sourcemap_basic.snap | 24 ++-- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 80 ++++++++++++ 7 files changed, 328 insertions(+), 109 deletions(-) diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 12d806b18..6614c74d7 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -77,7 +77,17 @@ class _CriboSourceMapRuntime(object): def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): self._mode = mode + # As-given path for frame matching (co_filename uses the invocation + # spelling), plus a startup-anchored absolute path for file I/O so a + # later os.chdir() in bundled code cannot orphan a relative path. self._bundle = bundle_file + if bundle_file == "": + self._bundle_anchor = bundle_file + else: + self._bundle_anchor = os_mod.path.abspath(bundle_file) + # Marks this instance so another cribo runtime chained behind it can + # recognize it (see _notify_custom_hook). + self._cribo_sm_runtime_marker = True self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -143,25 +153,19 @@ def _map_location(self): if env not in ("", "1", "true", "yes", "on"): path = env return (path, self._os.path.dirname(self._os.path.abspath(path))) - bundle = self._bundle + bundle = self._bundle_anchor if self._mode == "inline": if bundle == "": return None # stdin cannot be re-opened; use CRIBO_SOURCE_MAPS= - return (None, self._os.path.dirname(self._os.path.abspath(bundle))) + return (None, self._os.path.dirname(bundle)) sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): - return ( - sibling, - self._os.path.dirname(self._os.path.abspath(sibling)), - ) + return (sibling, self._os.path.dirname(sibling)) return None # external: opt in via CRIBO_SOURCE_MAPS=1 (a path was handled above) if env in ("1", "true", "yes", "on"): - return ( - sibling, - self._os.path.dirname(self._os.path.abspath(sibling)), - ) + return (sibling, self._os.path.dirname(sibling)) return None def _file_chunks(self, path, *, _open=open): @@ -236,27 +240,41 @@ def _skip_ws(self, stream, byte): byte = stream.read_byte() return byte + def _read_hex4(self, stream, *, _int=int, _chr=chr, _range=range, _error=ValueError): + """Read the four hex digits of a \\uXXXX escape; return the code unit.""" + code = 0 + for _ in _range(4): + digit = stream.read_byte() + if digit < 0: + raise _error("unterminated unicode escape") + code = code * 16 + _int(_chr(digit), 16) + return code + def _read_string( self, stream, collect, *, _bytearray=bytearray, - _int=int, _chr=chr, - _range=range, _error=ValueError, ): """Consume a JSON string whose opening quote was already read. Returns the decoded text when collect is true, else None (contents are - discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are - handled, so string *values* containing text like '"mappings":' cannot - confuse the key scanner. + discarded byte-by-byte). Escaped quotes, \\uXXXX sequences, and UTF-16 + surrogate pairs (non-BMP characters) are handled, so string *values* + containing text like '"mappings":' cannot confuse the key scanner and + emoji-bearing paths survive intact. """ + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf = _bytearray() if collect else None + pending = None # one byte of lookahead pushed back by surrogate handling while True: - byte = stream.read_byte() + if pending is None: + byte = stream.read_byte() + else: + byte, pending = pending, None if byte < 0: raise _error("unterminated JSON string") if byte == 34: # '"' @@ -268,18 +286,37 @@ def _read_string( escape = stream.read_byte() if escape < 0: raise _error("unterminated JSON escape") - if escape == 117: # 'u' - code = 0 - for _ in _range(4): - digit = stream.read_byte() - if digit < 0: - raise _error("unterminated unicode escape") - code = code * 16 + _int(_chr(digit), 16) + if not escape == 117: # 'u' if buf is not None: + buf.append(table.get(escape, escape)) + continue + code = self._read_hex4(stream) + if buf is None: + continue + if 0xD800 <= code <= 0xDBFF: + # High surrogate: a following \uXXXX low surrogate combines + # into one non-BMP code point. + nxt = stream.read_byte() + if nxt == 92: + escape2 = stream.read_byte() + if escape2 == 117: + low = self._read_hex4(stream) + if 0xDC00 <= low <= 0xDFFF: + code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00) + buf.extend(_chr(code).encode("utf-8")) + else: + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(low).encode("utf-8", "surrogatepass")) + continue + if escape2 < 0: + raise _error("unterminated JSON escape") buf.extend(_chr(code).encode("utf-8", "surrogatepass")) - elif buf is not None: - table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} - buf.append(table.get(escape, escape)) + buf.append(table.get(escape2, escape2)) + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + pending = nxt # includes EOF/quote; the main loop handles both + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) def _skip_value(self, stream, byte, *, _error=ValueError): """Skip one JSON value; return the first byte after it (or -1).""" @@ -445,7 +482,7 @@ def _load(self, needed_lines, *, _set=set, _max=max): needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) if map_path is None: - chunks = self._inline_chunks(self._bundle) + chunks = self._inline_chunks(self._bundle_anchor) else: chunks = self._file_chunks(map_path) sources, table0 = self._scan(chunks, needed0, max_needed) @@ -463,7 +500,7 @@ def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): json = self._import("json") if map_path is None: - raw = b"".join(self._inline_chunks(self._bundle)) + raw = b"".join(self._inline_chunks(self._bundle_anchor)) else: handle = _open(map_path, "rb") try: @@ -779,21 +816,28 @@ def _try_render( pass self._local.in_hook = False - def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException): + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): """Invoke a chained hook after a successful remap when it is custom. A successful remap replaces the *default* printer, but preinstalled custom hooks (error reporters, sitecustomize) must still observe the - exception; their own output is theirs to manage. When the interpreter - default is unavailable for comparison (e.g. `threading.__excepthook__` - before Python 3.10), no notification happens — better to skip a custom - hook than to double-print via the default one. + exception; their own output is theirs to manage. Two exclusions: when + the interpreter default is unavailable for comparison (e.g. + `threading.__excepthook__` before Python 3.10) no notification happens + — better to skip a custom hook than to double-print via the default + one; and an earlier cribo runtime's hook is skipped, since it would + find no frames for its own bundle and delegate to the default printer, + duplicating the traceback. """ - if default is not None and prev is not None and prev is not default: - try: - call(prev) - except _bex: - pass + if default is None or prev is None or prev is default: + return + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return + try: + call(prev) + except _bex: + pass # -- installed hooks ------------------------------------------------------ diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index 717acaf23..9692299af 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -131,6 +131,19 @@ def test_scan_handles_mappings_before_sources(rt): assert table == {0: (0, 0)}, table +def test_scan_combines_surrogate_pairs(rt): + # Non-BMP characters arrive as JSON surrogate pairs; they must decode to + # one code point, not two replacement characters. + json_text = '{"sources":["\\ud83d\\ude00.py"],"mappings":"AAAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources == ["\U0001f600.py"], sources + # A lone high surrogate followed by a plain character stays recoverable + # (replacement character), and the rest of the string is intact. + json_text = '{"sources":["\\ud83dx.py"],"mappings":"AAAA"}' + sources, _table = _scan(rt, json_text, {0}, 0) + assert sources[0].endswith("x.py"), sources + + def make_inline_bundle(payload_json): """Create a temp file shaped like an inline-mode bundle; return its path.""" import base64 diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 0cc57aa13..7b88cb343 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -45,6 +45,11 @@ class _CriboSourceMapRuntime(object): def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): self._mode = mode self._bundle = bundle_file + if bundle_file == "": + self._bundle_anchor = bundle_file + else: + self._bundle_anchor = os_mod.path.abspath(bundle_file) + self._cribo_sm_runtime_marker = True self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -84,18 +89,18 @@ class _CriboSourceMapRuntime(object): if env not in ("", "1", "true", "yes", "on"): path = env return path, self._os.path.dirname(self._os.path.abspath(path)) - bundle = self._bundle + bundle = self._bundle_anchor if self._mode == "inline": if bundle == "": return None - return None, self._os.path.dirname(self._os.path.abspath(bundle)) + return None, self._os.path.dirname(bundle) sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): - return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return sibling, self._os.path.dirname(sibling) return None if env in ("1", "true", "yes", "on"): - return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return sibling, self._os.path.dirname(sibling) return None def _file_chunks(self, path, *, _open=open): @@ -164,11 +169,26 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range, _error=ValueError): - """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ + def _read_hex4(self, stream, *, _int=int, _chr=chr, _range=range, _error=ValueError): + """Read the four hex digits of a \\uXXXX escape; return the code unit.""" + code = 0 + for _ in _range(4): + digit = stream.read_byte() + if digit < 0: + raise _error("unterminated unicode escape") + code = code * 16 + _int(_chr(digit), 16) + return code + + def _read_string(self, stream, collect, *, _bytearray=bytearray, _chr=chr, _error=ValueError): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes, \\uXXXX sequences, and UTF-16\n surrogate pairs (non-BMP characters) are handled, so string *values*\n containing text like '\"mappings\":' cannot confuse the key scanner and\n emoji-bearing paths survive intact.\n """ + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf = _bytearray() if collect else None + pending = None while True: - byte = stream.read_byte() + if pending is None: + byte = stream.read_byte() + else: + byte, pending = pending, None if byte < 0: raise _error("unterminated JSON string") if byte == 34: @@ -180,18 +200,35 @@ class _CriboSourceMapRuntime(object): escape = stream.read_byte() if escape < 0: raise _error("unterminated JSON escape") - if escape == 117: - code = 0 - for _ in _range(4): - digit = stream.read_byte() - if digit < 0: - raise _error("unterminated unicode escape") - code = code * 16 + _int(_chr(digit), 16) + if not escape == 117: if buf is not None: + buf.append(table.get(escape, escape)) + continue + code = self._read_hex4(stream) + if buf is None: + continue + if 55296 <= code <= 56319: + nxt = stream.read_byte() + if nxt == 92: + escape2 = stream.read_byte() + if escape2 == 117: + low = self._read_hex4(stream) + if 56320 <= low <= 57343: + code = 65536 + (code - 55296 << 10) + (low - 56320) + buf.extend(_chr(code).encode("utf-8")) + else: + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(low).encode("utf-8", "surrogatepass")) + continue + if escape2 < 0: + raise _error("unterminated JSON escape") buf.extend(_chr(code).encode("utf-8", "surrogatepass")) - elif buf is not None: - table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} - buf.append(table.get(escape, escape)) + buf.append(table.get(escape2, escape2)) + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + pending = nxt + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) def _skip_value(self, stream, byte, *, _error=ValueError): """Skip one JSON value; return the first byte after it (or -1).""" @@ -333,7 +370,7 @@ class _CriboSourceMapRuntime(object): needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) if map_path is None: - chunks = self._inline_chunks(self._bundle) + chunks = self._inline_chunks(self._bundle_anchor) else: chunks = self._file_chunks(map_path) sources, table0 = self._scan(chunks, needed0, max_needed) @@ -350,7 +387,7 @@ class _CriboSourceMapRuntime(object): map_path, map_dir = location json = self._import("json") if map_path is None: - raw = b"".join(self._inline_chunks(self._bundle)) + raw = b"".join(self._inline_chunks(self._bundle_anchor)) else: handle = _open(map_path, "rb") try: @@ -593,13 +630,17 @@ class _CriboSourceMapRuntime(object): pass self._local.in_hook = False - def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. When the interpreter\n default is unavailable for comparison (e.g. `threading.__excepthook__`\n before Python 3.10), no notification happens — better to skip a custom\n hook than to double-print via the default one.\n """ - if default is not None and prev is not None and prev is not default: - try: - call(prev) - except _bex: - pass + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Two exclusions: when\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one; and an earlier cribo runtime's hook is skipped, since it would\n find no frames for its own bundle and delegate to the default printer,\n duplicating the traceback.\n """ + if default is None or prev is None or prev is default: + return + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return + try: + call(prev) + except _bex: + pass def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 05d7276bb..f05274b80 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -45,6 +45,11 @@ class _CriboSourceMapRuntime(object): def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): self._mode = mode self._bundle = bundle_file + if bundle_file == "": + self._bundle_anchor = bundle_file + else: + self._bundle_anchor = os_mod.path.abspath(bundle_file) + self._cribo_sm_runtime_marker = True self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -84,18 +89,18 @@ class _CriboSourceMapRuntime(object): if env not in ("", "1", "true", "yes", "on"): path = env return path, self._os.path.dirname(self._os.path.abspath(path)) - bundle = self._bundle + bundle = self._bundle_anchor if self._mode == "inline": if bundle == "": return None - return None, self._os.path.dirname(self._os.path.abspath(bundle)) + return None, self._os.path.dirname(bundle) sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): - return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return sibling, self._os.path.dirname(sibling) return None if env in ("1", "true", "yes", "on"): - return sibling, self._os.path.dirname(self._os.path.abspath(sibling)) + return sibling, self._os.path.dirname(sibling) return None def _file_chunks(self, path, *, _open=open): @@ -164,11 +169,26 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string(self, stream, collect, *, _bytearray=bytearray, _int=int, _chr=chr, _range=range, _error=ValueError): - """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes and \\uXXXX sequences are\n handled, so string *values* containing text like '\"mappings\":' cannot\n confuse the key scanner.\n """ + def _read_hex4(self, stream, *, _int=int, _chr=chr, _range=range, _error=ValueError): + """Read the four hex digits of a \\uXXXX escape; return the code unit.""" + code = 0 + for _ in _range(4): + digit = stream.read_byte() + if digit < 0: + raise _error("unterminated unicode escape") + code = code * 16 + _int(_chr(digit), 16) + return code + + def _read_string(self, stream, collect, *, _bytearray=bytearray, _chr=chr, _error=ValueError): + """Consume a JSON string whose opening quote was already read.\n\n Returns the decoded text when collect is true, else None (contents are\n discarded byte-by-byte). Escaped quotes, \\uXXXX sequences, and UTF-16\n surrogate pairs (non-BMP characters) are handled, so string *values*\n containing text like '\"mappings\":' cannot confuse the key scanner and\n emoji-bearing paths survive intact.\n """ + table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} buf = _bytearray() if collect else None + pending = None while True: - byte = stream.read_byte() + if pending is None: + byte = stream.read_byte() + else: + byte, pending = pending, None if byte < 0: raise _error("unterminated JSON string") if byte == 34: @@ -180,18 +200,35 @@ class _CriboSourceMapRuntime(object): escape = stream.read_byte() if escape < 0: raise _error("unterminated JSON escape") - if escape == 117: - code = 0 - for _ in _range(4): - digit = stream.read_byte() - if digit < 0: - raise _error("unterminated unicode escape") - code = code * 16 + _int(_chr(digit), 16) + if not escape == 117: if buf is not None: + buf.append(table.get(escape, escape)) + continue + code = self._read_hex4(stream) + if buf is None: + continue + if 55296 <= code <= 56319: + nxt = stream.read_byte() + if nxt == 92: + escape2 = stream.read_byte() + if escape2 == 117: + low = self._read_hex4(stream) + if 56320 <= low <= 57343: + code = 65536 + (code - 55296 << 10) + (low - 56320) + buf.extend(_chr(code).encode("utf-8")) + else: + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + buf.extend(_chr(low).encode("utf-8", "surrogatepass")) + continue + if escape2 < 0: + raise _error("unterminated JSON escape") buf.extend(_chr(code).encode("utf-8", "surrogatepass")) - elif buf is not None: - table = {98: 8, 102: 12, 110: 10, 114: 13, 116: 9} - buf.append(table.get(escape, escape)) + buf.append(table.get(escape2, escape2)) + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) + pending = nxt + continue + buf.extend(_chr(code).encode("utf-8", "surrogatepass")) def _skip_value(self, stream, byte, *, _error=ValueError): """Skip one JSON value; return the first byte after it (or -1).""" @@ -333,7 +370,7 @@ class _CriboSourceMapRuntime(object): needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) if map_path is None: - chunks = self._inline_chunks(self._bundle) + chunks = self._inline_chunks(self._bundle_anchor) else: chunks = self._file_chunks(map_path) sources, table0 = self._scan(chunks, needed0, max_needed) @@ -350,7 +387,7 @@ class _CriboSourceMapRuntime(object): map_path, map_dir = location json = self._import("json") if map_path is None: - raw = b"".join(self._inline_chunks(self._bundle)) + raw = b"".join(self._inline_chunks(self._bundle_anchor)) else: handle = _open(map_path, "rb") try: @@ -593,13 +630,17 @@ class _CriboSourceMapRuntime(object): pass self._local.in_hook = False - def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. When the interpreter\n default is unavailable for comparison (e.g. `threading.__excepthook__`\n before Python 3.10), no notification happens — better to skip a custom\n hook than to double-print via the default one.\n """ - if default is not None and prev is not None and prev is not default: - try: - call(prev) - except _bex: - pass + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Two exclusions: when\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one; and an earlier cribo runtime's hook is skipped, since it would\n find no frames for its own bundle and delegate to the default printer,\n duplicating the traceback.\n """ + if default is None or prev is None or prev is default: + return + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return + try: + call(prev) + except _bex: + pass def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 742b93a60..31cee40dc 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:754 `def add(a, b):` -> calculator.py:1 -bundle:755 `result = a + b` -> calculator.py:2 -bundle:756 `return result` -> calculator.py:3 -bundle:758 `def multiply(a, b):` -> calculator.py:6 -bundle:759 `result = a * b` -> calculator.py:7 -bundle:760 `return result` -> calculator.py:8 -bundle:765 `def describe(name, value):` -> utils.py:1 -bundle:766 `return f"{name} = {value}"` -> utils.py:2 -bundle:772 `total = add(2, 3)` -> main.py:4 -bundle:773 `product = multiply(total, 4)` -> main.py:5 -bundle:774 `print(describe("total", total))` -> main.py:6 -bundle:775 `print(describe("product", product))` -> main.py:7 +bundle:795 `def add(a, b):` -> calculator.py:1 +bundle:796 `result = a + b` -> calculator.py:2 +bundle:797 `return result` -> calculator.py:3 +bundle:799 `def multiply(a, b):` -> calculator.py:6 +bundle:800 `result = a * b` -> calculator.py:7 +bundle:801 `return result` -> calculator.py:8 +bundle:806 `def describe(name, value):` -> utils.py:1 +bundle:807 `return f"{name} = {value}"` -> utils.py:2 +bundle:813 `total = add(2, 3)` -> main.py:4 +bundle:814 `product = multiply(total, 4)` -> main.py:5 +bundle:815 `print(describe("total", total))` -> main.py:6 +bundle:816 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index a7bd65f69..5ff50106e 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:763 `print("effects module loading")` -> effects.py:1 -bundle:764 `COUNTER = 1` -> effects.py:3 -bundle:767 `def boost(value):` -> effects.py:6 -bundle:768 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:769 `return boosted` -> effects.py:8 -bundle:781 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:782 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:804 `print("effects module loading")` -> effects.py:1 +bundle:805 `COUNTER = 1` -> effects.py:3 +bundle:808 `def boost(value):` -> effects.py:6 +bundle:809 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:810 `return boosted` -> effects.py:8 +bundle:822 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:823 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index ad917db5e..300d3395d 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1136,3 +1136,83 @@ fn runtime_survives_shadowed_builtins() { ); assert!(!stderr.contains("bundle.py\", line"), "{stderr}"); } + +#[test] +fn runtime_survives_chdir_before_crash() { + // A bundle launched via a relative path that chdirs away before crashing + // must still locate itself and its sibling map (paths are anchored at + // startup). + let dir = make_project(&[ + ( + "main.py", + "import os\nfrom helper import boom\n\nos.chdir(\"/\")\nboom()\n", + ), + ( + "helper.py", + "def boom():\n raise ValueError(\"kaboom after chdir\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + + let mut command = Command::new(common::get_python_executable()); + // Invoke through the relative file name, from the bundle's directory. + command.arg(bundle.file_name().expect("bundle name")); + command.current_dir(dir.path()); + command.env_remove("CRIBO_SOURCE_MAPS"); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert!( + stderr.contains("helper.py\", line 2, in boom"), + "remapping must survive os.chdir before the crash: {stderr}" + ); + assert!(stderr.contains("kaboom after chdir")); +} + +#[test] +fn stacked_runtimes_do_not_duplicate_tracebacks() { + // Two source-mapped bundles executed in one interpreter stack their hooks; + // the outer runtime must recognize the inner one and not chain into it + // (which would fall through to the default printer and duplicate output). + let quiet = fixture_project(); + let quiet_bundle = quiet.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&quiet), + "--output", + &quiet_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let crash = crash_project(); + let crash_bundle = bundle_crash_project(&crash, "--sourcemap=linked"); + + let driver = crash.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({quiet:?})\nrunpy.run_path({crash:?})\n", + quiet = quiet_bundle.to_string_lossy(), + crash = crash_bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let (ok, _, stderr) = run_python(&driver, &[]); + assert!(!ok); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "stacked runtimes must render exactly one traceback: {stderr}" + ); + assert_eq!( + stderr.matches("ValueError: kaboom").count(), + 1, + "the exception line must print exactly once: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "the crashing bundle's frames must be remapped: {stderr}" + ); +} From f0a598a1ad12be2f98245eb92292e3d2d1643dba Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 22:46:01 +0200 Subject: [PATCH 08/17] fix: address seventh review round on source map runtime - shadow-proof importer removes every sys.path entry resolving to the script directory (PYTHONPATH=. exposes it beyond index 0) - anchor a relative CRIBO_SOURCE_MAPS override to the startup cwd - two-pass streaming scan: pass 1 decodes only mappings, pass 2 collects only the source paths referenced by the traceback, so memory scales with needed frames instead of the bundle's full source table - snapshot the interpreter default hooks at bootstrap so rebinding sys.__excepthook__ cannot cause double printing - stage maps with create_new (no symlink following, no truncation of a planted target; collision-retry keeps concurrent builds apart) and copy an existing map's permissions before writing content Addresses seventh-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 68 ++++-- crates/cribo/src/python/sourcemap_runtime.py | 198 +++++++++++------- .../tests/python/test_sourcemap_runtime.py | 26 +-- .../bundled_code@sourcemap_basic.snap | 156 ++++++++------ .../bundled_code@sourcemap_wrapper.snap | 156 ++++++++------ .../snapshots/source_map@sourcemap_basic.snap | 24 +-- .../source_map@sourcemap_wrapper.snap | 14 +- 7 files changed, 399 insertions(+), 243 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 49408c038..6e1a999c2 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -49,6 +49,57 @@ fn source_map_path_for(output_path: &Path) -> PathBuf { output_path.with_file_name(file_name) } +/// Stage the source map into a collision-resistant temp file next to `map_path`. +/// +/// The file is opened with `create_new` (`O_EXCL` semantics), which neither +/// follows a pre-planted symlink nor truncates an existing file — a requirement +/// for outputs in shared sticky directories, where a temp name could otherwise +/// be predicted and pointed elsewhere. Name collisions retry with a new suffix, +/// which also keeps concurrent builds from stomping each other's staged map. +/// When a previous map exists, its permissions are copied onto the staged file +/// before any content is written, so a restricted map (e.g. 0600 protecting +/// `sourcesContent`) stays restricted across rebuilds. +fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { + use std::io::Write as _; + + let pid = std::process::id(); + let base_name = map_path.file_name().map_or_else( + || std::ffi::OsString::from("bundle.py.map"), + std::ffi::OsStr::to_os_string, + ); + let mut attempt = 0_u32; + loop { + let mut tmp_name = base_name.clone(); + tmp_name.push(format!(".{pid}.{attempt}.tmp")); + let tmp_path = map_path.with_file_name(tmp_name); + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path) + { + Ok(mut file) => { + let write_result = fs::metadata(map_path) + .ok() + .map_or(Ok(()), |metadata| { + file.set_permissions(metadata.permissions()) + }) + .and_then(|()| file.write_all(map_json.as_bytes())) + .and_then(|()| file.flush()); + if let Err(err) = write_result { + drop(file); + let _ = fs::remove_file(&tmp_path); + return Err(err); + } + return Ok(tmp_path); + } + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists && attempt < 32 => { + attempt += 1; + } + Err(err) => return Err(err), + } + } +} + /// Type alias for module processing queue type ModuleQueue = Vec<(ModuleId, PathBuf)>; /// Type alias for processed modules set @@ -745,18 +796,11 @@ impl BundleOrchestrator { // mismatch window is a rename failure, which requires directory-level // problems that would have failed the bundle write too. let staged_map = if let Some((map_path, map_json)) = pending_map { - let mut tmp_name = map_path.file_name().map_or_else( - || std::ffi::OsString::from("bundle.py.map"), - std::ffi::OsStr::to_os_string, - ); - // Process-unique staging name: concurrent builds targeting the same - // output must not stomp each other's staged map (concurrent writers - // to one output path remain externally undefined, as for the bundle - // file itself, but each publish stays internally consistent). - tmp_name.push(format!(".{}.tmp", std::process::id())); - let tmp_path = map_path.with_file_name(tmp_name); - fs::write(&tmp_path, map_json).with_context(|| { - format!("Failed to stage source map file: {}", tmp_path.display()) + let tmp_path = stage_map_file(&map_path, map_json).with_context(|| { + format!( + "Failed to stage source map file next to: {}", + map_path.display() + ) })?; Some((tmp_path, map_path)) } else { diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 6614c74d7..a4727f71f 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -17,17 +17,28 @@ def _cribo_sm_import(name, *, _list=list, _import=__import__): """Import a stdlib module immune to script-directory shadowing. - The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a + The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and + `PYTHONPATH=.` can expose the same directory again at later indices — so a project file named e.g. `threading.py` sitting next to the bundle would otherwise shadow the stdlib for this runtime. Already-imported modules are - taken from `sys.modules`; otherwise the import runs with that first path - entry dropped. (`sys` itself is a builtin and can never be shadowed.) + taken from `sys.modules`; otherwise the import runs with every path entry + resolving to the script directory removed. (`sys` itself is a builtin and + can never be shadowed.) """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - _cribo_sys.path = _list(saved_path[1:]) + filtered = _list(saved_path[1:]) + os_mod = _cribo_sys.modules.get("os") + if os_mod is not None and saved_path: + script_dir = os_mod.path.normcase(os_mod.path.abspath(saved_path[0] or ".")) + filtered = [ + entry + for entry in filtered + if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir + ] + _cribo_sys.path = filtered try: return _import(name) finally: @@ -88,6 +99,12 @@ def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, trace # Marks this instance so another cribo runtime chained behind it can # recognize it (see _notify_custom_hook). self._cribo_sm_runtime_marker = True + # Startup working directory, for anchoring a relative + # CRIBO_SOURCE_MAPS override before user code can chdir away. + try: + self._startup_cwd = os_mod.getcwd() + except OSError: + self._startup_cwd = "." self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -104,6 +121,12 @@ def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, trace self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook self._prev_threading_hook = threading_mod.excepthook + # Interpreter defaults, snapshotted now so user code rebinding e.g. + # sys.__excepthook__ later cannot make the captured previous hook look + # custom (which would double-print) or raise from the hook. + self._default_excepthook = _cribo_sys.__excepthook__ + self._default_unraisablehook = _cribo_sys.__unraisablehook__ + self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) try: self._group_type = BaseExceptionGroup except NameError: # Python < 3.11 @@ -149,9 +172,12 @@ def _map_location(self): return None # An explicit path wins for every mode. This is also the only way to # supply a map to a bundle executed via `python -` (stdin), whose - # source cannot be re-read at hook time. + # source cannot be re-read at hook time. A relative override is + # anchored to the startup working directory, immune to later chdir. if env not in ("", "1", "true", "yes", "on"): path = env + if not self._os.path.isabs(path): + path = self._os.path.join(self._startup_cwd, path) return (path, self._os.path.dirname(self._os.path.abspath(path))) bundle = self._bundle_anchor if self._mode == "inline": @@ -341,27 +367,6 @@ def _skip_value(self, stream, byte, *, _error=ValueError): byte = stream.read_byte() return byte - def _read_string_array(self, stream, byte, *, _error=ValueError): - """Read a JSON array of strings/nulls; return (list, byte after array).""" - if not byte == 91: # '[' - raise _error("expected array") - items = [] - byte = self._skip_ws(stream, stream.read_byte()) - if byte == 93: # ']' - return items, stream.read_byte() - while True: - if byte == 34: - items.append(self._read_string(stream, True)) - byte = self._skip_ws(stream, stream.read_byte()) - else: - items.append(None) - byte = self._skip_ws(stream, self._skip_value(stream, byte)) - if byte == 93: - return items, stream.read_byte() - if not byte == 44: # ',' - raise _error("malformed array") - byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq( self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError ): @@ -420,15 +425,28 @@ def end_segment(): vlq_value = 0 vlq_shift = 0 - def _scan(self, chunks, needed, max_needed, *, _error=ValueError): - """Scan the map's top-level object; return (sources, line table).""" - stream = self._stream_cls(chunks) + def _scan(self, chunks_factory, needed, max_needed, *, _set=set, _error=ValueError): + """Two-pass scan of the map; return (sources dict, line table). + + Pass 1 decodes only `mappings` (skipping the sources array entirely); + pass 2 re-streams the map and collects only the source paths actually + referenced by the decoded lines. Memory therefore scales with the + traceback's needed frames, not with the bundle's full source table — + the property the whole streaming decoder exists for. `chunks_factory` + is a zero-argument callable producing a fresh chunk iterator per pass. + """ + table = self._scan_mappings(self._stream_cls(chunks_factory()), needed, max_needed) + wanted = _set(src_idx for (src_idx, _line) in table.values()) + sources = {} + if wanted: + sources = self._scan_sources(self._stream_cls(chunks_factory()), wanted) + return sources, table + + def _scan_mappings(self, stream, needed, max_needed, *, _error=ValueError): + """Pass 1: decode the `mappings` field; every other value is skipped.""" byte = self._skip_ws(stream, stream.read_byte()) if not byte == 123: # '{' raise _error("not a JSON object") - sources = [] - table = {} - saw_mappings = False byte = self._skip_ws(stream, stream.read_byte()) while byte == 34: # '"' starting a key key = self._read_string(stream, True) @@ -436,35 +454,56 @@ def _scan(self, chunks, needed, max_needed, *, _error=ValueError): if not byte == 58: # ':' raise _error("malformed object") byte = self._skip_ws(stream, stream.read_byte()) - if key == "sources": - sources, byte = self._read_string_array(stream, byte) - elif key == "mappings": + if key == "mappings": if not byte == 34: raise _error("mappings is not a string") - table, terminated = self._decode_vlq(stream, needed, max_needed) - saw_mappings = True - if sources: - # Both fields consumed; stop without reading further (the - # early exit inside the VLQ machine is preserved). - break - if not terminated: - # Early exit left the stream inside the mappings string; - # skim to its closing quote so a `sources` field that - # follows `mappings` still parses correctly. VLQ data - # contains no escapes, so a bare '"' terminates. - while True: - byte = stream.read_byte() - if byte < 0 or byte == 34: - break - byte = stream.read_byte() - else: - byte = self._skip_value(stream, byte) - byte = self._skip_ws(stream, byte) + table, _terminated = self._decode_vlq(stream, needed, max_needed) + return table + byte = self._skip_ws(stream, self._skip_value(stream, byte)) if byte == 44: # ',' byte = self._skip_ws(stream, stream.read_byte()) - if not saw_mappings: - raise _error("no mappings field") - return sources, table + raise _error("no mappings field") + + def _scan_sources(self, stream, wanted, *, _error=ValueError): + """Pass 2: collect only the `sources` entries whose index is wanted.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: # '{' + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: # '"' starting a key + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: # ':' + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + if not byte == 91: # '[' + raise _error("sources is not an array") + return self._read_wanted_array_items(stream, wanted) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: # ',' + byte = self._skip_ws(stream, stream.read_byte()) + return {} + + def _read_wanted_array_items(self, stream, wanted, *, _len=len, _error=ValueError): + """Read a JSON array, collecting only string items at wanted indices.""" + items = {} + index = 0 + byte = self._skip_ws(stream, stream.read_byte()) + if byte == 93: # ']' + return items + while True: + if byte == 34 and index in wanted: + items[index] = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + else: + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + index += 1 + if byte == 93 or _len(items) == _len(wanted): + return items + if not byte == 44: # ',' + raise _error("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) # -- loading ------------------------------------------------------------- @@ -482,16 +521,24 @@ def _load(self, needed_lines, *, _set=set, _max=max): needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) if map_path is None: - chunks = self._inline_chunks(self._bundle_anchor) + + def chunks_factory(): + return self._inline_chunks(self._bundle_anchor) + else: - chunks = self._file_chunks(map_path) - sources, table0 = self._scan(chunks, needed0, max_needed) + + def chunks_factory(): + return self._file_chunks(map_path) + + sources, table0 = self._scan(chunks_factory, needed0, max_needed) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = (src_idx, src_line0 + 1) return (table, sources, map_dir) - def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): + def _load_json_fallback( + self, needed_lines, *, _open=open, _set=set, _max=max, _enumerate=enumerate + ): """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" location = self._map_location() if location is None: @@ -508,7 +555,9 @@ def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): finally: handle.close() data = json.loads(raw.decode("utf-8")) - sources = data.get("sources") or [] + sources = {} + for index, source in _enumerate(data.get("sources") or []): + sources[index] = source mappings = data.get("mappings") or "" needed0 = _set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) @@ -590,9 +639,7 @@ def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int= limit = _getattr(self._sys, "tracebacklimit", None) return limit if _isinstance(limit, _int) else None - def _write_frames( - self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len - ): + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): """Write remapped frame lines, collapsing repeated frames like CPython. Consecutive identical frames (recursion) print at most 3 times followed @@ -631,13 +678,14 @@ def emit(entry): name = frame.f_code.co_name if filename == self._bundle: mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < _len(sources) and sources[mapped[0]]: - source = sources[mapped[0]] - if not self._os.path.isabs(source): - source = self._os.path.normpath( - self._os.path.join(map_dir, source) - ) - filename, lineno = source, mapped[1] + if mapped is not None: + source = sources.get(mapped[0]) + if source: + if not self._os.path.isabs(source): + source = self._os.path.normpath( + self._os.path.join(map_dir, source) + ) + filename, lineno = source, mapped[1] entry = (filename, lineno, name) if entry == last: repeats += 1 @@ -845,7 +893,7 @@ def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): self._notify_custom_hook( self._prev_excepthook, - self._sys.__excepthook__, + self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), ) return @@ -865,7 +913,7 @@ def threading_hook( if self._try_render(args.exc_value, args.exc_traceback, prefix): self._notify_custom_hook( self._prev_threading_hook, - _getattr(self._threading, "__excepthook__", None), + self._default_threading_hook, lambda hook: hook(args), ) return @@ -880,7 +928,7 @@ def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): self._notify_custom_hook( self._prev_unraisablehook, - self._sys.__unraisablehook__, + self._default_unraisablehook, lambda hook: hook(unraisable), ) return diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index 9692299af..b50414de3 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -49,14 +49,14 @@ def test_stream_reads_across_chunk_boundaries(rt): def _scan(rt, json_text, needed, max_needed): - return rt._scan([json_text.encode("utf-8")], needed, max_needed) + return rt._scan(lambda: [json_text.encode("utf-8")], needed, max_needed) def test_scan_extracts_sources_and_mappings(rt): # AAAA;AACA;AACA: one segment per line, source line advancing by one. json_text = '{"version":3,"sources":["a.py","b.py"],"mappings":"AAAA;AACA;AACA"}' sources, table = _scan(rt, json_text, {0, 2}, 2) - assert sources == ["a.py", "b.py"], sources + assert sources == {0: "a.py"}, sources # only referenced indices collected assert table == {0: (0, 0), 2: (0, 2)}, table @@ -82,20 +82,20 @@ def test_scan_ignores_adversarial_sources_content(rt): '{"sources":["a.py"],"sourcesContent":["' + evil + '"],"mappings":"AAAA"}' ) sources, table = _scan(rt, json_text, {0}, 0) - assert sources == ["a.py"], sources + assert sources == {0: "a.py"}, sources assert table == {0: (0, 0)}, table def test_scan_handles_unicode_escapes_in_sources(rt): json_text = '{"sources":["\\u00e9t\\u00e9.py"],"mappings":"AAAA"}' sources, _table = _scan(rt, json_text, {0}, 0) - assert sources == ["\u00e9t\u00e9.py"], sources + assert sources == {0: "\u00e9t\u00e9.py"}, sources def test_scan_null_in_sources_array(rt): - json_text = '{"sources":["a.py",null,"c.py"],"mappings":"AAAA"}' + json_text = '{"sources":["a.py",null,"c.py"],"mappings":"ACAA"}' sources, _table = _scan(rt, json_text, {0}, 0) - assert sources == ["a.py", None, "c.py"], sources + assert sources == {}, sources # a null entry is simply not collected def test_vlq_early_exit_stops_reading(rt): @@ -104,11 +104,11 @@ def test_vlq_early_exit_stops_reading(rt): class Boom(Exception): pass - def chunks(): + def chunks_factory(): yield b'{"sources":["a.py"],"mappings":"AAAA;AACA;' raise Boom("decoder read past its early-exit point") - _sources, table = rt._scan(chunks(), {0}, 0) + _sources, table = rt._scan(chunks_factory, {0}, 0) assert table == {0: (0, 0)}, table @@ -127,7 +127,7 @@ def test_scan_handles_mappings_before_sources(rt): # mappings string must not derail parsing of a later sources field. json_text = '{"mappings":"AAAA;AACA;AACA","sources":["a.py","b.py"]}' sources, table = _scan(rt, json_text, {0}, 0) # early exit after line 0 - assert sources == ["a.py", "b.py"], sources + assert sources == {0: "a.py"}, sources assert table == {0: (0, 0)}, table @@ -136,7 +136,7 @@ def test_scan_combines_surrogate_pairs(rt): # one code point, not two replacement characters. json_text = '{"sources":["\\ud83d\\ude00.py"],"mappings":"AAAA"}' sources, _table = _scan(rt, json_text, {0}, 0) - assert sources == ["\U0001f600.py"], sources + assert sources == {0: "\U0001f600.py"}, sources # A lone high surrogate followed by a plain character stays recoverable # (replacement character), and the rest of the string is intact. json_text = '{"sources":["\\ud83dx.py"],"mappings":"AAAA"}' @@ -168,8 +168,8 @@ def test_inline_payload_scan_and_chunked_base64(rt): decoded = b"".join(rt._inline_chunks(path)) assert decoded.decode("utf-8") == json_text # And end-to-end through the scanner: - sources, table = rt._scan(rt._inline_chunks(path), {0}, 0) - assert sources == ["a.py"], sources + sources, table = rt._scan(lambda: rt._inline_chunks(path), {0}, 0) + assert sources == {0: "a.py"}, sources assert table == {0: (0, 0)}, table finally: os.unlink(path) @@ -241,7 +241,7 @@ def test_env_path_wins_for_every_mode(rt): loaded = inline_stdin._load({1}) assert loaded is not None, "env path must activate a inline bundle" table, sources, _map_dir = loaded - assert sources == ["a.py"] + assert sources == {0: "a.py"}, sources assert table == {1: (0, 1)}, table # Without the env override, a inline bundle stays inactive. os.environ.pop("CRIBO_SOURCE_MAPS", None) diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 7b88cb343..3098058c0 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -8,12 +8,17 @@ input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py import sys as _cribo_sys def _cribo_sm_import(name, *, _list=list, _import=__import__): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with that first path\n entry dropped. (`sys` itself is a builtin and can never be shadowed.)\n """ + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. (`sys` itself is a builtin and\n can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - _cribo_sys.path = _list(saved_path[1:]) + filtered = _list(saved_path[1:]) + os_mod = _cribo_sys.modules.get("os") + if os_mod is not None and saved_path: + script_dir = os_mod.path.normcase(os_mod.path.abspath(saved_path[0] or ".")) + filtered = [entry for entry in filtered if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir] + _cribo_sys.path = filtered try: return _import(name) finally: @@ -50,6 +55,10 @@ class _CriboSourceMapRuntime(object): else: self._bundle_anchor = os_mod.path.abspath(bundle_file) self._cribo_sm_runtime_marker = True + try: + self._startup_cwd = os_mod.getcwd() + except OSError: + self._startup_cwd = "." self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -61,6 +70,9 @@ class _CriboSourceMapRuntime(object): self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook self._prev_threading_hook = threading_mod.excepthook + self._default_excepthook = _cribo_sys.__excepthook__ + self._default_unraisablehook = _cribo_sys.__unraisablehook__ + self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) try: self._group_type = BaseExceptionGroup except NameError: @@ -88,6 +100,8 @@ class _CriboSourceMapRuntime(object): return None if env not in ("", "1", "true", "yes", "on"): path = env + if not self._os.path.isabs(path): + path = self._os.path.join(self._startup_cwd, path) return path, self._os.path.dirname(self._os.path.abspath(path)) bundle = self._bundle_anchor if self._mode == "inline": @@ -252,27 +266,6 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string_array(self, stream, byte, *, _error=ValueError): - """Read a JSON array of strings/nulls; return (list, byte after array).""" - if not byte == 91: - raise _error("expected array") - items = [] - byte = self._skip_ws(stream, stream.read_byte()) - if byte == 93: - return items, stream.read_byte() - while True: - if byte == 34: - items.append(self._read_string(stream, True)) - byte = self._skip_ws(stream, stream.read_byte()) - else: - items.append(None) - byte = self._skip_ws(stream, self._skip_value(stream, byte)) - if byte == 93: - return items, stream.read_byte() - if not byte == 44: - raise _error("malformed array") - byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError): """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} @@ -321,15 +314,20 @@ class _CriboSourceMapRuntime(object): vlq_value = 0 vlq_shift = 0 - def _scan(self, chunks, needed, max_needed, *, _error=ValueError): - """Scan the map's top-level object; return (sources, line table).""" - stream = self._stream_cls(chunks) + def _scan(self, chunks_factory, needed, max_needed, *, _set=set, _error=ValueError): + """Two-pass scan of the map; return (sources dict, line table).\n\n Pass 1 decodes only `mappings` (skipping the sources array entirely);\n pass 2 re-streams the map and collects only the source paths actually\n referenced by the decoded lines. Memory therefore scales with the\n traceback's needed frames, not with the bundle's full source table —\n the property the whole streaming decoder exists for. `chunks_factory`\n is a zero-argument callable producing a fresh chunk iterator per pass.\n """ + table = self._scan_mappings(self._stream_cls(chunks_factory()), needed, max_needed) + wanted = _set(src_idx for src_idx, _line in table.values()) + sources = {} + if wanted: + sources = self._scan_sources(self._stream_cls(chunks_factory()), wanted) + return sources, table + + def _scan_mappings(self, stream, needed, max_needed, *, _error=ValueError): + """Pass 1: decode the `mappings` field; every other value is skipped.""" byte = self._skip_ws(stream, stream.read_byte()) if not byte == 123: raise _error("not a JSON object") - sources = [] - table = {} - saw_mappings = False byte = self._skip_ws(stream, stream.read_byte()) while byte == 34: key = self._read_string(stream, True) @@ -337,29 +335,56 @@ class _CriboSourceMapRuntime(object): if not byte == 58: raise _error("malformed object") byte = self._skip_ws(stream, stream.read_byte()) - if key == "sources": - sources, byte = self._read_string_array(stream, byte) - elif key == "mappings": + if key == "mappings": if not byte == 34: raise _error("mappings is not a string") - table, terminated = self._decode_vlq(stream, needed, max_needed) - saw_mappings = True - if sources: - break - if not terminated: - while True: - byte = stream.read_byte() - if byte < 0 or byte == 34: - break - byte = stream.read_byte() - else: - byte = self._skip_value(stream, byte) - byte = self._skip_ws(stream, byte) + table, _terminated = self._decode_vlq(stream, needed, max_needed) + return table + byte = self._skip_ws(stream, self._skip_value(stream, byte)) if byte == 44: byte = self._skip_ws(stream, stream.read_byte()) - if not saw_mappings: - raise _error("no mappings field") - return sources, table + raise _error("no mappings field") + + def _scan_sources(self, stream, wanted, *, _error=ValueError): + """Pass 2: collect only the `sources` entries whose index is wanted.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + if not byte == 91: + raise _error("sources is not an array") + return self._read_wanted_array_items(stream, wanted) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + return {} + + def _read_wanted_array_items(self, stream, wanted, *, _len=len, _error=ValueError): + """Read a JSON array, collecting only string items at wanted indices.""" + items = {} + index = 0 + byte = self._skip_ws(stream, stream.read_byte()) + if byte == 93: + return items + while True: + if byte == 34 and index in wanted: + items[index] = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + else: + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + index += 1 + if byte == 93 or _len(items) == _len(wanted): + return items + if not byte == 44: + raise _error("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) def _load(self, needed_lines, *, _set=set, _max=max): """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ @@ -370,16 +395,20 @@ class _CriboSourceMapRuntime(object): needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) if map_path is None: - chunks = self._inline_chunks(self._bundle_anchor) + + def chunks_factory(): + return self._inline_chunks(self._bundle_anchor) else: - chunks = self._file_chunks(map_path) - sources, table0 = self._scan(chunks, needed0, max_needed) + + def chunks_factory(): + return self._file_chunks(map_path) + sources, table0 = self._scan(chunks_factory, needed0, max_needed) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): + def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max, _enumerate=enumerate): """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" location = self._map_location() if location is None: @@ -395,7 +424,9 @@ class _CriboSourceMapRuntime(object): finally: handle.close() data = json.loads(raw.decode("utf-8")) - sources = data.get("sources") or [] + sources = {} + for index, source in _enumerate(data.get("sources") or []): + sources[index] = source mappings = data.get("mappings") or "" needed0 = _set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) @@ -465,7 +496,7 @@ class _CriboSourceMapRuntime(object): limit = _getattr(self._sys, "tracebacklimit", None) return limit if _isinstance(limit, _int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len): + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ if limit is not None: total = 0 @@ -495,11 +526,12 @@ class _CriboSourceMapRuntime(object): name = frame.f_code.co_name if filename == self._bundle: mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < _len(sources) and sources[mapped[0]]: - source = sources[mapped[0]] - if not self._os.path.isabs(source): - source = self._os.path.normpath(self._os.path.join(map_dir, source)) - filename, lineno = source, mapped[1] + if mapped is not None: + source = sources.get(mapped[0]) + if source: + if not self._os.path.isabs(source): + source = self._os.path.normpath(self._os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] entry = filename, lineno, name if entry == last: repeats += 1 @@ -644,7 +676,7 @@ class _CriboSourceMapRuntime(object): def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): - self._notify_custom_hook(self._prev_excepthook, self._sys.__excepthook__, lambda hook: hook(exc_type, exc_value, traceback_obj)) + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj)) return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -656,7 +688,7 @@ class _CriboSourceMapRuntime(object): name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, _getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args)) return self._prev_threading_hook(args) @@ -667,7 +699,7 @@ class _CriboSourceMapRuntime(object): except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable)) return self._prev_unraisablehook(unraisable) _CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index f05274b80..bc62a8b60 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -8,12 +8,17 @@ input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py import sys as _cribo_sys def _cribo_sm_import(name, *, _list=list, _import=__import__): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with that first path\n entry dropped. (`sys` itself is a builtin and can never be shadowed.)\n """ + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. (`sys` itself is a builtin and\n can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - _cribo_sys.path = _list(saved_path[1:]) + filtered = _list(saved_path[1:]) + os_mod = _cribo_sys.modules.get("os") + if os_mod is not None and saved_path: + script_dir = os_mod.path.normcase(os_mod.path.abspath(saved_path[0] or ".")) + filtered = [entry for entry in filtered if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir] + _cribo_sys.path = filtered try: return _import(name) finally: @@ -50,6 +55,10 @@ class _CriboSourceMapRuntime(object): else: self._bundle_anchor = os_mod.path.abspath(bundle_file) self._cribo_sm_runtime_marker = True + try: + self._startup_cwd = os_mod.getcwd() + except OSError: + self._startup_cwd = "." self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -61,6 +70,9 @@ class _CriboSourceMapRuntime(object): self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook self._prev_threading_hook = threading_mod.excepthook + self._default_excepthook = _cribo_sys.__excepthook__ + self._default_unraisablehook = _cribo_sys.__unraisablehook__ + self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) try: self._group_type = BaseExceptionGroup except NameError: @@ -88,6 +100,8 @@ class _CriboSourceMapRuntime(object): return None if env not in ("", "1", "true", "yes", "on"): path = env + if not self._os.path.isabs(path): + path = self._os.path.join(self._startup_cwd, path) return path, self._os.path.dirname(self._os.path.abspath(path)) bundle = self._bundle_anchor if self._mode == "inline": @@ -252,27 +266,6 @@ class _CriboSourceMapRuntime(object): byte = stream.read_byte() return byte - def _read_string_array(self, stream, byte, *, _error=ValueError): - """Read a JSON array of strings/nulls; return (list, byte after array).""" - if not byte == 91: - raise _error("expected array") - items = [] - byte = self._skip_ws(stream, stream.read_byte()) - if byte == 93: - return items, stream.read_byte() - while True: - if byte == 34: - items.append(self._read_string(stream, True)) - byte = self._skip_ws(stream, stream.read_byte()) - else: - items.append(None) - byte = self._skip_ws(stream, self._skip_value(stream, byte)) - if byte == 93: - return items, stream.read_byte() - if not byte == 44: - raise _error("malformed array") - byte = self._skip_ws(stream, stream.read_byte()) - def _decode_vlq(self, stream, needed, max_needed, *, _range=range, _len=len, _error=ValueError): """Streaming VLQ state machine over the raw bytes of the mappings string.\n\n Constant state: line/segment counters plus running deltas. Records the\n first segment per needed generated line; exits as soon as every needed\n line is resolved or the max needed line is passed. Returns\n ``(table, terminated)`` where ``terminated`` says whether the closing\n quote was consumed (early exits leave the stream inside the string).\n """ lut = {} @@ -321,15 +314,20 @@ class _CriboSourceMapRuntime(object): vlq_value = 0 vlq_shift = 0 - def _scan(self, chunks, needed, max_needed, *, _error=ValueError): - """Scan the map's top-level object; return (sources, line table).""" - stream = self._stream_cls(chunks) + def _scan(self, chunks_factory, needed, max_needed, *, _set=set, _error=ValueError): + """Two-pass scan of the map; return (sources dict, line table).\n\n Pass 1 decodes only `mappings` (skipping the sources array entirely);\n pass 2 re-streams the map and collects only the source paths actually\n referenced by the decoded lines. Memory therefore scales with the\n traceback's needed frames, not with the bundle's full source table —\n the property the whole streaming decoder exists for. `chunks_factory`\n is a zero-argument callable producing a fresh chunk iterator per pass.\n """ + table = self._scan_mappings(self._stream_cls(chunks_factory()), needed, max_needed) + wanted = _set(src_idx for src_idx, _line in table.values()) + sources = {} + if wanted: + sources = self._scan_sources(self._stream_cls(chunks_factory()), wanted) + return sources, table + + def _scan_mappings(self, stream, needed, max_needed, *, _error=ValueError): + """Pass 1: decode the `mappings` field; every other value is skipped.""" byte = self._skip_ws(stream, stream.read_byte()) if not byte == 123: raise _error("not a JSON object") - sources = [] - table = {} - saw_mappings = False byte = self._skip_ws(stream, stream.read_byte()) while byte == 34: key = self._read_string(stream, True) @@ -337,29 +335,56 @@ class _CriboSourceMapRuntime(object): if not byte == 58: raise _error("malformed object") byte = self._skip_ws(stream, stream.read_byte()) - if key == "sources": - sources, byte = self._read_string_array(stream, byte) - elif key == "mappings": + if key == "mappings": if not byte == 34: raise _error("mappings is not a string") - table, terminated = self._decode_vlq(stream, needed, max_needed) - saw_mappings = True - if sources: - break - if not terminated: - while True: - byte = stream.read_byte() - if byte < 0 or byte == 34: - break - byte = stream.read_byte() - else: - byte = self._skip_value(stream, byte) - byte = self._skip_ws(stream, byte) + table, _terminated = self._decode_vlq(stream, needed, max_needed) + return table + byte = self._skip_ws(stream, self._skip_value(stream, byte)) if byte == 44: byte = self._skip_ws(stream, stream.read_byte()) - if not saw_mappings: - raise _error("no mappings field") - return sources, table + raise _error("no mappings field") + + def _scan_sources(self, stream, wanted, *, _error=ValueError): + """Pass 2: collect only the `sources` entries whose index is wanted.""" + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 123: + raise _error("not a JSON object") + byte = self._skip_ws(stream, stream.read_byte()) + while byte == 34: + key = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + if not byte == 58: + raise _error("malformed object") + byte = self._skip_ws(stream, stream.read_byte()) + if key == "sources": + if not byte == 91: + raise _error("sources is not an array") + return self._read_wanted_array_items(stream, wanted) + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + if byte == 44: + byte = self._skip_ws(stream, stream.read_byte()) + return {} + + def _read_wanted_array_items(self, stream, wanted, *, _len=len, _error=ValueError): + """Read a JSON array, collecting only string items at wanted indices.""" + items = {} + index = 0 + byte = self._skip_ws(stream, stream.read_byte()) + if byte == 93: + return items + while True: + if byte == 34 and index in wanted: + items[index] = self._read_string(stream, True) + byte = self._skip_ws(stream, stream.read_byte()) + else: + byte = self._skip_ws(stream, self._skip_value(stream, byte)) + index += 1 + if byte == 93 or _len(items) == _len(wanted): + return items + if not byte == 44: + raise _error("malformed array") + byte = self._skip_ws(stream, stream.read_byte()) def _load(self, needed_lines, *, _set=set, _max=max): """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ @@ -370,16 +395,20 @@ class _CriboSourceMapRuntime(object): needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) if map_path is None: - chunks = self._inline_chunks(self._bundle_anchor) + + def chunks_factory(): + return self._inline_chunks(self._bundle_anchor) else: - chunks = self._file_chunks(map_path) - sources, table0 = self._scan(chunks, needed0, max_needed) + + def chunks_factory(): + return self._file_chunks(map_path) + sources, table0 = self._scan(chunks_factory, needed0, max_needed) table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max): + def _load_json_fallback(self, needed_lines, *, _open=open, _set=set, _max=max, _enumerate=enumerate): """Fallback: full json.loads parse, reusing the VLQ machine on the result.""" location = self._map_location() if location is None: @@ -395,7 +424,9 @@ class _CriboSourceMapRuntime(object): finally: handle.close() data = json.loads(raw.decode("utf-8")) - sources = data.get("sources") or [] + sources = {} + for index, source in _enumerate(data.get("sources") or []): + sources[index] = source mappings = data.get("mappings") or "" needed0 = _set(line - 1 for line in needed_lines) stream = self._stream_cls([mappings.encode("ascii"), b'"']) @@ -465,7 +496,7 @@ class _CriboSourceMapRuntime(object): limit = _getattr(self._sys, "tracebacklimit", None) return limit if _isinstance(limit, _int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit, *, _len=len): + def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ if limit is not None: total = 0 @@ -495,11 +526,12 @@ class _CriboSourceMapRuntime(object): name = frame.f_code.co_name if filename == self._bundle: mapped = table.get(lineno) - if mapped is not None and 0 <= mapped[0] < _len(sources) and sources[mapped[0]]: - source = sources[mapped[0]] - if not self._os.path.isabs(source): - source = self._os.path.normpath(self._os.path.join(map_dir, source)) - filename, lineno = source, mapped[1] + if mapped is not None: + source = sources.get(mapped[0]) + if source: + if not self._os.path.isabs(source): + source = self._os.path.normpath(self._os.path.join(map_dir, source)) + filename, lineno = source, mapped[1] entry = filename, lineno, name if entry == last: repeats += 1 @@ -644,7 +676,7 @@ class _CriboSourceMapRuntime(object): def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): - self._notify_custom_hook(self._prev_excepthook, self._sys.__excepthook__, lambda hook: hook(exc_type, exc_value, traceback_obj)) + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj)) return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -656,7 +688,7 @@ class _CriboSourceMapRuntime(object): name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, _getattr(self._threading, "__excepthook__", None), lambda hook: hook(args)) + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args)) return self._prev_threading_hook(args) @@ -667,7 +699,7 @@ class _CriboSourceMapRuntime(object): except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - self._notify_custom_hook(self._prev_unraisablehook, self._sys.__unraisablehook__, lambda hook: hook(unraisable)) + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable)) return self._prev_unraisablehook(unraisable) _CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 31cee40dc..c500f0d64 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:795 `def add(a, b):` -> calculator.py:1 -bundle:796 `result = a + b` -> calculator.py:2 -bundle:797 `return result` -> calculator.py:3 -bundle:799 `def multiply(a, b):` -> calculator.py:6 -bundle:800 `result = a * b` -> calculator.py:7 -bundle:801 `return result` -> calculator.py:8 -bundle:806 `def describe(name, value):` -> utils.py:1 -bundle:807 `return f"{name} = {value}"` -> utils.py:2 -bundle:813 `total = add(2, 3)` -> main.py:4 -bundle:814 `product = multiply(total, 4)` -> main.py:5 -bundle:815 `print(describe("total", total))` -> main.py:6 -bundle:816 `print(describe("product", product))` -> main.py:7 +bundle:827 `def add(a, b):` -> calculator.py:1 +bundle:828 `result = a + b` -> calculator.py:2 +bundle:829 `return result` -> calculator.py:3 +bundle:831 `def multiply(a, b):` -> calculator.py:6 +bundle:832 `result = a * b` -> calculator.py:7 +bundle:833 `return result` -> calculator.py:8 +bundle:838 `def describe(name, value):` -> utils.py:1 +bundle:839 `return f"{name} = {value}"` -> utils.py:2 +bundle:845 `total = add(2, 3)` -> main.py:4 +bundle:846 `product = multiply(total, 4)` -> main.py:5 +bundle:847 `print(describe("total", total))` -> main.py:6 +bundle:848 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 5ff50106e..b9db4b353 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:804 `print("effects module loading")` -> effects.py:1 -bundle:805 `COUNTER = 1` -> effects.py:3 -bundle:808 `def boost(value):` -> effects.py:6 -bundle:809 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:810 `return boosted` -> effects.py:8 -bundle:822 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:823 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:836 `print("effects module loading")` -> effects.py:1 +bundle:837 `COUNTER = 1` -> effects.py:3 +bundle:840 `def boost(value):` -> effects.py:6 +bundle:841 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:842 `return boosted` -> effects.py:8 +bundle:854 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:855 `print("boosted:", effects.boost(10))` -> main.py:4 From d9ee813da65e289cf2db8d1b995ccfae7f8214ea Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Thu, 20 Aug 2026 23:00:04 +0200 Subject: [PATCH 09/17] fix: address eighth review round on source map runtime - keep source paths absolute for stdout bundles (no anchor directory) - map the def/class header line of decorated definitions (statement ranges start at the first decorator), with validity-guarded anchors since symbol renaming can regenerate identifiers with synthetic ranges - _chain_has_group no longer follows suppressed contexts, so a group hidden by raise-from-None cannot force the unremapped fallback Addresses eighth-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 6 +-- crates/cribo/src/python/sourcemap_runtime.py | 10 +++- crates/cribo/src/source_map.rs | 33 +++++++++++++ .../bundled_code@sourcemap_basic.snap | 8 +++- .../bundled_code@sourcemap_wrapper.snap | 8 +++- .../snapshots/source_map@sourcemap_basic.snap | 24 +++++----- .../source_map@sourcemap_wrapper.snap | 14 +++--- crates/cribo/tests/test_source_maps.rs | 47 +++++++++++++++++-- 8 files changed, 119 insertions(+), 31 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 6e1a999c2..25edf0aee 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -2293,12 +2293,12 @@ impl BundleOrchestrator { |name| name.to_string_lossy().into_owned(), ); // Source paths are relative to the directory the map lives in (the - // output directory), or the current directory for stdout output. + // output directory). For stdout output the bundle's eventual location + // is unknown, so paths stay absolute. let base_dir = params .output_path .and_then(Path::parent) - .map(|dir| std::path::absolute(dir).unwrap_or_else(|_| dir.to_path_buf())) - .or_else(|| std::env::current_dir().ok()); + .map(|dir| std::path::absolute(dir).unwrap_or_else(|_| dir.to_path_buf())); let options = SourceMapOptions { file: &file_name, diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index a4727f71f..698f4337e 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -610,8 +610,14 @@ def _chain_has_group( if _isinstance(exc, self._group_type): return True cause = _getattr(exc, "__cause__", None) - context = _getattr(exc, "__context__", None) - exc = cause if cause is not None else context + if cause is not None: + exc = cause + continue + # A suppressed context (`raise ... from None`) is never rendered, + # so a group hidden there must not force the unremapped fallback. + if _getattr(exc, "__suppress_context__", False): + return False + exc = _getattr(exc, "__context__", None) return False def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index 47a69920c..2338602c3 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -344,6 +344,39 @@ impl ParallelWalker<'_> { }); } } + // With decorators present, the statement range starts at the first + // `@...` line, so the mapping recorded above covers that decorator + // — the actual `def`/`class` header still needs its own record. + // The name identifier sits on the header line on both sides, but + // symbol renaming can regenerate the original identifier with a + // synthetic (default) range; only ranges inside the statement are + // trusted, with the parameter list as a fallback anchor. + let header_anchor = match (generated, original) { + (Stmt::FunctionDef(g), Stmt::FunctionDef(o)) => { + let orig_anchor = Some(o.name.range()) + .filter(|range| o.range().contains(range.start())) + .or_else(|| { + Some(o.parameters.range()) + .filter(|range| o.range().contains(range.start())) + }); + orig_anchor.map(|range| (g.name.range(), range, o.node_index().load())) + } + (Stmt::ClassDef(g), Stmt::ClassDef(o)) => Some(o.name.range()) + .filter(|range| o.range().contains(range.start())) + .map(|range| (g.name.range(), range, o.node_index().load())), + _ => None, + }; + if !gen_decorators.is_empty() + && let Some((gen_anchor, orig_anchor, orig_index)) = header_anchor + && let Some((module_ordinal, original_line)) = + self.provenance.resolve(orig_index, orig_anchor.start()) + { + self.records.push(MappingRecord { + generated_line: self.line_index.line_of(gen_anchor.start()), + module_ordinal, + original_line, + }); + } } // Recurse into nested statement bodies even when the statement itself is diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 3098058c0..84fd8aaee 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -467,8 +467,12 @@ class _CriboSourceMapRuntime(object): if _isinstance(exc, self._group_type): return True cause = _getattr(exc, "__cause__", None) - context = _getattr(exc, "__context__", None) - exc = cause if cause is not None else context + if cause is not None: + exc = cause + continue + if _getattr(exc, "__suppress_context__", False): + return False + exc = _getattr(exc, "__context__", None) return False def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index bc62a8b60..dcdc571db 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -467,8 +467,12 @@ class _CriboSourceMapRuntime(object): if _isinstance(exc, self._group_type): return True cause = _getattr(exc, "__cause__", None) - context = _getattr(exc, "__context__", None) - exc = cause if cause is not None else context + if cause is not None: + exc = cause + continue + if _getattr(exc, "__suppress_context__", False): + return False + exc = _getattr(exc, "__context__", None) return False def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index c500f0d64..a5e7cade7 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:827 `def add(a, b):` -> calculator.py:1 -bundle:828 `result = a + b` -> calculator.py:2 -bundle:829 `return result` -> calculator.py:3 -bundle:831 `def multiply(a, b):` -> calculator.py:6 -bundle:832 `result = a * b` -> calculator.py:7 -bundle:833 `return result` -> calculator.py:8 -bundle:838 `def describe(name, value):` -> utils.py:1 -bundle:839 `return f"{name} = {value}"` -> utils.py:2 -bundle:845 `total = add(2, 3)` -> main.py:4 -bundle:846 `product = multiply(total, 4)` -> main.py:5 -bundle:847 `print(describe("total", total))` -> main.py:6 -bundle:848 `print(describe("product", product))` -> main.py:7 +bundle:831 `def add(a, b):` -> calculator.py:1 +bundle:832 `result = a + b` -> calculator.py:2 +bundle:833 `return result` -> calculator.py:3 +bundle:835 `def multiply(a, b):` -> calculator.py:6 +bundle:836 `result = a * b` -> calculator.py:7 +bundle:837 `return result` -> calculator.py:8 +bundle:842 `def describe(name, value):` -> utils.py:1 +bundle:843 `return f"{name} = {value}"` -> utils.py:2 +bundle:849 `total = add(2, 3)` -> main.py:4 +bundle:850 `product = multiply(total, 4)` -> main.py:5 +bundle:851 `print(describe("total", total))` -> main.py:6 +bundle:852 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index b9db4b353..418549698 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:836 `print("effects module loading")` -> effects.py:1 -bundle:837 `COUNTER = 1` -> effects.py:3 -bundle:840 `def boost(value):` -> effects.py:6 -bundle:841 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:842 `return boosted` -> effects.py:8 -bundle:854 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:855 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:840 `print("effects module loading")` -> effects.py:1 +bundle:841 `COUNTER = 1` -> effects.py:3 +bundle:844 `def boost(value):` -> effects.py:6 +bundle:845 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:846 `return boosted` -> effects.py:8 +bundle:858 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:859 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 300d3395d..9723a8cde 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -170,6 +170,15 @@ fn stdout_with_bare_sourcemap_selects_inline() { assert!(ok, "bundling must succeed: {stderr}"); let map_json = decode_inline_map(&stdout); assert_map_covers_helper(&map_json, ""); + // A stdout bundle can be redirected anywhere, so its map has no anchor + // directory: source paths must stay absolute. + let map = parse_map(&map_json); + assert!( + map.get_sources() + .all(|source| Path::new(source).is_absolute()), + "stdout maps must carry absolute source paths: {:?}", + map.get_sources().collect::>() + ); } #[test] @@ -1097,9 +1106,10 @@ fn map_covers_match_case_headers_and_decorators() { .filter(|token| token.get_source_id() == Some(helper_id)) .map(|token| token.get_src_line()) .collect(); - // 0-based original lines: 4 and 5 are the two decorators; 8, 10, and 12 - // are the `case` headers. - for header_line in [4, 5, 8, 10, 12] { + // 0-based original lines: 4 and 5 are the two decorators; 6 is the + // decorated `def run(value):` header itself; 8, 10, and 12 are the `case` + // headers. + for header_line in [4, 5, 6, 8, 10, 12] { assert!( mapped_helper_lines.contains(&header_line), "helper.py 0-based line {header_line} (decorator or case header) must be mapped; \ @@ -1216,3 +1226,34 @@ fn stacked_runtimes_do_not_duplicate_tracebacks() { "the crashing bundle's frames must be remapped: {stderr}" ); } + +#[test] +fn runtime_remaps_when_group_is_suppressed() { + // A caught ExceptionGroup replaced via `raise ... from None` never + // renders; its hidden presence in __context__ must not force the + // unremapped fallback for the visible ordinary exception. + let dir = make_project(&[ + ("main.py", "from helper import convert\n\nconvert()\n"), + ( + "helper.py", + "def convert():\n try:\n raise ExceptionGroup(\"grp\", \ + [ValueError(\"inner\")])\n except ExceptionGroup:\n raise \ + RuntimeError(\"converted kaboom\") from None\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!( + stderr.contains("RuntimeError: converted kaboom"), + "{stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in convert"), + "a suppressed group in __context__ must not disable remapping: {stderr}" + ); + assert!( + !stderr.contains("ExceptionGroup"), + "the suppressed group must not be rendered: {stderr}" + ); +} From 69c043b42a7e0921a3527175d59e33955119d919 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 21 Aug 2026 02:30:19 +0200 Subject: [PATCH 10/17] fix: address ninth review round on source map publication and comments - linked bundles embed a SHA-256 of their map; the runtime verifies the sibling map against it and ignores a mismatch, so no interleaving of concurrent builds (or manual file shuffling) can pair a bundle with another build's mappings - staged map files use unpredictable hashed names with fresh entropy per retry, closing the pre-created-name denial of service - percent-encode control characters in linked sourceMappingURL comments so a newline-bearing output filename cannot inject executable text Addresses ninth-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 40 ++++++++-- crates/cribo/src/python/sourcemap_runtime.py | 77 ++++++++++++++++--- crates/cribo/src/source_map.rs | 30 +++++++- .../tests/python/test_sourcemap_runtime.py | 2 + .../bundled_code@sourcemap_basic.snap | 59 +++++++++++--- .../bundled_code@sourcemap_wrapper.snap | 59 +++++++++++--- .../snapshots/source_map@sourcemap_basic.snap | 24 +++--- .../source_map@sourcemap_wrapper.snap | 14 ++-- crates/cribo/tests/test_source_maps.rs | 59 ++++++++++++++ 9 files changed, 307 insertions(+), 57 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 25edf0aee..339e40903 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -49,20 +49,48 @@ fn source_map_path_for(output_path: &Path) -> PathBuf { output_path.with_file_name(file_name) } +/// Collision-resistant, unpredictable suffix for a staged map file. +/// +/// In a shared writable directory a predictable temp name (e.g. PID-derived) +/// could be pre-created by another local user, turning every `create_new` +/// attempt into a denial of service. The suffix hashes process identity, an +/// ASLR-randomized stack address, wall-clock nanoseconds, and the attempt +/// counter — not guessable ahead of time, and fresh entropy per retry. This +/// names a transient file only; deterministic-output rules are unaffected. +fn staging_suffix(attempt: u32) -> String { + use sha2::{Digest as _, Sha256}; + + let stack_probe = 0_u8; + let mut hasher = Sha256::new(); + hasher.update(std::process::id().to_le_bytes()); + hasher.update(attempt.to_le_bytes()); + hasher.update((&raw const stack_probe as usize).to_le_bytes()); + if let Ok(elapsed) = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) { + hasher.update(elapsed.as_secs().to_le_bytes()); + hasher.update(elapsed.subsec_nanos().to_le_bytes()); + } + let digest = hasher.finalize(); + let mut hex = String::with_capacity(16); + for byte in &digest[..8] { + write!(hex, "{byte:02x}").expect("Writing to String never fails"); + } + hex +} + /// Stage the source map into a collision-resistant temp file next to `map_path`. /// /// The file is opened with `create_new` (`O_EXCL` semantics), which neither /// follows a pre-planted symlink nor truncates an existing file — a requirement /// for outputs in shared sticky directories, where a temp name could otherwise -/// be predicted and pointed elsewhere. Name collisions retry with a new suffix, -/// which also keeps concurrent builds from stomping each other's staged map. +/// be predicted and pointed elsewhere. Names are unpredictable (see +/// [`staging_suffix`]) and collisions retry with fresh entropy, which also +/// keeps concurrent builds from stomping each other's staged map. /// When a previous map exists, its permissions are copied onto the staged file /// before any content is written, so a restricted map (e.g. 0600 protecting /// `sourcesContent`) stays restricted across rebuilds. fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { use std::io::Write as _; - let pid = std::process::id(); let base_name = map_path.file_name().map_or_else( || std::ffi::OsString::from("bundle.py.map"), std::ffi::OsStr::to_os_string, @@ -70,7 +98,7 @@ fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { let mut attempt = 0_u32; loop { let mut tmp_name = base_name.clone(); - tmp_name.push(format!(".{pid}.{attempt}.tmp")); + tmp_name.push(format!(".{}.tmp", staging_suffix(attempt))); let tmp_path = map_path.with_file_name(tmp_name); match fs::OpenOptions::new() .write(true) @@ -92,7 +120,7 @@ fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { } return Ok(tmp_path); } - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists && attempt < 32 => { + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists && attempt < 1024 => { attempt += 1; } Err(err) => return Err(err), @@ -769,6 +797,8 @@ impl BundleOrchestrator { |name| name.to_string_lossy().into_owned(), ); bundled_code.push('\n'); + bundled_code + .push_str(&crate::source_map::linked_map_digest_comment(map_json)); bundled_code.push_str(&crate::source_map::linked_source_mapping_comment( &map_file_name, )); diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 698f4337e..4c76632e4 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -86,7 +86,16 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): + def __init__( + self, + mode, + bundle_file, + os_mod, + binascii_mod, + threading_mod, + traceback_mod, + hashlib_mod, + ): self._mode = mode # As-given path for frame matching (co_filename uses the invocation # spelling), plus a startup-anchored absolute path for file I/O so a @@ -115,6 +124,7 @@ def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, trace # first-party module registering sys.modules["traceback"] later cannot # degrade exception formatting. self._traceback = traceback_mod + self._hashlib = hashlib_mod # Re-entrancy guard; thread-local so a hook firing on one thread never # disables remapping on another. self._local = threading_mod.local() @@ -153,6 +163,7 @@ def _bootstrap(cls, mode, bundle_file): _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback"), + _cribo_sm_import("hashlib"), ) runtime.install() except BaseException: @@ -186,7 +197,7 @@ def _map_location(self): return (None, self._os.path.dirname(bundle)) sibling = bundle + ".map" if self._mode == "linked": - if self._os.path.exists(sibling): + if self._os.path.exists(sibling) and self._map_matches_bundle(sibling): return (sibling, self._os.path.dirname(sibling)) return None # external: opt in via CRIBO_SOURCE_MAPS=1 (a path was handled above) @@ -206,17 +217,15 @@ def _file_chunks(self, path, *, _open=open): finally: handle.close() - def _find_inline_payload(self, handle, *, _len=len): - """Backward-scan the bundle for the last inline map marker. + def _find_marker_tail(self, handle, marker, *, _len=len): + """Backward-scan a file for the last occurrence of `marker`. - Returns the byte offset of the base64 payload, or -1. Only the tail of - the file is examined; the bundle body is never read. + Returns the offset of the payload following the marker, or -1. Only the + tail of the file is examined; the body is never read. """ - marker = b"# sourceMappingURL=data:" handle.seek(0, 2) position = handle.tell() overlap = b"" - found = -1 while position > 0: step = self._CHUNK if position >= self._CHUNK else position position -= step @@ -224,9 +233,13 @@ def _find_inline_payload(self, handle, *, _len=len): data = handle.read(step) + overlap index = data.rfind(marker) if index >= 0: - found = position + index - break + return position + index + _len(marker) overlap = data[: _len(marker) - 1] + return -1 + + def _find_inline_payload(self, handle): + """Offset of the inline map's base64 payload in the bundle, or -1.""" + found = self._find_marker_tail(handle, b"# sourceMappingURL=data:") if found < 0: return -1 handle.seek(found) @@ -234,7 +247,49 @@ def _find_inline_payload(self, handle, *, _len=len): base64_at = head.find(b"base64,") if base64_at < 0: return -1 - return found + base64_at + _len(b"base64,") + return found + base64_at + 7 + + def _bundle_expected_digest(self, *, _open=open, _len=len, _int=int): + """SHA-256 hex the bundle records for its linked map, or None.""" + try: + handle = _open(self._bundle_anchor, "rb") + except OSError: + return None + try: + found = self._find_marker_tail(handle, b"# cribo-sourcemap-sha256=") + if found < 0: + return None + handle.seek(found) + digest = handle.read(64) + finally: + handle.close() + if not _len(digest) == 64: + return None + try: + _int(digest, 16) + except ValueError: + return None + return digest.decode("ascii").lower() + + def _map_matches_bundle(self, map_path, *, _bex=BaseException): + """False only when the bundle records a digest and the map disagrees. + + This is what makes linked-mode publication safe against interleaved + concurrent builds and manual file shuffling: the digest travels inside + the bundle, which is always internally consistent, so a sibling map + from a different build is detected and ignored. Verification errors + fail open (the subsequent map read would surface them anyway). + """ + try: + expected = self._bundle_expected_digest() + if expected is None: + return True + hasher = self._hashlib.sha256() + for chunk in self._file_chunks(map_path): + hasher.update(chunk) + return hasher.hexdigest() == expected + except _bex: + return True def _inline_chunks(self, path, *, _open=open, _len=len): """Yield decoded chunks of an inline (base64 data URL) source map.""" diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index 2338602c3..b8b6488cc 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -526,9 +526,35 @@ fn relative_path(base: &std::path::Path, target: &std::path::Path) -> std::path: /// Comment linking the bundle to an adjacent source map file. /// /// The `sourceMappingURL` convention is borrowed from the JS ecosystem; Python -/// treats the line as a plain comment. +/// treats the line as a plain comment. The file name is percent-encoded so a +/// hostile or accidental control character (e.g. a newline in a Unix filename) +/// cannot terminate the comment and inject executable text into the bundle. pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { - format!("# sourceMappingURL={map_file_name}\n") + let mut encoded = String::with_capacity(map_file_name.len()); + for byte in map_file_name.bytes() { + if byte < 0x20 || byte == 0x7F || byte == b'%' { + encoded.push_str(&format!("%{byte:02X}")); + } else { + encoded.push(byte as char); + } + } + format!("# sourceMappingURL={encoded}\n") +} + +/// Comment recording the SHA-256 of the linked map at build time. +/// +/// The runtime refuses a sibling map whose digest does not match, so no +/// interleaving of concurrent builds (or manual file shuffling) can pair a +/// bundle with another build's mappings — the digest travels inside the bundle +/// itself, which is always internally consistent. +pub(crate) fn linked_map_digest_comment(map_json: &str) -> String { + use sha2::{Digest as _, Sha256}; + let digest = Sha256::digest(map_json.as_bytes()); + let mut hex = String::with_capacity(64); + for byte in digest { + hex.push_str(&format!("{byte:02x}")); + } + format!("# cribo-sourcemap-sha256={hex}\n") } /// Comment embedding the source map as a base64 data URL. diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index b50414de3..fff65a24f 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -33,6 +33,7 @@ def load_runtime(path): __import__("binascii"), threading, __import__("traceback"), + __import__("hashlib"), ) @@ -237,6 +238,7 @@ def test_env_path_wins_for_every_mode(rt): __import__("binascii"), threading, __import__("traceback"), + __import__("hashlib"), ) loaded = inline_stdin._load({1}) assert loaded is not None, "env path must activate a inline bundle" diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 84fd8aaee..72ab851fd 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -47,7 +47,7 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): self._mode = mode self._bundle = bundle_file if bundle_file == "": @@ -66,6 +66,7 @@ class _CriboSourceMapRuntime(object): self._stream_cls = _CriboSmStream self._import = _cribo_sm_import self._traceback = traceback_mod + self._hashlib = hashlib_mod self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook @@ -88,7 +89,7 @@ class _CriboSourceMapRuntime(object): def _bootstrap(cls, mode, bundle_file): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: - runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback")) + runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback"), _cribo_sm_import("hashlib")) runtime.install() except BaseException: pass @@ -110,7 +111,7 @@ class _CriboSourceMapRuntime(object): return None, self._os.path.dirname(bundle) sibling = bundle + ".map" if self._mode == "linked": - if self._os.path.exists(sibling): + if self._os.path.exists(sibling) and self._map_matches_bundle(sibling): return sibling, self._os.path.dirname(sibling) return None if env in ("1", "true", "yes", "on"): @@ -129,13 +130,11 @@ class _CriboSourceMapRuntime(object): finally: handle.close() - def _find_inline_payload(self, handle, *, _len=len): - """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of\n the file is examined; the bundle body is never read.\n """ - marker = b"# sourceMappingURL=data:" + def _find_marker_tail(self, handle, marker, *, _len=len): + """Backward-scan a file for the last occurrence of `marker`.\n\n Returns the offset of the payload following the marker, or -1. Only the\n tail of the file is examined; the body is never read.\n """ handle.seek(0, 2) position = handle.tell() overlap = b"" - found = -1 while position > 0: step = self._CHUNK if position >= self._CHUNK else position position -= step @@ -143,9 +142,13 @@ class _CriboSourceMapRuntime(object): data = handle.read(step) + overlap index = data.rfind(marker) if index >= 0: - found = position + index - break + return position + index + _len(marker) overlap = data[:_len(marker) - 1] + return -1 + + def _find_inline_payload(self, handle): + """Offset of the inline map's base64 payload in the bundle, or -1.""" + found = self._find_marker_tail(handle, b"# sourceMappingURL=data:") if found < 0: return -1 handle.seek(found) @@ -153,7 +156,42 @@ class _CriboSourceMapRuntime(object): base64_at = head.find(b"base64,") if base64_at < 0: return -1 - return found + base64_at + _len(b"base64,") + return found + base64_at + 7 + + def _bundle_expected_digest(self, *, _open=open, _len=len, _int=int): + """SHA-256 hex the bundle records for its linked map, or None.""" + try: + handle = _open(self._bundle_anchor, "rb") + except OSError: + return None + try: + found = self._find_marker_tail(handle, b"# cribo-sourcemap-sha256=") + if found < 0: + return None + handle.seek(found) + digest = handle.read(64) + finally: + handle.close() + if not _len(digest) == 64: + return None + try: + _int(digest, 16) + except ValueError: + return None + return digest.decode("ascii").lower() + + def _map_matches_bundle(self, map_path, *, _bex=BaseException): + """False only when the bundle records a digest and the map disagrees.\n\n This is what makes linked-mode publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Verification errors\n fail open (the subsequent map read would surface them anyway).\n """ + try: + expected = self._bundle_expected_digest() + if expected is None: + return True + hasher = self._hashlib.sha256() + for chunk in self._file_chunks(map_path): + hasher.update(chunk) + return hasher.hexdigest() == expected + except _bex: + return True def _inline_chunks(self, path, *, _open=open, _len=len): """Yield decoded chunks of an inline (base64 data URL) source map.""" @@ -854,4 +892,5 @@ total = add(2, 3) product = multiply(total, 4) print(describe("total", total)) print(describe("product", product)) +# cribo-sourcemap-sha256=e62b7c23f3a0b0127209f929705d9e5e0dbb57a5a8582a707bb0ac7933284d4c # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index dcdc571db..2d7dd24be 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -47,7 +47,7 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod): + def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): self._mode = mode self._bundle = bundle_file if bundle_file == "": @@ -66,6 +66,7 @@ class _CriboSourceMapRuntime(object): self._stream_cls = _CriboSmStream self._import = _cribo_sm_import self._traceback = traceback_mod + self._hashlib = hashlib_mod self._local = threading_mod.local() self._prev_excepthook = _cribo_sys.excepthook self._prev_unraisablehook = _cribo_sys.unraisablehook @@ -88,7 +89,7 @@ class _CriboSourceMapRuntime(object): def _bootstrap(cls, mode, bundle_file): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: - runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback")) + runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback"), _cribo_sm_import("hashlib")) runtime.install() except BaseException: pass @@ -110,7 +111,7 @@ class _CriboSourceMapRuntime(object): return None, self._os.path.dirname(bundle) sibling = bundle + ".map" if self._mode == "linked": - if self._os.path.exists(sibling): + if self._os.path.exists(sibling) and self._map_matches_bundle(sibling): return sibling, self._os.path.dirname(sibling) return None if env in ("1", "true", "yes", "on"): @@ -129,13 +130,11 @@ class _CriboSourceMapRuntime(object): finally: handle.close() - def _find_inline_payload(self, handle, *, _len=len): - """Backward-scan the bundle for the last inline map marker.\n\n Returns the byte offset of the base64 payload, or -1. Only the tail of\n the file is examined; the bundle body is never read.\n """ - marker = b"# sourceMappingURL=data:" + def _find_marker_tail(self, handle, marker, *, _len=len): + """Backward-scan a file for the last occurrence of `marker`.\n\n Returns the offset of the payload following the marker, or -1. Only the\n tail of the file is examined; the body is never read.\n """ handle.seek(0, 2) position = handle.tell() overlap = b"" - found = -1 while position > 0: step = self._CHUNK if position >= self._CHUNK else position position -= step @@ -143,9 +142,13 @@ class _CriboSourceMapRuntime(object): data = handle.read(step) + overlap index = data.rfind(marker) if index >= 0: - found = position + index - break + return position + index + _len(marker) overlap = data[:_len(marker) - 1] + return -1 + + def _find_inline_payload(self, handle): + """Offset of the inline map's base64 payload in the bundle, or -1.""" + found = self._find_marker_tail(handle, b"# sourceMappingURL=data:") if found < 0: return -1 handle.seek(found) @@ -153,7 +156,42 @@ class _CriboSourceMapRuntime(object): base64_at = head.find(b"base64,") if base64_at < 0: return -1 - return found + base64_at + _len(b"base64,") + return found + base64_at + 7 + + def _bundle_expected_digest(self, *, _open=open, _len=len, _int=int): + """SHA-256 hex the bundle records for its linked map, or None.""" + try: + handle = _open(self._bundle_anchor, "rb") + except OSError: + return None + try: + found = self._find_marker_tail(handle, b"# cribo-sourcemap-sha256=") + if found < 0: + return None + handle.seek(found) + digest = handle.read(64) + finally: + handle.close() + if not _len(digest) == 64: + return None + try: + _int(digest, 16) + except ValueError: + return None + return digest.decode("ascii").lower() + + def _map_matches_bundle(self, map_path, *, _bex=BaseException): + """False only when the bundle records a digest and the map disagrees.\n\n This is what makes linked-mode publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Verification errors\n fail open (the subsequent map read would surface them anyway).\n """ + try: + expected = self._bundle_expected_digest() + if expected is None: + return True + hasher = self._hashlib.sha256() + for chunk in self._file_chunks(map_path): + hasher.update(chunk) + return hasher.hexdigest() == expected + except _bex: + return True def _inline_chunks(self, path, *, _open=open, _len=len): """Yield decoded chunks of an inline (base64 data URL) source map.""" @@ -861,4 +899,5 @@ effects.__init__ = _cribo_init___cribo_503b17_effects effects = _cribo_init___cribo_503b17_effects(effects) print("counter:", effects.COUNTER) print("boosted:", effects.boost(10)) +# cribo-sourcemap-sha256=3d0cc0c794af20e88f7ecfaa4ae7812c9e5a85caa0e8130e1bcd5c394445121c # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index a5e7cade7..5511d97dc 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:831 `def add(a, b):` -> calculator.py:1 -bundle:832 `result = a + b` -> calculator.py:2 -bundle:833 `return result` -> calculator.py:3 -bundle:835 `def multiply(a, b):` -> calculator.py:6 -bundle:836 `result = a * b` -> calculator.py:7 -bundle:837 `return result` -> calculator.py:8 -bundle:842 `def describe(name, value):` -> utils.py:1 -bundle:843 `return f"{name} = {value}"` -> utils.py:2 -bundle:849 `total = add(2, 3)` -> main.py:4 -bundle:850 `product = multiply(total, 4)` -> main.py:5 -bundle:851 `print(describe("total", total))` -> main.py:6 -bundle:852 `print(describe("product", product))` -> main.py:7 +bundle:869 `def add(a, b):` -> calculator.py:1 +bundle:870 `result = a + b` -> calculator.py:2 +bundle:871 `return result` -> calculator.py:3 +bundle:873 `def multiply(a, b):` -> calculator.py:6 +bundle:874 `result = a * b` -> calculator.py:7 +bundle:875 `return result` -> calculator.py:8 +bundle:880 `def describe(name, value):` -> utils.py:1 +bundle:881 `return f"{name} = {value}"` -> utils.py:2 +bundle:887 `total = add(2, 3)` -> main.py:4 +bundle:888 `product = multiply(total, 4)` -> main.py:5 +bundle:889 `print(describe("total", total))` -> main.py:6 +bundle:890 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 418549698..838995ad8 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:840 `print("effects module loading")` -> effects.py:1 -bundle:841 `COUNTER = 1` -> effects.py:3 -bundle:844 `def boost(value):` -> effects.py:6 -bundle:845 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:846 `return boosted` -> effects.py:8 -bundle:858 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:859 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:878 `print("effects module loading")` -> effects.py:1 +bundle:879 `COUNTER = 1` -> effects.py:3 +bundle:882 `def boost(value):` -> effects.py:6 +bundle:883 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:884 `return boosted` -> effects.py:8 +bundle:896 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:897 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 9723a8cde..99a4d354b 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1257,3 +1257,62 @@ fn runtime_remaps_when_group_is_suppressed() { "the suppressed group must not be rendered: {stderr}" ); } + +#[test] +fn runtime_rejects_mismatched_sibling_map() { + // Linked bundles embed a SHA-256 of their map; a sibling map from a + // different build (concurrent-build interleaving, manual copying) must be + // ignored rather than silently applying wrong mappings. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + + // Build a different project and steal its map. + let other = make_project(&[ + ("main.py", "from helper import other\n\nprint(other())\n"), + ("helper.py", "def other():\n return 42\n"), + ]); + let other_bundle = other.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&other), + "--output", + &other_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + fs::copy( + other.path().join("bundle.py.map"), + dir.path().join("bundle.py.map"), + ) + .expect("swap in a foreign map"); + + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[cfg(unix)] +#[test] +fn linked_comment_escapes_control_characters_in_names() { + // A newline inside the output file name must not break out of the + // sourceMappingURL comment (which would inject executable text). + let dir = fixture_project(); + let out = dir.path().join("bun\ndle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bun%0Adle.py.map"), + "control characters in the map name must be percent-encoded" + ); + // The bundle must remain valid, runnable Python. + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!(stdout.contains("hello world")); +} From 5b4750aab7e86fe17770088d1434a64d27f02af5 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 21 Aug 2026 02:37:48 +0200 Subject: [PATCH 11/17] fix: address tenth review round on source map runtime - shadow-proof importer de-duplicates script-directory path entries lexically (exact spelling, trailing separators, cwd aliases) so the filter works even before os is importable (python -S with PYTHONPATH=.), with the abspath-based comparison layered on once os is available - skip modules whose filesystem path is not valid UTF-8 instead of storing a lossy rendering: replacement-character paths displayed nonexistent files and distinct byte paths could collide onto one source id, mispairing mappings and sourcesContent Addresses tenth-round review comments on #570 --- crates/cribo/src/python/sourcemap_runtime.py | 25 ++++++++++---- crates/cribo/src/source_map.rs | 33 +++++++++++++++---- .../bundled_code@sourcemap_basic.snap | 21 ++++++++---- .../bundled_code@sourcemap_wrapper.snap | 21 ++++++++---- .../snapshots/source_map@sourcemap_basic.snap | 24 +++++++------- .../source_map@sourcemap_wrapper.snap | 14 ++++---- 6 files changed, 95 insertions(+), 43 deletions(-) diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 4c76632e4..e0a450838 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -22,23 +22,36 @@ def _cribo_sm_import(name, *, _list=list, _import=__import__): project file named e.g. `threading.py` sitting next to the bundle would otherwise shadow the stdlib for this runtime. Already-imported modules are taken from `sys.modules`; otherwise the import runs with every path entry - resolving to the script directory removed. (`sys` itself is a builtin and - can never be shadowed.) + resolving to the script directory removed. Lexical de-duplication (exact + spelling, trailing separators, cwd aliases) works even before `os` itself + is importable (`python -S`); once `os` is available, entries are also + compared by absolute path. (`sys` is a builtin and can never be shadowed.) """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - filtered = _list(saved_path[1:]) + first = saved_path[0] if saved_path else None + first_trimmed = first.rstrip("/\\") if first else first + filtered = [] + for entry in saved_path[1:]: + try: + if entry in ("", ".", "./", first) or ( + first_trimmed and entry.rstrip("/\\") == first_trimmed + ): + continue + except (TypeError, AttributeError): + pass # exotic non-str path entry: keep it + filtered.append(entry) os_mod = _cribo_sys.modules.get("os") - if os_mod is not None and saved_path: - script_dir = os_mod.path.normcase(os_mod.path.abspath(saved_path[0] or ".")) + if os_mod is not None and first is not None: + script_dir = os_mod.path.normcase(os_mod.path.abspath(first or ".")) filtered = [ entry for entry in filtered if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir ] - _cribo_sys.path = filtered + _cribo_sys.path = _list(filtered) try: return _import(name) finally: diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index b8b6488cc..5abc390ef 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -645,7 +645,9 @@ pub(crate) fn build_source_map( let records = extract_statement_mappings(bundle_text, bundled_ast, provenance)?; let mut generator = SourceMapGenerator::new(options.file); - let mut ordinal_to_source: Vec> = vec![None; provenance.modules().len()]; + // None = not yet seen; Some(None) = seen but unmappable (non-UTF-8 path). + let mut ordinal_to_source: Vec>> = + vec![None; provenance.modules().len()]; for record in records { let module = &provenance.modules()[record.module_ordinal]; @@ -654,12 +656,31 @@ pub(crate) fn build_source_map( || module.path.clone(), |base| relative_path(base, &module.path), ); - let content = options.include_contents.then(|| module.source.clone()); - let id = generator.add_source(&display_path.to_string_lossy(), content); - ordinal_to_source[record.module_ordinal] = Some(id); - id + // Source Map v3 is JSON: a path that is not valid UTF-8 has no + // lossless representation. A lossy rendering would display a + // nonexistent replacement-character path and could collide with a + // *different* non-UTF-8 path, mispairing mappings and + // sourcesContent — skip such modules instead (their frames stay on + // bundle coordinates). + let resolved = display_path.to_str().map_or_else( + || { + log::debug!( + "source map: skipping module with non-UTF-8 path: {}", + display_path.display() + ); + None + }, + |display| { + let content = options.include_contents.then(|| module.source.clone()); + Some(generator.add_source(display, content)) + }, + ); + ordinal_to_source[record.module_ordinal] = Some(resolved); + resolved }); - generator.add_mapping(record.generated_line, source_id, record.original_line); + if let Some(source_id) = source_id { + generator.add_mapping(record.generated_line, source_id, record.original_line); + } } Ok(generator.into_json()) diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 72ab851fd..cab36ce94 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -8,17 +8,26 @@ input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py import sys as _cribo_sys def _cribo_sm_import(name, *, _list=list, _import=__import__): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. (`sys` itself is a builtin and\n can never be shadowed.)\n """ + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. Lexical de-duplication (exact\n spelling, trailing separators, cwd aliases) works even before `os` itself\n is importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. (`sys` is a builtin and can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - filtered = _list(saved_path[1:]) + first = saved_path[0] if saved_path else None + first_trimmed = first.rstrip("/\\") if first else first + filtered = [] + for entry in saved_path[1:]: + try: + if entry in ("", ".", "./", first) or first_trimmed and entry.rstrip("/\\") == first_trimmed: + continue + except (TypeError, AttributeError): + pass + filtered.append(entry) os_mod = _cribo_sys.modules.get("os") - if os_mod is not None and saved_path: - script_dir = os_mod.path.normcase(os_mod.path.abspath(saved_path[0] or ".")) + if os_mod is not None and first is not None: + script_dir = os_mod.path.normcase(os_mod.path.abspath(first or ".")) filtered = [entry for entry in filtered if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir] - _cribo_sys.path = filtered + _cribo_sys.path = _list(filtered) try: return _import(name) finally: @@ -892,5 +901,5 @@ total = add(2, 3) product = multiply(total, 4) print(describe("total", total)) print(describe("product", product)) -# cribo-sourcemap-sha256=e62b7c23f3a0b0127209f929705d9e5e0dbb57a5a8582a707bb0ac7933284d4c +# cribo-sourcemap-sha256=c0fd18c37fc76a6f88625c54619e79a1f7d76a09a174871699fb835cfedaeb35 # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 2d7dd24be..7affa0c51 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -8,17 +8,26 @@ input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py import sys as _cribo_sys def _cribo_sm_import(name, *, _list=list, _import=__import__): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. (`sys` itself is a builtin and\n can never be shadowed.)\n """ + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. Lexical de-duplication (exact\n spelling, trailing separators, cwd aliases) works even before `os` itself\n is importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. (`sys` is a builtin and can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - filtered = _list(saved_path[1:]) + first = saved_path[0] if saved_path else None + first_trimmed = first.rstrip("/\\") if first else first + filtered = [] + for entry in saved_path[1:]: + try: + if entry in ("", ".", "./", first) or first_trimmed and entry.rstrip("/\\") == first_trimmed: + continue + except (TypeError, AttributeError): + pass + filtered.append(entry) os_mod = _cribo_sys.modules.get("os") - if os_mod is not None and saved_path: - script_dir = os_mod.path.normcase(os_mod.path.abspath(saved_path[0] or ".")) + if os_mod is not None and first is not None: + script_dir = os_mod.path.normcase(os_mod.path.abspath(first or ".")) filtered = [entry for entry in filtered if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir] - _cribo_sys.path = filtered + _cribo_sys.path = _list(filtered) try: return _import(name) finally: @@ -899,5 +908,5 @@ effects.__init__ = _cribo_init___cribo_503b17_effects effects = _cribo_init___cribo_503b17_effects(effects) print("counter:", effects.COUNTER) print("boosted:", effects.boost(10)) -# cribo-sourcemap-sha256=3d0cc0c794af20e88f7ecfaa4ae7812c9e5a85caa0e8130e1bcd5c394445121c +# cribo-sourcemap-sha256=fd8814e47b6c032610e09cd0d1c0e24f31b29af75a32a0dadfecb6196527bb69 # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 5511d97dc..d3a7b36c4 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:869 `def add(a, b):` -> calculator.py:1 -bundle:870 `result = a + b` -> calculator.py:2 -bundle:871 `return result` -> calculator.py:3 -bundle:873 `def multiply(a, b):` -> calculator.py:6 -bundle:874 `result = a * b` -> calculator.py:7 -bundle:875 `return result` -> calculator.py:8 -bundle:880 `def describe(name, value):` -> utils.py:1 -bundle:881 `return f"{name} = {value}"` -> utils.py:2 -bundle:887 `total = add(2, 3)` -> main.py:4 -bundle:888 `product = multiply(total, 4)` -> main.py:5 -bundle:889 `print(describe("total", total))` -> main.py:6 -bundle:890 `print(describe("product", product))` -> main.py:7 +bundle:878 `def add(a, b):` -> calculator.py:1 +bundle:879 `result = a + b` -> calculator.py:2 +bundle:880 `return result` -> calculator.py:3 +bundle:882 `def multiply(a, b):` -> calculator.py:6 +bundle:883 `result = a * b` -> calculator.py:7 +bundle:884 `return result` -> calculator.py:8 +bundle:889 `def describe(name, value):` -> utils.py:1 +bundle:890 `return f"{name} = {value}"` -> utils.py:2 +bundle:896 `total = add(2, 3)` -> main.py:4 +bundle:897 `product = multiply(total, 4)` -> main.py:5 +bundle:898 `print(describe("total", total))` -> main.py:6 +bundle:899 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 838995ad8..6bc707b0c 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:878 `print("effects module loading")` -> effects.py:1 -bundle:879 `COUNTER = 1` -> effects.py:3 -bundle:882 `def boost(value):` -> effects.py:6 -bundle:883 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:884 `return boosted` -> effects.py:8 -bundle:896 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:897 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:887 `print("effects module loading")` -> effects.py:1 +bundle:888 `COUNTER = 1` -> effects.py:3 +bundle:891 `def boost(value):` -> effects.py:6 +bundle:892 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:893 `return boosted` -> effects.py:8 +bundle:905 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:906 `print("boosted:", effects.boost(10))` -> main.py:4 From e18a1ec8c4a771fbf0e1a4f45eb97f7c807caec4 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 21 Aug 2026 02:54:51 +0200 Subject: [PATCH 12/17] fix: address eleventh review round on source map runtime - decorated class headers get a fallback anchor (base-class argument list) since the inliner regenerates class names with synthetic ranges - linked sourceMappingURL comments keep non-ASCII characters verbatim; percent-encoding is reserved for control characters (the previous byte-cast mangled UTF-8 names) Addresses eleventh-round review comments on #570 --- crates/cribo/src/source_map.rs | 26 ++++++++++++----- crates/cribo/tests/test_source_maps.rs | 40 ++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index 5abc390ef..0733c1f3e 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -361,9 +361,20 @@ impl ParallelWalker<'_> { }); orig_anchor.map(|range| (g.name.range(), range, o.node_index().load())) } - (Stmt::ClassDef(g), Stmt::ClassDef(o)) => Some(o.name.range()) - .filter(|range| o.range().contains(range.start())) - .map(|range| (g.name.range(), range, o.node_index().load())), + (Stmt::ClassDef(g), Stmt::ClassDef(o)) => { + // The inliner regenerates inlined class names with a + // default range, so fall back to the base-class / + // metaclass argument list, which also sits on the header. + let orig_anchor = Some(o.name.range()) + .filter(|range| o.range().contains(range.start())) + .or_else(|| { + o.arguments + .as_deref() + .map(ruff_text_size::Ranged::range) + .filter(|range| o.range().contains(range.start())) + }); + orig_anchor.map(|range| (g.name.range(), range, o.node_index().load())) + } _ => None, }; if !gen_decorators.is_empty() @@ -531,11 +542,12 @@ fn relative_path(base: &std::path::Path, target: &std::path::Path) -> std::path: /// cannot terminate the comment and inject executable text into the bundle. pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { let mut encoded = String::with_capacity(map_file_name.len()); - for byte in map_file_name.bytes() { - if byte < 0x20 || byte == 0x7F || byte == b'%' { - encoded.push_str(&format!("%{byte:02X}")); + for character in map_file_name.chars() { + if character < '\u{20}' || character == '\u{7F}' || character == '%' { + let _ = + std::fmt::Write::write_fmt(&mut encoded, format_args!("%{:02X}", character as u32)); } else { - encoded.push(byte as char); + encoded.push(character); } } format!("# sourceMappingURL={encoded}\n") diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 99a4d354b..832837511 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1074,13 +1074,16 @@ fn runtime_renders_long_exception_chains_fully() { #[test] fn map_covers_match_case_headers_and_decorators() { let dir = make_project(&[ - ("main.py", "from helper import run\n\nprint(run(1))\n"), + ( + "main.py", + "from helper import Widget, run\n\nprint(run(1))\nprint(Widget())\n", + ), ( "helper.py", "def trace(func):\n return func\n\n\n@trace\n@trace\ndef run(value):\n match \ value:\n case 0:\n return \"zero\"\n case _ if value > \ 0:\n return \"positive\"\n case _:\n return \ - \"negative\"\n", + \"negative\"\n\n\n@trace\nclass Widget(dict):\n pass\n", ), ]); let out = dir.path().join("bundle.py"); @@ -1106,10 +1109,12 @@ fn map_covers_match_case_headers_and_decorators() { .filter(|token| token.get_source_id() == Some(helper_id)) .map(|token| token.get_src_line()) .collect(); - // 0-based original lines: 4 and 5 are the two decorators; 6 is the + // 0-based original lines: 4 and 5 are the def decorators; 6 is the // decorated `def run(value):` header itself; 8, 10, and 12 are the `case` - // headers. - for header_line in [4, 5, 6, 8, 10, 12] { + // headers; 16 and 17 are the decorated class's decorator and header (the + // inliner regenerates class names with synthetic ranges, so the header + // anchor falls back to the base-class list). + for header_line in [4, 5, 6, 8, 10, 12, 16, 17] { assert!( mapped_helper_lines.contains(&header_line), "helper.py 0-based line {header_line} (decorator or case header) must be mapped; \ @@ -1316,3 +1321,28 @@ fn linked_comment_escapes_control_characters_in_names() { assert!(ok, "bundle must run: {stderr}"); assert!(stdout.contains("hello world")); } + +#[test] +fn linked_comment_preserves_unicode_names() { + // Non-ASCII characters in the output name must pass through verbatim + // (percent-encoding is reserved for control characters), so external map + // consumers following the comment can locate the sibling file. + let dir = fixture_project(); + let out = dir.path().join("bündle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bündle.py.map"), + "unicode map names must not be mangled" + ); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!(stdout.contains("hello world")); +} From 62dae9342b56f4e387d381a5a69ed2d622f1cfc0 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 21 Aug 2026 03:21:40 +0200 Subject: [PATCH 13/17] fix: address twelfth review round on source map runtime - external mode embeds and verifies the same map digest as linked mode, so CRIBO_SOURCE_MAPS=1 cannot silently apply a foreign sibling map - digest verification and both decode passes read from one pinned file handle, so the verified bytes are exactly the decoded bytes even if a concurrent build renames a new map into place mid-decode - percent-encode URL delimiters (#, ?, etc.) in sourceMappingURL comments - map interior physical lines of simple multiline statements (offset-aligned; multiline string content is preserved verbatim by the generator) - unmapped frames render through traceback.StackSummary, preserving PEP 657 caret and anchor indicators - stacked cribo runtimes register on sys and merge mappings per bundle, so a traceback crossing bundle boundaries remaps every frame Addresses twelfth-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 8 +- crates/cribo/src/python/sourcemap_runtime.py | 270 ++++++++++++------ crates/cribo/src/source_map.rs | 55 +++- .../tests/python/test_sourcemap_runtime.py | 11 +- .../bundled_code@sourcemap_basic.snap | 211 +++++++++----- .../bundled_code@sourcemap_wrapper.snap | 211 +++++++++----- .../snapshots/source_map@sourcemap_basic.snap | 24 +- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 107 +++++++ 9 files changed, 640 insertions(+), 271 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 339e40903..e8fe1826f 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -791,14 +791,16 @@ impl BundleOrchestrator { match mode { SourceMapMode::Linked | SourceMapMode::External => { let map_path = source_map_path_for(output_path); + // Both sibling-map modes embed the map digest so the + // runtime can reject a mismatched pair; only linked mode + // adds the sourceMappingURL reference. + bundled_code.push('\n'); + bundled_code.push_str(&crate::source_map::linked_map_digest_comment(map_json)); if mode == SourceMapMode::Linked { let map_file_name = map_path.file_name().map_or_else( || map_path.to_string_lossy().into_owned(), |name| name.to_string_lossy().into_owned(), ); - bundled_code.push('\n'); - bundled_code - .push_str(&crate::source_map::linked_map_digest_comment(map_json)); bundled_code.push_str(&crate::source_map::linked_source_mapping_comment( &map_file_name, )); diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index e0a450838..6a03d5961 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -156,7 +156,17 @@ def __init__( self._group_type = None def install(self): - """Install the three hooks; the previous hooks stay chained.""" + """Install the three hooks; the previous hooks stay chained. + + Also joins the interpreter-wide runtime registry (stored on the + unshadowable `sys` module) so stacked bundles can merge each other's + mappings when a traceback crosses bundle boundaries. + """ + registry = getattr(self._sys, "_cribo_sm_runtimes", None) + if not isinstance(registry, list): + registry = [] + self._sys._cribo_sm_runtimes = registry + registry.append(self) self._sys.excepthook = self.excepthook self._sys.unraisablehook = self.unraisablehook self._threading.excepthook = self.threading_hook @@ -187,14 +197,17 @@ def _bootstrap(cls, mode, bundle_file): def _map_location(self): """Resolve the map location per delivery mode, or None when inactive. - Returns (map_path, map_dir); map_path is None for inline mode (the map - lives inside the bundle file itself). Called lazily at hook-fire time - so the happy path never touches the environment or the filesystem. + Returns (map_path, map_dir, verify); map_path is None for inline mode + (the map lives inside the bundle file itself) and `verify` says whether + the sibling map must match the digest recorded in the bundle. Called + lazily at hook-fire time so the happy path never touches the + environment or the filesystem. """ env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None - # An explicit path wins for every mode. This is also the only way to + # An explicit path wins for every mode (and skips digest verification — + # it is the user's explicit choice). This is also the only way to # supply a map to a bundle executed via `python -` (stdin), whose # source cannot be re-read at hook time. A relative override is # anchored to the startup working directory, immune to later chdir. @@ -202,20 +215,20 @@ def _map_location(self): path = env if not self._os.path.isabs(path): path = self._os.path.join(self._startup_cwd, path) - return (path, self._os.path.dirname(self._os.path.abspath(path))) + return (path, self._os.path.dirname(self._os.path.abspath(path)), False) bundle = self._bundle_anchor if self._mode == "inline": if bundle == "": return None # stdin cannot be re-opened; use CRIBO_SOURCE_MAPS= - return (None, self._os.path.dirname(bundle)) + return (None, self._os.path.dirname(bundle), False) sibling = bundle + ".map" if self._mode == "linked": - if self._os.path.exists(sibling) and self._map_matches_bundle(sibling): - return (sibling, self._os.path.dirname(sibling)) + if self._os.path.exists(sibling): + return (sibling, self._os.path.dirname(sibling), True) return None # external: opt in via CRIBO_SOURCE_MAPS=1 (a path was handled above) if env in ("1", "true", "yes", "on"): - return (sibling, self._os.path.dirname(sibling)) + return (sibling, self._os.path.dirname(sibling), True) return None def _file_chunks(self, path, *, _open=open): @@ -230,6 +243,20 @@ def _file_chunks(self, path, *, _open=open): finally: handle.close() + def _handle_chunks(self, handle): + """Yield fixed-size chunks from an already-open handle, from offset 0. + + Keeping one open handle across the verify/decode passes pins the inode: + a concurrent build renaming a new map into place cannot swap the bytes + between passes. + """ + handle.seek(0) + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + def _find_marker_tail(self, handle, marker, *, _len=len): """Backward-scan a file for the last occurrence of `marker`. @@ -284,48 +311,49 @@ def _bundle_expected_digest(self, *, _open=open, _len=len, _int=int): return None return digest.decode("ascii").lower() - def _map_matches_bundle(self, map_path, *, _bex=BaseException): + def _map_matches_bundle(self, handle, *, _bex=BaseException): """False only when the bundle records a digest and the map disagrees. - This is what makes linked-mode publication safe against interleaved + This is what makes sibling-map publication safe against interleaved concurrent builds and manual file shuffling: the digest travels inside the bundle, which is always internally consistent, so a sibling map - from a different build is detected and ignored. Verification errors - fail open (the subsequent map read would surface them anyway). + from a different build is detected and ignored. Hashing reads from the + same open handle later used for decoding, so the verified bytes are the + decoded bytes. Verification errors fail open (the subsequent map read + would surface them anyway). """ try: expected = self._bundle_expected_digest() if expected is None: return True hasher = self._hashlib.sha256() - for chunk in self._file_chunks(map_path): + for chunk in self._handle_chunks(handle): hasher.update(chunk) return hasher.hexdigest() == expected except _bex: return True - def _inline_chunks(self, path, *, _open=open, _len=len): - """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = _open(path, "rb") - try: - start = self._find_inline_payload(handle) - if start < 0: - return - handle.seek(start) - pending = b"" - while True: - raw = handle.read(self._CHUNK) - if not raw: - break - data = pending + raw.translate(None, b"\r\n") - usable = _len(data) - (_len(data) % 4) - pending = data[usable:] - if usable: - yield self._binascii.a2b_base64(data[:usable]) - if pending: - yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) - finally: - handle.close() + def _inline_chunks(self, handle, *, _len=len): + """Yield decoded chunks of an inline (base64 data URL) source map. + + Reads from an already-open bundle handle so all passes see one inode. + """ + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = _len(data) - (_len(data) % 4) + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) # -- streaming JSON field scanner --------------------------------------- @@ -575,30 +603,39 @@ def _read_wanted_array_items(self, stream, wanted, *, _len=len, _error=ValueErro # -- loading ------------------------------------------------------------- - def _load(self, needed_lines, *, _set=set, _max=max): + def _load(self, needed_lines, *, _open=open, _set=set, _max=max): """Load (table, sources, map_dir) for 1-based bundle line numbers. Returns None when the runtime is inactive for the current mode. The returned table is keyed by 1-based bundle lines mapping to - (source_index, 1-based original line). + (source_index, 1-based original line). The map is opened exactly once: + digest verification and both decode passes read from the same pinned + handle, so the verified bytes are the decoded bytes even if a + concurrent build renames a new map into place mid-decode. """ location = self._map_location() if location is None: return None - map_path, map_dir = location + map_path, map_dir, verify = location needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) - if map_path is None: + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is not None and verify and not self._map_matches_bundle(handle): + return None + if map_path is None: - def chunks_factory(): - return self._inline_chunks(self._bundle_anchor) + def chunks_factory(): + return self._inline_chunks(handle) - else: + else: - def chunks_factory(): - return self._file_chunks(map_path) + def chunks_factory(): + return self._handle_chunks(handle) - sources, table0 = self._scan(chunks_factory, needed0, max_needed) + sources, table0 = self._scan(chunks_factory, needed0, max_needed) + finally: + handle.close() table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = (src_idx, src_line0 + 1) @@ -611,17 +648,19 @@ def _load_json_fallback( location = self._map_location() if location is None: return None - map_path, map_dir = location + map_path, map_dir, verify = location json = self._import("json") - if map_path is None: - raw = b"".join(self._inline_chunks(self._bundle_anchor)) - else: - handle = _open(map_path, "rb") - try: - raw = handle.read() - finally: - handle.close() + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is not None and verify and not self._map_matches_bundle(handle): + return None + if map_path is None: + raw = b"".join(self._inline_chunks(handle)) + else: + raw = b"".join(self._handle_chunks(handle)) + finally: + handle.close() data = json.loads(raw.decode("utf-8")) sources = {} for index, source in _enumerate(data.get("sources") or []): @@ -638,14 +677,14 @@ def _load_json_fallback( # -- traceback collection and rendering ---------------------------------- def _collect_needed( - self, exc_value, traceback_obj, *, _set=set, _id=id, _getattr=getattr + self, exc_value, traceback_obj, bundle_file, *, _set=set, _id=id, _getattr=getattr ): - """1-based bundle lines referenced by the traceback (and its chain).""" + """1-based lines of `bundle_file` referenced by the traceback chain.""" needed = _set() def add(tb): while tb is not None: - if tb.tb_frame.f_code.co_filename == self._bundle: + if tb.tb_frame.f_code.co_filename == bundle_file: needed.add(tb.tb_lineno) tb = tb.tb_next @@ -713,15 +752,20 @@ def _effective_tb_limit(self, *, _getattr=getattr, _isinstance=isinstance, _int= limit = _getattr(self._sys, "tracebacklimit", None) return limit if _isinstance(limit, _int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): + def _write_frames(self, traceback_obj, maps_by_file, write, limit, summaries, *, _bex=BaseException, _len=len): """Write remapped frame lines, collapsing repeated frames like CPython. - Consecutive identical frames (recursion) print at most 3 times followed - by a "[Previous line repeated N more times]" marker; source line text - is cached per (file, line) within one rendering to avoid re-reading - files. A positive `limit` keeps only the last `limit` frames, matching - the interpreter's `sys.tracebacklimit` handling in the default hook. + `maps_by_file` holds one (table, sources, map_dir) triple per known + bundle file, so a traceback crossing several stacked cribo bundles + remaps every frame. Frames that stay unmapped render through the + standard traceback machinery (`summaries`, when available) to preserve + PEP 657 position indicators. Consecutive identical frames (recursion) + print at most 3 times followed by a "[Previous line repeated N more + times]" marker; source line text is cached per (file, line) within one + rendering. A positive `limit` keeps only the last `limit` frames, + matching the interpreter's `sys.tracebacklimit` handling. """ + index = 0 if limit is not None: total = 0 probe = traceback_obj @@ -732,12 +776,26 @@ def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): while skip > 0 and traceback_obj is not None: traceback_obj = traceback_obj.tb_next skip -= 1 + index += 1 cache = {} last = None repeats = 0 - def emit(entry): + def emit(entry, frame_index, mapped_frame): + if not mapped_frame and summaries is not None and frame_index < _len(summaries): + # Standard rendering for untouched frames keeps PEP 657 + # carets and anchors intact. + try: + rendered = self._traceback.StackSummary.from_list( + [summaries[frame_index]] + ).format() + except _bex: + rendered = None + if rendered: + for line in rendered: + write(line) + return write(' File "%s", line %d, in %s\n' % entry) key = (entry[0], entry[1]) if key not in cache: @@ -750,7 +808,10 @@ def emit(entry): filename = frame.f_code.co_filename lineno = traceback_obj.tb_lineno name = frame.f_code.co_name - if filename == self._bundle: + mapped_frame = False + bundle_map = maps_by_file.get(filename) + if bundle_map is not None: + table, sources, map_dir = bundle_map mapped = table.get(lineno) if mapped is not None: source = sources.get(mapped[0]) @@ -760,18 +821,20 @@ def emit(entry): self._os.path.join(map_dir, source) ) filename, lineno = source, mapped[1] + mapped_frame = True entry = (filename, lineno, name) if entry == last: repeats += 1 if repeats <= 3: - emit(entry) + emit(entry, index, mapped_frame) else: if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) last = entry repeats = 1 - emit(entry) + emit(entry, index, mapped_frame) traceback_obj = traceback_obj.tb_next + index += 1 if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) @@ -825,9 +888,7 @@ def _write_exception_only( def _render( self, exc_value, - table, - sources, - map_dir, + maps_by_file, write, *, _getattr=getattr, @@ -836,6 +897,8 @@ def _render( _list=list, _reversed=reversed, _enumerate=enumerate, + _type=type, + _bex=BaseException, ): """Render the exception (with its cause/context chain) like CPython.""" chain = [] @@ -874,8 +937,16 @@ def _render( tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): + # Standard per-frame summaries (with PEP 657 positions on + # 3.11+) render the frames this runtime does not remap. + try: + summaries = self._traceback.TracebackException( + _type(exc), exc, tb, lookup_lines=False + ).stack + except _bex: + summaries = None write("Traceback (most recent call last):\n") - self._write_frames(tb, table, sources, map_dir, write, limit) + self._write_frames(tb, maps_by_file, write, limit, summaries) self._write_exception_only(exc, write) def _try_render( @@ -894,21 +965,30 @@ def _try_render( try: if self._chain_has_group(exc_value): return False - needed = self._collect_needed(exc_value, traceback_obj) - if not needed: - return False - loaded = None - try: - loaded = self._load(needed) - except _bex: + # Every stacked cribo runtime contributes mappings for its own + # bundle, so a traceback crossing several bundles remaps fully. + maps_by_file = {} + for runtime in self._registered_runtimes(): try: - loaded = self._load_json_fallback(needed) - except _bex: + bundle_file = runtime._bundle + if bundle_file in maps_by_file: + continue + needed = self._collect_needed(exc_value, traceback_obj, bundle_file) + if not needed: + continue loaded = None - if not loaded: - return False - table, sources, map_dir = loaded - if not table: + try: + loaded = runtime._load(needed) + except _bex: + try: + loaded = runtime._load_json_fallback(needed) + except _bex: + loaded = None + if loaded and loaded[0]: + maps_by_file[bundle_file] = loaded + except _bex: + continue + if not maps_by_file: return False try: old_limit = self._sys.getrecursionlimit() @@ -920,7 +1000,7 @@ def _try_render( parts = [] if prefix: parts.append(prefix) - self._render(exc_value, table, sources, map_dir, parts.append) + self._render(exc_value, maps_by_file, parts.append) stderr = self._sys.stderr stderr.write("".join(parts)) try: @@ -938,6 +1018,22 @@ def _try_render( pass self._local.in_hook = False + def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): + """All installed cribo runtimes in this interpreter, self included. + + The registry lives on the (unshadowable) `sys` module so stacked + bundles can find each other's mappings. + """ + registry = _getattr(self._sys, "_cribo_sm_runtimes", None) + runtimes = [] + if _isinstance(registry, _list): + for runtime in registry: + if _getattr(runtime, "_cribo_sm_runtime_marker", False): + runtimes.append(runtime) + if self not in runtimes: + runtimes.append(self) + return runtimes + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): """Invoke a chained hook after a successful remap when it is custom. diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index 0733c1f3e..fb010b315 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -312,11 +312,46 @@ impl ParallelWalker<'_> { .provenance .resolve(original.node_index().load(), original.range().start()) { + let generated_start = self.line_index.line_of(generated.range().start()); self.records.push(MappingRecord { - generated_line: self.line_index.line_of(generated.range().start()), + generated_line: generated_start, module_ordinal, original_line, }); + // Simple (body-less) statements can still span several physical + // lines — ruff's generator emits one line per statement except for + // multiline string/f-string literals, whose content is preserved + // verbatim. A raising expression inside such a literal is + // attributed to its own physical line, so fill the span with + // offset-aligned mappings. Compound statements are excluded: their + // interiors get precise mappings from the recursion below, which + // must not be shadowed (first mapping per line wins). + let has_nested_body = matches!( + generated, + Stmt::FunctionDef(_) + | Stmt::ClassDef(_) + | Stmt::If(_) + | Stmt::While(_) + | Stmt::For(_) + | Stmt::With(_) + | Stmt::Try(_) + | Stmt::Match(_) + ); + if !has_nested_body { + let generated_span = + self.line_index.line_of(generated.range().end()) - generated_start; + let original_span = self + .provenance + .resolve(original.node_index().load(), original.range().end()) + .map_or(0, |(_, end_line)| end_line.saturating_sub(original_line)); + for offset in 1..=generated_span.min(original_span) { + self.records.push(MappingRecord { + generated_line: generated_start + offset, + module_ordinal, + original_line: original_line + offset, + }); + } + } } // Evaluating a decorator can raise on its own `@...` line; give every @@ -543,7 +578,17 @@ fn relative_path(base: &std::path::Path, target: &std::path::Path) -> std::path: pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { let mut encoded = String::with_capacity(map_file_name.len()); for character in map_file_name.chars() { - if character < '\u{20}' || character == '\u{7F}' || character == '%' { + // Control characters would break out of the comment; '#', '?', and + // other URL delimiters would truncate the reference for + // standards-compliant URL consumers. Non-ASCII Unicode passes through + // verbatim. + if character < '\u{21}' + || character == '\u{7F}' + || matches!( + character, + '%' | '#' | '?' | '"' | '<' | '>' | '\\' | '^' | '`' | '|' + ) + { let _ = std::fmt::Write::write_fmt(&mut encoded, format_args!("%{:02X}", character as u32)); } else { @@ -553,12 +598,14 @@ pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { format!("# sourceMappingURL={encoded}\n") } -/// Comment recording the SHA-256 of the linked map at build time. +/// Comment recording the SHA-256 of the sibling map at build time. /// /// The runtime refuses a sibling map whose digest does not match, so no /// interleaving of concurrent builds (or manual file shuffling) can pair a /// bundle with another build's mappings — the digest travels inside the bundle -/// itself, which is always internally consistent. +/// itself, which is always internally consistent. Emitted for both linked and +/// external modes (the sibling-map pairing problem is identical); an explicit +/// `CRIBO_SOURCE_MAPS=` override skips verification. pub(crate) fn linked_map_digest_comment(map_json: &str) -> String { use sha2::{Digest as _, Sha256}; let digest = Sha256::digest(map_json.as_bytes()); diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index fff65a24f..fb72331f1 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -166,10 +166,12 @@ def test_inline_payload_scan_and_chunked_base64(rt): json_text = '{"filler":"' + filler + '","sources":["a.py"],"mappings":"AAAA"}' path = make_inline_bundle(json_text) try: - decoded = b"".join(rt._inline_chunks(path)) + with open(path, "rb") as handle: + decoded = b"".join(rt._inline_chunks(handle)) assert decoded.decode("utf-8") == json_text - # And end-to-end through the scanner: - sources, table = rt._scan(lambda: rt._inline_chunks(path), {0}, 0) + # And end-to-end through the scanner (one pinned handle, two passes): + with open(path, "rb") as handle: + sources, table = rt._scan(lambda: rt._inline_chunks(handle), {0}, 0) assert sources == {0: "a.py"}, sources assert table == {0: (0, 0)}, table finally: @@ -183,7 +185,8 @@ def test_inline_scan_without_marker_yields_nothing(rt): with handle as f: f.write("print('no map here')\n" * 50) try: - assert b"".join(rt._inline_chunks(handle.name)) == b"" + with open(handle.name, "rb") as bundle: + assert b"".join(rt._inline_chunks(bundle)) == b"" finally: os.unlink(handle.name) diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index cab36ce94..6f6a40eb2 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -89,7 +89,12 @@ class _CriboSourceMapRuntime(object): self._group_type = None def install(self): - """Install the three hooks; the previous hooks stay chained.""" + """Install the three hooks; the previous hooks stay chained.\n\n Also joins the interpreter-wide runtime registry (stored on the\n unshadowable `sys` module) so stacked bundles can merge each other's\n mappings when a traceback crosses bundle boundaries.\n """ + registry = getattr(self._sys, "_cribo_sm_runtimes", None) + if not isinstance(registry, list): + registry = [] + self._sys._cribo_sm_runtimes = registry + registry.append(self) self._sys.excepthook = self.excepthook self._sys.unraisablehook = self.unraisablehook self._threading.excepthook = self.threading_hook @@ -104,7 +109,7 @@ class _CriboSourceMapRuntime(object): pass def _map_location(self): - """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time\n so the happy path never touches the environment or the filesystem.\n """ + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir, verify); map_path is None for inline mode\n (the map lives inside the bundle file itself) and `verify` says whether\n the sibling map must match the digest recorded in the bundle. Called\n lazily at hook-fire time so the happy path never touches the\n environment or the filesystem.\n """ env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None @@ -112,19 +117,19 @@ class _CriboSourceMapRuntime(object): path = env if not self._os.path.isabs(path): path = self._os.path.join(self._startup_cwd, path) - return path, self._os.path.dirname(self._os.path.abspath(path)) + return path, self._os.path.dirname(self._os.path.abspath(path)), False bundle = self._bundle_anchor if self._mode == "inline": if bundle == "": return None - return None, self._os.path.dirname(bundle) + return None, self._os.path.dirname(bundle), False sibling = bundle + ".map" if self._mode == "linked": - if self._os.path.exists(sibling) and self._map_matches_bundle(sibling): - return sibling, self._os.path.dirname(sibling) + if self._os.path.exists(sibling): + return sibling, self._os.path.dirname(sibling), True return None if env in ("1", "true", "yes", "on"): - return sibling, self._os.path.dirname(sibling) + return sibling, self._os.path.dirname(sibling), True return None def _file_chunks(self, path, *, _open=open): @@ -139,6 +144,15 @@ class _CriboSourceMapRuntime(object): finally: handle.close() + def _handle_chunks(self, handle): + """Yield fixed-size chunks from an already-open handle, from offset 0.\n\n Keeping one open handle across the verify/decode passes pins the inode:\n a concurrent build renaming a new map into place cannot swap the bytes\n between passes.\n """ + handle.seek(0) + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + def _find_marker_tail(self, handle, marker, *, _len=len): """Backward-scan a file for the last occurrence of `marker`.\n\n Returns the offset of the payload following the marker, or -1. Only the\n tail of the file is examined; the body is never read.\n """ handle.seek(0, 2) @@ -189,41 +203,37 @@ class _CriboSourceMapRuntime(object): return None return digest.decode("ascii").lower() - def _map_matches_bundle(self, map_path, *, _bex=BaseException): - """False only when the bundle records a digest and the map disagrees.\n\n This is what makes linked-mode publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Verification errors\n fail open (the subsequent map read would surface them anyway).\n """ + def _map_matches_bundle(self, handle, *, _bex=BaseException): + """False only when the bundle records a digest and the map disagrees.\n\n This is what makes sibling-map publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Hashing reads from the\n same open handle later used for decoding, so the verified bytes are the\n decoded bytes. Verification errors fail open (the subsequent map read\n would surface them anyway).\n """ try: expected = self._bundle_expected_digest() if expected is None: return True hasher = self._hashlib.sha256() - for chunk in self._file_chunks(map_path): + for chunk in self._handle_chunks(handle): hasher.update(chunk) return hasher.hexdigest() == expected except _bex: return True - def _inline_chunks(self, path, *, _open=open, _len=len): - """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = _open(path, "rb") - try: - start = self._find_inline_payload(handle) - if start < 0: - return - handle.seek(start) - pending = b"" - while True: - raw = handle.read(self._CHUNK) - if not raw: - break - data = pending + raw.translate(None, b"\r\n") - usable = _len(data) - _len(data) % 4 - pending = data[usable:] - if usable: - yield self._binascii.a2b_base64(data[:usable]) - if pending: - yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) - finally: - handle.close() + def _inline_chunks(self, handle, *, _len=len): + """Yield decoded chunks of an inline (base64 data URL) source map.\n\n Reads from an already-open bundle handle so all passes see one inode.\n """ + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = _len(data) - _len(data) % 4 + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) def _skip_ws(self, stream, byte): while byte in (32, 9, 10, 13): @@ -433,23 +443,29 @@ class _CriboSourceMapRuntime(object): raise _error("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _load(self, needed_lines, *, _set=set, _max=max): - """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ + def _load(self, needed_lines, *, _open=open, _set=set, _max=max): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line). The map is opened exactly once:\n digest verification and both decode passes read from the same pinned\n handle, so the verified bytes are the decoded bytes even if a\n concurrent build renames a new map into place mid-decode.\n """ location = self._map_location() if location is None: return None - map_path, map_dir = location + map_path, map_dir, verify = location needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) - if map_path is None: + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is not None and verify and not self._map_matches_bundle(handle): + return None + if map_path is None: - def chunks_factory(): - return self._inline_chunks(self._bundle_anchor) - else: + def chunks_factory(): + return self._inline_chunks(handle) + else: - def chunks_factory(): - return self._file_chunks(map_path) - sources, table0 = self._scan(chunks_factory, needed0, max_needed) + def chunks_factory(): + return self._handle_chunks(handle) + sources, table0 = self._scan(chunks_factory, needed0, max_needed) + finally: + handle.close() table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 @@ -460,16 +476,18 @@ class _CriboSourceMapRuntime(object): location = self._map_location() if location is None: return None - map_path, map_dir = location + map_path, map_dir, verify = location json = self._import("json") - if map_path is None: - raw = b"".join(self._inline_chunks(self._bundle_anchor)) - else: - handle = _open(map_path, "rb") - try: - raw = handle.read() - finally: - handle.close() + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is not None and verify and not self._map_matches_bundle(handle): + return None + if map_path is None: + raw = b"".join(self._inline_chunks(handle)) + else: + raw = b"".join(self._handle_chunks(handle)) + finally: + handle.close() data = json.loads(raw.decode("utf-8")) sources = {} for index, source in _enumerate(data.get("sources") or []): @@ -483,13 +501,13 @@ class _CriboSourceMapRuntime(object): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _collect_needed(self, exc_value, traceback_obj, *, _set=set, _id=id, _getattr=getattr): - """1-based bundle lines referenced by the traceback (and its chain).""" + def _collect_needed(self, exc_value, traceback_obj, bundle_file, *, _set=set, _id=id, _getattr=getattr): + """1-based lines of `bundle_file` referenced by the traceback chain.""" needed = _set() def add(tb): while tb is not None: - if tb.tb_frame.f_code.co_filename == self._bundle: + if tb.tb_frame.f_code.co_filename == bundle_file: needed.add(tb.tb_lineno) tb = tb.tb_next add(traceback_obj) @@ -547,8 +565,9 @@ class _CriboSourceMapRuntime(object): limit = _getattr(self._sys, "tracebacklimit", None) return limit if _isinstance(limit, _int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): - """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ + def _write_frames(self, traceback_obj, maps_by_file, write, limit, summaries, *, _bex=BaseException, _len=len): + """Write remapped frame lines, collapsing repeated frames like CPython.\n\n `maps_by_file` holds one (table, sources, map_dir) triple per known\n bundle file, so a traceback crossing several stacked cribo bundles\n remaps every frame. Frames that stay unmapped render through the\n standard traceback machinery (`summaries`, when available) to preserve\n PEP 657 position indicators. Consecutive identical frames (recursion)\n print at most 3 times followed by a \"[Previous line repeated N more\n times]\" marker; source line text is cached per (file, line) within one\n rendering. A positive `limit` keeps only the last `limit` frames,\n matching the interpreter's `sys.tracebacklimit` handling.\n """ + index = 0 if limit is not None: total = 0 probe = traceback_obj @@ -559,11 +578,21 @@ class _CriboSourceMapRuntime(object): while skip > 0 and traceback_obj is not None: traceback_obj = traceback_obj.tb_next skip -= 1 + index += 1 cache = {} last = None repeats = 0 - def emit(entry): + def emit(entry, frame_index, mapped_frame): + if not mapped_frame and summaries is not None and frame_index < _len(summaries): + try: + rendered = self._traceback.StackSummary.from_list([summaries[frame_index]]).format() + except _bex: + rendered = None + if rendered: + for line in rendered: + write(line) + return write(' File "%s", line %d, in %s\n' % entry) key = entry[0], entry[1] if key not in cache: @@ -575,7 +604,10 @@ class _CriboSourceMapRuntime(object): filename = frame.f_code.co_filename lineno = traceback_obj.tb_lineno name = frame.f_code.co_name - if filename == self._bundle: + mapped_frame = False + bundle_map = maps_by_file.get(filename) + if bundle_map is not None: + table, sources, map_dir = bundle_map mapped = table.get(lineno) if mapped is not None: source = sources.get(mapped[0]) @@ -583,18 +615,20 @@ class _CriboSourceMapRuntime(object): if not self._os.path.isabs(source): source = self._os.path.normpath(self._os.path.join(map_dir, source)) filename, lineno = source, mapped[1] + mapped_frame = True entry = filename, lineno, name if entry == last: repeats += 1 if repeats <= 3: - emit(entry) + emit(entry, index, mapped_frame) else: if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) last = entry repeats = 1 - emit(entry) + emit(entry, index, mapped_frame) traceback_obj = traceback_obj.tb_next + index += 1 if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) @@ -629,7 +663,7 @@ class _CriboSourceMapRuntime(object): except _bex: pass - def _render(self, exc_value, table, sources, map_dir, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate): + def _render(self, exc_value, maps_by_file, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate, _type=type, _bex=BaseException): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value @@ -658,8 +692,12 @@ class _CriboSourceMapRuntime(object): tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): + try: + summaries = self._traceback.TracebackException(_type(exc), exc, tb, lookup_lines=False).stack + except _bex: + summaries = None write("Traceback (most recent call last):\n") - self._write_frames(tb, table, sources, map_dir, write, limit) + self._write_frames(tb, maps_by_file, write, limit, summaries) self._write_exception_only(exc, write) def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException): @@ -671,21 +709,28 @@ class _CriboSourceMapRuntime(object): try: if self._chain_has_group(exc_value): return False - needed = self._collect_needed(exc_value, traceback_obj) - if not needed: - return False - loaded = None - try: - loaded = self._load(needed) - except _bex: + maps_by_file = {} + for runtime in self._registered_runtimes(): try: - loaded = self._load_json_fallback(needed) - except _bex: + bundle_file = runtime._bundle + if bundle_file in maps_by_file: + continue + needed = self._collect_needed(exc_value, traceback_obj, bundle_file) + if not needed: + continue loaded = None - if not loaded: - return False - table, sources, map_dir = loaded - if not table: + try: + loaded = runtime._load(needed) + except _bex: + try: + loaded = runtime._load_json_fallback(needed) + except _bex: + loaded = None + if loaded and loaded[0]: + maps_by_file[bundle_file] = loaded + except _bex: + continue + if not maps_by_file: return False try: old_limit = self._sys.getrecursionlimit() @@ -695,7 +740,7 @@ class _CriboSourceMapRuntime(object): parts = [] if prefix: parts.append(prefix) - self._render(exc_value, table, sources, map_dir, parts.append) + self._render(exc_value, maps_by_file, parts.append) stderr = self._sys.stderr stderr.write("".join(parts)) try: @@ -713,6 +758,18 @@ class _CriboSourceMapRuntime(object): pass self._local.in_hook = False + def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): + """All installed cribo runtimes in this interpreter, self included.\n\n The registry lives on the (unshadowable) `sys` module so stacked\n bundles can find each other's mappings.\n """ + registry = _getattr(self._sys, "_cribo_sm_runtimes", None) + runtimes = [] + if _isinstance(registry, _list): + for runtime in registry: + if _getattr(runtime, "_cribo_sm_runtime_marker", False): + runtimes.append(runtime) + if self not in runtimes: + runtimes.append(self) + return runtimes + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Two exclusions: when\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one; and an earlier cribo runtime's hook is skipped, since it would\n find no frames for its own bundle and delegate to the default printer,\n duplicating the traceback.\n """ if default is None or prev is None or prev is default: @@ -901,5 +958,5 @@ total = add(2, 3) product = multiply(total, 4) print(describe("total", total)) print(describe("product", product)) -# cribo-sourcemap-sha256=c0fd18c37fc76a6f88625c54619e79a1f7d76a09a174871699fb835cfedaeb35 +# cribo-sourcemap-sha256=b6b82631d8f5e7c89c6803f2cd7c21b12df55242fd1aa31984d5236dcba71d4a # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 7affa0c51..8d6f69098 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -89,7 +89,12 @@ class _CriboSourceMapRuntime(object): self._group_type = None def install(self): - """Install the three hooks; the previous hooks stay chained.""" + """Install the three hooks; the previous hooks stay chained.\n\n Also joins the interpreter-wide runtime registry (stored on the\n unshadowable `sys` module) so stacked bundles can merge each other's\n mappings when a traceback crosses bundle boundaries.\n """ + registry = getattr(self._sys, "_cribo_sm_runtimes", None) + if not isinstance(registry, list): + registry = [] + self._sys._cribo_sm_runtimes = registry + registry.append(self) self._sys.excepthook = self.excepthook self._sys.unraisablehook = self.unraisablehook self._threading.excepthook = self.threading_hook @@ -104,7 +109,7 @@ class _CriboSourceMapRuntime(object): pass def _map_location(self): - """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir); map_path is None for inline mode (the map\n lives inside the bundle file itself). Called lazily at hook-fire time\n so the happy path never touches the environment or the filesystem.\n """ + """Resolve the map location per delivery mode, or None when inactive.\n\n Returns (map_path, map_dir, verify); map_path is None for inline mode\n (the map lives inside the bundle file itself) and `verify` says whether\n the sibling map must match the digest recorded in the bundle. Called\n lazily at hook-fire time so the happy path never touches the\n environment or the filesystem.\n """ env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None @@ -112,19 +117,19 @@ class _CriboSourceMapRuntime(object): path = env if not self._os.path.isabs(path): path = self._os.path.join(self._startup_cwd, path) - return path, self._os.path.dirname(self._os.path.abspath(path)) + return path, self._os.path.dirname(self._os.path.abspath(path)), False bundle = self._bundle_anchor if self._mode == "inline": if bundle == "": return None - return None, self._os.path.dirname(bundle) + return None, self._os.path.dirname(bundle), False sibling = bundle + ".map" if self._mode == "linked": - if self._os.path.exists(sibling) and self._map_matches_bundle(sibling): - return sibling, self._os.path.dirname(sibling) + if self._os.path.exists(sibling): + return sibling, self._os.path.dirname(sibling), True return None if env in ("1", "true", "yes", "on"): - return sibling, self._os.path.dirname(sibling) + return sibling, self._os.path.dirname(sibling), True return None def _file_chunks(self, path, *, _open=open): @@ -139,6 +144,15 @@ class _CriboSourceMapRuntime(object): finally: handle.close() + def _handle_chunks(self, handle): + """Yield fixed-size chunks from an already-open handle, from offset 0.\n\n Keeping one open handle across the verify/decode passes pins the inode:\n a concurrent build renaming a new map into place cannot swap the bytes\n between passes.\n """ + handle.seek(0) + while True: + chunk = handle.read(self._CHUNK) + if not chunk: + break + yield chunk + def _find_marker_tail(self, handle, marker, *, _len=len): """Backward-scan a file for the last occurrence of `marker`.\n\n Returns the offset of the payload following the marker, or -1. Only the\n tail of the file is examined; the body is never read.\n """ handle.seek(0, 2) @@ -189,41 +203,37 @@ class _CriboSourceMapRuntime(object): return None return digest.decode("ascii").lower() - def _map_matches_bundle(self, map_path, *, _bex=BaseException): - """False only when the bundle records a digest and the map disagrees.\n\n This is what makes linked-mode publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Verification errors\n fail open (the subsequent map read would surface them anyway).\n """ + def _map_matches_bundle(self, handle, *, _bex=BaseException): + """False only when the bundle records a digest and the map disagrees.\n\n This is what makes sibling-map publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Hashing reads from the\n same open handle later used for decoding, so the verified bytes are the\n decoded bytes. Verification errors fail open (the subsequent map read\n would surface them anyway).\n """ try: expected = self._bundle_expected_digest() if expected is None: return True hasher = self._hashlib.sha256() - for chunk in self._file_chunks(map_path): + for chunk in self._handle_chunks(handle): hasher.update(chunk) return hasher.hexdigest() == expected except _bex: return True - def _inline_chunks(self, path, *, _open=open, _len=len): - """Yield decoded chunks of an inline (base64 data URL) source map.""" - handle = _open(path, "rb") - try: - start = self._find_inline_payload(handle) - if start < 0: - return - handle.seek(start) - pending = b"" - while True: - raw = handle.read(self._CHUNK) - if not raw: - break - data = pending + raw.translate(None, b"\r\n") - usable = _len(data) - _len(data) % 4 - pending = data[usable:] - if usable: - yield self._binascii.a2b_base64(data[:usable]) - if pending: - yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) - finally: - handle.close() + def _inline_chunks(self, handle, *, _len=len): + """Yield decoded chunks of an inline (base64 data URL) source map.\n\n Reads from an already-open bundle handle so all passes see one inode.\n """ + start = self._find_inline_payload(handle) + if start < 0: + return + handle.seek(start) + pending = b"" + while True: + raw = handle.read(self._CHUNK) + if not raw: + break + data = pending + raw.translate(None, b"\r\n") + usable = _len(data) - _len(data) % 4 + pending = data[usable:] + if usable: + yield self._binascii.a2b_base64(data[:usable]) + if pending: + yield self._binascii.a2b_base64(pending + b"=" * (-_len(pending) % 4)) def _skip_ws(self, stream, byte): while byte in (32, 9, 10, 13): @@ -433,23 +443,29 @@ class _CriboSourceMapRuntime(object): raise _error("malformed array") byte = self._skip_ws(stream, stream.read_byte()) - def _load(self, needed_lines, *, _set=set, _max=max): - """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line).\n """ + def _load(self, needed_lines, *, _open=open, _set=set, _max=max): + """Load (table, sources, map_dir) for 1-based bundle line numbers.\n\n Returns None when the runtime is inactive for the current mode. The\n returned table is keyed by 1-based bundle lines mapping to\n (source_index, 1-based original line). The map is opened exactly once:\n digest verification and both decode passes read from the same pinned\n handle, so the verified bytes are the decoded bytes even if a\n concurrent build renames a new map into place mid-decode.\n """ location = self._map_location() if location is None: return None - map_path, map_dir = location + map_path, map_dir, verify = location needed0 = _set(line - 1 for line in needed_lines) max_needed = _max(needed0) - if map_path is None: + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is not None and verify and not self._map_matches_bundle(handle): + return None + if map_path is None: - def chunks_factory(): - return self._inline_chunks(self._bundle_anchor) - else: + def chunks_factory(): + return self._inline_chunks(handle) + else: - def chunks_factory(): - return self._file_chunks(map_path) - sources, table0 = self._scan(chunks_factory, needed0, max_needed) + def chunks_factory(): + return self._handle_chunks(handle) + sources, table0 = self._scan(chunks_factory, needed0, max_needed) + finally: + handle.close() table = {} for line0, (src_idx, src_line0) in table0.items(): table[line0 + 1] = src_idx, src_line0 + 1 @@ -460,16 +476,18 @@ class _CriboSourceMapRuntime(object): location = self._map_location() if location is None: return None - map_path, map_dir = location + map_path, map_dir, verify = location json = self._import("json") - if map_path is None: - raw = b"".join(self._inline_chunks(self._bundle_anchor)) - else: - handle = _open(map_path, "rb") - try: - raw = handle.read() - finally: - handle.close() + handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") + try: + if map_path is not None and verify and not self._map_matches_bundle(handle): + return None + if map_path is None: + raw = b"".join(self._inline_chunks(handle)) + else: + raw = b"".join(self._handle_chunks(handle)) + finally: + handle.close() data = json.loads(raw.decode("utf-8")) sources = {} for index, source in _enumerate(data.get("sources") or []): @@ -483,13 +501,13 @@ class _CriboSourceMapRuntime(object): table[line0 + 1] = src_idx, src_line0 + 1 return table, sources, map_dir - def _collect_needed(self, exc_value, traceback_obj, *, _set=set, _id=id, _getattr=getattr): - """1-based bundle lines referenced by the traceback (and its chain).""" + def _collect_needed(self, exc_value, traceback_obj, bundle_file, *, _set=set, _id=id, _getattr=getattr): + """1-based lines of `bundle_file` referenced by the traceback chain.""" needed = _set() def add(tb): while tb is not None: - if tb.tb_frame.f_code.co_filename == self._bundle: + if tb.tb_frame.f_code.co_filename == bundle_file: needed.add(tb.tb_lineno) tb = tb.tb_next add(traceback_obj) @@ -547,8 +565,9 @@ class _CriboSourceMapRuntime(object): limit = _getattr(self._sys, "tracebacklimit", None) return limit if _isinstance(limit, _int) else None - def _write_frames(self, traceback_obj, table, sources, map_dir, write, limit): - """Write remapped frame lines, collapsing repeated frames like CPython.\n\n Consecutive identical frames (recursion) print at most 3 times followed\n by a \"[Previous line repeated N more times]\" marker; source line text\n is cached per (file, line) within one rendering to avoid re-reading\n files. A positive `limit` keeps only the last `limit` frames, matching\n the interpreter's `sys.tracebacklimit` handling in the default hook.\n """ + def _write_frames(self, traceback_obj, maps_by_file, write, limit, summaries, *, _bex=BaseException, _len=len): + """Write remapped frame lines, collapsing repeated frames like CPython.\n\n `maps_by_file` holds one (table, sources, map_dir) triple per known\n bundle file, so a traceback crossing several stacked cribo bundles\n remaps every frame. Frames that stay unmapped render through the\n standard traceback machinery (`summaries`, when available) to preserve\n PEP 657 position indicators. Consecutive identical frames (recursion)\n print at most 3 times followed by a \"[Previous line repeated N more\n times]\" marker; source line text is cached per (file, line) within one\n rendering. A positive `limit` keeps only the last `limit` frames,\n matching the interpreter's `sys.tracebacklimit` handling.\n """ + index = 0 if limit is not None: total = 0 probe = traceback_obj @@ -559,11 +578,21 @@ class _CriboSourceMapRuntime(object): while skip > 0 and traceback_obj is not None: traceback_obj = traceback_obj.tb_next skip -= 1 + index += 1 cache = {} last = None repeats = 0 - def emit(entry): + def emit(entry, frame_index, mapped_frame): + if not mapped_frame and summaries is not None and frame_index < _len(summaries): + try: + rendered = self._traceback.StackSummary.from_list([summaries[frame_index]]).format() + except _bex: + rendered = None + if rendered: + for line in rendered: + write(line) + return write(' File "%s", line %d, in %s\n' % entry) key = entry[0], entry[1] if key not in cache: @@ -575,7 +604,10 @@ class _CriboSourceMapRuntime(object): filename = frame.f_code.co_filename lineno = traceback_obj.tb_lineno name = frame.f_code.co_name - if filename == self._bundle: + mapped_frame = False + bundle_map = maps_by_file.get(filename) + if bundle_map is not None: + table, sources, map_dir = bundle_map mapped = table.get(lineno) if mapped is not None: source = sources.get(mapped[0]) @@ -583,18 +615,20 @@ class _CriboSourceMapRuntime(object): if not self._os.path.isabs(source): source = self._os.path.normpath(self._os.path.join(map_dir, source)) filename, lineno = source, mapped[1] + mapped_frame = True entry = filename, lineno, name if entry == last: repeats += 1 if repeats <= 3: - emit(entry) + emit(entry, index, mapped_frame) else: if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) last = entry repeats = 1 - emit(entry) + emit(entry, index, mapped_frame) traceback_obj = traceback_obj.tb_next + index += 1 if repeats > 3: write(" [Previous line repeated %d more times]\n" % (repeats - 3)) @@ -629,7 +663,7 @@ class _CriboSourceMapRuntime(object): except _bex: pass - def _render(self, exc_value, table, sources, map_dir, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate): + def _render(self, exc_value, maps_by_file, write, *, _getattr=getattr, _set=set, _id=id, _list=list, _reversed=reversed, _enumerate=enumerate, _type=type, _bex=BaseException): """Render the exception (with its cause/context chain) like CPython.""" chain = [] exc = exc_value @@ -658,8 +692,12 @@ class _CriboSourceMapRuntime(object): tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): + try: + summaries = self._traceback.TracebackException(_type(exc), exc, tb, lookup_lines=False).stack + except _bex: + summaries = None write("Traceback (most recent call last):\n") - self._write_frames(tb, table, sources, map_dir, write, limit) + self._write_frames(tb, maps_by_file, write, limit, summaries) self._write_exception_only(exc, write) def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException): @@ -671,21 +709,28 @@ class _CriboSourceMapRuntime(object): try: if self._chain_has_group(exc_value): return False - needed = self._collect_needed(exc_value, traceback_obj) - if not needed: - return False - loaded = None - try: - loaded = self._load(needed) - except _bex: + maps_by_file = {} + for runtime in self._registered_runtimes(): try: - loaded = self._load_json_fallback(needed) - except _bex: + bundle_file = runtime._bundle + if bundle_file in maps_by_file: + continue + needed = self._collect_needed(exc_value, traceback_obj, bundle_file) + if not needed: + continue loaded = None - if not loaded: - return False - table, sources, map_dir = loaded - if not table: + try: + loaded = runtime._load(needed) + except _bex: + try: + loaded = runtime._load_json_fallback(needed) + except _bex: + loaded = None + if loaded and loaded[0]: + maps_by_file[bundle_file] = loaded + except _bex: + continue + if not maps_by_file: return False try: old_limit = self._sys.getrecursionlimit() @@ -695,7 +740,7 @@ class _CriboSourceMapRuntime(object): parts = [] if prefix: parts.append(prefix) - self._render(exc_value, table, sources, map_dir, parts.append) + self._render(exc_value, maps_by_file, parts.append) stderr = self._sys.stderr stderr.write("".join(parts)) try: @@ -713,6 +758,18 @@ class _CriboSourceMapRuntime(object): pass self._local.in_hook = False + def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): + """All installed cribo runtimes in this interpreter, self included.\n\n The registry lives on the (unshadowable) `sys` module so stacked\n bundles can find each other's mappings.\n """ + registry = _getattr(self._sys, "_cribo_sm_runtimes", None) + runtimes = [] + if _isinstance(registry, _list): + for runtime in registry: + if _getattr(runtime, "_cribo_sm_runtime_marker", False): + runtimes.append(runtime) + if self not in runtimes: + runtimes.append(self) + return runtimes + def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Two exclusions: when\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one; and an earlier cribo runtime's hook is skipped, since it would\n find no frames for its own bundle and delegate to the default printer,\n duplicating the traceback.\n """ if default is None or prev is None or prev is default: @@ -908,5 +965,5 @@ effects.__init__ = _cribo_init___cribo_503b17_effects effects = _cribo_init___cribo_503b17_effects(effects) print("counter:", effects.COUNTER) print("boosted:", effects.boost(10)) -# cribo-sourcemap-sha256=fd8814e47b6c032610e09cd0d1c0e24f31b29af75a32a0dadfecb6196527bb69 +# cribo-sourcemap-sha256=3a91bc5dcbd3f9b4d889f6232ecbabbf740aaa1ac46aba62277c03732bc91b69 # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index d3a7b36c4..c3d5d05c3 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:878 `def add(a, b):` -> calculator.py:1 -bundle:879 `result = a + b` -> calculator.py:2 -bundle:880 `return result` -> calculator.py:3 -bundle:882 `def multiply(a, b):` -> calculator.py:6 -bundle:883 `result = a * b` -> calculator.py:7 -bundle:884 `return result` -> calculator.py:8 -bundle:889 `def describe(name, value):` -> utils.py:1 -bundle:890 `return f"{name} = {value}"` -> utils.py:2 -bundle:896 `total = add(2, 3)` -> main.py:4 -bundle:897 `product = multiply(total, 4)` -> main.py:5 -bundle:898 `print(describe("total", total))` -> main.py:6 -bundle:899 `print(describe("product", product))` -> main.py:7 +bundle:935 `def add(a, b):` -> calculator.py:1 +bundle:936 `result = a + b` -> calculator.py:2 +bundle:937 `return result` -> calculator.py:3 +bundle:939 `def multiply(a, b):` -> calculator.py:6 +bundle:940 `result = a * b` -> calculator.py:7 +bundle:941 `return result` -> calculator.py:8 +bundle:946 `def describe(name, value):` -> utils.py:1 +bundle:947 `return f"{name} = {value}"` -> utils.py:2 +bundle:953 `total = add(2, 3)` -> main.py:4 +bundle:954 `product = multiply(total, 4)` -> main.py:5 +bundle:955 `print(describe("total", total))` -> main.py:6 +bundle:956 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 6bc707b0c..35a28b400 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:887 `print("effects module loading")` -> effects.py:1 -bundle:888 `COUNTER = 1` -> effects.py:3 -bundle:891 `def boost(value):` -> effects.py:6 -bundle:892 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:893 `return boosted` -> effects.py:8 -bundle:905 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:906 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:944 `print("effects module loading")` -> effects.py:1 +bundle:945 `COUNTER = 1` -> effects.py:3 +bundle:948 `def boost(value):` -> effects.py:6 +bundle:949 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:950 `return boosted` -> effects.py:8 +bundle:962 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:963 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 832837511..10193f827 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1346,3 +1346,110 @@ fn linked_comment_preserves_unicode_names() { assert!(ok, "bundle must run: {stderr}"); assert!(stdout.contains("hello world")); } + +#[test] +fn external_mode_rejects_mismatched_sibling_map() { + // External mode embeds the same digest as linked mode; CRIBO_SOURCE_MAPS=1 + // against a foreign sibling map must fall back to the standard traceback. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=external"); + + let other = make_project(&[ + ("main.py", "from helper import other\n\nprint(other())\n"), + ("helper.py", "def other():\n return 42\n"), + ]); + let other_bundle = other.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&other), + "--output", + &other_bundle.to_string_lossy(), + "--sourcemap=external", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + fs::copy( + other.path().join("bundle.py.map"), + dir.path().join("bundle.py.map"), + ) + .expect("swap in a foreign map"); + + let (ok, _, stderr) = run_python(&bundle, &[("CRIBO_SOURCE_MAPS", "1")]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn stacked_runtimes_merge_maps_across_bundles() { + // A traceback crossing two source-mapped bundles (the crashing bundle + // calls a function retained from an earlier one) must remap the frames of + // BOTH bundles; unmapped driver frames keep standard rendering. + let provider = make_project(&[ + ( + "main.py", + // The real builtins module (bypassing the bundler's stdlib proxy, + // whose speculative import_module would add chained-context noise). + "from helper import provider_boom\n\n__import__(\"builtins\").provider_boom = \ + provider_boom\nprint(\"provider ready\")\n", + ), + ( + "helper.py", + "def provider_boom():\n raise ValueError(\"cross-bundle kaboom\")\n", + ), + ]); + let provider_bundle = provider.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&provider), + "--output", + &provider_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let caller = make_project(&[ + ( + "main.py", + "print(\"calling provider\")\n__import__(\"builtins\").provider_boom()\n", + ), + ("helper.py", "unused = True\n"), + ]); + let caller_bundle = caller.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&caller), + "--output", + &caller_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + + let driver = caller.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({provider:?})\nrunpy.run_path({caller:?})\n", + provider = provider_bundle.to_string_lossy(), + caller = caller_bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let (ok, _, stderr) = run_python(&driver, &[]); + assert!(!ok); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "exactly one traceback expected: {stderr}" + ); + // The caller bundle's frame (installed second, renders) is remapped... + assert!( + stderr.contains("main.py\", line 2, in "), + "the caller frame must be remapped: {stderr}" + ); + // ...and so is the provider bundle's frame, via the runtime registry. + assert!( + stderr.contains("helper.py\", line 2, in provider_boom"), + "the earlier bundle's frame must be remapped through the shared registry: {stderr}" + ); + assert!(stderr.contains("cross-bundle kaboom")); +} From 0eb008160fed1e7e7058b0814c4c77c48d462134 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 21 Aug 2026 06:58:30 +0200 Subject: [PATCH 14/17] fix: address thirteenth review round on source map runtime - shadow-proof importer takes the bundle's own path so runpy execution (where sys.path[0] is the driver's directory) still filters equivalent entries; non-string sys.path entries are kept untouched - refuse maps when the on-disk bundle changed since startup (stat identity captured at bootstrap), so a redeployed bundle's map is never applied to old in-memory code - traverse stacked cribo hooks to the original preinstalled custom hook so it stays notified without duplicate default rendering - percent-encode URL delimiters in sources entries (decoded by the runtime before filesystem access) so '#'/'?' in paths survive URL interpretation - inherit staged-map permissions via symlink_metadata from regular files only, closing the symlink-pointed permissions leak - document that ruff's generator emits statements on single physical lines (multiline literals use \n escapes), so no interior-line mappings exist; removed the dead span filler and locked the behavior in with a test Addresses thirteenth-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 5 +- crates/cribo/src/python/sourcemap_runtime.py | 167 +++++++++++---- crates/cribo/src/source_map.rs | 71 +++---- .../bundled_code@sourcemap_basic.snap | 112 ++++++++-- .../bundled_code@sourcemap_wrapper.snap | 112 ++++++++-- .../snapshots/source_map@sourcemap_basic.snap | 24 +-- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 194 ++++++++++++++++++ 8 files changed, 555 insertions(+), 144 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index e8fe1826f..dbe6fed93 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -106,8 +106,11 @@ fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { .open(&tmp_path) { Ok(mut file) => { - let write_result = fs::metadata(map_path) + // symlink_metadata: an attacker-planted symlink at the map + // path must not decide the staged file's permissions. + let write_result = fs::symlink_metadata(map_path) .ok() + .filter(fs::Metadata::is_file) .map_or(Ok(()), |metadata| { file.set_permissions(metadata.permissions()) }) diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 6a03d5961..d0e58eb7d 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -14,43 +14,71 @@ import sys as _cribo_sys -def _cribo_sm_import(name, *, _list=list, _import=__import__): +def _cribo_sm_import( + name, + script_path=None, + *, + _list=list, + _import=__import__, + _isinstance=isinstance, + _str=str, + _bex=BaseException, +): """Import a stdlib module immune to script-directory shadowing. - The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and - `PYTHONPATH=.` can expose the same directory again at later indices — so a - project file named e.g. `threading.py` sitting next to the bundle would - otherwise shadow the stdlib for this runtime. Already-imported modules are - taken from `sys.modules`; otherwise the import runs with every path entry - resolving to the script directory removed. Lexical de-duplication (exact - spelling, trailing separators, cwd aliases) works even before `os` itself - is importable (`python -S`); once `os` is available, entries are also - compared by absolute path. (`sys` is a builtin and can never be shadowed.) + The bundle's directory can reach `sys.path` several ways: as `sys.path[0]` + (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where + `sys.path[0]` is the *driver's* directory) — so the bundle's own path is + passed in explicitly and every equivalent entry is removed. Lexical + de-duplication (exact spelling, trailing separators, cwd aliases) works + even before `os` itself is importable (`python -S`); once `os` is + available, entries are also compared by absolute path. Non-string path + entries (embedded hosts) are kept untouched, as the importer itself would + do. (`sys` is a builtin and can never be shadowed.) """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - first = saved_path[0] if saved_path else None - first_trimmed = first.rstrip("/\\") if first else first + candidates = [] + if saved_path and _isinstance(saved_path[0], _str): + candidates.append(saved_path[0]) + if script_path and _isinstance(script_path, _str): + cut = script_path.rfind("/") + cut_windows = script_path.rfind("\\") + if cut_windows > cut: + cut = cut_windows + candidates.append(script_path[:cut] if cut >= 0 else ".") + trimmed = [] + for candidate in candidates: + trimmed.append(candidate.rstrip("/\\") or candidate) filtered = [] for entry in saved_path[1:]: - try: - if entry in ("", ".", "./", first) or ( - first_trimmed and entry.rstrip("/\\") == first_trimmed - ): + if _isinstance(entry, _str): + if entry in ("", ".", "./"): + continue + stripped = entry.rstrip("/\\") or entry + if stripped in trimmed: continue - except (TypeError, AttributeError): - pass # exotic non-str path entry: keep it filtered.append(entry) os_mod = _cribo_sys.modules.get("os") - if os_mod is not None and first is not None: - script_dir = os_mod.path.normcase(os_mod.path.abspath(first or ".")) - filtered = [ - entry - for entry in filtered - if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir - ] + if os_mod is not None and trimmed: + resolved = [] + for candidate in trimmed: + try: + resolved.append(os_mod.path.normcase(os_mod.path.abspath(candidate or "."))) + except _bex: + pass + kept = [] + for entry in filtered: + if _isinstance(entry, _str): + try: + if os_mod.path.normcase(os_mod.path.abspath(entry or ".")) in resolved: + continue + except _bex: + pass + kept.append(entry) + filtered = kept _cribo_sys.path = _list(filtered) try: return _import(name) @@ -127,6 +155,16 @@ def __init__( self._startup_cwd = os_mod.getcwd() except OSError: self._startup_cwd = "." + # Identity of the on-disk bundle at startup: a rebuild/redeploy at the + # same path must not have its (new) map applied to this (old) running + # code. None when unknowable (stdin); verified lazily at hook time. + self._bundle_stat = None + if not bundle_file == "": + try: + stat = os_mod.stat(self._bundle_anchor) + self._bundle_stat = (stat.st_ino, stat.st_size, stat.st_mtime_ns) + except (OSError, AttributeError): + self._bundle_stat = None self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -179,14 +217,15 @@ def _bootstrap(cls, mode, bundle_file): running without remapping instead of aborting it at startup. """ try: + script = None if bundle_file == "" else bundle_file runtime = cls( mode, bundle_file, - _cribo_sm_import("os"), - _cribo_sm_import("binascii"), - _cribo_sm_import("threading"), - _cribo_sm_import("traceback"), - _cribo_sm_import("hashlib"), + _cribo_sm_import("os", script), + _cribo_sm_import("binascii", script), + _cribo_sm_import("threading", script), + _cribo_sm_import("traceback", script), + _cribo_sm_import("hashlib", script), ) runtime.install() except BaseException: @@ -206,6 +245,19 @@ def _map_location(self): env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None + # A rebuilt/redeployed bundle at the same path carries a NEW map that + # cannot describe the OLD in-memory code objects; refuse remapping + # rather than mapping through a foreign table. (Explicit env-path + # overrides below are the user's deliberate choice and stay untouched.) + stale = False + if self._bundle_stat is not None: + try: + stat = self._os.stat(self._bundle_anchor) + stale = not ( + (stat.st_ino, stat.st_size, stat.st_mtime_ns) == self._bundle_stat + ) + except (OSError, AttributeError): + stale = True # An explicit path wins for every mode (and skips digest verification — # it is the user's explicit choice). This is also the only way to # supply a map to a bundle executed via `python -` (stdin), whose @@ -218,9 +270,11 @@ def _map_location(self): return (path, self._os.path.dirname(self._os.path.abspath(path)), False) bundle = self._bundle_anchor if self._mode == "inline": - if bundle == "": - return None # stdin cannot be re-opened; use CRIBO_SOURCE_MAPS= + if bundle == "" or stale: + return None # stdin/replaced bundles cannot be re-read reliably return (None, self._os.path.dirname(bundle), False) + if stale: + return None sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): @@ -727,6 +781,26 @@ def _chain_has_group( exc = _getattr(exc, "__context__", None) return False + def _percent_decode(self, text, *, _int=int, _chr=chr, _len=len, _bex=BaseException): + """Inverse of the build-time percent-encoding of `sources` entries.""" + if "%" not in text: + return text + out = [] + position = 0 + length = _len(text) + while position < length: + character = text[position] + if character == "%" and position + 2 < length: + try: + out.append(_chr(_int(text[position + 1 : position + 3], 16))) + position += 3 + continue + except _bex: + pass + out.append(character) + position += 1 + return "".join(out) + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): """Read a single 1-based line from a file without caching it.""" try: @@ -816,6 +890,7 @@ def emit(entry, frame_index, mapped_frame): if mapped is not None: source = sources.get(mapped[0]) if source: + source = self._percent_decode(source) if not self._os.path.isabs(source): source = self._os.path.normpath( self._os.path.join(map_dir, source) @@ -1034,24 +1109,33 @@ def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _lis runtimes.append(self) return runtimes - def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): + def _notify_custom_hook( + self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr + ): """Invoke a chained hook after a successful remap when it is custom. A successful remap replaces the *default* printer, but preinstalled custom hooks (error reporters, sitecustomize) must still observe the - exception; their own output is theirs to manage. Two exclusions: when + exception; their own output is theirs to manage. Earlier cribo + runtimes in the chain are traversed (via their own captured + predecessor, named by `prev_attr`) rather than invoked, so a custom + hook installed before any bundle still gets notified while the default + printer is never reached (which would duplicate the traceback). When the interpreter default is unavailable for comparison (e.g. `threading.__excepthook__` before Python 3.10) no notification happens — better to skip a custom hook than to double-print via the default - one; and an earlier cribo runtime's hook is skipped, since it would - find no frames for its own bundle and delegate to the default printer, - duplicating the traceback. + one. """ + hops = 0 + while prev is not None and hops < 32: + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + prev = _getattr(bound_to, prev_attr, None) + hops += 1 + continue + break if default is None or prev is None or prev is default: return - bound_to = _getattr(prev, "__self__", None) - if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): - return try: call(prev) except _bex: @@ -1065,6 +1149,7 @@ def excepthook(self, exc_type, exc_value, traceback_obj): self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), + "_prev_excepthook", ) return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -1085,6 +1170,7 @@ def threading_hook( self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), + "_prev_threading_hook", ) return self._prev_threading_hook(args) @@ -1100,6 +1186,7 @@ def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), + "_prev_unraisablehook", ) return self._prev_unraisablehook(unraisable) diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index fb010b315..973b15e5e 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -308,51 +308,22 @@ impl ParallelWalker<'_> { return; } - if let Some((module_ordinal, original_line)) = self + let stmt_provenance = self .provenance - .resolve(original.node_index().load(), original.range().start()) - { - let generated_start = self.line_index.line_of(generated.range().start()); + .resolve(original.node_index().load(), original.range().start()); + if let Some((module_ordinal, original_line)) = stmt_provenance { self.records.push(MappingRecord { - generated_line: generated_start, + generated_line: self.line_index.line_of(generated.range().start()), module_ordinal, original_line, }); - // Simple (body-less) statements can still span several physical - // lines — ruff's generator emits one line per statement except for - // multiline string/f-string literals, whose content is preserved - // verbatim. A raising expression inside such a literal is - // attributed to its own physical line, so fill the span with - // offset-aligned mappings. Compound statements are excluded: their - // interiors get precise mappings from the recursion below, which - // must not be shadowed (first mapping per line wins). - let has_nested_body = matches!( - generated, - Stmt::FunctionDef(_) - | Stmt::ClassDef(_) - | Stmt::If(_) - | Stmt::While(_) - | Stmt::For(_) - | Stmt::With(_) - | Stmt::Try(_) - | Stmt::Match(_) - ); - if !has_nested_body { - let generated_span = - self.line_index.line_of(generated.range().end()) - generated_start; - let original_span = self - .provenance - .resolve(original.node_index().load(), original.range().end()) - .map_or(0, |(_, end_line)| end_line.saturating_sub(original_line)); - for offset in 1..=generated_span.min(original_span) { - self.records.push(MappingRecord { - generated_line: generated_start + offset, - module_ordinal, - original_line: original_line + offset, - }); - } - } } + // Note on multiline statements: ruff's generator emits every statement + // on a single physical line — multiline string/f-string literals are + // rendered with `\n` escapes, and docstrings likewise. There are + // therefore no interior physical lines to map; a raising expression + // inside such a literal is attributed to the statement's single + // generated line, which the record above already covers. // Evaluating a decorator can raise on its own `@...` line; give every // decorator its own mapping (the statement mapping above only covers @@ -690,6 +661,26 @@ fn is_docstring(stmt: &Stmt) -> bool { matches!(stmt, Stmt::Expr(expr) if expr.value.is_string_literal_expr()) } +/// Percent-encode a source path for the `sources` array. +/// +/// Source Map v3 consumers resolve `sources` entries as URL references, so a +/// raw `#` or `?` in a filesystem name would be read as fragment/query +/// delimiters and point at the wrong resource. Only URL-breaking characters +/// are escaped; everything else (including non-ASCII) passes through. The +/// injected runtime applies the inverse decoding before filesystem access. +fn percent_encode_source(path: &str) -> String { + let mut encoded = String::with_capacity(path.len()); + for character in path.chars() { + if character < '\u{20}' || matches!(character, '\u{7F}' | '%' | '#' | '?') { + let _ = + std::fmt::Write::write_fmt(&mut encoded, format_args!("%{:02X}", character as u32)); + } else { + encoded.push(character); + } + } + encoded +} + /// Build the complete Source Map v3 JSON for an emitted bundle. /// /// Re-parses `bundle_text`, extracts statement mappings against `bundled_ast`, @@ -731,7 +722,7 @@ pub(crate) fn build_source_map( }, |display| { let content = options.include_contents.then(|| module.source.clone()); - Some(generator.add_source(display, content)) + Some(generator.add_source(&percent_encode_source(display), content)) }, ); ordinal_to_source[record.module_ordinal] = Some(resolved); diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 6f6a40eb2..c9935e70d 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -7,26 +7,51 @@ input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py # https://github.com/ophidiarium/cribo import sys as _cribo_sys -def _cribo_sm_import(name, *, _list=list, _import=__import__): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. Lexical de-duplication (exact\n spelling, trailing separators, cwd aliases) works even before `os` itself\n is importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. (`sys` is a builtin and can never be shadowed.)\n """ +def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed. Lexical\n de-duplication (exact spelling, trailing separators, cwd aliases) works\n even before `os` itself is importable (`python -S`); once `os` is\n available, entries are also compared by absolute path. Non-string path\n entries (embedded hosts) are kept untouched, as the importer itself would\n do. (`sys` is a builtin and can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - first = saved_path[0] if saved_path else None - first_trimmed = first.rstrip("/\\") if first else first + candidates = [] + if saved_path and _isinstance(saved_path[0], _str): + candidates.append(saved_path[0]) + if script_path and _isinstance(script_path, _str): + cut = script_path.rfind("/") + cut_windows = script_path.rfind("\\") + if cut_windows > cut: + cut = cut_windows + candidates.append(script_path[:cut] if cut >= 0 else ".") + trimmed = [] + for candidate in candidates: + trimmed.append(candidate.rstrip("/\\") or candidate) filtered = [] for entry in saved_path[1:]: - try: - if entry in ("", ".", "./", first) or first_trimmed and entry.rstrip("/\\") == first_trimmed: + if _isinstance(entry, _str): + if entry in ("", ".", "./"): + continue + stripped = entry.rstrip("/\\") or entry + if stripped in trimmed: continue - except (TypeError, AttributeError): - pass filtered.append(entry) os_mod = _cribo_sys.modules.get("os") - if os_mod is not None and first is not None: - script_dir = os_mod.path.normcase(os_mod.path.abspath(first or ".")) - filtered = [entry for entry in filtered if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir] + if os_mod is not None and trimmed: + resolved = [] + for candidate in trimmed: + try: + resolved.append(os_mod.path.normcase(os_mod.path.abspath(candidate or "."))) + except _bex: + pass + kept = [] + for entry in filtered: + if _isinstance(entry, _str): + try: + if os_mod.path.normcase(os_mod.path.abspath(entry or ".")) in resolved: + continue + except _bex: + pass + kept.append(entry) + filtered = kept _cribo_sys.path = _list(filtered) try: return _import(name) @@ -68,6 +93,13 @@ class _CriboSourceMapRuntime(object): self._startup_cwd = os_mod.getcwd() except OSError: self._startup_cwd = "." + self._bundle_stat = None + if not bundle_file == "": + try: + stat = os_mod.stat(self._bundle_anchor) + self._bundle_stat = stat.st_ino, stat.st_size, stat.st_mtime_ns + except (OSError, AttributeError): + self._bundle_stat = None self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -103,7 +135,8 @@ class _CriboSourceMapRuntime(object): def _bootstrap(cls, mode, bundle_file): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: - runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback"), _cribo_sm_import("hashlib")) + script = None if bundle_file == "" else bundle_file + runtime = cls(mode, bundle_file, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) runtime.install() except BaseException: pass @@ -113,6 +146,13 @@ class _CriboSourceMapRuntime(object): env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None + stale = False + if self._bundle_stat is not None: + try: + stat = self._os.stat(self._bundle_anchor) + stale = not (stat.st_ino, stat.st_size, stat.st_mtime_ns) == self._bundle_stat + except (OSError, AttributeError): + stale = True if env not in ("", "1", "true", "yes", "on"): path = env if not self._os.path.isabs(path): @@ -120,9 +160,11 @@ class _CriboSourceMapRuntime(object): return path, self._os.path.dirname(self._os.path.abspath(path)), False bundle = self._bundle_anchor if self._mode == "inline": - if bundle == "": + if bundle == "" or stale: return None return None, self._os.path.dirname(bundle), False + if stale: + return None sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): @@ -540,6 +582,26 @@ class _CriboSourceMapRuntime(object): exc = _getattr(exc, "__context__", None) return False + def _percent_decode(self, text, *, _int=int, _chr=chr, _len=len, _bex=BaseException): + """Inverse of the build-time percent-encoding of `sources` entries.""" + if "%" not in text: + return text + out = [] + position = 0 + length = _len(text) + while position < length: + character = text[position] + if character == "%" and position + 2 < length: + try: + out.append(_chr(_int(text[position + 1:position + 3], 16))) + position += 3 + continue + except _bex: + pass + out.append(character) + position += 1 + return "".join(out) + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): """Read a single 1-based line from a file without caching it.""" try: @@ -612,6 +674,7 @@ class _CriboSourceMapRuntime(object): if mapped is not None: source = sources.get(mapped[0]) if source: + source = self._percent_decode(source) if not self._os.path.isabs(source): source = self._os.path.normpath(self._os.path.join(map_dir, source)) filename, lineno = source, mapped[1] @@ -770,13 +833,18 @@ class _CriboSourceMapRuntime(object): runtimes.append(self) return runtimes - def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Two exclusions: when\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one; and an earlier cribo runtime's hook is skipped, since it would\n find no frames for its own bundle and delegate to the default printer,\n duplicating the traceback.\n """ + def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). When\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one.\n """ + hops = 0 + while prev is not None and hops < 32: + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + prev = _getattr(bound_to, prev_attr, None) + hops += 1 + continue + break if default is None or prev is None or prev is default: return - bound_to = _getattr(prev, "__self__", None) - if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): - return try: call(prev) except _bex: @@ -784,7 +852,7 @@ class _CriboSourceMapRuntime(object): def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): - self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj)) + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook") return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -796,7 +864,7 @@ class _CriboSourceMapRuntime(object): name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args)) + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook") return self._prev_threading_hook(args) @@ -807,7 +875,7 @@ class _CriboSourceMapRuntime(object): except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable)) + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook") return self._prev_unraisablehook(unraisable) _CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) @@ -958,5 +1026,5 @@ total = add(2, 3) product = multiply(total, 4) print(describe("total", total)) print(describe("product", product)) -# cribo-sourcemap-sha256=b6b82631d8f5e7c89c6803f2cd7c21b12df55242fd1aa31984d5236dcba71d4a +# cribo-sourcemap-sha256=c36dd7f51595374bafef64709ded727675b756b31a07e0e1dd59bfdc790bf228 # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 8d6f69098..9968100c2 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -7,26 +7,51 @@ input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py # https://github.com/ophidiarium/cribo import sys as _cribo_sys -def _cribo_sm_import(name, *, _list=list, _import=__import__): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's own directory is `sys.path[0]` (or `''` for `-c`/stdin), and\n `PYTHONPATH=.` can expose the same directory again at later indices — so a\n project file named e.g. `threading.py` sitting next to the bundle would\n otherwise shadow the stdlib for this runtime. Already-imported modules are\n taken from `sys.modules`; otherwise the import runs with every path entry\n resolving to the script directory removed. Lexical de-duplication (exact\n spelling, trailing separators, cwd aliases) works even before `os` itself\n is importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. (`sys` is a builtin and can never be shadowed.)\n """ +def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed. Lexical\n de-duplication (exact spelling, trailing separators, cwd aliases) works\n even before `os` itself is importable (`python -S`); once `os` is\n available, entries are also compared by absolute path. Non-string path\n entries (embedded hosts) are kept untouched, as the importer itself would\n do. (`sys` is a builtin and can never be shadowed.)\n """ module = _cribo_sys.modules.get(name) if module is not None: return module saved_path = _cribo_sys.path - first = saved_path[0] if saved_path else None - first_trimmed = first.rstrip("/\\") if first else first + candidates = [] + if saved_path and _isinstance(saved_path[0], _str): + candidates.append(saved_path[0]) + if script_path and _isinstance(script_path, _str): + cut = script_path.rfind("/") + cut_windows = script_path.rfind("\\") + if cut_windows > cut: + cut = cut_windows + candidates.append(script_path[:cut] if cut >= 0 else ".") + trimmed = [] + for candidate in candidates: + trimmed.append(candidate.rstrip("/\\") or candidate) filtered = [] for entry in saved_path[1:]: - try: - if entry in ("", ".", "./", first) or first_trimmed and entry.rstrip("/\\") == first_trimmed: + if _isinstance(entry, _str): + if entry in ("", ".", "./"): + continue + stripped = entry.rstrip("/\\") or entry + if stripped in trimmed: continue - except (TypeError, AttributeError): - pass filtered.append(entry) os_mod = _cribo_sys.modules.get("os") - if os_mod is not None and first is not None: - script_dir = os_mod.path.normcase(os_mod.path.abspath(first or ".")) - filtered = [entry for entry in filtered if not os_mod.path.normcase(os_mod.path.abspath(entry or ".")) == script_dir] + if os_mod is not None and trimmed: + resolved = [] + for candidate in trimmed: + try: + resolved.append(os_mod.path.normcase(os_mod.path.abspath(candidate or "."))) + except _bex: + pass + kept = [] + for entry in filtered: + if _isinstance(entry, _str): + try: + if os_mod.path.normcase(os_mod.path.abspath(entry or ".")) in resolved: + continue + except _bex: + pass + kept.append(entry) + filtered = kept _cribo_sys.path = _list(filtered) try: return _import(name) @@ -68,6 +93,13 @@ class _CriboSourceMapRuntime(object): self._startup_cwd = os_mod.getcwd() except OSError: self._startup_cwd = "." + self._bundle_stat = None + if not bundle_file == "": + try: + stat = os_mod.stat(self._bundle_anchor) + self._bundle_stat = stat.st_ino, stat.st_size, stat.st_mtime_ns + except (OSError, AttributeError): + self._bundle_stat = None self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -103,7 +135,8 @@ class _CriboSourceMapRuntime(object): def _bootstrap(cls, mode, bundle_file): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: - runtime = cls(mode, bundle_file, _cribo_sm_import("os"), _cribo_sm_import("binascii"), _cribo_sm_import("threading"), _cribo_sm_import("traceback"), _cribo_sm_import("hashlib")) + script = None if bundle_file == "" else bundle_file + runtime = cls(mode, bundle_file, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) runtime.install() except BaseException: pass @@ -113,6 +146,13 @@ class _CriboSourceMapRuntime(object): env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None + stale = False + if self._bundle_stat is not None: + try: + stat = self._os.stat(self._bundle_anchor) + stale = not (stat.st_ino, stat.st_size, stat.st_mtime_ns) == self._bundle_stat + except (OSError, AttributeError): + stale = True if env not in ("", "1", "true", "yes", "on"): path = env if not self._os.path.isabs(path): @@ -120,9 +160,11 @@ class _CriboSourceMapRuntime(object): return path, self._os.path.dirname(self._os.path.abspath(path)), False bundle = self._bundle_anchor if self._mode == "inline": - if bundle == "": + if bundle == "" or stale: return None return None, self._os.path.dirname(bundle), False + if stale: + return None sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): @@ -540,6 +582,26 @@ class _CriboSourceMapRuntime(object): exc = _getattr(exc, "__context__", None) return False + def _percent_decode(self, text, *, _int=int, _chr=chr, _len=len, _bex=BaseException): + """Inverse of the build-time percent-encoding of `sources` entries.""" + if "%" not in text: + return text + out = [] + position = 0 + length = _len(text) + while position < length: + character = text[position] + if character == "%" and position + 2 < length: + try: + out.append(_chr(_int(text[position + 1:position + 3], 16))) + position += 3 + continue + except _bex: + pass + out.append(character) + position += 1 + return "".join(out) + def _source_line(self, path, lineno, *, _open=open, _os_error=OSError): """Read a single 1-based line from a file without caching it.""" try: @@ -612,6 +674,7 @@ class _CriboSourceMapRuntime(object): if mapped is not None: source = sources.get(mapped[0]) if source: + source = self._percent_decode(source) if not self._os.path.isabs(source): source = self._os.path.normpath(self._os.path.join(map_dir, source)) filename, lineno = source, mapped[1] @@ -770,13 +833,18 @@ class _CriboSourceMapRuntime(object): runtimes.append(self) return runtimes - def _notify_custom_hook(self, prev, default, call, *, _bex=BaseException, _getattr=getattr): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Two exclusions: when\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one; and an earlier cribo runtime's hook is skipped, since it would\n find no frames for its own bundle and delegate to the default printer,\n duplicating the traceback.\n """ + def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). When\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one.\n """ + hops = 0 + while prev is not None and hops < 32: + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + prev = _getattr(bound_to, prev_attr, None) + hops += 1 + continue + break if default is None or prev is None or prev is default: return - bound_to = _getattr(prev, "__self__", None) - if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): - return try: call(prev) except _bex: @@ -784,7 +852,7 @@ class _CriboSourceMapRuntime(object): def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): - self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj)) + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook") return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -796,7 +864,7 @@ class _CriboSourceMapRuntime(object): name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args)) + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook") return self._prev_threading_hook(args) @@ -807,7 +875,7 @@ class _CriboSourceMapRuntime(object): except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable)) + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook") return self._prev_unraisablehook(unraisable) _CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) @@ -965,5 +1033,5 @@ effects.__init__ = _cribo_init___cribo_503b17_effects effects = _cribo_init___cribo_503b17_effects(effects) print("counter:", effects.COUNTER) print("boosted:", effects.boost(10)) -# cribo-sourcemap-sha256=3a91bc5dcbd3f9b4d889f6232ecbabbf740aaa1ac46aba62277c03732bc91b69 +# cribo-sourcemap-sha256=d1929b8451a8662aa287cb9620ed4718dbb5db55abe122d15d965adbc4bdc823 # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index c3d5d05c3..91f143d0b 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:935 `def add(a, b):` -> calculator.py:1 -bundle:936 `result = a + b` -> calculator.py:2 -bundle:937 `return result` -> calculator.py:3 -bundle:939 `def multiply(a, b):` -> calculator.py:6 -bundle:940 `result = a * b` -> calculator.py:7 -bundle:941 `return result` -> calculator.py:8 -bundle:946 `def describe(name, value):` -> utils.py:1 -bundle:947 `return f"{name} = {value}"` -> utils.py:2 -bundle:953 `total = add(2, 3)` -> main.py:4 -bundle:954 `product = multiply(total, 4)` -> main.py:5 -bundle:955 `print(describe("total", total))` -> main.py:6 -bundle:956 `print(describe("product", product))` -> main.py:7 +bundle:1003 `def add(a, b):` -> calculator.py:1 +bundle:1004 `result = a + b` -> calculator.py:2 +bundle:1005 `return result` -> calculator.py:3 +bundle:1007 `def multiply(a, b):` -> calculator.py:6 +bundle:1008 `result = a * b` -> calculator.py:7 +bundle:1009 `return result` -> calculator.py:8 +bundle:1014 `def describe(name, value):` -> utils.py:1 +bundle:1015 `return f"{name} = {value}"` -> utils.py:2 +bundle:1021 `total = add(2, 3)` -> main.py:4 +bundle:1022 `product = multiply(total, 4)` -> main.py:5 +bundle:1023 `print(describe("total", total))` -> main.py:6 +bundle:1024 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 35a28b400..9b0147233 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:944 `print("effects module loading")` -> effects.py:1 -bundle:945 `COUNTER = 1` -> effects.py:3 -bundle:948 `def boost(value):` -> effects.py:6 -bundle:949 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:950 `return boosted` -> effects.py:8 -bundle:962 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:963 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:1012 `print("effects module loading")` -> effects.py:1 +bundle:1013 `COUNTER = 1` -> effects.py:3 +bundle:1016 `def boost(value):` -> effects.py:6 +bundle:1017 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:1018 `return boosted` -> effects.py:8 +bundle:1030 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:1031 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 10193f827..360a737af 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1453,3 +1453,197 @@ fn stacked_runtimes_merge_maps_across_bundles() { ); assert!(stderr.contains("cross-bundle kaboom")); } + +#[test] +fn runtime_refuses_map_after_bundle_replacement() { + // A bundle rebuilt at the same path while an old instance is running must + // not have its (new) map applied to the old in-memory code. The fixture + // simulates the replacement by touching its own file before crashing. + let dir = make_project(&[ + ( + "main.py", + "from helper import boom\n\nwith open(__file__, \"a\") as handle:\n \ + handle.write(\"# rebuilt\\n\")\nboom()\n", + ), + ( + "helper.py", + "def boom():\n raise ValueError(\"kaboom\")\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert_standard_traceback(&stderr); +} + +#[test] +fn runtime_survives_shadowed_threading_under_runpy() { + // Under runpy, sys.path[0] is the DRIVER's directory; the bundle's own + // directory (reached via PYTHONPATH here) must still be filtered when the + // runtime imports its stdlib dependencies. + let dir = crash_project(); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + fs::write( + dir.path().join("threading.py"), + "raise RuntimeError(\"shadow module imported\")\n", + ) + .expect("write shadowing module"); + + let driver_dir = TempDir::new().expect("create driver dir"); + let driver = driver_dir.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({bundle:?}, run_name=\"__main__\")\n", + bundle = bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let mut command = Command::new(common::get_python_executable()); + command.arg(&driver); + command.env_remove("CRIBO_SOURCE_MAPS"); + command.env("PYTHONPATH", dir.path()); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert!( + !stderr.contains("shadow module imported"), + "the runtime must not import the bundle-adjacent threading.py: {stderr}" + ); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "remapping must work under runpy: {stderr}" + ); +} + +#[test] +fn stacked_runtimes_still_notify_preinstalled_custom_hook() { + // A custom hook installed before ANY bundle must still observe exceptions + // when several cribo runtimes have stacked on top of it: the notifier + // traverses through earlier cribo hooks to the original custom one. + let quiet = fixture_project(); + let quiet_bundle = quiet.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&quiet), + "--output", + &quiet_bundle.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let crash = crash_project(); + let crash_bundle = bundle_crash_project(&crash, "--sourcemap=linked"); + + let driver_dir = TempDir::new().expect("create driver dir"); + fs::write( + driver_dir.path().join("sitecustomize.py"), + "import sys\n\n\ndef reporting_hook(exc_type, exc_value, tb):\n print(\"REPORTER \ + SAW:\", exc_type.__name__, file=sys.stderr)\n\n\nsys.excepthook = reporting_hook\n", + ) + .expect("write sitecustomize"); + let driver = driver_dir.path().join("driver.py"); + fs::write( + &driver, + format!( + "import runpy\nrunpy.run_path({quiet:?})\nrunpy.run_path({crash:?})\n", + quiet = quiet_bundle.to_string_lossy(), + crash = crash_bundle.to_string_lossy(), + ), + ) + .expect("write driver"); + + let mut command = Command::new(common::get_python_executable()); + command.arg(&driver); + command.env_remove("CRIBO_SOURCE_MAPS"); + command.env("PYTHONPATH", driver_dir.path()); + let output = command.output().expect("run python"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert!( + stderr.contains("helper.py\", line 5, in inner"), + "remapping must work: {stderr}" + ); + assert!( + stderr.contains("REPORTER SAW: ValueError"), + "the custom hook below two stacked runtimes must still be notified: {stderr}" + ); + assert_eq!( + stderr.matches("Traceback (most recent call last):").count(), + 1, + "no duplicate rendering through the default printer: {stderr}" + ); +} + +#[test] +fn runtime_remaps_raise_inside_multiline_fstring() { + // ruff's generator emits multiline f-strings on a single physical line + // (`\n` escapes), so a replacement expression raising on what was an + // interior line in the ORIGINAL source is attributed to the statement's + // single generated line — which must remap to the statement's original + // starting line. + let dir = make_project(&[ + ("main.py", "from helper import render\n\nprint(render(0))\n"), + ( + "helper.py", + "def render(value):\n banner = f\"\"\"first {value}\nsecond {1 // value}\nthird \ + {value}\"\"\"\n return banner\n", + ), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + assert!(stderr.contains("ZeroDivisionError"), "{stderr}"); + assert!( + stderr.contains("helper.py\", line 2, in render"), + "a raise inside a multiline f-string must map to the statement's original start line: \ + {stderr}" + ); +} + +#[test] +fn sources_with_url_delimiters_are_encoded_and_decoded() { + // '#' in a directory name would be read as a URL fragment by map + // consumers; the map must percent-encode it and the runtime must decode + // it back before touching the filesystem. + let dir = TempDir::new().expect("create temp dir"); + let src_dir = dir.path().join("we#ird"); + fs::create_dir_all(&src_dir).expect("create source dir"); + fs::write( + src_dir.join("main.py"), + "from helper import boom\n\nboom()\n", + ) + .expect("write main.py"); + fs::write( + src_dir.join("helper.py"), + "def boom():\n raise ValueError(\"kaboom\")\n", + ) + .expect("write helper.py"); + + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &src_dir.join("main.py").to_string_lossy(), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + assert!( + map_json.contains("we%23ird"), + "URL delimiters in source paths must be percent-encoded: {map_json}" + ); + + let (ok, _, stderr) = run_python(&out, &[]); + assert!(!ok); + assert!( + stderr.contains("we#ird/helper.py\", line 2, in boom") + || stderr.contains("we#ird\\helper.py\", line 2, in boom"), + "the runtime must decode the path before display and file access: {stderr}" + ); + assert!( + stderr.contains("raise ValueError(\"kaboom\")"), + "the original source line must load from the decoded path: {stderr}" + ); +} From 0d60b715c80c97a55bc26cad2ac4eb88ff6de3b9 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sat, 22 Aug 2026 21:36:18 +0200 Subject: [PATCH 15/17] fix: address fourteenth review round on source map runtime - bake the map's SHA-256 into the executing code itself (placeholder in the prologue, substituted post-extraction without shifting lines), replacing the disk-read digest trailer: verification is now immune to every bundle replacement window, covers inline payloads, and restores external mode's comment-free output - decline remapping for frames whose filename spelling is claimed by several stacked runtimes with different anchored files - hook-chain traversal uses cycle detection and never invokes a hook that is still cribo-owned, eliminating the fixed-depth duplicate rendering - serialize Windows source paths with forward slashes (URL references) Addresses fourteenth-round review comments on #570 --- crates/cribo/src/orchestrator.rs | 21 ++- crates/cribo/src/python/sourcemap_runtime.py | 167 +++++++++--------- crates/cribo/src/source_map.rs | 47 +++-- .../tests/python/test_sourcemap_runtime.py | 2 + .../bundled_code@sourcemap_basic.snap | 105 +++++------ .../bundled_code@sourcemap_wrapper.snap | 105 +++++------ .../snapshots/source_map@sourcemap_basic.snap | 24 +-- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 31 ++-- 9 files changed, 263 insertions(+), 253 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index dbe6fed93..2bdc538e2 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -715,6 +715,13 @@ impl BundleOrchestrator { })?; let mut bundled_code = emitted.code; + // Bake the map digest into the runtime prologue (or blank the + // placeholder when no map was produced). + if self.config.sourcemap.is_some() { + bundled_code = + crate::source_map::apply_map_digest(&bundled_code, emitted.source_map.as_deref()); + } + // Stdout output can only carry an inline map; other modes are rejected // at CLI validation time. if self.config.sourcemap == Some(SourceMapMode::Inline) @@ -785,6 +792,14 @@ impl BundleOrchestrator { })?; let mut bundled_code = emitted.code; + // Bake the map digest into the runtime prologue (or blank the + // placeholder when no map was produced): the digest lives inside the + // executing code itself, immune to on-disk replacement. + if self.config.sourcemap.is_some() { + bundled_code = + crate::source_map::apply_map_digest(&bundled_code, emitted.source_map.as_deref()); + } + // Apply the configured source map delivery mode. The map file itself is // written only after the bundle write succeeds, so a failed run never // leaves an orphaned (and potentially stale) map next to an old bundle. @@ -794,16 +809,12 @@ impl BundleOrchestrator { match mode { SourceMapMode::Linked | SourceMapMode::External => { let map_path = source_map_path_for(output_path); - // Both sibling-map modes embed the map digest so the - // runtime can reject a mismatched pair; only linked mode - // adds the sourceMappingURL reference. - bundled_code.push('\n'); - bundled_code.push_str(&crate::source_map::linked_map_digest_comment(map_json)); if mode == SourceMapMode::Linked { let map_file_name = map_path.file_name().map_or_else( || map_path.to_string_lossy().into_owned(), |name| name.to_string_lossy().into_owned(), ); + bundled_code.push('\n'); bundled_code.push_str(&crate::source_map::linked_source_mapping_comment( &map_file_name, )); diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index d0e58eb7d..5b5c6c9f8 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -131,6 +131,7 @@ def __init__( self, mode, bundle_file, + expected_digest, os_mod, binascii_mod, threading_mod, @@ -146,6 +147,16 @@ def __init__( self._bundle_anchor = bundle_file else: self._bundle_anchor = os_mod.path.abspath(bundle_file) + # SHA-256 of this build's map, baked into the executing code itself at + # build time — immune to any on-disk replacement of the bundle, so a + # redeployed bundle's map can never be applied to old in-memory code. + self._expected_digest = None + try: + if isinstance(expected_digest, str) and len(expected_digest) == 64: + int(expected_digest, 16) + self._expected_digest = expected_digest.lower() + except ValueError: + self._expected_digest = None # Marks this instance so another cribo runtime chained behind it can # recognize it (see _notify_custom_hook). self._cribo_sm_runtime_marker = True @@ -155,16 +166,6 @@ def __init__( self._startup_cwd = os_mod.getcwd() except OSError: self._startup_cwd = "." - # Identity of the on-disk bundle at startup: a rebuild/redeploy at the - # same path must not have its (new) map applied to this (old) running - # code. None when unknowable (stdin); verified lazily at hook time. - self._bundle_stat = None - if not bundle_file == "": - try: - stat = os_mod.stat(self._bundle_anchor) - self._bundle_stat = (stat.st_ino, stat.st_size, stat.st_mtime_ns) - except (OSError, AttributeError): - self._bundle_stat = None self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -210,7 +211,7 @@ def install(self): self._threading.excepthook = self.threading_hook @classmethod - def _bootstrap(cls, mode, bundle_file): + def _bootstrap(cls, mode, bundle_file, expected_digest): """Import dependencies safely, construct, and install — fail-open. Any failure (however exotic the host environment) leaves the program @@ -221,6 +222,7 @@ def _bootstrap(cls, mode, bundle_file): runtime = cls( mode, bundle_file, + expected_digest, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), @@ -245,19 +247,6 @@ def _map_location(self): env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None - # A rebuilt/redeployed bundle at the same path carries a NEW map that - # cannot describe the OLD in-memory code objects; refuse remapping - # rather than mapping through a foreign table. (Explicit env-path - # overrides below are the user's deliberate choice and stay untouched.) - stale = False - if self._bundle_stat is not None: - try: - stat = self._os.stat(self._bundle_anchor) - stale = not ( - (stat.st_ino, stat.st_size, stat.st_mtime_ns) == self._bundle_stat - ) - except (OSError, AttributeError): - stale = True # An explicit path wins for every mode (and skips digest verification — # it is the user's explicit choice). This is also the only way to # supply a map to a bundle executed via `python -` (stdin), whose @@ -270,11 +259,9 @@ def _map_location(self): return (path, self._os.path.dirname(self._os.path.abspath(path)), False) bundle = self._bundle_anchor if self._mode == "inline": - if bundle == "" or stale: - return None # stdin/replaced bundles cannot be re-read reliably - return (None, self._os.path.dirname(bundle), False) - if stale: - return None + if bundle == "": + return None # stdin cannot be re-opened; use CRIBO_SOURCE_MAPS= + return (None, self._os.path.dirname(bundle), True) sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): @@ -343,45 +330,23 @@ def _find_inline_payload(self, handle): return -1 return found + base64_at + 7 - def _bundle_expected_digest(self, *, _open=open, _len=len, _int=int): - """SHA-256 hex the bundle records for its linked map, or None.""" - try: - handle = _open(self._bundle_anchor, "rb") - except OSError: - return None - try: - found = self._find_marker_tail(handle, b"# cribo-sourcemap-sha256=") - if found < 0: - return None - handle.seek(found) - digest = handle.read(64) - finally: - handle.close() - if not _len(digest) == 64: - return None - try: - _int(digest, 16) - except ValueError: - return None - return digest.decode("ascii").lower() - - def _map_matches_bundle(self, handle, *, _bex=BaseException): - """False only when the bundle records a digest and the map disagrees. - - This is what makes sibling-map publication safe against interleaved - concurrent builds and manual file shuffling: the digest travels inside - the bundle, which is always internally consistent, so a sibling map - from a different build is detected and ignored. Hashing reads from the - same open handle later used for decoding, so the verified bytes are the - decoded bytes. Verification errors fail open (the subsequent map read - would surface them anyway). + def _map_matches_digest(self, chunks_factory, *, _bex=BaseException): + """False only when a build digest is known and the map bytes disagree. + + The SHA-256 of this build's map is baked into the executing code at + build time, so it is immune to any on-disk replacement of the bundle — + no interleaving of concurrent builds, manual file shuffling, or + redeploy-while-running can pair these code objects with another + build's mappings. Hashing streams the same chunk source later used for + decoding. Verification errors fail open (the subsequent map read would + surface them anyway). """ try: - expected = self._bundle_expected_digest() + expected = self._expected_digest if expected is None: return True hasher = self._hashlib.sha256() - for chunk in self._handle_chunks(handle): + for chunk in chunks_factory(): hasher.update(chunk) return hasher.hexdigest() == expected except _bex: @@ -675,8 +640,6 @@ def _load(self, needed_lines, *, _open=open, _set=set, _max=max): max_needed = _max(needed0) handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") try: - if map_path is not None and verify and not self._map_matches_bundle(handle): - return None if map_path is None: def chunks_factory(): @@ -687,6 +650,8 @@ def chunks_factory(): def chunks_factory(): return self._handle_chunks(handle) + if verify and not self._map_matches_digest(chunks_factory): + return None sources, table0 = self._scan(chunks_factory, needed0, max_needed) finally: handle.close() @@ -707,12 +672,19 @@ def _load_json_fallback( handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") try: - if map_path is not None and verify and not self._map_matches_bundle(handle): - return None if map_path is None: - raw = b"".join(self._inline_chunks(handle)) + + def fallback_chunks(): + return self._inline_chunks(handle) + else: - raw = b"".join(self._handle_chunks(handle)) + + def fallback_chunks(): + return self._handle_chunks(handle) + + if verify and not self._map_matches_digest(fallback_chunks): + return None + raw = b"".join(fallback_chunks()) finally: handle.close() data = json.loads(raw.decode("utf-8")) @@ -1025,7 +997,15 @@ def _render( self._write_exception_only(exc, write) def _try_render( - self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException + self, + exc_value, + traceback_obj, + prefix, + *, + _getattr=getattr, + _bex=BaseException, + _set=set, + _len=len, ): """Attempt a remapped rendering to stderr; True on success. @@ -1042,12 +1022,23 @@ def _try_render( return False # Every stacked cribo runtime contributes mappings for its own # bundle, so a traceback crossing several bundles remaps fully. + # When two runtimes share one filename spelling (e.g. both loaded + # as a relative "bundle.py" from different directories), frames + # carrying that spelling are ambiguous — decline rather than remap + # through the wrong project's sources. + anchors_by_name = {} + for runtime in self._registered_runtimes(): + bundle_file = _getattr(runtime, "_bundle", None) + anchor = _getattr(runtime, "_bundle_anchor", None) + anchors_by_name.setdefault(bundle_file, _set()).add(anchor) maps_by_file = {} for runtime in self._registered_runtimes(): try: bundle_file = runtime._bundle if bundle_file in maps_by_file: continue + if _len(anchors_by_name.get(bundle_file, ())) > 1: + continue # ambiguous spelling: skip these frames needed = self._collect_needed(exc_value, traceback_obj, bundle_file) if not needed: continue @@ -1110,7 +1101,16 @@ def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _lis return runtimes def _notify_custom_hook( - self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr + self, + prev, + default, + call, + prev_attr, + *, + _bex=BaseException, + _getattr=getattr, + _set=set, + _id=id, ): """Invoke a chained hook after a successful remap when it is custom. @@ -1120,20 +1120,25 @@ def _notify_custom_hook( runtimes in the chain are traversed (via their own captured predecessor, named by `prev_attr`) rather than invoked, so a custom hook installed before any bundle still gets notified while the default - printer is never reached (which would duplicate the traceback). When - the interpreter default is unavailable for comparison (e.g. - `threading.__excepthook__` before Python 3.10) no notification happens - — better to skip a custom hook than to double-print via the default - one. + printer is never reached (which would duplicate the traceback). Cycle + detection guards the traversal, and a chain that still ends on a cribo + hook invokes nothing (its registry-aware renderer would print the + traceback a second time). When the interpreter default is unavailable + for comparison (e.g. `threading.__excepthook__` before Python 3.10) no + notification happens — better to skip a custom hook than to + double-print via the default one. """ - hops = 0 - while prev is not None and hops < 32: + seen = _set() + while prev is not None and _id(prev) not in seen: + seen.add(_id(prev)) bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): prev = _getattr(bound_to, prev_attr, None) - hops += 1 continue break + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return if default is None or prev is None or prev is default: return try: @@ -1193,5 +1198,7 @@ def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): _CriboSourceMapRuntime._bootstrap( - "__CRIBO_SOURCEMAP_MODE__", globals().get("__file__", "") + "__CRIBO_SOURCEMAP_MODE__", + globals().get("__file__", ""), + "__CRIBO_SOURCEMAP_DIGEST__", ) diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index 973b15e5e..b73e6314f 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -569,22 +569,33 @@ pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { format!("# sourceMappingURL={encoded}\n") } -/// Comment recording the SHA-256 of the sibling map at build time. +/// Placeholder in the runtime template replaced with this build's map digest. +const RUNTIME_DIGEST_PLACEHOLDER: &str = "__CRIBO_SOURCEMAP_DIGEST__"; + +/// Bake the map's SHA-256 into the emitted bundle code. /// -/// The runtime refuses a sibling map whose digest does not match, so no -/// interleaving of concurrent builds (or manual file shuffling) can pair a -/// bundle with another build's mappings — the digest travels inside the bundle -/// itself, which is always internally consistent. Emitted for both linked and -/// external modes (the sibling-map pairing problem is identical); an explicit -/// `CRIBO_SOURCE_MAPS=` override skips verification. -pub(crate) fn linked_map_digest_comment(map_json: &str) -> String { +/// The digest lives inside the executing code itself (not in a trailer read +/// back from disk), so it is immune to any on-disk replacement of the bundle: +/// no interleaving of concurrent builds, manual file shuffling, or +/// redeploy-while-running can pair in-memory code objects with another build's +/// mappings. The placeholder sits inside a single-line string literal, so the +/// substitution never shifts line numbers and the extracted mappings stay +/// valid. With no map, the placeholder becomes an empty string, which the +/// runtime treats as "no digest known" (verification is skipped). +pub(crate) fn apply_map_digest(code: &str, map_json: Option<&str>) -> String { + use cow_utils::CowUtils as _; use sha2::{Digest as _, Sha256}; - let digest = Sha256::digest(map_json.as_bytes()); - let mut hex = String::with_capacity(64); - for byte in digest { - hex.push_str(&format!("{byte:02x}")); - } - format!("# cribo-sourcemap-sha256={hex}\n") + + let digest_hex = map_json.map_or_else(String::new, |json| { + let digest = Sha256::digest(json.as_bytes()); + let mut hex = String::with_capacity(64); + for byte in digest { + let _ = std::fmt::Write::write_fmt(&mut hex, format_args!("{byte:02x}")); + } + hex + }); + code.cow_replace(RUNTIME_DIGEST_PLACEHOLDER, &digest_hex) + .into_owned() } /// Comment embedding the source map as a base64 data URL. @@ -671,6 +682,14 @@ fn is_docstring(stmt: &Stmt) -> bool { fn percent_encode_source(path: &str) -> String { let mut encoded = String::with_capacity(path.len()); for character in path.chars() { + // On Windows the native separator is '\'; URL consumers only treat + // '/' as a path separator, so normalize. (On Unix a backslash is a + // legal filename character and passes through untouched.) + #[cfg(windows)] + if character == '\\' { + encoded.push('/'); + continue; + } if character < '\u{20}' || matches!(character, '\u{7F}' | '%' | '#' | '?') { let _ = std::fmt::Write::write_fmt(&mut encoded, format_args!("%{:02X}", character as u32)); diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index fb72331f1..cdc6dd313 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -29,6 +29,7 @@ def load_runtime(path): return module._CriboSourceMapRuntime( "external", "", + "", # no build digest: verification is skipped os, __import__("binascii"), threading, @@ -237,6 +238,7 @@ def test_env_path_wins_for_every_mode(rt): inline_stdin = type(rt)( "inline", "", + "", os, __import__("binascii"), threading, diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index c9935e70d..55d7f5c08 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -81,25 +81,25 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): + def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): self._mode = mode self._bundle = bundle_file if bundle_file == "": self._bundle_anchor = bundle_file else: self._bundle_anchor = os_mod.path.abspath(bundle_file) + self._expected_digest = None + try: + if isinstance(expected_digest, str) and len(expected_digest) == 64: + int(expected_digest, 16) + self._expected_digest = expected_digest.lower() + except ValueError: + self._expected_digest = None self._cribo_sm_runtime_marker = True try: self._startup_cwd = os_mod.getcwd() except OSError: self._startup_cwd = "." - self._bundle_stat = None - if not bundle_file == "": - try: - stat = os_mod.stat(self._bundle_anchor) - self._bundle_stat = stat.st_ino, stat.st_size, stat.st_mtime_ns - except (OSError, AttributeError): - self._bundle_stat = None self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -132,11 +132,11 @@ class _CriboSourceMapRuntime(object): self._threading.excepthook = self.threading_hook @classmethod - def _bootstrap(cls, mode, bundle_file): + def _bootstrap(cls, mode, bundle_file, expected_digest): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: script = None if bundle_file == "" else bundle_file - runtime = cls(mode, bundle_file, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) + runtime = cls(mode, bundle_file, expected_digest, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) runtime.install() except BaseException: pass @@ -146,13 +146,6 @@ class _CriboSourceMapRuntime(object): env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None - stale = False - if self._bundle_stat is not None: - try: - stat = self._os.stat(self._bundle_anchor) - stale = not (stat.st_ino, stat.st_size, stat.st_mtime_ns) == self._bundle_stat - except (OSError, AttributeError): - stale = True if env not in ("", "1", "true", "yes", "on"): path = env if not self._os.path.isabs(path): @@ -160,11 +153,9 @@ class _CriboSourceMapRuntime(object): return path, self._os.path.dirname(self._os.path.abspath(path)), False bundle = self._bundle_anchor if self._mode == "inline": - if bundle == "" or stale: + if bundle == "": return None - return None, self._os.path.dirname(bundle), False - if stale: - return None + return None, self._os.path.dirname(bundle), True sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): @@ -223,36 +214,14 @@ class _CriboSourceMapRuntime(object): return -1 return found + base64_at + 7 - def _bundle_expected_digest(self, *, _open=open, _len=len, _int=int): - """SHA-256 hex the bundle records for its linked map, or None.""" - try: - handle = _open(self._bundle_anchor, "rb") - except OSError: - return None + def _map_matches_digest(self, chunks_factory, *, _bex=BaseException): + """False only when a build digest is known and the map bytes disagree.\n\n The SHA-256 of this build's map is baked into the executing code at\n build time, so it is immune to any on-disk replacement of the bundle —\n no interleaving of concurrent builds, manual file shuffling, or\n redeploy-while-running can pair these code objects with another\n build's mappings. Hashing streams the same chunk source later used for\n decoding. Verification errors fail open (the subsequent map read would\n surface them anyway).\n """ try: - found = self._find_marker_tail(handle, b"# cribo-sourcemap-sha256=") - if found < 0: - return None - handle.seek(found) - digest = handle.read(64) - finally: - handle.close() - if not _len(digest) == 64: - return None - try: - _int(digest, 16) - except ValueError: - return None - return digest.decode("ascii").lower() - - def _map_matches_bundle(self, handle, *, _bex=BaseException): - """False only when the bundle records a digest and the map disagrees.\n\n This is what makes sibling-map publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Hashing reads from the\n same open handle later used for decoding, so the verified bytes are the\n decoded bytes. Verification errors fail open (the subsequent map read\n would surface them anyway).\n """ - try: - expected = self._bundle_expected_digest() + expected = self._expected_digest if expected is None: return True hasher = self._hashlib.sha256() - for chunk in self._handle_chunks(handle): + for chunk in chunks_factory(): hasher.update(chunk) return hasher.hexdigest() == expected except _bex: @@ -495,8 +464,6 @@ class _CriboSourceMapRuntime(object): max_needed = _max(needed0) handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") try: - if map_path is not None and verify and not self._map_matches_bundle(handle): - return None if map_path is None: def chunks_factory(): @@ -505,6 +472,8 @@ class _CriboSourceMapRuntime(object): def chunks_factory(): return self._handle_chunks(handle) + if verify and not self._map_matches_digest(chunks_factory): + return None sources, table0 = self._scan(chunks_factory, needed0, max_needed) finally: handle.close() @@ -522,12 +491,17 @@ class _CriboSourceMapRuntime(object): json = self._import("json") handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") try: - if map_path is not None and verify and not self._map_matches_bundle(handle): - return None if map_path is None: - raw = b"".join(self._inline_chunks(handle)) + + def fallback_chunks(): + return self._inline_chunks(handle) else: - raw = b"".join(self._handle_chunks(handle)) + + def fallback_chunks(): + return self._handle_chunks(handle) + if verify and not self._map_matches_digest(fallback_chunks): + return None + raw = b"".join(fallback_chunks()) finally: handle.close() data = json.loads(raw.decode("utf-8")) @@ -763,7 +737,7 @@ class _CriboSourceMapRuntime(object): self._write_frames(tb, maps_by_file, write, limit, summaries) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException): + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException, _set=set, _len=len): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ if _getattr(self._local, "in_hook", False) or exc_value is None: return False @@ -772,12 +746,19 @@ class _CriboSourceMapRuntime(object): try: if self._chain_has_group(exc_value): return False + anchors_by_name = {} + for runtime in self._registered_runtimes(): + bundle_file = _getattr(runtime, "_bundle", None) + anchor = _getattr(runtime, "_bundle_anchor", None) + anchors_by_name.setdefault(bundle_file, _set()).add(anchor) maps_by_file = {} for runtime in self._registered_runtimes(): try: bundle_file = runtime._bundle if bundle_file in maps_by_file: continue + if _len(anchors_by_name.get(bundle_file, ())) > 1: + continue needed = self._collect_needed(exc_value, traceback_obj, bundle_file) if not needed: continue @@ -833,16 +814,19 @@ class _CriboSourceMapRuntime(object): runtimes.append(self) return runtimes - def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). When\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one.\n """ - hops = 0 - while prev is not None and hops < 32: + def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Cycle\n detection guards the traversal, and a chain that still ends on a cribo\n hook invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + seen = _set() + while prev is not None and _id(prev) not in seen: + seen.add(_id(prev)) bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): prev = _getattr(bound_to, prev_attr, None) - hops += 1 continue break + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return if default is None or prev is None or prev is default: return try: @@ -878,7 +862,7 @@ class _CriboSourceMapRuntime(object): self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook") return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "a12cc2500424ec3363fdd28632bb2c8aefea11f8d96e1abfe1257d55772a8631") import sys as _sys import importlib as _importlib class _CriboModule(): @@ -1026,5 +1010,4 @@ total = add(2, 3) product = multiply(total, 4) print(describe("total", total)) print(describe("product", product)) -# cribo-sourcemap-sha256=c36dd7f51595374bafef64709ded727675b756b31a07e0e1dd59bfdc790bf228 # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 9968100c2..f8cd2e0bf 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -81,25 +81,25 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): + def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): self._mode = mode self._bundle = bundle_file if bundle_file == "": self._bundle_anchor = bundle_file else: self._bundle_anchor = os_mod.path.abspath(bundle_file) + self._expected_digest = None + try: + if isinstance(expected_digest, str) and len(expected_digest) == 64: + int(expected_digest, 16) + self._expected_digest = expected_digest.lower() + except ValueError: + self._expected_digest = None self._cribo_sm_runtime_marker = True try: self._startup_cwd = os_mod.getcwd() except OSError: self._startup_cwd = "." - self._bundle_stat = None - if not bundle_file == "": - try: - stat = os_mod.stat(self._bundle_anchor) - self._bundle_stat = stat.st_ino, stat.st_size, stat.st_mtime_ns - except (OSError, AttributeError): - self._bundle_stat = None self._os = os_mod self._sys = _cribo_sys self._binascii = binascii_mod @@ -132,11 +132,11 @@ class _CriboSourceMapRuntime(object): self._threading.excepthook = self.threading_hook @classmethod - def _bootstrap(cls, mode, bundle_file): + def _bootstrap(cls, mode, bundle_file, expected_digest): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: script = None if bundle_file == "" else bundle_file - runtime = cls(mode, bundle_file, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) + runtime = cls(mode, bundle_file, expected_digest, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) runtime.install() except BaseException: pass @@ -146,13 +146,6 @@ class _CriboSourceMapRuntime(object): env = self._os.environ.get("CRIBO_SOURCE_MAPS", "") if env == "0": return None - stale = False - if self._bundle_stat is not None: - try: - stat = self._os.stat(self._bundle_anchor) - stale = not (stat.st_ino, stat.st_size, stat.st_mtime_ns) == self._bundle_stat - except (OSError, AttributeError): - stale = True if env not in ("", "1", "true", "yes", "on"): path = env if not self._os.path.isabs(path): @@ -160,11 +153,9 @@ class _CriboSourceMapRuntime(object): return path, self._os.path.dirname(self._os.path.abspath(path)), False bundle = self._bundle_anchor if self._mode == "inline": - if bundle == "" or stale: + if bundle == "": return None - return None, self._os.path.dirname(bundle), False - if stale: - return None + return None, self._os.path.dirname(bundle), True sibling = bundle + ".map" if self._mode == "linked": if self._os.path.exists(sibling): @@ -223,36 +214,14 @@ class _CriboSourceMapRuntime(object): return -1 return found + base64_at + 7 - def _bundle_expected_digest(self, *, _open=open, _len=len, _int=int): - """SHA-256 hex the bundle records for its linked map, or None.""" - try: - handle = _open(self._bundle_anchor, "rb") - except OSError: - return None + def _map_matches_digest(self, chunks_factory, *, _bex=BaseException): + """False only when a build digest is known and the map bytes disagree.\n\n The SHA-256 of this build's map is baked into the executing code at\n build time, so it is immune to any on-disk replacement of the bundle —\n no interleaving of concurrent builds, manual file shuffling, or\n redeploy-while-running can pair these code objects with another\n build's mappings. Hashing streams the same chunk source later used for\n decoding. Verification errors fail open (the subsequent map read would\n surface them anyway).\n """ try: - found = self._find_marker_tail(handle, b"# cribo-sourcemap-sha256=") - if found < 0: - return None - handle.seek(found) - digest = handle.read(64) - finally: - handle.close() - if not _len(digest) == 64: - return None - try: - _int(digest, 16) - except ValueError: - return None - return digest.decode("ascii").lower() - - def _map_matches_bundle(self, handle, *, _bex=BaseException): - """False only when the bundle records a digest and the map disagrees.\n\n This is what makes sibling-map publication safe against interleaved\n concurrent builds and manual file shuffling: the digest travels inside\n the bundle, which is always internally consistent, so a sibling map\n from a different build is detected and ignored. Hashing reads from the\n same open handle later used for decoding, so the verified bytes are the\n decoded bytes. Verification errors fail open (the subsequent map read\n would surface them anyway).\n """ - try: - expected = self._bundle_expected_digest() + expected = self._expected_digest if expected is None: return True hasher = self._hashlib.sha256() - for chunk in self._handle_chunks(handle): + for chunk in chunks_factory(): hasher.update(chunk) return hasher.hexdigest() == expected except _bex: @@ -495,8 +464,6 @@ class _CriboSourceMapRuntime(object): max_needed = _max(needed0) handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") try: - if map_path is not None and verify and not self._map_matches_bundle(handle): - return None if map_path is None: def chunks_factory(): @@ -505,6 +472,8 @@ class _CriboSourceMapRuntime(object): def chunks_factory(): return self._handle_chunks(handle) + if verify and not self._map_matches_digest(chunks_factory): + return None sources, table0 = self._scan(chunks_factory, needed0, max_needed) finally: handle.close() @@ -522,12 +491,17 @@ class _CriboSourceMapRuntime(object): json = self._import("json") handle = _open(self._bundle_anchor if map_path is None else map_path, "rb") try: - if map_path is not None and verify and not self._map_matches_bundle(handle): - return None if map_path is None: - raw = b"".join(self._inline_chunks(handle)) + + def fallback_chunks(): + return self._inline_chunks(handle) else: - raw = b"".join(self._handle_chunks(handle)) + + def fallback_chunks(): + return self._handle_chunks(handle) + if verify and not self._map_matches_digest(fallback_chunks): + return None + raw = b"".join(fallback_chunks()) finally: handle.close() data = json.loads(raw.decode("utf-8")) @@ -763,7 +737,7 @@ class _CriboSourceMapRuntime(object): self._write_frames(tb, maps_by_file, write, limit, summaries) self._write_exception_only(exc, write) - def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException): + def _try_render(self, exc_value, traceback_obj, prefix, *, _getattr=getattr, _bex=BaseException, _set=set, _len=len): """Attempt a remapped rendering to stderr; True on success.\n\n Never raises and never masks the original exception: any failure in\n this runtime returns False so callers can delegate to the previous\n hook.\n """ if _getattr(self._local, "in_hook", False) or exc_value is None: return False @@ -772,12 +746,19 @@ class _CriboSourceMapRuntime(object): try: if self._chain_has_group(exc_value): return False + anchors_by_name = {} + for runtime in self._registered_runtimes(): + bundle_file = _getattr(runtime, "_bundle", None) + anchor = _getattr(runtime, "_bundle_anchor", None) + anchors_by_name.setdefault(bundle_file, _set()).add(anchor) maps_by_file = {} for runtime in self._registered_runtimes(): try: bundle_file = runtime._bundle if bundle_file in maps_by_file: continue + if _len(anchors_by_name.get(bundle_file, ())) > 1: + continue needed = self._collect_needed(exc_value, traceback_obj, bundle_file) if not needed: continue @@ -833,16 +814,19 @@ class _CriboSourceMapRuntime(object): runtimes.append(self) return runtimes - def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). When\n the interpreter default is unavailable for comparison (e.g.\n `threading.__excepthook__` before Python 3.10) no notification happens\n — better to skip a custom hook than to double-print via the default\n one.\n """ - hops = 0 - while prev is not None and hops < 32: + def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Cycle\n detection guards the traversal, and a chain that still ends on a cribo\n hook invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + seen = _set() + while prev is not None and _id(prev) not in seen: + seen.add(_id(prev)) bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): prev = _getattr(bound_to, prev_attr, None) - hops += 1 continue break + bound_to = _getattr(prev, "__self__", None) + if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + return if default is None or prev is None or prev is default: return try: @@ -878,7 +862,7 @@ class _CriboSourceMapRuntime(object): self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook") return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", "")) +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "d04b33c25f114129eb46735bdb46a45ae5a9636163102bd44bb41678e13b2608") import sys as _sys import importlib as _importlib class _CriboModule(): @@ -1033,5 +1017,4 @@ effects.__init__ = _cribo_init___cribo_503b17_effects effects = _cribo_init___cribo_503b17_effects(effects) print("counter:", effects.COUNTER) print("boosted:", effects.boost(10)) -# cribo-sourcemap-sha256=d1929b8451a8662aa287cb9620ed4718dbb5db55abe122d15d965adbc4bdc823 # sourceMappingURL=bundled.py.map diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 91f143d0b..708628dce 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:1003 `def add(a, b):` -> calculator.py:1 -bundle:1004 `result = a + b` -> calculator.py:2 -bundle:1005 `return result` -> calculator.py:3 -bundle:1007 `def multiply(a, b):` -> calculator.py:6 -bundle:1008 `result = a * b` -> calculator.py:7 -bundle:1009 `return result` -> calculator.py:8 -bundle:1014 `def describe(name, value):` -> utils.py:1 -bundle:1015 `return f"{name} = {value}"` -> utils.py:2 -bundle:1021 `total = add(2, 3)` -> main.py:4 -bundle:1022 `product = multiply(total, 4)` -> main.py:5 -bundle:1023 `print(describe("total", total))` -> main.py:6 -bundle:1024 `print(describe("product", product))` -> main.py:7 +bundle:987 `def add(a, b):` -> calculator.py:1 +bundle:988 `result = a + b` -> calculator.py:2 +bundle:989 `return result` -> calculator.py:3 +bundle:991 `def multiply(a, b):` -> calculator.py:6 +bundle:992 `result = a * b` -> calculator.py:7 +bundle:993 `return result` -> calculator.py:8 +bundle:998 `def describe(name, value):` -> utils.py:1 +bundle:999 `return f"{name} = {value}"` -> utils.py:2 +bundle:1005 `total = add(2, 3)` -> main.py:4 +bundle:1006 `product = multiply(total, 4)` -> main.py:5 +bundle:1007 `print(describe("total", total))` -> main.py:6 +bundle:1008 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 9b0147233..8271d47d9 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:1012 `print("effects module loading")` -> effects.py:1 -bundle:1013 `COUNTER = 1` -> effects.py:3 -bundle:1016 `def boost(value):` -> effects.py:6 -bundle:1017 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:1018 `return boosted` -> effects.py:8 -bundle:1030 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:1031 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:996 `print("effects module loading")` -> effects.py:1 +bundle:997 `COUNTER = 1` -> effects.py:3 +bundle:1000 `def boost(value):` -> effects.py:6 +bundle:1001 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:1002 `return boosted` -> effects.py:8 +bundle:1014 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:1015 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 360a737af..b9392dbc6 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1457,20 +1457,25 @@ fn stacked_runtimes_merge_maps_across_bundles() { #[test] fn runtime_refuses_map_after_bundle_replacement() { // A bundle rebuilt at the same path while an old instance is running must - // not have its (new) map applied to the old in-memory code. The fixture - // simulates the replacement by touching its own file before crashing. - let dir = make_project(&[ - ( - "main.py", - "from helper import boom\n\nwith open(__file__, \"a\") as handle:\n \ - handle.write(\"# rebuilt\\n\")\nboom()\n", - ), - ( - "helper.py", - "def boom():\n raise ValueError(\"kaboom\")\n", - ), - ]); + // not have its (new) map applied to the old in-memory code. The digest of + // this build's map is baked into the executing code, so the old process + // rejects the replacement map. Simulated here by rebuilding with shifted + // lines and running the ORIGINAL bundle against the REBUILT map. + let dir = crash_project(); let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + let original_bundle = fs::read(&bundle).expect("read original bundle"); + + // Rebuild with different line numbers → different map (and digest). + fs::write( + dir.path().join("helper.py"), + "# shifted\n# shifted again\ndef boom():\n inner()\n\ndef inner():\n raise \ + ValueError(\"kaboom\")\n", + ) + .expect("shift helper lines"); + bundle_crash_project(&dir, "--sourcemap=linked"); + + // Old bundle + new map: the replacement pairing must be refused. + fs::write(&bundle, original_bundle).expect("restore original bundle"); let (ok, _, stderr) = run_python(&bundle, &[]); assert!(!ok); assert_standard_traceback(&stderr); From 16b23cf7828a6d7fcbe11ef5be1e166209caa9c5 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sat, 22 Aug 2026 22:13:40 +0200 Subject: [PATCH 16/17] fix: address fifteenth review round on source map runtime - substitute the map digest into the first placeholder occurrence only, leaving user strings that share the spelling untouched - create the staged map file 0600 on unix (atomic at open) so no reader window exists before permissions are copied from a previous map - collect every stacked runtime's captured default hook during chain traversal so a chain ending on any snapshot of the default printer is never invoked (which would double-print) - scrub all prologue helper names from the bundle namespace with a final del; construction-time needs are snapshotted via keyword-only defaults - percent-encode raw non-UTF-8 bytes of the linked map name (OsStr in, byte-wise on unix) instead of baking U+FFFD into sourceMappingURL - resolve runtime imports through BuiltinImporter/FrozenImporter before PathFinder over a private path snapshot: no sys.path mutation, and builtin-compiled modules (e.g. binascii on some interpreters) resolve - drop the setrecursionlimit bump: hook runs post-unwind and the render path is iterative, so the global race bought nothing --- crates/cribo/src/orchestrator.rs | 31 +++-- crates/cribo/src/python/sourcemap_runtime.py | 127 ++++++++++++------ crates/cribo/src/source_map.rs | 62 ++++++++- .../tests/python/test_sourcemap_runtime.py | 12 +- .../bundled_code@sourcemap_basic.snap | 81 ++++++----- .../bundled_code@sourcemap_wrapper.snap | 81 ++++++----- .../snapshots/source_map@sourcemap_basic.snap | 24 ++-- .../source_map@sourcemap_wrapper.snap | 14 +- crates/cribo/tests/test_source_maps.rs | 113 ++++++++++++++++ docs/source-maps.md | 3 + 10 files changed, 399 insertions(+), 149 deletions(-) diff --git a/crates/cribo/src/orchestrator.rs b/crates/cribo/src/orchestrator.rs index 2bdc538e2..33968a0d4 100644 --- a/crates/cribo/src/orchestrator.rs +++ b/crates/cribo/src/orchestrator.rs @@ -85,9 +85,15 @@ fn staging_suffix(attempt: u32) -> String { /// be predicted and pointed elsewhere. Names are unpredictable (see /// [`staging_suffix`]) and collisions retry with fresh entropy, which also /// keeps concurrent builds from stomping each other's staged map. -/// When a previous map exists, its permissions are copied onto the staged file -/// before any content is written, so a restricted map (e.g. 0600 protecting -/// `sourcesContent`) stays restricted across rebuilds. +/// +/// The staged file is born `0600` on Unix (mode set atomically at `open(2)` +/// time), so no other user can grab a readable handle between creation and a +/// later `chmod`. When a previous map exists, its permissions are then copied +/// onto the staged file before any content is written, so a restricted map +/// (e.g. 0600 protecting `sourcesContent`) stays restricted across rebuilds. +/// With no previous map the file keeps `0600`; after the rename the map is +/// owner-only by default, and users who want it world-readable can `chmod` it +/// once — the permissions persist across subsequent rebuilds. fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { use std::io::Write as _; @@ -100,11 +106,14 @@ fn stage_map_file(map_path: &Path, map_json: &str) -> std::io::Result { let mut tmp_name = base_name.clone(); tmp_name.push(format!(".{}.tmp", staging_suffix(attempt))); let tmp_path = map_path.with_file_name(tmp_name); - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp_path) + let mut open_options = fs::OpenOptions::new(); + open_options.write(true).create_new(true); + #[cfg(unix)] { + use std::os::unix::fs::OpenOptionsExt as _; + open_options.mode(0o600); + } + match open_options.open(&tmp_path) { Ok(mut file) => { // symlink_metadata: an attacker-planted symlink at the map // path must not decide the staged file's permissions. @@ -810,13 +819,11 @@ impl BundleOrchestrator { SourceMapMode::Linked | SourceMapMode::External => { let map_path = source_map_path_for(output_path); if mode == SourceMapMode::Linked { - let map_file_name = map_path.file_name().map_or_else( - || map_path.to_string_lossy().into_owned(), - |name| name.to_string_lossy().into_owned(), - ); + let map_file_name = + map_path.file_name().unwrap_or_else(|| map_path.as_os_str()); bundled_code.push('\n'); bundled_code.push_str(&crate::source_map::linked_source_mapping_comment( - &map_file_name, + map_file_name, )); } pending_map = Some((map_path, map_json)); diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 5b5c6c9f8..47e75620c 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -18,6 +18,7 @@ def _cribo_sm_import( name, script_path=None, *, + _sys=_cribo_sys, _list=list, _import=__import__, _isinstance=isinstance, @@ -29,17 +30,20 @@ def _cribo_sm_import( The bundle's directory can reach `sys.path` several ways: as `sys.path[0]` (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where `sys.path[0]` is the *driver's* directory) — so the bundle's own path is - passed in explicitly and every equivalent entry is removed. Lexical - de-duplication (exact spelling, trailing separators, cwd aliases) works - even before `os` itself is importable (`python -S`); once `os` is - available, entries are also compared by absolute path. Non-string path - entries (embedded hosts) are kept untouched, as the importer itself would - do. (`sys` is a builtin and can never be shadowed.) + passed in explicitly and every equivalent entry is removed from a private + *snapshot* of the search path. Resolution goes through the frozen importer + machinery (`PathFinder.find_spec` with that explicit snapshot), so global + `sys.path` is never mutated and concurrent bootstraps cannot race each + other's save/restore. Lexical de-duplication works even before `os` is + importable (`python -S`); once `os` is available, entries are also + compared by absolute path. Non-string entries are kept untouched. + (`sys` and the frozen importlib modules are builtins and can never be + shadowed.) """ - module = _cribo_sys.modules.get(name) + module = _sys.modules.get(name) if module is not None: return module - saved_path = _cribo_sys.path + saved_path = _list(_sys.path) candidates = [] if saved_path and _isinstance(saved_path[0], _str): candidates.append(saved_path[0]) @@ -61,7 +65,7 @@ def _cribo_sm_import( if stripped in trimmed: continue filtered.append(entry) - os_mod = _cribo_sys.modules.get("os") + os_mod = _sys.modules.get("os") if os_mod is not None and trimmed: resolved = [] for candidate in trimmed: @@ -79,11 +83,36 @@ def _cribo_sm_import( pass kept.append(entry) filtered = kept - _cribo_sys.path = _list(filtered) + # Primary path: resolve against the private snapshot via the frozen + # importer machinery — no global state is touched. Builtin and frozen + # modules (which sys.path shadowing can never affect, and which PathFinder + # cannot see — e.g. `binascii` is compiled into some interpreters) are + # consulted first, mirroring the interpreter's own finder order. + frozen_external = _sys.modules.get("_frozen_importlib_external") + frozen_bootstrap = _sys.modules.get("_frozen_importlib") + if frozen_external is not None and frozen_bootstrap is not None: + spec = ( + frozen_bootstrap.BuiltinImporter.find_spec(name) + or frozen_bootstrap.FrozenImporter.find_spec(name) + or frozen_external.PathFinder.find_spec(name, filtered) + ) + if spec is None: + raise ImportError("cribo runtime could not resolve " + name) + module = frozen_bootstrap.module_from_spec(spec) + _sys.modules[name] = module + try: + spec.loader.exec_module(module) + except _bex: + _sys.modules.pop(name, None) + raise + return module + # Fallback for exotic hosts without the frozen modules: brief global + # swap (the historical behavior). + _sys.path = filtered try: return _import(name) finally: - _cribo_sys.path = saved_path + _sys.path = saved_path class _CriboSmStream(object): @@ -137,6 +166,14 @@ def __init__( threading_mod, traceback_mod, hashlib_mod, + *, + # Class-definition-time snapshots: the module-level helper names are + # deleted at the end of the prologue, so they must never be looked up + # globally at construction time (test harnesses construct instances + # long after that cleanup). + _sys=_cribo_sys, + _stream_cls=_CriboSmStream, + _sm_import=_cribo_sm_import, ): self._mode = mode # As-given path for frame matching (co_filename uses the invocation @@ -167,11 +204,11 @@ def __init__( except OSError: self._startup_cwd = "." self._os = os_mod - self._sys = _cribo_sys + self._sys = _sys self._binascii = binascii_mod self._threading = threading_mod - self._stream_cls = _CriboSmStream - self._import = _cribo_sm_import + self._stream_cls = _stream_cls + self._import = _sm_import # Captured at construction (before any bundled user code runs) so a # first-party module registering sys.modules["traceback"] later cannot # degrade exception formatting. @@ -180,14 +217,14 @@ def __init__( # Re-entrancy guard; thread-local so a hook firing on one thread never # disables remapping on another. self._local = threading_mod.local() - self._prev_excepthook = _cribo_sys.excepthook - self._prev_unraisablehook = _cribo_sys.unraisablehook + self._prev_excepthook = _sys.excepthook + self._prev_unraisablehook = _sys.unraisablehook self._prev_threading_hook = threading_mod.excepthook # Interpreter defaults, snapshotted now so user code rebinding e.g. # sys.__excepthook__ later cannot make the captured previous hook look # custom (which would double-print) or raise from the hook. - self._default_excepthook = _cribo_sys.__excepthook__ - self._default_unraisablehook = _cribo_sys.__unraisablehook__ + self._default_excepthook = _sys.__excepthook__ + self._default_unraisablehook = _sys.__unraisablehook__ self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) try: self._group_type = BaseExceptionGroup @@ -211,7 +248,7 @@ def install(self): self._threading.excepthook = self.threading_hook @classmethod - def _bootstrap(cls, mode, bundle_file, expected_digest): + def _bootstrap(cls, mode, bundle_file, expected_digest, *, _sm_import=_cribo_sm_import): """Import dependencies safely, construct, and install — fail-open. Any failure (however exotic the host environment) leaves the program @@ -223,11 +260,11 @@ def _bootstrap(cls, mode, bundle_file, expected_digest): mode, bundle_file, expected_digest, - _cribo_sm_import("os", script), - _cribo_sm_import("binascii", script), - _cribo_sm_import("threading", script), - _cribo_sm_import("traceback", script), - _cribo_sm_import("hashlib", script), + _sm_import("os", script), + _sm_import("binascii", script), + _sm_import("threading", script), + _sm_import("traceback", script), + _sm_import("hashlib", script), ) runtime.install() except BaseException: @@ -1016,7 +1053,6 @@ def _try_render( if _getattr(self._local, "in_hook", False) or exc_value is None: return False self._local.in_hook = True - old_limit = None try: if self._chain_has_group(exc_value): return False @@ -1056,11 +1092,6 @@ def _try_render( continue if not maps_by_file: return False - try: - old_limit = self._sys.getrecursionlimit() - self._sys.setrecursionlimit(old_limit + 64) - except _bex: - old_limit = None # Buffer the rendering so a mid-render failure produces no partial # output before the previous hook prints the standard traceback. parts = [] @@ -1077,11 +1108,6 @@ def _try_render( except _bex: return False finally: - if old_limit is not None: - try: - self._sys.setrecursionlimit(old_limit) - except _bex: - pass self._local.in_hook = False def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): @@ -1106,6 +1132,7 @@ def _notify_custom_hook( default, call, prev_attr, + default_attr, *, _bex=BaseException, _getattr=getattr, @@ -1120,27 +1147,40 @@ def _notify_custom_hook( runtimes in the chain are traversed (via their own captured predecessor, named by `prev_attr`) rather than invoked, so a custom hook installed before any bundle still gets notified while the default - printer is never reached (which would duplicate the traceback). Cycle - detection guards the traversal, and a chain that still ends on a cribo - hook invokes nothing (its registry-aware renderer would print the + printer is never reached (which would duplicate the traceback). Each + traversed runtime's own captured default (named by `default_attr`) is + collected along the way: an earlier runtime may have snapshotted a + different default object than this one (e.g. after someone swapped + `sys.__excepthook__` between bundle imports), and a chain ending on + *any* of those defaults must invoke nothing — every default is a + traceback printer, and the remap already printed. Cycle detection + guards the traversal, and a chain that still ends on a cribo hook + likewise invokes nothing (its registry-aware renderer would print the traceback a second time). When the interpreter default is unavailable for comparison (e.g. `threading.__excepthook__` before Python 3.10) no notification happens — better to skip a custom hook than to double-print via the default one. """ + defaults = [default] seen = _set() while prev is not None and _id(prev) not in seen: seen.add(_id(prev)) bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + other_default = _getattr(bound_to, default_attr, None) + if other_default is not None: + defaults.append(other_default) prev = _getattr(bound_to, prev_attr, None) continue break bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): return - if default is None or prev is None or prev is default: + if default is None or prev is None: return + for known_default in defaults: + if prev is known_default: + return try: call(prev) except _bex: @@ -1155,6 +1195,7 @@ def excepthook(self, exc_type, exc_value, traceback_obj): self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook", + "_default_excepthook", ) return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -1176,6 +1217,7 @@ def threading_hook( self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook", + "_default_threading_hook", ) return self._prev_threading_hook(args) @@ -1192,6 +1234,7 @@ def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook", + "_default_unraisablehook", ) return self._prev_unraisablehook(unraisable) @@ -1202,3 +1245,9 @@ def unraisablehook(self, unraisable, *, _getattr=getattr, _bex=BaseException): globals().get("__file__", ""), "__CRIBO_SOURCEMAP_DIGEST__", ) +# Leave no trace in the bundle's namespace: user code must not see (or +# collide with) runtime helpers, and `from bundle import *` must not export +# them. The installed instance holds every reference it needs (captured in +# __init__/keyword defaults), so nothing here is looked up globally after +# bootstrap. +del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index b73e6314f..a9248fff4 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -546,9 +546,54 @@ fn relative_path(base: &std::path::Path, target: &std::path::Path) -> std::path: /// treats the line as a plain comment. The file name is percent-encoded so a /// hostile or accidental control character (e.g. a newline in a Unix filename) /// cannot terminate the comment and inject executable text into the bundle. -pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { +/// +/// Takes the name as `OsStr` because Unix filenames need not be UTF-8: invalid +/// bytes are percent-encoded verbatim (`%XX` of the raw byte), which is both +/// injection-safe and the faithful URL form of the on-disk name — a lossy +/// U+FFFD substitution would point consumers at a file that does not exist. +pub(crate) fn linked_source_mapping_comment(map_file_name: &std::ffi::OsStr) -> String { let mut encoded = String::with_capacity(map_file_name.len()); - for character in map_file_name.chars() { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt as _; + let mut bytes = map_file_name.as_bytes(); + while !bytes.is_empty() { + match std::str::from_utf8(bytes) { + Ok(valid) => { + encode_map_url_into(valid, &mut encoded); + break; + } + Err(err) => { + let (valid, rest) = bytes.split_at(err.valid_up_to()); + encode_map_url_into( + std::str::from_utf8(valid).expect("valid_up_to prefix is UTF-8"), + &mut encoded, + ); + // error_len is None only at end-of-input (truncated + // sequence); encode every remaining byte in that case. + let invalid_len = err.error_len().unwrap_or(rest.len()); + for byte in &rest[..invalid_len] { + let _ = + std::fmt::Write::write_fmt(&mut encoded, format_args!("%{byte:02X}")); + } + bytes = &rest[invalid_len..]; + } + } + } + } + #[cfg(not(unix))] + { + // Windows filenames are UTF-16; unpaired surrogates cannot be + // expressed in a UTF-8 comment at all, so lossy substitution is the + // only option there. + encode_map_url_into(&map_file_name.to_string_lossy(), &mut encoded); + } + format!("# sourceMappingURL={encoded}\n") +} + +/// Percent-encode `text` for a `sourceMappingURL` comment, appending to `out`. +fn encode_map_url_into(text: &str, out: &mut String) { + for character in text.chars() { // Control characters would break out of the comment; '#', '?', and // other URL delimiters would truncate the reference for // standards-compliant URL consumers. Non-ASCII Unicode passes through @@ -560,13 +605,11 @@ pub(crate) fn linked_source_mapping_comment(map_file_name: &str) -> String { '%' | '#' | '?' | '"' | '<' | '>' | '\\' | '^' | '`' | '|' ) { - let _ = - std::fmt::Write::write_fmt(&mut encoded, format_args!("%{:02X}", character as u32)); + let _ = std::fmt::Write::write_fmt(out, format_args!("%{:02X}", character as u32)); } else { - encoded.push(character); + out.push(character); } } - format!("# sourceMappingURL={encoded}\n") } /// Placeholder in the runtime template replaced with this build's map digest. @@ -582,6 +625,11 @@ const RUNTIME_DIGEST_PLACEHOLDER: &str = "__CRIBO_SOURCEMAP_DIGEST__"; /// substitution never shifts line numbers and the extracted mappings stay /// valid. With no map, the placeholder becomes an empty string, which the /// runtime treats as "no digest known" (verification is skipped). +/// +/// Only the first occurrence is replaced: the prologue is emitted before any +/// user code, so its placeholder is always the first one in the bundle, and +/// user code that happens to contain the same spelling (in a string literal, +/// comment, or identifier) is left untouched. pub(crate) fn apply_map_digest(code: &str, map_json: Option<&str>) -> String { use cow_utils::CowUtils as _; use sha2::{Digest as _, Sha256}; @@ -594,7 +642,7 @@ pub(crate) fn apply_map_digest(code: &str, map_json: Option<&str>) -> String { } hex }); - code.cow_replace(RUNTIME_DIGEST_PLACEHOLDER, &digest_hex) + code.cow_replacen(RUNTIME_DIGEST_PLACEHOLDER, &digest_hex, 1) .into_owned() } diff --git a/crates/cribo/tests/python/test_sourcemap_runtime.py b/crates/cribo/tests/python/test_sourcemap_runtime.py index cdc6dd313..504b957d4 100644 --- a/crates/cribo/tests/python/test_sourcemap_runtime.py +++ b/crates/cribo/tests/python/test_sourcemap_runtime.py @@ -19,14 +19,22 @@ def load_runtime(path): """Import the runtime module and return a runtime instance for testing. The import installs the hooks; they are restored immediately so failures - in this harness surface as normal tracebacks. + in this harness surface as normal tracebacks. The runtime class is + recovered from the installed hook's bound instance — the template's last + statement deletes every module-level helper name, so the module namespace + is intentionally empty after import. """ prev_hooks = (sys.excepthook, sys.unraisablehook, threading.excepthook) spec = importlib.util.spec_from_file_location("cribo_sm_runtime", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + installed_hook = sys.excepthook sys.excepthook, sys.unraisablehook, threading.excepthook = prev_hooks - return module._CriboSourceMapRuntime( + assert not hasattr(module, "_CriboSourceMapRuntime"), ( + "template must scrub its helper names from the bundle namespace" + ) + runtime_cls = type(installed_hook.__self__) + return runtime_cls( "external", "", "", # no build digest: verification is skipped diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 55d7f5c08..1359b3167 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -7,12 +7,12 @@ input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py # https://github.com/ophidiarium/cribo import sys as _cribo_sys -def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed. Lexical\n de-duplication (exact spelling, trailing separators, cwd aliases) works\n even before `os` itself is importable (`python -S`); once `os` is\n available, entries are also compared by absolute path. Non-string path\n entries (embedded hosts) are kept untouched, as the importer itself would\n do. (`sys` is a builtin and can never be shadowed.)\n """ - module = _cribo_sys.modules.get(name) +def _cribo_sm_import(name, script_path=None, *, _sys=_cribo_sys, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed from a private\n *snapshot* of the search path. Resolution goes through the frozen importer\n machinery (`PathFinder.find_spec` with that explicit snapshot), so global\n `sys.path` is never mutated and concurrent bootstraps cannot race each\n other's save/restore. Lexical de-duplication works even before `os` is\n importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. Non-string entries are kept untouched.\n (`sys` and the frozen importlib modules are builtins and can never be\n shadowed.)\n """ + module = _sys.modules.get(name) if module is not None: return module - saved_path = _cribo_sys.path + saved_path = _list(_sys.path) candidates = [] if saved_path and _isinstance(saved_path[0], _str): candidates.append(saved_path[0]) @@ -34,7 +34,7 @@ def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, if stripped in trimmed: continue filtered.append(entry) - os_mod = _cribo_sys.modules.get("os") + os_mod = _sys.modules.get("os") if os_mod is not None and trimmed: resolved = [] for candidate in trimmed: @@ -52,11 +52,25 @@ def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, pass kept.append(entry) filtered = kept - _cribo_sys.path = _list(filtered) + frozen_external = _sys.modules.get("_frozen_importlib_external") + frozen_bootstrap = _sys.modules.get("_frozen_importlib") + if frozen_external is not None and frozen_bootstrap is not None: + spec = frozen_bootstrap.BuiltinImporter.find_spec(name) or frozen_bootstrap.FrozenImporter.find_spec(name) or frozen_external.PathFinder.find_spec(name, filtered) + if spec is None: + raise ImportError("cribo runtime could not resolve " + name) + module = frozen_bootstrap.module_from_spec(spec) + _sys.modules[name] = module + try: + spec.loader.exec_module(module) + except _bex: + _sys.modules.pop(name, None) + raise + return module + _sys.path = filtered try: return _import(name) finally: - _cribo_sys.path = saved_path + _sys.path = saved_path class _CriboSmStream(object): """Byte-at-a-time reader over an iterator of byte chunks.\n\n Keyword-only defaults snapshot the builtins at definition time (before any\n bundled user code runs), so later shadowing of e.g. `len` cannot break the\n reader — the same idiom cribo's generated module proxies use.\n """ __slots__ = "_chunks", "_buf", "_pos" @@ -81,7 +95,7 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): + def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod, *, _sys=_cribo_sys, _stream_cls=_CriboSmStream, _sm_import=_cribo_sm_import): self._mode = mode self._bundle = bundle_file if bundle_file == "": @@ -101,19 +115,19 @@ class _CriboSourceMapRuntime(object): except OSError: self._startup_cwd = "." self._os = os_mod - self._sys = _cribo_sys + self._sys = _sys self._binascii = binascii_mod self._threading = threading_mod - self._stream_cls = _CriboSmStream - self._import = _cribo_sm_import + self._stream_cls = _stream_cls + self._import = _sm_import self._traceback = traceback_mod self._hashlib = hashlib_mod self._local = threading_mod.local() - self._prev_excepthook = _cribo_sys.excepthook - self._prev_unraisablehook = _cribo_sys.unraisablehook + self._prev_excepthook = _sys.excepthook + self._prev_unraisablehook = _sys.unraisablehook self._prev_threading_hook = threading_mod.excepthook - self._default_excepthook = _cribo_sys.__excepthook__ - self._default_unraisablehook = _cribo_sys.__unraisablehook__ + self._default_excepthook = _sys.__excepthook__ + self._default_unraisablehook = _sys.__unraisablehook__ self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) try: self._group_type = BaseExceptionGroup @@ -132,11 +146,11 @@ class _CriboSourceMapRuntime(object): self._threading.excepthook = self.threading_hook @classmethod - def _bootstrap(cls, mode, bundle_file, expected_digest): + def _bootstrap(cls, mode, bundle_file, expected_digest, *, _sm_import=_cribo_sm_import): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: script = None if bundle_file == "" else bundle_file - runtime = cls(mode, bundle_file, expected_digest, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) + runtime = cls(mode, bundle_file, expected_digest, _sm_import("os", script), _sm_import("binascii", script), _sm_import("threading", script), _sm_import("traceback", script), _sm_import("hashlib", script)) runtime.install() except BaseException: pass @@ -742,7 +756,6 @@ class _CriboSourceMapRuntime(object): if _getattr(self._local, "in_hook", False) or exc_value is None: return False self._local.in_hook = True - old_limit = None try: if self._chain_has_group(exc_value): return False @@ -776,11 +789,6 @@ class _CriboSourceMapRuntime(object): continue if not maps_by_file: return False - try: - old_limit = self._sys.getrecursionlimit() - self._sys.setrecursionlimit(old_limit + 64) - except _bex: - old_limit = None parts = [] if prefix: parts.append(prefix) @@ -795,11 +803,6 @@ class _CriboSourceMapRuntime(object): except _bex: return False finally: - if old_limit is not None: - try: - self._sys.setrecursionlimit(old_limit) - except _bex: - pass self._local.in_hook = False def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): @@ -814,21 +817,28 @@ class _CriboSourceMapRuntime(object): runtimes.append(self) return runtimes - def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Cycle\n detection guards the traversal, and a chain that still ends on a cribo\n hook invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + def _notify_custom_hook(self, prev, default, call, prev_attr, default_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Each\n traversed runtime's own captured default (named by `default_attr`) is\n collected along the way: an earlier runtime may have snapshotted a\n different default object than this one (e.g. after someone swapped\n `sys.__excepthook__` between bundle imports), and a chain ending on\n *any* of those defaults must invoke nothing — every default is a\n traceback printer, and the remap already printed. Cycle detection\n guards the traversal, and a chain that still ends on a cribo hook\n likewise invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + defaults = [default] seen = _set() while prev is not None and _id(prev) not in seen: seen.add(_id(prev)) bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + other_default = _getattr(bound_to, default_attr, None) + if other_default is not None: + defaults.append(other_default) prev = _getattr(bound_to, prev_attr, None) continue break bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): return - if default is None or prev is None or prev is default: + if default is None or prev is None: return + for known_default in defaults: + if prev is known_default: + return try: call(prev) except _bex: @@ -836,7 +846,7 @@ class _CriboSourceMapRuntime(object): def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): - self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook") + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook", "_default_excepthook") return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -848,7 +858,7 @@ class _CriboSourceMapRuntime(object): name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook") + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook", "_default_threading_hook") return self._prev_threading_hook(args) @@ -859,10 +869,11 @@ class _CriboSourceMapRuntime(object): except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook") + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook", "_default_unraisablehook") return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "a12cc2500424ec3363fdd28632bb2c8aefea11f8d96e1abfe1257d55772a8631") +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "49c287797b013f17ab9e979132db7215f3eb9d0f7c3299d27043ab12f15a2a3b") +del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime import sys as _sys import importlib as _importlib class _CriboModule(): diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index f8cd2e0bf..16c570068 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -7,12 +7,12 @@ input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py # https://github.com/ophidiarium/cribo import sys as _cribo_sys -def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): - """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed. Lexical\n de-duplication (exact spelling, trailing separators, cwd aliases) works\n even before `os` itself is importable (`python -S`); once `os` is\n available, entries are also compared by absolute path. Non-string path\n entries (embedded hosts) are kept untouched, as the importer itself would\n do. (`sys` is a builtin and can never be shadowed.)\n """ - module = _cribo_sys.modules.get(name) +def _cribo_sm_import(name, script_path=None, *, _sys=_cribo_sys, _list=list, _import=__import__, _isinstance=isinstance, _str=str, _bex=BaseException): + """Import a stdlib module immune to script-directory shadowing.\n\n The bundle's directory can reach `sys.path` several ways: as `sys.path[0]`\n (direct execution), via `PYTHONPATH`, or not at all under `runpy` (where\n `sys.path[0]` is the *driver's* directory) — so the bundle's own path is\n passed in explicitly and every equivalent entry is removed from a private\n *snapshot* of the search path. Resolution goes through the frozen importer\n machinery (`PathFinder.find_spec` with that explicit snapshot), so global\n `sys.path` is never mutated and concurrent bootstraps cannot race each\n other's save/restore. Lexical de-duplication works even before `os` is\n importable (`python -S`); once `os` is available, entries are also\n compared by absolute path. Non-string entries are kept untouched.\n (`sys` and the frozen importlib modules are builtins and can never be\n shadowed.)\n """ + module = _sys.modules.get(name) if module is not None: return module - saved_path = _cribo_sys.path + saved_path = _list(_sys.path) candidates = [] if saved_path and _isinstance(saved_path[0], _str): candidates.append(saved_path[0]) @@ -34,7 +34,7 @@ def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, if stripped in trimmed: continue filtered.append(entry) - os_mod = _cribo_sys.modules.get("os") + os_mod = _sys.modules.get("os") if os_mod is not None and trimmed: resolved = [] for candidate in trimmed: @@ -52,11 +52,25 @@ def _cribo_sm_import(name, script_path=None, *, _list=list, _import=__import__, pass kept.append(entry) filtered = kept - _cribo_sys.path = _list(filtered) + frozen_external = _sys.modules.get("_frozen_importlib_external") + frozen_bootstrap = _sys.modules.get("_frozen_importlib") + if frozen_external is not None and frozen_bootstrap is not None: + spec = frozen_bootstrap.BuiltinImporter.find_spec(name) or frozen_bootstrap.FrozenImporter.find_spec(name) or frozen_external.PathFinder.find_spec(name, filtered) + if spec is None: + raise ImportError("cribo runtime could not resolve " + name) + module = frozen_bootstrap.module_from_spec(spec) + _sys.modules[name] = module + try: + spec.loader.exec_module(module) + except _bex: + _sys.modules.pop(name, None) + raise + return module + _sys.path = filtered try: return _import(name) finally: - _cribo_sys.path = saved_path + _sys.path = saved_path class _CriboSmStream(object): """Byte-at-a-time reader over an iterator of byte chunks.\n\n Keyword-only defaults snapshot the builtins at definition time (before any\n bundled user code runs), so later shadowing of e.g. `len` cannot break the\n reader — the same idiom cribo's generated module proxies use.\n """ __slots__ = "_chunks", "_buf", "_pos" @@ -81,7 +95,7 @@ class _CriboSourceMapRuntime(object): _B64 = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" _CHUNK = 8192 - def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod): + def __init__(self, mode, bundle_file, expected_digest, os_mod, binascii_mod, threading_mod, traceback_mod, hashlib_mod, *, _sys=_cribo_sys, _stream_cls=_CriboSmStream, _sm_import=_cribo_sm_import): self._mode = mode self._bundle = bundle_file if bundle_file == "": @@ -101,19 +115,19 @@ class _CriboSourceMapRuntime(object): except OSError: self._startup_cwd = "." self._os = os_mod - self._sys = _cribo_sys + self._sys = _sys self._binascii = binascii_mod self._threading = threading_mod - self._stream_cls = _CriboSmStream - self._import = _cribo_sm_import + self._stream_cls = _stream_cls + self._import = _sm_import self._traceback = traceback_mod self._hashlib = hashlib_mod self._local = threading_mod.local() - self._prev_excepthook = _cribo_sys.excepthook - self._prev_unraisablehook = _cribo_sys.unraisablehook + self._prev_excepthook = _sys.excepthook + self._prev_unraisablehook = _sys.unraisablehook self._prev_threading_hook = threading_mod.excepthook - self._default_excepthook = _cribo_sys.__excepthook__ - self._default_unraisablehook = _cribo_sys.__unraisablehook__ + self._default_excepthook = _sys.__excepthook__ + self._default_unraisablehook = _sys.__unraisablehook__ self._default_threading_hook = getattr(threading_mod, "__excepthook__", None) try: self._group_type = BaseExceptionGroup @@ -132,11 +146,11 @@ class _CriboSourceMapRuntime(object): self._threading.excepthook = self.threading_hook @classmethod - def _bootstrap(cls, mode, bundle_file, expected_digest): + def _bootstrap(cls, mode, bundle_file, expected_digest, *, _sm_import=_cribo_sm_import): """Import dependencies safely, construct, and install — fail-open.\n\n Any failure (however exotic the host environment) leaves the program\n running without remapping instead of aborting it at startup.\n """ try: script = None if bundle_file == "" else bundle_file - runtime = cls(mode, bundle_file, expected_digest, _cribo_sm_import("os", script), _cribo_sm_import("binascii", script), _cribo_sm_import("threading", script), _cribo_sm_import("traceback", script), _cribo_sm_import("hashlib", script)) + runtime = cls(mode, bundle_file, expected_digest, _sm_import("os", script), _sm_import("binascii", script), _sm_import("threading", script), _sm_import("traceback", script), _sm_import("hashlib", script)) runtime.install() except BaseException: pass @@ -742,7 +756,6 @@ class _CriboSourceMapRuntime(object): if _getattr(self._local, "in_hook", False) or exc_value is None: return False self._local.in_hook = True - old_limit = None try: if self._chain_has_group(exc_value): return False @@ -776,11 +789,6 @@ class _CriboSourceMapRuntime(object): continue if not maps_by_file: return False - try: - old_limit = self._sys.getrecursionlimit() - self._sys.setrecursionlimit(old_limit + 64) - except _bex: - old_limit = None parts = [] if prefix: parts.append(prefix) @@ -795,11 +803,6 @@ class _CriboSourceMapRuntime(object): except _bex: return False finally: - if old_limit is not None: - try: - self._sys.setrecursionlimit(old_limit) - except _bex: - pass self._local.in_hook = False def _registered_runtimes(self, *, _getattr=getattr, _isinstance=isinstance, _list=list): @@ -814,21 +817,28 @@ class _CriboSourceMapRuntime(object): runtimes.append(self) return runtimes - def _notify_custom_hook(self, prev, default, call, prev_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): - """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Cycle\n detection guards the traversal, and a chain that still ends on a cribo\n hook invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + def _notify_custom_hook(self, prev, default, call, prev_attr, default_attr, *, _bex=BaseException, _getattr=getattr, _set=set, _id=id): + """Invoke a chained hook after a successful remap when it is custom.\n\n A successful remap replaces the *default* printer, but preinstalled\n custom hooks (error reporters, sitecustomize) must still observe the\n exception; their own output is theirs to manage. Earlier cribo\n runtimes in the chain are traversed (via their own captured\n predecessor, named by `prev_attr`) rather than invoked, so a custom\n hook installed before any bundle still gets notified while the default\n printer is never reached (which would duplicate the traceback). Each\n traversed runtime's own captured default (named by `default_attr`) is\n collected along the way: an earlier runtime may have snapshotted a\n different default object than this one (e.g. after someone swapped\n `sys.__excepthook__` between bundle imports), and a chain ending on\n *any* of those defaults must invoke nothing — every default is a\n traceback printer, and the remap already printed. Cycle detection\n guards the traversal, and a chain that still ends on a cribo hook\n likewise invokes nothing (its registry-aware renderer would print the\n traceback a second time). When the interpreter default is unavailable\n for comparison (e.g. `threading.__excepthook__` before Python 3.10) no\n notification happens — better to skip a custom hook than to\n double-print via the default one.\n """ + defaults = [default] seen = _set() while prev is not None and _id(prev) not in seen: seen.add(_id(prev)) bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): + other_default = _getattr(bound_to, default_attr, None) + if other_default is not None: + defaults.append(other_default) prev = _getattr(bound_to, prev_attr, None) continue break bound_to = _getattr(prev, "__self__", None) if bound_to is not None and _getattr(bound_to, "_cribo_sm_runtime_marker", False): return - if default is None or prev is None or prev is default: + if default is None or prev is None: return + for known_default in defaults: + if prev is known_default: + return try: call(prev) except _bex: @@ -836,7 +846,7 @@ class _CriboSourceMapRuntime(object): def excepthook(self, exc_type, exc_value, traceback_obj): if self._try_render(exc_value, traceback_obj, None): - self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook") + self._notify_custom_hook(self._prev_excepthook, self._default_excepthook, lambda hook: hook(exc_type, exc_value, traceback_obj), "_prev_excepthook", "_default_excepthook") return self._prev_excepthook(exc_type, exc_value, traceback_obj) @@ -848,7 +858,7 @@ class _CriboSourceMapRuntime(object): name = _getattr(thread, "name", None) or "Thread" prefix = "Exception in thread %s:\n" % name if self._try_render(args.exc_value, args.exc_traceback, prefix): - self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook") + self._notify_custom_hook(self._prev_threading_hook, self._default_threading_hook, lambda hook: hook(args), "_prev_threading_hook", "_default_threading_hook") return self._prev_threading_hook(args) @@ -859,10 +869,11 @@ class _CriboSourceMapRuntime(object): except _bex: prefix = "%s\n" % message if self._try_render(unraisable.exc_value, unraisable.exc_traceback, prefix): - self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook") + self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook", "_default_unraisablehook") return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "d04b33c25f114129eb46735bdb46a45ae5a9636163102bd44bb41678e13b2608") +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "e892aae89904ceb493d1b2e4def1f0b19e81b73861a30ac51c7c6f8a2f5d395f") +del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime import sys as _sys import importlib as _importlib class _CriboModule(): diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index 708628dce..e699daffb 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:987 `def add(a, b):` -> calculator.py:1 -bundle:988 `result = a + b` -> calculator.py:2 -bundle:989 `return result` -> calculator.py:3 -bundle:991 `def multiply(a, b):` -> calculator.py:6 -bundle:992 `result = a * b` -> calculator.py:7 -bundle:993 `return result` -> calculator.py:8 -bundle:998 `def describe(name, value):` -> utils.py:1 -bundle:999 `return f"{name} = {value}"` -> utils.py:2 -bundle:1005 `total = add(2, 3)` -> main.py:4 -bundle:1006 `product = multiply(total, 4)` -> main.py:5 -bundle:1007 `print(describe("total", total))` -> main.py:6 -bundle:1008 `print(describe("product", product))` -> main.py:7 +bundle:998 `def add(a, b):` -> calculator.py:1 +bundle:999 `result = a + b` -> calculator.py:2 +bundle:1000 `return result` -> calculator.py:3 +bundle:1002 `def multiply(a, b):` -> calculator.py:6 +bundle:1003 `result = a * b` -> calculator.py:7 +bundle:1004 `return result` -> calculator.py:8 +bundle:1009 `def describe(name, value):` -> utils.py:1 +bundle:1010 `return f"{name} = {value}"` -> utils.py:2 +bundle:1016 `total = add(2, 3)` -> main.py:4 +bundle:1017 `product = multiply(total, 4)` -> main.py:5 +bundle:1018 `print(describe("total", total))` -> main.py:6 +bundle:1019 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index 8271d47d9..ac5ed393c 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:996 `print("effects module loading")` -> effects.py:1 -bundle:997 `COUNTER = 1` -> effects.py:3 -bundle:1000 `def boost(value):` -> effects.py:6 -bundle:1001 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:1002 `return boosted` -> effects.py:8 -bundle:1014 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:1015 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:1007 `print("effects module loading")` -> effects.py:1 +bundle:1008 `COUNTER = 1` -> effects.py:3 +bundle:1011 `def boost(value):` -> effects.py:6 +bundle:1012 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:1013 `return boosted` -> effects.py:8 +bundle:1025 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:1026 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index b9392dbc6..09afcf68e 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1652,3 +1652,116 @@ fn sources_with_url_delimiters_are_encoded_and_decoded() { "the original source line must load from the decoded path: {stderr}" ); } + +#[test] +fn digest_placeholder_in_user_code_is_left_untouched() { + // Only the prologue's own placeholder receives the digest; a user string + // that happens to share the spelling must survive verbatim, and the + // runtime must still verify (i.e. the prologue occurrence was the one + // substituted). + let dir = make_project(&[ + ( + "main.py", + "from helper import boom\n\nprint(\"__CRIBO_SOURCEMAP_DIGEST__\")\nboom()\n", + ), + ( + "helper.py", + "def boom():\n raise ValueError(\"kaboom\")\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert_eq!( + bundle.matches("__CRIBO_SOURCEMAP_DIGEST__").count(), + 1, + "exactly the user occurrence must remain (prologue one substituted)" + ); + + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(!ok); + assert!( + stdout.contains("__CRIBO_SOURCEMAP_DIGEST__"), + "user string must print unchanged: {stdout}" + ); + // Remapping proves the prologue digest matched the sibling map. + assert!( + stderr.contains("helper.py\", line 2, in boom"), + "traceback must remap, proving digest verification passed: {stderr}" + ); + assert!(stderr.contains("raise ValueError(\"kaboom\")"), "{stderr}"); +} + +#[cfg(unix)] +#[test] +fn linked_comment_encodes_non_utf8_names() { + // Unix filenames need not be UTF-8. A lossy conversion would bake U+FFFD + // into the sourceMappingURL comment — a name that does not exist on disk. + // The raw invalid byte must be percent-encoded instead. + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt as _; + + let dir = fixture_project(); + let mut name = b"bun".to_vec(); + name.push(0xFF); + name.extend_from_slice(b"dle.py"); + let out = dir.path().join(OsString::from_vec(name)); + let output = Command::new(env!("CARGO_BIN_EXE_cribo")) + .arg("--entry") + .arg(dir.path().join("main.py")) + .arg("--output") + .arg(&out) + .arg("--sourcemap=linked") + .output() + .expect("run cribo"); + assert!( + output.status.success(), + "bundling must succeed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let bundle = fs::read_to_string(&out).expect("read bundle"); + assert!( + bundle.contains("# sourceMappingURL=bun%FFdle.py.map"), + "invalid bytes must be percent-encoded, not replaced with U+FFFD" + ); + assert!( + !bundle.contains('\u{FFFD}'), + "no lossy replacement character may reach the comment" + ); + // The bundle must remain valid, runnable Python (with its sibling map). + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!(stdout.contains("hello world")); +} + +#[test] +fn bundle_namespace_is_clean_of_runtime_helpers() { + // The prologue must leave no helper names behind: user code and + // `from bundle import *` consumers must not see (or collide with) them. + let dir = make_project(&[( + "main.py", + "prologue_names = (\"_cribo_sys\", \"_cribo_sm_import\", \"_CriboSmStream\", \"_CriboSourceMapRuntime\")\nleaked = sorted(name for name in prologue_names if name in globals())\nprint(\"LEAKED:\", leaked)\n", + )]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=inline", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let (ok, stdout, stderr) = run_python(&out, &[]); + assert!(ok, "bundle must run: {stderr}"); + assert!( + stdout.contains("LEAKED: []"), + "runtime helper names must not leak into the bundle namespace: {stdout}" + ); +} diff --git a/docs/source-maps.md b/docs/source-maps.md index 180ac05d5..c2dde216b 100644 --- a/docs/source-maps.md +++ b/docs/source-maps.md @@ -19,6 +19,9 @@ tracebacks to original sources at run time, analogous to `node --enable-source-m `linked`). - `linked`: write `.map` next to the output and append a `# sourceMappingURL=.map` comment as the last line. + A newly created map file is owner-only (`0600` on Unix, set atomically at + creation) since it may embed `sourcesContent`; rebuilds preserve whatever + permissions the existing map carries, so a one-time `chmod` sticks. - `inline`: append a `# sourceMappingURL=data:application/json;base64,...` comment. - `external`: write the `.map` file with no comment. 3. **Runtime traceback injection** is bundled into the output whenever `--sourcemap` From e99322a2af539d964cdf0dd65ac4ab96743b09e3 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Tue, 25 Aug 2026 22:11:52 +0200 Subject: [PATCH 17/17] fix: address sixteenth review round on source map runtime - extract PEP 657 frame summaries with an explicit limit spanning the whole traceback: the traceback module honors a positive sys.tracebacklimit by keeping the first n frames while the renderer (like the C printer) keeps the last n, so an implicit limit misaligned summary indices and dropped caret anchors on retained unmapped frames - preserve the original identifier range when the inliner renames a function or class: the header anchor of a decorated 'class C:' (no argument list to fall back on) now maps, so exceptions CPython attributes to the class header land on original coordinates --- crates/cribo/src/code_generator/inliner.rs | 16 ++- crates/cribo/src/python/sourcemap_runtime.py | 16 ++- crates/cribo/src/source_map.rs | 9 +- .../bundled_code@sourcemap_basic.snap | 9 +- .../bundled_code@sourcemap_wrapper.snap | 9 +- .../snapshots/source_map@sourcemap_basic.snap | 24 ++-- .../source_map@sourcemap_wrapper.snap | 14 +-- crates/cribo/tests/test_source_maps.rs | 104 +++++++++++++++++- 8 files changed, 167 insertions(+), 34 deletions(-) diff --git a/crates/cribo/src/code_generator/inliner.rs b/crates/cribo/src/code_generator/inliner.rs index 245af355c..f8e8d2ce3 100644 --- a/crates/cribo/src/code_generator/inliner.rs +++ b/crates/cribo/src/code_generator/inliner.rs @@ -265,9 +265,12 @@ impl Bundler<'_> { module_renames.insert(func_name.clone(), renamed_name.clone()); ctx.global_symbols.insert(renamed_name.clone()); - // Clone and rename the function + // Clone and rename the function, preserving the original + // identifier's range for source-map provenance (see the + // class rename below for the rationale). + let original_name_range = func_def.name.range; let mut func_def_clone = func_def.clone(); - func_def_clone.name = Identifier::new(renamed_name, TextRange::default()); + func_def_clone.name = Identifier::new(renamed_name, original_name_range); // Apply renames to function annotations (parameters and return type) if let Some(ref mut returns) = func_def_clone.returns { @@ -626,9 +629,14 @@ impl Bundler<'_> { module_renames.insert(class_name.clone(), renamed_name.clone()); ctx.global_symbols.insert(renamed_name.clone()); - // Clone and rename the class + // Clone and rename the class. The original identifier's range is + // preserved on the renamed name: these ranges point into the original + // module source and feed source-map provenance (the code generator + // never reads them), and the name is the only header token that + // exists on every class form (`class C:` has no argument list). + let original_name_range = class_def.name.range; let mut class_def_clone = class_def.clone(); - class_def_clone.name = Identifier::new(renamed_name.clone(), TextRange::default()); + class_def_clone.name = Identifier::new(renamed_name.clone(), original_name_range); // Apply renames to base classes and keyword arguments // CRITICAL: For cross-module inheritance, we need to apply renames from the diff --git a/crates/cribo/src/python/sourcemap_runtime.py b/crates/cribo/src/python/sourcemap_runtime.py index 47e75620c..605017e02 100644 --- a/crates/cribo/src/python/sourcemap_runtime.py +++ b/crates/cribo/src/python/sourcemap_runtime.py @@ -1022,10 +1022,22 @@ def _render( limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): # Standard per-frame summaries (with PEP 657 positions on - # 3.11+) render the frames this runtime does not remap. + # 3.11+) render the frames this runtime does not remap. The + # extraction passes an explicit limit spanning the whole + # traceback: left implicit, the traceback module would honor a + # positive sys.tracebacklimit by keeping the FIRST n frames, + # while this renderer (like the interpreter's C printer) keeps + # the LAST n — the truncated list would misalign or fall out + # of range against raw traceback indices in _write_frames, + # which applies the real limit itself. + total_frames = 0 + probe = tb + while probe is not None: + total_frames += 1 + probe = probe.tb_next try: summaries = self._traceback.TracebackException( - _type(exc), exc, tb, lookup_lines=False + _type(exc), exc, tb, limit=total_frames, lookup_lines=False ).stack except _bex: summaries = None diff --git a/crates/cribo/src/source_map.rs b/crates/cribo/src/source_map.rs index a9248fff4..8f3265ac0 100644 --- a/crates/cribo/src/source_map.rs +++ b/crates/cribo/src/source_map.rs @@ -368,9 +368,12 @@ impl ParallelWalker<'_> { orig_anchor.map(|range| (g.name.range(), range, o.node_index().load())) } (Stmt::ClassDef(g), Stmt::ClassDef(o)) => { - // The inliner regenerates inlined class names with a - // default range, so fall back to the base-class / - // metaclass argument list, which also sits on the header. + // The inliner preserves the original identifier range on + // renamed class names, so the name anchor holds even for + // `class C:` headers with no argument list. The + // base-class / metaclass argument list stays as a + // fallback for any other transformation that regenerates + // the identifier with a synthetic (default) range. let orig_anchor = Some(o.name.range()) .filter(|range| o.range().contains(range.start())) .or_else(|| { diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap index 1359b3167..38f2fb2c8 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_basic.snap @@ -743,8 +743,13 @@ class _CriboSourceMapRuntime(object): tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): + total_frames = 0 + probe = tb + while probe is not None: + total_frames += 1 + probe = probe.tb_next try: - summaries = self._traceback.TracebackException(_type(exc), exc, tb, lookup_lines=False).stack + summaries = self._traceback.TracebackException(_type(exc), exc, tb, limit=total_frames, lookup_lines=False).stack except _bex: summaries = None write("Traceback (most recent call last):\n") @@ -872,7 +877,7 @@ class _CriboSourceMapRuntime(object): self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook", "_default_unraisablehook") return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "49c287797b013f17ab9e979132db7215f3eb9d0f7c3299d27043ab12f15a2a3b") +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "c36dd7f51595374bafef64709ded727675b756b31a07e0e1dd59bfdc790bf228") del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime import sys as _sys import importlib as _importlib diff --git a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap index 16c570068..05751ee45 100644 --- a/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/bundled_code@sourcemap_wrapper.snap @@ -743,8 +743,13 @@ class _CriboSourceMapRuntime(object): tb = _getattr(exc, "__traceback__", None) limit = self._effective_tb_limit() if tb is not None and (limit is None or limit > 0): + total_frames = 0 + probe = tb + while probe is not None: + total_frames += 1 + probe = probe.tb_next try: - summaries = self._traceback.TracebackException(_type(exc), exc, tb, lookup_lines=False).stack + summaries = self._traceback.TracebackException(_type(exc), exc, tb, limit=total_frames, lookup_lines=False).stack except _bex: summaries = None write("Traceback (most recent call last):\n") @@ -872,7 +877,7 @@ class _CriboSourceMapRuntime(object): self._notify_custom_hook(self._prev_unraisablehook, self._default_unraisablehook, lambda hook: hook(unraisable), "_prev_unraisablehook", "_default_unraisablehook") return self._prev_unraisablehook(unraisable) -_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "e892aae89904ceb493d1b2e4def1f0b19e81b73861a30ac51c7c6f8a2f5d395f") +_CriboSourceMapRuntime._bootstrap("linked", globals().get("__file__", ""), "d1929b8451a8662aa287cb9620ed4718dbb5db55abe122d15d965adbc4bdc823") del _cribo_sys, _cribo_sm_import, _CriboSmStream, _CriboSourceMapRuntime import sys as _sys import importlib as _importlib diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap index e699daffb..91f143d0b 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_basic.snap @@ -2,15 +2,15 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_basic/main.py --- -bundle:998 `def add(a, b):` -> calculator.py:1 -bundle:999 `result = a + b` -> calculator.py:2 -bundle:1000 `return result` -> calculator.py:3 -bundle:1002 `def multiply(a, b):` -> calculator.py:6 -bundle:1003 `result = a * b` -> calculator.py:7 -bundle:1004 `return result` -> calculator.py:8 -bundle:1009 `def describe(name, value):` -> utils.py:1 -bundle:1010 `return f"{name} = {value}"` -> utils.py:2 -bundle:1016 `total = add(2, 3)` -> main.py:4 -bundle:1017 `product = multiply(total, 4)` -> main.py:5 -bundle:1018 `print(describe("total", total))` -> main.py:6 -bundle:1019 `print(describe("product", product))` -> main.py:7 +bundle:1003 `def add(a, b):` -> calculator.py:1 +bundle:1004 `result = a + b` -> calculator.py:2 +bundle:1005 `return result` -> calculator.py:3 +bundle:1007 `def multiply(a, b):` -> calculator.py:6 +bundle:1008 `result = a * b` -> calculator.py:7 +bundle:1009 `return result` -> calculator.py:8 +bundle:1014 `def describe(name, value):` -> utils.py:1 +bundle:1015 `return f"{name} = {value}"` -> utils.py:2 +bundle:1021 `total = add(2, 3)` -> main.py:4 +bundle:1022 `product = multiply(total, 4)` -> main.py:5 +bundle:1023 `print(describe("total", total))` -> main.py:6 +bundle:1024 `print(describe("product", product))` -> main.py:7 diff --git a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap index ac5ed393c..9b0147233 100644 --- a/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap +++ b/crates/cribo/tests/snapshots/source_map@sourcemap_wrapper.snap @@ -2,10 +2,10 @@ source: crates/cribo/tests/test_bundling_snapshots.rs input_file: crates/cribo/tests/fixtures/sourcemap_wrapper/main.py --- -bundle:1007 `print("effects module loading")` -> effects.py:1 -bundle:1008 `COUNTER = 1` -> effects.py:3 -bundle:1011 `def boost(value):` -> effects.py:6 -bundle:1012 `boosted = value * 2 + COUNTER` -> effects.py:7 -bundle:1013 `return boosted` -> effects.py:8 -bundle:1025 `print("counter:", effects.COUNTER)` -> main.py:3 -bundle:1026 `print("boosted:", effects.boost(10))` -> main.py:4 +bundle:1012 `print("effects module loading")` -> effects.py:1 +bundle:1013 `COUNTER = 1` -> effects.py:3 +bundle:1016 `def boost(value):` -> effects.py:6 +bundle:1017 `boosted = value * 2 + COUNTER` -> effects.py:7 +bundle:1018 `return boosted` -> effects.py:8 +bundle:1030 `print("counter:", effects.COUNTER)` -> main.py:3 +bundle:1031 `print("boosted:", effects.boost(10))` -> main.py:4 diff --git a/crates/cribo/tests/test_source_maps.rs b/crates/cribo/tests/test_source_maps.rs index 09afcf68e..28c14c073 100644 --- a/crates/cribo/tests/test_source_maps.rs +++ b/crates/cribo/tests/test_source_maps.rs @@ -1112,8 +1112,8 @@ fn map_covers_match_case_headers_and_decorators() { // 0-based original lines: 4 and 5 are the def decorators; 6 is the // decorated `def run(value):` header itself; 8, 10, and 12 are the `case` // headers; 16 and 17 are the decorated class's decorator and header (the - // inliner regenerates class names with synthetic ranges, so the header - // anchor falls back to the base-class list). + // inliner preserves the original name range on renamed identifiers, and + // the base-class list serves as a fallback anchor). for header_line in [4, 5, 6, 8, 10, 12, 16, 17] { assert!( mapped_helper_lines.contains(&header_line), @@ -1765,3 +1765,103 @@ fn bundle_namespace_is_clean_of_runtime_helpers() { "runtime helper names must not leak into the bundle namespace: {stdout}" ); } + +#[test] +fn map_covers_decorated_class_without_argument_list() { + // `class Widget:` has no base-class list to fall back on, and the inliner + // rewrites every inlined class name — the header anchor must survive via + // the preserved identifier range, or exceptions CPython attributes to the + // class header during construction (e.g. a descriptor raising from + // `__set_name__`) stay on bundle coordinates. + let dir = make_project(&[ + ( + "main.py", + "from helper import Widget\n\nprint(Widget().value)\n", + ), + ( + "helper.py", + "def trace(cls):\n return cls\n\n\n@trace\nclass Widget:\n def \ + __init__(self):\n self.value = \"widget value\"\n", + ), + ]); + let out = dir.path().join("bundle.py"); + let (ok, _, stderr) = run_cribo(&[ + "--entry", + &entry_arg(&dir), + "--output", + &out.to_string_lossy(), + "--sourcemap=linked", + ]); + assert!(ok, "bundling must succeed: {stderr}"); + let map_json = fs::read_to_string(dir.path().join("bundle.py.map")).expect("read map"); + let map = parse_map(&map_json); + + let helper_id = (0..map.get_sources().count() as u32) + .find(|id| { + map.get_source(*id) + .is_some_and(|s| s.ends_with("helper.py")) + }) + .expect("helper.py in sources"); + let mapped_helper_lines: Vec = map + .get_tokens() + .filter(|token| token.get_source_id() == Some(helper_id)) + .map(|token| token.get_src_line()) + .collect(); + // 0-based: 4 is the `@trace` decorator, 5 the bare `class Widget:` header. + for header_line in [4, 5] { + assert!( + mapped_helper_lines.contains(&header_line), + "helper.py 0-based line {header_line} (decorator or bare class header) must be \ + mapped; mapped lines: {mapped_helper_lines:?}" + ); + } +} + +#[test] +fn truncated_traceback_keeps_pep657_on_unmapped_frames() { + // With a positive sys.tracebacklimit the runtime keeps the LAST n frames + // (like the interpreter's C printer), while the traceback module keeps + // the FIRST n. The PEP 657 summaries must therefore be extracted + // untruncated, or a retained unmapped frame would index past (or into the + // wrong slot of) the shortened list and lose its caret anchors. + let dir = make_project(&[ + ( + "main.py", + "import importlib.util\nimport os\n\nfrom helper import boom\n\n__import__(\"sys\")\ + .tracebacklimit = 2\nspec = importlib.util.spec_from_file_location(\"ext\", \ + os.path.join(os.path.dirname(os.path.abspath(__file__)), \"ext.py\"))\next = \ + importlib.util.module_from_spec(spec)\nspec.loader.exec_module(ext)\nboom(ext)\n", + ), + ("helper.py", "def boom(ext):\n ext.go()\n"), + ]); + let bundle = bundle_crash_project(&dir, "--sourcemap=linked"); + // Placed next to the bundle and loaded by explicit path at run time, so + // cribo never bundles it: its frames stay unmapped by design. + fs::write( + dir.path().join("ext.py"), + "def go():\n left = 1\n return left + \"boom\"\n", + ) + .expect("write ext.py"); + + let (ok, _, stderr) = run_python(&bundle, &[]); + assert!(!ok); + // Retained frames: helper.boom (remapped) and ext.go (unmapped). + assert!( + stderr.contains("helper.py\", line 2, in boom"), + "the retained bundle frame must remap: {stderr}" + ); + assert!( + stderr.contains("ext.py\", line 3, in go"), + "the retained external frame must render: {stderr}" + ); + assert!( + !stderr.contains("main.py"), + "tracebacklimit = 2 must drop the module frame: {stderr}" + ); + // The caret line proves the frame came from the standard summaries (the + // plain fallback prints only the file line and source text). + assert!( + stderr.contains('^'), + "the unmapped frame must keep its PEP 657 carets: {stderr}" + ); +}