Add -Zrmeta-content-svh, -Zrmeta-strip-spans, -Zrmeta-normalize-src-hash: byte-stable .rmeta under non-interface edits

Cargo.lock

@@ -4380,6 +4380,7 @@ dependencies = [
  "rustc_expand",
  "rustc_feature",
  "rustc_fs_util",
+ "rustc_hashes",
  "rustc_hir",
  "rustc_hir_pretty",
  "rustc_index",

compiler/rustc_interface/src/tests.rs

@@ -16,8 +16,9 @@ use rustc_session::config::{
     InstrumentCoverage, InstrumentMcount, InstrumentMcountOpts, InstrumentXRay, LinkSelfContained,
     LinkerPluginLto, LocationDetail, LtoCli, MirIncludeSpans, NextSolverConfig, Offload, Options,
     OutFileName, OutputType, OutputTypes, PAuthKey, PacRet, Passes, PatchableFunctionEntry,
-    Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion,
-    WasiExecModel, build_configuration, build_session_options, rustc_optgroups,
+    Polonius, ProcMacroExecutionStrategy, RmetaStripSpans, Strip, SwitchWithOptPath,
+    SymbolManglingVersion, WasiExecModel, build_configuration, build_session_options,
+    rustc_optgroups,
 };
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
@@ -877,6 +878,9 @@ fn test_unstable_options_tracking_hash() {
     tracked!(regparm, Some(3));
     tracked!(relax_elf_relocations, Some(true));
     tracked!(remap_cwd_prefix, Some(PathBuf::from("abc")));
+    tracked!(rmeta_content_svh, true);
+    tracked!(rmeta_normalize_src_hash, true);
+    tracked!(rmeta_strip_spans, RmetaStripSpans::All);
     tracked!(sanitizer, SanitizerSet::ADDRESS);
     tracked!(sanitizer_cfi_canonical_jump_tables, None);
     tracked!(sanitizer_cfi_generalize_pointers, Some(true));

compiler/rustc_metadata/Cargo.toml

@@ -16,6 +16,7 @@ rustc_errors = { path = "../rustc_errors" }
 rustc_expand = { path = "../rustc_expand" }
 rustc_feature = { path = "../rustc_feature" }
 rustc_fs_util = { path = "../rustc_fs_util" }
+rustc_hashes = { path = "../rustc_hashes" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_index = { path = "../rustc_index" }

compiler/rustc_session/src/config.rs

@@ -637,6 +637,69 @@ impl FromStr for SplitDwarfKind {
     }
 }
 
+/// The value of `-Zrmeta-strip-spans`, controlling which spans are replaced by
+/// `DUMMY_SP` when encoding crate metadata.
+///
+/// Spans in crate metadata are encoded as byte offsets into this crate's
+/// source files, so any edit that shifts source positions (adding a comment, a
+/// blank line, ...) changes the encoded bytes of the `.rmeta` even when the
+/// crate's interface is unchanged. Build systems that content-address the
+/// `.rmeta` (to skip rebuilding dependents when it is unchanged) can opt into
+/// stripping those spans, trading diagnostic and debuginfo quality in
+/// *dependent* crates for byte-stability of the metadata. See the variants for
+/// exactly what each level gives up. Has no effect on proc-macro crates, whose
+/// metadata is dominated by span and hygiene data that consumers rely on.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum RmetaStripSpans {
+    /// Encode all spans faithfully (default).
+    None,
+    /// Strip spans everywhere except in the parts of metadata that dependent
+    /// crates compile into their own output: encoded MIR bodies (used for
+    /// cross-crate inlining, generic instantiation, and const evaluation),
+    /// hygiene expansion data, and the definition spans of items whose MIR is
+    /// exported (a dependent deriving debuginfo for an inlined function takes
+    /// its declaration file/line from the definition span; stripping it while
+    /// keeping body spans was verified to produce line rows bound to the
+    /// wrong file). What is given up: diagnostics reported in dependent
+    /// crates lose the ability to point into this crate's source for
+    /// item-level spans of items without exported MIR (e.g. "function defined
+    /// here" notes fall back to dummy spans).
+    ///
+    /// Stability boundary (measured, deliberate): because the preserved spans
+    /// are byte offsets and the referenced source files' length and line
+    /// tables must stay real for those spans to resolve correctly in
+    /// dependents, only edits that preserve byte positions (same-length
+    /// comment rewrites, same-length body edits) leave the metadata
+    /// byte-identical in this mode. Any length-changing edit, even a comment
+    /// appended at end of file, perturbs the source file record and thus the
+    /// metadata. Use `All` when byte-stability under general non-interface
+    /// edits is the goal.
+    NonExported,
+    /// Additionally strip spans inside encoded MIR bodies and hygiene
+    /// expansion data, and replace expansion hashes with span-independent
+    /// ones. What is given up on top of `NonExported`: debuginfo line
+    /// information in dependent crates for code inlined or instantiated from
+    /// this crate, const-eval error backtraces pointing into this crate, and
+    /// macro-backtrace notes for this crate's macros. In exchange, the
+    /// metadata bytes no longer depend on source positions at all, so
+    /// non-interface edits (comments, whitespace, private non-inlined
+    /// function bodies) leave the `.rmeta` byte-identical.
+    All,
+}
+
+impl FromStr for RmetaStripSpans {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, ()> {
+        Ok(match s {
+            "none" => RmetaStripSpans::None,
+            "non-exported" => RmetaStripSpans::NonExported,
+            "all" => RmetaStripSpans::All,
+            _ => return Err(()),
+        })
+    }
+}
+
 macro_rules! define_output_types {
     (
         $(
@@ -2736,6 +2799,16 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
 
     let incremental = cg.incremental.as_ref().map(PathBuf::from);
 
+    // `-Zrmeta-content-svh` derives the metadata SVH from the freshly encoded
+    // metadata bytes. With incremental compilation the metadata work product
+    // can be reused from a previous session without re-encoding, in which case
+    // there are no fresh bytes to hash (and the incremental system separately
+    // uses the HIR-based SVH to name session directories). Rather than
+    // silently mixing the two schemes, reject the combination.
+    if unstable_opts.rmeta_content_svh && incremental.is_some() {
+        early_dcx.early_fatal("option `-Z rmeta-content-svh` cannot be used with `-C incremental`");
+    }
+
     if cg.profile_generate.enabled() && cg.profile_use.is_some() {
         early_dcx.early_fatal("options `-C profile-generate` and `-C profile-use` are exclusive");
     }
@@ -3316,8 +3389,8 @@ pub(crate) mod dep_tracking {
         InstrumentMcountOpts, InstrumentXRay, LinkerPluginLto, LocationDetail, LtoCli,
         MirStripDebugInfo, NextSolverConfig, Offload, OptLevel, OutFileName, OutputType,
         OutputTypes, PatchableFunctionEntry, PointerAuthOption, Polonius, ResolveDocLinks,
-        SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion,
-        WasiExecModel,
+        RmetaStripSpans, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath,
+        SymbolManglingVersion, WasiExecModel,
     };
     use crate::lint;
     use crate::utils::NativeLib;
@@ -3402,6 +3475,7 @@ pub(crate) mod dep_tracking {
         Edition,
         LinkerPluginLto,
         ResolveDocLinks,
+        RmetaStripSpans,
         SplitDebuginfo,
         SplitDwarfKind,
         StackProtector,

compiler/rustc_session/src/options.rs

@@ -887,6 +887,7 @@ mod desc {
         "one of supported split-debuginfo modes (`off`, `packed`, or `unpacked`)";
     pub(crate) const parse_split_dwarf_kind: &str =
         "one of supported split dwarf modes (`split` or `single`)";
+    pub(crate) const parse_rmeta_strip_spans: &str = "one of `none`, `non-exported` or `all`";
     pub(crate) const parse_link_self_contained: &str = "one of: `y`, `yes`, `on`, `n`, `no`, `off`, or a list of enabled (`+` prefix) and disabled (`-` prefix) \
         components: `crto`, `libc`, `unwind`, `linker`, `sanitizers`, `mingw`";
     pub(crate) const parse_linker_features: &str =
@@ -1997,6 +1998,14 @@ pub mod parse {
         true
     }
 
+    pub(crate) fn parse_rmeta_strip_spans(slot: &mut RmetaStripSpans, v: Option<&str>) -> bool {
+        match v.and_then(|s| RmetaStripSpans::from_str(s).ok()) {
+            Some(e) => *slot = e,
+            _ => return false,
+        }
+        true
+    }
+
     pub(crate) fn parse_stack_protector(slot: &mut StackProtector, v: Option<&str>) -> bool {
         match v.and_then(|s| StackProtector::from_str(s).ok()) {
             Some(ssp) => *slot = ssp,
@@ -2792,6 +2801,23 @@ written to standard error output)"),
         "do not skip rigid aliases in normalization for internal debugging"),
     retpoline: bool = (false, parse_bool, [TRACKED] { TARGET_MODIFIER: Retpoline },
         "enables retpoline-indirect-branches and retpoline-indirect-calls target features (default: no)"),
+    rmeta_content_svh: bool = (false, parse_bool, [TRACKED],
+        "derive the SVH stored in crate metadata from the encoded metadata bytes themselves \
+        instead of from the HIR, so that it is stable exactly when the rest of the metadata is; \
+        incompatible with incremental compilation (default: no)"),
+    rmeta_normalize_src_hash: bool = (false, parse_bool, [TRACKED],
+        "replace per-file source content hashes in crate metadata with zeroes, so that the \
+        metadata does not vary with source file contents that no encoded item depends on; \
+        dependent crates lose cross-crate diagnostic source snippets and debuginfo file \
+        checksums for this crate (default: no)"),
+    rmeta_strip_spans: RmetaStripSpans = (RmetaStripSpans::None, parse_rmeta_strip_spans, [TRACKED],
+        "replace spans with dummy spans when encoding crate metadata, so that the encoded bytes \
+        do not depend on source positions; degrades diagnostics and (with `all`) debuginfo \
+        reported in dependent crates against this crate
+
+        `none`: encode all spans faithfully (default)
+        `non-exported`: strip spans except in encoded MIR bodies and hygiene expansion data
+        `all`: strip every span, including exported MIR and hygiene expansion data"),
     retpoline_external_thunk: bool = (false, parse_bool, [TRACKED] { TARGET_MODIFIER: RetpolineExternalThunk },
         "enables retpoline-external-thunk, retpoline-indirect-branches and retpoline-indirect-calls \
         target features (default: no)"),

compiler/rustc_span/src/hygiene.rs

@@ -158,6 +158,20 @@ impl ExpnHash {
     fn new(stable_crate_id: StableCrateId, local_hash: Hash64) -> ExpnHash {
         ExpnHash(Fingerprint::new(stable_crate_id.0, local_hash))
     }
+
+    /// Builds an [ExpnHash] from its parts, bypassing the usual computation
+    /// over the expansion's [ExpnData].
+    ///
+    /// This is only meant for `-Zrmeta-strip-spans=all` metadata encoding,
+    /// which replaces the real expansion hashes (whose inputs include source
+    /// positions) with hashes derived from the crate-local expansion index, so
+    /// that the encoded metadata does not vary with source positions.
+    /// `local_hash` must be unique within the crate identified by
+    /// `stable_crate_id`, and must be non-zero for non-root expansions so the
+    /// result cannot be confused with the root [ExpnHash].
+    pub fn from_parts(stable_crate_id: StableCrateId, local_hash: Hash64) -> ExpnHash {
+        ExpnHash::new(stable_crate_id, local_hash)
+    }
 }
 
 /// A property of a macro expansion that determines how identifiers

compiler/rustc_span/src/lib.rs

@@ -1761,6 +1761,18 @@ impl Display for SourceFileHash {
 }
 
 impl SourceFileHash {
+    /// Returns a `SourceFileHash` of the given `kind` with an all-zero value.
+    ///
+    /// This is only meant for `-Zrmeta-normalize-src-hash`, which replaces
+    /// the per-file content hashes recorded in crate metadata so that they do
+    /// not vary with file contents. An all-zero value does not match the hash
+    /// of any actual source file (up to hash collisions), so consumers that
+    /// verify on-disk source against the recorded hash treat the source as
+    /// unavailable rather than silently using stale source.
+    pub fn zeroed(kind: SourceFileHashAlgorithm) -> SourceFileHash {
+        SourceFileHash { kind, value: [0; 32] }
+    }
+
     pub fn new_in_memory(kind: SourceFileHashAlgorithm, src: impl AsRef<[u8]>) -> SourceFileHash {
         let mut hash = SourceFileHash { kind, value: Default::default() };
         let len = hash.hash_len();

src/doc/unstable-book/src/compiler-flags/rmeta-content-svh.md

@@ -0,0 +1,23 @@
+# `rmeta-content-svh`
+
+---
+
+This flag derives the strict version hash (SVH) stored in crate metadata from
+the encoded metadata bytes themselves, instead of from the crate's HIR.
+
+The default HIR-based SVH covers essentially the whole crate, including every
+function body and (indirectly) source positions, so any edit to the crate
+changes the SVH and with it the metadata, even when nothing a dependent crate
+consumes has changed. With this flag, the SVH is a hash of everything else in
+the metadata file (plus the session's dependency-tracking hash and the stable
+crate id), so it is stable exactly when the rest of the metadata is. Combined
+with `-Zrmeta-strip-spans` and `-Zrmeta-normalize-src-hash`, this makes the
+`.rmeta` byte-identical across rebuilds after non-interface edits, which lets
+content-addressed build systems skip rebuilding dependents.
+
+The link-time consistency check is unaffected: the `.rmeta` and `.rlib`
+produced by one compiler invocation embed the same metadata and therefore the
+same SVH, and the crate loader continues to compare SVHs for equality.
+
+This flag is rejected in combination with `-Cincremental`, and it does not
+change the `crate_hash` query used within the compiling session.

src/doc/unstable-book/src/compiler-flags/rmeta-normalize-src-hash.md

@@ -0,0 +1,26 @@
+# `rmeta-normalize-src-hash`
+
+---
+
+This flag replaces the per-file source content hashes recorded in crate
+metadata with zeroes. The recorded hash covers a source file's entire raw
+text, so by default any edit to a file referenced from metadata, even a
+comment edit that moves no token, changes the crate's metadata bytes.
+
+Costs, per consumer of the recorded hash:
+
+* Cross-crate diagnostic snippets: a dependent crate verifies this crate's
+  on-disk source against the recorded hash before quoting it in diagnostics.
+  The zero hash never matches, so such diagnostics degrade to plain
+  `file:line:col` references without a quoted snippet (rather than risking
+  quoting stale source).
+* Debuginfo: file checksums recorded for this crate's files in dependent
+  crates' debug info become zero, so debuggers cannot detect source
+  staleness for them.
+
+On its own, this flag stabilizes the metadata against comment edits that
+preserve byte length and line positions. For general comment, whitespace,
+and private function body edits, combine it with `-Zrmeta-strip-spans=all`
+and `-Zrmeta-content-svh`.
+
+The flag has no effect on proc-macro crates.

src/doc/unstable-book/src/compiler-flags/rmeta-strip-spans.md

@@ -0,0 +1,52 @@
+# `rmeta-strip-spans`
+
+---
+
+This flag replaces spans with dummy spans when encoding crate metadata, so
+that the encoded bytes do not depend on source positions. Spans in metadata
+are byte offsets into this crate's source files, so by default any edit that
+shifts source text (adding a comment, a blank line, editing a function body)
+changes the metadata of the crate even when its interface is unchanged.
+
+* `-Zrmeta-strip-spans=none` (default): encode all spans faithfully.
+* `-Zrmeta-strip-spans=non-exported`: strip spans except in the metadata that
+  dependent crates compile into their own output: encoded MIR bodies (used
+  for cross-crate inlining, generic instantiation, and const evaluation),
+  hygiene expansion data, and the definition spans of items whose MIR is
+  exported. The definition spans must accompany the body spans: a dependent
+  crate derives an inlined function's debuginfo declaration file/line from
+  the definition span and binds the inlined line rows to that file, so
+  stripping it while keeping body spans yields line rows bound to the wrong
+  file. Cost: diagnostics reported in dependent crates can no longer point
+  into this crate's source for item-level spans of items without exported
+  MIR ("function defined here" style notes degrade to dummy spans).
+  Stability boundary: because the preserved spans are byte offsets and the
+  referenced files' length and line tables must stay real for those spans
+  to resolve correctly in dependents, only byte-position-preserving edits
+  (same-length comment rewrites, same-length body edits of non-exported
+  functions) leave the metadata byte-identical in this mode; any
+  length-changing edit, even a comment appended at end of file, perturbs
+  the encoded source file record. This mode is for toolchains that want
+  the SVH and source-hash normalization plus reduced item-span surface
+  with zero debuginfo cost; use `all` when byte-stability under general
+  non-interface edits is the goal.
+* `-Zrmeta-strip-spans=all`: additionally strip spans inside exported MIR and
+  hygiene data, and replace expansion hashes with span-independent ones.
+  Additional cost: debuginfo line information in dependent crates for code
+  inlined or instantiated from this crate, const-eval error backtraces
+  pointing into this crate, and macro-backtrace notes for this crate's
+  macros. Concretely, the declaration debuginfo a dependent crate emits for
+  a function inlined from this crate carries the dummy span, which resolves
+  to the dependent's own first source file at line 1 instead of this
+  crate's source; call-site line information in the dependent is
+  unaffected. In exchange the metadata does not depend on source positions at
+  all: with `-Zrmeta-content-svh` and `-Zrmeta-normalize-src-hash` the
+  `.rmeta` is byte-identical after comment, whitespace, and private
+  non-inlinable function body edits.
+
+Interface-relevant changes (signatures, visibility, adding or removing items,
+and bodies of generic or inlinable functions, whose MIR is exported) still
+change the metadata under every mode.
+
+The flag has no effect on proc-macro crates: their metadata consists largely
+of span and hygiene data that dependent crates rely on for expansion.