From b946278601d31bd237629155d8c233dbb8cebb00 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 07:34:56 +0200 Subject: [PATCH 01/25] new crates zip reader --- Cargo.lock | 14 +++ Cargo.toml | 2 + crates/lib/docs_rs_crate_zip/Cargo.toml | 18 +++ .../lib/docs_rs_crate_zip/examples/fetch.rs | 31 +++++ crates/lib/docs_rs_crate_zip/src/error.rs | 22 ++++ crates/lib/docs_rs_crate_zip/src/lib.rs | 7 ++ crates/lib/docs_rs_crate_zip/src/manifest.rs | 24 ++++ .../docs_rs_crate_zip/src/source_archive.rs | 119 ++++++++++++++++++ crates/lib/docs_rs_storage/Cargo.toml | 4 +- 9 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 crates/lib/docs_rs_crate_zip/Cargo.toml create mode 100644 crates/lib/docs_rs_crate_zip/examples/fetch.rs create mode 100644 crates/lib/docs_rs_crate_zip/src/error.rs create mode 100644 crates/lib/docs_rs_crate_zip/src/lib.rs create mode 100644 crates/lib/docs_rs_crate_zip/src/manifest.rs create mode 100644 crates/lib/docs_rs_crate_zip/src/source_archive.rs diff --git a/Cargo.lock b/Cargo.lock index 0810f40d8..e07314edd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2108,6 +2108,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "docs_rs_crate_zip" +version = "0.1.0" +dependencies = [ + "async-compression", + "docs_rs_utils", + "futures-util", + "reqwest", + "serde", + "thiserror", + "tokio", + "tokio-util", +] + [[package]] name = "docs_rs_database" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 5ac02140f..206f5fab6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,9 @@ repository = "https://github.com/rust-lang/docs.rs" edition = "2024" [workspace.dependencies] +async-compression = { version = "0.4.32", features = ["bzip2", "deflate", "gzip", "tokio", "zstd"] } anyhow = { version = "1.0.42", features = ["backtrace"] } +tokio-util = { version = "0.7.15", default-features = false, features = ["io-util"] } askama = "0.16.0" async-stream = "0.3.5" axum-extra = { version = "0.12.0", features = ["middleware", "routing", "typed-header"] } diff --git a/crates/lib/docs_rs_crate_zip/Cargo.toml b/crates/lib/docs_rs_crate_zip/Cargo.toml new file mode 100644 index 000000000..5dadc21ad --- /dev/null +++ b/crates/lib/docs_rs_crate_zip/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "docs_rs_crate_zip" +description = "read specific files from the crates.io source zip archive" +version = "0.1.0" +edition = "2024" + +[dependencies] +# async-compression = { version = "0.4.42", features = ["deflate", "tokio"] } +async-compression = { workspace = true } +# futures-util = { version = "0.3.32", default-features = false, features = ["std"] } +futures-util = { workspace = true } +# reqwest = { version = "0.13.4", default-features = false, features = ["json", "stream", "rustls"] } +reqwest = { workspace = true } +serde = { workspace = true } +thiserror = "2.0.18" +tokio = { workspace = true } +tokio-util = { workspace = true } +docs_rs_utils = { path = "../docs_rs_utils" } diff --git a/crates/lib/docs_rs_crate_zip/examples/fetch.rs b/crates/lib/docs_rs_crate_zip/examples/fetch.rs new file mode 100644 index 000000000..3b5f97290 --- /dev/null +++ b/crates/lib/docs_rs_crate_zip/examples/fetch.rs @@ -0,0 +1,31 @@ +use fetch_from_crate::SourceArchive; +use tokio::io; + +#[tokio::main] +async fn main() { + let mut args = std::env::args().skip(1); + let (name, version) = match (args.next(), args.next()) { + (Some(name), Some(version)) => (name, version), + _ => { + eprintln!("usage: fetch [file]"); + return; + } + }; + let file = args.next().unwrap_or_else(|| "Cargo.toml".to_string()); + + let archive = match SourceArchive::load(&name, &version).await { + Ok(archive) => archive, + Err(err) => { + eprintln!("error loading: {:?}", err); + return; + } + }; + + let entry = archive + .by_name(&file) + .unwrap_or_else(|| panic!("no {file} in archive")); + + if let Err(err) = archive.fetch(entry, &mut io::stdout()).await { + eprintln!("error fetching: {:?}", err); + } +} diff --git a/crates/lib/docs_rs_crate_zip/src/error.rs b/crates/lib/docs_rs_crate_zip/src/error.rs new file mode 100644 index 000000000..6df9e2878 --- /dev/null +++ b/crates/lib/docs_rs_crate_zip/src/error.rs @@ -0,0 +1,22 @@ +use reqwest::Url; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("manifest not found: {0}")] + ManifestNotFound(Url), + + #[error("source archive not found: {0}, range={1}-{2}")] + ArchiveNotFound(Url, u64, u64), + + #[error("url-parsing error")] + UrlParse, + + #[error("i/o error: {0}")] + Io(#[from] std::io::Error), + + #[error("reqwest error")] + Request(#[from] reqwest::Error), + + #[error("unsupported zip compression method: {0}")] + UnsupportedZipCompression(String), +} diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs new file mode 100644 index 000000000..40e2e67e2 --- /dev/null +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -0,0 +1,7 @@ +mod error; +mod manifest; +mod source_archive; + +pub use error::Error; +pub use manifest::{FileEntry, Manifest}; +pub use source_archive::SourceArchive; diff --git a/crates/lib/docs_rs_crate_zip/src/manifest.rs b/crates/lib/docs_rs_crate_zip/src/manifest.rs new file mode 100644 index 000000000..321d88f6f --- /dev/null +++ b/crates/lib/docs_rs_crate_zip/src/manifest.rs @@ -0,0 +1,24 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Manifest { + /// One entry per file in the zip, sorted alphabetically by path. + pub files: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileEntry { + /// Realtive path (without the leading `{name}-{version}/` component of + /// the tarball). + pub path: String, + /// Byte offset in the zip where this entry's compressed payload begins. + pub data_offset: u64, + /// Length of the compressed contents in bytes. + pub compressed_size: u64, + /// Length of the uncompressed contents in bytes. + pub uncompressed_size: u64, + /// How the payload is compressed: `"deflate"` or `"store"`. + pub compression: String, + /// Lowercase hex sha256 of the uncompressed contents. + pub sha256: String, +} diff --git a/crates/lib/docs_rs_crate_zip/src/source_archive.rs b/crates/lib/docs_rs_crate_zip/src/source_archive.rs new file mode 100644 index 000000000..488d6af73 --- /dev/null +++ b/crates/lib/docs_rs_crate_zip/src/source_archive.rs @@ -0,0 +1,119 @@ +use crate::{ + error::Error, + manifest::{FileEntry, Manifest}, +}; +use async_compression::tokio::bufread::DeflateDecoder; +use docs_rs_utils::APP_USER_AGENT; +use futures_util::TryStreamExt as _; +use reqwest::{ + IntoUrl, StatusCode, Url, + header::{HeaderValue, RANGE, USER_AGENT}, +}; +use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; +use tokio_util::io::StreamReader; + +pub struct SourceArchive { + manifest: Manifest, + zip_url: Url, + client: reqwest::Client, +} + +impl SourceArchive { + pub async fn load(name: impl AsRef, version: impl AsRef) -> Result { + Self::load_from("https://static.crates.io/", name, version).await + } + + pub async fn load_from( + base_url: impl IntoUrl, + name: impl AsRef, + version: impl AsRef, + ) -> Result { + let mut base_url = base_url.into_url().map_err(|_| Error::UrlParse)?; + base_url.set_path("crates/"); + + let index_url = base_url + .join(&format!( + "{0}/{0}-{1}.zip.json", + name.as_ref(), + version.as_ref() + )) + .map_err(|_| Error::UrlParse)?; + + let headers = vec![(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT))] + .into_iter() + .collect(); + + let client = reqwest::Client::builder() + .default_headers(headers) + .build()?; + + let response = client.get(index_url.clone()).send().await?; + if is_not_found_error(&response.status()) { + return Err(Error::ManifestNotFound(index_url)); + } + let response = response.error_for_status()?; + + Ok(Self { + manifest: response.json().await?, + zip_url: base_url + .join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref())) + .map_err(|_| Error::UrlParse)?, + client, + }) + } + + pub fn entries(&self) -> impl Iterator { + self.manifest.files.iter() + } + + pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { + let path = path.as_ref(); + self.manifest.files.iter().find(|e| e.path == path) + } + + pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<(), Error> + where + W: AsyncWrite + Unpin, + { + let range_start = entry.data_offset; + let range_end = entry.data_offset + entry.compressed_size - 1; + + let response = self + .client + .get(self.zip_url.clone()) + .header(RANGE, format!("bytes={range_start}-{range_end}",)) + .send() + .await?; + + if is_not_found_error(&response.status()) { + return Err(Error::ArchiveNotFound( + self.zip_url.clone(), + range_start, + range_end, + )); + } + let response = response.error_for_status()?; + + let stream = response.bytes_stream().map_err(std::io::Error::other); + let mut reader = StreamReader::new(stream); + + match entry.compression.as_str() { + "deflate" => { + let mut decoder = DeflateDecoder::new(reader); + io::copy(&mut decoder, writer).await?; + } + "store" => { + io::copy(&mut reader, writer).await?; + } + compression => return Err(Error::UnsupportedZipCompression(compression.into())), + } + + writer.flush().await?; + + Ok(()) + } +} + +fn is_not_found_error(status: &StatusCode) -> bool { + matches!(status, &StatusCode::NOT_FOUND | &StatusCode::FORBIDDEN) +} diff --git a/crates/lib/docs_rs_storage/Cargo.toml b/crates/lib/docs_rs_storage/Cargo.toml index 153de2489..07562bdd6 100644 --- a/crates/lib/docs_rs_storage/Cargo.toml +++ b/crates/lib/docs_rs_storage/Cargo.toml @@ -14,7 +14,7 @@ testing = [ [dependencies] anyhow = { workspace = true } -async-compression = { version = "0.4.32", features = ["bzip2", "deflate", "gzip", "tokio", "zstd"] } +async-compression = { workspace = true } async-stream = { workspace = true } # The default `rustls` feature pulls in the legacy hyper 0.14 + rustls 0.21 # stack via `aws-smithy-runtime/tls-rustls`, which includes the vulnerable @@ -51,7 +51,7 @@ strum = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tokio-util = { version = "0.7.15", default-features = false, features = ["io-util"] } +tokio-util = { workspace = true } tracing = { workspace = true } walkdir = { workspace = true } zip = { workspace = true } From 2277de768855ca7a38b0cf0eb97e09479eb7a6b6 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 07:46:09 +0200 Subject: [PATCH 02/25] sue readme --- Cargo.lock | 1 + crates/bin/docs_rs_web/Cargo.toml | 1 + .../docs_rs_web/src/handlers/crate_details.rs | 62 +++++++------------ 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e07314edd..780493640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2509,6 +2509,7 @@ dependencies = [ "docs_rs_cargo_metadata", "docs_rs_config", "docs_rs_context", + "docs_rs_crate_zip", "docs_rs_database", "docs_rs_env_vars", "docs_rs_headers", diff --git a/crates/bin/docs_rs_web/Cargo.toml b/crates/bin/docs_rs_web/Cargo.toml index f2d3d86fb..059b5c42d 100644 --- a/crates/bin/docs_rs_web/Cargo.toml +++ b/crates/bin/docs_rs_web/Cargo.toml @@ -29,6 +29,7 @@ constant_time_eq = "0.5.0" docs_rs_build_limits = { path = "../../lib/docs_rs_build_limits" } docs_rs_build_queue = { path = "../../lib/docs_rs_build_queue" } docs_rs_cargo_metadata = { path = "../../lib/docs_rs_cargo_metadata" } +docs_rs_crate_zip= { path = "../../lib/docs_rs_crate_zip" } docs_rs_config = { path = "../../lib/docs_rs_config" } docs_rs_context = { path = "../../lib/docs_rs_context" } docs_rs_database = { path = "../../lib/docs_rs_database" } diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index 67409ee41..d7eb35ca1 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -20,6 +20,7 @@ use axum::{ }; use chrono::{DateTime, Utc}; use docs_rs_cargo_metadata::{Dependency, ReleaseDependencyList}; +use docs_rs_crate_zip::{Error as CratesIoZipError, SourceArchive}; use docs_rs_database::crate_details::{Release, parse_doc_targets}; use docs_rs_headers::CanonicalUrl; use docs_rs_registry_api::OwnerKind; @@ -293,25 +294,20 @@ impl CrateDetails { } async fn fetch_readme(&self, storage: &AsyncStorage) -> anyhow::Result> { - let manifest = match storage - .fetch_source_file( - &self.name, - &self.version, - self.latest_build_id, - "Cargo.toml", - self.archive_storage, - ) - .await - { - Ok(manifest) => manifest, - Err(err) if err.is::() => { - return Ok(None); - } - Err(err) => { - return Err(err); - } + let source_archive = match SourceArchive::load(&self.name, self.version.to_string()).await { + Ok(source_archive) => source_archive, + Err(err) if matches!(err, CratesIoZipError::ManifestNotFound(_)) => return Ok(None), + Err(err) => return Err(err.into()), }; - let manifest = String::from_utf8(manifest.content) + + let Some(cargo_toml) = source_archive.by_name("Cargo.toml") else { + return Ok(None); + }; + + let mut manifest = Vec::new(); + source_archive.fetch(&cargo_toml, &mut manifest).await?; + + let manifest = String::from_utf8(manifest) .context("parsing Cargo.toml")? .parse::() .context("parsing Cargo.toml")?; @@ -322,27 +318,15 @@ impl CrateDetails { _ => vec!["README.md", "README.txt", "README"], }; for path in &paths { - match storage - .fetch_source_file( - &self.name, - &self.version, - self.latest_build_id, - path, - self.archive_storage, - ) - .await - { - Ok(readme) => { - let readme = String::from_utf8(readme.content) - .with_context(|| format!("parsing {path} content"))?; - return Ok(Some(readme)); - } - Err(err) if err.is::() => { - continue; - } - Err(err) => { - return Err(err); - } + let Some(readme) = source_archive.by_name(path) else { + continue; + }; + + let mut content = Vec::new(); + match source_archive.fetch(&readme, &mut content).await { + Ok(()) => return Ok(Some(String::from_utf8_lossy(&content).to_string())), + Err(err) if matches!(err, CratesIoZipError::ArchiveNotFound(_, _, _)) => continue, + Err(err) => return Err(err.into()), } } Ok(None) From 725fa74d5cf0a0175f2da44b7fb0c3ced81c6f39 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 07:48:28 +0200 Subject: [PATCH 03/25] fix --- .../docs_rs_web/src/handlers/crate_details.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index d7eb35ca1..e4c6ef319 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -14,23 +14,18 @@ use crate::{ }; use anyhow::{Context, Result}; use askama::Template; -use axum::{ - extract::Extension, - response::{IntoResponse, Response as AxumResponse}, -}; +use axum::response::{IntoResponse, Response as AxumResponse}; use chrono::{DateTime, Utc}; use docs_rs_cargo_metadata::{Dependency, ReleaseDependencyList}; use docs_rs_crate_zip::{Error as CratesIoZipError, SourceArchive}; use docs_rs_database::crate_details::{Release, parse_doc_targets}; use docs_rs_headers::CanonicalUrl; use docs_rs_registry_api::OwnerKind; -use docs_rs_storage::{AsyncStorage, PathNotFoundError}; use docs_rs_types::{ BuildId, BuildStatus, CrateId, Duration, KrateName, ReleaseId, ReqVersion, Version, }; use futures_util::stream::TryStreamExt; use serde_json::Value; -use std::sync::Arc; use tracing::warn; // TODO: Add target name and versions @@ -293,7 +288,7 @@ impl CrateDetails { Ok(Some(crate_details)) } - async fn fetch_readme(&self, storage: &AsyncStorage) -> anyhow::Result> { + async fn fetch_readme(&self) -> anyhow::Result> { let source_archive = match SourceArchive::load(&self.name, self.version.to_string()).await { Ok(source_archive) => source_archive, Err(err) if matches!(err, CratesIoZipError::ManifestNotFound(_)) => return Ok(None), @@ -434,10 +429,9 @@ impl_axum_webpage! { cpu_intensive_rendering = true, } -#[tracing::instrument(skip(conn, storage))] +#[tracing::instrument(skip(conn))] pub(crate) async fn crate_details_handler( params: RustdocParams, - Extension(storage): Extension>, mut conn: DbConnection, ) -> AxumResult { let matched_release = match_version(&mut conn, params.name(), params.req_version()) @@ -471,7 +465,7 @@ pub(crate) async fn crate_details_handler( // before we do the long S3 requests. drop(conn); - match details.fetch_readme(&storage).await { + match details.fetch_readme().await { Ok(readme) => details.readme = readme.or(details.readme), Err(e) => warn!(?e, "error fetching readme"), } @@ -1973,10 +1967,7 @@ mod tests { let mut conn = env.async_conn().await?; let details = crate_details(&mut conn, "dummy", "0.5.0", None).await; - assert!(matches!( - details.fetch_readme(env.storage()?).await, - Ok(None) - )); + assert!(matches!(details.fetch_readme().await, Ok(None))); Ok(()) }); } From 79a5f43efaed15faf9f9693090f336138bc6e434 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 07:54:36 +0200 Subject: [PATCH 04/25] more --- crates/bin/docs_rs_web/src/handlers/crate_details.rs | 12 ++++++------ crates/lib/docs_rs_crate_zip/src/source_archive.rs | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index e4c6ef319..d88439a08 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -299,8 +299,7 @@ impl CrateDetails { return Ok(None); }; - let mut manifest = Vec::new(); - source_archive.fetch(&cargo_toml, &mut manifest).await?; + let manifest = source_archive.fetch_bytes(&cargo_toml).await?; let manifest = String::from_utf8(manifest) .context("parsing Cargo.toml")? @@ -317,10 +316,11 @@ impl CrateDetails { continue; }; - let mut content = Vec::new(); - match source_archive.fetch(&readme, &mut content).await { - Ok(()) => return Ok(Some(String::from_utf8_lossy(&content).to_string())), - Err(err) if matches!(err, CratesIoZipError::ArchiveNotFound(_, _, _)) => continue, + match source_archive.fetch_bytes(&readme).await { + Ok(content) => return Ok(Some(String::from_utf8_lossy(&content).to_string())), + Err(err) if matches!(err, CratesIoZipError::ArchiveNotFound(_, _, _)) => { + return Ok(None); + } Err(err) => return Err(err.into()), } } diff --git a/crates/lib/docs_rs_crate_zip/src/source_archive.rs b/crates/lib/docs_rs_crate_zip/src/source_archive.rs index 488d6af73..771e50481 100644 --- a/crates/lib/docs_rs_crate_zip/src/source_archive.rs +++ b/crates/lib/docs_rs_crate_zip/src/source_archive.rs @@ -112,6 +112,12 @@ impl SourceArchive { Ok(()) } + + pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result, Error> { + let mut buf = Vec::new(); + self.fetch(entry, &mut buf).await?; + Ok(buf) + } } fn is_not_found_error(status: &StatusCode) -> bool { From 0eefa23583d8224dcc84478b726558d6d06b66cc Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 08:13:21 +0200 Subject: [PATCH 05/25] kk --- crates/lib/docs_rs_crate_zip/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs index 40e2e67e2..459bd93bd 100644 --- a/crates/lib/docs_rs_crate_zip/src/lib.rs +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -1,3 +1,10 @@ +//! Library to read the crates.io source archives (manifest & zip), and +//! fetch single files from the remote archives. +//! +//! Archives are created here: +//! https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs +//! Also we copied the manifest structs from there. + mod error; mod manifest; mod source_archive; From d95d10d9b2e51f3e4e34e7616663c52625943152 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 08:25:42 +0200 Subject: [PATCH 06/25] move --- crates/lib/docs_rs_crate_zip/src/lib.rs | 4 +-- crates/lib/docs_rs_crate_zip/src/manifest.rs | 24 -------------- .../docs_rs_crate_zip/src/source_archive.rs | 31 ++++++++++++++++--- 3 files changed, 28 insertions(+), 31 deletions(-) delete mode 100644 crates/lib/docs_rs_crate_zip/src/manifest.rs diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs index 459bd93bd..da04cf9cb 100644 --- a/crates/lib/docs_rs_crate_zip/src/lib.rs +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -6,9 +6,7 @@ //! Also we copied the manifest structs from there. mod error; -mod manifest; mod source_archive; pub use error::Error; -pub use manifest::{FileEntry, Manifest}; -pub use source_archive::SourceArchive; +pub use source_archive::{FileEntry, Manifest, SourceArchive}; diff --git a/crates/lib/docs_rs_crate_zip/src/manifest.rs b/crates/lib/docs_rs_crate_zip/src/manifest.rs deleted file mode 100644 index 321d88f6f..000000000 --- a/crates/lib/docs_rs_crate_zip/src/manifest.rs +++ /dev/null @@ -1,24 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Manifest { - /// One entry per file in the zip, sorted alphabetically by path. - pub files: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FileEntry { - /// Realtive path (without the leading `{name}-{version}/` component of - /// the tarball). - pub path: String, - /// Byte offset in the zip where this entry's compressed payload begins. - pub data_offset: u64, - /// Length of the compressed contents in bytes. - pub compressed_size: u64, - /// Length of the uncompressed contents in bytes. - pub uncompressed_size: u64, - /// How the payload is compressed: `"deflate"` or `"store"`. - pub compression: String, - /// Lowercase hex sha256 of the uncompressed contents. - pub sha256: String, -} diff --git a/crates/lib/docs_rs_crate_zip/src/source_archive.rs b/crates/lib/docs_rs_crate_zip/src/source_archive.rs index 771e50481..b7fc6d2c4 100644 --- a/crates/lib/docs_rs_crate_zip/src/source_archive.rs +++ b/crates/lib/docs_rs_crate_zip/src/source_archive.rs @@ -1,7 +1,4 @@ -use crate::{ - error::Error, - manifest::{FileEntry, Manifest}, -}; +use crate::error::Error; use async_compression::tokio::bufread::DeflateDecoder; use docs_rs_utils::APP_USER_AGENT; use futures_util::TryStreamExt as _; @@ -9,9 +6,35 @@ use reqwest::{ IntoUrl, StatusCode, Url, header::{HeaderValue, RANGE, USER_AGENT}, }; +use serde::{Deserialize, Serialize}; use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; use tokio_util::io::StreamReader; +/// archive manifest serde structs, copied from +/// https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Manifest { + /// One entry per file in the zip, sorted alphabetically by path. + pub files: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileEntry { + /// Realtive path (without the leading `{name}-{version}/` component of + /// the tarball). + pub path: String, + /// Byte offset in the zip where this entry's compressed payload begins. + pub data_offset: u64, + /// Length of the compressed contents in bytes. + pub compressed_size: u64, + /// Length of the uncompressed contents in bytes. + pub uncompressed_size: u64, + /// How the payload is compressed: `"deflate"` or `"store"`. + pub compression: String, + /// Lowercase hex sha256 of the uncompressed contents. + pub sha256: String, +} + pub struct SourceArchive { manifest: Manifest, zip_url: Url, From b2a31bdeeb4580b5c1a973ce9871c2bcf6ae83fa Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 08:38:28 +0200 Subject: [PATCH 07/25] kk --- Cargo.lock | 1 + .../docs_rs_web/src/handlers/crate_details.rs | 25 ++++------ crates/lib/docs_rs_crate_zip/Cargo.toml | 1 + crates/lib/docs_rs_crate_zip/src/error.rs | 22 --------- crates/lib/docs_rs_crate_zip/src/lib.rs | 2 - .../docs_rs_crate_zip/src/source_archive.rs | 48 +++++++------------ 6 files changed, 30 insertions(+), 69 deletions(-) delete mode 100644 crates/lib/docs_rs_crate_zip/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index 780493640..233aad4e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2112,6 +2112,7 @@ dependencies = [ name = "docs_rs_crate_zip" version = "0.1.0" dependencies = [ + "anyhow", "async-compression", "docs_rs_utils", "futures-util", diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index d88439a08..58817eaa2 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -17,7 +17,7 @@ use askama::Template; use axum::response::{IntoResponse, Response as AxumResponse}; use chrono::{DateTime, Utc}; use docs_rs_cargo_metadata::{Dependency, ReleaseDependencyList}; -use docs_rs_crate_zip::{Error as CratesIoZipError, SourceArchive}; +use docs_rs_crate_zip::SourceArchive; use docs_rs_database::crate_details::{Release, parse_doc_targets}; use docs_rs_headers::CanonicalUrl; use docs_rs_registry_api::OwnerKind; @@ -289,19 +289,17 @@ impl CrateDetails { } async fn fetch_readme(&self) -> anyhow::Result> { - let source_archive = match SourceArchive::load(&self.name, self.version.to_string()).await { - Ok(source_archive) => source_archive, - Err(err) if matches!(err, CratesIoZipError::ManifestNotFound(_)) => return Ok(None), - Err(err) => return Err(err.into()), + let Some(source_archive) = + SourceArchive::load(&self.name, self.version.to_string()).await? + else { + return Ok(None); }; let Some(cargo_toml) = source_archive.by_name("Cargo.toml") else { return Ok(None); }; - let manifest = source_archive.fetch_bytes(&cargo_toml).await?; - - let manifest = String::from_utf8(manifest) + let manifest = String::from_utf8(source_archive.fetch_bytes(cargo_toml).await?) .context("parsing Cargo.toml")? .parse::() .context("parsing Cargo.toml")?; @@ -311,18 +309,15 @@ impl CrateDetails { Some(toml::Value::String(path)) => vec![path.as_ref()], _ => vec!["README.md", "README.txt", "README"], }; + for path in &paths { let Some(readme) = source_archive.by_name(path) else { continue; }; - match source_archive.fetch_bytes(&readme).await { - Ok(content) => return Ok(Some(String::from_utf8_lossy(&content).to_string())), - Err(err) if matches!(err, CratesIoZipError::ArchiveNotFound(_, _, _)) => { - return Ok(None); - } - Err(err) => return Err(err.into()), - } + return Ok(Some( + String::from_utf8_lossy(&source_archive.fetch_bytes(readme).await?).to_string(), + )); } Ok(None) } diff --git a/crates/lib/docs_rs_crate_zip/Cargo.toml b/crates/lib/docs_rs_crate_zip/Cargo.toml index 5dadc21ad..a72096902 100644 --- a/crates/lib/docs_rs_crate_zip/Cargo.toml +++ b/crates/lib/docs_rs_crate_zip/Cargo.toml @@ -5,6 +5,7 @@ version = "0.1.0" edition = "2024" [dependencies] +anyhow = { workspace = true } # async-compression = { version = "0.4.42", features = ["deflate", "tokio"] } async-compression = { workspace = true } # futures-util = { version = "0.3.32", default-features = false, features = ["std"] } diff --git a/crates/lib/docs_rs_crate_zip/src/error.rs b/crates/lib/docs_rs_crate_zip/src/error.rs deleted file mode 100644 index 6df9e2878..000000000 --- a/crates/lib/docs_rs_crate_zip/src/error.rs +++ /dev/null @@ -1,22 +0,0 @@ -use reqwest::Url; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("manifest not found: {0}")] - ManifestNotFound(Url), - - #[error("source archive not found: {0}, range={1}-{2}")] - ArchiveNotFound(Url, u64, u64), - - #[error("url-parsing error")] - UrlParse, - - #[error("i/o error: {0}")] - Io(#[from] std::io::Error), - - #[error("reqwest error")] - Request(#[from] reqwest::Error), - - #[error("unsupported zip compression method: {0}")] - UnsupportedZipCompression(String), -} diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs index da04cf9cb..c7e05da40 100644 --- a/crates/lib/docs_rs_crate_zip/src/lib.rs +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -5,8 +5,6 @@ //! https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs //! Also we copied the manifest structs from there. -mod error; mod source_archive; -pub use error::Error; pub use source_archive::{FileEntry, Manifest, SourceArchive}; diff --git a/crates/lib/docs_rs_crate_zip/src/source_archive.rs b/crates/lib/docs_rs_crate_zip/src/source_archive.rs index b7fc6d2c4..533fb8026 100644 --- a/crates/lib/docs_rs_crate_zip/src/source_archive.rs +++ b/crates/lib/docs_rs_crate_zip/src/source_archive.rs @@ -1,4 +1,4 @@ -use crate::error::Error; +use anyhow::{Result, bail}; use async_compression::tokio::bufread::DeflateDecoder; use docs_rs_utils::APP_USER_AGENT; use futures_util::TryStreamExt as _; @@ -42,7 +42,7 @@ pub struct SourceArchive { } impl SourceArchive { - pub async fn load(name: impl AsRef, version: impl AsRef) -> Result { + pub async fn load(name: impl AsRef, version: impl AsRef) -> Result> { Self::load_from("https://static.crates.io/", name, version).await } @@ -50,17 +50,15 @@ impl SourceArchive { base_url: impl IntoUrl, name: impl AsRef, version: impl AsRef, - ) -> Result { - let mut base_url = base_url.into_url().map_err(|_| Error::UrlParse)?; + ) -> Result> { + let mut base_url = base_url.into_url()?; base_url.set_path("crates/"); - let index_url = base_url - .join(&format!( - "{0}/{0}-{1}.zip.json", - name.as_ref(), - version.as_ref() - )) - .map_err(|_| Error::UrlParse)?; + let index_url = base_url.join(&format!( + "{0}/{0}-{1}.zip.json", + name.as_ref(), + version.as_ref() + ))?; let headers = vec![(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT))] .into_iter() @@ -72,17 +70,15 @@ impl SourceArchive { let response = client.get(index_url.clone()).send().await?; if is_not_found_error(&response.status()) { - return Err(Error::ManifestNotFound(index_url)); + return Ok(None); } let response = response.error_for_status()?; - Ok(Self { + Ok(Some(Self { manifest: response.json().await?, - zip_url: base_url - .join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref())) - .map_err(|_| Error::UrlParse)?, + zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, client, - }) + })) } pub fn entries(&self) -> impl Iterator { @@ -94,7 +90,7 @@ impl SourceArchive { self.manifest.files.iter().find(|e| e.path == path) } - pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<(), Error> + pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> where W: AsyncWrite + Unpin, { @@ -106,16 +102,8 @@ impl SourceArchive { .get(self.zip_url.clone()) .header(RANGE, format!("bytes={range_start}-{range_end}",)) .send() - .await?; - - if is_not_found_error(&response.status()) { - return Err(Error::ArchiveNotFound( - self.zip_url.clone(), - range_start, - range_end, - )); - } - let response = response.error_for_status()?; + .await? + .error_for_status()?; let stream = response.bytes_stream().map_err(std::io::Error::other); let mut reader = StreamReader::new(stream); @@ -128,7 +116,7 @@ impl SourceArchive { "store" => { io::copy(&mut reader, writer).await?; } - compression => return Err(Error::UnsupportedZipCompression(compression.into())), + compression => bail!("unsupported zip compression: {}", compression), } writer.flush().await?; @@ -136,7 +124,7 @@ impl SourceArchive { Ok(()) } - pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result, Error> { + pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { let mut buf = Vec::new(); self.fetch(entry, &mut buf).await?; Ok(buf) From 8ee78bd3b068a319edb105ce91c79f7abae73353 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 08:45:04 +0200 Subject: [PATCH 08/25] fix --- Cargo.lock | 1 - Cargo.toml | 4 +- crates/bin/docs_rs_web/Cargo.toml | 2 +- crates/lib/docs_rs_crate_zip/Cargo.toml | 6 +- crates/lib/docs_rs_crate_zip/src/lib.rs | 136 +++++++++++++++++- .../docs_rs_crate_zip/src/source_archive.rs | 136 ------------------ 6 files changed, 138 insertions(+), 147 deletions(-) delete mode 100644 crates/lib/docs_rs_crate_zip/src/source_archive.rs diff --git a/Cargo.lock b/Cargo.lock index 233aad4e4..ffe2cf745 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2118,7 +2118,6 @@ dependencies = [ "futures-util", "reqwest", "serde", - "thiserror", "tokio", "tokio-util", ] diff --git a/Cargo.toml b/Cargo.toml index 206f5fab6..939b4c647 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,10 +27,9 @@ repository = "https://github.com/rust-lang/docs.rs" edition = "2024" [workspace.dependencies] -async-compression = { version = "0.4.32", features = ["bzip2", "deflate", "gzip", "tokio", "zstd"] } anyhow = { version = "1.0.42", features = ["backtrace"] } -tokio-util = { version = "0.7.15", default-features = false, features = ["io-util"] } askama = "0.16.0" +async-compression = { version = "0.4.32", features = ["bzip2", "deflate", "gzip", "tokio", "zstd"] } async-stream = "0.3.5" axum-extra = { version = "0.12.0", features = ["middleware", "routing", "typed-header"] } base64 = "0.22" @@ -64,6 +63,7 @@ tempfile = "3.1.0" test-case = "3.0.0" thiserror = "2.0.3" tokio = { version = "1.0", features = ["macros", "process", "rt-multi-thread", "signal", "sync"] } +tokio-util = { version = "0.7.15", default-features = false, features = ["io-util"] } toml = "1.0.0" tracing = "0.1.37" url = { version = "2.1.1", features = ["serde"] } diff --git a/crates/bin/docs_rs_web/Cargo.toml b/crates/bin/docs_rs_web/Cargo.toml index 059b5c42d..1f562e3cd 100644 --- a/crates/bin/docs_rs_web/Cargo.toml +++ b/crates/bin/docs_rs_web/Cargo.toml @@ -29,9 +29,9 @@ constant_time_eq = "0.5.0" docs_rs_build_limits = { path = "../../lib/docs_rs_build_limits" } docs_rs_build_queue = { path = "../../lib/docs_rs_build_queue" } docs_rs_cargo_metadata = { path = "../../lib/docs_rs_cargo_metadata" } -docs_rs_crate_zip= { path = "../../lib/docs_rs_crate_zip" } docs_rs_config = { path = "../../lib/docs_rs_config" } docs_rs_context = { path = "../../lib/docs_rs_context" } +docs_rs_crate_zip = { path = "../../lib/docs_rs_crate_zip" } docs_rs_database = { path = "../../lib/docs_rs_database" } docs_rs_env_vars = { path = "../../lib/docs_rs_env_vars" } docs_rs_headers = { path = "../../lib/docs_rs_headers" } diff --git a/crates/lib/docs_rs_crate_zip/Cargo.toml b/crates/lib/docs_rs_crate_zip/Cargo.toml index a72096902..024bb6e95 100644 --- a/crates/lib/docs_rs_crate_zip/Cargo.toml +++ b/crates/lib/docs_rs_crate_zip/Cargo.toml @@ -6,14 +6,10 @@ edition = "2024" [dependencies] anyhow = { workspace = true } -# async-compression = { version = "0.4.42", features = ["deflate", "tokio"] } async-compression = { workspace = true } -# futures-util = { version = "0.3.32", default-features = false, features = ["std"] } +docs_rs_utils = { path = "../docs_rs_utils" } futures-util = { workspace = true } -# reqwest = { version = "0.13.4", default-features = false, features = ["json", "stream", "rustls"] } reqwest = { workspace = true } serde = { workspace = true } -thiserror = "2.0.18" tokio = { workspace = true } tokio-util = { workspace = true } -docs_rs_utils = { path = "../docs_rs_utils" } diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs index c7e05da40..9802d6372 100644 --- a/crates/lib/docs_rs_crate_zip/src/lib.rs +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -5,6 +5,138 @@ //! https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs //! Also we copied the manifest structs from there. -mod source_archive; +use anyhow::{Result, bail}; +use async_compression::tokio::bufread::DeflateDecoder; +use docs_rs_utils::APP_USER_AGENT; +use futures_util::TryStreamExt as _; +use reqwest::{ + IntoUrl, StatusCode, Url, + header::{HeaderValue, RANGE, USER_AGENT}, +}; +use serde::{Deserialize, Serialize}; +use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; +use tokio_util::io::StreamReader; -pub use source_archive::{FileEntry, Manifest, SourceArchive}; +/// archive manifest serde structs, copied from +/// https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Manifest { + /// One entry per file in the zip, sorted alphabetically by path. + pub files: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileEntry { + /// Realtive path (without the leading `{name}-{version}/` component of + /// the tarball). + pub path: String, + /// Byte offset in the zip where this entry's compressed payload begins. + pub data_offset: u64, + /// Length of the compressed contents in bytes. + pub compressed_size: u64, + /// Length of the uncompressed contents in bytes. + pub uncompressed_size: u64, + /// How the payload is compressed: `"deflate"` or `"store"`. + pub compression: String, + /// Lowercase hex sha256 of the uncompressed contents. + pub sha256: String, +} + +pub struct SourceArchive { + manifest: Manifest, + zip_url: Url, + client: reqwest::Client, +} + +impl SourceArchive { + pub async fn load(name: impl AsRef, version: impl AsRef) -> Result> { + Self::load_from("https://static.crates.io/", name, version).await + } + + pub async fn load_from( + base_url: impl IntoUrl, + name: impl AsRef, + version: impl AsRef, + ) -> Result> { + let mut base_url = base_url.into_url()?; + base_url.set_path("crates/"); + + let index_url = base_url.join(&format!( + "{0}/{0}-{1}.zip.json", + name.as_ref(), + version.as_ref() + ))?; + + let headers = vec![(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT))] + .into_iter() + .collect(); + + let client = reqwest::Client::builder() + .default_headers(headers) + .build()?; + + let response = client.get(index_url.clone()).send().await?; + if matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::FORBIDDEN + ) { + return Ok(None); + } + let response = response.error_for_status()?; + + Ok(Some(Self { + manifest: response.json().await?, + zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, + client, + })) + } + + pub fn entries(&self) -> impl Iterator { + self.manifest.files.iter() + } + + pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { + let path = path.as_ref(); + self.manifest.files.iter().find(|e| e.path == path) + } + + pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> + where + W: AsyncWrite + Unpin, + { + let range_start = entry.data_offset; + let range_end = entry.data_offset + entry.compressed_size - 1; + + let response = self + .client + .get(self.zip_url.clone()) + .header(RANGE, format!("bytes={range_start}-{range_end}",)) + .send() + .await? + .error_for_status()?; + + let stream = response.bytes_stream().map_err(std::io::Error::other); + let mut reader = StreamReader::new(stream); + + match entry.compression.as_str() { + "deflate" => { + let mut decoder = DeflateDecoder::new(reader); + io::copy(&mut decoder, writer).await?; + } + "store" => { + io::copy(&mut reader, writer).await?; + } + compression => bail!("unsupported zip compression: {}", compression), + } + + writer.flush().await?; + + Ok(()) + } + + pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { + let mut buf = Vec::new(); + self.fetch(entry, &mut buf).await?; + Ok(buf) + } +} diff --git a/crates/lib/docs_rs_crate_zip/src/source_archive.rs b/crates/lib/docs_rs_crate_zip/src/source_archive.rs deleted file mode 100644 index 533fb8026..000000000 --- a/crates/lib/docs_rs_crate_zip/src/source_archive.rs +++ /dev/null @@ -1,136 +0,0 @@ -use anyhow::{Result, bail}; -use async_compression::tokio::bufread::DeflateDecoder; -use docs_rs_utils::APP_USER_AGENT; -use futures_util::TryStreamExt as _; -use reqwest::{ - IntoUrl, StatusCode, Url, - header::{HeaderValue, RANGE, USER_AGENT}, -}; -use serde::{Deserialize, Serialize}; -use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; -use tokio_util::io::StreamReader; - -/// archive manifest serde structs, copied from -/// https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Manifest { - /// One entry per file in the zip, sorted alphabetically by path. - pub files: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FileEntry { - /// Realtive path (without the leading `{name}-{version}/` component of - /// the tarball). - pub path: String, - /// Byte offset in the zip where this entry's compressed payload begins. - pub data_offset: u64, - /// Length of the compressed contents in bytes. - pub compressed_size: u64, - /// Length of the uncompressed contents in bytes. - pub uncompressed_size: u64, - /// How the payload is compressed: `"deflate"` or `"store"`. - pub compression: String, - /// Lowercase hex sha256 of the uncompressed contents. - pub sha256: String, -} - -pub struct SourceArchive { - manifest: Manifest, - zip_url: Url, - client: reqwest::Client, -} - -impl SourceArchive { - pub async fn load(name: impl AsRef, version: impl AsRef) -> Result> { - Self::load_from("https://static.crates.io/", name, version).await - } - - pub async fn load_from( - base_url: impl IntoUrl, - name: impl AsRef, - version: impl AsRef, - ) -> Result> { - let mut base_url = base_url.into_url()?; - base_url.set_path("crates/"); - - let index_url = base_url.join(&format!( - "{0}/{0}-{1}.zip.json", - name.as_ref(), - version.as_ref() - ))?; - - let headers = vec![(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT))] - .into_iter() - .collect(); - - let client = reqwest::Client::builder() - .default_headers(headers) - .build()?; - - let response = client.get(index_url.clone()).send().await?; - if is_not_found_error(&response.status()) { - return Ok(None); - } - let response = response.error_for_status()?; - - Ok(Some(Self { - manifest: response.json().await?, - zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, - client, - })) - } - - pub fn entries(&self) -> impl Iterator { - self.manifest.files.iter() - } - - pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { - let path = path.as_ref(); - self.manifest.files.iter().find(|e| e.path == path) - } - - pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> - where - W: AsyncWrite + Unpin, - { - let range_start = entry.data_offset; - let range_end = entry.data_offset + entry.compressed_size - 1; - - let response = self - .client - .get(self.zip_url.clone()) - .header(RANGE, format!("bytes={range_start}-{range_end}",)) - .send() - .await? - .error_for_status()?; - - let stream = response.bytes_stream().map_err(std::io::Error::other); - let mut reader = StreamReader::new(stream); - - match entry.compression.as_str() { - "deflate" => { - let mut decoder = DeflateDecoder::new(reader); - io::copy(&mut decoder, writer).await?; - } - "store" => { - io::copy(&mut reader, writer).await?; - } - compression => bail!("unsupported zip compression: {}", compression), - } - - writer.flush().await?; - - Ok(()) - } - - pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { - let mut buf = Vec::new(); - self.fetch(entry, &mut buf).await?; - Ok(buf) - } -} - -fn is_not_found_error(status: &StatusCode) -> bool { - matches!(status, &StatusCode::NOT_FOUND | &StatusCode::FORBIDDEN) -} From b9e3a26e4d8e3e77abbb10ade7a45a3af6cb9e7b Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 09:16:29 +0200 Subject: [PATCH 09/25] simple test --- Cargo.lock | 4 + crates/lib/docs_rs_crate_zip/Cargo.toml | 7 ++ .../lib/docs_rs_crate_zip/examples/fetch.rs | 5 +- crates/lib/docs_rs_crate_zip/src/lib.rs | 101 ++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffe2cf745..bebc58533 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2116,10 +2116,14 @@ dependencies = [ "async-compression", "docs_rs_utils", "futures-util", + "mockito", "reqwest", "serde", + "serde_json", + "tempfile", "tokio", "tokio-util", + "zip", ] [[package]] diff --git a/crates/lib/docs_rs_crate_zip/Cargo.toml b/crates/lib/docs_rs_crate_zip/Cargo.toml index 024bb6e95..bdc14bdaf 100644 --- a/crates/lib/docs_rs_crate_zip/Cargo.toml +++ b/crates/lib/docs_rs_crate_zip/Cargo.toml @@ -13,3 +13,10 @@ reqwest = { workspace = true } serde = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } + +[dev-dependencies] +zip = { workspace = true } +mockito = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["io-std"] } +serde_json = { workspace = true } diff --git a/crates/lib/docs_rs_crate_zip/examples/fetch.rs b/crates/lib/docs_rs_crate_zip/examples/fetch.rs index 3b5f97290..627098797 100644 --- a/crates/lib/docs_rs_crate_zip/examples/fetch.rs +++ b/crates/lib/docs_rs_crate_zip/examples/fetch.rs @@ -1,4 +1,4 @@ -use fetch_from_crate::SourceArchive; +use docs_rs_crate_zip::SourceArchive; use tokio::io; #[tokio::main] @@ -19,7 +19,8 @@ async fn main() { eprintln!("error loading: {:?}", err); return; } - }; + } + .expect("krate / version not found"); let entry = archive .by_name(&file) diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs index 9802d6372..1fdc17394 100644 --- a/crates/lib/docs_rs_crate_zip/src/lib.rs +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -140,3 +140,104 @@ impl SourceArchive { Ok(buf) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{self, Write}; + use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + + fn test_archive() -> Result<(Manifest, Vec)> { + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Deflated) + .compression_level(Some(9)); + + let buf = Vec::new(); + let mut zip = ZipWriter::new(io::Cursor::new(buf)); + + for filename in ["src/main.rs", "Cargo.toml"] { + zip.start_file(filename, options)?; + zip.write_all(filename.as_bytes())?; + } + + let mut archive = zip.finish_into_readable()?; + + let mut files = Vec::with_capacity(archive.len()); + for i in 0..archive.len() { + // `_raw` because we only read each entry's metadata, never its bytes, + // so there is no need to set up a decompressor. + let entry = archive.by_index_raw(i)?; + + let path = entry.name().to_string(); + let data_offset = entry.data_start().expect("missing data start"); + + debug_assert!(matches!(entry.compression(), CompressionMethod::Deflated)); + + files.push(FileEntry { + data_offset, + compressed_size: entry.compressed_size(), + uncompressed_size: entry.size(), + compression: "deflate".into(), + sha256: "dummy".into(), + path, + }); + } + + // Order the manifest alphabetically (case-insensitive) by path. + files.sort_by_cached_key(|f| (f.path.to_lowercase(), f.path.clone())); + + let bytes = archive.into_inner().into_inner(); + + Ok((Manifest { files }, bytes)) + } + + #[tokio::test] + async fn test_fetch() -> anyhow::Result<()> { + let mut server = mockito::Server::new_async().await; + + let (manifest, zip) = test_archive()?; + + let _json_mock = server + .mock("GET", "/crates/krate/krate-0.1.0.zip.json") + .with_body(serde_json::to_string(&manifest).unwrap()) + .create_async() + .await; + + let _content_mock = server + .mock("GET", "/crates/krate/krate-0.1.0.zip") + .with_body_from_request(move |request| { + let range = request + .headers() + .get(RANGE) + .expect("range header must exists"); + let range = range.to_str().unwrap(); + + let bytes = range.strip_prefix("bytes=").unwrap(); + + let (lhs, rhs) = bytes.split_once("-").unwrap(); + + let lhs: usize = lhs.parse().unwrap(); + let rhs: usize = rhs.parse().unwrap(); + + zip.get(lhs..=rhs).unwrap().to_vec() + }) + .create_async() + .await; + + let source_archive = SourceArchive::load_from(server.url(), "krate", "0.1.0") + .await? + .expect("not found"); + + { + let info = source_archive.by_name("src/main.rs").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(&info).await?, b"src/main.rs"); + } + + { + let info = source_archive.by_name("Cargo.toml").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(&info).await?, b"Cargo.toml"); + } + + Ok(()) + } +} From 29b8585eb4ebdab8d6de120199e75f665ce02cac Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 09:50:41 +0200 Subject: [PATCH 10/25] wip --- Cargo.lock | 1 - crates/bin/docs_rs_web/Cargo.toml | 1 + .../docs_rs_web/src/handlers/crate_details.rs | 47 ++++++++ crates/lib/docs_rs_crate_zip/Cargo.toml | 11 +- crates/lib/docs_rs_crate_zip/src/lib.rs | 84 ++----------- crates/lib/docs_rs_crate_zip/src/test_env.rs | 112 ++++++++++++++++++ 6 files changed, 178 insertions(+), 78 deletions(-) create mode 100644 crates/lib/docs_rs_crate_zip/src/test_env.rs diff --git a/Cargo.lock b/Cargo.lock index bebc58533..2e19183ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2120,7 +2120,6 @@ dependencies = [ "reqwest", "serde", "serde_json", - "tempfile", "tokio", "tokio-util", "zip", diff --git a/crates/bin/docs_rs_web/Cargo.toml b/crates/bin/docs_rs_web/Cargo.toml index 1f562e3cd..0809055db 100644 --- a/crates/bin/docs_rs_web/Cargo.toml +++ b/crates/bin/docs_rs_web/Cargo.toml @@ -85,6 +85,7 @@ docs_rs_context = { path = "../../lib/docs_rs_context", features = ["testing"] } docs_rs_database = { path = "../../lib/docs_rs_database", features = ["testing"] } docs_rs_headers = { path = "../../lib/docs_rs_headers", features = ["testing"] } docs_rs_storage = { path = "../../lib/docs_rs_storage", features = ["testing"] } +docs_rs_crate_zip = { path = "../../lib/docs_rs_crate_zip", features = ["testing"] } docs_rs_test_fakes = { path = "../../lib/docs_rs_test_fakes" } docs_rs_types = { path = "../../lib/docs_rs_types", features = ["testing"] } http-body-util = "0.1.0" diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index 58817eaa2..a4c35783d 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -1901,6 +1901,14 @@ mod tests { #[test] fn readme() { async_wrapper(|env| async move { + let (storage_readme_manifest, storage_readme_archive) = + docs_rs_crate_zip::test_env::create_test_archive([( + "README.md", + "storage readme", + )])?; + + let mut static_crates_io = docs_rs_crate_zip::test_env::TestEnv::new().await?; + env.fake_release() .await .name("dummy") @@ -1918,6 +1926,15 @@ mod tests { .create() .await?; + static_crates_io + .add( + "dummy", + "0.2.0", + storage_readme_manifest.clone(), + storage_readme_archive.clone(), + ) + .await?; + env.fake_release() .await .name("dummy") @@ -1926,6 +1943,15 @@ mod tests { .create() .await?; + static_crates_io + .add( + "dummy", + "0.3.0", + storage_readme_manifest.clone(), + storage_readme_archive.clone(), + ) + .await?; + env.fake_release() .await .name("dummy") @@ -1936,6 +1962,18 @@ mod tests { .create() .await?; + let (storage_readme_manifest_2, storage_readme_archive_2) = + docs_rs_crate_zip::test_env::create_test_archive([("MEREAD", "storage meread")])?; + + static_crates_io + .add( + "dummy", + "0.4.0", + storage_readme_manifest_2, + storage_readme_archive_2, + ) + .await?; + env.fake_release() .await .name("dummy") @@ -1946,6 +1984,15 @@ mod tests { .create() .await?; + static_crates_io + .add( + "dummy", + "0.5.0", + storage_readme_manifest.clone(), + storage_readme_archive.clone(), + ) + .await?; + let check_readme = |path: String, content: String| { let env = env.clone(); async move { diff --git a/crates/lib/docs_rs_crate_zip/Cargo.toml b/crates/lib/docs_rs_crate_zip/Cargo.toml index bdc14bdaf..eee214ba6 100644 --- a/crates/lib/docs_rs_crate_zip/Cargo.toml +++ b/crates/lib/docs_rs_crate_zip/Cargo.toml @@ -4,6 +4,13 @@ description = "read specific files from the crates.io source zip archive" version = "0.1.0" edition = "2024" +[features] +testing = [ + "dep:zip", + "dep:mockito", + "dep:serde_json", +] + [dependencies] anyhow = { workspace = true } async-compression = { workspace = true } @@ -13,10 +20,12 @@ reqwest = { workspace = true } serde = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } +zip = { workspace = true, optional = true } +mockito = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } [dev-dependencies] zip = { workspace = true } mockito = { workspace = true } -tempfile = { workspace = true } tokio = { workspace = true, features = ["io-std"] } serde_json = { workspace = true } diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs index 1fdc17394..8a45aaa73 100644 --- a/crates/lib/docs_rs_crate_zip/src/lib.rs +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -5,6 +5,9 @@ //! https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs //! Also we copied the manifest structs from there. +#[cfg(any(test, feature = "testing"))] +pub mod test_env; + use anyhow::{Result, bail}; use async_compression::tokio::bufread::DeflateDecoder; use docs_rs_utils::APP_USER_AGENT; @@ -144,87 +147,16 @@ impl SourceArchive { #[cfg(test)] mod tests { use super::*; - use std::io::{self, Write}; - use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; - - fn test_archive() -> Result<(Manifest, Vec)> { - let options = SimpleFileOptions::default() - .compression_method(CompressionMethod::Deflated) - .compression_level(Some(9)); - - let buf = Vec::new(); - let mut zip = ZipWriter::new(io::Cursor::new(buf)); - - for filename in ["src/main.rs", "Cargo.toml"] { - zip.start_file(filename, options)?; - zip.write_all(filename.as_bytes())?; - } - - let mut archive = zip.finish_into_readable()?; - - let mut files = Vec::with_capacity(archive.len()); - for i in 0..archive.len() { - // `_raw` because we only read each entry's metadata, never its bytes, - // so there is no need to set up a decompressor. - let entry = archive.by_index_raw(i)?; - - let path = entry.name().to_string(); - let data_offset = entry.data_start().expect("missing data start"); - - debug_assert!(matches!(entry.compression(), CompressionMethod::Deflated)); - - files.push(FileEntry { - data_offset, - compressed_size: entry.compressed_size(), - uncompressed_size: entry.size(), - compression: "deflate".into(), - sha256: "dummy".into(), - path, - }); - } - - // Order the manifest alphabetically (case-insensitive) by path. - files.sort_by_cached_key(|f| (f.path.to_lowercase(), f.path.clone())); - - let bytes = archive.into_inner().into_inner(); - - Ok((Manifest { files }, bytes)) - } + use crate::test_env::{TestEnv, create_test_archive}; #[tokio::test] async fn test_fetch() -> anyhow::Result<()> { - let mut server = mockito::Server::new_async().await; - - let (manifest, zip) = test_archive()?; - - let _json_mock = server - .mock("GET", "/crates/krate/krate-0.1.0.zip.json") - .with_body(serde_json::to_string(&manifest).unwrap()) - .create_async() - .await; - - let _content_mock = server - .mock("GET", "/crates/krate/krate-0.1.0.zip") - .with_body_from_request(move |request| { - let range = request - .headers() - .get(RANGE) - .expect("range header must exists"); - let range = range.to_str().unwrap(); - - let bytes = range.strip_prefix("bytes=").unwrap(); - - let (lhs, rhs) = bytes.split_once("-").unwrap(); - - let lhs: usize = lhs.parse().unwrap(); - let rhs: usize = rhs.parse().unwrap(); + let (manifest, zip) = + create_test_archive([("src/main.rs", "src/main.rs"), ("Cargo.toml", "Cargo.toml")])?; - zip.get(lhs..=rhs).unwrap().to_vec() - }) - .create_async() - .await; + let test_env = TestEnv::new("krate", "0.1.0", manifest, zip).await?; - let source_archive = SourceArchive::load_from(server.url(), "krate", "0.1.0") + let source_archive = SourceArchive::load_from(test_env.url(), "krate", "0.1.0") .await? .expect("not found"); diff --git a/crates/lib/docs_rs_crate_zip/src/test_env.rs b/crates/lib/docs_rs_crate_zip/src/test_env.rs new file mode 100644 index 000000000..80e061eeb --- /dev/null +++ b/crates/lib/docs_rs_crate_zip/src/test_env.rs @@ -0,0 +1,112 @@ +use crate::{FileEntry, Manifest}; +use anyhow::Result; +use reqwest::header::RANGE; +use std::io::{self, Write as _}; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + +pub fn create_test_archive(files: I) -> Result<(Manifest, Vec)> +where + I: IntoIterator, + N: ToString, + B: AsRef<[u8]>, +{ + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Deflated) + .compression_level(Some(9)); + + let buf = Vec::new(); + let mut zip = ZipWriter::new(io::Cursor::new(buf)); + + for (filename, content) in files { + zip.start_file(filename, options)?; + zip.write_all(content.as_ref())?; + } + + let mut archive = zip.finish_into_readable()?; + + let mut files = Vec::with_capacity(archive.len()); + for i in 0..archive.len() { + // `_raw` because we only read each entry's metadata, never its bytes, + // so there is no need to set up a decompressor. + let entry = archive.by_index_raw(i)?; + + let path = entry.name().to_string(); + let data_offset = entry.data_start().expect("missing data start"); + + debug_assert!(matches!(entry.compression(), CompressionMethod::Deflated)); + + files.push(FileEntry { + data_offset, + compressed_size: entry.compressed_size(), + uncompressed_size: entry.size(), + compression: "deflate".into(), + sha256: "dummy".into(), + path, + }); + } + + let bytes = archive.into_inner().into_inner(); + + Ok((Manifest { files }, bytes)) +} + +pub struct TestEnv { + mocks: Vec, + server: mockito::ServerGuard, +} + +impl TestEnv { + pub async fn new() -> Result { + Ok(Self { + mocks: Vec::new(), + server: mockito::Server::new_async().await, + }) + } + pub async fn add( + &mut self, + name: impl AsRef, + version: impl AsRef, + manifest: Manifest, + zip: Vec, + ) -> Result<()> { + let name = name.as_ref(); + let version = version.as_ref(); + + self.mocks.push( + self.server + .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip.json")) + .with_body(serde_json::to_string(&manifest).unwrap()) + .create_async() + .await, + ); + + self.mocks.push( + self.server + .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip")) + .with_body_from_request(move |request| { + let range = request + .headers() + .get(RANGE) + .expect("range header must exists"); + let range = range.to_str().unwrap(); + + let bytes = range.strip_prefix("bytes=").unwrap(); + + let (lhs, rhs) = bytes.split_once("-").unwrap(); + + let lhs: usize = lhs.parse().unwrap(); + let rhs: usize = rhs.parse().unwrap(); + + zip.get(lhs..=rhs).unwrap().to_vec() + }) + .create_async() + .await, + ); + + Ok(()) + } + + pub fn url(&self) -> String { + self.server.url() + } +} From ce6ba609e80afd317b5e2d295d11127a7482a8d1 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 09:57:17 +0200 Subject: [PATCH 11/25] WIP --- Cargo.lock | 3 + crates/lib/docs_rs_crate_zip/src/lib.rs | 175 ------------------ crates/lib/docs_rs_registry_api/Cargo.toml | 4 + crates/lib/docs_rs_registry_api/src/config.rs | 4 + crates/lib/docs_rs_registry_api/src/lib.rs | 1 + .../src/source_archive/manifest.rs | 26 +++ .../src/source_archive/mod.rs | 2 + .../src/source_archive/source_archive.rs | 140 ++++++++++++++ 8 files changed, 180 insertions(+), 175 deletions(-) create mode 100644 crates/lib/docs_rs_registry_api/src/source_archive/manifest.rs create mode 100644 crates/lib/docs_rs_registry_api/src/source_archive/mod.rs create mode 100644 crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs diff --git a/Cargo.lock b/Cargo.lock index 2e19183ac..0589416cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2291,12 +2291,14 @@ name = "docs_rs_registry_api" version = "0.1.0" dependencies = [ "anyhow", + "async-compression", "bon", "chrono", "docs_rs_config", "docs_rs_env_vars", "docs_rs_types", "docs_rs_utils", + "futures-util", "mime", "mockito", "reqwest", @@ -2306,6 +2308,7 @@ dependencies = [ "test-case", "thiserror", "tokio", + "tokio-util", "tracing", "url", ] diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs index 8a45aaa73..e69de29bb 100644 --- a/crates/lib/docs_rs_crate_zip/src/lib.rs +++ b/crates/lib/docs_rs_crate_zip/src/lib.rs @@ -1,175 +0,0 @@ -//! Library to read the crates.io source archives (manifest & zip), and -//! fetch single files from the remote archives. -//! -//! Archives are created here: -//! https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs -//! Also we copied the manifest structs from there. - -#[cfg(any(test, feature = "testing"))] -pub mod test_env; - -use anyhow::{Result, bail}; -use async_compression::tokio::bufread::DeflateDecoder; -use docs_rs_utils::APP_USER_AGENT; -use futures_util::TryStreamExt as _; -use reqwest::{ - IntoUrl, StatusCode, Url, - header::{HeaderValue, RANGE, USER_AGENT}, -}; -use serde::{Deserialize, Serialize}; -use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; -use tokio_util::io::StreamReader; - -/// archive manifest serde structs, copied from -/// https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Manifest { - /// One entry per file in the zip, sorted alphabetically by path. - pub files: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FileEntry { - /// Realtive path (without the leading `{name}-{version}/` component of - /// the tarball). - pub path: String, - /// Byte offset in the zip where this entry's compressed payload begins. - pub data_offset: u64, - /// Length of the compressed contents in bytes. - pub compressed_size: u64, - /// Length of the uncompressed contents in bytes. - pub uncompressed_size: u64, - /// How the payload is compressed: `"deflate"` or `"store"`. - pub compression: String, - /// Lowercase hex sha256 of the uncompressed contents. - pub sha256: String, -} - -pub struct SourceArchive { - manifest: Manifest, - zip_url: Url, - client: reqwest::Client, -} - -impl SourceArchive { - pub async fn load(name: impl AsRef, version: impl AsRef) -> Result> { - Self::load_from("https://static.crates.io/", name, version).await - } - - pub async fn load_from( - base_url: impl IntoUrl, - name: impl AsRef, - version: impl AsRef, - ) -> Result> { - let mut base_url = base_url.into_url()?; - base_url.set_path("crates/"); - - let index_url = base_url.join(&format!( - "{0}/{0}-{1}.zip.json", - name.as_ref(), - version.as_ref() - ))?; - - let headers = vec![(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT))] - .into_iter() - .collect(); - - let client = reqwest::Client::builder() - .default_headers(headers) - .build()?; - - let response = client.get(index_url.clone()).send().await?; - if matches!( - response.status(), - StatusCode::NOT_FOUND | StatusCode::FORBIDDEN - ) { - return Ok(None); - } - let response = response.error_for_status()?; - - Ok(Some(Self { - manifest: response.json().await?, - zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, - client, - })) - } - - pub fn entries(&self) -> impl Iterator { - self.manifest.files.iter() - } - - pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { - let path = path.as_ref(); - self.manifest.files.iter().find(|e| e.path == path) - } - - pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> - where - W: AsyncWrite + Unpin, - { - let range_start = entry.data_offset; - let range_end = entry.data_offset + entry.compressed_size - 1; - - let response = self - .client - .get(self.zip_url.clone()) - .header(RANGE, format!("bytes={range_start}-{range_end}",)) - .send() - .await? - .error_for_status()?; - - let stream = response.bytes_stream().map_err(std::io::Error::other); - let mut reader = StreamReader::new(stream); - - match entry.compression.as_str() { - "deflate" => { - let mut decoder = DeflateDecoder::new(reader); - io::copy(&mut decoder, writer).await?; - } - "store" => { - io::copy(&mut reader, writer).await?; - } - compression => bail!("unsupported zip compression: {}", compression), - } - - writer.flush().await?; - - Ok(()) - } - - pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { - let mut buf = Vec::new(); - self.fetch(entry, &mut buf).await?; - Ok(buf) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_env::{TestEnv, create_test_archive}; - - #[tokio::test] - async fn test_fetch() -> anyhow::Result<()> { - let (manifest, zip) = - create_test_archive([("src/main.rs", "src/main.rs"), ("Cargo.toml", "Cargo.toml")])?; - - let test_env = TestEnv::new("krate", "0.1.0", manifest, zip).await?; - - let source_archive = SourceArchive::load_from(test_env.url(), "krate", "0.1.0") - .await? - .expect("not found"); - - { - let info = source_archive.by_name("src/main.rs").expect("should exist"); - assert_eq!(source_archive.fetch_bytes(&info).await?, b"src/main.rs"); - } - - { - let info = source_archive.by_name("Cargo.toml").expect("should exist"); - assert_eq!(source_archive.fetch_bytes(&info).await?, b"Cargo.toml"); - } - - Ok(()) - } -} diff --git a/crates/lib/docs_rs_registry_api/Cargo.toml b/crates/lib/docs_rs_registry_api/Cargo.toml index ccbfa8083..84a3ce54d 100644 --- a/crates/lib/docs_rs_registry_api/Cargo.toml +++ b/crates/lib/docs_rs_registry_api/Cargo.toml @@ -13,6 +13,10 @@ docs_rs_config = { path = "../docs_rs_config" } docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_types = { path = "../docs_rs_types" } docs_rs_utils = { path = "../docs_rs_utils" } +tokio = { workspace = true } +futures-util = { workspace = true } +async-compression = { workspace = true } +tokio-util = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/lib/docs_rs_registry_api/src/config.rs b/crates/lib/docs_rs_registry_api/src/config.rs index ef5f8b118..02aa33604 100644 --- a/crates/lib/docs_rs_registry_api/src/config.rs +++ b/crates/lib/docs_rs_registry_api/src/config.rs @@ -8,6 +8,9 @@ pub struct Config { #[builder(default = "https://crates.io".parse().unwrap())] pub registry_api_host: Url, + #[builder(default = "https://static.crates.io".parse().unwrap())] + pub static_host: Url, + // amount of retries for external API calls, mostly crates.io #[builder(default = 3)] pub crates_io_api_call_retries: u32, @@ -18,6 +21,7 @@ impl AppConfig for Config { Ok(Self::builder() .maybe_crates_io_api_call_retries(maybe_env("DOCSRS_CRATESIO_API_CALL_RETRIES")?) .maybe_registry_api_host(maybe_env("DOCSRS_REGISTRY_API_HOST")?) + .maybe_static_host(maybe_env("DOCSRS_STATIC_HOST")?) .build()) } } diff --git a/crates/lib/docs_rs_registry_api/src/lib.rs b/crates/lib/docs_rs_registry_api/src/lib.rs index 1763fd6b2..940c365ef 100644 --- a/crates/lib/docs_rs_registry_api/src/lib.rs +++ b/crates/lib/docs_rs_registry_api/src/lib.rs @@ -2,6 +2,7 @@ mod api; mod config; mod error; mod models; +mod source_archive; pub use api::RegistryApi; pub use config::Config; diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/manifest.rs b/crates/lib/docs_rs_registry_api/src/source_archive/manifest.rs new file mode 100644 index 000000000..4f88ea5ec --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/source_archive/manifest.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; + +/// archive manifest serde structs, copied from +/// https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Manifest { + /// One entry per file in the zip, sorted alphabetically by path. + pub files: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileEntry { + /// Realtive path (without the leading `{name}-{version}/` component of + /// the tarball). + pub path: String, + /// Byte offset in the zip where this entry's compressed payload begins. + pub data_offset: u64, + /// Length of the compressed contents in bytes. + pub compressed_size: u64, + /// Length of the uncompressed contents in bytes. + pub uncompressed_size: u64, + /// How the payload is compressed: `"deflate"` or `"store"`. + pub compression: String, + /// Lowercase hex sha256 of the uncompressed contents. + pub sha256: String, +} diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs new file mode 100644 index 000000000..c4042239b --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -0,0 +1,2 @@ +mod manifest; +mod source_archive; diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs b/crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs new file mode 100644 index 000000000..937391e01 --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs @@ -0,0 +1,140 @@ +use anyhow::{Result, bail}; +use async_compression::tokio::bufread::DeflateDecoder; +use docs_rs_utils::APP_USER_AGENT; +use futures_util::TryStreamExt as _; +use reqwest::{ + IntoUrl, StatusCode, Url, + header::{HeaderValue, RANGE, USER_AGENT}, +}; +use serde::{Deserialize, Serialize}; +use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; +use tokio_util::io::StreamReader; + +pub struct SourceArchive { + manifest: Manifest, + zip_url: Url, + client: reqwest::Client, +} + +impl SourceArchive { + pub async fn load(name: impl AsRef, version: impl AsRef) -> Result> { + Self::load_from("https://static.crates.io/", name, version).await + } + + pub async fn load_from( + base_url: impl IntoUrl, + name: impl AsRef, + version: impl AsRef, + ) -> Result> { + let mut base_url = base_url.into_url()?; + base_url.set_path("crates/"); + + let index_url = base_url.join(&format!( + "{0}/{0}-{1}.zip.json", + name.as_ref(), + version.as_ref() + ))?; + + let headers = vec![(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT))] + .into_iter() + .collect(); + + let client = reqwest::Client::builder() + .default_headers(headers) + .build()?; + + let response = client.get(index_url.clone()).send().await?; + if matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::FORBIDDEN + ) { + return Ok(None); + } + let response = response.error_for_status()?; + + Ok(Some(Self { + manifest: response.json().await?, + zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, + client, + })) + } + + pub fn entries(&self) -> impl Iterator { + self.manifest.files.iter() + } + + pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { + let path = path.as_ref(); + self.manifest.files.iter().find(|e| e.path == path) + } + + pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> + where + W: AsyncWrite + Unpin, + { + let range_start = entry.data_offset; + let range_end = entry.data_offset + entry.compressed_size - 1; + + let response = self + .client + .get(self.zip_url.clone()) + .header(RANGE, format!("bytes={range_start}-{range_end}",)) + .send() + .await? + .error_for_status()?; + + let stream = response.bytes_stream().map_err(std::io::Error::other); + let mut reader = StreamReader::new(stream); + + match entry.compression.as_str() { + "deflate" => { + let mut decoder = DeflateDecoder::new(reader); + io::copy(&mut decoder, writer).await?; + } + "store" => { + io::copy(&mut reader, writer).await?; + } + compression => bail!("unsupported zip compression: {}", compression), + } + + writer.flush().await?; + + Ok(()) + } + + pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { + let mut buf = Vec::new(); + self.fetch(entry, &mut buf).await?; + Ok(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_env::{TestEnv, create_test_archive}; + + #[tokio::test] + async fn test_fetch() -> anyhow::Result<()> { + let (manifest, zip) = + create_test_archive([("src/main.rs", "src/main.rs"), ("Cargo.toml", "Cargo.toml")])?; + + let test_env = TestEnv::new("krate", "0.1.0", manifest, zip).await?; + + let source_archive = SourceArchive::load_from(test_env.url(), "krate", "0.1.0") + .await? + .expect("not found"); + + { + let info = source_archive.by_name("src/main.rs").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(&info).await?, b"src/main.rs"); + } + + { + let info = source_archive.by_name("Cargo.toml").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(&info).await?, b"Cargo.toml"); + } + + Ok(()) + } +} From b0663bae6d016484ae4597ccf4e9d017f5d85356 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 10:18:37 +0200 Subject: [PATCH 12/25] migra --- Cargo.lock | 19 +-- crates/bin/docs_rs_web/Cargo.toml | 3 +- crates/lib/docs_rs_crate_zip/Cargo.toml | 31 ---- .../lib/docs_rs_crate_zip/examples/fetch.rs | 32 ---- crates/lib/docs_rs_crate_zip/src/lib.rs | 0 crates/lib/docs_rs_registry_api/Cargo.toml | 16 +- crates/lib/docs_rs_registry_api/src/api.rs | 42 +++++- crates/lib/docs_rs_registry_api/src/lib.rs | 7 + .../src/source_archive/mod.rs | 133 ++++++++++++++++- .../src/source_archive/source_archive.rs | 140 ------------------ .../docs_rs_registry_api/src/testing/mod.rs | 1 + .../src/testing/static_test_env.rs} | 12 +- 12 files changed, 201 insertions(+), 235 deletions(-) delete mode 100644 crates/lib/docs_rs_crate_zip/Cargo.toml delete mode 100644 crates/lib/docs_rs_crate_zip/examples/fetch.rs delete mode 100644 crates/lib/docs_rs_crate_zip/src/lib.rs delete mode 100644 crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs create mode 100644 crates/lib/docs_rs_registry_api/src/testing/mod.rs rename crates/lib/{docs_rs_crate_zip/src/test_env.rs => docs_rs_registry_api/src/testing/static_test_env.rs} (91%) diff --git a/Cargo.lock b/Cargo.lock index 0589416cb..84802de19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2108,23 +2108,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "docs_rs_crate_zip" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-compression", - "docs_rs_utils", - "futures-util", - "mockito", - "reqwest", - "serde", - "serde_json", - "tokio", - "tokio-util", - "zip", -] - [[package]] name = "docs_rs_database" version = "0.0.0" @@ -2311,6 +2294,7 @@ dependencies = [ "tokio-util", "tracing", "url", + "zip", ] [[package]] @@ -2515,7 +2499,6 @@ dependencies = [ "docs_rs_cargo_metadata", "docs_rs_config", "docs_rs_context", - "docs_rs_crate_zip", "docs_rs_database", "docs_rs_env_vars", "docs_rs_headers", diff --git a/crates/bin/docs_rs_web/Cargo.toml b/crates/bin/docs_rs_web/Cargo.toml index 0809055db..20d9ac969 100644 --- a/crates/bin/docs_rs_web/Cargo.toml +++ b/crates/bin/docs_rs_web/Cargo.toml @@ -31,7 +31,6 @@ docs_rs_build_queue = { path = "../../lib/docs_rs_build_queue" } docs_rs_cargo_metadata = { path = "../../lib/docs_rs_cargo_metadata" } docs_rs_config = { path = "../../lib/docs_rs_config" } docs_rs_context = { path = "../../lib/docs_rs_context" } -docs_rs_crate_zip = { path = "../../lib/docs_rs_crate_zip" } docs_rs_database = { path = "../../lib/docs_rs_database" } docs_rs_env_vars = { path = "../../lib/docs_rs_env_vars" } docs_rs_headers = { path = "../../lib/docs_rs_headers" } @@ -84,8 +83,8 @@ docs_rs_config = { path = "../../lib/docs_rs_config", features = ["testing"] } docs_rs_context = { path = "../../lib/docs_rs_context", features = ["testing"] } docs_rs_database = { path = "../../lib/docs_rs_database", features = ["testing"] } docs_rs_headers = { path = "../../lib/docs_rs_headers", features = ["testing"] } +docs_rs_registry_api = { path = "../../lib/docs_rs_registry_api", features = ["testing"] } docs_rs_storage = { path = "../../lib/docs_rs_storage", features = ["testing"] } -docs_rs_crate_zip = { path = "../../lib/docs_rs_crate_zip", features = ["testing"] } docs_rs_test_fakes = { path = "../../lib/docs_rs_test_fakes" } docs_rs_types = { path = "../../lib/docs_rs_types", features = ["testing"] } http-body-util = "0.1.0" diff --git a/crates/lib/docs_rs_crate_zip/Cargo.toml b/crates/lib/docs_rs_crate_zip/Cargo.toml deleted file mode 100644 index eee214ba6..000000000 --- a/crates/lib/docs_rs_crate_zip/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "docs_rs_crate_zip" -description = "read specific files from the crates.io source zip archive" -version = "0.1.0" -edition = "2024" - -[features] -testing = [ - "dep:zip", - "dep:mockito", - "dep:serde_json", -] - -[dependencies] -anyhow = { workspace = true } -async-compression = { workspace = true } -docs_rs_utils = { path = "../docs_rs_utils" } -futures-util = { workspace = true } -reqwest = { workspace = true } -serde = { workspace = true } -tokio = { workspace = true } -tokio-util = { workspace = true } -zip = { workspace = true, optional = true } -mockito = { workspace = true, optional = true } -serde_json = { workspace = true, optional = true } - -[dev-dependencies] -zip = { workspace = true } -mockito = { workspace = true } -tokio = { workspace = true, features = ["io-std"] } -serde_json = { workspace = true } diff --git a/crates/lib/docs_rs_crate_zip/examples/fetch.rs b/crates/lib/docs_rs_crate_zip/examples/fetch.rs deleted file mode 100644 index 627098797..000000000 --- a/crates/lib/docs_rs_crate_zip/examples/fetch.rs +++ /dev/null @@ -1,32 +0,0 @@ -use docs_rs_crate_zip::SourceArchive; -use tokio::io; - -#[tokio::main] -async fn main() { - let mut args = std::env::args().skip(1); - let (name, version) = match (args.next(), args.next()) { - (Some(name), Some(version)) => (name, version), - _ => { - eprintln!("usage: fetch [file]"); - return; - } - }; - let file = args.next().unwrap_or_else(|| "Cargo.toml".to_string()); - - let archive = match SourceArchive::load(&name, &version).await { - Ok(archive) => archive, - Err(err) => { - eprintln!("error loading: {:?}", err); - return; - } - } - .expect("krate / version not found"); - - let entry = archive - .by_name(&file) - .unwrap_or_else(|| panic!("no {file} in archive")); - - if let Err(err) = archive.fetch(entry, &mut io::stdout()).await { - eprintln!("error fetching: {:?}", err); - } -} diff --git a/crates/lib/docs_rs_crate_zip/src/lib.rs b/crates/lib/docs_rs_crate_zip/src/lib.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/crates/lib/docs_rs_registry_api/Cargo.toml b/crates/lib/docs_rs_registry_api/Cargo.toml index 84a3ce54d..ba0308d9b 100644 --- a/crates/lib/docs_rs_registry_api/Cargo.toml +++ b/crates/lib/docs_rs_registry_api/Cargo.toml @@ -5,25 +5,34 @@ license = "MIT" repository = "https://github.com/rust-lang/docs.rs" edition = "2024" +[features] +testing = [ + "dep:mockito", + "dep:zip", +] + [dependencies] anyhow = { workspace = true } +async-compression = { workspace = true } bon = { workspace = true } chrono = { workspace = true } docs_rs_config = { path = "../docs_rs_config" } docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_types = { path = "../docs_rs_types" } docs_rs_utils = { path = "../docs_rs_utils" } -tokio = { workspace = true } futures-util = { workspace = true } -async-compression = { workspace = true } -tokio-util = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } tracing = { workspace = true } url = { workspace = true } +mockito = { workspace = true, optional = true } +zip = { workspace = true, optional = true } + [dev-dependencies] docs_rs_types = { path = "../docs_rs_types", features = ["testing"] } @@ -31,6 +40,7 @@ mime = { workspace = true } mockito = { workspace = true } test-case = { workspace = true } tokio = { workspace = true } +zip = { workspace = true } [lints] workspace = true diff --git a/crates/lib/docs_rs_registry_api/src/api.rs b/crates/lib/docs_rs_registry_api/src/api.rs index 594162dd8..3e4ab3239 100644 --- a/crates/lib/docs_rs_registry_api/src/api.rs +++ b/crates/lib/docs_rs_registry_api/src/api.rs @@ -1,5 +1,5 @@ use crate::{ - Config, + Config, SourceArchive, error::{Error, Result}, models::{ApiErrors, CrateData, CrateOwner, OwnerKind, ReleaseData, Search, SearchResponse}, }; @@ -17,6 +17,7 @@ use url::Url; #[derive(Debug)] pub struct RegistryApi { api_base: Url, + static_base: Url, max_retries: u32, client: reqwest::Client, } @@ -25,11 +26,12 @@ impl RegistryApi { pub fn from_config(config: &Config) -> Result { Self::new( config.registry_api_host.clone(), + config.static_host.clone(), config.crates_io_api_call_retries, ) } - pub fn new(api_base: Url, max_retries: u32) -> Result { + pub fn new(api_base: Url, static_base: Url, max_retries: u32) -> Result { let headers = vec![ (USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)), (ACCEPT, HeaderValue::from_static("application/json")), @@ -43,6 +45,7 @@ impl RegistryApi { Ok(Self { api_base, + static_base, client, max_retries, }) @@ -224,6 +227,27 @@ impl RegistryApi { meta: response.meta.ok_or(Error::MissingMetadata)?, }) } + + /// open the crates.io source archive zip for this crate / version. + /// + /// We directly fetch the manifest JSON file. + /// + /// The returned object can be used to inspect the file-list, and + /// fetch single files from the archive via range request. + pub async fn open_source_archive( + &self, + name: &KrateName, + version: &Version, + ) -> Result { + SourceArchive::load( + self.client.clone(), + self.static_base.clone(), + &name, + &version.to_string(), + ) + .await? + .ok_or(Error::MissingReleases) + } } #[cfg(test)] @@ -238,6 +262,7 @@ mod tests { async fn test_search(status: StatusCode, body: impl Serialize) -> Result { let mut crates_io_api = mockito::Server::new_async().await; + let static_server = mockito::Server::new_async().await; let _m = crates_io_api .mock("GET", "/api/v1/crates?q=foo") @@ -247,7 +272,11 @@ mod tests { .create_async() .await; - let api = RegistryApi::new(crates_io_api.url().parse().unwrap(), 0)?; + let api = RegistryApi::new( + crates_io_api.url().parse().unwrap(), + static_server.url().parse().unwrap(), + 0, + )?; api.search("q=foo").await } @@ -257,6 +286,7 @@ mod tests { version: &Version, ) -> Result> { let mut crates_io_api = mockito::Server::new_async().await; + let static_server = mockito::Server::new_async().await; let _m = crates_io_api .mock("GET", "/api/v1/crates/krate/versions") @@ -266,7 +296,11 @@ mod tests { .create_async() .await; - let api = RegistryApi::new(crates_io_api.url().parse().unwrap(), 0)?; + let api = RegistryApi::new( + crates_io_api.url().parse().unwrap(), + static_server.url().parse().unwrap(), + 0, + )?; api.get_release_data(&KRATE, version).await } diff --git a/crates/lib/docs_rs_registry_api/src/lib.rs b/crates/lib/docs_rs_registry_api/src/lib.rs index 940c365ef..f54b14461 100644 --- a/crates/lib/docs_rs_registry_api/src/lib.rs +++ b/crates/lib/docs_rs_registry_api/src/lib.rs @@ -4,7 +4,14 @@ mod error; mod models; mod source_archive; +#[cfg(any(test, feature = "testing"))] +pub mod testing; + pub use api::RegistryApi; pub use config::Config; pub use error::Error; pub use models::{CrateData, CrateOwner, OwnerKind, ReleaseData, Search}; +pub use source_archive::{ + SourceArchive, + manifest::{FileEntry, Manifest}, +}; diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs index c4042239b..86e508bc5 100644 --- a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -1,2 +1,131 @@ -mod manifest; -mod source_archive; +pub mod manifest; + +use crate::source_archive::manifest::{FileEntry, Manifest}; +use anyhow::{Result, bail}; +use async_compression::tokio::bufread::DeflateDecoder; +use futures_util::TryStreamExt as _; +use reqwest::{IntoUrl, StatusCode, Url, header::RANGE}; +use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; +use tokio_util::io::StreamReader; + +pub struct SourceArchive { + manifest: Manifest, + zip_url: Url, + client: reqwest::Client, +} + +impl SourceArchive { + pub(crate) async fn load( + client: reqwest::Client, + base_url: impl IntoUrl, + name: impl AsRef, + version: impl AsRef, + ) -> Result> { + let mut base_url = base_url.into_url()?; + base_url.set_path("crates/"); + + let index_url = base_url.join(&format!( + "{0}/{0}-{1}.zip.json", + name.as_ref(), + version.as_ref() + ))?; + + let response = client.get(index_url.clone()).send().await?; + if matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::FORBIDDEN + ) { + return Ok(None); + } + let response = response.error_for_status()?; + + Ok(Some(Self { + manifest: response.json().await?, + zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, + client, + })) + } + + pub fn entries(&self) -> impl Iterator { + self.manifest.files.iter() + } + + pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { + let path = path.as_ref(); + self.manifest.files.iter().find(|e| e.path == path) + } + + pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> + where + W: AsyncWrite + Unpin, + { + let range_start = entry.data_offset; + let range_end = entry.data_offset + entry.compressed_size - 1; + + let response = self + .client + .get(self.zip_url.clone()) + .header(RANGE, format!("bytes={range_start}-{range_end}",)) + .send() + .await? + .error_for_status()?; + + let stream = response.bytes_stream().map_err(std::io::Error::other); + let mut reader = StreamReader::new(stream); + + match entry.compression.as_str() { + "deflate" => { + let mut decoder = DeflateDecoder::new(reader); + io::copy(&mut decoder, writer).await?; + } + "store" => { + io::copy(&mut reader, writer).await?; + } + compression => bail!("unsupported zip compression: {}", compression), + } + + writer.flush().await?; + + Ok(()) + } + + pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { + let mut buf = Vec::new(); + self.fetch(entry, &mut buf).await?; + Ok(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::static_test_env::{StaticTestEnv, create_test_source_archive}; + + #[tokio::test] + async fn test_fetch() -> anyhow::Result<()> { + let (manifest, zip) = create_test_source_archive([ + ("src/main.rs", "src/main.rs"), + ("Cargo.toml", "Cargo.toml"), + ])?; + + let mut test_env = StaticTestEnv::new().await?; + test_env.add("krate", "0.1.0", manifest, zip).await?; + + let source_archive = + SourceArchive::load(test_env.client().clone(), test_env.url(), "krate", "0.1.0") + .await? + .expect("not found"); + + { + let info = source_archive.by_name("src/main.rs").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(info).await?, b"src/main.rs"); + } + + { + let info = source_archive.by_name("Cargo.toml").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(info).await?, b"Cargo.toml"); + } + + Ok(()) + } +} diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs b/crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs deleted file mode 100644 index 937391e01..000000000 --- a/crates/lib/docs_rs_registry_api/src/source_archive/source_archive.rs +++ /dev/null @@ -1,140 +0,0 @@ -use anyhow::{Result, bail}; -use async_compression::tokio::bufread::DeflateDecoder; -use docs_rs_utils::APP_USER_AGENT; -use futures_util::TryStreamExt as _; -use reqwest::{ - IntoUrl, StatusCode, Url, - header::{HeaderValue, RANGE, USER_AGENT}, -}; -use serde::{Deserialize, Serialize}; -use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; -use tokio_util::io::StreamReader; - -pub struct SourceArchive { - manifest: Manifest, - zip_url: Url, - client: reqwest::Client, -} - -impl SourceArchive { - pub async fn load(name: impl AsRef, version: impl AsRef) -> Result> { - Self::load_from("https://static.crates.io/", name, version).await - } - - pub async fn load_from( - base_url: impl IntoUrl, - name: impl AsRef, - version: impl AsRef, - ) -> Result> { - let mut base_url = base_url.into_url()?; - base_url.set_path("crates/"); - - let index_url = base_url.join(&format!( - "{0}/{0}-{1}.zip.json", - name.as_ref(), - version.as_ref() - ))?; - - let headers = vec![(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT))] - .into_iter() - .collect(); - - let client = reqwest::Client::builder() - .default_headers(headers) - .build()?; - - let response = client.get(index_url.clone()).send().await?; - if matches!( - response.status(), - StatusCode::NOT_FOUND | StatusCode::FORBIDDEN - ) { - return Ok(None); - } - let response = response.error_for_status()?; - - Ok(Some(Self { - manifest: response.json().await?, - zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, - client, - })) - } - - pub fn entries(&self) -> impl Iterator { - self.manifest.files.iter() - } - - pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { - let path = path.as_ref(); - self.manifest.files.iter().find(|e| e.path == path) - } - - pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> - where - W: AsyncWrite + Unpin, - { - let range_start = entry.data_offset; - let range_end = entry.data_offset + entry.compressed_size - 1; - - let response = self - .client - .get(self.zip_url.clone()) - .header(RANGE, format!("bytes={range_start}-{range_end}",)) - .send() - .await? - .error_for_status()?; - - let stream = response.bytes_stream().map_err(std::io::Error::other); - let mut reader = StreamReader::new(stream); - - match entry.compression.as_str() { - "deflate" => { - let mut decoder = DeflateDecoder::new(reader); - io::copy(&mut decoder, writer).await?; - } - "store" => { - io::copy(&mut reader, writer).await?; - } - compression => bail!("unsupported zip compression: {}", compression), - } - - writer.flush().await?; - - Ok(()) - } - - pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { - let mut buf = Vec::new(); - self.fetch(entry, &mut buf).await?; - Ok(buf) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_env::{TestEnv, create_test_archive}; - - #[tokio::test] - async fn test_fetch() -> anyhow::Result<()> { - let (manifest, zip) = - create_test_archive([("src/main.rs", "src/main.rs"), ("Cargo.toml", "Cargo.toml")])?; - - let test_env = TestEnv::new("krate", "0.1.0", manifest, zip).await?; - - let source_archive = SourceArchive::load_from(test_env.url(), "krate", "0.1.0") - .await? - .expect("not found"); - - { - let info = source_archive.by_name("src/main.rs").expect("should exist"); - assert_eq!(source_archive.fetch_bytes(&info).await?, b"src/main.rs"); - } - - { - let info = source_archive.by_name("Cargo.toml").expect("should exist"); - assert_eq!(source_archive.fetch_bytes(&info).await?, b"Cargo.toml"); - } - - Ok(()) - } -} diff --git a/crates/lib/docs_rs_registry_api/src/testing/mod.rs b/crates/lib/docs_rs_registry_api/src/testing/mod.rs new file mode 100644 index 000000000..230d352e3 --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/testing/mod.rs @@ -0,0 +1 @@ +pub(crate) mod static_test_env; diff --git a/crates/lib/docs_rs_crate_zip/src/test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs similarity index 91% rename from crates/lib/docs_rs_crate_zip/src/test_env.rs rename to crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs index 80e061eeb..45e56486d 100644 --- a/crates/lib/docs_rs_crate_zip/src/test_env.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs @@ -4,7 +4,7 @@ use reqwest::header::RANGE; use std::io::{self, Write as _}; use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; -pub fn create_test_archive(files: I) -> Result<(Manifest, Vec)> +pub fn create_test_source_archive(files: I) -> Result<(Manifest, Vec)> where I: IntoIterator, N: ToString, @@ -50,16 +50,18 @@ where Ok((Manifest { files }, bytes)) } -pub struct TestEnv { +pub struct StaticTestEnv { mocks: Vec, server: mockito::ServerGuard, + client: reqwest::Client, } -impl TestEnv { +impl StaticTestEnv { pub async fn new() -> Result { Ok(Self { mocks: Vec::new(), server: mockito::Server::new_async().await, + client: reqwest::Client::builder().build()?, }) } pub async fn add( @@ -109,4 +111,8 @@ impl TestEnv { pub fn url(&self) -> String { self.server.url() } + + pub fn client(&self) -> &reqwest::Client { + &self.client + } } From 65dbc6147cdba2d5db7d0cd127f93f7991d4965d Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 10:45:07 +0200 Subject: [PATCH 13/25] tests pass --- .../docs_rs_web/src/handlers/crate_details.rs | 355 ++++++++++-------- .../docs_rs_registry_api/src/testing/mod.rs | 2 +- 2 files changed, 208 insertions(+), 149 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index a4c35783d..444c5a6d0 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use crate::{ cache::CachePolicy, error::{AxumNope, AxumResult}, @@ -14,13 +16,15 @@ use crate::{ }; use anyhow::{Context, Result}; use askama::Template; -use axum::response::{IntoResponse, Response as AxumResponse}; +use axum::{ + Extension, + response::{IntoResponse, Response as AxumResponse}, +}; use chrono::{DateTime, Utc}; use docs_rs_cargo_metadata::{Dependency, ReleaseDependencyList}; -use docs_rs_crate_zip::SourceArchive; use docs_rs_database::crate_details::{Release, parse_doc_targets}; use docs_rs_headers::CanonicalUrl; -use docs_rs_registry_api::OwnerKind; +use docs_rs_registry_api::{OwnerKind, RegistryApi}; use docs_rs_types::{ BuildId, BuildStatus, CrateId, Duration, KrateName, ReleaseId, ReqVersion, Version, }; @@ -288,11 +292,14 @@ impl CrateDetails { Ok(Some(crate_details)) } - async fn fetch_readme(&self) -> anyhow::Result> { - let Some(source_archive) = - SourceArchive::load(&self.name, self.version.to_string()).await? - else { - return Ok(None); + async fn fetch_readme(&self, registry_api: &RegistryApi) -> anyhow::Result> { + let source_archive = match registry_api + .open_source_archive(&self.name, &self.version) + .await + { + Ok(archive) => archive, + Err(docs_rs_registry_api::Error::MissingReleases) => return Ok(None), + Err(err) => return Err(err.into()), }; let Some(cargo_toml) = source_archive.by_name("Cargo.toml") else { @@ -427,6 +434,7 @@ impl_axum_webpage! { #[tracing::instrument(skip(conn))] pub(crate) async fn crate_details_handler( params: RustdocParams, + Extension(registry_api): Extension>, mut conn: DbConnection, ) -> AxumResult { let matched_release = match_version(&mut conn, params.name(), params.req_version()) @@ -460,7 +468,7 @@ pub(crate) async fn crate_details_handler( // before we do the long S3 requests. drop(conn); - match details.fetch_readme().await { + match details.fetch_readme(®istry_api).await { Ok(readme) => details.readme = readme.or(details.readme), Err(e) => warn!(?e, "error fetching readme"), } @@ -1898,132 +1906,150 @@ mod tests { Ok(()) } - #[test] - fn readme() { - async_wrapper(|env| async move { - let (storage_readme_manifest, storage_readme_archive) = - docs_rs_crate_zip::test_env::create_test_archive([( - "README.md", - "storage readme", - )])?; - - let mut static_crates_io = docs_rs_crate_zip::test_env::TestEnv::new().await?; - - env.fake_release() - .await - .name("dummy") - .version("0.1.0") - .readme_only_database("database readme") - .create() - .await?; + #[tokio::test(flavor = "multi_thread")] + async fn readme() -> Result<()> { + let (storage_readme_manifest, storage_readme_archive) = + docs_rs_registry_api::testing::static_test_env::create_test_source_archive([( + "README.md", + "storage readme", + )])?; + + let mut static_crates_io = + docs_rs_registry_api::testing::static_test_env::StaticTestEnv::new().await?; + + let env = TestEnvironment::builder() + .registry_api_config( + docs_rs_registry_api::Config::builder() + .static_host(static_crates_io.url().parse().unwrap()) + .build(), + ) + .build() + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.2.0") - .readme_only_database("database readme") - .source_file("README.md", b"storage readme") - .create() - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.1.0") + .readme_only_database("database readme") + .create() + .await?; - static_crates_io - .add( - "dummy", - "0.2.0", - storage_readme_manifest.clone(), - storage_readme_archive.clone(), - ) - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.2.0") + .readme_only_database("database readme") + .source_file("README.md", b"storage readme") + .create() + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.3.0") - .source_file("README.md", b"storage readme") - .create() - .await?; + static_crates_io + .add( + "dummy", + "0.2.0", + storage_readme_manifest.clone(), + storage_readme_archive.clone(), + ) + .await?; - static_crates_io - .add( - "dummy", - "0.3.0", - storage_readme_manifest.clone(), - storage_readme_archive.clone(), - ) - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.3.0") + .source_file("README.md", b"storage readme") + .create() + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.4.0") - .readme_only_database("database readme") - .source_file("MEREAD", b"storage meread") - .source_file("Cargo.toml", br#"package.readme = "MEREAD""#) - .create() - .await?; + static_crates_io + .add( + "dummy", + "0.3.0", + storage_readme_manifest.clone(), + storage_readme_archive.clone(), + ) + .await?; - let (storage_readme_manifest_2, storage_readme_archive_2) = - docs_rs_crate_zip::test_env::create_test_archive([("MEREAD", "storage meread")])?; + env.fake_release() + .await + .name("dummy") + .version("0.4.0") + .readme_only_database("database readme") + .source_file("MEREAD", b"storage meread") + .source_file("Cargo.toml", br#"package.readme = "MEREAD""#) + .create() + .await?; - static_crates_io - .add( - "dummy", - "0.4.0", - storage_readme_manifest_2, - storage_readme_archive_2, - ) - .await?; + let (storage_readme_manifest_2, storage_readme_archive_2) = + docs_rs_registry_api::testing::static_test_env::create_test_source_archive([ + ("MEREAD", "storage meread"), + ("Cargo.toml", r#"package.readme = "MEREAD""#), + ])?; + + static_crates_io + .add( + "dummy", + "0.4.0", + storage_readme_manifest_2, + storage_readme_archive_2, + ) + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.5.0") - .readme_only_database("database readme") - .source_file("README.md", b"storage readme") - .no_cargo_toml() - .create() - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.5.0") + .readme_only_database("database readme") + .source_file("README.md", b"storage readme") + .no_cargo_toml() + .create() + .await?; - static_crates_io - .add( - "dummy", - "0.5.0", - storage_readme_manifest.clone(), - storage_readme_archive.clone(), - ) - .await?; + static_crates_io + .add( + "dummy", + "0.5.0", + storage_readme_manifest.clone(), + storage_readme_archive.clone(), + ) + .await?; - let check_readme = |path: String, content: String| { - let env = env.clone(); - async move { - let resp = env.web_app().await.get(&path).await.unwrap(); - let body = resp.text().await.unwrap(); - assert!(body.contains(&content)); - } - }; + async fn check_readme(env: &TestEnvironment, path: &str, content: &str) { + let resp = env.web_app().await.assert_success(path).await.unwrap(); + let body = resp.text().await.unwrap(); + assert!(body.contains(content)); + } - check_readme("/crate/dummy/0.1.0".into(), "database readme".into()).await; - check_readme("/crate/dummy/0.2.0".into(), "storage readme".into()).await; - check_readme("/crate/dummy/0.3.0".into(), "storage readme".into()).await; - check_readme("/crate/dummy/0.4.0".into(), "storage meread".into()).await; + check_readme(&env, "/crate/dummy/0.1.0", "database readme").await; + check_readme(&env, "/crate/dummy/0.2.0", "storage readme").await; + check_readme(&env, "/crate/dummy/0.3.0", "storage readme").await; + check_readme(&env, "/crate/dummy/0.4.0", "storage meread").await; - let mut conn = env.async_conn().await?; - let details = crate_details(&mut conn, "dummy", "0.5.0", None).await; - assert!(matches!(details.fetch_readme().await, Ok(None))); - Ok(()) - }); + let mut conn = env.async_conn().await?; + let details = crate_details(&mut conn, "dummy", "0.5.0", None).await; + let registry_api = env.registry_api()?; + assert!(matches!(details.fetch_readme(registry_api).await, Ok(None))); + Ok(()) } - #[test] - fn no_readme() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("dummy") - .version("0.2.0") - .source_file( + #[tokio::test(flavor = "multi_thread")] + async fn no_readme() -> Result<()> { + let mut static_crates_io = + docs_rs_registry_api::testing::static_test_env::StaticTestEnv::new().await?; + + let env = TestEnvironment::builder() + .registry_api_config( + docs_rs_registry_api::Config::builder() + .static_host(static_crates_io.url().parse().unwrap()) + .build(), + ) + .build() + .await?; + + let (storage_readme_manifest, storage_readme_archive) = + docs_rs_registry_api::testing::static_test_env::create_test_source_archive([ + ( "Cargo.toml", - br#"[package] + r#"[package] name = "dummy" version = "0.2.0" @@ -2031,42 +2057,75 @@ version = "0.2.0" name = "dummy" path = "src/lib.rs" "#, - ) - .source_file( + ), + ( "src/lib.rs", - b"//! # Crate-level docs + "//! # Crate-level docs //! //! ``` //! let x = 21; //! ``` ", - ) - .target_source("src/lib.rs") - .create() - .await?; + ), + ])?; + + static_crates_io + .add( + "dummy", + "0.2.0", + storage_readme_manifest, + storage_readme_archive, + ) + .await?; - let web = env.web_app().await; - let response = web.get("/crate/dummy/0.2.0").await?; - assert!(response.status().is_success()); + env.fake_release() + .await + .name("dummy") + .version("0.2.0") + .source_file( + "Cargo.toml", + br#"[package] +name = "dummy" +version = "0.2.0" - let dom = kuchikiki::parse_html().one(response.text().await?); - dom.select_first("#main").expect("not main crate docs"); - // First we check that the crate-level docs have been rendered as expected. - assert_eq!( - dom.select_first("#main h1") - .expect("no h1 found") - .text_contents(), - "Crate-level docs" - ); - // Then we check that by default, the language used for highlighting is rust. - assert_eq!( - dom.select_first("#main pre .syntax-source.syntax-rust") - .expect("no rust code block found") - .text_contents(), - "let x = 21;\n" - ); - Ok(()) - }); +[lib] +name = "dummy" +path = "src/lib.rs" +"#, + ) + .source_file( + "src/lib.rs", + b"//! # Crate-level docs +//! +//! ``` +//! let x = 21; +//! ``` +", + ) + .target_source("src/lib.rs") + .create() + .await?; + + let web = env.web_app().await; + let response = web.assert_success("/crate/dummy/0.2.0").await?; + + let dom = kuchikiki::parse_html().one(dbg!(response.text().await?)); + dom.select_first("#main").expect("not main crate docs"); + // First we check that the crate-level docs have been rendered as expected. + assert_eq!( + dom.select_first("#main h1") + .expect("no h1 found") + .text_contents(), + "Crate-level docs" + ); + // Then we check that by default, the language used for highlighting is rust. + assert_eq!( + dom.select_first("#main pre .syntax-source.syntax-rust") + .expect("no rust code block found") + .text_contents(), + "let x = 21;\n" + ); + Ok(()) } #[test] diff --git a/crates/lib/docs_rs_registry_api/src/testing/mod.rs b/crates/lib/docs_rs_registry_api/src/testing/mod.rs index 230d352e3..7d93d6837 100644 --- a/crates/lib/docs_rs_registry_api/src/testing/mod.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/mod.rs @@ -1 +1 @@ -pub(crate) mod static_test_env; +pub mod static_test_env; From c440d0d346d7245eaf7c8689e59a0d408342de54 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 11:08:39 +0200 Subject: [PATCH 14/25] try global test env static --- .../docs_rs_web/src/handlers/crate_details.rs | 4 +- crates/lib/docs_rs_context/Cargo.toml | 1 + .../src/testing/test_env/non_blocking.rs | 20 ++-- .../src/source_archive/mod.rs | 4 +- .../src/testing/static_test_env.rs | 98 ++++++++++--------- crates/lib/docs_rs_test_fakes/Cargo.toml | 2 +- crates/lib/docs_rs_test_fakes/src/legacy.rs | 33 ++++++- 7 files changed, 97 insertions(+), 65 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index 444c5a6d0..222de8f11 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -1915,7 +1915,7 @@ mod tests { )])?; let mut static_crates_io = - docs_rs_registry_api::testing::static_test_env::StaticTestEnv::new().await?; + docs_rs_registry_api::testing::static_test_env::TestStaticCratesIo::new().await?; let env = TestEnvironment::builder() .registry_api_config( @@ -2034,7 +2034,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn no_readme() -> Result<()> { let mut static_crates_io = - docs_rs_registry_api::testing::static_test_env::StaticTestEnv::new().await?; + docs_rs_registry_api::testing::static_test_env::TestStaticCratesIo::new().await?; let env = TestEnvironment::builder() .registry_api_config( diff --git a/crates/lib/docs_rs_context/Cargo.toml b/crates/lib/docs_rs_context/Cargo.toml index 5702d7dee..8eac17529 100644 --- a/crates/lib/docs_rs_context/Cargo.toml +++ b/crates/lib/docs_rs_context/Cargo.toml @@ -12,6 +12,7 @@ testing = [ "docs_rs_fastly/testing", "docs_rs_logging/testing", "docs_rs_storage/testing", + "docs_rs_registry_api/testing", ] [dependencies] diff --git a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs index 98583d1f5..1ff24eaa3 100644 --- a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs +++ b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs @@ -6,7 +6,7 @@ use docs_rs_config::AppConfig; use docs_rs_database::{AsyncPoolClient, Config as DatabaseConfig, testing::TestDatabase}; use docs_rs_fastly::Cdn; use docs_rs_opentelemetry::testing::{CollectedMetrics, TestMetrics}; -use docs_rs_registry_api::RegistryApi; +use docs_rs_registry_api::{RegistryApi, testing::static_test_env::TestStaticCratesIo}; use docs_rs_storage::{Config as StorageConfig, testing::TestStorage}; use docs_rs_test_fakes::FakeRelease; use std::{ops::Deref, sync::Arc}; @@ -14,6 +14,7 @@ use std::{ops::Deref, sync::Arc}; pub struct TestEnvironment { context: Arc, config: Arc, + static_crates_io: Arc, // so we can allow asserting collected metrics later. metrics: TestMetrics, #[allow(dead_code)] // we need to keep the storage so it can be cleaned up. @@ -54,12 +55,14 @@ impl TestEnvironment { C::test_config()? }); - let registry_api_config = - Arc::new(if let Some(registry_api_config) = registry_api_config { - registry_api_config - } else { - docs_rs_registry_api::Config::from_environment()? - }); + let mut registry_api_config = if let Some(registry_api_config) = registry_api_config { + registry_api_config + } else { + docs_rs_registry_api::Config::from_environment()? + }; + + let static_crates_io = TestStaticCratesIo::new().await?; + registry_api_config.static_host = static_crates_io.url().parse().unwrap(); let registry_api = RegistryApi::from_config(®istry_api_config)?; @@ -90,6 +93,7 @@ impl TestEnvironment { )); Ok(Self { + static_crates_io: Arc::new(static_crates_io), config: app_config, context: Context::builder() .with_runtime() @@ -98,7 +102,7 @@ impl TestEnvironment { .pool(db_config.into(), db.pool().clone()) .storage(storage_config.clone(), test_storage.storage()) .build_queue(build_queue_config, build_queue) - .registry_api(registry_api_config, registry_api.into()) + .registry_api(Arc::new(registry_api_config), registry_api.into()) .with_repository_stats()? .maybe_cdn( Arc::new(docs_rs_fastly::Config::test_config()?), diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs index 86e508bc5..a0934ca96 100644 --- a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -99,7 +99,7 @@ impl SourceArchive { #[cfg(test)] mod tests { use super::*; - use crate::testing::static_test_env::{StaticTestEnv, create_test_source_archive}; + use crate::testing::static_test_env::{TestStaticCratesIo, create_test_source_archive}; #[tokio::test] async fn test_fetch() -> anyhow::Result<()> { @@ -108,7 +108,7 @@ mod tests { ("Cargo.toml", "Cargo.toml"), ])?; - let mut test_env = StaticTestEnv::new().await?; + let mut test_env = TestStaticCratesIo::new().await?; test_env.add("krate", "0.1.0", manifest, zip).await?; let source_archive = diff --git a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs index 45e56486d..ddd11b298 100644 --- a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs @@ -1,7 +1,9 @@ use crate::{FileEntry, Manifest}; use anyhow::Result; +use docs_rs_types::{KrateName, Version}; use reqwest::header::RANGE; use std::io::{self, Write as _}; +use tokio::sync::Mutex; use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; pub fn create_test_source_archive(files: I) -> Result<(Manifest, Vec)> @@ -50,69 +52,69 @@ where Ok((Manifest { files }, bytes)) } -pub struct StaticTestEnv { +pub struct TestStaticCratesIo { + inner: Mutex, +} + +struct TestStaticCratesIoInner { mocks: Vec, server: mockito::ServerGuard, - client: reqwest::Client, } -impl StaticTestEnv { +impl TestStaticCratesIo { pub async fn new() -> Result { Ok(Self { - mocks: Vec::new(), - server: mockito::Server::new_async().await, - client: reqwest::Client::builder().build()?, + inner: Mutex::new(TestStaticCratesIoInner { + mocks: Vec::new(), + server: mockito::Server::new_async().await, + }), }) } pub async fn add( - &mut self, - name: impl AsRef, - version: impl AsRef, + &self, + name: &KrateName, + version: &Version, manifest: Manifest, zip: Vec, ) -> Result<()> { - let name = name.as_ref(); - let version = version.as_ref(); - - self.mocks.push( - self.server - .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip.json")) - .with_body(serde_json::to_string(&manifest).unwrap()) - .create_async() - .await, - ); - - self.mocks.push( - self.server - .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip")) - .with_body_from_request(move |request| { - let range = request - .headers() - .get(RANGE) - .expect("range header must exists"); - let range = range.to_str().unwrap(); - - let bytes = range.strip_prefix("bytes=").unwrap(); - - let (lhs, rhs) = bytes.split_once("-").unwrap(); - - let lhs: usize = lhs.parse().unwrap(); - let rhs: usize = rhs.parse().unwrap(); - - zip.get(lhs..=rhs).unwrap().to_vec() - }) - .create_async() - .await, - ); + let mut inner = self.inner.lock().await; + + let mock_json = inner + .server + .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip.json")) + .with_body(serde_json::to_string(&manifest).unwrap()) + .create_async() + .await; + inner.mocks.push(mock_json); + + let mock_zip = inner + .server + .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip")) + .with_body_from_request(move |request| { + let range = request + .headers() + .get(RANGE) + .expect("range header must exists"); + let range = range.to_str().unwrap(); + + let bytes = range.strip_prefix("bytes=").unwrap(); + + let (lhs, rhs) = bytes.split_once("-").unwrap(); + + let lhs: usize = lhs.parse().unwrap(); + let rhs: usize = rhs.parse().unwrap(); + + zip.get(lhs..=rhs).unwrap().to_vec() + }) + .create_async() + .await; + inner.mocks.push(mock_zip); Ok(()) } - pub fn url(&self) -> String { - self.server.url() - } - - pub fn client(&self) -> &reqwest::Client { - &self.client + pub async fn url(&self) -> String { + let inner = self.inner.lock().await; + inner.server.url() } } diff --git a/crates/lib/docs_rs_test_fakes/Cargo.toml b/crates/lib/docs_rs_test_fakes/Cargo.toml index 5915e374a..ead4cc591 100644 --- a/crates/lib/docs_rs_test_fakes/Cargo.toml +++ b/crates/lib/docs_rs_test_fakes/Cargo.toml @@ -11,7 +11,7 @@ base64 = { workspace = true } chrono = { workspace = true } docs_rs_cargo_metadata = { path = "../docs_rs_cargo_metadata", features = ["testing"] } docs_rs_database = { path = "../docs_rs_database" } -docs_rs_registry_api = { path = "../docs_rs_registry_api" } +docs_rs_registry_api = { path = "../docs_rs_registry_api", features = ["testing"] } docs_rs_rustdoc_json = { path = "../../lib/docs_rs_rustdoc_json" } docs_rs_storage = { path = "../docs_rs_storage" } docs_rs_types = { path = "../docs_rs_types" } diff --git a/crates/lib/docs_rs_test_fakes/src/legacy.rs b/crates/lib/docs_rs_test_fakes/src/legacy.rs index be9460bfe..7e80050b9 100644 --- a/crates/lib/docs_rs_test_fakes/src/legacy.rs +++ b/crates/lib/docs_rs_test_fakes/src/legacy.rs @@ -8,7 +8,9 @@ use docs_rs_database::{ add_build_logs, initialize_build, initialize_crate, initialize_release, update_build_status, }, }; -use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData}; +use docs_rs_registry_api::{ + CrateData, CrateOwner, ReleaseData, testing::static_test_env::TestStaticCratesIo, +}; use docs_rs_rustdoc_json::{RUSTDOC_JSON_COMPRESSION_ALGORITHMS, RustdocJsonFormatVersion}; use docs_rs_storage::{ AsyncStorage, FileEntry, compress, file_list_to_json, rustdoc_archive_path, rustdoc_json_path, @@ -70,6 +72,7 @@ where pub struct FakeRelease<'a> { pool: Pool, storage: Arc, + static_crates_io: Arc, package: MetadataPackage, builds: Option>, /// name, content @@ -107,10 +110,15 @@ const DEFAULT_CONTENT: &[u8] = b"default content for test/fakes"; impl<'a> FakeRelease<'a> { - pub fn new(pool: Pool, storage: Arc) -> Self { + pub fn new( + pool: Pool, + storage: Arc, + static_crates_io: Arc, + ) -> Self { FakeRelease { pool, storage, + static_crates_io, package: MetadataPackage { id: "fake-package-id".into(), name: "fake-package".into(), @@ -454,6 +462,12 @@ impl<'a> FakeRelease<'a> { let source_tmp = create_temp_dir(); store_files_into(&self.source_files, source_tmp.path())?; + let mut owned_source_files: Vec<_> = self + .source_files + .iter() + .map(|(n, c)| (n.to_string(), c.to_vec())) + .collect(); + if !self.no_cargo_toml && !self .source_files @@ -469,6 +483,19 @@ impl<'a> FakeRelease<'a> { "# ); store_files_into(&[("Cargo.toml", content.as_bytes())], source_tmp.path())?; + owned_source_files.push(("Cargo.toml".into(), content.as_bytes().to_vec())); + } + + let krate_name: KrateName = package.name.parse()?; + + { + let (manifest, zip) = + docs_rs_registry_api::testing::static_test_env::create_test_source_archive( + owned_source_files, + )?; + self.static_crates_io + .add(&krate_name, &package.version, manifest, zip) + .await?; } let (source_meta, algs) = upload_files( @@ -536,8 +563,6 @@ impl<'a> FakeRelease<'a> { self.doc_targets.insert(0, default_target.to_owned()); } - let krate_name: KrateName = package.name.parse()?; - for target in &self.doc_targets { let dummy_rustdoc_json_content = serde_json::to_vec(&serde_json::json!({ "format_version": 42 From 9ef62eaffab09ada848b624de01312ff7ef3ee98 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 11:12:33 +0200 Subject: [PATCH 15/25] fix --- .../docs_rs_web/src/handlers/crate_details.rs | 101 +----------------- .../src/testing/test_env/non_blocking.rs | 3 +- 2 files changed, 5 insertions(+), 99 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index 222de8f11..a56609f82 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -1908,21 +1908,8 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn readme() -> Result<()> { - let (storage_readme_manifest, storage_readme_archive) = - docs_rs_registry_api::testing::static_test_env::create_test_source_archive([( - "README.md", - "storage readme", - )])?; - - let mut static_crates_io = - docs_rs_registry_api::testing::static_test_env::TestStaticCratesIo::new().await?; - let env = TestEnvironment::builder() - .registry_api_config( - docs_rs_registry_api::Config::builder() - .static_host(static_crates_io.url().parse().unwrap()) - .build(), - ) + .registry_api_config(docs_rs_registry_api::Config::builder().build()) .build() .await?; @@ -1943,15 +1930,6 @@ mod tests { .create() .await?; - static_crates_io - .add( - "dummy", - "0.2.0", - storage_readme_manifest.clone(), - storage_readme_archive.clone(), - ) - .await?; - env.fake_release() .await .name("dummy") @@ -1960,15 +1938,6 @@ mod tests { .create() .await?; - static_crates_io - .add( - "dummy", - "0.3.0", - storage_readme_manifest.clone(), - storage_readme_archive.clone(), - ) - .await?; - env.fake_release() .await .name("dummy") @@ -1979,21 +1948,6 @@ mod tests { .create() .await?; - let (storage_readme_manifest_2, storage_readme_archive_2) = - docs_rs_registry_api::testing::static_test_env::create_test_source_archive([ - ("MEREAD", "storage meread"), - ("Cargo.toml", r#"package.readme = "MEREAD""#), - ])?; - - static_crates_io - .add( - "dummy", - "0.4.0", - storage_readme_manifest_2, - storage_readme_archive_2, - ) - .await?; - env.fake_release() .await .name("dummy") @@ -2004,15 +1958,6 @@ mod tests { .create() .await?; - static_crates_io - .add( - "dummy", - "0.5.0", - storage_readme_manifest.clone(), - storage_readme_archive.clone(), - ) - .await?; - async fn check_readme(env: &TestEnvironment, path: &str, content: &str) { let resp = env.web_app().await.assert_success(path).await.unwrap(); let body = resp.text().await.unwrap(); @@ -2033,51 +1978,11 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn no_readme() -> Result<()> { - let mut static_crates_io = - docs_rs_registry_api::testing::static_test_env::TestStaticCratesIo::new().await?; - let env = TestEnvironment::builder() - .registry_api_config( - docs_rs_registry_api::Config::builder() - .static_host(static_crates_io.url().parse().unwrap()) - .build(), - ) + .registry_api_config(docs_rs_registry_api::Config::builder().build()) .build() .await?; - let (storage_readme_manifest, storage_readme_archive) = - docs_rs_registry_api::testing::static_test_env::create_test_source_archive([ - ( - "Cargo.toml", - r#"[package] -name = "dummy" -version = "0.2.0" - -[lib] -name = "dummy" -path = "src/lib.rs" -"#, - ), - ( - "src/lib.rs", - "//! # Crate-level docs -//! -//! ``` -//! let x = 21; -//! ``` -", - ), - ])?; - - static_crates_io - .add( - "dummy", - "0.2.0", - storage_readme_manifest, - storage_readme_archive, - ) - .await?; - env.fake_release() .await .name("dummy") @@ -2109,7 +2014,7 @@ path = "src/lib.rs" let web = env.web_app().await; let response = web.assert_success("/crate/dummy/0.2.0").await?; - let dom = kuchikiki::parse_html().one(dbg!(response.text().await?)); + let dom = kuchikiki::parse_html().one(response.text().await?); dom.select_first("#main").expect("not main crate docs"); // First we check that the crate-level docs have been rendered as expected. assert_eq!( diff --git a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs index 1ff24eaa3..2c47deeb2 100644 --- a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs +++ b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs @@ -62,7 +62,7 @@ impl TestEnvironment { }; let static_crates_io = TestStaticCratesIo::new().await?; - registry_api_config.static_host = static_crates_io.url().parse().unwrap(); + registry_api_config.static_host = static_crates_io.url().await.parse().unwrap(); let registry_api = RegistryApi::from_config(®istry_api_config)?; @@ -139,6 +139,7 @@ impl TestEnvironment { FakeRelease::new( self.context.pool().unwrap().clone(), self.context.storage().unwrap().clone(), + self.static_crates_io.clone(), ) } From 49366e203335744748efc0f8fff15ff702c761cb Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 11:23:06 +0200 Subject: [PATCH 16/25] kk --- .../src/source_archive/mod.rs | 16 ++++++++----- .../src/testing/static_test_env.rs | 24 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs index a0934ca96..949dc78e1 100644 --- a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -100,6 +100,11 @@ impl SourceArchive { mod tests { use super::*; use crate::testing::static_test_env::{TestStaticCratesIo, create_test_source_archive}; + use docs_rs_types::testing::{KRATE, V0_1}; + + fn client() -> reqwest::Client { + reqwest::Client::builder().build().unwrap() + } #[tokio::test] async fn test_fetch() -> anyhow::Result<()> { @@ -108,13 +113,12 @@ mod tests { ("Cargo.toml", "Cargo.toml"), ])?; - let mut test_env = TestStaticCratesIo::new().await?; - test_env.add("krate", "0.1.0", manifest, zip).await?; + let test_env = TestStaticCratesIo::new().await?; + test_env.add(&KRATE, &V0_1, manifest, zip).await?; - let source_archive = - SourceArchive::load(test_env.client().clone(), test_env.url(), "krate", "0.1.0") - .await? - .expect("not found"); + let source_archive = SourceArchive::load(client(), test_env.url().await, "krate", "0.1.0") + .await? + .expect("not found"); { let info = source_archive.by_name("src/main.rs").expect("should exist"); diff --git a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs index ddd11b298..1ce22064b 100644 --- a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs @@ -1,7 +1,7 @@ use crate::{FileEntry, Manifest}; use anyhow::Result; use docs_rs_types::{KrateName, Version}; -use reqwest::header::RANGE; +use reqwest::header::{HeaderMap, RANGE}; use std::io::{self, Write as _}; use tokio::sync::Mutex; use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; @@ -91,18 +91,20 @@ impl TestStaticCratesIo { .server .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip")) .with_body_from_request(move |request| { - let range = request + let Some((lhs, rhs)) = request .headers() .get(RANGE) - .expect("range header must exists"); - let range = range.to_str().unwrap(); - - let bytes = range.strip_prefix("bytes=").unwrap(); - - let (lhs, rhs) = bytes.split_once("-").unwrap(); - - let lhs: usize = lhs.parse().unwrap(); - let rhs: usize = rhs.parse().unwrap(); + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.strip_prefix("bytes=")) + .and_then(|r| r.split_once("-")) + .and_then(|(lhs, rhs)| { + let lhs: usize = lhs.parse().ok()?; + let rhs: usize = rhs.parse().ok()?; + Some((lhs, rhs)) + }) + else { + return zip.clone(); + }; zip.get(lhs..=rhs).unwrap().to_vec() }) From db28cd3432572c1b205ace7589a2f3343247eb50 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 11:28:22 +0200 Subject: [PATCH 17/25] notes --- .../src/testing/static_test_env.rs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs index 1ce22064b..3e4933e0e 100644 --- a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs @@ -1,11 +1,12 @@ use crate::{FileEntry, Manifest}; use anyhow::Result; use docs_rs_types::{KrateName, Version}; -use reqwest::header::{HeaderMap, RANGE}; +use reqwest::header::RANGE; use std::io::{self, Write as _}; use tokio::sync::Mutex; use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; +/// create a manifest & zip file the same way (roughly, for our tests) as crates.io does. pub fn create_test_source_archive(files: I) -> Result<(Manifest, Vec)> where I: IntoIterator, @@ -52,6 +53,12 @@ where Ok((Manifest { files }, bytes)) } +/// simlulated `static.crates.io` server for our tests, and configured by +/// default in our test environment. +/// +/// Right now just for the source-zip archives. +/// Our shared test-env, and also `FakeRelease`, fill it with +/// data when needed. pub struct TestStaticCratesIo { inner: Mutex, } @@ -91,7 +98,9 @@ impl TestStaticCratesIo { .server .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip")) .with_body_from_request(move |request| { - let Some((lhs, rhs)) = request + // NOTE: mockito itself doesn't understand range requests. + // So we have to parse the header ourselves, and return the correct chunk here. + if let Some((lhs, rhs)) = request .headers() .get(RANGE) .and_then(|h| h.to_str().ok()) @@ -102,11 +111,11 @@ impl TestStaticCratesIo { let rhs: usize = rhs.parse().ok()?; Some((lhs, rhs)) }) - else { - return zip.clone(); - }; - - zip.get(lhs..=rhs).unwrap().to_vec() + { + zip.get(lhs..=rhs).expect("should exist").to_vec() + } else { + zip.clone() + } }) .create_async() .await; From 73aa30a6da4d622e746aee9ba85e6d43036b036c Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 11:35:57 +0200 Subject: [PATCH 18/25] kk --- .../bin/docs_rs_web/src/handlers/crate_details.rs | 5 +---- .../src/testing/test_env/non_blocking.rs | 2 +- crates/lib/docs_rs_registry_api/src/api.rs | 15 ++++++--------- crates/lib/docs_rs_registry_api/src/config.rs | 4 ++-- .../src/testing/static_test_env.rs | 2 ++ 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index a56609f82..5120a1c18 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -293,10 +293,7 @@ impl CrateDetails { } async fn fetch_readme(&self, registry_api: &RegistryApi) -> anyhow::Result> { - let source_archive = match registry_api - .open_source_archive(&self.name, &self.version) - .await - { + let source_archive = match registry_api.source_archive(&self.name, &self.version).await { Ok(archive) => archive, Err(docs_rs_registry_api::Error::MissingReleases) => return Ok(None), Err(err) => return Err(err.into()), diff --git a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs index 2c47deeb2..1ac7646a2 100644 --- a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs +++ b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs @@ -62,7 +62,7 @@ impl TestEnvironment { }; let static_crates_io = TestStaticCratesIo::new().await?; - registry_api_config.static_host = static_crates_io.url().await.parse().unwrap(); + registry_api_config.registry_static_host = static_crates_io.url().await.parse().unwrap(); let registry_api = RegistryApi::from_config(®istry_api_config)?; diff --git a/crates/lib/docs_rs_registry_api/src/api.rs b/crates/lib/docs_rs_registry_api/src/api.rs index 3e4ab3239..be1f4eb0f 100644 --- a/crates/lib/docs_rs_registry_api/src/api.rs +++ b/crates/lib/docs_rs_registry_api/src/api.rs @@ -8,7 +8,7 @@ use docs_rs_types::{KrateName, Version}; use docs_rs_utils::{APP_USER_AGENT, retry_async}; use reqwest::{ StatusCode, - header::{ACCEPT, HeaderValue, USER_AGENT}, + header::{ACCEPT, HeaderMap, HeaderValue, USER_AGENT}, }; use serde::{Deserialize, de::DeserializeOwned}; use tracing::instrument; @@ -26,18 +26,15 @@ impl RegistryApi { pub fn from_config(config: &Config) -> Result { Self::new( config.registry_api_host.clone(), - config.static_host.clone(), + config.registry_static_host.clone(), config.crates_io_api_call_retries, ) } pub fn new(api_base: Url, static_base: Url, max_retries: u32) -> Result { - let headers = vec![ - (USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)), - (ACCEPT, HeaderValue::from_static("application/json")), - ] - .into_iter() - .collect(); + let mut headers = HeaderMap::with_capacity(2); + headers.insert(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); let client = reqwest::Client::builder() .default_headers(headers) @@ -234,7 +231,7 @@ impl RegistryApi { /// /// The returned object can be used to inspect the file-list, and /// fetch single files from the archive via range request. - pub async fn open_source_archive( + pub async fn source_archive( &self, name: &KrateName, version: &Version, diff --git a/crates/lib/docs_rs_registry_api/src/config.rs b/crates/lib/docs_rs_registry_api/src/config.rs index 02aa33604..a6f7a1ec4 100644 --- a/crates/lib/docs_rs_registry_api/src/config.rs +++ b/crates/lib/docs_rs_registry_api/src/config.rs @@ -9,7 +9,7 @@ pub struct Config { pub registry_api_host: Url, #[builder(default = "https://static.crates.io".parse().unwrap())] - pub static_host: Url, + pub registry_static_host: Url, // amount of retries for external API calls, mostly crates.io #[builder(default = 3)] @@ -21,7 +21,7 @@ impl AppConfig for Config { Ok(Self::builder() .maybe_crates_io_api_call_retries(maybe_env("DOCSRS_CRATESIO_API_CALL_RETRIES")?) .maybe_registry_api_host(maybe_env("DOCSRS_REGISTRY_API_HOST")?) - .maybe_static_host(maybe_env("DOCSRS_STATIC_HOST")?) + .maybe_registry_static_host(maybe_env("DOCSRS_REGISTRY_STATIC_HOST")?) .build()) } } diff --git a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs index 3e4933e0e..16e566cb4 100644 --- a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs @@ -7,6 +7,8 @@ use tokio::sync::Mutex; use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; /// create a manifest & zip file the same way (roughly, for our tests) as crates.io does. +/// +/// Source: the `crates_io_crate_zip` subcrate in the crates.io codebase. pub fn create_test_source_archive(files: I) -> Result<(Manifest, Vec)> where I: IntoIterator, From 4a292cf5aaaa4188ad4c5e8d78321e97719e3513 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 11:39:00 +0200 Subject: [PATCH 19/25] ckj --- crates/lib/docs_rs_registry_api/src/source_archive/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs index 949dc78e1..c8f195cc1 100644 --- a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -7,6 +7,7 @@ use futures_util::TryStreamExt as _; use reqwest::{IntoUrl, StatusCode, Url, header::RANGE}; use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; use tokio_util::io::StreamReader; +use tracing::info; pub struct SourceArchive { manifest: Manifest, @@ -30,6 +31,7 @@ impl SourceArchive { version.as_ref() ))?; + info!(%index_url, "fetching source archive manifest"); let response = client.get(index_url.clone()).send().await?; if matches!( response.status(), @@ -62,6 +64,7 @@ impl SourceArchive { let range_start = entry.data_offset; let range_end = entry.data_offset + entry.compressed_size - 1; + info!(%self.zip_url, entry.path, "fetching file from source archive"); let response = self .client .get(self.zip_url.clone()) From aace82a5997a0d68995f1c2741925d7c6d321b80 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 11:49:30 +0200 Subject: [PATCH 20/25] cache --- crates/lib/docs_rs_registry_api/src/api.rs | 2 +- .../src/source_archive/mod.rs | 39 ++++++++++++------- .../src/testing/static_test_env.rs | 5 ++- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/crates/lib/docs_rs_registry_api/src/api.rs b/crates/lib/docs_rs_registry_api/src/api.rs index be1f4eb0f..7051e9078 100644 --- a/crates/lib/docs_rs_registry_api/src/api.rs +++ b/crates/lib/docs_rs_registry_api/src/api.rs @@ -239,7 +239,7 @@ impl RegistryApi { SourceArchive::load( self.client.clone(), self.static_base.clone(), - &name, + name.as_str(), &version.to_string(), ) .await? diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs index c8f195cc1..00675fcf7 100644 --- a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -4,10 +4,22 @@ use crate::source_archive::manifest::{FileEntry, Manifest}; use anyhow::{Result, bail}; use async_compression::tokio::bufread::DeflateDecoder; use futures_util::TryStreamExt as _; -use reqwest::{IntoUrl, StatusCode, Url, header::RANGE}; +use reqwest::{ + StatusCode, Url, + header::{HeaderMap, HeaderName, RANGE}, +}; use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; use tokio_util::io::StreamReader; -use tracing::info; +use tracing::{field, info, instrument}; + +pub static X_CACHE: HeaderName = HeaderName::from_static("x-cache"); + +fn is_cache_hit(hm: &HeaderMap) -> bool { + hm.get(&X_CACHE) + .and_then(|hv| hv.to_str().ok()) + .map(|hv| hv.contains("HIT")) + .unwrap_or(false) +} pub struct SourceArchive { manifest: Manifest, @@ -16,20 +28,16 @@ pub struct SourceArchive { } impl SourceArchive { + #[instrument(skip_all, fields( %base_url, %name, %version, cache_hit=field::Empty))] pub(crate) async fn load( client: reqwest::Client, - base_url: impl IntoUrl, - name: impl AsRef, - version: impl AsRef, + mut base_url: Url, + name: &str, + version: &str, ) -> Result> { - let mut base_url = base_url.into_url()?; base_url.set_path("crates/"); - let index_url = base_url.join(&format!( - "{0}/{0}-{1}.zip.json", - name.as_ref(), - version.as_ref() - ))?; + let index_url = base_url.join(&format!("{0}/{0}-{1}.zip.json", name, version))?; info!(%index_url, "fetching source archive manifest"); let response = client.get(index_url.clone()).send().await?; @@ -41,9 +49,11 @@ impl SourceArchive { } let response = response.error_for_status()?; + tracing::Span::current().record("cache_hit", is_cache_hit(response.headers())); + Ok(Some(Self { manifest: response.json().await?, - zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name.as_ref(), version.as_ref()))?, + zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name, version))?, client, })) } @@ -57,6 +67,7 @@ impl SourceArchive { self.manifest.files.iter().find(|e| e.path == path) } + #[instrument(skip_all, fields(zip_url=%self.zip_url, path=%entry.path, cache_hit=field::Empty))] pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> where W: AsyncWrite + Unpin, @@ -64,7 +75,7 @@ impl SourceArchive { let range_start = entry.data_offset; let range_end = entry.data_offset + entry.compressed_size - 1; - info!(%self.zip_url, entry.path, "fetching file from source archive"); + info!("fetching file from source archive"); let response = self .client .get(self.zip_url.clone()) @@ -73,6 +84,8 @@ impl SourceArchive { .await? .error_for_status()?; + tracing::Span::current().record("cache_hit", is_cache_hit(response.headers())); + let stream = response.bytes_stream().map_err(std::io::Error::other); let mut reader = StreamReader::new(stream); diff --git a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs index 16e566cb4..5a0506fb4 100644 --- a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs +++ b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs @@ -4,6 +4,7 @@ use docs_rs_types::{KrateName, Version}; use reqwest::header::RANGE; use std::io::{self, Write as _}; use tokio::sync::Mutex; +use url::Url; use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; /// create a manifest & zip file the same way (roughly, for our tests) as crates.io does. @@ -126,8 +127,8 @@ impl TestStaticCratesIo { Ok(()) } - pub async fn url(&self) -> String { + pub async fn url(&self) -> Url { let inner = self.inner.lock().await; - inner.server.url() + Url::parse(&inner.server.url()).unwrap() } } From 21b4e9aff87802669215ae3d85eef6236c5da2f0 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 12:00:48 +0200 Subject: [PATCH 21/25] wip --- crates/lib/docs_rs_registry_api/src/source_archive/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs index 00675fcf7..db5649147 100644 --- a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -10,7 +10,7 @@ use reqwest::{ }; use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; use tokio_util::io::StreamReader; -use tracing::{field, info, instrument}; +use tracing::{debug, field, instrument}; pub static X_CACHE: HeaderName = HeaderName::from_static("x-cache"); @@ -39,7 +39,7 @@ impl SourceArchive { let index_url = base_url.join(&format!("{0}/{0}-{1}.zip.json", name, version))?; - info!(%index_url, "fetching source archive manifest"); + debug!(%index_url, "fetching source archive manifest"); let response = client.get(index_url.clone()).send().await?; if matches!( response.status(), @@ -75,7 +75,7 @@ impl SourceArchive { let range_start = entry.data_offset; let range_end = entry.data_offset + entry.compressed_size - 1; - info!("fetching file from source archive"); + debug!(range_start, range_end, "fetching file from source archive"); let response = self .client .get(self.zip_url.clone()) From 522c235d5b1e2412f183778c5b2f05792d494952 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 12:02:53 +0200 Subject: [PATCH 22/25] fmt --- crates/bin/docs_rs_web/src/handlers/crate_details.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index 5120a1c18..2ec4b1a51 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use crate::{ cache::CachePolicy, error::{AxumNope, AxumResult}, @@ -30,6 +28,7 @@ use docs_rs_types::{ }; use futures_util::stream::TryStreamExt; use serde_json::Value; +use std::sync::Arc; use tracing::warn; // TODO: Add target name and versions From 60d11727472370838e669b5806b464e6967043be Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 12:15:35 +0200 Subject: [PATCH 23/25] kk --- crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs | 2 +- .../docs_rs_context/src/testing/test_env/non_blocking.rs | 4 ++-- crates/lib/docs_rs_repository_stats/src/workspaces.rs | 2 +- crates/lib/docs_rs_test_fakes/src/legacy.rs | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs b/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs index ea85023bd..b04e4bd67 100644 --- a/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs +++ b/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs @@ -348,7 +348,7 @@ mod tests { impl TestEnv { async fn fake_release(&self) -> FakeRelease<'_> { - FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone()) + FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone(), None) } } diff --git a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs index 1ac7646a2..12af4bd56 100644 --- a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs +++ b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs @@ -62,7 +62,7 @@ impl TestEnvironment { }; let static_crates_io = TestStaticCratesIo::new().await?; - registry_api_config.registry_static_host = static_crates_io.url().await.parse().unwrap(); + registry_api_config.registry_static_host = static_crates_io.url().await; let registry_api = RegistryApi::from_config(®istry_api_config)?; @@ -139,7 +139,7 @@ impl TestEnvironment { FakeRelease::new( self.context.pool().unwrap().clone(), self.context.storage().unwrap().clone(), - self.static_crates_io.clone(), + Some(self.static_crates_io.clone()), ) } diff --git a/crates/lib/docs_rs_repository_stats/src/workspaces.rs b/crates/lib/docs_rs_repository_stats/src/workspaces.rs index f3485273b..dd3719bd0 100644 --- a/crates/lib/docs_rs_repository_stats/src/workspaces.rs +++ b/crates/lib/docs_rs_repository_stats/src/workspaces.rs @@ -218,7 +218,7 @@ mod tests { impl TestEnv { async fn fake_release(&self) -> FakeRelease<'_> { - FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone()) + FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone(), None) } } diff --git a/crates/lib/docs_rs_test_fakes/src/legacy.rs b/crates/lib/docs_rs_test_fakes/src/legacy.rs index 7e80050b9..01fcca6fd 100644 --- a/crates/lib/docs_rs_test_fakes/src/legacy.rs +++ b/crates/lib/docs_rs_test_fakes/src/legacy.rs @@ -72,7 +72,7 @@ where pub struct FakeRelease<'a> { pool: Pool, storage: Arc, - static_crates_io: Arc, + static_crates_io: Option>, package: MetadataPackage, builds: Option>, /// name, content @@ -113,7 +113,7 @@ impl<'a> FakeRelease<'a> { pub fn new( pool: Pool, storage: Arc, - static_crates_io: Arc, + static_crates_io: Option>, ) -> Self { FakeRelease { pool, @@ -488,12 +488,12 @@ impl<'a> FakeRelease<'a> { let krate_name: KrateName = package.name.parse()?; - { + if let Some(static_crates_io) = self.static_crates_io { let (manifest, zip) = docs_rs_registry_api::testing::static_test_env::create_test_source_archive( owned_source_files, )?; - self.static_crates_io + static_crates_io .add(&krate_name, &package.version, manifest, zip) .await?; } From 7fd208516a8b1a32d0702c7ff69eadc99fe3ec61 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 12:24:18 +0200 Subject: [PATCH 24/25] sort --- crates/lib/docs_rs_registry_api/Cargo.toml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/lib/docs_rs_registry_api/Cargo.toml b/crates/lib/docs_rs_registry_api/Cargo.toml index ba0308d9b..9a17e0302 100644 --- a/crates/lib/docs_rs_registry_api/Cargo.toml +++ b/crates/lib/docs_rs_registry_api/Cargo.toml @@ -6,10 +6,7 @@ repository = "https://github.com/rust-lang/docs.rs" edition = "2024" [features] -testing = [ - "dep:mockito", - "dep:zip", -] +testing = ["dep:mockito", "dep:zip"] [dependencies] anyhow = { workspace = true } @@ -21,6 +18,7 @@ docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_types = { path = "../docs_rs_types" } docs_rs_utils = { path = "../docs_rs_utils" } futures-util = { workspace = true } +mockito = { workspace = true, optional = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -30,10 +28,8 @@ tokio = { workspace = true } tokio-util = { workspace = true } tracing = { workspace = true } url = { workspace = true } -mockito = { workspace = true, optional = true } zip = { workspace = true, optional = true } - [dev-dependencies] docs_rs_types = { path = "../docs_rs_types", features = ["testing"] } mime = { workspace = true } From 5883407549d768523c335b483ab813da475206bc Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 2 Jul 2026 12:36:43 +0200 Subject: [PATCH 25/25] less copy --- crates/lib/docs_rs_test_fakes/src/legacy.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/lib/docs_rs_test_fakes/src/legacy.rs b/crates/lib/docs_rs_test_fakes/src/legacy.rs index 01fcca6fd..e1625361c 100644 --- a/crates/lib/docs_rs_test_fakes/src/legacy.rs +++ b/crates/lib/docs_rs_test_fakes/src/legacy.rs @@ -462,11 +462,7 @@ impl<'a> FakeRelease<'a> { let source_tmp = create_temp_dir(); store_files_into(&self.source_files, source_tmp.path())?; - let mut owned_source_files: Vec<_> = self - .source_files - .iter() - .map(|(n, c)| (n.to_string(), c.to_vec())) - .collect(); + let mut additional_source_files: Vec<(String, Vec)> = Vec::new(); if !self.no_cargo_toml && !self @@ -483,7 +479,7 @@ impl<'a> FakeRelease<'a> { "# ); store_files_into(&[("Cargo.toml", content.as_bytes())], source_tmp.path())?; - owned_source_files.push(("Cargo.toml".into(), content.as_bytes().to_vec())); + additional_source_files.push(("Cargo.toml".into(), content.as_bytes().to_vec())); } let krate_name: KrateName = package.name.parse()?; @@ -491,7 +487,11 @@ impl<'a> FakeRelease<'a> { if let Some(static_crates_io) = self.static_crates_io { let (manifest, zip) = docs_rs_registry_api::testing::static_test_env::create_test_source_archive( - owned_source_files, + self.source_files.iter().cloned().chain( + additional_source_files + .iter() + .map(|(n, c)| (n.as_str(), c.as_slice())), + ), )?; static_crates_io .add(&krate_name, &package.version, manifest, zip)