diff --git a/Cargo.lock b/Cargo.lock index 0810f40d8..84802de19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2274,12 +2274,14 @@ name = "docs_rs_registry_api" version = "0.1.0" dependencies = [ "anyhow", + "async-compression", "bon", "chrono", "docs_rs_config", "docs_rs_env_vars", "docs_rs_types", "docs_rs_utils", + "futures-util", "mime", "mockito", "reqwest", @@ -2289,8 +2291,10 @@ dependencies = [ "test-case", "thiserror", "tokio", + "tokio-util", "tracing", "url", + "zip", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5ac02140f..939b4c647 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ edition = "2024" [workspace.dependencies] anyhow = { version = "1.0.42", features = ["backtrace"] } askama = "0.16.0" +async-compression = { version = "0.4.32", features = ["bzip2", "deflate", "gzip", "tokio", "zstd"] } async-stream = "0.3.5" axum-extra = { version = "0.12.0", features = ["middleware", "routing", "typed-header"] } base64 = "0.22" @@ -62,6 +63,7 @@ tempfile = "3.1.0" test-case = "3.0.0" thiserror = "2.0.3" tokio = { version = "1.0", features = ["macros", "process", "rt-multi-thread", "signal", "sync"] } +tokio-util = { version = "0.7.15", default-features = false, features = ["io-util"] } toml = "1.0.0" tracing = "0.1.37" url = { version = "2.1.1", features = ["serde"] } diff --git a/crates/bin/docs_rs_web/Cargo.toml b/crates/bin/docs_rs_web/Cargo.toml index f2d3d86fb..20d9ac969 100644 --- a/crates/bin/docs_rs_web/Cargo.toml +++ b/crates/bin/docs_rs_web/Cargo.toml @@ -83,6 +83,7 @@ docs_rs_config = { path = "../../lib/docs_rs_config", features = ["testing"] } docs_rs_context = { path = "../../lib/docs_rs_context", features = ["testing"] } docs_rs_database = { path = "../../lib/docs_rs_database", features = ["testing"] } docs_rs_headers = { path = "../../lib/docs_rs_headers", features = ["testing"] } +docs_rs_registry_api = { path = "../../lib/docs_rs_registry_api", features = ["testing"] } docs_rs_storage = { path = "../../lib/docs_rs_storage", features = ["testing"] } docs_rs_test_fakes = { path = "../../lib/docs_rs_test_fakes" } docs_rs_types = { path = "../../lib/docs_rs_types", features = ["testing"] } diff --git a/crates/bin/docs_rs_web/src/handlers/crate_details.rs b/crates/bin/docs_rs_web/src/handlers/crate_details.rs index 67409ee41..2ec4b1a51 100644 --- a/crates/bin/docs_rs_web/src/handlers/crate_details.rs +++ b/crates/bin/docs_rs_web/src/handlers/crate_details.rs @@ -15,15 +15,14 @@ use crate::{ use anyhow::{Context, Result}; use askama::Template; use axum::{ - extract::Extension, + Extension, response::{IntoResponse, Response as AxumResponse}, }; use chrono::{DateTime, Utc}; use docs_rs_cargo_metadata::{Dependency, ReleaseDependencyList}; use docs_rs_database::crate_details::{Release, parse_doc_targets}; use docs_rs_headers::CanonicalUrl; -use docs_rs_registry_api::OwnerKind; -use docs_rs_storage::{AsyncStorage, PathNotFoundError}; +use docs_rs_registry_api::{OwnerKind, RegistryApi}; use docs_rs_types::{ BuildId, BuildStatus, CrateId, Duration, KrateName, ReleaseId, ReqVersion, Version, }; @@ -292,26 +291,18 @@ impl CrateDetails { Ok(Some(crate_details)) } - async fn fetch_readme(&self, storage: &AsyncStorage) -> anyhow::Result> { - let manifest = match storage - .fetch_source_file( - &self.name, - &self.version, - self.latest_build_id, - "Cargo.toml", - self.archive_storage, - ) - .await - { - Ok(manifest) => manifest, - Err(err) if err.is::() => { - return Ok(None); - } - Err(err) => { - return Err(err); - } + async fn fetch_readme(&self, registry_api: &RegistryApi) -> anyhow::Result> { + let source_archive = match registry_api.source_archive(&self.name, &self.version).await { + Ok(archive) => archive, + Err(docs_rs_registry_api::Error::MissingReleases) => return Ok(None), + Err(err) => return Err(err.into()), + }; + + let Some(cargo_toml) = source_archive.by_name("Cargo.toml") else { + return Ok(None); }; - let manifest = String::from_utf8(manifest.content) + + let manifest = String::from_utf8(source_archive.fetch_bytes(cargo_toml).await?) .context("parsing Cargo.toml")? .parse::() .context("parsing Cargo.toml")?; @@ -321,29 +312,15 @@ impl CrateDetails { Some(toml::Value::String(path)) => vec![path.as_ref()], _ => vec!["README.md", "README.txt", "README"], }; + for path in &paths { - match storage - .fetch_source_file( - &self.name, - &self.version, - self.latest_build_id, - path, - self.archive_storage, - ) - .await - { - Ok(readme) => { - let readme = String::from_utf8(readme.content) - .with_context(|| format!("parsing {path} content"))?; - return Ok(Some(readme)); - } - Err(err) if err.is::() => { - continue; - } - Err(err) => { - return Err(err); - } - } + let Some(readme) = source_archive.by_name(path) else { + continue; + }; + + return Ok(Some( + String::from_utf8_lossy(&source_archive.fetch_bytes(readme).await?).to_string(), + )); } Ok(None) } @@ -450,10 +427,10 @@ impl_axum_webpage! { cpu_intensive_rendering = true, } -#[tracing::instrument(skip(conn, storage))] +#[tracing::instrument(skip(conn))] pub(crate) async fn crate_details_handler( params: RustdocParams, - Extension(storage): Extension>, + Extension(registry_api): Extension>, mut conn: DbConnection, ) -> AxumResult { let matched_release = match_version(&mut conn, params.name(), params.req_version()) @@ -487,7 +464,7 @@ pub(crate) async fn crate_details_handler( // before we do the long S3 requests. drop(conn); - match details.fetch_readme(&storage).await { + match details.fetch_readme(®istry_api).await { Ok(readme) => details.readme = readme.or(details.readme), Err(e) => warn!(?e, "error fetching readme"), } @@ -1925,88 +1902,90 @@ mod tests { Ok(()) } - #[test] - fn readme() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("dummy") - .version("0.1.0") - .readme_only_database("database readme") - .create() - .await?; + #[tokio::test(flavor = "multi_thread")] + async fn readme() -> Result<()> { + let env = TestEnvironment::builder() + .registry_api_config(docs_rs_registry_api::Config::builder().build()) + .build() + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.2.0") - .readme_only_database("database readme") - .source_file("README.md", b"storage readme") - .create() - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.1.0") + .readme_only_database("database readme") + .create() + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.3.0") - .source_file("README.md", b"storage readme") - .create() - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.2.0") + .readme_only_database("database readme") + .source_file("README.md", b"storage readme") + .create() + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.4.0") - .readme_only_database("database readme") - .source_file("MEREAD", b"storage meread") - .source_file("Cargo.toml", br#"package.readme = "MEREAD""#) - .create() - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.3.0") + .source_file("README.md", b"storage readme") + .create() + .await?; - env.fake_release() - .await - .name("dummy") - .version("0.5.0") - .readme_only_database("database readme") - .source_file("README.md", b"storage readme") - .no_cargo_toml() - .create() - .await?; + env.fake_release() + .await + .name("dummy") + .version("0.4.0") + .readme_only_database("database readme") + .source_file("MEREAD", b"storage meread") + .source_file("Cargo.toml", br#"package.readme = "MEREAD""#) + .create() + .await?; - let check_readme = |path: String, content: String| { - let env = env.clone(); - async move { - let resp = env.web_app().await.get(&path).await.unwrap(); - let body = resp.text().await.unwrap(); - assert!(body.contains(&content)); - } - }; + env.fake_release() + .await + .name("dummy") + .version("0.5.0") + .readme_only_database("database readme") + .source_file("README.md", b"storage readme") + .no_cargo_toml() + .create() + .await?; - check_readme("/crate/dummy/0.1.0".into(), "database readme".into()).await; - check_readme("/crate/dummy/0.2.0".into(), "storage readme".into()).await; - check_readme("/crate/dummy/0.3.0".into(), "storage readme".into()).await; - check_readme("/crate/dummy/0.4.0".into(), "storage meread".into()).await; + async fn check_readme(env: &TestEnvironment, path: &str, content: &str) { + let resp = env.web_app().await.assert_success(path).await.unwrap(); + let body = resp.text().await.unwrap(); + assert!(body.contains(content)); + } - let mut conn = env.async_conn().await?; - let details = crate_details(&mut conn, "dummy", "0.5.0", None).await; - assert!(matches!( - details.fetch_readme(env.storage()?).await, - Ok(None) - )); - Ok(()) - }); + check_readme(&env, "/crate/dummy/0.1.0", "database readme").await; + check_readme(&env, "/crate/dummy/0.2.0", "storage readme").await; + check_readme(&env, "/crate/dummy/0.3.0", "storage readme").await; + check_readme(&env, "/crate/dummy/0.4.0", "storage meread").await; + + let mut conn = env.async_conn().await?; + let details = crate_details(&mut conn, "dummy", "0.5.0", None).await; + let registry_api = env.registry_api()?; + assert!(matches!(details.fetch_readme(registry_api).await, Ok(None))); + Ok(()) } - #[test] - fn no_readme() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("dummy") - .version("0.2.0") - .source_file( - "Cargo.toml", - br#"[package] + #[tokio::test(flavor = "multi_thread")] + async fn no_readme() -> Result<()> { + let env = TestEnvironment::builder() + .registry_api_config(docs_rs_registry_api::Config::builder().build()) + .build() + .await?; + + env.fake_release() + .await + .name("dummy") + .version("0.2.0") + .source_file( + "Cargo.toml", + br#"[package] name = "dummy" version = "0.2.0" @@ -2014,42 +1993,40 @@ version = "0.2.0" name = "dummy" path = "src/lib.rs" "#, - ) - .source_file( - "src/lib.rs", - b"//! # Crate-level docs + ) + .source_file( + "src/lib.rs", + b"//! # Crate-level docs //! //! ``` //! let x = 21; //! ``` ", - ) - .target_source("src/lib.rs") - .create() - .await?; + ) + .target_source("src/lib.rs") + .create() + .await?; - let web = env.web_app().await; - let response = web.get("/crate/dummy/0.2.0").await?; - assert!(response.status().is_success()); + let web = env.web_app().await; + let response = web.assert_success("/crate/dummy/0.2.0").await?; - let dom = kuchikiki::parse_html().one(response.text().await?); - dom.select_first("#main").expect("not main crate docs"); - // First we check that the crate-level docs have been rendered as expected. - assert_eq!( - dom.select_first("#main h1") - .expect("no h1 found") - .text_contents(), - "Crate-level docs" - ); - // Then we check that by default, the language used for highlighting is rust. - assert_eq!( - dom.select_first("#main pre .syntax-source.syntax-rust") - .expect("no rust code block found") - .text_contents(), - "let x = 21;\n" - ); - Ok(()) - }); + let dom = kuchikiki::parse_html().one(response.text().await?); + dom.select_first("#main").expect("not main crate docs"); + // First we check that the crate-level docs have been rendered as expected. + assert_eq!( + dom.select_first("#main h1") + .expect("no h1 found") + .text_contents(), + "Crate-level docs" + ); + // Then we check that by default, the language used for highlighting is rust. + assert_eq!( + dom.select_first("#main pre .syntax-source.syntax-rust") + .expect("no rust code block found") + .text_contents(), + "let x = 21;\n" + ); + Ok(()) } #[test] diff --git a/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs b/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs index ea85023bd..b04e4bd67 100644 --- a/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs +++ b/crates/lib/docs_rs_build_queue/src/queue/non_blocking.rs @@ -348,7 +348,7 @@ mod tests { impl TestEnv { async fn fake_release(&self) -> FakeRelease<'_> { - FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone()) + FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone(), None) } } diff --git a/crates/lib/docs_rs_context/Cargo.toml b/crates/lib/docs_rs_context/Cargo.toml index 5702d7dee..8eac17529 100644 --- a/crates/lib/docs_rs_context/Cargo.toml +++ b/crates/lib/docs_rs_context/Cargo.toml @@ -12,6 +12,7 @@ testing = [ "docs_rs_fastly/testing", "docs_rs_logging/testing", "docs_rs_storage/testing", + "docs_rs_registry_api/testing", ] [dependencies] diff --git a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs index 98583d1f5..12af4bd56 100644 --- a/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs +++ b/crates/lib/docs_rs_context/src/testing/test_env/non_blocking.rs @@ -6,7 +6,7 @@ use docs_rs_config::AppConfig; use docs_rs_database::{AsyncPoolClient, Config as DatabaseConfig, testing::TestDatabase}; use docs_rs_fastly::Cdn; use docs_rs_opentelemetry::testing::{CollectedMetrics, TestMetrics}; -use docs_rs_registry_api::RegistryApi; +use docs_rs_registry_api::{RegistryApi, testing::static_test_env::TestStaticCratesIo}; use docs_rs_storage::{Config as StorageConfig, testing::TestStorage}; use docs_rs_test_fakes::FakeRelease; use std::{ops::Deref, sync::Arc}; @@ -14,6 +14,7 @@ use std::{ops::Deref, sync::Arc}; pub struct TestEnvironment { context: Arc, config: Arc, + static_crates_io: Arc, // so we can allow asserting collected metrics later. metrics: TestMetrics, #[allow(dead_code)] // we need to keep the storage so it can be cleaned up. @@ -54,12 +55,14 @@ impl TestEnvironment { C::test_config()? }); - let registry_api_config = - Arc::new(if let Some(registry_api_config) = registry_api_config { - registry_api_config - } else { - docs_rs_registry_api::Config::from_environment()? - }); + let mut registry_api_config = if let Some(registry_api_config) = registry_api_config { + registry_api_config + } else { + docs_rs_registry_api::Config::from_environment()? + }; + + let static_crates_io = TestStaticCratesIo::new().await?; + registry_api_config.registry_static_host = static_crates_io.url().await; let registry_api = RegistryApi::from_config(®istry_api_config)?; @@ -90,6 +93,7 @@ impl TestEnvironment { )); Ok(Self { + static_crates_io: Arc::new(static_crates_io), config: app_config, context: Context::builder() .with_runtime() @@ -98,7 +102,7 @@ impl TestEnvironment { .pool(db_config.into(), db.pool().clone()) .storage(storage_config.clone(), test_storage.storage()) .build_queue(build_queue_config, build_queue) - .registry_api(registry_api_config, registry_api.into()) + .registry_api(Arc::new(registry_api_config), registry_api.into()) .with_repository_stats()? .maybe_cdn( Arc::new(docs_rs_fastly::Config::test_config()?), @@ -135,6 +139,7 @@ impl TestEnvironment { FakeRelease::new( self.context.pool().unwrap().clone(), self.context.storage().unwrap().clone(), + Some(self.static_crates_io.clone()), ) } diff --git a/crates/lib/docs_rs_registry_api/Cargo.toml b/crates/lib/docs_rs_registry_api/Cargo.toml index ccbfa8083..9a17e0302 100644 --- a/crates/lib/docs_rs_registry_api/Cargo.toml +++ b/crates/lib/docs_rs_registry_api/Cargo.toml @@ -5,21 +5,30 @@ license = "MIT" repository = "https://github.com/rust-lang/docs.rs" edition = "2024" +[features] +testing = ["dep:mockito", "dep:zip"] + [dependencies] anyhow = { workspace = true } +async-compression = { workspace = true } bon = { workspace = true } chrono = { workspace = true } docs_rs_config = { path = "../docs_rs_config" } docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_types = { path = "../docs_rs_types" } docs_rs_utils = { path = "../docs_rs_utils" } +futures-util = { workspace = true } +mockito = { workspace = true, optional = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } tracing = { workspace = true } url = { workspace = true } +zip = { workspace = true, optional = true } [dev-dependencies] docs_rs_types = { path = "../docs_rs_types", features = ["testing"] } @@ -27,6 +36,7 @@ mime = { workspace = true } mockito = { workspace = true } test-case = { workspace = true } tokio = { workspace = true } +zip = { workspace = true } [lints] workspace = true diff --git a/crates/lib/docs_rs_registry_api/src/api.rs b/crates/lib/docs_rs_registry_api/src/api.rs index 594162dd8..7051e9078 100644 --- a/crates/lib/docs_rs_registry_api/src/api.rs +++ b/crates/lib/docs_rs_registry_api/src/api.rs @@ -1,5 +1,5 @@ use crate::{ - Config, + Config, SourceArchive, error::{Error, Result}, models::{ApiErrors, CrateData, CrateOwner, OwnerKind, ReleaseData, Search, SearchResponse}, }; @@ -8,7 +8,7 @@ use docs_rs_types::{KrateName, Version}; use docs_rs_utils::{APP_USER_AGENT, retry_async}; use reqwest::{ StatusCode, - header::{ACCEPT, HeaderValue, USER_AGENT}, + header::{ACCEPT, HeaderMap, HeaderValue, USER_AGENT}, }; use serde::{Deserialize, de::DeserializeOwned}; use tracing::instrument; @@ -17,6 +17,7 @@ use url::Url; #[derive(Debug)] pub struct RegistryApi { api_base: Url, + static_base: Url, max_retries: u32, client: reqwest::Client, } @@ -25,17 +26,15 @@ impl RegistryApi { pub fn from_config(config: &Config) -> Result { Self::new( config.registry_api_host.clone(), + config.registry_static_host.clone(), config.crates_io_api_call_retries, ) } - pub fn new(api_base: Url, max_retries: u32) -> Result { - let headers = vec![ - (USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)), - (ACCEPT, HeaderValue::from_static("application/json")), - ] - .into_iter() - .collect(); + pub fn new(api_base: Url, static_base: Url, max_retries: u32) -> Result { + let mut headers = HeaderMap::with_capacity(2); + headers.insert(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT)); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); let client = reqwest::Client::builder() .default_headers(headers) @@ -43,6 +42,7 @@ impl RegistryApi { Ok(Self { api_base, + static_base, client, max_retries, }) @@ -224,6 +224,27 @@ impl RegistryApi { meta: response.meta.ok_or(Error::MissingMetadata)?, }) } + + /// open the crates.io source archive zip for this crate / version. + /// + /// We directly fetch the manifest JSON file. + /// + /// The returned object can be used to inspect the file-list, and + /// fetch single files from the archive via range request. + pub async fn source_archive( + &self, + name: &KrateName, + version: &Version, + ) -> Result { + SourceArchive::load( + self.client.clone(), + self.static_base.clone(), + name.as_str(), + &version.to_string(), + ) + .await? + .ok_or(Error::MissingReleases) + } } #[cfg(test)] @@ -238,6 +259,7 @@ mod tests { async fn test_search(status: StatusCode, body: impl Serialize) -> Result { let mut crates_io_api = mockito::Server::new_async().await; + let static_server = mockito::Server::new_async().await; let _m = crates_io_api .mock("GET", "/api/v1/crates?q=foo") @@ -247,7 +269,11 @@ mod tests { .create_async() .await; - let api = RegistryApi::new(crates_io_api.url().parse().unwrap(), 0)?; + let api = RegistryApi::new( + crates_io_api.url().parse().unwrap(), + static_server.url().parse().unwrap(), + 0, + )?; api.search("q=foo").await } @@ -257,6 +283,7 @@ mod tests { version: &Version, ) -> Result> { let mut crates_io_api = mockito::Server::new_async().await; + let static_server = mockito::Server::new_async().await; let _m = crates_io_api .mock("GET", "/api/v1/crates/krate/versions") @@ -266,7 +293,11 @@ mod tests { .create_async() .await; - let api = RegistryApi::new(crates_io_api.url().parse().unwrap(), 0)?; + let api = RegistryApi::new( + crates_io_api.url().parse().unwrap(), + static_server.url().parse().unwrap(), + 0, + )?; api.get_release_data(&KRATE, version).await } diff --git a/crates/lib/docs_rs_registry_api/src/config.rs b/crates/lib/docs_rs_registry_api/src/config.rs index ef5f8b118..a6f7a1ec4 100644 --- a/crates/lib/docs_rs_registry_api/src/config.rs +++ b/crates/lib/docs_rs_registry_api/src/config.rs @@ -8,6 +8,9 @@ pub struct Config { #[builder(default = "https://crates.io".parse().unwrap())] pub registry_api_host: Url, + #[builder(default = "https://static.crates.io".parse().unwrap())] + pub registry_static_host: Url, + // amount of retries for external API calls, mostly crates.io #[builder(default = 3)] pub crates_io_api_call_retries: u32, @@ -18,6 +21,7 @@ impl AppConfig for Config { Ok(Self::builder() .maybe_crates_io_api_call_retries(maybe_env("DOCSRS_CRATESIO_API_CALL_RETRIES")?) .maybe_registry_api_host(maybe_env("DOCSRS_REGISTRY_API_HOST")?) + .maybe_registry_static_host(maybe_env("DOCSRS_REGISTRY_STATIC_HOST")?) .build()) } } diff --git a/crates/lib/docs_rs_registry_api/src/lib.rs b/crates/lib/docs_rs_registry_api/src/lib.rs index 1763fd6b2..f54b14461 100644 --- a/crates/lib/docs_rs_registry_api/src/lib.rs +++ b/crates/lib/docs_rs_registry_api/src/lib.rs @@ -2,8 +2,16 @@ mod api; mod config; mod error; mod models; +mod source_archive; + +#[cfg(any(test, feature = "testing"))] +pub mod testing; pub use api::RegistryApi; pub use config::Config; pub use error::Error; pub use models::{CrateData, CrateOwner, OwnerKind, ReleaseData, Search}; +pub use source_archive::{ + SourceArchive, + manifest::{FileEntry, Manifest}, +}; diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/manifest.rs b/crates/lib/docs_rs_registry_api/src/source_archive/manifest.rs new file mode 100644 index 000000000..4f88ea5ec --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/source_archive/manifest.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; + +/// archive manifest serde structs, copied from +/// https://github.com/rust-lang/crates.io/blob/5274087feb193ee490e9a6bbdf2e18e74e9ddaeb/crates/crates_io_crate_zip/src/lib.rs +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Manifest { + /// One entry per file in the zip, sorted alphabetically by path. + pub files: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileEntry { + /// Realtive path (without the leading `{name}-{version}/` component of + /// the tarball). + pub path: String, + /// Byte offset in the zip where this entry's compressed payload begins. + pub data_offset: u64, + /// Length of the compressed contents in bytes. + pub compressed_size: u64, + /// Length of the uncompressed contents in bytes. + pub uncompressed_size: u64, + /// How the payload is compressed: `"deflate"` or `"store"`. + pub compression: String, + /// Lowercase hex sha256 of the uncompressed contents. + pub sha256: String, +} diff --git a/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs new file mode 100644 index 000000000..db5649147 --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/source_archive/mod.rs @@ -0,0 +1,151 @@ +pub mod manifest; + +use crate::source_archive::manifest::{FileEntry, Manifest}; +use anyhow::{Result, bail}; +use async_compression::tokio::bufread::DeflateDecoder; +use futures_util::TryStreamExt as _; +use reqwest::{ + StatusCode, Url, + header::{HeaderMap, HeaderName, RANGE}, +}; +use tokio::io::{self, AsyncWrite, AsyncWriteExt as _}; +use tokio_util::io::StreamReader; +use tracing::{debug, field, instrument}; + +pub static X_CACHE: HeaderName = HeaderName::from_static("x-cache"); + +fn is_cache_hit(hm: &HeaderMap) -> bool { + hm.get(&X_CACHE) + .and_then(|hv| hv.to_str().ok()) + .map(|hv| hv.contains("HIT")) + .unwrap_or(false) +} + +pub struct SourceArchive { + manifest: Manifest, + zip_url: Url, + client: reqwest::Client, +} + +impl SourceArchive { + #[instrument(skip_all, fields( %base_url, %name, %version, cache_hit=field::Empty))] + pub(crate) async fn load( + client: reqwest::Client, + mut base_url: Url, + name: &str, + version: &str, + ) -> Result> { + base_url.set_path("crates/"); + + let index_url = base_url.join(&format!("{0}/{0}-{1}.zip.json", name, version))?; + + debug!(%index_url, "fetching source archive manifest"); + let response = client.get(index_url.clone()).send().await?; + if matches!( + response.status(), + StatusCode::NOT_FOUND | StatusCode::FORBIDDEN + ) { + return Ok(None); + } + let response = response.error_for_status()?; + + tracing::Span::current().record("cache_hit", is_cache_hit(response.headers())); + + Ok(Some(Self { + manifest: response.json().await?, + zip_url: base_url.join(&format!("{0}/{0}-{1}.zip", name, version))?, + client, + })) + } + + pub fn entries(&self) -> impl Iterator { + self.manifest.files.iter() + } + + pub fn by_name(&self, path: impl AsRef) -> Option<&FileEntry> { + let path = path.as_ref(); + self.manifest.files.iter().find(|e| e.path == path) + } + + #[instrument(skip_all, fields(zip_url=%self.zip_url, path=%entry.path, cache_hit=field::Empty))] + pub async fn fetch(&self, entry: &FileEntry, writer: &mut W) -> Result<()> + where + W: AsyncWrite + Unpin, + { + let range_start = entry.data_offset; + let range_end = entry.data_offset + entry.compressed_size - 1; + + debug!(range_start, range_end, "fetching file from source archive"); + let response = self + .client + .get(self.zip_url.clone()) + .header(RANGE, format!("bytes={range_start}-{range_end}",)) + .send() + .await? + .error_for_status()?; + + tracing::Span::current().record("cache_hit", is_cache_hit(response.headers())); + + let stream = response.bytes_stream().map_err(std::io::Error::other); + let mut reader = StreamReader::new(stream); + + match entry.compression.as_str() { + "deflate" => { + let mut decoder = DeflateDecoder::new(reader); + io::copy(&mut decoder, writer).await?; + } + "store" => { + io::copy(&mut reader, writer).await?; + } + compression => bail!("unsupported zip compression: {}", compression), + } + + writer.flush().await?; + + Ok(()) + } + + pub async fn fetch_bytes(&self, entry: &FileEntry) -> Result> { + let mut buf = Vec::new(); + self.fetch(entry, &mut buf).await?; + Ok(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::static_test_env::{TestStaticCratesIo, create_test_source_archive}; + use docs_rs_types::testing::{KRATE, V0_1}; + + fn client() -> reqwest::Client { + reqwest::Client::builder().build().unwrap() + } + + #[tokio::test] + async fn test_fetch() -> anyhow::Result<()> { + let (manifest, zip) = create_test_source_archive([ + ("src/main.rs", "src/main.rs"), + ("Cargo.toml", "Cargo.toml"), + ])?; + + let test_env = TestStaticCratesIo::new().await?; + test_env.add(&KRATE, &V0_1, manifest, zip).await?; + + let source_archive = SourceArchive::load(client(), test_env.url().await, "krate", "0.1.0") + .await? + .expect("not found"); + + { + let info = source_archive.by_name("src/main.rs").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(info).await?, b"src/main.rs"); + } + + { + let info = source_archive.by_name("Cargo.toml").expect("should exist"); + assert_eq!(source_archive.fetch_bytes(info).await?, b"Cargo.toml"); + } + + Ok(()) + } +} diff --git a/crates/lib/docs_rs_registry_api/src/testing/mod.rs b/crates/lib/docs_rs_registry_api/src/testing/mod.rs new file mode 100644 index 000000000..7d93d6837 --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/testing/mod.rs @@ -0,0 +1 @@ +pub mod static_test_env; diff --git a/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs new file mode 100644 index 000000000..5a0506fb4 --- /dev/null +++ b/crates/lib/docs_rs_registry_api/src/testing/static_test_env.rs @@ -0,0 +1,134 @@ +use crate::{FileEntry, Manifest}; +use anyhow::Result; +use docs_rs_types::{KrateName, Version}; +use reqwest::header::RANGE; +use std::io::{self, Write as _}; +use tokio::sync::Mutex; +use url::Url; +use zip::{CompressionMethod, ZipWriter, write::SimpleFileOptions}; + +/// create a manifest & zip file the same way (roughly, for our tests) as crates.io does. +/// +/// Source: the `crates_io_crate_zip` subcrate in the crates.io codebase. +pub fn create_test_source_archive(files: I) -> Result<(Manifest, Vec)> +where + I: IntoIterator, + N: ToString, + B: AsRef<[u8]>, +{ + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Deflated) + .compression_level(Some(9)); + + let buf = Vec::new(); + let mut zip = ZipWriter::new(io::Cursor::new(buf)); + + for (filename, content) in files { + zip.start_file(filename, options)?; + zip.write_all(content.as_ref())?; + } + + let mut archive = zip.finish_into_readable()?; + + let mut files = Vec::with_capacity(archive.len()); + for i in 0..archive.len() { + // `_raw` because we only read each entry's metadata, never its bytes, + // so there is no need to set up a decompressor. + let entry = archive.by_index_raw(i)?; + + let path = entry.name().to_string(); + let data_offset = entry.data_start().expect("missing data start"); + + debug_assert!(matches!(entry.compression(), CompressionMethod::Deflated)); + + files.push(FileEntry { + data_offset, + compressed_size: entry.compressed_size(), + uncompressed_size: entry.size(), + compression: "deflate".into(), + sha256: "dummy".into(), + path, + }); + } + + let bytes = archive.into_inner().into_inner(); + + Ok((Manifest { files }, bytes)) +} + +/// simlulated `static.crates.io` server for our tests, and configured by +/// default in our test environment. +/// +/// Right now just for the source-zip archives. +/// Our shared test-env, and also `FakeRelease`, fill it with +/// data when needed. +pub struct TestStaticCratesIo { + inner: Mutex, +} + +struct TestStaticCratesIoInner { + mocks: Vec, + server: mockito::ServerGuard, +} + +impl TestStaticCratesIo { + pub async fn new() -> Result { + Ok(Self { + inner: Mutex::new(TestStaticCratesIoInner { + mocks: Vec::new(), + server: mockito::Server::new_async().await, + }), + }) + } + pub async fn add( + &self, + name: &KrateName, + version: &Version, + manifest: Manifest, + zip: Vec, + ) -> Result<()> { + let mut inner = self.inner.lock().await; + + let mock_json = inner + .server + .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip.json")) + .with_body(serde_json::to_string(&manifest).unwrap()) + .create_async() + .await; + inner.mocks.push(mock_json); + + let mock_zip = inner + .server + .mock("GET", &*format!("/crates/{name}/{name}-{version}.zip")) + .with_body_from_request(move |request| { + // NOTE: mockito itself doesn't understand range requests. + // So we have to parse the header ourselves, and return the correct chunk here. + if let Some((lhs, rhs)) = request + .headers() + .get(RANGE) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.strip_prefix("bytes=")) + .and_then(|r| r.split_once("-")) + .and_then(|(lhs, rhs)| { + let lhs: usize = lhs.parse().ok()?; + let rhs: usize = rhs.parse().ok()?; + Some((lhs, rhs)) + }) + { + zip.get(lhs..=rhs).expect("should exist").to_vec() + } else { + zip.clone() + } + }) + .create_async() + .await; + inner.mocks.push(mock_zip); + + Ok(()) + } + + pub async fn url(&self) -> Url { + let inner = self.inner.lock().await; + Url::parse(&inner.server.url()).unwrap() + } +} diff --git a/crates/lib/docs_rs_repository_stats/src/workspaces.rs b/crates/lib/docs_rs_repository_stats/src/workspaces.rs index f3485273b..dd3719bd0 100644 --- a/crates/lib/docs_rs_repository_stats/src/workspaces.rs +++ b/crates/lib/docs_rs_repository_stats/src/workspaces.rs @@ -218,7 +218,7 @@ mod tests { impl TestEnv { async fn fake_release(&self) -> FakeRelease<'_> { - FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone()) + FakeRelease::new(self.db.pool().clone(), self.storage.storage().clone(), None) } } diff --git a/crates/lib/docs_rs_storage/Cargo.toml b/crates/lib/docs_rs_storage/Cargo.toml index 153de2489..07562bdd6 100644 --- a/crates/lib/docs_rs_storage/Cargo.toml +++ b/crates/lib/docs_rs_storage/Cargo.toml @@ -14,7 +14,7 @@ testing = [ [dependencies] anyhow = { workspace = true } -async-compression = { version = "0.4.32", features = ["bzip2", "deflate", "gzip", "tokio", "zstd"] } +async-compression = { workspace = true } async-stream = { workspace = true } # The default `rustls` feature pulls in the legacy hyper 0.14 + rustls 0.21 # stack via `aws-smithy-runtime/tls-rustls`, which includes the vulnerable @@ -51,7 +51,7 @@ strum = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } -tokio-util = { version = "0.7.15", default-features = false, features = ["io-util"] } +tokio-util = { workspace = true } tracing = { workspace = true } walkdir = { workspace = true } zip = { workspace = true } diff --git a/crates/lib/docs_rs_test_fakes/Cargo.toml b/crates/lib/docs_rs_test_fakes/Cargo.toml index 5915e374a..ead4cc591 100644 --- a/crates/lib/docs_rs_test_fakes/Cargo.toml +++ b/crates/lib/docs_rs_test_fakes/Cargo.toml @@ -11,7 +11,7 @@ base64 = { workspace = true } chrono = { workspace = true } docs_rs_cargo_metadata = { path = "../docs_rs_cargo_metadata", features = ["testing"] } docs_rs_database = { path = "../docs_rs_database" } -docs_rs_registry_api = { path = "../docs_rs_registry_api" } +docs_rs_registry_api = { path = "../docs_rs_registry_api", features = ["testing"] } docs_rs_rustdoc_json = { path = "../../lib/docs_rs_rustdoc_json" } docs_rs_storage = { path = "../docs_rs_storage" } docs_rs_types = { path = "../docs_rs_types" } diff --git a/crates/lib/docs_rs_test_fakes/src/legacy.rs b/crates/lib/docs_rs_test_fakes/src/legacy.rs index be9460bfe..e1625361c 100644 --- a/crates/lib/docs_rs_test_fakes/src/legacy.rs +++ b/crates/lib/docs_rs_test_fakes/src/legacy.rs @@ -8,7 +8,9 @@ use docs_rs_database::{ add_build_logs, initialize_build, initialize_crate, initialize_release, update_build_status, }, }; -use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData}; +use docs_rs_registry_api::{ + CrateData, CrateOwner, ReleaseData, testing::static_test_env::TestStaticCratesIo, +}; use docs_rs_rustdoc_json::{RUSTDOC_JSON_COMPRESSION_ALGORITHMS, RustdocJsonFormatVersion}; use docs_rs_storage::{ AsyncStorage, FileEntry, compress, file_list_to_json, rustdoc_archive_path, rustdoc_json_path, @@ -70,6 +72,7 @@ where pub struct FakeRelease<'a> { pool: Pool, storage: Arc, + static_crates_io: Option>, package: MetadataPackage, builds: Option>, /// name, content @@ -107,10 +110,15 @@ const DEFAULT_CONTENT: &[u8] = b"default content for test/fakes"; impl<'a> FakeRelease<'a> { - pub fn new(pool: Pool, storage: Arc) -> Self { + pub fn new( + pool: Pool, + storage: Arc, + static_crates_io: Option>, + ) -> Self { FakeRelease { pool, storage, + static_crates_io, package: MetadataPackage { id: "fake-package-id".into(), name: "fake-package".into(), @@ -454,6 +462,8 @@ impl<'a> FakeRelease<'a> { let source_tmp = create_temp_dir(); store_files_into(&self.source_files, source_tmp.path())?; + let mut additional_source_files: Vec<(String, Vec)> = Vec::new(); + if !self.no_cargo_toml && !self .source_files @@ -469,6 +479,23 @@ impl<'a> FakeRelease<'a> { "# ); store_files_into(&[("Cargo.toml", content.as_bytes())], source_tmp.path())?; + additional_source_files.push(("Cargo.toml".into(), content.as_bytes().to_vec())); + } + + let krate_name: KrateName = package.name.parse()?; + + if let Some(static_crates_io) = self.static_crates_io { + let (manifest, zip) = + docs_rs_registry_api::testing::static_test_env::create_test_source_archive( + self.source_files.iter().cloned().chain( + additional_source_files + .iter() + .map(|(n, c)| (n.as_str(), c.as_slice())), + ), + )?; + static_crates_io + .add(&krate_name, &package.version, manifest, zip) + .await?; } let (source_meta, algs) = upload_files( @@ -536,8 +563,6 @@ impl<'a> FakeRelease<'a> { self.doc_targets.insert(0, default_target.to_owned()); } - let krate_name: KrateName = package.name.parse()?; - for target in &self.doc_targets { let dummy_rustdoc_json_content = serde_json::to_vec(&serde_json::json!({ "format_version": 42