diff --git a/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json b/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json deleted file mode 100644 index e5b5773b4..000000000 --- a/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT releases.files\n FROM releases\n INNER JOIN crates ON crates.id = releases.crate_id\n WHERE crates.name = $1 AND releases.version = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "files", - "type_info": "Json", - "origin": { - "Table": { - "table": "releases", - "name": "files" - } - } - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc" -} diff --git a/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json b/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json deleted file mode 100644 index 4f5d74702..000000000 --- a/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE releases\n SET files = NULL\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [] - }, - "hash": "8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8" -} diff --git a/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json b/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json deleted file mode 100644 index 25f8b00a7..000000000 --- a/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n releases.archive_storage,\n (\n SELECT id\n FROM builds\n WHERE\n builds.rid = releases.id AND\n builds.build_status = 'success'\n ORDER BY build_finished DESC\n LIMIT 1\n ) AS \"latest_build_id?: BuildId\"\n FROM releases\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n name = $1 AND\n version = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "archive_storage", - "type_info": "Bool", - "origin": { - "Table": { - "table": "releases", - "name": "archive_storage" - } - } - }, - { - "ordinal": 1, - "name": "latest_build_id?: BuildId", - "type_info": "Int4", - "origin": "Expression" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20" -} diff --git a/crates/bin/cratesfyi/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json b/crates/bin/cratesfyi/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json deleted file mode 100644 index e5b5773b4..000000000 --- a/crates/bin/cratesfyi/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT releases.files\n FROM releases\n INNER JOIN crates ON crates.id = releases.crate_id\n WHERE crates.name = $1 AND releases.version = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "files", - "type_info": "Json", - "origin": { - "Table": { - "table": "releases", - "name": "files" - } - } - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc" -} diff --git a/crates/bin/cratesfyi/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json b/crates/bin/cratesfyi/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json deleted file mode 100644 index 25f8b00a7..000000000 --- a/crates/bin/cratesfyi/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n releases.archive_storage,\n (\n SELECT id\n FROM builds\n WHERE\n builds.rid = releases.id AND\n builds.build_status = 'success'\n ORDER BY build_finished DESC\n LIMIT 1\n ) AS \"latest_build_id?: BuildId\"\n FROM releases\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n name = $1 AND\n version = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "archive_storage", - "type_info": "Bool", - "origin": { - "Table": { - "table": "releases", - "name": "archive_storage" - } - } - }, - { - "ordinal": 1, - "name": "latest_build_id?: BuildId", - "type_info": "Int4", - "origin": "Expression" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20" -} diff --git a/crates/bin/docs_rs_web/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json b/crates/bin/docs_rs_web/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json deleted file mode 100644 index e5b5773b4..000000000 --- a/crates/bin/docs_rs_web/.sqlx/query-2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT releases.files\n FROM releases\n INNER JOIN crates ON crates.id = releases.crate_id\n WHERE crates.name = $1 AND releases.version = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "files", - "type_info": "Json", - "origin": { - "Table": { - "table": "releases", - "name": "files" - } - } - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "2b8b1aae3740a05cb7179be9c7d5af30e8362c3cba0b07bc18fa32ff1a2232cc" -} diff --git a/crates/bin/docs_rs_web/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json b/crates/bin/docs_rs_web/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json deleted file mode 100644 index 4f5d74702..000000000 --- a/crates/bin/docs_rs_web/.sqlx/query-8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE releases\n SET files = NULL\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [] - }, - "hash": "8cdeaffe16408187897e5c8f2b0d6054a31d0a2df37a8a1d7659208092929ce8" -} diff --git a/crates/bin/docs_rs_web/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json b/crates/bin/docs_rs_web/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json deleted file mode 100644 index 25f8b00a7..000000000 --- a/crates/bin/docs_rs_web/.sqlx/query-d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n releases.archive_storage,\n (\n SELECT id\n FROM builds\n WHERE\n builds.rid = releases.id AND\n builds.build_status = 'success'\n ORDER BY build_finished DESC\n LIMIT 1\n ) AS \"latest_build_id?: BuildId\"\n FROM releases\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n name = $1 AND\n version = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "archive_storage", - "type_info": "Bool", - "origin": { - "Table": { - "table": "releases", - "name": "archive_storage" - } - } - }, - { - "ordinal": 1, - "name": "latest_build_id?: BuildId", - "type_info": "Int4", - "origin": "Expression" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "d2c3a85119eea791d48d035c4593b0fd730a139cb52ae92ac16cd662286dbe20" -} diff --git a/crates/bin/docs_rs_web/src/extractors/rustdoc.rs b/crates/bin/docs_rs_web/src/extractors/rustdoc.rs index dea6f1793..568ec574d 100644 --- a/crates/bin/docs_rs_web/src/extractors/rustdoc.rs +++ b/crates/bin/docs_rs_web/src/extractors/rustdoc.rs @@ -7,9 +7,10 @@ use axum::{ extract::{FromRequestParts, MatchedPath}, http::{Uri, request::Parts}, }; -use docs_rs_types::{BuildId, CompressionAlgorithm, KrateName, ReqVersion}; -use docs_rs_uri::{EscapedURI, url_decode}; +use docs_rs_types::{BuildId, CompressionAlgorithm, KrateName, ReqVersion, Version}; +use docs_rs_uri::{EscapedURI, encode_url_path, url_decode}; use serde::{Deserialize, Serialize}; +use tracing::warn; const INDEX_HTML: &str = "index.html"; const FOLDER_AND_INDEX_HTML: &str = "/index.html"; @@ -45,6 +46,7 @@ pub(crate) struct RustdocParams { original_uri: Option, name: KrateName, req_version: ReqVersion, + matched_version: Option, doc_target: Option, inner_path: Option, static_route_suffix: Option, @@ -63,6 +65,7 @@ impl std::fmt::Debug for RustdocParams { .field("original_uri", &self.original_uri) .field("name", &self.name) .field("req_version", &self.req_version) + .field("matched_version", &self.matched_version) .field("doc_target", &self.doc_target) .field("inner_path", &self.inner_path) .field("doc_targets", &self.doc_targets) @@ -149,6 +152,7 @@ impl RustdocParams { Self { name: name.into(), req_version: ReqVersion::default(), + matched_version: None, original_uri: None, doc_target: None, inner_path: None, @@ -212,6 +216,7 @@ impl RustdocParams { pub(crate) fn apply_metadata(self, metadata: &MetaData) -> RustdocParams { self.with_name(metadata.name.clone()) .with_req_version(&metadata.req_version) + .with_matched_version(metadata.version.clone()) // first set the doc-target list .with_maybe_doc_targets(metadata.doc_targets.clone()) // then the default target, so we can validate it. @@ -227,6 +232,7 @@ impl RustdocParams { let release = &matched_release.release; self.with_name(matched_release.name.clone()) .with_req_version(&matched_release.req_version) + .with_matched_version(matched_release.release.version.clone()) .with_maybe_doc_targets(release.doc_targets.as_deref()) .with_maybe_default_target(release.default_target.as_deref()) .with_maybe_target_name(release.target_name.as_deref()) @@ -264,6 +270,19 @@ impl RustdocParams { }) } + pub(crate) fn matched_version(&self) -> Option<&Version> { + self.matched_version.as_ref() + } + pub(crate) fn with_matched_version(self, version: impl Into) -> Self { + self.with_maybe_matched_version(Some(version)) + } + pub(crate) fn with_maybe_matched_version(self, version: Option>) -> Self { + self.update(|mut params| { + params.matched_version = version.map(Into::into); + params + }) + } + pub(crate) fn inner_path(&self) -> &str { if self.page_kind == Some(PageKind::Rustdoc) && let Some(merged_inner_path) = self.merged_inner_path.as_deref() @@ -627,10 +646,47 @@ impl RustdocParams { } else { "" }; - EscapedURI::from_path(format!( - "/crate/{}/{}/source/{}", - &self.name, &self.req_version, &inner_path - )) + + // crates.io doesn't have semver redirects, just a special "latest version" URL. + let crates_io_version = if let Some(matched_version) = &self.matched_version { + // if these `RustdocParams` got handed a specific matched version, we will use that + // for the redirect to crates.io. + format!("/{matched_version}") + } else { + match &self.req_version { + // exact version can be redirected to exact version. + ReqVersion::Exact(version) => format!("/{version}"), + // latest / * can be redirect to the root / latest on crates.io + ReqVersion::Latest => "".to_string(), + // This is just a fallback for an edge case, for the cases where these + // `RustdocParams` are used without applied metadata or matched-release. + // + // This shouldn't happen in our use-cases / handlers. + // + // In these cases we just redirect to the source root of the latest version. + ReqVersion::Semver(version_req) => { + warn!( + %self.name, + %version_req, + "got request to generate source_url with semver. falling back to latest version" + ); + + "".to_string() + } + } + }; + + EscapedURI::from_uri( + Uri::builder() + .scheme("https") + .authority("crates.io") + .path_and_query(encode_url_path(&format!( + "/crates/{}{}/code/{}", + &self.name, &crates_io_version, &inner_path + ))) + .build() + .unwrap(), + ) } pub(crate) fn target_redirect_url(&self) -> EscapedURI { @@ -1280,7 +1336,7 @@ mod tests { assert_eq!(params.rustdoc_url().to_string(), "/dummy/0.4.0/dummy/"); assert_eq!( params.source_url().to_string(), - "/crate/dummy/0.4.0/source/README.md" + "https://crates.io/crates/dummy/0.4.0/code/README.md" ); assert_eq!( params.target_redirect_url().to_string(), @@ -1288,6 +1344,65 @@ mod tests { ); } + #[test_case("0.4.0", "/0.4.0"; "crates.io can handle exact version")] + #[test_case("latest", ""; "latest will become latest")] + #[test_case("^0.4", ""; "semver will fall back to latest")] + fn test_source_url_with_req_version_only( + req_version: &'static str, + crates_io_version: &'static str, + ) { + let params = RustdocParams::new(DUMMY) + .try_with_req_version(req_version) + .unwrap() + .with_maybe_matched_version(None::) + .with_inner_path("README.md") + .with_page_kind(PageKind::Source) + .try_with_original_uri(format!("/crate/dummy/{req_version}/source/README.md")) + .unwrap() + .with_default_target(DEFAULT_TARGET) + .with_target_name("dummy") + .with_doc_targets(TARGETS.iter().cloned()); + + assert_eq!( + params.rustdoc_url().to_string(), + format!("/dummy/{req_version}/dummy/") + ); + assert_eq!( + params.source_url().to_string(), + format!("https://crates.io/crates/dummy{crates_io_version}/code/README.md") + ); + assert_eq!( + params.target_redirect_url().to_string(), + format!("/crate/dummy/{req_version}/target-redirect/dummy/") + ); + } + + #[test] + fn test_source_url_with_overwritten_matched_version() { + let params = RustdocParams::new(DUMMY) + // semver would normally fall back to the latest/root on crates.io + .try_with_req_version("^0.4.0") + .unwrap() + .with_matched_version(Version::parse("0.4.0").unwrap()) + .with_inner_path("README.md") + .with_page_kind(PageKind::Source) + .try_with_original_uri("/crate/dummy/^0.4.0/source/README.md") + .unwrap() + .with_default_target(DEFAULT_TARGET) + .with_target_name("dummy") + .with_doc_targets(TARGETS.iter().cloned()); + + assert_eq!(params.rustdoc_url().to_string(), "/dummy/^0.4.0/dummy/"); + assert_eq!( + params.source_url().to_string(), + "https://crates.io/crates/dummy/0.4.0/code/README.md" + ); + assert_eq!( + params.target_redirect_url().to_string(), + "/crate/dummy/^0.4.0/target-redirect/dummy/" + ); + } + #[test_case( None, None, None, None => "" )] @@ -1521,7 +1636,7 @@ mod tests { // without page kind, rustdoc path doesn' thave a path, and static suffix ignored assert_eq!(params.rustdoc_url(), "/krate/latest/krate/"); - assert_eq!(params.source_url(), "/crate/krate/latest/source/"); + assert_eq!(params.source_url(), "https://crates.io/crates/krate/code/"); assert_eq!( params.target_redirect_url(), "/crate/krate/latest/target-redirect/krate/" @@ -1529,7 +1644,7 @@ mod tests { let params = params.with_page_kind(PageKind::Rustdoc); assert_eq!(params.rustdoc_url(), "/krate/latest/path_add/static.html"); - assert_eq!(params.source_url(), "/crate/krate/latest/source/"); + assert_eq!(params.source_url(), "https://crates.io/crates/krate/code/"); assert_eq!( params.target_redirect_url(), "/crate/krate/latest/target-redirect/path_add/static.html" @@ -1538,7 +1653,10 @@ mod tests { let params = params.with_page_kind(PageKind::Source); assert_eq!(params.rustdoc_url(), "/krate/latest/krate/"); // just path added, not static suffix - assert_eq!(params.source_url(), "/crate/krate/latest/source/path_add"); + assert_eq!( + params.source_url(), + "https://crates.io/crates/krate/code/path_add" + ); assert_eq!( params.target_redirect_url(), "/crate/krate/latest/target-redirect/krate/" @@ -1562,7 +1680,8 @@ mod tests { params.rustdoc_url(), format!("/krate/latest/{OTHER_TARGET}/krate/") ); - assert_eq!(params.source_url(), "/crate/krate/latest/source/"); + assert_eq!(params.source_url(), "https://crates.io/crates/krate/code/"); + assert_eq!( params.target_redirect_url(), format!("/crate/krate/latest/target-redirect/{OTHER_TARGET}/krate/") @@ -1574,7 +1693,10 @@ mod tests { params.rustdoc_url(), format!("/krate/latest/{OTHER_TARGET}/krate/") ); - assert_eq!(params.source_url(), "/crate/krate/latest/source/path_add"); + assert_eq!( + params.source_url(), + "https://crates.io/crates/krate/code/path_add" + ); assert_eq!( params.target_redirect_url(), format!("/crate/krate/latest/target-redirect/{OTHER_TARGET}/krate/") @@ -1586,7 +1708,7 @@ mod tests { params.rustdoc_url(), format!("/krate/latest/{OTHER_TARGET}/path_add/static.html") ); - assert_eq!(params.source_url(), format!("/crate/krate/latest/source/")); + assert_eq!(params.source_url(), "https://crates.io/crates/krate/code/"); assert_eq!( params.target_redirect_url(), format!("/crate/krate/latest/target-redirect/{OTHER_TARGET}/path_add/static.html") diff --git a/crates/bin/docs_rs_web/src/handlers/builds.rs b/crates/bin/docs_rs_web/src/handlers/builds.rs index e5b3991e9..f68691e7c 100644 --- a/crates/bin/docs_rs_web/src/handlers/builds.rs +++ b/crates/bin/docs_rs_web/src/handlers/builds.rs @@ -670,7 +670,7 @@ mod tests { let body = resp.text().await?; assert!(body.contains(", -} - -impl FileList { - async fn from_archive_index( - storage: &AsyncStorage, - name: &KrateName, - version: &Version, - latest_build_id: Option, - folder: Option<&str>, - ) -> Result { - let mut files = storage - .find_archive_index(&source_archive_path(name, version), latest_build_id) - .await? - .folder_contents(folder) - .try_filter(|entry| std::future::ready(entry.name() != ".cargo-ok")) - .try_collect::>() - .await?; - - files.sort_unstable(); - - Ok(Self { files }) - } - - /// Gets legacy FileList from a request path - /// - /// All paths stored in database have this format: - /// - /// ```text - /// [ - /// ["text/plain", ".gitignore"], - /// ["text/x-c", "src/reseeding.rs"], - /// ["text/x-c", "src/lib.rs"], - /// ["text/x-c", "README.md"], - /// ... - /// ] - /// ``` - /// - /// This function is only returning FileList for requested directory. If is empty, - /// it will return list of files (and dirs) for root directory. req_path must be a - /// directory or empty for root directory. - #[instrument(skip(conn))] - async fn from_path( - conn: &mut sqlx::PgConnection, - name: &KrateName, - version: &Version, - folder: &str, - ) -> Result> { - let row = match sqlx::query!( - "SELECT releases.files - FROM releases - INNER JOIN crates ON crates.id = releases.crate_id - WHERE crates.name = $1 AND releases.version = $2", - name as _, - version as _, - ) - .fetch_optional(&mut *conn) - .await? - { - Some(row) => row, - None => return Ok(None), - }; - - let files = if let Some(files) = row.files { - files - } else { - return Ok(None); - }; - - let mut file_list = Vec::new(); - if let Some(files) = files.as_array() { - file_list.reserve(files.len()); - - for file in files { - if let Some(file) = file.as_array() { - let path = file[1].as_str().unwrap(); - - // skip .cargo-ok generated by cargo - if path == ".cargo-ok" { - continue; - } - - // look only files for req_path - if let Some(path) = path.strip_prefix(folder) { - let file = FolderEntry::from_path(path); - - // avoid adding duplicates, a directory may occur more than once - if !file_list.contains(&file) { - file_list.push(file); - } - } - } - } - - if file_list.is_empty() { - return Ok(None); - } - - file_list.sort_unstable(); - - Ok(Some(FileList { files: file_list })) - } else { - Ok(None) - } - } -} - -#[derive(Template)] -#[template(path = "crate/source.html")] -#[derive(Debug, Clone)] -struct SourcePage { - file_list: FileList, - metadata: MetaData, - show_parent_link: bool, - file: Option, - file_content: Option, - canonical_url: CanonicalUrl, - is_file_too_large: bool, - is_latest_url: bool, - params: RustdocParams, -} - -impl_axum_webpage! { - SourcePage, - canonical_url = |page| Some(page.canonical_url.clone()), - cache_policy = |page| if page.is_latest_url { - CachePolicy::ForeverInCdn( - page.metadata.name.clone().into() - ) - } else { - CachePolicy::ForeverInCdnAndStaleInBrowser( - page.metadata.name.clone().into() - ) - }, - cpu_intensive_rendering = true, -} - -// Used in templates. -impl SourcePage { - pub(crate) fn use_direct_platform_links(&self) -> bool { - true - } -} - -#[instrument(skip(conn, storage))] +#[instrument(skip(conn))] pub(crate) async fn source_browser_handler( params: RustdocParams, - Extension(storage): Extension>, - Extension(config): Extension>, mut conn: DbConnection, - if_none_match: Option>, ) -> AxumResult { let params = params.with_page_kind(PageKind::Source); let matched_release = match_version(&mut conn, params.name(), params.req_version()) .await? - .into_exactly_named_or_else(|corrected_name, req_version| { - AxumNope::Redirect( - params - .clone() - .with_name(corrected_name) - .with_req_version(req_version) - .source_url(), - CachePolicy::NoCaching, - ) - })? - .into_canonical_req_version_or_else(|confirmed_name, version| { - let params = params - .clone() - .with_name(confirmed_name) - .with_req_version(version); - AxumNope::Redirect( - params.source_url(), - CachePolicy::ForeverInCdn(confirmed_name.into()), - ) - })?; + .into_exactly_named() + .into_canonical_req_version(); let params = params.apply_matched_release(&matched_release); - let version = &matched_release.release.version; - - let row = sqlx::query!( - r#"SELECT - releases.archive_storage, - ( - SELECT id - FROM builds - WHERE - builds.rid = releases.id AND - builds.build_status = 'success' - ORDER BY build_finished DESC - LIMIT 1 - ) AS "latest_build_id?: BuildId" - FROM releases - INNER JOIN crates ON releases.crate_id = crates.id - WHERE - name = $1 AND - version = $2"#, - params.name() as _, - version as _, - ) - .fetch_one(&mut *conn) - .await?; - - let inner_path = params.inner_path(); - - let current_folder = inner_path - .rfind('/') - .map(|last_slash_pos| &inner_path[..last_slash_pos + 1]); - - let show_parent_link = current_folder.is_some(); - - let file_list = if row.archive_storage { - FileList::from_archive_index( - &storage, - params.name(), - version, - row.latest_build_id, - current_folder, - ) - .await? - } else { - FileList::from_path( - &mut conn, - params.name(), - version, - current_folder.unwrap_or(""), - ) - .await? - .unwrap_or_default() - }; - - let metadata = MetaData::from_crate( - &mut conn, - params.name(), - version, - Some(params.req_version().clone()), - ) - .await?; - - // NOTE: we want to give back the db connection to the pool - // before we do the long S3 requests. - drop(conn); - // try to get actual file first - // skip if request is a directory - let stream = if !params.path_is_folder() { - match storage - .stream_source_file( - params.name(), - version, - row.latest_build_id, - inner_path, - row.archive_storage, - ) - .await - .context("error fetching source file") - { - Ok(stream) => Some(stream), - Err(err) => match err { - err if err.is::() => None, - _ => return Err(err.into()), - }, - } - } else { - None - }; - - let canonical_url = CanonicalUrl::from_uri( - params - .clone() - .with_req_version(ReqVersion::Latest) - .source_url(), - ); - - let mut is_file_too_large = false; - - let (file, file_content) = if let Some(stream) = stream { - let is_text = stream.mime.type_() == mime::TEXT || stream.mime == mime::APPLICATION_JSON; - if !is_text { - // if the file isn't text, serve it directly to the client - let mut response = StreamingFile(stream) - .into_response(if_none_match.as_deref(), STATIC_ASSET_CACHE_POLICY); - response.headers_mut().typed_insert(canonical_url); - response - .extensions_mut() - .insert(CachePolicy::ForeverInCdnAndStaleInBrowser( - matched_release.name.into(), - )); - return Ok(response); + Ok(axum_cached_redirect( + params.source_url(), + if params.req_version().is_latest() { + CachePolicy::ForeverInCdn(params.name().into()) } else { - let max_file_size = storage.config().max_file_size_for(&stream.path); - - // otherwise we'll now download the content to render it into our template. - match stream.materialize(max_file_size).await { - Ok(blob) => { - let path = blob - .path - .rsplit_once('/') - .map(|(_, path)| path) - .unwrap_or(&blob.path); - ( - Some(FolderEntry::from_path(path)), - Some(String::from_utf8_lossy(&blob.content).to_string()), - ) - } - Err(err) - // if file is too large, set is_file_too_large to true - if err.downcast_ref::().is_some_and(|err| { - err.get_ref() - .map(|err| err.is::()) - .unwrap_or(false) - }) => - { - is_file_too_large = true; - (None, None) - } - Err(err) => return Err(err.into()), - } - } - } else { - (None, None) - }; - - Ok(SourcePage { - file_list, - metadata, - show_parent_link, - file, - file_content, - canonical_url, - is_file_too_large, - is_latest_url: params.req_version().is_latest(), - params, - } - .into_response()) + CachePolicy::ForeverInCdnAndStaleInBrowser(params.name().into()) + }, + )?) } #[cfg(test)] mod tests { use crate::{ cache::CachePolicy, - testing::{ - AxumResponseTestExt, AxumRouterTestExt, TestEnvironment, TestEnvironmentExt as _, - async_wrapper, - }, + testing::{AxumRouterTestExt, TestEnvironmentExt as _, async_wrapper}, }; - use anyhow::Result; - use axum_extra::headers::{ContentType, ETag, HeaderMapExt as _}; - use docs_rs_headers::IfNoneMatch; - use docs_rs_storage::StorageKind; - use docs_rs_types::KrateName; - use docs_rs_uri::encode_url_path; - use kuchikiki::traits::TendrilSink; - use mime::APPLICATION_PDF; - use reqwest::StatusCode; - use std::str::FromStr as _; + use docs_rs_types::testing::{KRATE, V1, V2}; use test_case::test_case; - fn get_file_list_links(body: &str) -> Vec { - let dom = kuchikiki::parse_html().one(body); - - dom.select(".package-menu > ul > li > a") - .expect("invalid selector") - .map(|el| { - let attributes = el.attributes.borrow(); - attributes.get("href").unwrap().to_string() - }) - .collect() - } - - #[test_case(true)] - #[test_case(false)] - fn fetch_source_file_utf8_path(archive_storage: bool) { + #[test_case("src/something.rs", "src/something.rs")] + #[test_case("a/序.pdf", "a/%E5%BA%8F.pdf")] + fn legacy_source_redirects_to_crates_io(filename: &'static str, expected: &'static str) { async_wrapper(|env| async move { - let filename = "序.pdf"; - env.fake_release() .await - .archive_storage(archive_storage) - .name("fake") - .version("0.1.0") + .archive_storage(true) + .name(&KRATE) + .version(V1) .source_file(filename, b"some_random_content") .create() .await?; let web = env.web_app().await; - let response = web - .get(&format!( - "/crate/fake/0.1.0/source/{}", - encode_url_path(filename) - )) - .await?; - assert!(response.status().is_success()); - assert_eq!( - response.headers().get("link").unwrap(), - "; rel=\"canonical\"", - ); - assert!(response.text().await?.contains("some_random_content")); - Ok(()) - }); - } - #[test_case(true)] - #[test_case(false)] - fn fetch_source_file_content(archive_storage: bool) { - async_wrapper(|env| async move { - env.fake_release() - .await - .archive_storage(archive_storage) - .name("fake") - .version("0.1.0") - .source_file("some_filename.rs", b"some_random_content") - .create() - .await?; - let web = env.web_app().await; - web.assert_success_cached( - "/crate/fake/0.1.0/source/", - CachePolicy::ForeverInCdnAndStaleInBrowser( - KrateName::from_str("fake").unwrap().into(), - ), + // source root redirect + web.assert_redirect_cached_unchecked( + &format!("/crate/{KRATE}/{V1}/source/"), + &format!("https://crates.io/crates/{KRATE}/{V1}/code/"), + CachePolicy::ForeverInCdnAndStaleInBrowser(KRATE.into()), env.config(), ) .await?; - let response = web.get("/crate/fake/0.1.0/source/some_filename.rs").await?; - assert!(response.status().is_success()); - assert_eq!( - response.headers().get("link").unwrap(), - "; rel=\"canonical\"" - ); - response.assert_cache_control( - CachePolicy::ForeverInCdnAndStaleInBrowser( - KrateName::from_str("fake").unwrap().into(), - ), - env.config(), - ); - assert!(response.text().await?.contains("some_random_content")); - Ok(()) - }); - } - #[test_case(true)] - #[test_case(false)] - fn fetch_binary(archive_storage: bool) { - async_wrapper(|env| async move { - env.fake_release() - .await - .archive_storage(archive_storage) - .name("fake") - .version("0.1.0") - .source_file("some_file.pdf", b"some_random_content") - .create() - .await?; - let web = env.web_app().await; - - const URL: &str = "/crate/fake/0.1.0/source/some_file.pdf"; - - // first request, uncached - let response = web.get(URL).await?; - assert!(response.status().is_success()); - let headers = response.headers(); - assert_eq!( - headers.get("link").unwrap(), - "; rel=\"canonical\"" - ); - assert_eq!( - headers.typed_get::().unwrap(), - APPLICATION_PDF.into(), - ); - response.assert_cache_control( - CachePolicy::ForeverInCdnAndStaleInBrowser( - KrateName::from_str("fake").unwrap().into(), - ), + // source path redirect + web.assert_redirect_cached_unchecked( + &format!("/crate/{KRATE}/{V1}/source/{filename}"), + &format!("https://crates.io/crates/{KRATE}/{V1}/code/{expected}"), + CachePolicy::ForeverInCdnAndStaleInBrowser(KRATE.into()), env.config(), - ); - - let etag: ETag = headers.typed_get().unwrap(); - - assert!(response.text().await?.contains("some_random_content")); - - let response = web - .get_with_headers(URL, |headers| { - headers.typed_insert(IfNoneMatch::from(etag)); - }) - .await?; - assert_eq!(response.status(), StatusCode::NOT_MODIFIED); - - Ok(()) - }); - } - - #[test_case(true)] - #[test_case(false)] - fn cargo_ok_not_skipped(archive_storage: bool) { - async_wrapper(|env| async move { - env.fake_release() - .await - .archive_storage(archive_storage) - .name("fake") - .version("0.1.0") - .source_file(".cargo-ok", b"ok") - .source_file("README.md", b"hello") - .create() - .await?; - let web = env.web_app().await; - web.assert_success("/crate/fake/0.1.0/source/").await?; - Ok(()) - }); - } - - #[test_case(true)] - #[test_case(false)] - fn empty_file_list_dont_break_the_view(archive_storage: bool) { - async_wrapper(|env| async move { - let release_id = env - .fake_release() - .await - .archive_storage(archive_storage) - .name("fake") - .version("0.1.0") - .source_file("README.md", b"hello") - .create() - .await?; - - let path = "/crate/fake/0.1.0/source/README.md"; - let web = env.web_app().await; - web.assert_success(path).await?; - - let mut conn = env.async_conn().await?; - sqlx::query!( - "UPDATE releases - SET files = NULL - WHERE id = $1", - release_id.0, ) - .execute(&mut *conn) .await?; - assert!(web.get(path).await?.status().is_success()); - - Ok(()) - }); - } - - #[test] - fn latest_contains_links_to_latest() { - async_wrapper(|env| async move { - env.fake_release() - .await - .archive_storage(true) - .name("fake") - .version("0.1.0") - .source_file(".cargo-ok", b"ok") - .source_file("README.md", b"hello") - .create() - .await?; - let resp = env - .web_app() - .await - .get("/crate/fake/latest/source/") - .await?; - resp.assert_cache_control( - CachePolicy::ForeverInCdn(KrateName::from_str("fake").unwrap().into()), - env.config(), - ); - let body = resp.text().await?; - assert!(body.contains(""#)); - - let response = web - .get("/crate/fake/0.1.0/source/Cargo.lock") - .await? - .text() - .await?; - assert!(response.contains(r#""#)); - - Ok(()) - }); - } - - #[test] - fn dotfiles_with_extension_are_highlighted() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("fake") - .version("0.1.0") - .source_file(".rustfmt.toml", b"[rustfmt]") - .create() - .await?; - - let web = env.web_app().await; - - let response = web - .get("/crate/fake/0.1.0/source/.rustfmt.toml") - .await? - .text() - .await?; - assert!(response.contains(r#""#)); - - Ok(()) - }); - } - - #[test] - fn json_is_served_as_rendered_html() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("fake") - .version("0.1.0") - .source_file("Cargo.toml", b"") - .source_file("config.json", b"{}") - .create() - .await?; - - let web = env.web_app().await; - - let response = web.get("/crate/fake/0.1.0/source/config.json").await?; - assert!( - response - .headers() - .get("content-type") - .unwrap() - .to_str() - .unwrap() - .starts_with("text/html") - ); - - let text = response.text().await?; - assert!(text.starts_with(r#""#)); - - // file list doesn't show "../" - assert_eq!( - get_file_list_links(&text), - vec!["./Cargo.toml", "./config.json"] - ); - - Ok(()) - }); - } - - #[test] - fn root_file_list() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("fake") - .version("0.1.0") - .source_file("Cargo.toml", b"some_random_content") - .source_file("folder1/some_filename.rs", b"some_random_content") - .source_file("folder2/another_filename.rs", b"some_random_content") - .source_file("root_filename.rs", b"some_random_content") - .create() - .await?; - - let web = env.web_app().await; - let response = web.get("/crate/fake/0.1.0/source/").await?; - assert!(response.status().is_success()); - response.assert_cache_control( - CachePolicy::ForeverInCdnAndStaleInBrowser( - KrateName::from_str("fake").unwrap().into(), - ), - env.config(), - ); - - assert_eq!( - get_file_list_links(&response.text().await?), - vec![ - "./folder1/", - "./folder2/", - "./Cargo.toml", - "./root_filename.rs" - ] - ); - Ok(()) - }); - } - - #[test] - fn child_file_list() { - async_wrapper(|env| async move { - env.fake_release() - .await - .name("fake") - .version("0.1.0") - .source_file("folder1/some_filename.rs", b"some_random_content") - .source_file("folder1/more_filenames.rs", b"some_random_content") - .source_file("folder2/another_filename.rs", b"some_random_content") - .source_file("root_filename.rs", b"some_random_content") - .create() - .await?; - - let web = env.web_app().await; - let response = web - .get("/crate/fake/0.1.0/source/folder1/some_filename.rs") - .await?; - assert!(response.status().is_success()); - response.assert_cache_control( - CachePolicy::ForeverInCdnAndStaleInBrowser( - KrateName::from_str("fake").unwrap().into(), - ), - env.config(), - ); - - assert_eq!( - get_file_list_links(&response.text().await?), - vec!["../", "./more_filenames.rs", "./some_filename.rs"], - ); - Ok(()) - }); - } - - #[tokio::test(flavor = "multi_thread")] - async fn large_file_test() -> Result<()> { - let env = TestEnvironment::builder() - .storage_config( - docs_rs_storage::Config::test_config_with_kind(StorageKind::Memory)?.set( - |mut cfg| { - cfg.max_file_size = 1; - cfg.max_file_size_html = 1; - cfg - }, - ), - ) - .build() - .await?; - - env.fake_release() - .await - .name("fake") - .version("0.1.0") - .source_file("large_file.rs", b"some_random_content") - .create() - .await?; - - let web = env.web_app().await; - let response = web.get("/crate/fake/0.1.0/source/large_file.rs").await?; - assert_eq!(response.status(), StatusCode::OK); - assert!( - response - .text() - .await? - .contains("This file is too large to display") - ); - Ok(()) - } } diff --git a/crates/bin/docs_rs_web/src/handlers/statics.rs b/crates/bin/docs_rs_web/src/handlers/statics.rs index 5a38b04be..c400a379e 100644 --- a/crates/bin/docs_rs_web/src/handlers/statics.rs +++ b/crates/bin/docs_rs_web/src/handlers/statics.rs @@ -338,7 +338,6 @@ mod tests { #[test_case("/-/static/index.js", "resetClipboardTimeout")] #[test_case("/-/static/menu.js", "closeMenu")] #[test_case("/-/static/keyboard.js", "handleKey")] - #[test_case("/-/static/source.js", "toggleSource")] fn js_content(path: &str, expected_content: &str) { async_wrapper(|env| async move { let web = env.web_app().await; diff --git a/crates/bin/docs_rs_web/src/metrics.rs b/crates/bin/docs_rs_web/src/metrics.rs index 1548f7abf..38145cfc1 100644 --- a/crates/bin/docs_rs_web/src/metrics.rs +++ b/crates/bin/docs_rs_web/src/metrics.rs @@ -137,7 +137,6 @@ mod tests { ("/-/static/index.js", "static resource"), ("/-/static/menu.js", "static resource"), ("/-/static/keyboard.js", "static resource"), - ("/-/static/source.js", "static resource"), ("/-/static/opensearch.xml", "static resource"), ("/releases", "/releases"), ("/releases/feed", "/releases/feed"), @@ -156,8 +155,8 @@ mod tests { ), ("/-/static/style.css", "static resource"), ("/-/static/vendored.css", "static resource"), - ("/rustdoc/rcc/0.0.0/rcc/index.html", "rustdoc page"), - ("/rustdoc/gcc/0.0.0/gcc/index.html", "rustdoc page"), + ("/rcc/0.0.0/rcc/index.html", "rustdoc page"), + ("/gcc/0.0.0/gcc/index.html", "rustdoc page"), ]; async_wrapper(|env| async move { @@ -165,9 +164,17 @@ mod tests { .await .name("rcc") .version("0.0.0") + .rustdoc_file("rcc/index.html") .repo("https://github.com/jyn514/rcc") .create() .await?; + env.fake_release() + .await + .name("gcc") + .version("0.0.0") + .rustdoc_file("gcc/index.html") + .create() + .await?; env.fake_release() .await .name("rcc") @@ -185,8 +192,8 @@ mod tests { let frontend = env.web_app().await; for (route, _) in ROUTES.iter() { - frontend.get(route).await?; - frontend.get(route).await?; + frontend.assert_success(route).await?; + frontend.assert_success(route).await?; } let mut expected = HashMap::new(); @@ -231,7 +238,7 @@ mod tests { ("/releases/recent/{page}", 2), ("/sitemap.xml", 2), ("rustdoc page", 4), - ("static resource", 16), + ("static resource", 14), ] .into_iter() .map(|(k, v)| (k.to_string(), v)) @@ -274,7 +281,7 @@ mod tests { ("/releases/recent/{page}", 2), ("/sitemap.xml", 2), ("rustdoc page", 4), - ("static resource", 16), + ("static resource", 14), ] .into_iter() .map(|(k, v)| (k.to_string(), v)) diff --git a/crates/bin/docs_rs_web/src/utils/mod.rs b/crates/bin/docs_rs_web/src/utils/mod.rs index 86ac95084..b7181d683 100644 --- a/crates/bin/docs_rs_web/src/utils/mod.rs +++ b/crates/bin/docs_rs_web/src/utils/mod.rs @@ -3,42 +3,10 @@ pub(crate) mod html_rewrite; pub(crate) mod licenses; pub(crate) mod markdown; -use crate::{ - icons::{ - IconFile, IconFileLines, IconFolderOpen, IconGitAlt, IconLock, IconMarkdown, IconRust, - }, - page::templates::{RenderBrands, RenderRegular, RenderSolid}, -}; use anyhow::Result; -use askama::filters::Safe; use chrono::{DateTime, NaiveDate, Utc}; -use docs_rs_mimes as mimes; -use docs_rs_storage::FolderEntry; use docs_rs_utils::rustc_version::parse_rustc_date; -pub fn folder_entry_icon(entry: &FolderEntry) -> Safe { - if entry.is_dir() { - return IconFolderOpen.render_regular(false, false, ""); - } - - let mime = entry.mime().expect("files always have mime"); - let name = entry.name(); - - if *mime == *mimes::TEXT_RUST { - IconRust.render_brands(false, false, "") - } else if *mime == mime::TEXT_PLAIN && name == "Cargo.lock" { - IconLock.render_solid(false, false, "") - } else if *mime == *mimes::TEXT_MARKDOWN { - IconMarkdown.render_brands(false, false, "") - } else if *mime == mime::TEXT_PLAIN && name == ".gitignore" { - IconGitAlt.render_brands(false, false, "") - } else if *mime == mime::TEXT_PLAIN || mime.type_() == "text" { - IconFileLines.render_regular(false, false, "") - } else { - IconFile.render_regular(false, false, "") - } -} - /// Picks the correct "rustdoc.css" static file depending on which rustdoc version was used to /// generate this version of this crate. pub(crate) fn get_correct_docsrs_style_file(version: &str) -> Result { diff --git a/crates/bin/docs_rs_web/static/source.js b/crates/bin/docs_rs_web/static/source.js deleted file mode 100644 index 0801441c4..000000000 --- a/crates/bin/docs_rs_web/static/source.js +++ /dev/null @@ -1,131 +0,0 @@ -(function() { - let oldLabel; - - function showSourceFiles(button, sideMenu, sourceCode) { - button.title = oldLabel; - button.setAttribute("aria-label", button.title); - button.setAttribute("aria-expanded", "true"); - - sideMenu.classList.remove("collapsed"); - sourceCode.classList.remove("expanded"); - } - - function hideSourceFiles(button, sideMenu, sourceCode) { - button.title = "Show source sidebar"; - button.setAttribute("aria-label", button.title); - button.setAttribute("aria-expanded", "false"); - - sideMenu.classList.add("collapsed"); - sourceCode.classList.add("expanded"); - } - - function toggleSource(button) { - const sideMenu = document.getElementById("side-menu"); - const sourceCode = document.getElementById("source-code"); - - if (sideMenu.classList.contains("collapsed")) { - showSourceFiles(button, sideMenu, sourceCode); - } else { - hideSourceFiles(button, sideMenu, sourceCode); - } - } - - document.addEventListener("DOMContentLoaded", () => { - const toggleSourceButton = document.querySelector("li.toggle-source button"); - oldLabel = toggleSourceButton.getAttribute("aria-label"); - - toggleSourceButton.addEventListener("click", () => { - toggleSource(toggleSourceButton); - }); - }); - - // This code has been adapted from the rustdoc implementation - function highlightLineNumbers() { - const match = window.location.hash.match(/^#?(\d+)(?:-(\d+))?$/); - if (!match) { - return; - } - let from = parseInt(match[1], 10); - let to = from; - if (typeof match[2] !== "undefined") { - to = parseInt(match[2], 10); - } - if (to < from) { - const tmp = to; - to = from; - from = tmp; - } - let elem = document.getElementById(from); - if (!elem) { - return; - } - const x = document.getElementById(from); - if (x) { - x.scrollIntoView(); - } - Array.from(document.getElementsByClassName("line-number-highlighted")).forEach(e => { - e.classList.remove("line-number-highlighted"); - }); - for (let i = from; i <= to; ++i) { - elem = document.getElementById(i); - if (!elem) { - break; - } - elem.classList.add("line-number-highlighted"); - } - } - - const handleLineNumbers = (function() { - let prev_line_id = 0; - - const set_fragment = name => { - const x = window.scrollX, - y = window.scrollY; - if (window.history && typeof window.history.pushState === "function") { - history.replaceState(null, null, "#" + name); - highlightLineNumbers(); - } else { - location.replace("#" + name); - } - // Prevent jumps when selecting one or many lines - window.scrollTo(x, y); - }; - - return ev => { - let cur_line_id = parseInt(ev.target.id, 10); - // This event handler is attached to the entire line number column, but it should only - // be run if one of the anchors is clicked. It also shouldn't do anything if the anchor - // is clicked with a modifier key (to open a new browser tab). - if (isNaN(cur_line_id) || - ev.ctrlKey || - ev.altKey || - ev.metaKey) { - return; - } - ev.preventDefault(); - - if (ev.shiftKey && prev_line_id) { - // Swap selection if needed - if (prev_line_id > cur_line_id) { - const tmp = prev_line_id; - prev_line_id = cur_line_id; - cur_line_id = tmp; - } - - set_fragment(prev_line_id + "-" + cur_line_id); - } else { - prev_line_id = cur_line_id; - - set_fragment(cur_line_id); - } - }; - }()); - - window.addEventListener("hashchange", highlightLineNumbers); - - Array.from(document.getElementById("line-numbers").children[0].children).forEach(el => { - el.addEventListener("click", handleLineNumbers); - }); - - highlightLineNumbers(); -})(); diff --git a/crates/bin/docs_rs_web/templates/crate/source.html b/crates/bin/docs_rs_web/templates/crate/source.html deleted file mode 100644 index c496cc250..000000000 --- a/crates/bin/docs_rs_web/templates/crate/source.html +++ /dev/null @@ -1,95 +0,0 @@ -{% extends "base.html" %} -{%- import "header/package_navigation.html" as navigation -%} - -{%- block title -%} - {% call macros::doc_title(name=metadata.name, version=metadata.version) %}{% endcall %} -{%- endblock title -%} - -{%- block topbar -%} - {%- include "rustdoc/topbar.html" -%} -{%- endblock topbar -%} - -{%- block header -%} - {# Set the active tab to the `source` tab #} - {% call navigation::package_navigation(metadata=metadata, active_tab="source") %}{% endcall %} -{%- endblock header -%} - -{%- block body_classes -%} - {%- if file_content.is_some() -%}flex{%- endif -%} -{%- endblock body_classes -%} - -{%- block body -%} - {% set has_file_content = file_content.is_some() %} -
-
-
-
- -
-
- - {# If the file exceeds the maximum size, display a warning #} - {%- if is_file_too_large -%} -
-
-

- This file is too large to display. -

-
-
- {%- endif -%} - - {# If the file has content, then display it in a codeblock #} - {%- if let Some(file_content) = file_content -%} - {% decl file_name %} - {% if let Some(file) = file %} - {% set file_name = file.name() %} - {% else %} - {% set file_name = "" %} - {% endif %} -
- 

-                        {%- for line in 1..=file_content.lines().count() -%}
-                            {{line}}
-                        {%~ endfor -%}
-
-
- {{- file_content|highlight(file_name) -}} -
-
- {%- endif -%} -
-
-{%- endblock body -%} - -{%- block javascript -%} - {% if file_content.is_some() %} - - {% endif %} -{%- endblock javascript -%} diff --git a/crates/lib/docs_rs_storage/src/archive_index.rs b/crates/lib/docs_rs_storage/src/archive_index.rs index 21fa0ba46..950907c9f 100644 --- a/crates/lib/docs_rs_storage/src/archive_index.rs +++ b/crates/lib/docs_rs_storage/src/archive_index.rs @@ -1,10 +1,9 @@ use crate::{ - PathNotFoundError, blob::StreamingBlob, config::ArchiveIndexCacheConfig, file::FolderEntry, - types::FileRange, utils::file_list::walk_dir_recursive, + PathNotFoundError, blob::StreamingBlob, config::ArchiveIndexCacheConfig, types::FileRange, + utils::file_list::walk_dir_recursive, }; use anyhow::{Context as _, Result, anyhow, bail}; use async_stream::try_stream; -use docs_rs_mimes::detect_mime; use docs_rs_opentelemetry::AnyMeterProvider; use docs_rs_types::{BuildId, CompressionAlgorithm}; use docs_rs_utils::spawn_blocking; @@ -16,7 +15,6 @@ use opentelemetry::{ }; use sqlx::{ConnectOptions as _, Connection as _, QueryBuilder, Row as _, Sqlite}; use std::{ - collections::HashSet, fmt, future::Future, path::{Path, PathBuf}, @@ -904,80 +902,6 @@ impl Index { } } } - - /// get the folder contents inside the zip archive. - /// * missing folder = list the root - /// * given folder: just lists the files in there, and subfolders, but not their contents. - /// - /// You'll need this method when you build a file-browser for the archive, like - /// in our source pages. - #[instrument(skip(self))] - pub fn folder_contents

( - &mut self, - folder: Option

, - ) -> impl Stream> + '_ - where - P: AsRef + std::fmt::Debug, - { - // Build the path prefix string used in GLOB patterns. - // For root (None): prefix = "" - // For a folder: prefix = "some/folder/" - let prefix: Option = folder.as_ref().map(|f| { - let s = f.as_ref().to_string_lossy(); - // Normalize: strip any trailing slash, then re-add exactly one. - format!("{}/", s.trim_end_matches('/')) - }); - - try_stream! { - // Seen-dirs is the only state we must accumulate: one String per unique - // immediate subdirectory name. File rows are yielded as they arrive. - let mut seen_dirs: HashSet = HashSet::new(); - - - let mut rows = if let Some(prefix) = &prefix { - let prefix_upper_bound = format!("{prefix}\u{10ffff}"); - - // NOTE: we're using >= and < for the prefix matching here. - // Using `GLOB` would mean we have to escape the path. - // Other techniques like sqlite string functions would mean the index on the - // table can't be used. - - sqlx::query("SELECT path FROM files WHERE path >= ? AND path < ?") - .bind(prefix) - .bind(prefix_upper_bound) - .fetch(&mut self.conn) - } else { - sqlx::query("SELECT path FROM files") - .fetch(&mut self.conn) - }; - - while let Some(row) = rows.try_next().await.context("error fetching entries from SQLite")? { - let full_path: String = row.try_get(0)?; - // The relative part is everything after the prefix. - let rel = if let Some(prefix) = &prefix { - // Archive paths are stored as UTF-8 strings, and `full_path` comes from - // the same prefix string used in the range query above. - debug_assert!(full_path.is_char_boundary(prefix.len())); - &full_path[prefix.len()..] - } else { - &full_path - }; - - if let Some(slash_pos) = rel.find('/') { - // It's inside a subdirectory. Extract and deduplicate the first component. - let dir_name = &rel[..slash_pos]; - if seen_dirs.insert(dir_name.to_string()) { - yield FolderEntry::Dir(dir_name.to_string()); - } - } else { - // Direct file — yield only the name relative to the queried folder. - let rel = rel.to_string(); - let mime = detect_mime(&rel); - yield FolderEntry::File(rel, mime); - } - } - } - } } #[cfg(test)] @@ -1757,153 +1681,6 @@ mod tests { Index::open(&tmp).await } - async fn collect_folder_contents( - index: &mut Index, - folder: Option<&str>, - ) -> Result<(Vec, Vec)> { - let entries: Vec = index - .folder_contents(folder.map(Path::new)) - .try_collect() - .await?; - - let mut files = Vec::new(); - let mut dirs = Vec::new(); - for entry in entries { - match entry { - FolderEntry::File(path, _) => files.push(path), - FolderEntry::Dir(name) => dirs.push(name), - } - } - files.sort(); - dirs.sort(); - Ok((files, dirs)) - } - - #[tokio::test] - async fn folder_contents_root_lists_files_and_dirs() -> Result<()> { - let mut index = index_from_entries(vec![ - ("index.html", b""), - ("style.css", b""), - ("sub/page.html", b""), - ("other/file.js", b""), - ]) - .await?; - - let (files, dirs) = collect_folder_contents(&mut index, None).await?; - - assert_eq!(files, vec!["index.html", "style.css"]); - assert_eq!(dirs, vec!["other", "sub"]); - - Ok(()) - } - - #[tokio::test] - async fn folder_contents_subfolder_lists_direct_children_only() -> Result<()> { - let mut index = index_from_entries(vec![ - ("src/main.rs", b""), - ("src/lib.rs", b""), - ("src/utils/helper.rs", b""), - ("src/utils/mod.rs", b""), - ("README.md", b""), - ]) - .await?; - - let (files, dirs) = collect_folder_contents(&mut index, Some("src")).await?; - - assert_eq!(files, vec!["lib.rs", "main.rs"]); - assert_eq!(dirs, vec!["utils"]); - - Ok(()) - } - - #[tokio::test] - async fn folder_contents_nested_subfolder() -> Result<()> { - let mut index = index_from_entries(vec![ - ("a/b/c/deep.txt", b""), - ("a/b/file.txt", b""), - ("a/b/other.txt", b""), - ]) - .await?; - - let (files, dirs) = collect_folder_contents(&mut index, Some("a/b")).await?; - - assert_eq!(files, vec!["file.txt", "other.txt"]); - assert_eq!(dirs, vec!["c"]); - - Ok(()) - } - - #[tokio::test] - async fn folder_contents_empty_folder_returns_nothing() -> Result<()> { - let mut index = index_from_entries(vec![("a/file.txt", b"")]).await?; - - let (files, dirs) = collect_folder_contents(&mut index, Some("nonexistent")).await?; - - assert!(files.is_empty()); - assert!(dirs.is_empty()); - - Ok(()) - } - - #[tokio::test] - async fn folder_contents_root_with_only_files() -> Result<()> { - let mut index = index_from_entries(vec![("a.txt", b""), ("b.txt", b"")]).await?; - - let (files, dirs) = collect_folder_contents(&mut index, None).await?; - - assert_eq!(files, vec!["a.txt", "b.txt"]); - assert!(dirs.is_empty()); - - Ok(()) - } - - #[tokio::test] - async fn folder_contents_subdir_deduplicated() -> Result<()> { - let mut index = index_from_entries(vec![ - ("sub/a.txt", b""), - ("sub/b.txt", b""), - ("sub/c.txt", b""), - ]) - .await?; - - let (files, dirs) = collect_folder_contents(&mut index, None).await?; - - assert!(files.is_empty()); - // "sub" should appear exactly once despite three files inside it - assert_eq!(dirs, vec!["sub"]); - - Ok(()) - } - - #[tokio::test] - async fn folder_contents_treats_glob_chars_literally() -> Result<()> { - let mut index = index_from_entries(vec![ - ("src[abc]/literal.rs", b""), - ("srca/wildcard.rs", b""), - ("srcb/wildcard.rs", b""), - ("srcc/wildcard.rs", b""), - ("src*/star.rs", b""), - ("srcx/star.rs", b""), - ("src?/question.rs", b""), - ("srcy/question.rs", b""), - ]) - .await?; - - let (files, dirs) = collect_folder_contents(&mut index, Some("src[abc]")).await?; - assert_eq!(files, vec!["literal.rs"]); - assert!(dirs.is_empty()); - - let (files, dirs) = collect_folder_contents(&mut index, Some("src*")).await?; - assert_eq!(files, vec!["star.rs"]); - assert!(dirs.is_empty()); - - let (files, dirs) = collect_folder_contents(&mut index, Some("src?")).await?; - assert_eq!(files, vec!["question.rs"]); - assert!(dirs.is_empty()); - - Ok(()) - } - #[tokio::test] async fn list_returns_all_entries() -> Result<()> { let mut index = index_from_entries(vec![ @@ -1951,56 +1728,4 @@ mod tests { Ok(()) } - - #[tokio::test] - async fn folder_contents_file_mime_correct() -> Result<()> { - let mut index = index_from_entries(vec![ - ("main.rs", b""), - ("README.md", b""), - ("style.css", b""), - ("data.json", b""), - ("index.html", b""), - ]) - .await?; - - let entries: Vec = index.folder_contents(None::<&Path>).try_collect().await?; - - let mut mime_map: Vec<(&str, String)> = entries - .iter() - .filter_map(|e| match e { - FolderEntry::File(name, mime) => Some((name.as_str(), mime.to_string())), - FolderEntry::Dir(_) => None, - }) - .collect(); - mime_map.sort_by_key(|(name, _)| *name); - - assert_eq!( - mime_map, - vec![ - ("README.md", "text/markdown".to_string()), - ("data.json", "application/json".to_string()), - ("index.html", "text/html".to_string()), - ("main.rs", "text/rust".to_string()), - ("style.css", "text/css".to_string()), - ] - ); - - Ok(()) - } - - #[tokio::test] - async fn folder_contents_dirs_have_no_mime() -> Result<()> { - let mut index = - index_from_entries(vec![("src/main.rs", b""), ("docs/readme.md", b"")]).await?; - - let entries: Vec = index.folder_contents(None::<&Path>).try_collect().await?; - - for entry in &entries { - if let FolderEntry::Dir(_) = entry { - assert!(entry.mime().is_none()); - } - } - - Ok(()) - } } diff --git a/crates/lib/docs_rs_storage/src/file.rs b/crates/lib/docs_rs_storage/src/file.rs index b26485d0e..f744022af 100644 --- a/crates/lib/docs_rs_storage/src/file.rs +++ b/crates/lib/docs_rs_storage/src/file.rs @@ -31,151 +31,3 @@ pub fn file_list_to_json(files: impl IntoIterator) -> Value { .collect(), ) } - -#[derive(Debug, Clone, Eq)] -pub enum FolderEntry { - File(String, Mime), - Dir(String), -} - -impl FolderEntry { - pub fn from_path(path: &str) -> Self { - if let Some((dir, _)) = path.split_once('/') { - Self::Dir(dir.to_string()) - } else { - Self::File(path.to_string(), detect_mime(path)) - } - } - - pub fn name(&self) -> &str { - match self { - FolderEntry::File(name, _) => name, - FolderEntry::Dir(name) => name, - } - } - - pub fn is_dir(&self) -> bool { - matches!(self, Self::Dir(_)) - } - - pub fn mime(&self) -> Option<&Mime> { - match self { - Self::File(_, mime) => Some(mime), - Self::Dir(_) => None, - } - } -} - -impl PartialEq for FolderEntry { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (FolderEntry::File(a, _), FolderEntry::File(b, _)) => a == b, - (FolderEntry::Dir(a), FolderEntry::Dir(b)) => a == b, - _ => false, - } - } -} - -impl PartialOrd for FolderEntry { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for FolderEntry { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - fn cmp_name(a: &str, b: &str) -> std::cmp::Ordering { - a.chars() - .flat_map(char::to_lowercase) - .cmp(b.chars().flat_map(char::to_lowercase)) - .then_with(|| a.cmp(b)) - } - - match (self, other) { - (FolderEntry::Dir(a), FolderEntry::Dir(b)) => cmp_name(a, b), - (FolderEntry::File(a, _), FolderEntry::File(b, _)) => cmp_name(a, b), - (FolderEntry::Dir(_), FolderEntry::File(_, _)) => std::cmp::Ordering::Less, - (FolderEntry::File(_, _), FolderEntry::Dir(_)) => std::cmp::Ordering::Greater, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn file(name: &str) -> FolderEntry { - FolderEntry::File(name.to_string(), detect_mime(name)) - } - - fn dir(name: &str) -> FolderEntry { - FolderEntry::Dir(name.to_string()) - } - - #[test] - fn folder_entry_ordering_sorts_dirs_before_files() { - let mut entries = vec![ - file("apple.txt"), - dir("zulu"), - file("zebra.txt"), - dir("alpha"), - ]; - - entries.sort(); - - assert_eq!( - entries, - vec![ - dir("alpha"), - dir("zulu"), - file("apple.txt"), - file("zebra.txt") - ] - ); - } - - #[test] - fn folder_entry_ordering_is_case_insensitive() { - let mut entries = vec![file("lib.rs"), file("Cargo.toml"), file("apple.txt")]; - - entries.sort(); - - assert_eq!( - entries, - vec![file("apple.txt"), file("Cargo.toml"), file("lib.rs")] - ); - } - - #[test] - fn folder_entry_ordering_dirs_is_case_insensitive() { - let mut entries = vec![dir("src"), dir("Cargo"), dir("apple")]; - - entries.sort(); - - assert_eq!(entries, vec![dir("apple"), dir("Cargo"), dir("src")]); - } - - #[test] - fn folder_entry_ordering_uses_name_tie_breaker() { - let mut entries = vec![file("readme.md"), file("README.md")]; - - entries.sort(); - - assert_eq!(entries, vec![file("README.md"), file("readme.md")]); - } - - #[test] - fn folder_entry_equality_ignores_mime() { - assert_eq!( - FolderEntry::File("same".to_string(), mime::TEXT_PLAIN), - FolderEntry::File("same".to_string(), mime::TEXT_HTML), - ); - } - - #[test] - fn folder_entry_equality_compares_variants_and_names() { - assert_eq!(dir("src"), dir("src")); - assert_ne!(dir("src"), dir("lib")); - assert_ne!(file("src"), dir("src")); - } -} diff --git a/crates/lib/docs_rs_storage/src/lib.rs b/crates/lib/docs_rs_storage/src/lib.rs index 26f97733c..036ef96f3 100644 --- a/crates/lib/docs_rs_storage/src/lib.rs +++ b/crates/lib/docs_rs_storage/src/lib.rs @@ -16,7 +16,7 @@ pub use blob::{Blob, BlobUpload, StreamingBlob}; pub use compression::{compress, compress_async, decompress}; pub use config::Config; pub use errors::{PathNotFoundError, SizeLimitReached}; -pub use file::{FileEntry, FolderEntry, file_list_to_json}; +pub use file::{FileEntry, file_list_to_json}; pub use storage::blocking::Storage; pub use storage::non_blocking::AsyncStorage; pub use types::StorageKind; diff --git a/gui-tests/source-viewer.goml b/gui-tests/source-viewer.goml deleted file mode 100644 index 89de3aa2a..000000000 --- a/gui-tests/source-viewer.goml +++ /dev/null @@ -1,39 +0,0 @@ -// Checks that the docs.rs source code viewer page looks as expected. -go-to: |DOC_PATH| + "/crate/sysinfo/latest/source/Cargo.toml" -// We enforce a theme to ensure that the color checks will succeed. -set-local-storage: {"rustdoc-theme": "dark", "rustdoc-use-system-theme": "false"} -reload: - -// Bugs appears when window size is at maximum given width so we set it. -set-window-size: (800, 800) - -// First we ensure that the "documentation link button" is taking only one line. -store-size: (".doc-link > span", {"height": icon_height}) -store-size: (".doc-link", {"height": button_height}) -assert-css: (".doc-link", {"padding-top": "10px", "padding-bottom": "10px"}) -// As confirmed just above, there is 20px of padding, so we subtract it from the check below. -assert: |button_height| - 20 < |icon_height| * 2 - -// We check that hovering the side menu toggle will update the border color and also change the -// mouse cursor. -assert-css: (".toggle-source", {"border-color": "#4e4e4e", "cursor": "pointer"}) -assert-css: (".toggle-source > button", {"cursor": "pointer"}) -move-cursor-to: ".toggle-source > button" -assert-css: (".toggle-source:hover", {"border-color": "#c0c0c0", "cursor": "pointer"}) -assert-css: (".toggle-source >button:hover", {"cursor": "pointer"}) - -store-size: ("#side-menu", {"width": side_menu_width}) -store-size: ("#source-code-container", {"width": source_width}) - -// Now we hide the side menu. -click: ".toggle-source > button" -wait-for: "#side-menu.collapsed" - -store-size: ("#side-menu", {"width": side_menu_width2}) -store-size: ("#source-code-container", {"width": source_width2}) - -// The elements width should have changed (smaller side menu and bigger source). -assert: |side_menu_width| > |side_menu_width2| -assert: |source_width| < |source_width2| -// The total width should not have changed. -assert: |source_width2| + |side_menu_width2| == |source_width| + |side_menu_width|