From 7d9d0b8c7320d363c2f13c9c5c77f3b4dc202abd Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Sun, 19 Jul 2026 09:45:45 -0400 Subject: [PATCH] Add HAR sidecar recording for final HTTP exchanges --- README.md | 4 + docs/advanced-features.md | 18 ++ docs/cli-reference.md | 22 ++ docs/getting-started.md | 10 + src/app.rs | 23 ++ src/cli.rs | 3 + src/flag_registry.rs | 3 + src/grpc/reflection.rs | 1 + src/har.rs | 400 ++++++++++++++++++++++++++++++++ src/http/client.rs | 8 +- src/http/mod.rs | 32 ++- src/http/request.rs | 84 +++++-- src/http/response.rs | 94 +++++++- src/http/response/formatters.rs | 6 + src/http/response/stream.rs | 106 ++++++++- src/http/transport/body.rs | 19 +- src/http/transport/client.rs | 13 ++ src/lib.rs | 1 + src/output/mod.rs | 166 +++++++++++++ src/update/client.rs | 1 + tests/har.rs | 257 ++++++++++++++++++++ 21 files changed, 1231 insertions(+), 40 deletions(-) create mode 100644 src/har.rs create mode 100644 tests/har.rs diff --git a/README.md b/README.md index df6bd58..2cfb83c 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,10 @@ default, and binary responses are protected from accidental terminal output. See [Output Formatting](docs/output-formatting.md) for pager, color, binary, clipboard, and file behavior. +Use `--har request.har` to record the final HTTP exchange as a HAR 1.2 sidecar +without changing normal response output. HAR files can contain credentials, +cookies, and bodies and should be treated as sensitive data. + ## Documentation Start with the **[documentation index](docs/README.md)**, or jump directly to: diff --git a/docs/advanced-features.md b/docs/advanced-features.md index 1a2fda4..cf7074c 100644 --- a/docs/advanced-features.md +++ b/docs/advanced-features.md @@ -616,6 +616,24 @@ Sessions are stored as JSON in the user's cache directory: ## Debugging Network Issues +### HAR Sidecars + +Use `--har PATH` to record the final HTTP exchange in HAR 1.2 format while +leaving normal response output unchanged: + +```sh +fetch --har request.har https://api.example.com/users +fetch -o response.json --har request.har https://api.example.com/users +``` + +The destination is reserved before network I/O and atomically installed after +the response completes. It honors `--clobber`. Redirect, retry, and +authentication challenge exchanges are not included; only the final exchange +is recorded. Captures larger than 16 MiB are counted but omitted from the HAR. + +HAR files may contain authorization headers, cookies, request bodies, and +response bodies. Store and share them as sensitive data. + ### Timing Waterfall `--timing` (or `-T`) displays a timing waterfall chart after the response, showing how time was spent across DNS resolution, TCP connection setup, TLS handshake, time to first byte, and body download: diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 5721539..5beb8e1 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -212,6 +212,28 @@ Overwrite existing output file (default behavior is to fail if file exists). fetch -o output.json --clobber example.com/data ``` +### `--har PATH` + +Write a HAR 1.2 sidecar containing the final HTTP exchange while preserving the +normal response output. The capture includes the effective request, finalized +headers and streamed request body, decoded response body, remote IP, and timing +information. Unary gRPC is supported; protobuf frames are stored as base64. + +```sh +fetch --har request.har https://api.example.com/users +fetch -o response.json --har request.har https://api.example.com/users +``` + +The destination is reserved before the request, is installed atomically after +the response completes, and follows `--clobber`. `PATH` cannot be `-` or match +`--output`. WebSocket, inspection, gRPC discovery, and `--dry-run` modes are not +supported. The initial HAR contains only the final exchange after redirects, +retries, or authentication challenges. + +HAR files may contain credentials, cookies, request bodies, and response bodies. +Treat them as sensitive data. Body capture is bounded at 16 MiB; larger bodies +are counted completely but their text is omitted with a truncation comment. + ### `--copy` Copy the response body to the system clipboard. The response is still printed diff --git a/docs/getting-started.md b/docs/getting-started.md index 5929305..b9297d2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -304,6 +304,16 @@ Use `-O` to save using the filename from the URL: fetch -O httpbin.org/image/png ``` +Record the final HTTP exchange as a HAR 1.2 sidecar without changing response +output: + +```sh +fetch --har request.har httpbin.org/json +``` + +HAR files can contain credentials, cookies, and bodies, so handle them as +sensitive data. + ### Viewing Images `fetch` can render images directly in terminals that support inline images (Kitty, iTerm2), with a block-character fallback for other terminals. diff --git a/src/app.rs b/src/app.rs index 1128ffc..7b05498 100644 --- a/src/app.rs +++ b/src/app.rs @@ -352,6 +352,29 @@ async fn run_inner(cli: &mut Cli) -> Result { return Err("flag '--remote-header-name' requires '--remote-name'".into()); } + if let Some(path) = cli.har.as_deref() { + if path == "-" { + return Err( + "invalid value '-' for option '--har': stdout is reserved for the response body" + .into(), + ); + } + if cli.output.as_deref().is_some_and(|output| { + output != "-" && crate::output::destinations_conflict(path, output) + }) { + return Err("flags '--har' and '--output' cannot use the same path".into()); + } + if cli.dry_run { + return Err("flag '--har' cannot be used with '--dry-run'".into()); + } + if cli.inspect_dns || cli.inspect_tls { + return Err("flag '--har' cannot be used with inspection modes".into()); + } + if cli.has_grpc_discovery() { + return Err("flag '--har' cannot be used with gRPC discovery modes".into()); + } + } + if cli.url.is_none() && cli.has_grpc_discovery() && !cli.has_proto_schema() { return Err(" must be provided unless --proto-file or --proto-desc is set".into()); } diff --git a/src/cli.rs b/src/cli.rs index a15cb30..6113601 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -422,6 +422,9 @@ pub struct Cli { )] pub output: Option, + #[arg(long, value_name = "PATH", help = "Write a HAR 1.2 sidecar file")] + pub har: Option, + #[arg( long = "proto-desc", value_name = "PATH", diff --git a/src/flag_registry.rs b/src/flag_registry.rs index 35e9408..e006d8f 100644 --- a/src/flag_registry.rs +++ b/src/flag_registry.rs @@ -136,6 +136,9 @@ pub(crate) static FLAGS: &[FlagDef] = &[ }) .with_from_curl() .with_ws_always(), + FlagDef::new("--har", Some(FlagCategory::Response), |c| c.har.is_some()) + .with_from_curl() + .with_ws_always(), FlagDef::new("--remote-name", Some(FlagCategory::Request), |c| { c.remote_name }) diff --git a/src/grpc/reflection.rs b/src/grpc/reflection.rs index 8f26494..c7a26d4 100644 --- a/src/grpc/reflection.rs +++ b/src/grpc/reflection.rs @@ -61,6 +61,7 @@ pub async fn execute_discovery(cli: &Cli) -> Result { request_start, session: session.as_ref(), connect_timing: Some(&connect_timing), + har: None, }; let client = crate::http::client::build_client_for_url(cli, &url, &client_build) .await? diff --git a/src/har.rs b/src/har.rs new file mode 100644 index 0000000..21dde58 --- /dev/null +++ b/src/har.rs @@ -0,0 +1,400 @@ +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +use base64::Engine; +use http::{HeaderMap, Method, Version}; +use serde::Serialize; +use url::Url; + +use crate::core; +use crate::error::FetchError; +use crate::output::PreparedOutput; +use crate::timing::ResponseTiming; + +pub(crate) const CAPTURE_LIMIT: usize = 16 * 1024 * 1024; + +#[derive(Clone, Default)] +pub(crate) struct Capture(Arc>); + +#[derive(Default)] +struct CaptureState { + bytes: Vec, + size: i64, + truncated: bool, + receive: Duration, +} + +impl Capture { + pub(crate) fn push(&self, bytes: &[u8]) { + let Ok(mut state) = self.0.lock() else { return }; + state.size = state + .size + .saturating_add(i64::try_from(bytes.len()).unwrap_or(i64::MAX)); + let remaining = CAPTURE_LIMIT.saturating_sub(state.bytes.len()); + state + .bytes + .extend_from_slice(&bytes[..bytes.len().min(remaining)]); + state.truncated |= bytes.len() > remaining; + } + + pub(crate) fn add_receive_time(&self, started: Instant) { + if let Ok(mut state) = self.0.lock() { + state.receive = state.receive.saturating_add(started.elapsed()); + } + } + + #[cfg(test)] + pub(crate) fn receive_time(&self) -> Duration { + self.0.lock().map(|state| state.receive).unwrap_or_default() + } + + fn snapshot(&self) -> CaptureSnapshot { + let state = self.0.lock().expect("HAR capture lock poisoned"); + CaptureSnapshot { + bytes: state.bytes.clone(), + size: state.size, + truncated: state.truncated, + receive: state.receive, + } + } +} + +struct CaptureSnapshot { + bytes: Vec, + size: i64, + truncated: bool, + receive: Duration, +} + +#[derive(Clone)] +pub(crate) struct Recorder(Arc>); + +struct RecorderState { + request: Option, + response_body: Capture, +} + +struct RequestCapture { + method: Method, + url: Url, + headers: HeaderMap, + body: Capture, +} + +pub(crate) struct ResponseMeta { + pub status: u16, + pub status_text: String, + pub headers: HeaderMap, + pub version: Version, + pub redirect_url: String, + pub remote_ip: Option, + pub content_type: String, + pub timing: Option, + pub started: SystemTime, +} + +impl Recorder { + pub(crate) fn new() -> Self { + Self(Arc::new(Mutex::new(RecorderState { + request: None, + response_body: Capture::default(), + }))) + } + + pub(crate) fn observe_request(&self, method: Method, url: Url, headers: HeaderMap) -> Capture { + let body = Capture::default(); + let mut state = self.0.lock().expect("HAR recorder lock poisoned"); + state.request = Some(RequestCapture { + method, + url, + headers, + body: body.clone(), + }); + body + } + + pub(crate) fn response_capture(&self) -> Capture { + self.0 + .lock() + .expect("HAR recorder lock poisoned") + .response_body + .clone() + } + + pub(crate) fn serialize(&self, meta: ResponseMeta) -> Result, FetchError> { + let state = self.0.lock().expect("HAR recorder lock poisoned"); + let request = state + .request + .as_ref() + .ok_or_else(|| FetchError::Message("unable to capture final request for HAR".into()))?; + let request_body = request.body.snapshot(); + let response_body = state.response_body.snapshot(); + let request_headers = header_entries(&request.headers); + let response_headers = header_entries(&meta.headers); + let query = request + .url + .query_pairs() + .map(|(name, value)| NameValue { + name: name.into_owned(), + value: value.into_owned(), + }) + .collect(); + let post_data = (request_body.size > 0).then(|| PostData { + mime_type: content_type(&request.headers), + text: body_text(&request_body), + params: Vec::new(), + comment: truncation_comment(&request_body), + }); + let timing = meta.timing; + let receive = duration_ms(response_body.receive); + let dns = timing + .and_then(|value| value.dns) + .map(duration_ms) + .unwrap_or(-1.0); + let connect = timing + .and_then(|value| value.tcp.or(value.quic)) + .map(duration_ms) + .unwrap_or(-1.0); + let ssl = timing + .and_then(|value| value.tls) + .map(duration_ms) + .unwrap_or(-1.0); + let wait = timing + .map(|value| duration_ms(value.ttfb.saturating_sub(value.dns.unwrap_or_default()))) + .unwrap_or(-1.0); + let total_time = [dns, connect, ssl, wait, receive] + .into_iter() + .filter(|value| *value >= 0.0) + .sum(); + let doc = Har { + log: Log { + version: "1.2", + creator: Creator { + name: "fetch", + version: core::version(), + }, + entries: vec![Entry { + started_date_time: rfc3339(meta.started), + time: total_time, + request: HarRequest { + method: request.method.as_str(), + url: request.url.as_str(), + http_version: http_version(meta.version), + cookies: Vec::new(), + headers: request_headers, + query_string: query, + post_data, + headers_size: -1, + body_size: request_body.size, + }, + response: HarResponse { + status: meta.status, + status_text: meta.status_text, + http_version: http_version(meta.version), + cookies: Vec::new(), + headers: response_headers, + content: Content { + size: response_body.size, + mime_type: meta.content_type, + text: body_text(&response_body), + encoding: body_encoding(&response_body), + comment: truncation_comment(&response_body), + }, + redirect_url: meta.redirect_url, + headers_size: -1, + body_size: response_body.size, + }, + cache: Empty {}, + timings: Timings { + blocked: -1.0, + dns, + connect, + send: -1.0, + wait, + receive, + ssl, + }, + server_ip_address: meta.remote_ip, + comment: "fetch records only the final HTTP exchange", + }], + }, + }; + serde_json::to_vec_pretty(&doc).map_err(|err| FetchError::Message(err.to_string())) + } +} + +pub(crate) struct Destination(PreparedOutput); + +impl Destination { + pub(crate) fn reserve(path: &str, clobber: bool) -> Result { + PreparedOutput::create(path, clobber) + .map(Self) + .map_err(|err| FetchError::Message(err.to_string())) + } + + pub(crate) fn commit(self, bytes: &[u8]) -> Result<(), FetchError> { + self.0 + .commit(bytes) + .map_err(|err| FetchError::Message(err.to_string())) + } +} + +#[derive(Serialize)] +struct Har<'a> { + log: Log<'a>, +} +#[derive(Serialize)] +struct Log<'a> { + version: &'a str, + creator: Creator<'a>, + entries: Vec>, +} +#[derive(Serialize)] +struct Creator<'a> { + name: &'a str, + version: &'a str, +} +#[derive(Serialize)] +struct Entry<'a> { + #[serde(rename = "startedDateTime")] + started_date_time: String, + time: f64, + request: HarRequest<'a>, + response: HarResponse, + cache: Empty, + timings: Timings, + #[serde(rename = "serverIPAddress", skip_serializing_if = "Option::is_none")] + server_ip_address: Option, + comment: &'a str, +} +#[derive(Serialize)] +struct HarRequest<'a> { + method: &'a str, + url: &'a str, + #[serde(rename = "httpVersion")] + http_version: &'static str, + cookies: Vec, + headers: Vec, + #[serde(rename = "queryString")] + query_string: Vec, + #[serde(rename = "postData", skip_serializing_if = "Option::is_none")] + post_data: Option, + #[serde(rename = "headersSize")] + headers_size: i64, + #[serde(rename = "bodySize")] + body_size: i64, +} +#[derive(Serialize)] +struct HarResponse { + status: u16, + #[serde(rename = "statusText")] + status_text: String, + #[serde(rename = "httpVersion")] + http_version: &'static str, + cookies: Vec, + headers: Vec, + content: Content, + #[serde(rename = "redirectURL")] + redirect_url: String, + #[serde(rename = "headersSize")] + headers_size: i64, + #[serde(rename = "bodySize")] + body_size: i64, +} +#[derive(Serialize)] +struct PostData { + #[serde(rename = "mimeType")] + mime_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + text: Option, + params: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + comment: Option<&'static str>, +} +#[derive(Serialize)] +struct Content { + size: i64, + #[serde(rename = "mimeType")] + mime_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + text: Option, + #[serde(skip_serializing_if = "Option::is_none")] + encoding: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + comment: Option<&'static str>, +} +#[derive(Serialize)] +struct NameValue { + name: String, + value: String, +} +#[derive(Serialize)] +struct Timings { + blocked: f64, + dns: f64, + connect: f64, + send: f64, + wait: f64, + receive: f64, + ssl: f64, +} +#[derive(Serialize)] +struct Empty {} + +fn header_entries(headers: &HeaderMap) -> Vec { + headers + .iter() + .map(|(name, value)| NameValue { + name: name.as_str().to_string(), + value: String::from_utf8_lossy(value.as_bytes()).into_owned(), + }) + .collect() +} +fn content_type(headers: &HeaderMap) -> String { + headers + .get(http::header::CONTENT_TYPE) + .map(|v| String::from_utf8_lossy(v.as_bytes()).into_owned()) + .unwrap_or_default() +} +fn body_text(body: &CaptureSnapshot) -> Option { + if body.truncated { + None + } else if let Ok(text) = std::str::from_utf8(&body.bytes) { + Some(text.to_string()) + } else { + Some(base64::engine::general_purpose::STANDARD.encode(&body.bytes)) + } +} +fn body_encoding(body: &CaptureSnapshot) -> Option<&'static str> { + (!body.truncated && std::str::from_utf8(&body.bytes).is_err()).then_some("base64") +} +fn truncation_comment(body: &CaptureSnapshot) -> Option<&'static str> { + body.truncated + .then_some("Body omitted by fetch because it exceeds the 16 MiB HAR capture limit") +} +fn duration_ms(value: Duration) -> f64 { + value.as_secs_f64() * 1000.0 +} +fn http_version(version: Version) -> &'static str { + match version { + Version::HTTP_09 => "HTTP/0.9", + Version::HTTP_10 => "HTTP/1.0", + Version::HTTP_11 => "HTTP/1.1", + Version::HTTP_2 => "HTTP/2", + Version::HTTP_3 => "HTTP/3", + _ => "HTTP", + } +} +fn rfc3339(value: SystemTime) -> String { + let dt = time::OffsetDateTime::from(value); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z", + dt.year(), + u8::from(dt.month()), + dt.day(), + dt.hour(), + dt.minute(), + dt.second(), + dt.millisecond() + ) +} diff --git a/src/http/client.rs b/src/http/client.rs index f6172ae..37c7c1e 100644 --- a/src/http/client.rs +++ b/src/http/client.rs @@ -78,6 +78,7 @@ pub(crate) struct ClientBuildContext<'a> { pub(crate) request_start: Instant, pub(crate) session: Option<&'a crate::session::Session>, pub(crate) connect_timing: Option<&'a ConnectionTiming>, + pub(crate) har: Option<&'a crate::har::Recorder>, } #[derive(Clone, Debug)] @@ -116,7 +117,7 @@ pub(crate) async fn build_client_for_url( let effective_proxy = effective_proxy_for_url(cli.proxy.as_deref(), http_version, url)?; let auto_http3 = auto_http3_allowed(context.mode, url, cli.unix.as_deref(), effective_proxy); let discovery = if dynamic_dns_for_client(cli, url, effective_proxy) { - let debug_dns = cli.timing || (cli.verbose >= 3 && !cli.silent); + let debug_dns = cli.timing || cli.har.is_some() || (cli.verbose >= 3 && !cli.silent); ClientDnsDiscovery { dns_resolution: None, runtime_dns_resolution: debug_dns.then(DnsResolutionHandle::default), @@ -167,10 +168,13 @@ pub(crate) async fn build_client_for_url( } builder = configure_dns_resolution(builder, url.host_str(), dns_resolution.as_ref()); if let Some(connect_timing) = context.connect_timing - && (cli.timing || (cli.verbose >= 3 && !cli.silent)) + && (cli.timing || cli.har.is_some() || (cli.verbose >= 3 && !cli.silent)) { builder = builder.connection_timing(connect_timing.clone()); } + if let Some(recorder) = context.har { + builder = builder.har_recorder(recorder.clone()); + } if should_configure_tls(cli, url) { let hard_fail = matches!(cli.ech.as_deref(), Some("on")); builder = builder.ech_hard_fail(hard_fail); diff --git a/src/http/mod.rs b/src/http/mod.rs index 741fea6..458370e 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -132,6 +132,12 @@ async fn execute_request( mut grpc_method: Option, session: Option<&crate::session::Session>, ) -> Result { + let har_recorder = cli.har.as_ref().map(|_| crate::har::Recorder::new()); + let har_destination = cli + .har + .as_deref() + .map(|path| crate::har::Destination::reserve(path, cli.clobber)) + .transpose()?; let request_start = Instant::now(); let request_timeout = cli .timeout @@ -153,6 +159,7 @@ async fn execute_request( request_start, session, connect_timing: Some(&connect_timing), + har: har_recorder.as_ref(), }; let mut initial_client = None; if cli.grpc && grpc_method.is_none() { @@ -293,6 +300,7 @@ async fn execute_request( }, )?; let req = apply_request_timeout(req, request_timeout, request_start)?; + let exchange_started = SystemTime::now(); request_client.clear_runtime_dns_resolution(); connect_timing.clear(); match Box::pin(req.send()).await { @@ -323,10 +331,13 @@ async fn execute_request( )) .await?; } + if cli.har.is_some() { + timing = AttemptTiming::start(); + } redirect_count += 1; continue; } - break Ok(response); + break Ok((response, exchange_started)); } Err(err) => { if protocol_nack_retries < MAX_PROTOCOL_NACK_RETRIES @@ -334,6 +345,9 @@ async fn execute_request( { ensure_request_body_replayable(&request_body, "protocol retry")?; protocol_nack_retries += 1; + if cli.har.is_some() { + timing = AttemptTiming::start(); + } continue; } record_request_dns_timing(cli, &request_client, &mut timing); @@ -342,7 +356,7 @@ async fn execute_request( } }; match attempt_result { - Ok(response) => { + Ok((response, mut exchange_started)) => { timing.mark_response_headers(); timing.set_transport(connect_timing.timing()); if cli.verbose >= 3 && !cli.silent { @@ -351,10 +365,10 @@ async fn execute_request( connect_debug_target(&response, &request_url, dns_resolution.as_ref()); timing::print_debug_lines(&timing, &connect_target, cli.color.as_deref()); } - let response = Box::pin(apply_digest_challenge( + let digest_result = Box::pin(apply_digest_challenge( response, DigestRetryContext { - client: &request_client.client, + client: &request_client, client_build: &client_build, method: request_method.clone(), headers: headers.clone(), @@ -366,6 +380,13 @@ async fn execute_request( digest_credentials.as_ref(), )) .await?; + let response = digest_result.response; + if let Some(digest_timing) = digest_result.timing { + timing = digest_timing; + } + if let Some(digest_started) = digest_result.started { + exchange_started = digest_started; + } let status = response.status(); let retry_sse_uncompressed = should_retry_sse_without_compression(&response, compression); @@ -431,6 +452,9 @@ async fn execute_request( compression, Some(timing), grpc_method.as_ref(), + har_recorder.as_ref(), + har_destination, + exchange_started, ) .await; } diff --git a/src/http/request.rs b/src/http/request.rs index 64db7f9..9744baa 100644 --- a/src/http/request.rs +++ b/src/http/request.rs @@ -192,7 +192,7 @@ pub(super) fn transport_request_version_for_cli( } pub(super) struct DigestRetryContext<'a> { - pub(super) client: &'a Client, + pub(super) client: &'a client::UrlClient, pub(super) client_build: &'a client::ClientBuildContext<'a>, pub(super) method: Method, pub(super) headers: HeaderMap, @@ -202,19 +202,35 @@ pub(super) struct DigestRetryContext<'a> { pub(super) auth_allowed: bool, } +pub(super) struct DigestChallengeResult { + pub(super) response: Response, + pub(super) timing: Option, + pub(super) started: Option, +} + +impl DigestChallengeResult { + fn unchanged(response: Response) -> Self { + Self { + response, + timing: None, + started: None, + } + } +} + pub(super) async fn apply_digest_challenge( response: Response, context: DigestRetryContext<'_>, credentials: Option<&(String, String)>, -) -> Result { +) -> Result { let Some((username, password)) = credentials else { - return Ok(response); + return Ok(DigestChallengeResult::unchanged(response)); }; if response.status() != StatusCode::UNAUTHORIZED { - return Ok(response); + return Ok(DigestChallengeResult::unchanged(response)); } if !context.auth_allowed { - return Ok(response); + return Ok(DigestChallengeResult::unchanged(response)); } let challenge = digest::find_digest_challenge( @@ -225,7 +241,7 @@ pub(super) async fn apply_digest_challenge( .filter_map(|value| value.to_str().ok()), ); let Some(challenge) = challenge else { - return Ok(response); + return Ok(DigestChallengeResult::unchanged(response)); }; let challenge = digest::parse_challenge(&challenge).map_err(digest_challenge_error)?; @@ -257,11 +273,9 @@ pub(super) async fn apply_digest_challenge( } else { None }; - let client = retry_client - .as_ref() - .map_or(context.client, |client| &client.client); + let client = retry_client.as_ref().unwrap_or(context.client); let retry_request = build_request( - client, + &client.client, challenged_method, challenged_url, challenged_headers, @@ -275,10 +289,41 @@ pub(super) async fn apply_digest_challenge( context.client_build.request_start, )?; + let mut challenge_response = Some(response); + if !retry_before_drain { + drain_response_body_bounded( + challenge_response + .take() + .expect("Digest challenge response is available before draining"), + ) + .await; + } + + client.clear_runtime_dns_resolution(); + if let Some(connect_timing) = context.client_build.connect_timing { + connect_timing.clear(); + } + let started = SystemTime::now(); + let mut timing = AttemptTiming::start(); + let retry_response: Result = + retry_request.send().await.map_err(Into::into); + timing.mark_response_headers(); + timing.set_transport( + context + .client_build + .connect_timing + .and_then(client::ConnectionTiming::timing), + ); + timing.set_dns( + client + .current_dns_resolution() + .as_ref() + .and_then(|resolution| resolution.timing.as_ref()) + .map(|dns| dns.duration), + ); + if retry_before_drain { - let retry_response: Result = - retry_request.send().await.map_err(Into::into); - drop(response); + drop(challenge_response.take()); retry_response.map(|mut response| { // The temporary client's pool owns the connection task. Retain it until the // response body is consumed so dropping the pool cannot cancel that body. @@ -287,11 +332,18 @@ pub(super) async fn apply_digest_challenge( .expect("retry-before-drain builds a fresh client") .client, ); - response + DigestChallengeResult { + response, + timing: Some(timing), + started: Some(started), + } }) } else { - drain_response_body_bounded(response).await; - retry_request.send().await.map_err(Into::into) + retry_response.map(|response| DigestChallengeResult { + response, + timing: Some(timing), + started: Some(started), + }) } } diff --git a/src/http/response.rs b/src/http/response.rs index 8d92d30..47b6053 100644 --- a/src/http/response.rs +++ b/src/http/response.rs @@ -26,12 +26,63 @@ use stream::{ stream_response_to_discard, stream_response_to_output, stream_response_to_stdout, }; +#[allow(clippy::too_many_arguments)] pub(super) async fn finish_response( cli: &Cli, response: Response, compression: CompressionMode, timing: Option, grpc_method: Option<&prost_reflect::MethodDescriptor>, + har_recorder: Option<&crate::har::Recorder>, + har_destination: Option, + har_started: SystemTime, +) -> Result { + let response_timing = timing.and_then(AttemptTiming::response_timing); + let status = response.status(); + let headers = response.headers().clone(); + let response_meta = har_recorder.map(|_| crate::har::ResponseMeta { + status: status.as_u16(), + status_text: status.canonical_reason().unwrap_or_default().to_string(), + redirect_url: headers + .get(LOCATION) + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .unwrap_or_default(), + content_type: headers + .get(CONTENT_TYPE) + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .unwrap_or_default(), + headers, + version: response.version(), + remote_ip: response.remote_addr().map(|addr| addr.ip().to_string()), + timing: response_timing, + started: har_started, + }); + let result = finish_response_output( + cli, + response, + compression, + response_timing, + grpc_method, + har_recorder.map(crate::har::Recorder::response_capture), + ) + .await; + let code = result?; + if let (Some(recorder), Some(destination), Some(meta)) = + (har_recorder, har_destination, response_meta) + { + let bytes = recorder.serialize(meta)?; + destination.commit(&bytes)?; + } + Ok(code) +} + +async fn finish_response_output( + cli: &Cli, + response: Response, + compression: CompressionMode, + response_timing: Option, + grpc_method: Option<&prost_reflect::MethodDescriptor>, + har_capture: Option, ) -> Result { let status = response.status(); print_response_metadata(cli, &response); @@ -43,13 +94,17 @@ pub(super) async fn finish_response( let output_progress_total = output_progress_total_bytes(compression, &response_headers, response_content_length); let method_is_head = cli.method().eq_ignore_ascii_case("HEAD"); - let response_timing = timing.and_then(AttemptTiming::response_timing); let stdio = core::stdio(); if cli.discard { let body_start = Instant::now(); - let streamed = - stream_response_to_discard(response, response_headers.clone(), compression).await?; + let streamed = stream_response_to_discard( + response, + response_headers.clone(), + compression, + har_capture, + ) + .await?; return Ok(finalize_streamed_response( cli, status, @@ -72,6 +127,14 @@ pub(super) async fn finish_response( if let Some(warning) = &resolved_output.warning { write_warning(cli, warning); } + if let (Some(har), Some(response_output)) = + (cli.har.as_deref(), resolved_output.path.as_deref()) + && output::destinations_conflict(har, response_output) + { + return Err(FetchError::Message( + "flags '--har' and response output cannot use the same path".into(), + )); + } if cli.article { return finish_article_response( cli, @@ -83,6 +146,7 @@ pub(super) async fn finish_response( response_timing, method_is_head, resolved_output.path.as_deref(), + har_capture, ) .await; } @@ -101,6 +165,7 @@ pub(super) async fn finish_response( cli.clobber, progress, cli.copy, + har_capture, ) .await?; return Ok(finalize_streamed_response( @@ -124,6 +189,7 @@ pub(super) async fn finish_response( compression, cli.copy, use_color, + har_capture, ) .await?; return Ok(finalize_streamed_response( @@ -144,6 +210,7 @@ pub(super) async fn finish_response( compression, cli.copy, use_color, + har_capture, ) .await?; return Ok(finalize_streamed_response( @@ -165,6 +232,7 @@ pub(super) async fn finish_response( cli.copy, grpc_method.map(|method| method.output()), use_color, + har_capture, ) .await?; return Ok(finalize_streamed_response( @@ -186,6 +254,7 @@ pub(super) async fn finish_response( cli.copy, target, stdout_is_terminal, + har_capture, ) .await?; return Ok(finalize_streamed_response( @@ -199,8 +268,13 @@ pub(super) async fn finish_response( )); } - let (bytes, trailers) = - read_decoded_response_body_limited(response, response_headers.clone(), compression).await?; + let (bytes, trailers) = read_decoded_response_body_limited( + response, + response_headers.clone(), + compression, + har_capture, + ) + .await?; let body_duration = body_duration(method_is_head, bytes.as_ref(), body_start); if cli.copy { handle_clipboard_outcome(cli, clipboard::copy_bytes(&bytes)); @@ -229,10 +303,16 @@ async fn finish_article_response( response_timing: Option, method_is_head: bool, output_path: Option<&str>, + har_capture: Option, ) -> Result { let body_start = Instant::now(); - let (bytes, trailers) = - read_decoded_article_body_limited(response, response_headers.clone(), compression).await?; + let (bytes, trailers) = read_decoded_article_body_limited( + response, + response_headers.clone(), + compression, + har_capture, + ) + .await?; let body_duration = body_duration(method_is_head, &bytes, body_start); let input_kind = article_response_content_kind(&response_headers, &bytes); diff --git a/src/http/response/formatters.rs b/src/http/response/formatters.rs index 63c02d3..94dd5ce 100644 --- a/src/http/response/formatters.rs +++ b/src/http/response/formatters.rs @@ -53,6 +53,7 @@ pub(super) async fn stream_response_to_formatted_sse_stdout( compression: CompressionMode, copy: bool, use_color: bool, + har_capture: Option, ) -> Result { super::stream::stream_formatted_response_to_stdout( response, @@ -60,6 +61,7 @@ pub(super) async fn stream_response_to_formatted_sse_stdout( compression, copy, FormattedSseStream::new(use_color), + har_capture, ) .await } @@ -70,6 +72,7 @@ pub(super) async fn stream_response_to_formatted_ndjson_stdout( compression: CompressionMode, copy: bool, use_color: bool, + har_capture: Option, ) -> Result { super::stream::stream_formatted_response_to_stdout( response, @@ -77,6 +80,7 @@ pub(super) async fn stream_response_to_formatted_ndjson_stdout( compression, copy, FormattedNdjsonStream::new(use_color), + har_capture, ) .await } @@ -88,6 +92,7 @@ pub(super) async fn stream_response_to_formatted_grpc_stdout( copy: bool, grpc_response_desc: Option, use_color: bool, + har_capture: Option, ) -> Result { let formatter = FormattedGrpcStream::new(&response_headers, grpc_response_desc, use_color); super::stream::stream_formatted_response_to_stdout( @@ -96,6 +101,7 @@ pub(super) async fn stream_response_to_formatted_grpc_stdout( compression, copy, formatter, + har_capture, ) .await } diff --git a/src/http/response/stream.rs b/src/http/response/stream.rs index 1907e74..4618e2f 100644 --- a/src/http/response/stream.rs +++ b/src/http/response/stream.rs @@ -15,11 +15,13 @@ pub(super) async fn read_decoded_response_body_limited( response: Response, response_headers: HeaderMap, compression: CompressionMode, + har_capture: Option, ) -> Result<(Vec, HeaderMap), FetchError> { read_decoded_response_body_with_limit_message( response, response_headers, compression, + har_capture, "cannot be buffered; use '--format off' or write to a file to stream it", ) .await @@ -29,11 +31,13 @@ pub(super) async fn read_decoded_article_body_limited( response: Response, response_headers: HeaderMap, compression: CompressionMode, + har_capture: Option, ) -> Result<(Vec, HeaderMap), FetchError> { read_decoded_response_body_with_limit_message( response, response_headers, compression, + har_capture, "cannot be extracted as an article", ) .await @@ -43,10 +47,11 @@ async fn read_decoded_response_body_with_limit_message( response: Response, response_headers: HeaderMap, compression: CompressionMode, + har_capture: Option, limit_message: &str, ) -> Result<(Vec, HeaderMap), FetchError> { - let (reader, trailers) = async_response_reader(response); - let mut reader = decoded_async_response_reader(reader, compression, &response_headers)?; + let (mut reader, trailers) = + decoded_capturing_response_reader(response, compression, &response_headers, har_capture)?; let mut bytes = Vec::new(); let mut buf = vec![0; 16 * 1024]; loop { @@ -114,9 +119,10 @@ pub(super) async fn stream_response_to_discard( response: Response, response_headers: HeaderMap, compression: CompressionMode, + har_capture: Option, ) -> Result { - let (reader, trailers) = async_response_reader(response); - let mut reader = decoded_async_response_reader(reader, compression, &response_headers)?; + let (mut reader, trailers) = + decoded_capturing_response_reader(response, compression, &response_headers, har_capture)?; let mut sink = tokio::io::sink(); let bytes_written = copy_async_reader_to_writer(&mut reader, &mut sink, None).await?; let trailers = captured_trailers(&trailers); @@ -127,6 +133,7 @@ pub(super) async fn stream_response_to_discard( }) } +#[allow(clippy::too_many_arguments)] pub(super) async fn stream_response_to_stdout( cli: &Cli, response: Response, @@ -135,9 +142,10 @@ pub(super) async fn stream_response_to_stdout( copy: bool, target: StdoutStreamTarget, stdout_is_terminal: bool, + har_capture: Option, ) -> Result { - let (reader, trailers) = async_response_reader(response); - let mut reader = decoded_async_response_reader(reader, compression, &response_headers)?; + let (mut reader, trailers) = + decoded_capturing_response_reader(response, compression, &response_headers, har_capture)?; let mut capture = copy.then(clipboard::Capture::default); let bytes_written = if terminal_binary_stdout_guard_enabled(cli, stdout_is_terminal) { stream_response_to_stdout_with_binary_check( @@ -166,12 +174,13 @@ pub(super) async fn stream_formatted_response_to_stdout( compression: CompressionMode, copy: bool, mut formatter: F, + har_capture: Option, ) -> Result where F: StdoutStreamFormatter, { - let (reader, trailers) = async_response_reader(response); - let mut reader = decoded_async_response_reader(reader, compression, &response_headers)?; + let (mut reader, trailers) = + decoded_capturing_response_reader(response, compression, &response_headers, har_capture)?; let mut stdout = tokio::io::stdout(); let mut capture = copy.then(clipboard::Capture::default); let mut buf = vec![0; 16 * 1024]; @@ -255,6 +264,7 @@ async fn flush_stdout( Ok(core::stdout_write_status(stdout.flush().await)?) } +#[allow(clippy::too_many_arguments)] pub(super) async fn stream_response_to_output( response: Response, response_headers: HeaderMap, @@ -263,9 +273,10 @@ pub(super) async fn stream_response_to_output( clobber: bool, progress: output::WriteProgress, copy: bool, + har_capture: Option, ) -> Result { - let (reader, trailers) = async_response_reader(response); - let mut reader = decoded_async_response_reader(reader, compression, &response_headers)?; + let (mut reader, trailers) = + decoded_capturing_response_reader(response, compression, &response_headers, har_capture)?; let mut capture = copy.then(clipboard::Capture::default); let bytes_written = if let Some(capture) = capture.as_mut() { let mut reader = AsyncClipboardTeeReader { reader, capture }; @@ -349,6 +360,56 @@ fn async_response_reader(response: Response) -> (AsyncReadBox, ResponseTrailers) (Box::pin(StreamReader::new(stream)), trailers) } +fn decoded_capturing_response_reader( + response: Response, + compression: CompressionMode, + response_headers: &HeaderMap, + capture: Option, +) -> Result<(AsyncReadBox, ResponseTrailers), FetchError> { + let (reader, trailers) = async_response_reader(response); + let reader = decoded_async_response_reader(reader, compression, response_headers)?; + let reader: AsyncReadBox = match capture { + Some(capture) => Box::pin(AsyncHarTeeReader { + reader, + capture, + read_started: None, + }), + None => reader, + }; + Ok((reader, trailers)) +} + +struct AsyncHarTeeReader { + reader: AsyncReadBox, + capture: crate::har::Capture, + read_started: Option, +} + +impl AsyncRead for AsyncHarTeeReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let started = *self.read_started.get_or_insert_with(Instant::now); + let before = buf.filled().len(); + match self.reader.as_mut().poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + self.capture.add_receive_time(started); + self.read_started = None; + self.capture.push(&buf.filled()[before..]); + Poll::Ready(Ok(())) + } + Poll::Ready(Err(err)) => { + self.capture.add_receive_time(started); + self.read_started = None; + Poll::Ready(Err(err)) + } + other => other, + } + } +} + async fn stream_response_to_stdout_with_binary_check( cli: &Cli, response_headers: &HeaderMap, @@ -1115,6 +1176,31 @@ mod tests { assert_eq!(writer.bytes, body); } + #[tokio::test] + async fn har_receive_timing_excludes_time_between_body_reads() { + let capture = crate::har::Capture::default(); + let reader: AsyncReadBox = Box::pin(ChunkedReader { + inner: std::io::Cursor::new(b"ab"), + chunk_size: 1, + }); + let mut reader = AsyncHarTeeReader { + reader, + capture: capture.clone(), + read_started: None, + }; + let mut byte = [0_u8; 1]; + assert_eq!(reader.read(&mut byte).await.unwrap(), 1); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(reader.read(&mut byte).await.unwrap(), 1); + assert_eq!(reader.read(&mut byte).await.unwrap(), 0); + + assert!( + capture.receive_time() < Duration::from_millis(50), + "HAR receive time included local delay: {:?}", + capture.receive_time() + ); + } + /// Test helper that wraps a [`std::io::Read`] and limits each /// [`AsyncRead::poll_read`] call to at most `chunk_size` bytes. struct ChunkedReader { diff --git a/src/http/transport/body.rs b/src/http/transport/body.rs index 9850f3c..428de32 100644 --- a/src/http/transport/body.rs +++ b/src/http/transport/body.rs @@ -47,6 +47,7 @@ impl BodyDeadline { pub struct Body { inner: UnsyncBoxBody, _client_keepalive: Option>, + har_capture: Option, } impl Body { @@ -68,9 +69,15 @@ impl Body { Self { inner: BodyExt::boxed_unsync(body), _client_keepalive: None, + har_capture: None, } } + pub(crate) fn with_har_capture(mut self, capture: crate::har::Capture) -> Self { + self.har_capture = Some(capture); + self + } + fn keep_client_alive(&mut self, client: super::client::Client) { self._client_keepalive = Some(Box::new(client)); } @@ -118,7 +125,17 @@ impl http_body::Body for Body { mut self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll, Self::Error>>> { - Pin::new(&mut self.inner).poll_frame(cx) + match Pin::new(&mut self.inner).poll_frame(cx) { + Poll::Ready(Some(Ok(frame))) => { + if let Some(data) = frame.data_ref() + && let Some(capture) = &self.har_capture + { + capture.push(data); + } + Poll::Ready(Some(Ok(frame))) + } + other => other, + } } fn is_end_stream(&self) -> bool { diff --git a/src/http/transport/client.rs b/src/http/transport/client.rs index b880496..4b769a1 100644 --- a/src/http/transport/client.rs +++ b/src/http/transport/client.rs @@ -71,6 +71,7 @@ pub(super) struct ClientConfig { pub(super) http3_cache: Option>, pub(super) learn_alt_svc: bool, pub(super) ech_hard_fail: bool, + pub(super) har: Option, } pub(crate) struct ClientBuilder { @@ -107,6 +108,7 @@ impl Client { http3_cache: None, learn_alt_svc: false, ech_hard_fail: false, + har: None, }, } } @@ -181,6 +183,12 @@ impl Client { self.apply_session_cookies(&url, &mut headers); self.apply_proxy_authorization(&url, &mut headers)?; apply_host_header(&url, &mut headers)?; + let body = if let Some(recorder) = &self.config.har { + let capture = recorder.observe_request(method.clone(), url.clone(), headers.clone()); + body.map(|body| body.with_har_capture(capture)) + } else { + body + }; let response = match version { None if (self.config.auto_http3.is_some() || self.config.auto_http3_discovery @@ -541,6 +549,11 @@ impl ClientBuilder { self } + pub(crate) fn har_recorder(mut self, recorder: crate::har::Recorder) -> Self { + self.config.har = Some(recorder); + self + } + pub(crate) fn ech_hard_fail(mut self, hard_fail: bool) -> Self { self.config.ech_hard_fail = hard_fail; self diff --git a/src/lib.rs b/src/lib.rs index 1869d3d..0dac382 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod fileutil; pub(crate) mod flag_registry; pub mod format; pub mod grpc; +pub(crate) mod har; pub mod http; pub mod image; pub(crate) mod inspection; diff --git a/src/output/mod.rs b/src/output/mod.rs index 0223a79..6b222d0 100644 --- a/src/output/mod.rs +++ b/src/output/mod.rs @@ -1,6 +1,7 @@ use std::fs::{File, OpenOptions}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use http::header::{CONTENT_DISPOSITION, HeaderMap}; @@ -34,6 +35,8 @@ pub enum OutputError { path: String, source: std::io::Error, }, + #[error("output file '{0}' changed while the request was running; refusing to overwrite it")] + TargetChanged(String), #[error(transparent)] Io(#[from] std::io::Error), } @@ -151,10 +154,161 @@ pub fn resolve_output_path( Err(OutputError::UnableToInferFileName) } +/// Return whether two output spellings identify the same destination. +/// +/// Existing targets are canonicalized so symlink aliases compare equal. For a +/// new target, canonicalizing its parent still resolves directory symlinks; +/// lexical normalization is the fallback when the parent does not yet exist. +pub(crate) fn destinations_conflict(first: &str, second: &str) -> bool { + let first = destination_identity(first); + let second = destination_identity(second); + if first == second { + return true; + } + let (Some(first_parent), Some(second_parent)) = (first.parent(), second.parent()) else { + return false; + }; + if first_parent != second_parent { + return false; + } + let (Some(first_name), Some(second_name)) = (first.file_name(), second.file_name()) else { + return false; + }; + if first_name.to_string_lossy().to_lowercase() != second_name.to_string_lossy().to_lowercase() { + return false; + } + filesystem_is_case_insensitive(first_parent) +} + +static CASE_PROBE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +fn filesystem_is_case_insensitive(parent: &Path) -> bool { + let sequence = CASE_PROBE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let exact_name = format!(".fetch-case-probe-{}-{sequence}-Aa", std::process::id()); + let folded_name = exact_name.to_ascii_lowercase(); + let exact_path = parent.join(&exact_name); + let folded_path = parent.join(folded_name); + let Ok(file) = OpenOptions::new() + .write(true) + .create_new(true) + .open(&exact_path) + else { + return cfg!(windows); + }; + drop(file); + let insensitive = std::fs::metadata(&folded_path).is_ok(); + let _ = std::fs::remove_file(exact_path); + insensitive +} + +fn destination_identity(path: &str) -> PathBuf { + let absolute = absolute_path(Path::new(path)).unwrap_or_else(|_| PathBuf::from(path)); + if let Ok(canonical) = std::fs::canonicalize(&absolute) { + return canonical; + } + let Some(parent) = absolute.parent() else { + return normalize_lexically(&absolute); + }; + let Some(name) = absolute.file_name() else { + return normalize_lexically(&absolute); + }; + std::fs::canonicalize(parent) + .map(|parent| parent.join(name)) + .unwrap_or_else(|_| normalize_lexically(&absolute)) +} + +fn normalize_lexically(path: &Path) -> PathBuf { + use std::path::Component; + + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + Component::Prefix(_) | Component::RootDir | Component::Normal(_) => { + normalized.push(component.as_os_str()); + } + } + } + normalized +} + pub async fn write_output(path: &str, bytes: &[u8], clobber: bool) -> Result<(), OutputError> { write_output_with_progress(path, bytes, clobber, WriteProgress::disabled()).await } +/// An atomic output reserved before long-running work begins. +pub(crate) struct PreparedOutput { + download: DownloadTemp, + file: File, + target_before: Option, +} + +impl PreparedOutput { + pub(crate) fn create(path: &str, clobber: bool) -> Result { + let (download, file) = DownloadTemp::create(path, clobber)?; + let target_before = target_snapshot(&download.target_path)?; + Ok(Self { + download, + file, + target_before, + }) + } + + pub(crate) fn commit(mut self, bytes: &[u8]) -> Result<(), OutputError> { + self.file.write_all(bytes)?; + self.file.flush()?; + if target_snapshot(&self.download.target_path)? != self.target_before { + return Err(OutputError::TargetChanged(self.download.requested_path)); + } + let bytes_written = i64::try_from(bytes.len()).unwrap_or(i64::MAX); + install_download_temp( + self.download, + self.file, + WriteOutcome { + bytes_written, + summary: None, + }, + )?; + Ok(()) + } +} + +#[derive(Debug, Eq, PartialEq)] +struct TargetSnapshot { + len: u64, + modified: Option, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, +} + +fn target_snapshot(path: &Path) -> Result, OutputError> { + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + + Ok(Some(TargetSnapshot { + len: metadata.len(), + modified: metadata.modified().ok(), + #[cfg(unix)] + device: metadata.dev(), + #[cfg(unix)] + inode: metadata.ino(), + })) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(OutputError::FileCheck { + path: path.to_string_lossy().into_owned(), + source: err, + }), + } +} + pub async fn write_output_with_progress( path: &str, bytes: &[u8], @@ -791,6 +945,18 @@ mod tests { use std::time::Duration; use tokio::io::ReadBuf; + #[test] + fn prepared_output_refuses_target_created_after_reservation() { + let dir = tempfile::TempDir::new().unwrap(); + let target = dir.path().join("sidecar.har"); + let prepared = PreparedOutput::create(target.to_str().unwrap(), true).unwrap(); + std::fs::write(&target, "response").unwrap(); + + let error = prepared.commit(b"har").unwrap_err(); + assert!(matches!(error, OutputError::TargetChanged(_))); + assert_eq!(std::fs::read_to_string(target).unwrap(), "response"); + } + #[derive(Clone)] struct SharedBuffer(Arc>>); diff --git a/src/update/client.rs b/src/update/client.rs index b73d8ed..7b582ad 100644 --- a/src/update/client.rs +++ b/src/update/client.rs @@ -233,6 +233,7 @@ impl UpdateClient { request_start: Instant::now(), session: None, connect_timing: None, + har: None, }; client::build_client_for_url(cli, url, &context).await } diff --git a/tests/har.rs b/tests/har.rs new file mode 100644 index 0000000..7c3665c --- /dev/null +++ b/tests/har.rs @@ -0,0 +1,257 @@ +mod support; + +use std::fs; +use std::net::Ipv4Addr; + +use serde_json::Value; +use support::common::{FetchOpts, assert_exit, host_port, run_fetch, run_fetch_opts}; +use support::dns::start_udp_dns_server; +use support::http::{TestResponse, TestServer}; +use tempfile::TempDir; + +#[test] +fn writes_final_exchange_har_without_changing_response_output() { + let server = TestServer::start(|request| { + assert_eq!(request.body, b"hello"); + TestResponse::status(201, "Created", "world") + .header("Content-Type", "text/plain; charset=utf-8") + .header("Set-Cookie", "a=1") + .header("Set-Cookie", "b=2") + }); + let dir = TempDir::new().unwrap(); + let path = dir.path().join("request.har"); + let res = run_fetch(&[ + &format!("{}?first=one&first=two", server.url), + "--data", + "hello", + "--header", + "X-Test: value", + "--har", + path.to_str().unwrap(), + ]); + assert_exit(&res, 0); + assert_eq!(res.stdout, "world"); + + let har: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!(har["log"]["version"], "1.2"); + assert_eq!(har["log"]["creator"]["name"], "fetch"); + let entry = &har["log"]["entries"][0]; + assert_eq!(entry["request"]["method"], "POST"); + assert_eq!(entry["request"]["postData"]["text"], "hello"); + assert_eq!(entry["request"]["queryString"][0]["name"], "first"); + assert_eq!(entry["request"]["queryString"][1]["value"], "two"); + assert_eq!(entry["response"]["status"], 201); + assert_eq!(entry["response"]["content"]["text"], "world"); + assert_eq!(entry["response"]["content"]["size"], 5); + assert_eq!(entry["response"]["httpVersion"], "HTTP/1.1"); + assert!(entry["serverIPAddress"].as_str().is_some()); + assert!(entry["timings"]["receive"].as_f64().unwrap() >= 0.0); + let set_cookies = entry["response"]["headers"] + .as_array() + .unwrap() + .iter() + .filter(|header| header["name"].as_str() == Some("set-cookie")) + .count(); + assert_eq!(set_cookies, 2); +} + +#[test] +fn encodes_binary_response_and_honors_clobber() { + let server = TestServer::start(|_| TestResponse::ok(vec![0, 159, 146, 150])); + let dir = TempDir::new().unwrap(); + let path = dir.path().join("binary.har"); + fs::write(&path, "existing").unwrap(); + + let res = run_fetch(&[&server.url, "--har", path.to_str().unwrap()]); + assert_exit(&res, 1); + assert!(res.stderr.contains("already exists")); + + let res = run_fetch(&[ + &server.url, + "--output", + "-", + "--har", + path.to_str().unwrap(), + "--clobber", + ]); + assert_exit(&res, 0); + let har: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let content = &har["log"]["entries"][0]["response"]["content"]; + assert_eq!(content["encoding"], "base64"); + assert_eq!(content["text"], "AJ+Slg=="); +} + +#[test] +fn rejects_unsupported_har_destinations_and_modes() { + for args in [ + vec!["--har", "-", "http://example.com"], + vec!["--har", "same", "--output", "same", "http://example.com"], + vec!["--har", "out.har", "--dry-run", "http://example.com"], + vec!["--har", "out.har", "--inspect-dns", "example.com"], + vec!["--har", "out.har", "ws://example.com"], + ] { + let res = run_fetch(&args); + assert_exit(&res, 1); + } +} + +#[test] +fn rejects_equivalent_response_and_har_paths_before_clobbering() { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let requests = Arc::new(AtomicUsize::new(0)); + let requests_for_server = Arc::clone(&requests); + let server = TestServer::start(move |_| { + requests_for_server.fetch_add(1, Ordering::SeqCst); + TestResponse::ok("response") + }); + let dir = TempDir::new().unwrap(); + + for (output, har) in [ + ("result", "./result".to_string()), + ( + "result", + dir.path().join("result").to_string_lossy().into_owned(), + ), + ] { + let target = dir.path().join("result"); + fs::write(&target, "original").unwrap(); + let res = run_fetch_opts( + FetchOpts { + cwd: Some(dir.path().to_path_buf()), + ..FetchOpts::default() + }, + &[&server.url, "--output", output, "--har", &har, "--clobber"], + ); + assert_exit(&res, 1); + assert!(res.stderr.contains("cannot use the same path")); + assert_eq!(fs::read_to_string(&target).unwrap(), "original"); + assert_eq!(requests.load(Ordering::SeqCst), 0); + } + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + + let real = dir.path().join("real"); + fs::create_dir(&real).unwrap(); + symlink(&real, dir.path().join("alias")).unwrap(); + let target = real.join("response"); + let alias = dir.path().join("alias/response"); + fs::write(&target, "original").unwrap(); + let res = run_fetch(&[ + &server.url, + "--output", + target.to_str().unwrap(), + "--har", + alias.to_str().unwrap(), + "--clobber", + ]); + assert_exit(&res, 1); + assert_eq!(fs::read_to_string(target).unwrap(), "original"); + assert_eq!(requests.load(Ordering::SeqCst), 0); + } + + if filesystem_is_case_insensitive(dir.path()) { + let lower = "case-result"; + let upper = "CASE-RESULT"; + let res = run_fetch_opts( + FetchOpts { + cwd: Some(dir.path().to_path_buf()), + ..FetchOpts::default() + }, + &[&server.url, "--output", lower, "--har", upper, "--clobber"], + ); + assert_exit(&res, 1); + assert_eq!(requests.load(Ordering::SeqCst), 0); + assert!(!dir.path().join(lower).exists()); + } +} + +#[test] +fn har_enables_runtime_dns_timing() { + let server = TestServer::start(|_| TestResponse::ok("dns")); + let port = host_port(&server.url).split(':').nth(1).unwrap(); + let dns = start_udp_dns_server("har-dns.test.", Ipv4Addr::LOCALHOST); + let dir = TempDir::new().unwrap(); + let path = dir.path().join("dns.har"); + let res = run_fetch(&[ + "--dns-server", + &dns, + "--har", + path.to_str().unwrap(), + &format!("http://har-dns.test:{port}"), + ]); + assert_exit(&res, 0); + let har: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert!(har["log"]["entries"][0]["timings"]["dns"].as_f64().unwrap() >= 0.0); +} + +#[test] +fn digest_har_timing_describes_authenticated_exchange() { + use std::sync::{Arc, Mutex}; + use std::thread; + use std::time::{Duration, SystemTime}; + + let challenge_finished = Arc::new(Mutex::new(None)); + let marker = Arc::clone(&challenge_finished); + let server = TestServer::start(move |request| { + if request.header("authorization").is_empty() { + thread::sleep(Duration::from_millis(250)); + *marker.lock().unwrap() = Some(rfc3339(SystemTime::now())); + return TestResponse::status(401, "Unauthorized", "").header( + "WWW-Authenticate", + r#"Digest realm="test", nonce="abc123", qop="auth", algorithm="MD5""#, + ); + } + TestResponse::ok("authenticated") + }); + let dir = TempDir::new().unwrap(); + let path = dir.path().join("digest.har"); + let res = run_fetch(&[ + &server.url, + "--digest", + "user:pass", + "--har", + path.to_str().unwrap(), + ]); + assert_exit(&res, 0); + + let har: Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + let entry = &har["log"]["entries"][0]; + let marker = challenge_finished.lock().unwrap().clone().unwrap(); + assert!(entry["startedDateTime"].as_str().unwrap() >= marker.as_str()); + assert!(entry["timings"]["wait"].as_f64().unwrap() < 200.0); + assert!(entry["time"].as_f64().unwrap() < 200.0); + assert!( + entry["request"]["headers"] + .as_array() + .unwrap() + .iter() + .any(|header| header["name"] == "authorization") + ); +} + +fn rfc3339(value: std::time::SystemTime) -> String { + let dt = time::OffsetDateTime::from(value); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z", + dt.year(), + u8::from(dt.month()), + dt.day(), + dt.hour(), + dt.minute(), + dt.second(), + dt.millisecond() + ) +} + +fn filesystem_is_case_insensitive(dir: &std::path::Path) -> bool { + let lower = dir.join("case-probe"); + let upper = dir.join("CASE-PROBE"); + fs::write(&lower, "probe").unwrap(); + let insensitive = upper.exists(); + fs::remove_file(lower).unwrap(); + insensitive +}