From c0fbeaf2d768d465c472b8db6d934c176fb31247 Mon Sep 17 00:00:00 2001 From: Nathan Herald Date: Wed, 29 Apr 2026 23:27:08 +0200 Subject: [PATCH 01/11] Add frontmatter, --main, --head, --skip-frontmatter flags CLI now emits a fake-YAML frontmatter block by default for every fetch (status, final-url, title, description, fetched-at). The big motivator: many sites 302 to a /?err=404 home page and the previous output gave no signal that the original URL was dead. With final-url surfaced, those cases are now obvious. New flags: - --main: strip nav/header/footer/cookie/sidebar/related elements via injected JS before HTML extraction. Useful for cutting noise on noisy product/listing pages. - --head: print only the frontmatter block, skip the body (fast URL validation in batch). - --skip-frontmatter: backward-compat escape hatch for scripts that just want the body markdown. Implementation: - WebPageFetcher.fetch() returns a FetchedPage struct with html, statusCode, finalURL. Backwards-compatible fetchHTML() wrapper kept. - WKNavigationDelegate captures HTTP status via decidePolicyFor navigationResponse:; final URL read from webView.url at didFinish. - HTMLToMarkdown.extractMetadata() returns PageMetadata (title and description) using SwiftSoup; falls back from to og:title and from meta[name=description] to og:description. - yamlEscape() in CLI quotes values containing colons, quotes, backslashes, comments, or leading/trailing whitespace; flattens newlines to spaces; escapes embedded quotes and backslashes. Tests: 8 new PageMetadataTests covering title/description extraction, OG fallbacks, whitespace trimming, empty-title fallback. All 54 tests pass. --- Sources/WebToMarkdown/HTMLToMarkdown.swift | 45 +++++ Sources/WebToMarkdown/WebPageFetcher.swift | 181 ++++++++++++------ .../WebToMarkdownCommand.swift | 92 ++++++++- .../HTMLToMarkdownTests.swift | 88 +++++++++ 4 files changed, 339 insertions(+), 67 deletions(-) diff --git a/Sources/WebToMarkdown/HTMLToMarkdown.swift b/Sources/WebToMarkdown/HTMLToMarkdown.swift index 7b09115..0c95c96 100644 --- a/Sources/WebToMarkdown/HTMLToMarkdown.swift +++ b/Sources/WebToMarkdown/HTMLToMarkdown.swift @@ -1,6 +1,16 @@ import Foundation import SwiftSoup +public struct PageMetadata: Sendable { + public let title: String? + public let description: String? + + public init(title: String?, description: String?) { + self.title = title + self.description = description + } +} + public enum HTMLToMarkdown { public enum Error: Swift.Error { case parsingFailed(String) @@ -19,6 +29,41 @@ public enum HTMLToMarkdown { return collapseExcessiveNewlines(markdown) } + /// Extract the page `<title>` and meta `description` (falling back to + /// Open Graph variants when standard tags are missing). + public static func extractMetadata(_ html: String) throws -> PageMetadata { + let document = try SwiftSoup.parse(html) + + let title: String? = try { + if let el = try document.select("title").first() { + let text = try el.text().trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + if let el = try document.select("meta[property=og:title]").first() { + let text = try el.attr("content") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + return nil + }() + + let description: String? = try { + if let el = try document.select("meta[name=description]").first() { + let text = try el.attr("content") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + if let el = try document.select("meta[property=og:description]").first() { + let text = try el.attr("content") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + return nil + }() + + return PageMetadata(title: title, description: description) + } + private static func collapseExcessiveNewlines(_ text: String) -> String { var result = text while result.contains("\n\n\n") { diff --git a/Sources/WebToMarkdown/WebPageFetcher.swift b/Sources/WebToMarkdown/WebPageFetcher.swift index d18b40c..43268a0 100644 --- a/Sources/WebToMarkdown/WebPageFetcher.swift +++ b/Sources/WebToMarkdown/WebPageFetcher.swift @@ -5,6 +5,18 @@ import WebKit private let log = OSLog(subsystem: "com.shareup.web-to-markdown", category: "web-page-fetcher") +public struct FetchedPage: Sendable { + public let html: String + public let statusCode: Int? + public let finalURL: URL + + public init(html: String, statusCode: Int?, finalURL: URL) { + self.html = html + self.statusCode = statusCode + self.finalURL = finalURL + } +} + public enum WebPageFetcher { public enum Error: Swift.Error { case loadFailed(String) @@ -12,10 +24,17 @@ public enum WebPageFetcher { case noHTML } - public static func fetchHTML( + /// Fetch a web page and return HTML plus response metadata. + /// + /// - Parameter extractMainOnly: when `true`, common page chrome (nav, + /// header, footer, cookies, sidebars, related/recommended sections) is + /// stripped from the DOM before the HTML is returned. Cuts noise on + /// listing/article pages dramatically. + public static func fetch( from url: URL, - timeout: TimeInterval = 30 - ) async throws -> String { + timeout: TimeInterval = 30, + extractMainOnly: Bool = false + ) async throws -> FetchedPage { os_log( .info, log: log, @@ -55,7 +74,8 @@ public enum WebPageFetcher { let delegate = NavigationDelegate( webView: webView, state: state, - timeout: timeout + timeout: timeout, + extractMainOnly: extractMainOnly ) state.access { state in @@ -78,32 +98,6 @@ public enum WebPageFetcher { } }, onCancel: { - // NOTE: Even though `isolation: MainActor.shared` is specified - // below, neither `operation` nor `onCancel` are called - // on `MainActor` if they weren't already running on - // `MainActor`. - // - // I'm no Swift Foundation engineer, but it doesn't seem - // like `isolation` is used anywhere in the current version - // of `withTaskCancellationHandler()`: - // - // ``` - // public func withTaskCancellationHandler<T>( - // operation: () async throws -> T, - // onCancel handler: @Sendable () -> Void, - // isolation: isolated (any Actor)? = #isolation - // ) async rethrows -> T { - // // unconditionally add the cancellation record to the task. - // // if the task was already cancelled, it will be executed right away. - // let record = unsafe _taskAddCancellationHandler(handler: handler) - // defer { unsafe _taskRemoveCancellationHandler(record: record) } - // - // - // return try await operation() - // } - // ``` - // - // https://github.com/swiftlang/swift/blob/5d480ef063859a0f459f4149df536db4fb330a50/stdlib/public/Concurrency/TaskCancellation.swift#L73-L84 Task { @MainActor in state.access { $0.cancel() } } @@ -111,21 +105,62 @@ public enum WebPageFetcher { isolation: MainActor.shared ) } + + /// Backwards-compatible wrapper returning just HTML. + public static func fetchHTML( + from url: URL, + timeout: TimeInterval = 30 + ) async throws -> String { + try await fetch(from: url, timeout: timeout).html + } } +/// JS that removes common chrome from a loaded page (nav, header, footer, +/// cookie banners, recommendations, sidebars, etc.) and then returns the +/// stripped outerHTML. Used by `extractMainOnly`. +private let stripChromeAndExtractJS: String = """ +(function() { + var selectors = [ + 'nav', 'header', 'footer', 'aside', + '[role="banner"]', '[role="contentinfo"]', '[role="navigation"]', + '[role="complementary"]', + '[id*="cookie" i]', '[class*="cookie" i]', + '[id*="consent" i]', '[class*="consent" i]', + '[id*="newsletter" i]', '[class*="newsletter" i]', + '[id*="subscribe" i]', '[class*="subscribe" i]', + '[class*="related" i]', '[class*="recommend" i]', + '[class*="sidebar" i]', '[id*="sidebar" i]', + '[id*="banner" i]', '[class*="promo" i]', + 'noscript', 'script[src]', 'style' + ]; + selectors.forEach(function(sel) { + try { + document.querySelectorAll(sel).forEach(function(el) { el.remove(); }); + } catch (e) {} + }); + return document.documentElement.outerHTML; +})() +""" + +private let extractJS = "document.documentElement.outerHTML" + @MainActor private final class NavigationDelegate: NSObject, WKNavigationDelegate { let webView: WKWebView let state: Locked<State> + let extractMainOnly: Bool var timeoutTask: Task<Void, Never>? + var capturedStatusCode: Int? init( webView: WKWebView, state: Locked<State>, - timeout: TimeInterval + timeout: TimeInterval, + extractMainOnly: Bool ) { self.webView = webView self.state = state + self.extractMainOnly = extractMainOnly super.init() timeoutTask = Task { @MainActor in @@ -143,6 +178,22 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { } } + func webView( + _: WKWebView, + decidePolicyFor navigationResponse: WKNavigationResponse + ) async -> WKNavigationResponsePolicy { + if let httpResponse = navigationResponse.response as? HTTPURLResponse { + capturedStatusCode = httpResponse.statusCode + os_log( + .info, + log: log, + "🔧TOOLCALL🔧 WebPageFetcher: Response status: %d", + httpResponse.statusCode + ) + } + return .allow + } + func webView(_ webView: WKWebView, didFinish _: WKNavigation!) { guard state.access({ $0.shouldLoadJavaScript }) else { return @@ -150,42 +201,48 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { os_log(.info, log: log, "🔧TOOLCALL🔧 WebPageFetcher: Page loaded, extracting HTML") - webView - .evaluateJavaScript("document.documentElement.outerHTML") { [weak self] result, error in - guard let self else { return } + let js = extractMainOnly ? stripChromeAndExtractJS : extractJS + let finalURL = webView.url + let status = capturedStatusCode + + webView.evaluateJavaScript(js) { [weak self] result, error in + guard let self else { return } - timeoutTask?.cancel() + timeoutTask?.cancel() - if let error, state.access({ $0.fail(with: error) }) { + if let error, state.access({ $0.fail(with: error) }) { + os_log( + .error, + log: log, + "🔧TOOLCALL🔧 WebPageFetcher: JavaScript error: %{public}s", + error.localizedDescription + ) + return + } + + guard let html = result as? String else { + if state.access({ $0.fail(with: WebPageFetcher.Error.noHTML) }) { os_log( .error, log: log, - "🔧TOOLCALL🔧 WebPageFetcher: JavaScript error: %{public}s", - error.localizedDescription + "🔧TOOLCALL🔧 WebPageFetcher: No HTML returned" ) - return } + return + } - guard let html = result as? String, - state.access({ $0.finish(with: html) }) - else { - if state.access({ $0.fail(with: WebPageFetcher.Error.noHTML) }) { - os_log( - .error, - log: log, - "🔧TOOLCALL🔧 WebPageFetcher: No HTML returned" - ) - } - return - } + let resolved = finalURL ?? webView.url ?? URL(string: "about:blank")! + let page = FetchedPage(html: html, statusCode: status, finalURL: resolved) - os_log( - .info, - log: log, - "🔧TOOLCALL🔧 WebPageFetcher: Extracted HTML of length: %d", - html.count - ) - } + _ = state.access { $0.finish(with: page) } + + os_log( + .info, + log: log, + "🔧TOOLCALL🔧 WebPageFetcher: Extracted HTML of length: %d", + html.count + ) + } } func webView(_: WKWebView, didFail _: WKNavigation!, withError error: Swift.Error) { @@ -240,7 +297,7 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { } } -private typealias FetchContinuation = CheckedContinuation<String, Swift.Error> +private typealias FetchContinuation = CheckedContinuation<FetchedPage, Swift.Error> private enum State: Sendable { case initial case inProgress(WKWebView, NavigationDelegate, FetchContinuation) @@ -313,7 +370,7 @@ private enum State: Sendable { } } - mutating func finish(with html: String) -> Bool { + mutating func finish(with page: FetchedPage) -> Bool { MainActor.assertIsolated() switch self { case .initial: @@ -323,7 +380,7 @@ private enum State: Sendable { case let .inProgress(_, _, continuation): self = .terminal - continuation.resume(returning: html) + continuation.resume(returning: page) return true case .terminal: @@ -332,7 +389,7 @@ private enum State: Sendable { case let .waitingForWebView(continuation): assertionFailure() self = .terminal - continuation.resume(returning: html) + continuation.resume(returning: page) return true } } diff --git a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift index 50b1603..b512d9f 100644 --- a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift +++ b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift @@ -18,23 +18,105 @@ struct WebToMarkdownCommand: AsyncParsableCommand { @Flag(name: .shortAndLong, help: "Output verbose logging") var verbose: Bool = false + @Flag( + name: .long, + help: "Strip page chrome (nav, header, footer, cookies, sidebars) before conversion" + ) + var main: Bool = false + + @Flag( + name: .long, + help: "Output only the frontmatter block (status, final-url, title, description, fetched-at) — skip the body" + ) + var head: Bool = false + + @Flag( + name: .long, + help: "Suppress the frontmatter block (output the body markdown only)" + ) + var skipFrontmatter: Bool = false + mutating func run() async throws { - guard let url = URL(string: url) else { + guard let parsedURL = URL(string: url) else { throw ValidationError("Invalid URL: \(url)") } if verbose { - fputs("Fetching \(url.absoluteString)...\n", stderr) + fputs("Fetching \(parsedURL.absoluteString)...\n", stderr) } - let html = try await WebPageFetcher.fetchHTML(from: url, timeout: timeout) + let page = try await WebPageFetcher.fetch( + from: parsedURL, + timeout: timeout, + extractMainOnly: main + ) + + if !skipFrontmatter { + let metadata = (try? HTMLToMarkdown.extractMetadata(page.html)) + ?? PageMetadata(title: nil, description: nil) + print(formatFrontmatter(page: page, metadata: metadata)) + if !head { print("") } + } + + if head { return } if verbose { fputs("Converting to markdown...\n", stderr) } - let markdown = try HTMLToMarkdown.convert(html, baseURL: url) - + let markdown = try HTMLToMarkdown.convert(page.html, baseURL: parsedURL) print(markdown) } + + private func formatFrontmatter(page: FetchedPage, metadata: PageMetadata) -> String { + var lines = ["---"] + if let status = page.statusCode { + lines.append("status: \(status)") + } + lines.append("final-url: \(yamlEscape(page.finalURL.absoluteString))") + if let title = metadata.title, !title.isEmpty { + lines.append("title: \(yamlEscape(title))") + } + if let description = metadata.description, !description.isEmpty { + lines.append("description: \(yamlEscape(truncate(description, max: 200)))") + } + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + lines.append("fetched-at: \(formatter.string(from: Date()))") + lines.append("---") + return lines.joined(separator: "\n") + } + + /// Escape a string for use as a fake-YAML frontmatter scalar value. + /// Quotes the value when it contains characters that would confuse a + /// downstream YAML reader (`:`, `"`, `\\`, leading/trailing whitespace). + /// Newlines are flattened to spaces — frontmatter is single-line per key. + private func yamlEscape(_ value: String) -> String { + let flattened = value + .replacingOccurrences(of: "\r\n", with: " ") + .replacingOccurrences(of: "\n", with: " ") + .replacingOccurrences(of: "\r", with: " ") + + let needsQuoting = flattened.contains(":") + || flattened.contains("\"") + || flattened.contains("\\") + || flattened.contains("#") + || flattened.first == " " + || flattened.last == " " + || flattened.isEmpty + + if needsQuoting { + let escaped = flattened + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + return "\"\(escaped)\"" + } + return flattened + } + + private func truncate(_ s: String, max: Int) -> String { + if s.count <= max { return s } + let end = s.index(s.startIndex, offsetBy: max) + return String(s[s.startIndex ..< end]) + "…" + } } diff --git a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift index f4a231c..dbdbeca 100644 --- a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift +++ b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift @@ -2,6 +2,94 @@ import Foundation import Testing @testable import WebToMarkdown +@Suite +struct PageMetadataTests { + @Test + func extractsTitleFromTitleTag() throws { + let html = "<html><head><title>My Pagex" + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "My Page") + } + + @Test + func fallsBackToOgTitleWhenNoTitleTag() throws { + let html = """ + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "OG Title Here") + } + + @Test + func prefersTitleTagOverOgTitle() throws { + let html = """ + + Real Title + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "Real Title") + } + + @Test + func extractsMetaDescription() throws { + let html = """ + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.description == "A nice description.") + } + + @Test + func fallsBackToOgDescription() throws { + let html = """ + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.description == "OG description fallback.") + } + + @Test + func returnsNilWhenMetadataMissing() throws { + let html = "just body" + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == nil) + #expect(meta.description == nil) + } + + @Test + func trimsWhitespaceInExtractedValues() throws { + let html = """ + + Spaced Title + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "Spaced Title") + #expect(meta.description == "spaced description") + } + + @Test + func ignoresEmptyTitleTag() throws { + let html = """ + + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "OG Backup") + } +} + @Suite struct HTMLToMarkdownTests { @Test From b35fcadcde317cd0fad0bd2e638929b5004a3236 Mon Sep 17 00:00:00 2001 From: Nathan Herald Date: Wed, 29 Apr 2026 23:29:47 +0200 Subject: [PATCH 02/11] install.sh: prefer Homebrew prefix, dedupe candidates, fail clearly Pick install destination by trying these in order, first writable wins: 1. $INSTALL_DIR (env override) 2. $(brew --prefix)/bin 3. /opt/homebrew/bin (Apple Silicon Homebrew default) 4. /usr/local/bin (Intel Homebrew / classic default; usually root-owned) Fixes the previous script always copying to /usr/local/bin/ which on Apple Silicon required sudo and didn't match where the existing binary lived. With Homebrew installed on Apple Silicon, the script now lands the binary at /opt/homebrew/bin/web-to-markdown and prints the chosen path. Fails with a clear error and the candidate list when nothing is writable. --- bin/install.sh | 60 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/bin/install.sh b/bin/install.sh index 262b44c..002bc34 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -9,7 +9,18 @@ if [[ "${TRACE-0}" == "1" ]]; then fi if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then - echo 'Usage: ./install.sh' + cat <<'EOF' +Usage: ./install.sh + +Builds web-to-markdown in release mode and copies the binary into a +directory on PATH. Picks the first writable destination from: + 1. $INSTALL_DIR (override) + 2. $(brew --prefix)/bin (if Homebrew is installed) + 3. /opt/homebrew/bin (Apple Silicon Homebrew default) + 4. /usr/local/bin (Intel Homebrew / classic default; usually requires sudo) + +If none are writable, runs with sudo or fails with a clear error. +EOF exit fi @@ -18,8 +29,51 @@ pushd "$DIR/.." &>/dev/null swift build -c release -cp .build/release/web-to-markdown /usr/local/bin/ +# Build the candidate list in priority order. +candidates=() + +if [[ -n "${INSTALL_DIR-}" ]]; then + candidates+=("$INSTALL_DIR") +fi + +if command -v brew >/dev/null 2>&1; then + brew_prefix="$(brew --prefix 2>/dev/null || true)" + if [[ -n "$brew_prefix" ]]; then + candidates+=("$brew_prefix/bin") + fi +fi + +candidates+=("/opt/homebrew/bin" "/usr/local/bin") + +# Deduplicate while preserving order. +seen="" +unique_candidates=() +for c in "${candidates[@]}"; do + if [[ ":$seen:" != *":$c:"* ]]; then + unique_candidates+=("$c") + seen="$seen:$c" + fi +done + +# Pick the first existing & writable candidate. +dest="" +for candidate in "${unique_candidates[@]}"; do + if [[ -d "$candidate" ]] && [[ -w "$candidate" ]]; then + dest="$candidate" + break + fi +done + +if [[ -z "$dest" ]]; then + echo "Error: no writable directory found among:" >&2 + printf ' - %s\n' "${unique_candidates[@]}" >&2 + echo "" >&2 + echo "Re-run with sudo, or set INSTALL_DIR= to a writable directory." >&2 + exit 1 +fi + +cp .build/release/web-to-markdown "$dest/" -echo "$(ls /usr/local/bin/web-to-markdown)" +echo "Installed: $dest/web-to-markdown" popd &>/dev/null From c7fa109681867751b6ef4a7532a67ffab20e653f Mon Sep 17 00:00:00 2001 From: Nathan Herald Date: Wed, 29 Apr 2026 23:44:00 +0200 Subject: [PATCH 03/11] Add --wait, --wait-for, --wait-for-text flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three new CLI flags for bridging the gap between page-load and SPA hydration. All bounded by the existing --timeout. --wait Fixed delay after the page loads. Decimal allowed (e.g. --wait 0.5). Default 0. --wait-for Wait until at least one element matches the CSS selector. Event-driven via MutationObserver, no polling. --wait-for-text Wait until the body's visible text contains this substring. Event-driven via MutationObserver. The waits chain in order: load → fixed wait → wait-for selector → wait-for-text → extract. Combine freely. Implementation: - Switched extraction from evaluateJavaScript to callAsyncJavaScript so the wait sequence and the chrome-stripping (--main) live in one async JS function with arguments passed safely (no string-escape gymnastics). As a small bonus, the implicit microtask hop seems to give SPAs like YouTube enough time to set document.title before extraction even without an explicit --wait. - The JS uses MutationObserver to detect selector/text appearance — resolves on the first matching mutation, no polling loop. - Outer Swift-side timeout (--timeout, default 30s) bounds all waits; if the JS never resolves, the timeoutTask fails the fetch. Tests: 4 new WebPageFetcherTests cover status/final-url capture, wait-for matching immediately, wait-for timing out cleanly, and fixed wait actually delaying extraction. All 58 tests pass. --- Sources/WebToMarkdown/WebPageFetcher.swift | 164 +++++++++++++----- .../WebToMarkdownCommand.swift | 23 ++- .../WebPageFetcherTests.swift | 48 +++++ 3 files changed, 190 insertions(+), 45 deletions(-) diff --git a/Sources/WebToMarkdown/WebPageFetcher.swift b/Sources/WebToMarkdown/WebPageFetcher.swift index 43268a0..a0270e7 100644 --- a/Sources/WebToMarkdown/WebPageFetcher.swift +++ b/Sources/WebToMarkdown/WebPageFetcher.swift @@ -26,14 +26,27 @@ public enum WebPageFetcher { /// Fetch a web page and return HTML plus response metadata. /// - /// - Parameter extractMainOnly: when `true`, common page chrome (nav, - /// header, footer, cookies, sidebars, related/recommended sections) is - /// stripped from the DOM before the HTML is returned. Cuts noise on - /// listing/article pages dramatically. + /// - Parameters: + /// - extractMainOnly: when `true`, common page chrome (nav, header, + /// footer, cookies, sidebars, related/recommended sections) is + /// stripped from the DOM before the HTML is returned. + /// - waitSeconds: extra fixed delay (seconds) after the page finishes + /// loading, before HTML is extracted. Useful for SPAs that hydrate + /// async after `window.load`. + /// - waitForSelector: if non-nil, waits (event-driven via + /// `MutationObserver`) until at least one element matching the CSS + /// selector is present in the DOM, before extraction. + /// - waitForText: if non-nil, waits until the body's visible text + /// contains this substring before extraction. + /// + /// All wait operations are bounded by the overall `timeout`. public static func fetch( from url: URL, timeout: TimeInterval = 30, - extractMainOnly: Bool = false + extractMainOnly: Bool = false, + waitSeconds: TimeInterval = 0, + waitForSelector: String? = nil, + waitForText: String? = nil ) async throws -> FetchedPage { os_log( .info, @@ -75,7 +88,10 @@ public enum WebPageFetcher { webView: webView, state: state, timeout: timeout, - extractMainOnly: extractMainOnly + extractMainOnly: extractMainOnly, + waitSeconds: waitSeconds, + waitForSelector: waitForSelector, + waitForText: waitForText ) state.access { state in @@ -115,13 +131,53 @@ public enum WebPageFetcher { } } -/// JS that removes common chrome from a loaded page (nav, header, footer, -/// cookie banners, recommendations, sidebars, etc.) and then returns the -/// stripped outerHTML. Used by `extractMainOnly`. -private let stripChromeAndExtractJS: String = """ -(function() { +/// Async JS run via `callAsyncJavaScript` after the page reports `didFinish`. +/// Optionally waits a fixed number of seconds, then for a selector to appear, +/// then for body text to contain a substring (event-driven via +/// `MutationObserver` — no polling). Optionally strips common chrome from the +/// DOM. Always returns `document.documentElement.outerHTML`. +/// +/// All `waitFor*` operations are bounded by the outer Swift-side `timeout`, +/// which fails the fetch if the JS never resolves. +private let extractJS: String = #""" +if (typeof waitSeconds === "number" && waitSeconds > 0) { + await new Promise(function(r) { setTimeout(r, waitSeconds * 1000); }); +} + +if (typeof waitForSelector === "string" && waitForSelector.length > 0) { + await new Promise(function(resolve) { + function check() { + try { return document.querySelector(waitForSelector); } + catch (e) { return null; } + } + if (check()) { resolve(); return; } + var obs = new MutationObserver(function() { + if (check()) { obs.disconnect(); resolve(); } + }); + obs.observe(document.documentElement, { childList: true, subtree: true }); + }); +} + +if (typeof waitForText === "string" && waitForText.length > 0) { + await new Promise(function(resolve) { + function hasText() { + var body = document.body; + if (!body) return false; + var t = body.innerText || body.textContent || ""; + return t.indexOf(waitForText) !== -1; + } + if (hasText()) { resolve(); return; } + var target = document.body || document.documentElement; + var obs = new MutationObserver(function() { + if (hasText()) { obs.disconnect(); resolve(); } + }); + obs.observe(target, { childList: true, subtree: true, characterData: true }); + }); +} + +if (extractMainOnly) { var selectors = [ - 'nav', 'header', 'footer', 'aside', + "nav", "header", "footer", "aside", '[role="banner"]', '[role="contentinfo"]', '[role="navigation"]', '[role="complementary"]', '[id*="cookie" i]', '[class*="cookie" i]', @@ -131,24 +187,26 @@ private let stripChromeAndExtractJS: String = """ '[class*="related" i]', '[class*="recommend" i]', '[class*="sidebar" i]', '[id*="sidebar" i]', '[id*="banner" i]', '[class*="promo" i]', - 'noscript', 'script[src]', 'style' + "noscript", "script[src]", "style" ]; selectors.forEach(function(sel) { try { document.querySelectorAll(sel).forEach(function(el) { el.remove(); }); } catch (e) {} }); - return document.documentElement.outerHTML; -})() -""" +} -private let extractJS = "document.documentElement.outerHTML" +return document.documentElement.outerHTML; +"""# @MainActor private final class NavigationDelegate: NSObject, WKNavigationDelegate { let webView: WKWebView let state: Locked let extractMainOnly: Bool + let waitSeconds: TimeInterval + let waitForSelector: String? + let waitForText: String? var timeoutTask: Task? var capturedStatusCode: Int? @@ -156,11 +214,17 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { webView: WKWebView, state: Locked, timeout: TimeInterval, - extractMainOnly: Bool + extractMainOnly: Bool, + waitSeconds: TimeInterval, + waitForSelector: String?, + waitForText: String? ) { self.webView = webView self.state = state self.extractMainOnly = extractMainOnly + self.waitSeconds = waitSeconds + self.waitForSelector = waitForSelector + self.waitForText = waitForText super.init() timeoutTask = Task { @MainActor in @@ -201,47 +265,59 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { os_log(.info, log: log, "🔧TOOLCALL🔧 WebPageFetcher: Page loaded, extracting HTML") - let js = extractMainOnly ? stripChromeAndExtractJS : extractJS let finalURL = webView.url let status = capturedStatusCode - webView.evaluateJavaScript(js) { [weak self] result, error in + let arguments: [String: Any] = [ + "waitSeconds": waitSeconds, + "waitForSelector": waitForSelector ?? NSNull(), + "waitForText": waitForText ?? NSNull(), + "extractMainOnly": extractMainOnly, + ] + + Task { @MainActor [weak self] in guard let self else { return } + do { + let result = try await webView.callAsyncJavaScript( + extractJS, + arguments: arguments, + contentWorld: .page + ) - timeoutTask?.cancel() + timeoutTask?.cancel() + + guard let html = result as? String else { + if state.access({ $0.fail(with: WebPageFetcher.Error.noHTML) }) { + os_log( + .error, + log: log, + "🔧TOOLCALL🔧 WebPageFetcher: No HTML returned" + ) + } + return + } + + let resolved = finalURL ?? webView.url ?? URL(string: "about:blank")! + let page = FetchedPage(html: html, statusCode: status, finalURL: resolved) + + _ = state.access { $0.finish(with: page) } - if let error, state.access({ $0.fail(with: error) }) { os_log( - .error, + .info, log: log, - "🔧TOOLCALL🔧 WebPageFetcher: JavaScript error: %{public}s", - error.localizedDescription + "🔧TOOLCALL🔧 WebPageFetcher: Extracted HTML of length: %d", + html.count ) - return - } - - guard let html = result as? String else { - if state.access({ $0.fail(with: WebPageFetcher.Error.noHTML) }) { + } catch { + if state.access({ $0.fail(with: error) }) { os_log( .error, log: log, - "🔧TOOLCALL🔧 WebPageFetcher: No HTML returned" + "🔧TOOLCALL🔧 WebPageFetcher: JavaScript error: %{public}s", + error.localizedDescription ) } - return } - - let resolved = finalURL ?? webView.url ?? URL(string: "about:blank")! - let page = FetchedPage(html: html, statusCode: status, finalURL: resolved) - - _ = state.access { $0.finish(with: page) } - - os_log( - .info, - log: log, - "🔧TOOLCALL🔧 WebPageFetcher: Extracted HTML of length: %d", - html.count - ) } } diff --git a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift index b512d9f..16f8eba 100644 --- a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift +++ b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift @@ -36,6 +36,24 @@ struct WebToMarkdownCommand: AsyncParsableCommand { ) var skipFrontmatter: Bool = false + @Option( + name: .long, + help: "Wait this many seconds (decimal allowed) after the page loads, before extracting. Bounded by --timeout." + ) + var wait: Double = 0 + + @Option( + name: .long, + help: "Wait until at least one element matches this CSS selector before extracting. MutationObserver-driven, bounded by --timeout." + ) + var waitFor: String? + + @Option( + name: .long, + help: "Wait until the body's visible text contains this substring before extracting. MutationObserver-driven, bounded by --timeout." + ) + var waitForText: String? + mutating func run() async throws { guard let parsedURL = URL(string: url) else { throw ValidationError("Invalid URL: \(url)") @@ -48,7 +66,10 @@ struct WebToMarkdownCommand: AsyncParsableCommand { let page = try await WebPageFetcher.fetch( from: parsedURL, timeout: timeout, - extractMainOnly: main + extractMainOnly: main, + waitSeconds: wait, + waitForSelector: waitFor, + waitForText: waitForText ) if !skipFrontmatter { diff --git a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift index c8d1e3b..63b4d9e 100644 --- a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift +++ b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift @@ -27,6 +27,54 @@ struct WebPageFetcherTests { } } + @Test + func fetchReturnsStatusAndFinalURL() async throws { + let url = URL(string: "https://example.com")! + let page = try await WebPageFetcher.fetch(from: url) + #expect(page.statusCode == 200) + #expect(page.finalURL.host == "example.com") + #expect(!page.html.isEmpty) + } + + @Test + func waitForExistingSelectorReturnsImmediately() async throws { + let url = URL(string: "https://example.com")! + let start = Date() + let page = try await WebPageFetcher.fetch( + from: url, + timeout: 10, + waitForSelector: "h1" + ) + let elapsed = Date().timeIntervalSince(start) + #expect(!page.html.isEmpty) + #expect(elapsed < 5, "Selector that already exists should not delay extraction") + } + + @Test + func waitForNonexistentSelectorTimesOut() async throws { + let url = URL(string: "https://example.com")! + await #expect(throws: WebPageFetcher.Error.self) { + try await WebPageFetcher.fetch( + from: url, + timeout: 3, + waitForSelector: ".never-matches-xyz-12345" + ) + } + } + + @Test + func fixedWaitDelaysExtraction() async throws { + let url = URL(string: "https://example.com")! + let start = Date() + _ = try await WebPageFetcher.fetch( + from: url, + timeout: 10, + waitSeconds: 1.0 + ) + let elapsed = Date().timeIntervalSince(start) + #expect(elapsed >= 1.0, "Fixed wait should add at least its duration") + } + @Test func canCancelFetch() async throws { let url = URL(string: "https://example.com")! From 4fd3ebcfc507ec29f2ed4b206e1549086aaecd5b Mon Sep 17 00:00:00 2001 From: Nathan Herald Date: Wed, 29 Apr 2026 23:53:10 +0200 Subject: [PATCH 04/11] README: document new flags + YouTube example --- README.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 26102d8..ab43054 100644 --- a/README.md +++ b/README.md @@ -15,16 +15,76 @@ web-to-markdown https://example.com web-to-markdown https://example.com --timeout 30 --verbose ``` +By default the output is a fake-YAML frontmatter block followed by the body +markdown: + +``` +--- +status: 200 +final-url: https://example.com/ +title: Example Domain +fetched-at: 2026-04-29T21:42:56Z +--- + +# Example Domain +… +``` + +### Flags + +- `--main` — strip page chrome (nav, header, footer, cookies, sidebars, + recommendations) before conversion. Useful for noisy product/listing pages. +- `--head` — print only the frontmatter block, skip the body. Fast URL + validation in batch. +- `--skip-frontmatter` — output the body markdown only. +- `--wait ` — fixed delay (decimal allowed) after the page loads. + Useful for SPAs that hydrate after `window.load`. +- `--wait-for ` — wait until at least one element matches. + Event-driven via `MutationObserver`, no polling. Bounded by `--timeout`. +- `--wait-for-text ` — wait until the body's visible text contains + the substring. Same `MutationObserver` pattern. +- `--timeout ` — overall fetch timeout (default 30s). Bounds all + wait flags. + +### Example: YouTube watch pages + +YouTube hydrates async after `window.load`, so a plain fetch grabs the app +shell with `title: YouTube`. Wait for the subscribe button to render — it +appears only after page content is real: + +```bash +web-to-markdown 'https://www.youtube.com/watch?v=...' \ + --wait-for '#subscribe-button' --timeout 20 +``` + +Same idea works for other JS-heavy sites: find a selector or piece of body +text that only appears after meaningful hydration, and use it as the anchor. + ## Library ```swift import WebToMarkdown -// Fetch HTML from a URL using WKWebView -let html = try await WebPageFetcher.fetchHTML(from: url, timeout: 30) +// Fetch a page with full metadata +let page = try await WebPageFetcher.fetch( + from: url, + timeout: 30, + extractMainOnly: false, + waitSeconds: 0, + waitForSelector: nil, + waitForText: nil +) +// page.html, page.statusCode, page.finalURL // Convert HTML to Markdown -let markdown = try HTMLToMarkdown.convert(html, baseURL: url) +let markdown = try HTMLToMarkdown.convert(page.html, baseURL: page.finalURL) + +// Extract and meta description +let metadata = try HTMLToMarkdown.extractMetadata(page.html) +// metadata.title, metadata.description + +// Backwards-compatible HTML-only fetch +let html = try await WebPageFetcher.fetchHTML(from: url, timeout: 30) ``` ## Testing From 841c685501a29f7439fbb8eb77831c6e90d17aea Mon Sep 17 00:00:00 2001 From: Nathan Herald <me@nathanherald.com> Date: Thu, 30 Apr 2026 00:16:25 +0200 Subject: [PATCH 05/11] Add FetchOptions struct and Frontmatter formatter to library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ergonomics improvements for library consumers: 1. FetchOptions struct in WebToMarkdown: bundles the optional knobs so call sites don't grow with every new feature. New overload `WebPageFetcher.fetch(from:options:)` accepts it. Old positional overload kept for backward compat — it just builds a FetchOptions and forwards. 2. Frontmatter type (new file Frontmatter.swift): holds (page, metadata, fetchedAt) as raw Swift values, with a `format()` method that emits the YAML block the CLI produces. Consumers who want the data without formatting can read the stored properties directly; consumers who want the YAML can call format(). yamlEscape is also exposed as a public static for callers who want the same escape rules on their own values. CLI now uses both: builds a FetchOptions, calls `Frontmatter(page:, metadata:).format()`. Old private formatFrontmatter, yamlEscape, and truncate helpers in the CLI removed. Tests: 11 new FrontmatterTests covering the format output, value escaping (colons, quotes, backslashes, hashes, leading/trailing whitespace, empty values), newline flattening, plain-value passthrough, description truncation. All 69 tests pass. Also: README updated to use plain "YAML" (it parses as real YAML on the consumer side; the "fake" framing was misleading). Library example in README updated to show FetchOptions + Frontmatter usage. --- README.md | 39 ++++--- Sources/WebToMarkdown/Frontmatter.swift | 90 ++++++++++++++++ Sources/WebToMarkdown/WebPageFetcher.swift | 73 ++++++++++--- .../WebToMarkdownCommand.swift | 59 +---------- .../HTMLToMarkdownTests.swift | 100 ++++++++++++++++++ 5 files changed, 276 insertions(+), 85 deletions(-) create mode 100644 Sources/WebToMarkdown/Frontmatter.swift diff --git a/README.md b/README.md index ab43054..1bb39fa 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ web-to-markdown https://example.com web-to-markdown https://example.com --timeout 30 --verbose ``` -By default the output is a fake-YAML frontmatter block followed by the body +By default the output is a YAML frontmatter block followed by the body markdown: ``` @@ -65,25 +65,32 @@ text that only appears after meaningful hydration, and use it as the anchor. ```swift import WebToMarkdown -// Fetch a page with full metadata -let page = try await WebPageFetcher.fetch( - from: url, - timeout: 30, - extractMainOnly: false, - waitSeconds: 0, - waitForSelector: nil, - waitForText: nil -) -// page.html, page.statusCode, page.finalURL - -// Convert HTML to Markdown +// Fetch a page (defaults to a 30s timeout, no waits, no chrome stripping). +let page = try await WebPageFetcher.fetch(from: url) + +// Same fetch with all knobs available, via FetchOptions. +var options = FetchOptions() +options.extractMainOnly = true +options.waitSeconds = 0.5 +options.waitForSelector = "#subscribe-button" +options.timeout = 20 +let richPage = try await WebPageFetcher.fetch(from: url, options: options) + +// page.html, page.statusCode, page.finalURL — all on FetchedPage. +print(page.statusCode ?? -1, page.finalURL) + +// Convert HTML to Markdown. let markdown = try HTMLToMarkdown.convert(page.html, baseURL: page.finalURL) -// Extract <title> and meta description +// Extract <title> and meta description as Swift values. let metadata = try HTMLToMarkdown.extractMetadata(page.html) -// metadata.title, metadata.description +print(metadata.title ?? "", metadata.description ?? "") + +// Build the same YAML frontmatter the CLI emits. +let frontmatter = Frontmatter(page: page, metadata: metadata).format() +print(frontmatter) -// Backwards-compatible HTML-only fetch +// Backwards-compatible HTML-only fetch. let html = try await WebPageFetcher.fetchHTML(from: url, timeout: 30) ``` diff --git a/Sources/WebToMarkdown/Frontmatter.swift b/Sources/WebToMarkdown/Frontmatter.swift new file mode 100644 index 0000000..099eda4 --- /dev/null +++ b/Sources/WebToMarkdown/Frontmatter.swift @@ -0,0 +1,90 @@ +import Foundation + +/// Renders the standard YAML frontmatter block (`---` … `---`) used by the +/// `web-to-markdown` CLI. The struct holds the raw inputs as Swift types — +/// consumers who want the data without the formatting can just read the +/// stored properties or build their own representation. We emit YAML +/// directly without pulling in a YAML library; the output parses as real +/// YAML on the consumer side. +public struct Frontmatter: Sendable { + public let page: FetchedPage + public let metadata: PageMetadata + public let fetchedAt: Date + + /// Maximum length of the rendered `description` field (longer values are + /// truncated with an ellipsis). The default keeps frontmatter scannable. + public var descriptionMaxLength: Int = 200 + + public init( + page: FetchedPage, + metadata: PageMetadata, + fetchedAt: Date = Date(), + descriptionMaxLength: Int = 200 + ) { + self.page = page + self.metadata = metadata + self.fetchedAt = fetchedAt + self.descriptionMaxLength = descriptionMaxLength + } + + /// Format as a YAML block: `---` line, one `key: value` per line, + /// closing `---`. Values containing characters that would confuse a YAML + /// reader (`:`, `"`, `\\`, `#`, leading/trailing whitespace) are + /// double-quoted with `\\` and `"` escaped. Newlines in values are + /// flattened to spaces. + public func format() -> String { + var lines = ["---"] + if let status = page.statusCode { + lines.append("status: \(status)") + } + lines.append("final-url: \(yamlEscape(page.finalURL.absoluteString))") + if let title = metadata.title, !title.isEmpty { + lines.append("title: \(yamlEscape(title))") + } + if let description = metadata.description, !description.isEmpty { + lines.append( + "description: \(yamlEscape(truncate(description, max: descriptionMaxLength)))" + ) + } + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + lines.append("fetched-at: \(formatter.string(from: fetchedAt))") + lines.append("---") + return lines.joined(separator: "\n") + } + + /// Public for unit tests and library consumers who want to format their + /// own values with the same escape rules. + public static func yamlEscape(_ value: String) -> String { + let flattened = value + .replacingOccurrences(of: "\r\n", with: " ") + .replacingOccurrences(of: "\n", with: " ") + .replacingOccurrences(of: "\r", with: " ") + + let needsQuoting = flattened.contains(":") + || flattened.contains("\"") + || flattened.contains("\\") + || flattened.contains("#") + || flattened.first == " " + || flattened.last == " " + || flattened.isEmpty + + if needsQuoting { + let escaped = flattened + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + return "\"\(escaped)\"" + } + return flattened + } + + private func yamlEscape(_ value: String) -> String { + Self.yamlEscape(value) + } + + private func truncate(_ s: String, max: Int) -> String { + if s.count <= max { return s } + let end = s.index(s.startIndex, offsetBy: max) + return String(s[s.startIndex ..< end]) + "…" + } +} diff --git a/Sources/WebToMarkdown/WebPageFetcher.swift b/Sources/WebToMarkdown/WebPageFetcher.swift index a0270e7..fd35085 100644 --- a/Sources/WebToMarkdown/WebPageFetcher.swift +++ b/Sources/WebToMarkdown/WebPageFetcher.swift @@ -17,6 +17,47 @@ public struct FetchedPage: Sendable { } } +/// All optional knobs a caller can tweak when fetching a page. Use the +/// default-initialized value for the simple case, then mutate the fields you +/// care about, or pass them at the init site. +public struct FetchOptions: Sendable { + /// Overall fetch timeout, seconds. The fetch fails if everything + /// (loading + waits + extraction) doesn't complete by this deadline. + public var timeout: TimeInterval = 30 + + /// Strip page chrome (nav/header/footer/cookie/sidebar/recommended) + /// before HTML extraction. + public var extractMainOnly: Bool = false + + /// Fixed delay (seconds) after the page finishes loading, before HTML is + /// extracted. Useful for SPAs that hydrate after `window.load`. + public var waitSeconds: TimeInterval = 0 + + /// CSS selector to wait for. Extraction is delayed until at least one + /// element matches. Event-driven via `MutationObserver`. Bounded by + /// `timeout`. + public var waitForSelector: String? + + /// Substring to wait for in the body's visible text. Extraction is + /// delayed until found. Event-driven via `MutationObserver`. Bounded by + /// `timeout`. + public var waitForText: String? + + public init( + timeout: TimeInterval = 30, + extractMainOnly: Bool = false, + waitSeconds: TimeInterval = 0, + waitForSelector: String? = nil, + waitForText: String? = nil + ) { + self.timeout = timeout + self.extractMainOnly = extractMainOnly + self.waitSeconds = waitSeconds + self.waitForSelector = waitForSelector + self.waitForText = waitForText + } +} + public enum WebPageFetcher { public enum Error: Swift.Error { case loadFailed(String) @@ -26,20 +67,24 @@ public enum WebPageFetcher { /// Fetch a web page and return HTML plus response metadata. /// - /// - Parameters: - /// - extractMainOnly: when `true`, common page chrome (nav, header, - /// footer, cookies, sidebars, related/recommended sections) is - /// stripped from the DOM before the HTML is returned. - /// - waitSeconds: extra fixed delay (seconds) after the page finishes - /// loading, before HTML is extracted. Useful for SPAs that hydrate - /// async after `window.load`. - /// - waitForSelector: if non-nil, waits (event-driven via - /// `MutationObserver`) until at least one element matching the CSS - /// selector is present in the DOM, before extraction. - /// - waitForText: if non-nil, waits until the body's visible text - /// contains this substring before extraction. - /// - /// All wait operations are bounded by the overall `timeout`. + /// See ``FetchOptions`` for the available knobs. + public static func fetch( + from url: URL, + options: FetchOptions = FetchOptions() + ) async throws -> FetchedPage { + try await fetch( + from: url, + timeout: options.timeout, + extractMainOnly: options.extractMainOnly, + waitSeconds: options.waitSeconds, + waitForSelector: options.waitForSelector, + waitForText: options.waitForText + ) + } + + /// Fetch a web page using individual parameters. Equivalent to passing a + /// ``FetchOptions`` value; this overload keeps existing call sites + /// working without adapter code. public static func fetch( from url: URL, timeout: TimeInterval = 30, diff --git a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift index 16f8eba..6a205f8 100644 --- a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift +++ b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift @@ -63,8 +63,7 @@ struct WebToMarkdownCommand: AsyncParsableCommand { fputs("Fetching \(parsedURL.absoluteString)...\n", stderr) } - let page = try await WebPageFetcher.fetch( - from: parsedURL, + let options = FetchOptions( timeout: timeout, extractMainOnly: main, waitSeconds: wait, @@ -72,10 +71,12 @@ struct WebToMarkdownCommand: AsyncParsableCommand { waitForText: waitForText ) + let page = try await WebPageFetcher.fetch(from: parsedURL, options: options) + if !skipFrontmatter { let metadata = (try? HTMLToMarkdown.extractMetadata(page.html)) ?? PageMetadata(title: nil, description: nil) - print(formatFrontmatter(page: page, metadata: metadata)) + print(Frontmatter(page: page, metadata: metadata).format()) if !head { print("") } } @@ -88,56 +89,4 @@ struct WebToMarkdownCommand: AsyncParsableCommand { let markdown = try HTMLToMarkdown.convert(page.html, baseURL: parsedURL) print(markdown) } - - private func formatFrontmatter(page: FetchedPage, metadata: PageMetadata) -> String { - var lines = ["---"] - if let status = page.statusCode { - lines.append("status: \(status)") - } - lines.append("final-url: \(yamlEscape(page.finalURL.absoluteString))") - if let title = metadata.title, !title.isEmpty { - lines.append("title: \(yamlEscape(title))") - } - if let description = metadata.description, !description.isEmpty { - lines.append("description: \(yamlEscape(truncate(description, max: 200)))") - } - let formatter = ISO8601DateFormatter() - formatter.formatOptions = [.withInternetDateTime] - lines.append("fetched-at: \(formatter.string(from: Date()))") - lines.append("---") - return lines.joined(separator: "\n") - } - - /// Escape a string for use as a fake-YAML frontmatter scalar value. - /// Quotes the value when it contains characters that would confuse a - /// downstream YAML reader (`:`, `"`, `\\`, leading/trailing whitespace). - /// Newlines are flattened to spaces — frontmatter is single-line per key. - private func yamlEscape(_ value: String) -> String { - let flattened = value - .replacingOccurrences(of: "\r\n", with: " ") - .replacingOccurrences(of: "\n", with: " ") - .replacingOccurrences(of: "\r", with: " ") - - let needsQuoting = flattened.contains(":") - || flattened.contains("\"") - || flattened.contains("\\") - || flattened.contains("#") - || flattened.first == " " - || flattened.last == " " - || flattened.isEmpty - - if needsQuoting { - let escaped = flattened - .replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "\"", with: "\\\"") - return "\"\(escaped)\"" - } - return flattened - } - - private func truncate(_ s: String, max: Int) -> String { - if s.count <= max { return s } - let end = s.index(s.startIndex, offsetBy: max) - return String(s[s.startIndex ..< end]) + "…" - } } diff --git a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift index dbdbeca..545c781 100644 --- a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift +++ b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift @@ -2,6 +2,106 @@ import Foundation import Testing @testable import WebToMarkdown +@Suite +struct FrontmatterTests { + private func makePage( + url: String = "https://example.com/", + status: Int? = 200, + html: String = "<html></html>" + ) -> FetchedPage { + FetchedPage(html: html, statusCode: status, finalURL: URL(string: url)!) + } + + private let fixedDate = Date(timeIntervalSince1970: 1_700_000_000) // 2023-11-14T22:13:20Z + + @Test + func formatsAllFields() { + let page = makePage() + let meta = PageMetadata(title: "Hello", description: "A description.") + let fm = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate) + let out = fm.format() + #expect(out.hasPrefix("---\n")) + #expect(out.hasSuffix("\n---")) + #expect(out.contains("status: 200")) + // URL contains ':' so it gets quoted. + #expect(out.contains("final-url: \"https://example.com/\"")) + #expect(out.contains("title: Hello")) + #expect(out.contains("description: A description.")) + #expect(out.contains("fetched-at: 2023-11-14T22:13:20Z")) + } + + @Test + func omitsAbsentMetadata() { + let page = makePage() + let meta = PageMetadata(title: nil, description: nil) + let out = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate).format() + #expect(!out.contains("title:")) + #expect(!out.contains("description:")) + } + + @Test + func omitsAbsentStatus() { + let page = makePage(status: nil) + let meta = PageMetadata(title: "x", description: nil) + let out = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate).format() + #expect(!out.contains("status:")) + } + + @Test + func quotesValuesWithColons() { + let escaped = Frontmatter.yamlEscape("Foo: Bar") + #expect(escaped == "\"Foo: Bar\"") + } + + @Test + func quotesValuesWithQuotes() { + let escaped = Frontmatter.yamlEscape("She said \"hi\"") + #expect(escaped == "\"She said \\\"hi\\\"\"") + } + + @Test + func quotesValuesWithBackslash() { + let escaped = Frontmatter.yamlEscape("path\\to") + #expect(escaped == "\"path\\\\to\"") + } + + @Test + func quotesValuesWithHash() { + let escaped = Frontmatter.yamlEscape("foo #bar") + #expect(escaped == "\"foo #bar\"") + } + + @Test + func quotesEmptyAndWhitespacePadded() { + #expect(Frontmatter.yamlEscape("") == "\"\"") + #expect(Frontmatter.yamlEscape(" leading") == "\" leading\"") + #expect(Frontmatter.yamlEscape("trailing ") == "\"trailing \"") + } + + @Test + func flattensNewlinesInValues() { + let escaped = Frontmatter.yamlEscape("line one\nline two") + // No colon/quote/backslash/hash, so no quoting; just a flattened space. + #expect(escaped == "line one line two") + } + + @Test + func leavesPlainValuesAlone() { + #expect(Frontmatter.yamlEscape("Hello world") == "Hello world") + #expect(Frontmatter.yamlEscape("plain-value-123") == "plain-value-123") + } + + @Test + func truncatesLongDescription() { + let longDesc = String(repeating: "a", count: 250) + let page = makePage() + let meta = PageMetadata(title: nil, description: longDesc) + let out = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate).format() + // Description should be exactly 200 chars + "…" (default max). + #expect(out.contains("description: " + String(repeating: "a", count: 200) + "…")) + } +} + @Suite struct PageMetadataTests { @Test From ef81da63c038f76677869385af1e59692c588acf Mon Sep 17 00:00:00 2001 From: Nathan Herald <me@nathanherald.com> Date: Thu, 30 Apr 2026 16:33:48 +0200 Subject: [PATCH 06/11] Gate per-navigation logging behind --verbose / FetchOptions.verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decidePolicyFor navigationAction handler was logging every URL it saw, which on chrome-heavy pages produced a flood of about:blank / iframe / redirect entries in the Xcode console. The one-shot lifecycle events (Loading URL, Response status, Page loaded, Extracted HTML length) and all error paths are unaffected — they still always log. Adds FetchOptions.verbose (default false). The CLI's existing --verbose flag now passes through to it. --- Sources/WebToMarkdown/WebPageFetcher.swift | 29 +++++++++++++++---- .../WebToMarkdownCommand.swift | 3 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Sources/WebToMarkdown/WebPageFetcher.swift b/Sources/WebToMarkdown/WebPageFetcher.swift index fd35085..9137251 100644 --- a/Sources/WebToMarkdown/WebPageFetcher.swift +++ b/Sources/WebToMarkdown/WebPageFetcher.swift @@ -43,18 +43,26 @@ public struct FetchOptions: Sendable { /// `timeout`. public var waitForText: String? + /// When `true`, log every per-navigation event (including iframes, + /// `about:blank`, redirects). Off by default — the noise scales badly on + /// chrome-heavy pages. Errors and one-shot lifecycle events (loading, + /// response status, page loaded, extracted length) always log. + public var verbose: Bool = false + public init( timeout: TimeInterval = 30, extractMainOnly: Bool = false, waitSeconds: TimeInterval = 0, waitForSelector: String? = nil, - waitForText: String? = nil + waitForText: String? = nil, + verbose: Bool = false ) { self.timeout = timeout self.extractMainOnly = extractMainOnly self.waitSeconds = waitSeconds self.waitForSelector = waitForSelector self.waitForText = waitForText + self.verbose = verbose } } @@ -78,7 +86,8 @@ public enum WebPageFetcher { extractMainOnly: options.extractMainOnly, waitSeconds: options.waitSeconds, waitForSelector: options.waitForSelector, - waitForText: options.waitForText + waitForText: options.waitForText, + verbose: options.verbose ) } @@ -91,7 +100,8 @@ public enum WebPageFetcher { extractMainOnly: Bool = false, waitSeconds: TimeInterval = 0, waitForSelector: String? = nil, - waitForText: String? = nil + waitForText: String? = nil, + verbose: Bool = false ) async throws -> FetchedPage { os_log( .info, @@ -136,7 +146,8 @@ public enum WebPageFetcher { extractMainOnly: extractMainOnly, waitSeconds: waitSeconds, waitForSelector: waitForSelector, - waitForText: waitForText + waitForText: waitForText, + verbose: verbose ) state.access { state in @@ -252,6 +263,7 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { let waitSeconds: TimeInterval let waitForSelector: String? let waitForText: String? + let verbose: Bool var timeoutTask: Task<Void, Never>? var capturedStatusCode: Int? @@ -262,7 +274,8 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { extractMainOnly: Bool, waitSeconds: TimeInterval, waitForSelector: String?, - waitForText: String? + waitForText: String?, + verbose: Bool ) { self.webView = webView self.state = state @@ -270,6 +283,7 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { self.waitSeconds = waitSeconds self.waitForSelector = waitForSelector self.waitForText = waitForText + self.verbose = verbose super.init() timeoutTask = Task { @MainActor in @@ -406,7 +420,10 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { ) async -> WKNavigationActionPolicy { - if let url = navigationAction.request.url { + // Per-navigation events are noisy on chrome-heavy pages (every iframe, + // every about:blank initial state, every redirect fires this). Off by + // default; opt in via FetchOptions.verbose / CLI --verbose. + if verbose, let url = navigationAction.request.url { os_log( .info, log: log, diff --git a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift index 6a205f8..38a5a01 100644 --- a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift +++ b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift @@ -68,7 +68,8 @@ struct WebToMarkdownCommand: AsyncParsableCommand { extractMainOnly: main, waitSeconds: wait, waitForSelector: waitFor, - waitForText: waitForText + waitForText: waitForText, + verbose: verbose ) let page = try await WebPageFetcher.fetch(from: parsedURL, options: options) From 3e114fee5ac7dd3e5db3f3a6784ab0b1b3d56f96 Mon Sep 17 00:00:00 2001 From: Nathan Herald <me@nathanherald.com> Date: Tue, 5 May 2026 22:44:26 +0200 Subject: [PATCH 07/11] CLI: validate --head/--skip-frontmatter as mutually exclusive; use page.finalURL as markdown base Two review fixes: - Throw a ValidationError if both --head and --skip-frontmatter are passed. Previously the combination silently produced empty output (frontmatter suppressed + early return after head). - Pass page.finalURL (not the raw input URL) as the base URL into HTMLToMarkdown.convert so relative links resolve correctly when the fetch followed redirects. --- Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift index 38a5a01..a9555b2 100644 --- a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift +++ b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift @@ -54,6 +54,14 @@ struct WebToMarkdownCommand: AsyncParsableCommand { ) var waitForText: String? + func validate() throws { + if head, skipFrontmatter { + throw ValidationError( + "--head and --skip-frontmatter are mutually exclusive: --head prints only the frontmatter, --skip-frontmatter prints only the body. Pick one." + ) + } + } + mutating func run() async throws { guard let parsedURL = URL(string: url) else { throw ValidationError("Invalid URL: \(url)") @@ -87,7 +95,7 @@ struct WebToMarkdownCommand: AsyncParsableCommand { fputs("Converting to markdown...\n", stderr) } - let markdown = try HTMLToMarkdown.convert(page.html, baseURL: parsedURL) + let markdown = try HTMLToMarkdown.convert(page.html, baseURL: page.finalURL) print(markdown) } } From d54b9388280430ce172352ed42b88aa4aab2d8e6 Mon Sep 17 00:00:00 2001 From: Nathan Herald <me@nathanherald.com> Date: Tue, 5 May 2026 22:44:29 +0200 Subject: [PATCH 08/11] Frontmatter: yamlEscape the fetched-at timestamp ISO8601 strings contain ':' so they need the same quoting as URLs and other colon-containing values. Previously the timestamp was emitted raw, producing technically-invalid YAML. --- Sources/WebToMarkdown/Frontmatter.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/WebToMarkdown/Frontmatter.swift b/Sources/WebToMarkdown/Frontmatter.swift index 099eda4..5fd241a 100644 --- a/Sources/WebToMarkdown/Frontmatter.swift +++ b/Sources/WebToMarkdown/Frontmatter.swift @@ -48,7 +48,7 @@ public struct Frontmatter: Sendable { } let formatter = ISO8601DateFormatter() formatter.formatOptions = [.withInternetDateTime] - lines.append("fetched-at: \(formatter.string(from: fetchedAt))") + lines.append("fetched-at: \(yamlEscape(formatter.string(from: fetchedAt)))") lines.append("---") return lines.joined(separator: "\n") } From 3fab0a7559f40d385248193e77a69e08fe23efde Mon Sep 17 00:00:00 2001 From: Nathan Herald <me@nathanherald.com> Date: Tue, 5 May 2026 22:44:31 +0200 Subject: [PATCH 09/11] README: update example to show quoted URL/timestamp Frontmatter.format() quotes any value containing ':' (which both URLs and ISO timestamps do). The README example previously showed unquoted forms that didn't match real output. --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1bb39fa..9f647db 100644 --- a/README.md +++ b/README.md @@ -21,15 +21,18 @@ markdown: ``` --- status: 200 -final-url: https://example.com/ +final-url: "https://example.com/" title: Example Domain -fetched-at: 2026-04-29T21:42:56Z +fetched-at: "2026-04-29T21:42:56Z" --- # Example Domain … ``` +Values containing `:` (URLs, ISO timestamps) are double-quoted so the output +parses cleanly as YAML. + ### Flags - `--main` — strip page chrome (nav, header, footer, cookies, sidebars, From 8fd44c3a0b78db17c0cf43acf9849e974883e3bb Mon Sep 17 00:00:00 2001 From: Nathan Herald <me@nathanherald.com> Date: Tue, 5 May 2026 22:44:36 +0200 Subject: [PATCH 10/11] Tests: relax timing assertions to tolerate timer precision Three test fixes: - waitForExistingSelectorReturnsImmediately: replace hardcoded 'elapsed < 5' with 'elapsed < timeout - 1' parameterized against the configured timeout, so a slow CI/network doesn't flake the test. - fixedWaitDelaysExtraction: loosen 'elapsed >= 1.0' to '>= 0.9' for ~100ms timer-precision/scheduling tolerance. - formatsAllFields: update expectation to the quoted form now that fetched-at runs through yamlEscape. --- Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift | 3 ++- Tests/WebToMarkdownTests/WebPageFetcherTests.swift | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift index 545c781..c2d82cb 100644 --- a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift +++ b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift @@ -27,7 +27,8 @@ struct FrontmatterTests { #expect(out.contains("final-url: \"https://example.com/\"")) #expect(out.contains("title: Hello")) #expect(out.contains("description: A description.")) - #expect(out.contains("fetched-at: 2023-11-14T22:13:20Z")) + // ISO8601 timestamp contains ':' so it gets quoted. + #expect(out.contains("fetched-at: \"2023-11-14T22:13:20Z\"")) } @Test diff --git a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift index 63b4d9e..f344bcc 100644 --- a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift +++ b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift @@ -39,15 +39,19 @@ struct WebPageFetcherTests { @Test func waitForExistingSelectorReturnsImmediately() async throws { let url = URL(string: "https://example.com")! + let timeout: TimeInterval = 10 let start = Date() let page = try await WebPageFetcher.fetch( from: url, - timeout: 10, + timeout: timeout, waitForSelector: "h1" ) let elapsed = Date().timeIntervalSince(start) #expect(!page.html.isEmpty) - #expect(elapsed < 5, "Selector that already exists should not delay extraction") + #expect( + elapsed < timeout - 1, + "Selector that already exists should complete well before the configured timeout" + ) } @Test @@ -72,7 +76,10 @@ struct WebPageFetcherTests { waitSeconds: 1.0 ) let elapsed = Date().timeIntervalSince(start) - #expect(elapsed >= 1.0, "Fixed wait should add at least its duration") + #expect( + elapsed >= 0.9, + "Fixed wait should add approximately its duration (0.1s tolerance for timer precision)" + ) } @Test From b0a318502bb169618ff594cdc02be82204d6f24a Mon Sep 17 00:00:00 2001 From: Anthony Drendel <me@anthonydrendel.com> Date: Sat, 16 May 2026 16:03:30 +0200 Subject: [PATCH 11/11] Handle YAML indicators and fetcher edge cases - Quote frontmatter values that start with YAML indicator characters - Observe attribute mutations when waiting for selectors - Capture status only for main-frame responses and use the latest final URL --- Sources/WebToMarkdown/Frontmatter.swift | 1 + Sources/WebToMarkdown/WebPageFetcher.swift | 10 +++--- .../HTMLToMarkdownTests.swift | 7 +++++ .../WebPageFetcherTests.swift | 31 +++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Sources/WebToMarkdown/Frontmatter.swift b/Sources/WebToMarkdown/Frontmatter.swift index 5fd241a..82c1d1d 100644 --- a/Sources/WebToMarkdown/Frontmatter.swift +++ b/Sources/WebToMarkdown/Frontmatter.swift @@ -65,6 +65,7 @@ public struct Frontmatter: Sendable { || flattened.contains("\"") || flattened.contains("\\") || flattened.contains("#") + || flattened.first.map { "-?:,[]{}#&*!|>'\"%@`".contains($0) } == true || flattened.first == " " || flattened.last == " " || flattened.isEmpty diff --git a/Sources/WebToMarkdown/WebPageFetcher.swift b/Sources/WebToMarkdown/WebPageFetcher.swift index 9137251..41f4368 100644 --- a/Sources/WebToMarkdown/WebPageFetcher.swift +++ b/Sources/WebToMarkdown/WebPageFetcher.swift @@ -210,7 +210,7 @@ if (typeof waitForSelector === "string" && waitForSelector.length > 0) { var obs = new MutationObserver(function() { if (check()) { obs.disconnect(); resolve(); } }); - obs.observe(document.documentElement, { childList: true, subtree: true }); + obs.observe(document.documentElement, { attributes: true, childList: true, subtree: true }); }); } @@ -305,7 +305,9 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { _: WKWebView, decidePolicyFor navigationResponse: WKNavigationResponse ) async -> WKNavigationResponsePolicy { - if let httpResponse = navigationResponse.response as? HTTPURLResponse { + if navigationResponse.isForMainFrame, + let httpResponse = navigationResponse.response as? HTTPURLResponse + { capturedStatusCode = httpResponse.statusCode os_log( .info, @@ -324,7 +326,7 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { os_log(.info, log: log, "🔧TOOLCALL🔧 WebPageFetcher: Page loaded, extracting HTML") - let finalURL = webView.url + let fallbackURL = webView.url let status = capturedStatusCode let arguments: [String: Any] = [ @@ -356,7 +358,7 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { return } - let resolved = finalURL ?? webView.url ?? URL(string: "about:blank")! + let resolved = webView.url ?? fallbackURL ?? URL(string: "about:blank")! let page = FetchedPage(html: html, statusCode: status, finalURL: resolved) _ = state.access { $0.finish(with: page) } diff --git a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift index c2d82cb..79834a3 100644 --- a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift +++ b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift @@ -72,6 +72,13 @@ struct FrontmatterTests { #expect(escaped == "\"foo #bar\"") } + @Test + func quotesValuesWithLeadingYamlIndicators() { + #expect(Frontmatter.yamlEscape("[Draft]") == "\"[Draft]\"") + #expect(Frontmatter.yamlEscape("*alias") == "\"*alias\"") + #expect(Frontmatter.yamlEscape("> block") == "\"> block\"") + } + @Test func quotesEmptyAndWhitespacePadded() { #expect(Frontmatter.yamlEscape("") == "\"\"") diff --git a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift index f344bcc..42763f1 100644 --- a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift +++ b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift @@ -66,6 +66,32 @@ struct WebPageFetcherTests { } } + @Test + func waitForSelectorHandlesAttributeMutation() async throws { + let url = dataURL( + """ + <html> + <body> + <main id="content">Loading</main> + <script> + setTimeout(function() { + document.getElementById("content").setAttribute("data-ready", "true"); + }, 100); + </script> + </body> + </html> + """ + ) + + let page = try await WebPageFetcher.fetch( + from: url, + timeout: 5, + waitForSelector: "[data-ready='true']" + ) + + #expect(page.html.contains("data-ready=\"true\"")) + } + @Test func fixedWaitDelaysExtraction() async throws { let url = URL(string: "https://example.com")! @@ -110,3 +136,8 @@ private func yield(_ times: Int) async { await Task.yield() } } + +private func dataURL(_ html: String) -> URL { + let encoded = Data(html.utf8).base64EncodedString() + return URL(string: "data:text/html;charset=utf-8;base64,\(encoded)")! +}