diff --git a/README.md b/README.md index 26102d8..9f647db 100644 --- a/README.md +++ b/README.md @@ -15,16 +15,86 @@ web-to-markdown https://example.com web-to-markdown https://example.com --timeout 30 --verbose ``` +By default the output is a YAML frontmatter block followed by the body +markdown: + +``` +--- +status: 200 +final-url: "https://example.com/" +title: Example Domain +fetched-at: "2026-04-29T21:42:56Z" +--- + +# Example Domain +… +``` + +Values containing `:` (URLs, ISO timestamps) are double-quoted so the output +parses cleanly as YAML. + +### Flags + +- `--main` — strip page chrome (nav, header, footer, cookies, sidebars, + recommendations) before conversion. Useful for noisy product/listing pages. +- `--head` — print only the frontmatter block, skip the body. Fast URL + validation in batch. +- `--skip-frontmatter` — output the body markdown only. +- `--wait ` — fixed delay (decimal allowed) after the page loads. + Useful for SPAs that hydrate after `window.load`. +- `--wait-for ` — wait until at least one element matches. + Event-driven via `MutationObserver`, no polling. Bounded by `--timeout`. +- `--wait-for-text ` — wait until the body's visible text contains + the substring. Same `MutationObserver` pattern. +- `--timeout ` — overall fetch timeout (default 30s). Bounds all + wait flags. + +### Example: YouTube watch pages + +YouTube hydrates async after `window.load`, so a plain fetch grabs the app +shell with `title: YouTube`. Wait for the subscribe button to render — it +appears only after page content is real: + +```bash +web-to-markdown 'https://www.youtube.com/watch?v=...' \ + --wait-for '#subscribe-button' --timeout 20 +``` + +Same idea works for other JS-heavy sites: find a selector or piece of body +text that only appears after meaningful hydration, and use it as the anchor. + ## Library ```swift import WebToMarkdown -// Fetch HTML from a URL using WKWebView -let html = try await WebPageFetcher.fetchHTML(from: url, timeout: 30) +// Fetch a page (defaults to a 30s timeout, no waits, no chrome stripping). +let page = try await WebPageFetcher.fetch(from: url) + +// Same fetch with all knobs available, via FetchOptions. +var options = FetchOptions() +options.extractMainOnly = true +options.waitSeconds = 0.5 +options.waitForSelector = "#subscribe-button" +options.timeout = 20 +let richPage = try await WebPageFetcher.fetch(from: url, options: options) -// Convert HTML to Markdown -let markdown = try HTMLToMarkdown.convert(html, baseURL: url) +// page.html, page.statusCode, page.finalURL — all on FetchedPage. +print(page.statusCode ?? -1, page.finalURL) + +// Convert HTML to Markdown. +let markdown = try HTMLToMarkdown.convert(page.html, baseURL: page.finalURL) + +// Extract and meta description as Swift values. +let metadata = try HTMLToMarkdown.extractMetadata(page.html) +print(metadata.title ?? "", metadata.description ?? "") + +// Build the same YAML frontmatter the CLI emits. +let frontmatter = Frontmatter(page: page, metadata: metadata).format() +print(frontmatter) + +// Backwards-compatible HTML-only fetch. +let html = try await WebPageFetcher.fetchHTML(from: url, timeout: 30) ``` ## Testing diff --git a/Sources/WebToMarkdown/Frontmatter.swift b/Sources/WebToMarkdown/Frontmatter.swift new file mode 100644 index 0000000..82c1d1d --- /dev/null +++ b/Sources/WebToMarkdown/Frontmatter.swift @@ -0,0 +1,91 @@ +import Foundation + +/// Renders the standard YAML frontmatter block (`---` … `---`) used by the +/// `web-to-markdown` CLI. The struct holds the raw inputs as Swift types — +/// consumers who want the data without the formatting can just read the +/// stored properties or build their own representation. We emit YAML +/// directly without pulling in a YAML library; the output parses as real +/// YAML on the consumer side. +public struct Frontmatter: Sendable { + public let page: FetchedPage + public let metadata: PageMetadata + public let fetchedAt: Date + + /// Maximum length of the rendered `description` field (longer values are + /// truncated with an ellipsis). The default keeps frontmatter scannable. + public var descriptionMaxLength: Int = 200 + + public init( + page: FetchedPage, + metadata: PageMetadata, + fetchedAt: Date = Date(), + descriptionMaxLength: Int = 200 + ) { + self.page = page + self.metadata = metadata + self.fetchedAt = fetchedAt + self.descriptionMaxLength = descriptionMaxLength + } + + /// Format as a YAML block: `---` line, one `key: value` per line, + /// closing `---`. Values containing characters that would confuse a YAML + /// reader (`:`, `"`, `\\`, `#`, leading/trailing whitespace) are + /// double-quoted with `\\` and `"` escaped. Newlines in values are + /// flattened to spaces. + public func format() -> String { + var lines = ["---"] + if let status = page.statusCode { + lines.append("status: \(status)") + } + lines.append("final-url: \(yamlEscape(page.finalURL.absoluteString))") + if let title = metadata.title, !title.isEmpty { + lines.append("title: \(yamlEscape(title))") + } + if let description = metadata.description, !description.isEmpty { + lines.append( + "description: \(yamlEscape(truncate(description, max: descriptionMaxLength)))" + ) + } + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + lines.append("fetched-at: \(yamlEscape(formatter.string(from: fetchedAt)))") + lines.append("---") + return lines.joined(separator: "\n") + } + + /// Public for unit tests and library consumers who want to format their + /// own values with the same escape rules. + public static func yamlEscape(_ value: String) -> String { + let flattened = value + .replacingOccurrences(of: "\r\n", with: " ") + .replacingOccurrences(of: "\n", with: " ") + .replacingOccurrences(of: "\r", with: " ") + + let needsQuoting = flattened.contains(":") + || flattened.contains("\"") + || flattened.contains("\\") + || flattened.contains("#") + || flattened.first.map { "-?:,[]{}#&*!|>'\"%@`".contains($0) } == true + || flattened.first == " " + || flattened.last == " " + || flattened.isEmpty + + if needsQuoting { + let escaped = flattened + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + return "\"\(escaped)\"" + } + return flattened + } + + private func yamlEscape(_ value: String) -> String { + Self.yamlEscape(value) + } + + private func truncate(_ s: String, max: Int) -> String { + if s.count <= max { return s } + let end = s.index(s.startIndex, offsetBy: max) + return String(s[s.startIndex ..< end]) + "…" + } +} diff --git a/Sources/WebToMarkdown/HTMLToMarkdown.swift b/Sources/WebToMarkdown/HTMLToMarkdown.swift index 7b09115..0c95c96 100644 --- a/Sources/WebToMarkdown/HTMLToMarkdown.swift +++ b/Sources/WebToMarkdown/HTMLToMarkdown.swift @@ -1,6 +1,16 @@ import Foundation import SwiftSoup +public struct PageMetadata: Sendable { + public let title: String? + public let description: String? + + public init(title: String?, description: String?) { + self.title = title + self.description = description + } +} + public enum HTMLToMarkdown { public enum Error: Swift.Error { case parsingFailed(String) @@ -19,6 +29,41 @@ public enum HTMLToMarkdown { return collapseExcessiveNewlines(markdown) } + /// Extract the page `<title>` and meta `description` (falling back to + /// Open Graph variants when standard tags are missing). + public static func extractMetadata(_ html: String) throws -> PageMetadata { + let document = try SwiftSoup.parse(html) + + let title: String? = try { + if let el = try document.select("title").first() { + let text = try el.text().trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + if let el = try document.select("meta[property=og:title]").first() { + let text = try el.attr("content") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + return nil + }() + + let description: String? = try { + if let el = try document.select("meta[name=description]").first() { + let text = try el.attr("content") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + if let el = try document.select("meta[property=og:description]").first() { + let text = try el.attr("content") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !text.isEmpty { return text } + } + return nil + }() + + return PageMetadata(title: title, description: description) + } + private static func collapseExcessiveNewlines(_ text: String) -> String { var result = text while result.contains("\n\n\n") { diff --git a/Sources/WebToMarkdown/WebPageFetcher.swift b/Sources/WebToMarkdown/WebPageFetcher.swift index d18b40c..41f4368 100644 --- a/Sources/WebToMarkdown/WebPageFetcher.swift +++ b/Sources/WebToMarkdown/WebPageFetcher.swift @@ -5,6 +5,67 @@ import WebKit private let log = OSLog(subsystem: "com.shareup.web-to-markdown", category: "web-page-fetcher") +public struct FetchedPage: Sendable { + public let html: String + public let statusCode: Int? + public let finalURL: URL + + public init(html: String, statusCode: Int?, finalURL: URL) { + self.html = html + self.statusCode = statusCode + self.finalURL = finalURL + } +} + +/// All optional knobs a caller can tweak when fetching a page. Use the +/// default-initialized value for the simple case, then mutate the fields you +/// care about, or pass them at the init site. +public struct FetchOptions: Sendable { + /// Overall fetch timeout, seconds. The fetch fails if everything + /// (loading + waits + extraction) doesn't complete by this deadline. + public var timeout: TimeInterval = 30 + + /// Strip page chrome (nav/header/footer/cookie/sidebar/recommended) + /// before HTML extraction. + public var extractMainOnly: Bool = false + + /// Fixed delay (seconds) after the page finishes loading, before HTML is + /// extracted. Useful for SPAs that hydrate after `window.load`. + public var waitSeconds: TimeInterval = 0 + + /// CSS selector to wait for. Extraction is delayed until at least one + /// element matches. Event-driven via `MutationObserver`. Bounded by + /// `timeout`. + public var waitForSelector: String? + + /// Substring to wait for in the body's visible text. Extraction is + /// delayed until found. Event-driven via `MutationObserver`. Bounded by + /// `timeout`. + public var waitForText: String? + + /// When `true`, log every per-navigation event (including iframes, + /// `about:blank`, redirects). Off by default — the noise scales badly on + /// chrome-heavy pages. Errors and one-shot lifecycle events (loading, + /// response status, page loaded, extracted length) always log. + public var verbose: Bool = false + + public init( + timeout: TimeInterval = 30, + extractMainOnly: Bool = false, + waitSeconds: TimeInterval = 0, + waitForSelector: String? = nil, + waitForText: String? = nil, + verbose: Bool = false + ) { + self.timeout = timeout + self.extractMainOnly = extractMainOnly + self.waitSeconds = waitSeconds + self.waitForSelector = waitForSelector + self.waitForText = waitForText + self.verbose = verbose + } +} + public enum WebPageFetcher { public enum Error: Swift.Error { case loadFailed(String) @@ -12,10 +73,36 @@ public enum WebPageFetcher { case noHTML } - public static func fetchHTML( + /// Fetch a web page and return HTML plus response metadata. + /// + /// See ``FetchOptions`` for the available knobs. + public static func fetch( from url: URL, - timeout: TimeInterval = 30 - ) async throws -> String { + options: FetchOptions = FetchOptions() + ) async throws -> FetchedPage { + try await fetch( + from: url, + timeout: options.timeout, + extractMainOnly: options.extractMainOnly, + waitSeconds: options.waitSeconds, + waitForSelector: options.waitForSelector, + waitForText: options.waitForText, + verbose: options.verbose + ) + } + + /// Fetch a web page using individual parameters. Equivalent to passing a + /// ``FetchOptions`` value; this overload keeps existing call sites + /// working without adapter code. + public static func fetch( + from url: URL, + timeout: TimeInterval = 30, + extractMainOnly: Bool = false, + waitSeconds: TimeInterval = 0, + waitForSelector: String? = nil, + waitForText: String? = nil, + verbose: Bool = false + ) async throws -> FetchedPage { os_log( .info, log: log, @@ -55,7 +142,12 @@ public enum WebPageFetcher { let delegate = NavigationDelegate( webView: webView, state: state, - timeout: timeout + timeout: timeout, + extractMainOnly: extractMainOnly, + waitSeconds: waitSeconds, + waitForSelector: waitForSelector, + waitForText: waitForText, + verbose: verbose ) state.access { state in @@ -78,32 +170,6 @@ public enum WebPageFetcher { } }, onCancel: { - // NOTE: Even though `isolation: MainActor.shared` is specified - // below, neither `operation` nor `onCancel` are called - // on `MainActor` if they weren't already running on - // `MainActor`. - // - // I'm no Swift Foundation engineer, but it doesn't seem - // like `isolation` is used anywhere in the current version - // of `withTaskCancellationHandler()`: - // - // ``` - // public func withTaskCancellationHandler<T>( - // operation: () async throws -> T, - // onCancel handler: @Sendable () -> Void, - // isolation: isolated (any Actor)? = #isolation - // ) async rethrows -> T { - // // unconditionally add the cancellation record to the task. - // // if the task was already cancelled, it will be executed right away. - // let record = unsafe _taskAddCancellationHandler(handler: handler) - // defer { unsafe _taskRemoveCancellationHandler(record: record) } - // - // - // return try await operation() - // } - // ``` - // - // https://github.com/swiftlang/swift/blob/5d480ef063859a0f459f4149df536db4fb330a50/stdlib/public/Concurrency/TaskCancellation.swift#L73-L84 Task { @MainActor in state.access { $0.cancel() } } @@ -111,21 +177,113 @@ public enum WebPageFetcher { isolation: MainActor.shared ) } + + /// Backwards-compatible wrapper returning just HTML. + public static func fetchHTML( + from url: URL, + timeout: TimeInterval = 30 + ) async throws -> String { + try await fetch(from: url, timeout: timeout).html + } } +/// Async JS run via `callAsyncJavaScript` after the page reports `didFinish`. +/// Optionally waits a fixed number of seconds, then for a selector to appear, +/// then for body text to contain a substring (event-driven via +/// `MutationObserver` — no polling). Optionally strips common chrome from the +/// DOM. Always returns `document.documentElement.outerHTML`. +/// +/// All `waitFor*` operations are bounded by the outer Swift-side `timeout`, +/// which fails the fetch if the JS never resolves. +private let extractJS: String = #""" +if (typeof waitSeconds === "number" && waitSeconds > 0) { + await new Promise(function(r) { setTimeout(r, waitSeconds * 1000); }); +} + +if (typeof waitForSelector === "string" && waitForSelector.length > 0) { + await new Promise(function(resolve) { + function check() { + try { return document.querySelector(waitForSelector); } + catch (e) { return null; } + } + if (check()) { resolve(); return; } + var obs = new MutationObserver(function() { + if (check()) { obs.disconnect(); resolve(); } + }); + obs.observe(document.documentElement, { attributes: true, childList: true, subtree: true }); + }); +} + +if (typeof waitForText === "string" && waitForText.length > 0) { + await new Promise(function(resolve) { + function hasText() { + var body = document.body; + if (!body) return false; + var t = body.innerText || body.textContent || ""; + return t.indexOf(waitForText) !== -1; + } + if (hasText()) { resolve(); return; } + var target = document.body || document.documentElement; + var obs = new MutationObserver(function() { + if (hasText()) { obs.disconnect(); resolve(); } + }); + obs.observe(target, { childList: true, subtree: true, characterData: true }); + }); +} + +if (extractMainOnly) { + var selectors = [ + "nav", "header", "footer", "aside", + '[role="banner"]', '[role="contentinfo"]', '[role="navigation"]', + '[role="complementary"]', + '[id*="cookie" i]', '[class*="cookie" i]', + '[id*="consent" i]', '[class*="consent" i]', + '[id*="newsletter" i]', '[class*="newsletter" i]', + '[id*="subscribe" i]', '[class*="subscribe" i]', + '[class*="related" i]', '[class*="recommend" i]', + '[class*="sidebar" i]', '[id*="sidebar" i]', + '[id*="banner" i]', '[class*="promo" i]', + "noscript", "script[src]", "style" + ]; + selectors.forEach(function(sel) { + try { + document.querySelectorAll(sel).forEach(function(el) { el.remove(); }); + } catch (e) {} + }); +} + +return document.documentElement.outerHTML; +"""# + @MainActor private final class NavigationDelegate: NSObject, WKNavigationDelegate { let webView: WKWebView let state: Locked<State> + let extractMainOnly: Bool + let waitSeconds: TimeInterval + let waitForSelector: String? + let waitForText: String? + let verbose: Bool var timeoutTask: Task<Void, Never>? + var capturedStatusCode: Int? init( webView: WKWebView, state: Locked<State>, - timeout: TimeInterval + timeout: TimeInterval, + extractMainOnly: Bool, + waitSeconds: TimeInterval, + waitForSelector: String?, + waitForText: String?, + verbose: Bool ) { self.webView = webView self.state = state + self.extractMainOnly = extractMainOnly + self.waitSeconds = waitSeconds + self.waitForSelector = waitForSelector + self.waitForText = waitForText + self.verbose = verbose super.init() timeoutTask = Task { @MainActor in @@ -143,6 +301,24 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { } } + func webView( + _: WKWebView, + decidePolicyFor navigationResponse: WKNavigationResponse + ) async -> WKNavigationResponsePolicy { + if navigationResponse.isForMainFrame, + let httpResponse = navigationResponse.response as? HTTPURLResponse + { + capturedStatusCode = httpResponse.statusCode + os_log( + .info, + log: log, + "🔧TOOLCALL🔧 WebPageFetcher: Response status: %d", + httpResponse.statusCode + ) + } + return .allow + } + func webView(_ webView: WKWebView, didFinish _: WKNavigation!) { guard state.access({ $0.shouldLoadJavaScript }) else { return @@ -150,25 +326,28 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { os_log(.info, log: log, "🔧TOOLCALL🔧 WebPageFetcher: Page loaded, extracting HTML") - webView - .evaluateJavaScript("document.documentElement.outerHTML") { [weak self] result, error in - guard let self else { return } + let fallbackURL = webView.url + let status = capturedStatusCode + + let arguments: [String: Any] = [ + "waitSeconds": waitSeconds, + "waitForSelector": waitForSelector ?? NSNull(), + "waitForText": waitForText ?? NSNull(), + "extractMainOnly": extractMainOnly, + ] + + Task { @MainActor [weak self] in + guard let self else { return } + do { + let result = try await webView.callAsyncJavaScript( + extractJS, + arguments: arguments, + contentWorld: .page + ) timeoutTask?.cancel() - if let error, state.access({ $0.fail(with: error) }) { - os_log( - .error, - log: log, - "🔧TOOLCALL🔧 WebPageFetcher: JavaScript error: %{public}s", - error.localizedDescription - ) - return - } - - guard let html = result as? String, - state.access({ $0.finish(with: html) }) - else { + guard let html = result as? String else { if state.access({ $0.fail(with: WebPageFetcher.Error.noHTML) }) { os_log( .error, @@ -179,13 +358,28 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { return } + let resolved = webView.url ?? fallbackURL ?? URL(string: "about:blank")! + let page = FetchedPage(html: html, statusCode: status, finalURL: resolved) + + _ = state.access { $0.finish(with: page) } + os_log( .info, log: log, "🔧TOOLCALL🔧 WebPageFetcher: Extracted HTML of length: %d", html.count ) + } catch { + if state.access({ $0.fail(with: error) }) { + os_log( + .error, + log: log, + "🔧TOOLCALL🔧 WebPageFetcher: JavaScript error: %{public}s", + error.localizedDescription + ) + } } + } } func webView(_: WKWebView, didFail _: WKNavigation!, withError error: Swift.Error) { @@ -228,7 +422,10 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { ) async -> WKNavigationActionPolicy { - if let url = navigationAction.request.url { + // Per-navigation events are noisy on chrome-heavy pages (every iframe, + // every about:blank initial state, every redirect fires this). Off by + // default; opt in via FetchOptions.verbose / CLI --verbose. + if verbose, let url = navigationAction.request.url { os_log( .info, log: log, @@ -240,7 +437,7 @@ private final class NavigationDelegate: NSObject, WKNavigationDelegate { } } -private typealias FetchContinuation = CheckedContinuation<String, Swift.Error> +private typealias FetchContinuation = CheckedContinuation<FetchedPage, Swift.Error> private enum State: Sendable { case initial case inProgress(WKWebView, NavigationDelegate, FetchContinuation) @@ -313,7 +510,7 @@ private enum State: Sendable { } } - mutating func finish(with html: String) -> Bool { + mutating func finish(with page: FetchedPage) -> Bool { MainActor.assertIsolated() switch self { case .initial: @@ -323,7 +520,7 @@ private enum State: Sendable { case let .inProgress(_, _, continuation): self = .terminal - continuation.resume(returning: html) + continuation.resume(returning: page) return true case .terminal: @@ -332,7 +529,7 @@ private enum State: Sendable { case let .waitingForWebView(continuation): assertionFailure() self = .terminal - continuation.resume(returning: html) + continuation.resume(returning: page) return true } } diff --git a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift index 50b1603..a9555b2 100644 --- a/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift +++ b/Sources/WebToMarkdownCLI/WebToMarkdownCommand.swift @@ -18,23 +18,84 @@ struct WebToMarkdownCommand: AsyncParsableCommand { @Flag(name: .shortAndLong, help: "Output verbose logging") var verbose: Bool = false + @Flag( + name: .long, + help: "Strip page chrome (nav, header, footer, cookies, sidebars) before conversion" + ) + var main: Bool = false + + @Flag( + name: .long, + help: "Output only the frontmatter block (status, final-url, title, description, fetched-at) — skip the body" + ) + var head: Bool = false + + @Flag( + name: .long, + help: "Suppress the frontmatter block (output the body markdown only)" + ) + var skipFrontmatter: Bool = false + + @Option( + name: .long, + help: "Wait this many seconds (decimal allowed) after the page loads, before extracting. Bounded by --timeout." + ) + var wait: Double = 0 + + @Option( + name: .long, + help: "Wait until at least one element matches this CSS selector before extracting. MutationObserver-driven, bounded by --timeout." + ) + var waitFor: String? + + @Option( + name: .long, + help: "Wait until the body's visible text contains this substring before extracting. MutationObserver-driven, bounded by --timeout." + ) + var waitForText: String? + + func validate() throws { + if head, skipFrontmatter { + throw ValidationError( + "--head and --skip-frontmatter are mutually exclusive: --head prints only the frontmatter, --skip-frontmatter prints only the body. Pick one." + ) + } + } + mutating func run() async throws { - guard let url = URL(string: url) else { + guard let parsedURL = URL(string: url) else { throw ValidationError("Invalid URL: \(url)") } if verbose { - fputs("Fetching \(url.absoluteString)...\n", stderr) + fputs("Fetching \(parsedURL.absoluteString)...\n", stderr) } - let html = try await WebPageFetcher.fetchHTML(from: url, timeout: timeout) + let options = FetchOptions( + timeout: timeout, + extractMainOnly: main, + waitSeconds: wait, + waitForSelector: waitFor, + waitForText: waitForText, + verbose: verbose + ) + + let page = try await WebPageFetcher.fetch(from: parsedURL, options: options) + + if !skipFrontmatter { + let metadata = (try? HTMLToMarkdown.extractMetadata(page.html)) + ?? PageMetadata(title: nil, description: nil) + print(Frontmatter(page: page, metadata: metadata).format()) + if !head { print("") } + } + + if head { return } if verbose { fputs("Converting to markdown...\n", stderr) } - let markdown = try HTMLToMarkdown.convert(html, baseURL: url) - + let markdown = try HTMLToMarkdown.convert(page.html, baseURL: page.finalURL) print(markdown) } } diff --git a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift index f4a231c..79834a3 100644 --- a/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift +++ b/Tests/WebToMarkdownTests/HTMLToMarkdownTests.swift @@ -2,6 +2,202 @@ import Foundation import Testing @testable import WebToMarkdown +@Suite +struct FrontmatterTests { + private func makePage( + url: String = "https://example.com/", + status: Int? = 200, + html: String = "<html></html>" + ) -> FetchedPage { + FetchedPage(html: html, statusCode: status, finalURL: URL(string: url)!) + } + + private let fixedDate = Date(timeIntervalSince1970: 1_700_000_000) // 2023-11-14T22:13:20Z + + @Test + func formatsAllFields() { + let page = makePage() + let meta = PageMetadata(title: "Hello", description: "A description.") + let fm = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate) + let out = fm.format() + #expect(out.hasPrefix("---\n")) + #expect(out.hasSuffix("\n---")) + #expect(out.contains("status: 200")) + // URL contains ':' so it gets quoted. + #expect(out.contains("final-url: \"https://example.com/\"")) + #expect(out.contains("title: Hello")) + #expect(out.contains("description: A description.")) + // ISO8601 timestamp contains ':' so it gets quoted. + #expect(out.contains("fetched-at: \"2023-11-14T22:13:20Z\"")) + } + + @Test + func omitsAbsentMetadata() { + let page = makePage() + let meta = PageMetadata(title: nil, description: nil) + let out = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate).format() + #expect(!out.contains("title:")) + #expect(!out.contains("description:")) + } + + @Test + func omitsAbsentStatus() { + let page = makePage(status: nil) + let meta = PageMetadata(title: "x", description: nil) + let out = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate).format() + #expect(!out.contains("status:")) + } + + @Test + func quotesValuesWithColons() { + let escaped = Frontmatter.yamlEscape("Foo: Bar") + #expect(escaped == "\"Foo: Bar\"") + } + + @Test + func quotesValuesWithQuotes() { + let escaped = Frontmatter.yamlEscape("She said \"hi\"") + #expect(escaped == "\"She said \\\"hi\\\"\"") + } + + @Test + func quotesValuesWithBackslash() { + let escaped = Frontmatter.yamlEscape("path\\to") + #expect(escaped == "\"path\\\\to\"") + } + + @Test + func quotesValuesWithHash() { + let escaped = Frontmatter.yamlEscape("foo #bar") + #expect(escaped == "\"foo #bar\"") + } + + @Test + func quotesValuesWithLeadingYamlIndicators() { + #expect(Frontmatter.yamlEscape("[Draft]") == "\"[Draft]\"") + #expect(Frontmatter.yamlEscape("*alias") == "\"*alias\"") + #expect(Frontmatter.yamlEscape("> block") == "\"> block\"") + } + + @Test + func quotesEmptyAndWhitespacePadded() { + #expect(Frontmatter.yamlEscape("") == "\"\"") + #expect(Frontmatter.yamlEscape(" leading") == "\" leading\"") + #expect(Frontmatter.yamlEscape("trailing ") == "\"trailing \"") + } + + @Test + func flattensNewlinesInValues() { + let escaped = Frontmatter.yamlEscape("line one\nline two") + // No colon/quote/backslash/hash, so no quoting; just a flattened space. + #expect(escaped == "line one line two") + } + + @Test + func leavesPlainValuesAlone() { + #expect(Frontmatter.yamlEscape("Hello world") == "Hello world") + #expect(Frontmatter.yamlEscape("plain-value-123") == "plain-value-123") + } + + @Test + func truncatesLongDescription() { + let longDesc = String(repeating: "a", count: 250) + let page = makePage() + let meta = PageMetadata(title: nil, description: longDesc) + let out = Frontmatter(page: page, metadata: meta, fetchedAt: fixedDate).format() + // Description should be exactly 200 chars + "…" (default max). + #expect(out.contains("description: " + String(repeating: "a", count: 200) + "…")) + } +} + +@Suite +struct PageMetadataTests { + @Test + func extractsTitleFromTitleTag() throws { + let html = "<html><head><title>My Pagex" + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "My Page") + } + + @Test + func fallsBackToOgTitleWhenNoTitleTag() throws { + let html = """ + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "OG Title Here") + } + + @Test + func prefersTitleTagOverOgTitle() throws { + let html = """ + + Real Title + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "Real Title") + } + + @Test + func extractsMetaDescription() throws { + let html = """ + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.description == "A nice description.") + } + + @Test + func fallsBackToOgDescription() throws { + let html = """ + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.description == "OG description fallback.") + } + + @Test + func returnsNilWhenMetadataMissing() throws { + let html = "just body" + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == nil) + #expect(meta.description == nil) + } + + @Test + func trimsWhitespaceInExtractedValues() throws { + let html = """ + + Spaced Title + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "Spaced Title") + #expect(meta.description == "spaced description") + } + + @Test + func ignoresEmptyTitleTag() throws { + let html = """ + + + + x + """ + let meta = try HTMLToMarkdown.extractMetadata(html) + #expect(meta.title == "OG Backup") + } +} + @Suite struct HTMLToMarkdownTests { @Test diff --git a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift index c8d1e3b..42763f1 100644 --- a/Tests/WebToMarkdownTests/WebPageFetcherTests.swift +++ b/Tests/WebToMarkdownTests/WebPageFetcherTests.swift @@ -27,6 +27,87 @@ struct WebPageFetcherTests { } } + @Test + func fetchReturnsStatusAndFinalURL() async throws { + let url = URL(string: "https://example.com")! + let page = try await WebPageFetcher.fetch(from: url) + #expect(page.statusCode == 200) + #expect(page.finalURL.host == "example.com") + #expect(!page.html.isEmpty) + } + + @Test + func waitForExistingSelectorReturnsImmediately() async throws { + let url = URL(string: "https://example.com")! + let timeout: TimeInterval = 10 + let start = Date() + let page = try await WebPageFetcher.fetch( + from: url, + timeout: timeout, + waitForSelector: "h1" + ) + let elapsed = Date().timeIntervalSince(start) + #expect(!page.html.isEmpty) + #expect( + elapsed < timeout - 1, + "Selector that already exists should complete well before the configured timeout" + ) + } + + @Test + func waitForNonexistentSelectorTimesOut() async throws { + let url = URL(string: "https://example.com")! + await #expect(throws: WebPageFetcher.Error.self) { + try await WebPageFetcher.fetch( + from: url, + timeout: 3, + waitForSelector: ".never-matches-xyz-12345" + ) + } + } + + @Test + func waitForSelectorHandlesAttributeMutation() async throws { + let url = dataURL( + """ + + +
Loading
+ + + + """ + ) + + let page = try await WebPageFetcher.fetch( + from: url, + timeout: 5, + waitForSelector: "[data-ready='true']" + ) + + #expect(page.html.contains("data-ready=\"true\"")) + } + + @Test + func fixedWaitDelaysExtraction() async throws { + let url = URL(string: "https://example.com")! + let start = Date() + _ = try await WebPageFetcher.fetch( + from: url, + timeout: 10, + waitSeconds: 1.0 + ) + let elapsed = Date().timeIntervalSince(start) + #expect( + elapsed >= 0.9, + "Fixed wait should add approximately its duration (0.1s tolerance for timer precision)" + ) + } + @Test func canCancelFetch() async throws { let url = URL(string: "https://example.com")! @@ -55,3 +136,8 @@ private func yield(_ times: Int) async { await Task.yield() } } + +private func dataURL(_ html: String) -> URL { + let encoded = Data(html.utf8).base64EncodedString() + return URL(string: "data:text/html;charset=utf-8;base64,\(encoded)")! +} diff --git a/bin/install.sh b/bin/install.sh index 262b44c..002bc34 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -9,7 +9,18 @@ if [[ "${TRACE-0}" == "1" ]]; then fi if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then - echo 'Usage: ./install.sh' + cat <<'EOF' +Usage: ./install.sh + +Builds web-to-markdown in release mode and copies the binary into a +directory on PATH. Picks the first writable destination from: + 1. $INSTALL_DIR (override) + 2. $(brew --prefix)/bin (if Homebrew is installed) + 3. /opt/homebrew/bin (Apple Silicon Homebrew default) + 4. /usr/local/bin (Intel Homebrew / classic default; usually requires sudo) + +If none are writable, runs with sudo or fails with a clear error. +EOF exit fi @@ -18,8 +29,51 @@ pushd "$DIR/.." &>/dev/null swift build -c release -cp .build/release/web-to-markdown /usr/local/bin/ +# Build the candidate list in priority order. +candidates=() + +if [[ -n "${INSTALL_DIR-}" ]]; then + candidates+=("$INSTALL_DIR") +fi + +if command -v brew >/dev/null 2>&1; then + brew_prefix="$(brew --prefix 2>/dev/null || true)" + if [[ -n "$brew_prefix" ]]; then + candidates+=("$brew_prefix/bin") + fi +fi + +candidates+=("/opt/homebrew/bin" "/usr/local/bin") + +# Deduplicate while preserving order. +seen="" +unique_candidates=() +for c in "${candidates[@]}"; do + if [[ ":$seen:" != *":$c:"* ]]; then + unique_candidates+=("$c") + seen="$seen:$c" + fi +done + +# Pick the first existing & writable candidate. +dest="" +for candidate in "${unique_candidates[@]}"; do + if [[ -d "$candidate" ]] && [[ -w "$candidate" ]]; then + dest="$candidate" + break + fi +done + +if [[ -z "$dest" ]]; then + echo "Error: no writable directory found among:" >&2 + printf ' - %s\n' "${unique_candidates[@]}" >&2 + echo "" >&2 + echo "Re-run with sudo, or set INSTALL_DIR= to a writable directory." >&2 + exit 1 +fi + +cp .build/release/web-to-markdown "$dest/" -echo "$(ls /usr/local/bin/web-to-markdown)" +echo "Installed: $dest/web-to-markdown" popd &>/dev/null