Cli improvements + frontmatter

README.md

@@ -15,16 +15,86 @@ web-to-markdown https://example.com
 web-to-markdown https://example.com --timeout 30 --verbose
 ```
 
+By default the output is a YAML frontmatter block followed by the body
+markdown:
+
+```
+---
+status: 200
+final-url: "https://example.com/"
+title: Example Domain
+fetched-at: "2026-04-29T21:42:56Z"
+---
+
+# Example Domain
+…
+```
+
+Values containing `:` (URLs, ISO timestamps) are double-quoted so the output
+parses cleanly as YAML.
+
+### Flags
+
+- `--main` — strip page chrome (nav, header, footer, cookies, sidebars,
+  recommendations) before conversion. Useful for noisy product/listing pages.
+- `--head` — print only the frontmatter block, skip the body. Fast URL
+  validation in batch.
+- `--skip-frontmatter` — output the body markdown only.
+- `--wait <seconds>` — fixed delay (decimal allowed) after the page loads.
+  Useful for SPAs that hydrate after `window.load`.
+- `--wait-for <css-selector>` — wait until at least one element matches.
+  Event-driven via `MutationObserver`, no polling. Bounded by `--timeout`.
+- `--wait-for-text <substring>` — wait until the body's visible text contains
+  the substring. Same `MutationObserver` pattern.
+- `--timeout <seconds>` — overall fetch timeout (default 30s). Bounds all
+  wait flags.
+
+### Example: YouTube watch pages
+
+YouTube hydrates async after `window.load`, so a plain fetch grabs the app
+shell with `title: YouTube`. Wait for the subscribe button to render — it
+appears only after page content is real:
+
+```bash
+web-to-markdown 'https://www.youtube.com/watch?v=...' \
+  --wait-for '#subscribe-button' --timeout 20
+```
+
+Same idea works for other JS-heavy sites: find a selector or piece of body
+text that only appears after meaningful hydration, and use it as the anchor.
+
 ## Library
 
 ```swift
 import WebToMarkdown
 
-// Fetch HTML from a URL using WKWebView
-let html = try await WebPageFetcher.fetchHTML(from: url, timeout: 30)
+// Fetch a page (defaults to a 30s timeout, no waits, no chrome stripping).
+let page = try await WebPageFetcher.fetch(from: url)
+
+// Same fetch with all knobs available, via FetchOptions.
+var options = FetchOptions()
+options.extractMainOnly = true
+options.waitSeconds = 0.5
+options.waitForSelector = "#subscribe-button"
+options.timeout = 20
+let richPage = try await WebPageFetcher.fetch(from: url, options: options)
 
-// Convert HTML to Markdown
-let markdown = try HTMLToMarkdown.convert(html, baseURL: url)
+// page.html, page.statusCode, page.finalURL — all on FetchedPage.
+print(page.statusCode ?? -1, page.finalURL)
+
+// Convert HTML to Markdown.
+let markdown = try HTMLToMarkdown.convert(page.html, baseURL: page.finalURL)
+
+// Extract <title> and meta description as Swift values.
+let metadata = try HTMLToMarkdown.extractMetadata(page.html)
+print(metadata.title ?? "", metadata.description ?? "")
+
+// Build the same YAML frontmatter the CLI emits.
+let frontmatter = Frontmatter(page: page, metadata: metadata).format()
+print(frontmatter)
+
+// Backwards-compatible HTML-only fetch.
+let html = try await WebPageFetcher.fetchHTML(from: url, timeout: 30)
 ```
 
 ## Testing

Sources/WebToMarkdown/Frontmatter.swift

@@ -0,0 +1,91 @@
+import Foundation
+
+/// Renders the standard YAML frontmatter block (`---` … `---`) used by the
+/// `web-to-markdown` CLI. The struct holds the raw inputs as Swift types —
+/// consumers who want the data without the formatting can just read the
+/// stored properties or build their own representation. We emit YAML
+/// directly without pulling in a YAML library; the output parses as real
+/// YAML on the consumer side.
+public struct Frontmatter: Sendable {
+    public let page: FetchedPage
+    public let metadata: PageMetadata
+    public let fetchedAt: Date
+
+    /// Maximum length of the rendered `description` field (longer values are
+    /// truncated with an ellipsis). The default keeps frontmatter scannable.
+    public var descriptionMaxLength: Int = 200
+
+    public init(
+        page: FetchedPage,
+        metadata: PageMetadata,
+        fetchedAt: Date = Date(),
+        descriptionMaxLength: Int = 200
+    ) {
+        self.page = page
+        self.metadata = metadata
+        self.fetchedAt = fetchedAt
+        self.descriptionMaxLength = descriptionMaxLength
+    }
+
+    /// Format as a YAML block: `---` line, one `key: value` per line,
+    /// closing `---`. Values containing characters that would confuse a YAML
+    /// reader (`:`, `"`, `\\`, `#`, leading/trailing whitespace) are
+    /// double-quoted with `\\` and `"` escaped. Newlines in values are
+    /// flattened to spaces.
+    public func format() -> String {
+        var lines = ["---"]
+        if let status = page.statusCode {
+            lines.append("status: \(status)")
+        }
+        lines.append("final-url: \(yamlEscape(page.finalURL.absoluteString))")
+        if let title = metadata.title, !title.isEmpty {
+            lines.append("title: \(yamlEscape(title))")
+        }
+        if let description = metadata.description, !description.isEmpty {
+            lines.append(
+                "description: \(yamlEscape(truncate(description, max: descriptionMaxLength)))"
+            )
+        }
+        let formatter = ISO8601DateFormatter()
+        formatter.formatOptions = [.withInternetDateTime]
+        lines.append("fetched-at: \(yamlEscape(formatter.string(from: fetchedAt)))")
+        lines.append("---")
+        return lines.joined(separator: "\n")
+    }
+
+    /// Public for unit tests and library consumers who want to format their
+    /// own values with the same escape rules.
+    public static func yamlEscape(_ value: String) -> String {
+        let flattened = value
+            .replacingOccurrences(of: "\r\n", with: " ")
+            .replacingOccurrences(of: "\n", with: " ")
+            .replacingOccurrences(of: "\r", with: " ")
+
+        let needsQuoting = flattened.contains(":")
+            || flattened.contains("\"")
+            || flattened.contains("\\")
+            || flattened.contains("#")
+            || flattened.first.map { "-?:,[]{}#&*!|>'\"%@`".contains($0) } == true
+            || flattened.first == " "
+            || flattened.last == " "
+            || flattened.isEmpty
+
+        if needsQuoting {
+            let escaped = flattened
+                .replacingOccurrences(of: "\\", with: "\\\\")
+                .replacingOccurrences(of: "\"", with: "\\\"")
+            return "\"\(escaped)\""
+        }
+        return flattened
+    }
+
+    private func yamlEscape(_ value: String) -> String {
+        Self.yamlEscape(value)
+    }
+
+    private func truncate(_ s: String, max: Int) -> String {
+        if s.count <= max { return s }
+        let end = s.index(s.startIndex, offsetBy: max)
+        return String(s[s.startIndex ..< end]) + "…"
+    }
+}

Sources/WebToMarkdown/HTMLToMarkdown.swift

@@ -1,6 +1,16 @@
 import Foundation
 import SwiftSoup
 
+public struct PageMetadata: Sendable {
+    public let title: String?
+    public let description: String?
+
+    public init(title: String?, description: String?) {
+        self.title = title
+        self.description = description
+    }
+}
+
 public enum HTMLToMarkdown {
     public enum Error: Swift.Error {
         case parsingFailed(String)
@@ -19,6 +29,41 @@ public enum HTMLToMarkdown {
         return collapseExcessiveNewlines(markdown)
     }
 
+    /// Extract the page `<title>` and meta `description` (falling back to
+    /// Open Graph variants when standard tags are missing).
+    public static func extractMetadata(_ html: String) throws -> PageMetadata {
+        let document = try SwiftSoup.parse(html)
+
+        let title: String? = try {
+            if let el = try document.select("title").first() {
+                let text = try el.text().trimmingCharacters(in: .whitespacesAndNewlines)
+                if !text.isEmpty { return text }
+            }
+            if let el = try document.select("meta[property=og:title]").first() {
+                let text = try el.attr("content")
+                    .trimmingCharacters(in: .whitespacesAndNewlines)
+                if !text.isEmpty { return text }
+            }
+            return nil
+        }()
+
+        let description: String? = try {
+            if let el = try document.select("meta[name=description]").first() {
+                let text = try el.attr("content")
+                    .trimmingCharacters(in: .whitespacesAndNewlines)
+                if !text.isEmpty { return text }
+            }
+            if let el = try document.select("meta[property=og:description]").first() {
+                let text = try el.attr("content")
+                    .trimmingCharacters(in: .whitespacesAndNewlines)
+                if !text.isEmpty { return text }
+            }
+            return nil
+        }()
+
+        return PageMetadata(title: title, description: description)
+    }
+
     private static func collapseExcessiveNewlines(_ text: String) -> String {
         var result = text
         while result.contains("\n\n\n") {