diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py index 3adcf9be..53f0a6ac 100644 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py +++ b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_backends.py @@ -244,7 +244,7 @@ def test_default_chain_cloakbrowser_is_primary(monkeypatch): with build_default_fetcher(config) as fetcher: names = [b.name for b in fetcher._tiered.backends] # Note: exactly one browser tier (cloakbrowser), not vanilla + cloak. - assert names == ["cloakbrowser", "pdf", "hyperbrowser", "firecrawl"] + assert names == ["cloakbrowser", "pdf", "firecrawl", "hyperbrowser"] def test_default_chain_falls_back_to_playwright_and_skips_unkeyed(monkeypatch): diff --git a/forecasting_tools/agents_and_tools/source_archive/README.md b/forecasting_tools/agents_and_tools/source_archive/README.md index 9c815175..b9eee9f5 100644 --- a/forecasting_tools/agents_and_tools/source_archive/README.md +++ b/forecasting_tools/agents_and_tools/source_archive/README.md @@ -248,7 +248,7 @@ query). | `config.py` | Environment-driven `ArchiveConfig` | | `models.py` | `CaptureResult`, `StoredCapture`, `CitationRecord` | | `ingest/` | Build a manifest: URL extraction from text + traced bot runs | -| `fetchers/` | Playwright (primary) + CloakBrowser / Hyperbrowser / Firecrawl / PDF backups, tiered orchestrator | +| `fetchers/` | Playwright (primary) + CloakBrowser / PDF / Firecrawl / Hyperbrowser backups, tiered orchestrator | | `benchmark.py` | Backend bake-off: reliability + cost per backend over a manifest | | `quality.py` | Reject 404s, block pages, and thin content before archiving | | `storage/` | `BlobStore` interface with S3 and local backends | diff --git a/forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py b/forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py index b136ea66..e41b581d 100644 --- a/forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py +++ b/forecasting_tools/agents_and_tools/source_archive/fetchers/__init__.py @@ -2,7 +2,7 @@ Most callers want :func:`build_default_fetcher`, which wires the recommended cost-ordered tiered setup: self-hosted Playwright primary, then CloakBrowser, -PDF, Hyperbrowser, and Firecrawl backups. +PDF, Firecrawl, and Hyperbrowser backups. """ from __future__ import annotations @@ -69,10 +69,10 @@ def build_default_fetcher(config: ArchiveConfig | None = None) -> PlaywrightFetc single process, so cloak *replaces* vanilla rather than stacking with it. 2. **PdfFetcher** (local, free; Firecrawl OCR fallback) — captures PDFs, which the browsers can't render. - 3. **Hyperbrowser** (managed) — consolidated anti-bot fallback. Added when - ``HYPERBROWSER_API_KEY`` is set. - 4. **Firecrawl** (managed) — cheapest stealth + native-PDF safety net. Added - when ``FIRECRAWL_API_KEY`` is set. + 3. **Firecrawl** (managed) — cheapest stealth + native-PDF safety net + (~$0.0042/page stealth). Added when ``FIRECRAWL_API_KEY`` is set. + 4. **Hyperbrowser** (managed) — anti-bot fallback of last resort (~$0.01/page + with proxy, plus bandwidth). Added when ``HYPERBROWSER_API_KEY`` is set. The returned object is a :class:`PlaywrightFetcher` subclass so the single browser's lifecycle is managed by ``with``. @@ -111,10 +111,10 @@ def __enter__(self) -> "_ManagedTieredFetcher": # key is present). Cheap to keep in the chain unconditionally. backends.append(PdfFetcher(self.config)) - if self.config.hyperbrowser_api_key: - backends.append(HyperbrowserFetcher(self.config)) if self.config.firecrawl_api_key: backends.append(FirecrawlFetcher(self.config)) + if self.config.hyperbrowser_api_key: + backends.append(HyperbrowserFetcher(self.config)) self._tiered = TieredFetcher(*backends) return self