diff --git a/spiders/sitemap_spider.py b/spiders/sitemap_spider.py index 5f9e038..b244c63 100644 --- a/spiders/sitemap_spider.py +++ b/spiders/sitemap_spider.py @@ -1,7 +1,7 @@ from scrapy.spiders import SitemapSpider from core.db import get_db from core.models import Spider -from .base import BaseDBSpiderMixin +from .base import BaseDBSpiderMixin, _pdf_links from dateutil import parser as dateutil_parser from datetime import datetime, timedelta from urllib.parse import urljoin, urlparse @@ -122,6 +122,12 @@ async def parse_article(self, response): response, source_label="sitemap_spider" ): yield item + # links_only: record PDF links found on this page as URL-only items + # (no download) — mirrors database_spider.parse_article, so PDF + # collection works for sitemap-driven spiders too. + if self._pdf_mode() == "links_only": + for purl in _pdf_links(response): + yield self._url_only_pdf_item(purl, response, "sitemap_spider") def _parse_since_date(self): """Parse SITEMAP_SINCE setting into a datetime. diff --git a/tests/unit/test_pdf_collection.py b/tests/unit/test_pdf_collection.py index 2ae5bd8..06c5f4c 100644 --- a/tests/unit/test_pdf_collection.py +++ b/tests/unit/test_pdf_collection.py @@ -193,3 +193,49 @@ def test_clean_pdf_text_normalizes_carriage_returns(): def test_extract_pdf_text_empty_on_non_pdf_or_empty_bytes(): assert _extract_pdf_text(b"this is not a pdf") == "" assert _extract_pdf_text(b"") == "" + + +# --- sitemap spiders collect PDF links too ----------------------------------- + + +async def test_sitemap_links_only_yields_url_only_items(monkeypatch): + """PDF collection must work for sitemap-driven spiders, not just rule-based + ones — sitemap fleets would otherwise harvest no PDFs at all.""" + from unittest.mock import patch as _patch + + from spiders.sitemap_spider import SitemapDatabaseSpider + + rec = Mock(spec=Spider) + rec.name = "x_com" + rec.active = True + rec.allowed_domains = ["x.com"] + rec.start_urls = ["https://x.com/sitemap.xml"] + rec.rules = [] + rec.callbacks_config = {} + rec.settings = [] + rec.id = 7 + with _patch("spiders.sitemap_spider.get_db") as mock_get_db: + db = Mock() + db.query.return_value.filter.return_value.first.return_value = rec + cm = MagicMock() + cm.__enter__.return_value = db + mock_get_db.return_value = cm + spider = SitemapDatabaseSpider(spider_name="x_com") + spider.custom_settings = {"PDF_MODE": "links_only"} + + async def _no_article(*a, **k): + for _ in []: + yield + + monkeypatch.setattr(spider, "_extract_article", _no_article) + html = 'AB' + resp = HtmlResponse( + url="https://x.com/article", body=html.encode(), encoding="utf-8" + ) + items = [i async for i in spider.parse_article(resp)] + urls = {i["url"] for i in items} + assert urls == {"https://x.com/docs/a.pdf", "https://ext.org/b.pdf"} + assert all(i["content"] == "" for i in items) + assert all(i["metadata_json"]["content_type"] == "pdf" for i in items) + assert all(i["metadata_json"]["found_on"] == "https://x.com/article" for i in items) + assert all(i["source"] == "sitemap_spider" for i in items)