fix: rewrite extractor for the 2026 layout drift caught by preflight

src/nhs_scraper/pipeline/extract.py

@@ -4,33 +4,32 @@
 returns validated ``WaitingTimeRecord`` objects. Behaviour is pinned by
 the characterisation fixtures and golden dataset.
 
-Expected page shape (My Planned Care trust page):
-
-- ``h1`` — provider (trust) name
-- ``section.specialty`` blocks, each with an ``h3`` specialty name
-- within a section, ``h4`` headings ("First Outpatient Appointment" /
-  "Treatment") each followed by a table whose data row holds
-  "N weeks" values for average and 8-in-10 waits
-- footer text "Page last updated: DD/MM/YYYY"
+Two layouts are supported. The legacy layout (pre-2026, kept as the
+characterisation baseline) uses ``section.specialty`` blocks with ``h4``
+metric headings. The 2026 layout uses ``div.inner_details_holder`` blocks
+with table ``<caption>`` metric labels and ``n/a`` cells; its footer
+reads "This page was last updated on D Month YYYY". New-layout extraction
+is tried first; a page with no 2026 holders falls back to legacy.
 """
 
 from __future__ import annotations
 
 import re
 from datetime import date, datetime
 
-from bs4 import BeautifulSoup, Tag
+from bs4 import BeautifulSoup
 
 from nhs_scraper.domain import Metric, Page, WaitingTimeRecord
 
-_METRIC_BY_HEADING = {
+_METRIC_BY_LABEL = {
     "first outpatient appointment": Metric.FIRST_OUTPATIENT_APPOINTMENT,
     "treatment": Metric.TREATMENT,
 }
 
 _WEEKS_PATTERN = re.compile(r"(\d+)\s*weeks?", re.IGNORECASE)
-_LAST_UPDATED_PATTERN = re.compile(
-    r"page last updated:\s*(\d{2}/\d{2}/\d{4})", re.IGNORECASE
+_LEGACY_FOOTER = re.compile(r"page last updated:\s*(\d{2}/\d{2}/\d{4})", re.IGNORECASE)
+_2026_FOOTER = re.compile(
+    r"this page was last updated on\s+(\d{1,2}\s+\w+\s+\d{4})", re.IGNORECASE
 )
 
 
@@ -41,65 +40,136 @@ def _parse_weeks(cell_text: str) -> int | None:
 
 
 def _parse_page_last_updated(soup: BeautifulSoup) -> date | None:
-    match = _LAST_UPDATED_PATTERN.search(soup.get_text(" ", strip=True))
-    if not match:
-        return None
-    return datetime.strptime(match.group(1), "%d/%m/%Y").date()
+    text = soup.get_text(" ", strip=True)
+    if match := _LEGACY_FOOTER.search(text):
+        return datetime.strptime(match.group(1), "%d/%m/%Y").date()
+    if match := _2026_FOOTER.search(text):
+        return datetime.strptime(match.group(1), "%d %B %Y").date()
+    return None
 
 
 def _provider_name(soup: BeautifulSoup) -> str | None:
     heading = soup.find("h1")
     return heading.get_text(strip=True) if heading else None
 
 
-def _table_values(table: Tag) -> tuple[int | None, int | None]:
-    rows = table.find_all("tr")
-    if len(rows) < 2:
-        return None, None
-    cells = [c.get_text(strip=True) for c in rows[1].find_all(["td", "th"])]
-    average = _parse_weeks(cells[0]) if cells else None
-    within = _parse_weeks(cells[1]) if len(cells) > 1 else None
-    return average, within
-
+def _make_record(
+    *, region, provider, specialty, source_url, metric, average, within, updated
+) -> WaitingTimeRecord:
+    return WaitingTimeRecord(
+        region=region,
+        provider=provider,
+        specialty=specialty,
+        source_url=source_url,
+        metric=metric,
+        average_wait_weeks=average,
+        patients_seen_within_weeks=within,
+        page_last_updated=updated,
+    )
+
+
+def _extract_2026(
+    soup: BeautifulSoup, *, region, provider, source_url, updated
+) -> list[WaitingTimeRecord]:
+    """Extract from the 2026 ``div.inner_details_holder`` layout."""
+    records: list[WaitingTimeRecord] = []
+    for holder in soup.find_all("div", class_="inner_details_holder"):
+        heading = holder.find("h3", class_="nhsblue-text0")
+        if heading is None:
+            continue
+        specialty = heading.get_text(strip=True).removesuffix(" - Waiting Times")
+
+        for table in holder.find_all("table", class_="waiting-times-data"):
+            caption = table.find("caption")
+            metric = (
+                _METRIC_BY_LABEL.get(caption.get_text(strip=True).lower())
+                if caption
+                else None
+            )
+            if metric is None:
+                continue
+            average = within = None
+            for row in table.find_all("tr"):
+                th, td = row.find("th"), row.find("td")
+                if th is None or td is None:
+                    continue
+                label = th.get_text(strip=True).lower()
+                value = td.get_text(strip=True)
+                if "average waiting time" in label:
+                    average = _parse_weeks(value)
+                elif "8 in 10 patients" in label:
+                    within = _parse_weeks(value)
+            records.append(
+                _make_record(
+                    region=region,
+                    provider=provider,
+                    specialty=specialty,
+                    source_url=source_url,
+                    metric=metric,
+                    average=average,
+                    within=within,
+                    updated=updated,
+                )
+            )
+    return records
 
-def extract_waiting_times(page: Page, *, region: str) -> list[WaitingTimeRecord]:
-    """Extract every waiting-time record present on a trust ``page``.
 
-    Pure: same input, same output, no side effects. Specialties whose data
-    is unavailable contribute no records; pages without a provider heading
-    are rejected wholesale.
-    """
-    soup = BeautifulSoup(page.html, "html.parser")
-    provider = _provider_name(soup)
-    if provider is None:
-        return []
-
-    last_updated = _parse_page_last_updated(soup)
+def _extract_legacy(
+    soup: BeautifulSoup, *, region, provider, source_url, updated
+) -> list[WaitingTimeRecord]:
+    """Extract from the legacy ``section.specialty`` layout."""
     records: list[WaitingTimeRecord] = []
-
     for section in soup.find_all("section", class_="specialty"):
         specialty_tag = section.find("h3")
         if specialty_tag is None:
             continue
         specialty = specialty_tag.get_text(strip=True)
 
         for heading in section.find_all("h4"):
-            metric = _METRIC_BY_HEADING.get(heading.get_text(strip=True).lower())
+            metric = _METRIC_BY_LABEL.get(heading.get_text(strip=True).lower())
             table = heading.find_next_sibling("table")
             if metric is None or table is None:
                 continue
-            average, within = _table_values(table)
+            rows = table.find_all("tr")
+            cells = (
+                [c.get_text(strip=True) for c in rows[1].find_all(["td", "th"])]
+                if len(rows) > 1
+                else []
+            )
+            average = _parse_weeks(cells[0]) if cells else None
+            within = _parse_weeks(cells[1]) if len(cells) > 1 else None
             records.append(
-                WaitingTimeRecord(
+                _make_record(
                     region=region,
                     provider=provider,
                     specialty=specialty,
-                    source_url=page.url,
+                    source_url=source_url,
                     metric=metric,
-                    average_wait_weeks=average,
-                    patients_seen_within_weeks=within,
-                    page_last_updated=last_updated,
+                    average=average,
+                    within=within,
+                    updated=updated,
                 )
             )
-
     return records
+
+
+def extract_waiting_times(page: Page, *, region: str) -> list[WaitingTimeRecord]:
+    """Extract every waiting-time record present on a trust ``page``.
+
+    Pure: same input, same output, no side effects. Specialties whose data
+    is unavailable contribute no records; pages without a provider heading
+    are rejected wholesale.
+    """
+    soup = BeautifulSoup(page.html, "html.parser")
+    provider = _provider_name(soup)
+    if provider is None:
+        return []
+
+    updated = _parse_page_last_updated(soup)
+    context = dict(
+        region=region,
+        provider=provider,
+        source_url=page.url,
+        updated=updated,
+    )
+    return _extract_2026(soup, **context) or _extract_legacy(soup, **context)

src/nhs_scraper/pipeline/preflight.py

@@ -1,8 +1,8 @@
 """Pre-flight layout probe: detect site structure drift before crawling.
 
 Runs the structural signals the extractor depends on against a single
-canary page, then the extractor itself end-to-end. Far cheaper and more
-diagnosable than discovering an empty CSV after a full crawl.
+canary page, then the extractor itself end-to-end. Both the legacy and
+the 2026 layouts are recognised; a page matching neither fails.
 """
 
 from __future__ import annotations
@@ -15,9 +15,10 @@
 from nhs_scraper.domain import Page
 from nhs_scraper.pipeline.extract import extract_waiting_times
 
-_KNOWN_METRIC_HEADINGS = {"first outpatient appointment", "treatment"}
-_LAST_UPDATED_PATTERN = re.compile(
-    r"page last updated:\s*\d{2}/\d{2}/\d{4}", re.IGNORECASE
+_KNOWN_METRIC_LABELS = {"first outpatient appointment", "treatment"}
+_LEGACY_FOOTER = re.compile(r"page last updated:\s*\d{2}/\d{2}/\d{4}", re.IGNORECASE)
+_2026_FOOTER = re.compile(
+    r"this page was last updated on\s+\d{1,2}\s+\w+\s+\d{4}", re.IGNORECASE
 )
 
 
@@ -30,7 +31,7 @@ class LayoutProbeResult:
 
 
 class LayoutDriftError(RuntimeError):
-    """Raised when the canary page no longer matches the expected layout."""
+    """Raised when the canary page no longer matches any known layout."""
 
     def __init__(self, url: str, failures: tuple[str, ...]) -> None:
         self.url = url
@@ -45,29 +46,37 @@ def _structural_failures(page: Page) -> list[str]:
 
     if not soup.find("h1"):
         failures.append("no <h1> provider heading found")
-    if not soup.find_all("section", class_="specialty"):
-        failures.append("no <section class='specialty'> blocks found")
 
-    headings = {h.get_text(strip=True).lower() for h in soup.find_all("h4")}
-    if not headings & _KNOWN_METRIC_HEADINGS:
-        failures.append("no recognised metric headings (h4) found")
+    if not (
+        soup.find_all("section", class_="specialty")
+        or soup.find_all("div", class_="inner_details_holder")
+    ):
+        failures.append("no recognised specialty blocks found")
 
-    has_waiting_table = any(
-        (row := table.find("tr")) is not None
-        and "average waiting time" in row.get_text(strip=True).lower()
+    headings = {h.get_text(strip=True).lower() for h in soup.find_all("h4")}
+    captions = {
+        caption.get_text(strip=True).lower()
         for table in soup.find_all("table")
-    )
-    if not has_waiting_table:
+        if (caption := table.find("caption")) is not None
+    }
+    if not (headings | captions) & _KNOWN_METRIC_LABELS:
+        failures.append("no recognised metric labels found")
+
+    if not any(
+        "average waiting time" in th.get_text(strip=True).lower()
+        for th in soup.find_all("th")
+    ):
         failures.append("no waiting-time tables with 'Average waiting time' header found")
 
-    if not _LAST_UPDATED_PATTERN.search(soup.get_text(" ", strip=True)):
-        failures.append("no 'Page last updated: DD/MM/YYYY' footer found")
+    text = soup.get_text(" ", strip=True)
+    if not (_LEGACY_FOOTER.search(text) or _2026_FOOTER.search(text)):
+        failures.append("no recognised last-updated footer found")
 
     return failures
 
 
 def probe_layout(page: Page) -> LayoutProbeResult:
-    """Probe one canary page for the layout the extractor depends on.
+    """Probe one canary page for a layout the extractor understands.
 
     Structural checks run first; only when they all pass is the extractor
     run end-to-end — a structurally valid page that still yields no

tests/fixtures/trust_page_drifted.html

@@ -1,6 +1,7 @@
 <!DOCTYPE html>
-<!-- Drifted-layout fixture: the same trust restructured. Tables replaced
-     by divs, headings renamed, footer format changed. The pre-flight
+<!-- Drifted-layout fixture: the same trust restructured into markup that
+     matches neither the legacy nor the 2026 layout. Tables replaced by
+     divs, headings renamed, both footer formats changed. The pre-flight
      probe must report every one of these failures. Added 2026-08-08. -->
 <html lang="en">
 <head><title>Royal Berkshire Hospital NHS Foundation Trust</title></head>
@@ -24,4 +25,4 @@ <h2 class="trust-name">Royal Berkshire Hospital NHS Foundation Trust</h2>
   </footer>
 </main>
 </body>
-</html>
+</html>

tests/fixtures/trust_page_royal_berkshire.html

@@ -50,4 +50,4 @@ <h3>Paediatric Surgery</h3>
   </footer>
 </main>
 </body>
-</html>
+</html>