diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1cc19d5..5062e64 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,11 +5,30 @@ on: branches: [main] pull_request: workflow_dispatch: + schedule: + # The guide list can also drift when a guide is published or unpublished on + # getpatchrail.com, with no commit here to trigger a run. + - cron: "17 6 * * 1" permissions: contents: read jobs: + fix-guide-slugs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install patchrail and pytest + run: python -m pip install --upgrade patchrail pytest + + - name: Check the guide slugs against the CLI and the published guides + run: python -m pytest -q + smoke: runs-on: ubuntu-latest steps: diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..7f4542c --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +testpaths = tests +markers = + network: hits getpatchrail.com; skipped automatically when the site is unreachable diff --git a/scripts/annotate.py b/scripts/annotate.py index 4c8691c..3538ce1 100644 --- a/scripts/annotate.py +++ b/scripts/annotate.py @@ -14,8 +14,9 @@ FIX_GUIDE_BASE = "https://getpatchrail.com/fix" # Failure classes with a dedicated /fix/ remediation guide on getpatchrail.com. -# Unknown or unlisted classes link to the guide index instead. Mirrors -# patchrail.cli._FIX_GUIDE_SLUGS so the action links the same pages as the CLI. +# Unknown or unlisted classes link to the guide index instead, never to a 404. +# Every entry must be a real `patchrail ci classes` slug AND a published guide; +# tests/test_fix_guide_slugs.py checks both and fails if this list drifts. FIX_GUIDE_SLUGS = frozenset( { "artifact-or-cache-failure", diff --git a/tests/test_fix_guide_slugs.py b/tests/test_fix_guide_slugs.py new file mode 100644 index 0000000..b260d54 --- /dev/null +++ b/tests/test_fix_guide_slugs.py @@ -0,0 +1,111 @@ +"""Guard the action's FIX_GUIDE_SLUGS against drift. + +The action turns a failure class into a https://getpatchrail.com/fix/ +link. That list is hand-maintained here, so it can rot in two directions: + +* a slug that is not a real PatchRail failure class -> dead entry, and a + renamed class silently loses its guide link; +* a slug with no published guide page -> the action sends users to a 404. + +The two sources of truth are the CLI (`patchrail ci classes`) and the guide +index on getpatchrail.com. These tests derive both and fail on divergence. +""" +from __future__ import annotations + +import importlib.util +import json +import re +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +GUIDE_INDEX = "https://getpatchrail.com/fix" +_HREF = re.compile(r'href="/fix/([a-z0-9-]+)"') + + +def _load_annotate(): + path = Path(__file__).resolve().parent.parent / "scripts" / "annotate.py" + spec = importlib.util.spec_from_file_location("annotate", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +annotate = _load_annotate() + + +def _slug(failure_class: str) -> str: + return failure_class.replace("_", "-") + + +def _cli_failure_classes() -> set[str]: + """Every failure class the installed patchrail CLI can emit, as slugs.""" + raw = subprocess.run( + [sys.executable, "-m", "patchrail", "ci", "classes", "--format", "json"], + capture_output=True, + text=True, + check=True, + ).stdout + payload = json.loads(raw) + entries = payload["classes"] if isinstance(payload, dict) else payload + names = {e["failure_class"] if isinstance(e, dict) else e for e in entries} + return {_slug(name) for name in names if name != "unknown"} + + +def _published_guides() -> set[str]: + """Slugs the guide index actually links to. Skips if the site is unreachable.""" + try: + with urllib.request.urlopen(GUIDE_INDEX, timeout=20) as response: + if response.status != 200: + pytest.skip(f"guide index returned HTTP {response.status}") + html = response.read().decode("utf-8", "replace") + except (urllib.error.URLError, TimeoutError) as exc: + pytest.skip(f"guide index unreachable: {exc}") + slugs = set(_HREF.findall(html)) + if not slugs: + pytest.skip("guide index exposed no /fix/ links; markup may have changed") + return slugs + + +def test_every_slug_is_a_real_failure_class() -> None: + """A slug the CLI never emits is dead weight, and a rename must not go unnoticed.""" + unknown = annotate.FIX_GUIDE_SLUGS - _cli_failure_classes() + assert not unknown, ( + f"FIX_GUIDE_SLUGS contains slugs that are not PatchRail failure classes: " + f"{sorted(unknown)}. Remove them, or fix the spelling to match " + f"`patchrail ci classes`." + ) + + +def test_known_slug_links_to_its_guide() -> None: + assert ( + annotate.guide_url("python_dependency_resolution") + == f"{GUIDE_INDEX}/python-dependency-resolution" + ) + + +@pytest.mark.parametrize("failure_class", ["unknown", "", "some_class_with_no_guide"]) +def test_class_without_a_guide_falls_back_to_the_index(failure_class: str) -> None: + """A class with no published guide must reach the index, never a 404.""" + assert annotate.guide_url(failure_class) == GUIDE_INDEX + + +@pytest.mark.network +def test_slugs_match_the_published_guides() -> None: + """FIX_GUIDE_SLUGS must be exactly the guides published on getpatchrail.com.""" + published = _published_guides() + missing_page = annotate.FIX_GUIDE_SLUGS - published + assert not missing_page, ( + f"FIX_GUIDE_SLUGS points at guides that are not published: " + f"{sorted(missing_page)}. The action would send users to a 404." + ) + unlinked = published - annotate.FIX_GUIDE_SLUGS + assert not unlinked, ( + f"These guides are published but missing from FIX_GUIDE_SLUGS: " + f"{sorted(unlinked)}. The action sends those failures to the index " + f"instead of the guide that exists." + )