From 5a002fbc2395963be92ee45f44c31409aae8a8a0 Mon Sep 17 00:00:00 2001 From: strictshot Date: Thu, 16 Jul 2026 21:47:37 -0400 Subject: [PATCH] updated workflow and added links status check --- .github/workflows/link-check.yml | 41 ++++++++++ scripts/check_links.py | 106 ++++++++++++++++++++++++++ scripts/check_links_helper.py | 125 +++++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 .github/workflows/link-check.yml create mode 100644 scripts/check_links.py create mode 100644 scripts/check_links_helper.py diff --git a/.github/workflows/link-check.yml b/.github/workflows/link-check.yml new file mode 100644 index 0000000..b2890a9 --- /dev/null +++ b/.github/workflows/link-check.yml @@ -0,0 +1,41 @@ +name: Discord 404 checker (cron + PR paths + manual test) + +on: + schedule: + # Weekly (Monday 09:00 UTC) + - cron: "0 9 * * 1" + pull_request: + paths: + - ".github/workflows/test_list.json" + - ".github/workflows/list.json" + workflow_dispatch: {} + +jobs: + check-links: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + python-version: "3.12" + + - name: Sync dependencies + run: uv sync --frozen + + - name: Run script + working-directory: ${{ github.workspace }} + env: + PYTHONPATH: ${{ github.workspace }} + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + USER_ID: ${{ secrets.USER_ID }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + EVENT_NAME: ${{ github.event_name }} + run: | + uv run -m scripts.check_links diff --git a/scripts/check_links.py b/scripts/check_links.py new file mode 100644 index 0000000..5472e6a --- /dev/null +++ b/scripts/check_links.py @@ -0,0 +1,106 @@ +import asyncio +import json +import os +import sys +from pathlib import Path + +# import httpx +from .check_links_helper import ( + BROKEN_STATUS_CODES, + OKAY_STATUS_CODES, + WARNING_STATUS_CODES, + check, + log, + send_discord, +) + +# REPO_ROOT works only if this script is in /scripts +REPO_ROOT = Path(__file__).resolve().parents[1] +URL_LIST_PATH_1 = REPO_ROOT / "data" / "webpages" / "test_list.json" +URL_LIST_PATH_2 = REPO_ROOT / "data" / "webpages" / "list.json" + + +def main(): + # webhook for the discord channel + webhook = os.environ["DISCORD_WEBHOOK_URL"] + # # user id of the user we want to ping + user_id = os.environ["USER_ID"] + summary_path = os.environ["GITHUB_STEP_SUMMARY"] + run_url = os.environ["RUN_URL"] + event_name = os.environ.get("EVENT_NAME", "") + + urls = [] + broken_links = [] + # Unverifiable links <--> Warning links + warning_links = [] + okay_links = [] + + for path in (URL_LIST_PATH_1, URL_LIST_PATH_2): + with open(path) as f: + urls.extend(json.load(f)) + + payload = asyncio.run(check(urls)) + + for key, value in payload.items(): + match value: + case _ if value in BROKEN_STATUS_CODES: + broken_links.append(f"❌ **status code: {value}** | {key}") + case _ if value in WARNING_STATUS_CODES: + warning_links.append(f"🟨 **status code: {value}** | {key}") + case _ if value in OKAY_STATUS_CODES: + okay_links.append(f"🟩 **status code: {value}** | {key}") + + log.info("event_name=%s", event_name) + log.info( + "broken_links=%d unverifiable_links=%d okay_links=%d", + len(broken_links), + len(warning_links), + len(okay_links), + ) + + if broken_links and event_name == "schedule": + log.info("Sending Discord webhook with %d BROKEN links", len(broken_links)) + broken_links.insert(0, f"<@{user_id}>, these links are broken! ⚠️") + broken_links.insert(1, f"[Here is the summary log!]({run_url})") + send_discord(webhook, broken_links) + + # if you want to test the workflow, use the okay_links + # if okay_links and event_name in ("schedule", "workflow_dispatch"): + # log.info("Sending Discord webhook with %d OK lines", len(okay_links)) + # okay_links.insert(0, f"<@{user_id}>, these links are broken! ⚠️") + # okay_links.append(f"<{run_url}|Here is the WORKFLOW!>") + # send_discord(webhook, okay_links) + + with open(summary_path, "a", encoding="utf-8") as f: + f.write("### Link check results\n\n") + + f.write("## 🟥 Broken\n") + if broken_links: + for line in broken_links: + f.write(f"--> {line}\n") + else: + f.write("--> None\n") + f.write("\n") + + f.write("## 🟨 Unverifiable/Warning\n") + if warning_links: + for line in warning_links: + f.write(f"--> {line}\n") + else: + f.write("--> None\n") + f.write("\n") + + f.write("## 🟩 OK\n") + if okay_links: + for line in okay_links: + f.write(f"--> {line}\n") + else: + f.write("--> None\n") + f.write("\n") + + if broken_links: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_links_helper.py b/scripts/check_links_helper.py new file mode 100644 index 0000000..fbe7c8b --- /dev/null +++ b/scripts/check_links_helper.py @@ -0,0 +1,125 @@ +import asyncio +import json +import logging +import random +import urllib.error +import urllib.request + +import httpx + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + +BROKEN_STATUS_CODES = {404, 410} +WARNING_STATUS_CODES = {429, 403} | set(range(500, 600)) +OKAY_STATUS_CODES = set(range(200, 300)) + +DEFAULT_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" + ) +} + + +# sends relevant info to webhook +def send_discord(webhook_url: str, content: list[str]): + # Discord has a 2000 char limit - limiting content to <2000 chars + # right now, if the content payload is >2000 chars, we can't send all the urls + MAX_CONTENT = 2000 + + joined = "\n".join(content) + + if not isinstance(joined, str): + joined = str(joined) + + if len(joined) > MAX_CONTENT: + final_str = "\nCheck the summary logs for more info!" + joined = joined[: MAX_CONTENT - 3 - len(final_str)] + "..." + final_str + + # avoiding encoding issues if any + joined = joined.encode("utf-8", errors="replace").decode("utf-8") + + payload = {"content": joined} + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + + log.info( + "Discord POST bytes=%d content_chars=%d content_preview=%r", + len(data), + len(joined), + joined[:120], + ) + + req = urllib.request.Request( + webhook_url, + data=data, + headers={ + "Content-Type": "application/json", + "User-Agent": "link-checker/1.0", + }, + method="POST", + ) + + try: + with urllib.request.urlopen(req) as resp: + body = resp.read().decode("utf-8", errors="ignore") + log.info("Discord success status=%s body_len=%d", resp.status, len(body)) + if body: + log.info("Discord body: %s", body) + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="ignore") + log.error( + "Discord HTTPError status=%s code=%s response_body=%s", + getattr(e, "status", None), + e.code, + body, + ) + raise + except Exception as e: + log.exception("Discord request failed with unexpected exception: %s", e) + raise + + +# runs using asyncio.run(check(urls)) +# concurrently runs and checks the status code, puts in a dict and returns +async def check(urls: list[str]) -> dict[str, int | None]: + timeout = httpx.Timeout(10.0) + async with httpx.AsyncClient( + timeout=timeout, + follow_redirects=True, + headers=DEFAULT_HEADERS, + ) as client: + tasks = [fetch_status_with_retries(url, client) for url in urls] + results = await asyncio.gather(*tasks, return_exceptions=False) + return dict(zip(urls, results)) + + +# run with: asyncio.run(check_one(url)) +async def fetch_status_with_retries( + url: str, + client: httpx.AsyncClient, + *, + retries: int = 2, # total attempts = 1 + retries + base_backoff_s: float = 0.5, +) -> int | None: + attempt = 0 + while True: + try: + res = await client.get(url) + status = res.status_code + + if status not in WARNING_STATUS_CODES: + return status + + if attempt >= retries: + return status + + except httpx.HTTPError: + if attempt >= retries: + return None + + # backoff + backoff = base_backoff_s * (2**attempt) # 0.5, 1.0, 2.0, ... + backoff = backoff + random.uniform(0, 0.1) + await asyncio.sleep(backoff) + attempt += 1