updated workflow and added links status check

.github/workflows/link-check.yml

@@ -0,0 +1,41 @@
+name: Discord 404 checker (cron + PR paths + manual test)
+
+on:
+  schedule:
+    # Weekly (Monday 09:00 UTC)
+    - cron: "0 9 * * 1"
+  pull_request:
+    paths:
+      - ".github/workflows/test_list.json"
+      - ".github/workflows/list.json"
+  workflow_dispatch: {}
+
+jobs:
+  check-links:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      - name: Sync dependencies
+        run: uv sync --frozen
+
+      - name: Run script
+        working-directory: ${{ github.workspace }}
+        env:
+          PYTHONPATH: ${{ github.workspace }}
+          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
+          USER_ID: ${{ secrets.USER_ID }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          uv run -m scripts.check_links

scripts/check_links.py

@@ -0,0 +1,106 @@
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+# import httpx
+from .check_links_helper import (
+    BROKEN_STATUS_CODES,
+    OKAY_STATUS_CODES,
+    WARNING_STATUS_CODES,
+    check,
+    log,
+    send_discord,
+)
+
+# REPO_ROOT works only if this script is in /scripts
+REPO_ROOT = Path(__file__).resolve().parents[1]
+URL_LIST_PATH_1 = REPO_ROOT / "data" / "webpages" / "test_list.json"
+URL_LIST_PATH_2 = REPO_ROOT / "data" / "webpages" / "list.json"
+
+
+def main():
+    # webhook for the discord channel
+    webhook = os.environ["DISCORD_WEBHOOK_URL"]
+    # # user id of the user we want to ping
+    user_id = os.environ["USER_ID"]
+    summary_path = os.environ["GITHUB_STEP_SUMMARY"]
+    run_url = os.environ["RUN_URL"]
+    event_name = os.environ.get("EVENT_NAME", "")
+
+    urls = []
+    broken_links = []
+    # Unverifiable links <--> Warning links
+    warning_links = []
+    okay_links = []
+
+    for path in (URL_LIST_PATH_1, URL_LIST_PATH_2):
+        with open(path) as f:
+            urls.extend(json.load(f))
+
+    payload = asyncio.run(check(urls))
+
+    for key, value in payload.items():
+        match value:
+            case _ if value in BROKEN_STATUS_CODES:
+                broken_links.append(f"❌ **status code: {value}** | {key}")
+            case _ if value in WARNING_STATUS_CODES:
+                warning_links.append(f"🟨 **status code: {value}** | {key}")
+            case _ if value in OKAY_STATUS_CODES:
+                okay_links.append(f"🟩 **status code: {value}** | {key}")
+
+    log.info("event_name=%s", event_name)
+    log.info(
+        "broken_links=%d unverifiable_links=%d okay_links=%d",
+        len(broken_links),
+        len(warning_links),
+        len(okay_links),
+    )
+
+    if broken_links and event_name == "schedule":
+        log.info("Sending Discord webhook with %d BROKEN links", len(broken_links))
+        broken_links.insert(0, f"<@{user_id}>, these links are broken! ⚠️")
+        broken_links.insert(1, f"[Here is the summary log!]({run_url})")
+        send_discord(webhook, broken_links)
+
+    # if you want to test the workflow, use the okay_links
+    # if okay_links and event_name in ("schedule", "workflow_dispatch"):
+    #     log.info("Sending Discord webhook with %d OK lines", len(okay_links))
+    #     okay_links.insert(0, f"<@{user_id}>, these links are broken! ⚠️")
+    #     okay_links.append(f"<{run_url}|Here is the WORKFLOW!>")
+    #     send_discord(webhook, okay_links)
+
+    with open(summary_path, "a", encoding="utf-8") as f:
+        f.write("### Link check results\n\n")
+
+        f.write("## 🟥 Broken\n")
+        if broken_links:
+            for line in broken_links:
+                f.write(f"--> {line}\n")
+        else:
+            f.write("--> None\n")
+        f.write("\n")
+
+        f.write("## 🟨 Unverifiable/Warning\n")
+        if warning_links:
+            for line in warning_links:
+                f.write(f"--> {line}\n")
+        else:
+            f.write("--> None\n")
+        f.write("\n")
+
+        f.write("## 🟩 OK\n")
+        if okay_links:
+            for line in okay_links:
+                f.write(f"--> {line}\n")
+        else:
+            f.write("--> None\n")
+        f.write("\n")
+
+        if broken_links:
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/check_links_helper.py

@@ -0,0 +1,125 @@
+import asyncio
+import json
+import logging
+import random
+import urllib.error
+import urllib.request
+
+import httpx
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+
+BROKEN_STATUS_CODES = {404, 410}
+WARNING_STATUS_CODES = {429, 403} | set(range(500, 600))
+OKAY_STATUS_CODES = set(range(200, 300))
+
+DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+    )
+}
+
+
+# sends relevant info to webhook
+def send_discord(webhook_url: str, content: list[str]):
+    # Discord has a 2000 char limit - limiting content to <2000 chars
+    # right now, if the content payload is >2000 chars, we can't send all the urls
+    MAX_CONTENT = 2000
+
+    joined = "\n".join(content)
+
+    if not isinstance(joined, str):
+        joined = str(joined)
+
+    if len(joined) > MAX_CONTENT:
+        final_str = "\nCheck the summary logs for more info!"
+        joined = joined[: MAX_CONTENT - 3 - len(final_str)] + "..." + final_str
+
+    # avoiding encoding issues if any
+    joined = joined.encode("utf-8", errors="replace").decode("utf-8")
+
+    payload = {"content": joined}
+    data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
+
+    log.info(
+        "Discord POST bytes=%d content_chars=%d content_preview=%r",
+        len(data),
+        len(joined),
+        joined[:120],
+    )
+
+    req = urllib.request.Request(
+        webhook_url,
+        data=data,
+        headers={
+            "Content-Type": "application/json",
+            "User-Agent": "link-checker/1.0",
+        },
+        method="POST",
+    )
+
+    try:
+        with urllib.request.urlopen(req) as resp:
+            body = resp.read().decode("utf-8", errors="ignore")
+            log.info("Discord success status=%s body_len=%d", resp.status, len(body))
+            if body:
+                log.info("Discord body: %s", body)
+    except urllib.error.HTTPError as e:
+        body = e.read().decode("utf-8", errors="ignore")
+        log.error(
+            "Discord HTTPError status=%s code=%s response_body=%s",
+            getattr(e, "status", None),
+            e.code,
+            body,
+        )
+        raise
+    except Exception as e:
+        log.exception("Discord request failed with unexpected exception: %s", e)
+        raise
+
+
+# runs using asyncio.run(check(urls))
+# concurrently runs and checks the status code, puts in a dict and returns
+async def check(urls: list[str]) -> dict[str, int | None]:
+    timeout = httpx.Timeout(10.0)
+    async with httpx.AsyncClient(
+        timeout=timeout,
+        follow_redirects=True,
+        headers=DEFAULT_HEADERS,
+    ) as client:
+        tasks = [fetch_status_with_retries(url, client) for url in urls]
+        results = await asyncio.gather(*tasks, return_exceptions=False)
+        return dict(zip(urls, results))
+
+
+# run with: asyncio.run(check_one(url))
+async def fetch_status_with_retries(
+    url: str,
+    client: httpx.AsyncClient,
+    *,
+    retries: int = 2,  # total attempts = 1 + retries
+    base_backoff_s: float = 0.5,
+) -> int | None:
+    attempt = 0
+    while True:
+        try:
+            res = await client.get(url)
+            status = res.status_code
+
+            if status not in WARNING_STATUS_CODES:
+                return status
+
+            if attempt >= retries:
+                return status
+
+        except httpx.HTTPError:
+            if attempt >= retries:
+                return None
+
+        # backoff
+        backoff = base_backoff_s * (2**attempt)  # 0.5, 1.0, 2.0, ...
+        backoff = backoff + random.uniform(0, 0.1)
+        await asyncio.sleep(backoff)
+        attempt += 1