From 1451c146fc54cc0b3843f80997fe330c850a2274 Mon Sep 17 00:00:00 2001 From: strictshot Date: Sun, 5 Jul 2026 01:44:29 -0400 Subject: [PATCH 1/6] added the link_checker.py, up next is the cronjob .yml --- scripts/link_checker.py | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 scripts/link_checker.py diff --git a/scripts/link_checker.py b/scripts/link_checker.py new file mode 100644 index 0000000..b908803 --- /dev/null +++ b/scripts/link_checker.py @@ -0,0 +1,55 @@ +import os, json, urllib.request, urllib.error +from pathlib import Path + + +def send_discord(webhook_url: str, content: list[str]): + payload = {"content": "\n".join(content)} + print(payload) + req = urllib.request.Request( + webhook_url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json", "User-Agent": "link-checker/1.0"}, + method="POST", + ) + # lets try to use logs instead of print + try: + with urllib.request.urlopen(req) as resp: + print("Discord status:", resp.status) + print(resp.read().decode("utf-8", errors="ignore")) + except urllib.error.HTTPError as e: + print("Discord HTTPError:", e.code) + body = e.read().decode("utf-8", errors="ignore") + print("Discord response body:", body) + raise + + +# can add a hardcoded webhook for the time being (when testing the file itself) +# webhook = os.environ["DISCORD_WEBHOOK_URL"] +# here goes the user id of the user we want to ping +# user_id = os.environ["USER_ID"] + +REPO_ROOT = Path(__file__).resolve().parents[1] # works only if the script is in /scripts +URL_LIST_PATH = REPO_ROOT / "data" / "webpages" / "test_list.json" + +failed_urls = [] + +with open(URL_LIST_PATH) as f: + urls = json.load(f) + +for url in urls: + url_status = urllib.request.urlopen(url).getcode() + print(f"url status: {url_status}") + # uncomment this so that ts actually works with 404 calls or any other calls which arent 200 + # if url_status != 200: + # failed_urls.append(f"FAILED URL: {url}") + # adds "FAILED URL:" before each url + failed_urls.append("FAILED URL: " + url) # remove this after testing + + +payload = failed_urls +# this cmd is when we actually implement user_id +# payload.insert(0, f"<@{user_id}>, these links are returning a 404! Check these links.") +payload.insert(0, "@test, these links are returning a 404! Check these links.") +# if failed_urls: # add 'not' later after testing and shit +# send_discord(webhook, payload) +print(payload) # remove this after testing From 8a0761349d1add465b06eddb2b066512ef4a727f Mon Sep 17 00:00:00 2001 From: strictshot Date: Sun, 5 Jul 2026 19:12:50 -0400 Subject: [PATCH 2/6] pushing .yml workflow and .py script, check if cronjob works --- .github/workflows/link_checker.yml | 33 ++++++++++++++++++++++++++++++ scripts/link_checker.py | 15 +++++++------- 2 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/link_checker.yml diff --git a/.github/workflows/link_checker.yml b/.github/workflows/link_checker.yml new file mode 100644 index 0000000..3a8d702 --- /dev/null +++ b/.github/workflows/link_checker.yml @@ -0,0 +1,33 @@ +name: Discord 404 checker (cron + manual test) + +on: + schedule: + # Runs every 15 minutes (UTC). Change as needed. + - cron: "*/15 * * * *" + workflow_dispatch: {} + +jobs: + check-links: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install deps (none for this script) + run: python -m pip install --upgrade pip + + - name: Run script + env: + DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + USER_ID: ${{ secrets.USER_ID }} + run: | + python -u scripts/link_checker.py + diff --git a/scripts/link_checker.py b/scripts/link_checker.py index b908803..c86468d 100644 --- a/scripts/link_checker.py +++ b/scripts/link_checker.py @@ -24,9 +24,11 @@ def send_discord(webhook_url: str, content: list[str]): # can add a hardcoded webhook for the time being (when testing the file itself) -# webhook = os.environ["DISCORD_WEBHOOK_URL"] +webhook = os.environ["DISCORD_WEBHOOK_URL"] +# webhook = "https://discord.com/api/webhooks/1523440196232937632/K-aVVdgRNxrTOIUbpcGzhfsE4Jon8Rw_g9w94UqPDubew3Hkvfsw9PSrcsq38iRyvOjR" # here goes the user id of the user we want to ping -# user_id = os.environ["USER_ID"] +user_id = os.environ["USER_ID"] +# user_id = "1523427702571536500" REPO_ROOT = Path(__file__).resolve().parents[1] # works only if the script is in /scripts URL_LIST_PATH = REPO_ROOT / "data" / "webpages" / "test_list.json" @@ -47,9 +49,6 @@ def send_discord(webhook_url: str, content: list[str]): payload = failed_urls -# this cmd is when we actually implement user_id -# payload.insert(0, f"<@{user_id}>, these links are returning a 404! Check these links.") -payload.insert(0, "@test, these links are returning a 404! Check these links.") -# if failed_urls: # add 'not' later after testing and shit -# send_discord(webhook, payload) -print(payload) # remove this after testing +payload.insert(0, f"<@{user_id}>, these links are returning a 404! Check these links.") +if failed_urls: # add 'not' later after testing and shit + send_discord(webhook, payload) From 938b6c300014991caf09941490da5302793b38d5 Mon Sep 17 00:00:00 2001 From: strictshot Date: Sun, 5 Jul 2026 19:18:18 -0400 Subject: [PATCH 3/6] lol --- scripts/link_checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/link_checker.py b/scripts/link_checker.py index c86468d..6eebd6a 100644 --- a/scripts/link_checker.py +++ b/scripts/link_checker.py @@ -25,10 +25,10 @@ def send_discord(webhook_url: str, content: list[str]): # can add a hardcoded webhook for the time being (when testing the file itself) webhook = os.environ["DISCORD_WEBHOOK_URL"] -# webhook = "https://discord.com/api/webhooks/1523440196232937632/K-aVVdgRNxrTOIUbpcGzhfsE4Jon8Rw_g9w94UqPDubew3Hkvfsw9PSrcsq38iRyvOjR" +# webhook = "" # here goes the user id of the user we want to ping user_id = os.environ["USER_ID"] -# user_id = "1523427702571536500" +# user_id = "" REPO_ROOT = Path(__file__).resolve().parents[1] # works only if the script is in /scripts URL_LIST_PATH = REPO_ROOT / "data" / "webpages" / "test_list.json" From b3b5487047e108954b9a756a9cc93b19596aa54a Mon Sep 17 00:00:00 2001 From: strictshot Date: Sun, 5 Jul 2026 20:17:01 -0400 Subject: [PATCH 4/6] from 15min to 5min --- .github/workflows/link_checker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/link_checker.yml b/.github/workflows/link_checker.yml index 3a8d702..2bedf03 100644 --- a/.github/workflows/link_checker.yml +++ b/.github/workflows/link_checker.yml @@ -3,7 +3,7 @@ name: Discord 404 checker (cron + manual test) on: schedule: # Runs every 15 minutes (UTC). Change as needed. - - cron: "*/15 * * * *" + - cron: "*/5 * * * *" workflow_dispatch: {} jobs: From 06e88275d78c050ff1c3f426fca761c218cdd37d Mon Sep 17 00:00:00 2001 From: strictshot Date: Sun, 5 Jul 2026 21:07:16 -0400 Subject: [PATCH 5/6] fixed lint test --- .github/workflows/link_checker.yml | 1 - scripts/link_checker.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/link_checker.yml b/.github/workflows/link_checker.yml index 2bedf03..b6f0039 100644 --- a/.github/workflows/link_checker.yml +++ b/.github/workflows/link_checker.yml @@ -30,4 +30,3 @@ jobs: USER_ID: ${{ secrets.USER_ID }} run: | python -u scripts/link_checker.py - diff --git a/scripts/link_checker.py b/scripts/link_checker.py index 6eebd6a..777f005 100644 --- a/scripts/link_checker.py +++ b/scripts/link_checker.py @@ -1,4 +1,7 @@ -import os, json, urllib.request, urllib.error +import json +import os +import urllib.error +import urllib.request from pathlib import Path From b7e6ce0d12da0275506c3badec4db1cf0424e296 Mon Sep 17 00:00:00 2001 From: strictshot Date: Mon, 6 Jul 2026 10:52:00 -0400 Subject: [PATCH 6/6] cleaning the script up --- scripts/link_checker.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/link_checker.py b/scripts/link_checker.py index 777f005..70b5ea6 100644 --- a/scripts/link_checker.py +++ b/scripts/link_checker.py @@ -4,36 +4,38 @@ import urllib.request from pathlib import Path +from src.config.logger import get_logger +log = get_logger(__name__) + + +# sends relevant info to webhook def send_discord(webhook_url: str, content: list[str]): payload = {"content": "\n".join(content)} - print(payload) req = urllib.request.Request( webhook_url, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json", "User-Agent": "link-checker/1.0"}, method="POST", ) - # lets try to use logs instead of print try: with urllib.request.urlopen(req) as resp: - print("Discord status:", resp.status) - print(resp.read().decode("utf-8", errors="ignore")) + log.info("Discord status:", status=resp.status) + log.info(resp.read().decode("utf-8", errors="ignore")) except urllib.error.HTTPError as e: - print("Discord HTTPError:", e.code) body = e.read().decode("utf-8", errors="ignore") - print("Discord response body:", body) + log.error("Discord HTTPError:", error=e.code, response_body=body) raise -# can add a hardcoded webhook for the time being (when testing the file itself) +# webhook for the discord channel webhook = os.environ["DISCORD_WEBHOOK_URL"] -# webhook = "" -# here goes the user id of the user we want to ping +# user id of the user we want to ping user_id = os.environ["USER_ID"] -# user_id = "" -REPO_ROOT = Path(__file__).resolve().parents[1] # works only if the script is in /scripts +# works only if this script is in /scripts +REPO_ROOT = Path(__file__).resolve().parents[1] +# replace with any list URL_LIST_PATH = REPO_ROOT / "data" / "webpages" / "test_list.json" failed_urls = [] @@ -43,15 +45,13 @@ def send_discord(webhook_url: str, content: list[str]): for url in urls: url_status = urllib.request.urlopen(url).getcode() - print(f"url status: {url_status}") - # uncomment this so that ts actually works with 404 calls or any other calls which arent 200 - # if url_status != 200: - # failed_urls.append(f"FAILED URL: {url}") - # adds "FAILED URL:" before each url - failed_urls.append("FAILED URL: " + url) # remove this after testing - + if url_status != 200: + failed_urls.append(f"FAILED URL: {url}") + # uncomment this if you want to try the functionality with any other statuscode (or none) + # failed_urls.append("FAILED URL: " + url) payload = failed_urls payload.insert(0, f"<@{user_id}>, these links are returning a 404! Check these links.") -if failed_urls: # add 'not' later after testing and shit + +if failed_urls: send_discord(webhook, payload)