diff --git a/backend/test_deep_audit_regressions.py b/backend/test_deep_audit_regressions.py index 836f592..e5e1c55 100644 --- a/backend/test_deep_audit_regressions.py +++ b/backend/test_deep_audit_regressions.py @@ -6,11 +6,13 @@ import re import sqlite3 import tempfile +import unittest +import xml.etree.ElementTree as ET +from html.parser import HTMLParser from pathlib import Path from types import ModuleType, SimpleNamespace from typing import Any, cast from unittest import mock -import unittest MODULE_PATH = Path(__file__).with_name("api_core.py") REPO_ROOT = MODULE_PATH.parents[1] @@ -2060,6 +2062,83 @@ def test_public_logo_shell_is_owned_and_accessible_once(self): offenders.append(str(path.relative_to(REPO_ROOT))) self.assertEqual(offenders, []) + @staticmethod + def _sitemap_urls(): + root = ET.parse(REPO_ROOT / "frontend/sitemap.xml").getroot() + return [ + element.text.strip() + for element in root.iter() + if element.tag.rsplit("}", 1)[-1] == "loc" and element.text + ] + + @staticmethod + def _html_declares_noindex(text): + class RobotsMetaParser(HTMLParser): + has_noindex = False + + def handle_starttag(self, tag, attrs): + if tag.lower() != "meta": + return + attributes = { + key.lower(): (value or "").lower() + for key, value in attrs + if key + } + if attributes.get("name") not in {"robots", "googlebot"}: + return + directives = attributes.get("content", "").replace(",", " ").replace(";", " ").split() + if "noindex" in directives: + self.has_noindex = True + + parser = RobotsMetaParser() + parser.feed(text) + return parser.has_noindex + + def test_sitemap_urls_are_unique(self): + urls = self._sitemap_urls() + self.assertGreater(len(urls), 0) + self.assertIn("https://www.gohirehumans.com/", urls) + seen = set() + duplicates = set() + for url in urls: + if url in seen: + duplicates.add(url) + seen.add(url) + self.assertEqual(sorted(duplicates), []) + + def test_sitemapped_html_pages_do_not_opt_out_of_indexing(self): + urls = self._sitemap_urls() + self.assertGreater(len(urls), 0) + offenders = [] + public_origin = "https://www.gohirehumans.com" + for url in urls: + self.assertTrue(url.startswith(public_origin), url) + loc = url.removeprefix(public_origin) + if loc in ("", "/"): + page = REPO_ROOT / "frontend/index.html" + elif loc.endswith("/"): + page = REPO_ROOT / f"frontend{loc}index.html" + elif loc.endswith(".html"): + page = REPO_ROOT / f"frontend{loc}" + else: + continue + if not page.exists(): + continue + text = page.read_text(encoding="utf-8", errors="ignore") + if self._html_declares_noindex(text): + offenders.append(str(page.relative_to(REPO_ROOT))) + self.assertEqual(offenders, []) + + def test_sitemap_noindex_guard_recognizes_attribute_order_and_directive_tokens(self): + samples = [ + '', + '', + ] + for sample in samples: + with self.subTest(sample=sample): + self.assertTrue(self._html_declares_noindex(sample)) + self.assertFalse(self._html_declares_noindex('')) + def test_sitemapped_html_pages_use_single_canonical_public_nav(self): expected_labels = [ "GoHireHumans", diff --git a/frontend/sitemap.xml b/frontend/sitemap.xml index a754444..d895fe8 100644 --- a/frontend/sitemap.xml +++ b/frontend/sitemap.xml @@ -298,12 +298,6 @@ monthly 0.8 - - https://www.gohirehumans.com/blog/ai-agent-marketplace-guide.html - 2026-03-19 - monthly - 0.7 - https://www.gohirehumans.com/blog/freelance-vs-full-time-2026.html 2026-03-19 @@ -528,12 +522,6 @@ weekly 0.84 - - https://www.gohirehumans.com/ai-human-qa/ai-citation-source-verification.html - 2026-05-25 - weekly - 0.84 - https://www.gohirehumans.com/ai-human-qa/rag-groundedness-human-review.html 2026-05-25