Fix sitemap indexing consistency

backend/test_deep_audit_regressions.py

@@ -6,11 +6,13 @@
 import re
 import sqlite3
 import tempfile
+import unittest
+import xml.etree.ElementTree as ET
+from html.parser import HTMLParser
 from pathlib import Path
 from types import ModuleType, SimpleNamespace
 from typing import Any, cast
 from unittest import mock
-import unittest
 
 MODULE_PATH = Path(__file__).with_name("api_core.py")
 REPO_ROOT = MODULE_PATH.parents[1]
@@ -2060,6 +2062,83 @@ def test_public_logo_shell_is_owned_and_accessible_once(self):
                 offenders.append(str(path.relative_to(REPO_ROOT)))
         self.assertEqual(offenders, [])
 
+    @staticmethod
+    def _sitemap_urls():
+        root = ET.parse(REPO_ROOT / "frontend/sitemap.xml").getroot()
+        return [
+            element.text.strip()
+            for element in root.iter()
+            if element.tag.rsplit("}", 1)[-1] == "loc" and element.text
+        ]
+
+    @staticmethod
+    def _html_declares_noindex(text):
+        class RobotsMetaParser(HTMLParser):
+            has_noindex = False
+
+            def handle_starttag(self, tag, attrs):
+                if tag.lower() != "meta":
+                    return
+                attributes = {
+                    key.lower(): (value or "").lower()
+                    for key, value in attrs
+                    if key
+                }
+                if attributes.get("name") not in {"robots", "googlebot"}:
+                    return
+                directives = attributes.get("content", "").replace(",", " ").replace(";", " ").split()
+                if "noindex" in directives:
+                    self.has_noindex = True
+
+        parser = RobotsMetaParser()
+        parser.feed(text)
+        return parser.has_noindex
+
+    def test_sitemap_urls_are_unique(self):
+        urls = self._sitemap_urls()
+        self.assertGreater(len(urls), 0)
+        self.assertIn("https://www.gohirehumans.com/", urls)
+        seen = set()
+        duplicates = set()
+        for url in urls:
+            if url in seen:
+                duplicates.add(url)
+            seen.add(url)
+        self.assertEqual(sorted(duplicates), [])
+
+    def test_sitemapped_html_pages_do_not_opt_out_of_indexing(self):
+        urls = self._sitemap_urls()
+        self.assertGreater(len(urls), 0)
+        offenders = []
+        public_origin = "https://www.gohirehumans.com"
+        for url in urls:
+            self.assertTrue(url.startswith(public_origin), url)
+            loc = url.removeprefix(public_origin)
+            if loc in ("", "/"):
+                page = REPO_ROOT / "frontend/index.html"
+            elif loc.endswith("/"):
+                page = REPO_ROOT / f"frontend{loc}index.html"
+            elif loc.endswith(".html"):
+                page = REPO_ROOT / f"frontend{loc}"
+            else:
+                continue
+            if not page.exists():
+                continue
+            text = page.read_text(encoding="utf-8", errors="ignore")
+            if self._html_declares_noindex(text):
+                offenders.append(str(page.relative_to(REPO_ROOT)))
+        self.assertEqual(offenders, [])
+
+    def test_sitemap_noindex_guard_recognizes_attribute_order_and_directive_tokens(self):
+        samples = [
+            '<meta name="robots" content="noindex, nofollow">',
+            '<meta content="follow, NOINDEX" name="robots">',
+        ]
+        for sample in samples:
+            with self.subTest(sample=sample):
+                self.assertTrue(self._html_declares_noindex(sample))
+        self.assertFalse(self._html_declares_noindex('<meta name="robots" content="index, follow">'))
+
     def test_sitemapped_html_pages_use_single_canonical_public_nav(self):
         expected_labels = [
             "GoHireHumans",

frontend/sitemap.xml

@@ -298,12 +298,6 @@
     <changefreq>monthly</changefreq>
     <priority>0.8</priority>
   </url>
-  <url>
-    <loc>https://www.gohirehumans.com/blog/ai-agent-marketplace-guide.html</loc>
-    <lastmod>2026-03-19</lastmod>
-    <changefreq>monthly</changefreq>
-    <priority>0.7</priority>
-  </url>
   <url>
     <loc>https://www.gohirehumans.com/blog/freelance-vs-full-time-2026.html</loc>
     <lastmod>2026-03-19</lastmod>
@@ -528,12 +522,6 @@
     <changefreq>weekly</changefreq>
     <priority>0.84</priority>
   </url>
-  <url>
-    <loc>https://www.gohirehumans.com/ai-human-qa/ai-citation-source-verification.html</loc>
-    <lastmod>2026-05-25</lastmod>
-    <changefreq>weekly</changefreq>
-    <priority>0.84</priority>
-  </url>
   <url>
     <loc>https://www.gohirehumans.com/ai-human-qa/rag-groundedness-human-review.html</loc>
     <lastmod>2026-05-25</lastmod>