From c9cae6675ef2492c27659d8bdcbcb01a37db1e2a Mon Sep 17 00:00:00 2001
From: MirjamOdile <48677969+MirjamOdile@users.noreply.github.com>
Date: Thu, 9 Jul 2026 12:57:58 +0000
Subject: [PATCH] Warn instead of blocking when spider name differs from its
 domain
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The importer hard-rejected any spider whose name didn't equal the
domain-derived folder name whenever source_url was set. But a domain
can legitimately host more than one spider (a second template, a
sub-domain grouped under the main site), and crawls + analysis both key
off the spider NAME consistently — nothing actually scatters. The check
forced legit sub-spiders to drop source_url just to get imported.

Keep the check as a warning that states where the data will live.

See docs/requests/05-spider-name-domain-warn.md (ships with the quality
tool PR) for the full write-up.
---
 cli/spiders.py | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/cli/spiders.py b/cli/spiders.py
index 5241d42..26f9d0f 100644
--- a/cli/spiders.py
+++ b/cli/spiders.py
@@ -163,7 +163,13 @@ def import_spider(file, project, skip_validation):
                     )
                     return
 
-            # Validate spider name matches inspector's folder structure
+            # Sanity-check spider name vs its domain. A domain can legitimately host
+            # MORE THAN ONE spider (e.g. noaa_gov + noaa_gov_gc, or a subdomain like
+            # ncei.noaa.gov grouped as noaa_gov_ncei), so the name won't always equal
+            # the domain. Everything downstream keys off the spider NAME (only the
+            # inspector's analysis folder stays domain-derived) — warn, don't block
+            # (previously this rejected legit sub-spiders, forcing them to drop
+            # source_url to sneak in).
             if source_url:
                 from urllib.parse import urlparse
 
@@ -172,21 +178,12 @@ def import_spider(file, project, skip_validation):
                 expected_name = domain.replace(".", "_")
 
                 if spider_name != expected_name:
-                    click.echo("❌ Spider name mismatch detected!")
                     click.echo(
-                        f"   Inspector created folder: data/{project}/{expected_name}/analysis/"
+                        f"⚠️  Spider name '{spider_name}' differs from its domain "
+                        f"('{expected_name}') — fine for a secondary or sub-domain "
+                        f"spider. Crawls will live under data/{project}/{spider_name}/ "
+                        f"(inspector analysis stays under data/{project}/{expected_name}/)."
                     )
-                    click.echo(f"   But spider name is: '{spider_name}'")
-                    click.echo(
-                        f"   Crawls will save to: data/{project}/{spider_name}/crawls/"
-                    )
-                    click.echo(
-                        "\n   ⚠️  Files will be scattered across different folders!"
-                    )
-                    click.echo(
-                        f"\n   Fix: Change spider name to '{expected_name}' in the JSON config."
-                    )
-                    return
 
             # Check for existing spider
             existing = db.query(Spider).filter(Spider.name == spider_name).first()