From 3148a769d32db999ff3193f608063855eba353b4 Mon Sep 17 00:00:00 2001
From: MirjamOdile <48677969+MirjamOdile@users.noreply.github.com>
Date: Thu, 9 Jul 2026 12:58:51 +0000
Subject: [PATCH] crawl-all: enqueue full crawls via Pueue instead of running
 inline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Since production crawls auto-detach into Pueue, a single `scrapai
crawl` is parallel-safe and disconnect-safe — but crawl-all bypassed
that (detached=True forced every spider inline), so "run the whole
project" meant strictly sequential, one spider at a time, bound to the
terminal, and the whole batch died on Ctrl+C or an SSH drop.

crawl_all now calls each spider through the same path as a single
crawl: a full run submits itself to Pueue and returns, so the whole
project is queued in seconds, runs parallel (Pueue group default), and
survives disconnects. --limit keeps the old inline sequential
behaviour (test crawls are foreground by design). A closing hint
points at ./scrapai crawl-status.

See docs/requests/14-crawl-all-pueue.md (ships with the quality tool
PR) for the full write-up.
---
 cli/crawl.py                   | 12 +++++--
 tests/unit/test_crawl_pueue.py | 62 ++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/cli/crawl.py b/cli/crawl.py
index a087859..ef4b0b4 100644
--- a/cli/crawl.py
+++ b/cli/crawl.py
@@ -225,11 +225,19 @@ def crawl_all(project, limit):
         click.echo(f"\n{'='*50}")
         click.echo(f"Running: {name}")
         click.echo(f"{'='*50}")
-        # detached=True keeps crawl_all running each spider inline, unchanged.
+        # Full crawls self-submit to Pueue like a single `crawl` (parallel,
+        # disconnect-safe); --limit runs stay inline.
         _run_spider(
-            project, name, None, limit, None, "auto", False, None, False, False, True
+            project, name, None, limit, None, "auto", False, None, False, False, False
         )
 
+    if not limit:
+        click.echo(
+            f"\n📋 All {len(spider_names)} spiders queued in Pueue "
+            "(detached; concurrency = your `pueue parallel` setting)."
+        )
+        click.echo("   Monitor: ./scrapai crawl-status   ·   pueue status")
+
 
 @click.command("crawl-status")
 @click.argument("spider", required=False)
diff --git a/tests/unit/test_crawl_pueue.py b/tests/unit/test_crawl_pueue.py
index 4e5d72c..8a5ac44 100644
--- a/tests/unit/test_crawl_pueue.py
+++ b/tests/unit/test_crawl_pueue.py
@@ -41,3 +41,65 @@ def test_passes_flags_through():
     ]:
         i = cmd.index(flag)
         assert cmd[i] == flag and cmd[i + 1] == val
+
+
+# --- crawl-all queues every spider through the same detach path ---------------
+
+
+def test_crawl_all_enqueues_each_spider(monkeypatch):
+    """`crawl-all` (no --limit) must submit EVERY spider through the same
+    auto-detach path as a single `crawl` (detached=False → Pueue), not run them
+    inline — inline was sequential, terminal-bound, and died on disconnect."""
+    from unittest.mock import MagicMock, Mock, patch
+
+    from click.testing import CliRunner
+
+    import importlib
+
+    crawl_mod = importlib.import_module("cli.crawl")  # cli/__init__ shadows the
+    #                                                    attribute with the command
+    calls = []
+    monkeypatch.setattr(crawl_mod, "_run_spider", lambda *a, **k: calls.append(a))
+
+    recs = []
+    for n in ("a_org", "b_org"):
+        r = Mock()
+        r.name = n
+        recs.append(r)
+    with patch("core.db.get_db") as mock_get_db:
+        db = Mock()
+        db.query.return_value.filter.return_value.all.return_value = recs
+        cm = MagicMock()
+        cm.__enter__.return_value = db
+        mock_get_db.return_value = cm
+        res = CliRunner().invoke(crawl_mod.crawl_all, ["--project", "proj"])
+    assert res.exit_code == 0, res.output
+    assert [c[1] for c in calls] == ["a_org", "b_org"]  # every spider
+    assert all(c[10] is False for c in calls)  # detached=False → Pueue path
+    assert "queued in Pueue" in res.output
+
+
+def test_crawl_all_with_limit_stays_inline(monkeypatch):
+    from unittest.mock import MagicMock, Mock, patch
+
+    from click.testing import CliRunner
+
+    import importlib
+
+    crawl_mod = importlib.import_module("cli.crawl")
+    calls = []
+    monkeypatch.setattr(crawl_mod, "_run_spider", lambda *a, **k: calls.append(a))
+    r = Mock()
+    r.name = "a_org"
+    with patch("core.db.get_db") as mock_get_db:
+        db = Mock()
+        db.query.return_value.filter.return_value.all.return_value = [r]
+        cm = MagicMock()
+        cm.__enter__.return_value = db
+        mock_get_db.return_value = cm
+        res = CliRunner().invoke(
+            crawl_mod.crawl_all, ["--project", "proj", "--limit", "5"]
+        )
+    assert res.exit_code == 0, res.output
+    assert calls[0][3] == 5  # limit threads through
+    assert "queued in Pueue" not in res.output  # inline test mode