From d676ec428486fae11411ecdf1fc9734e3aa674df Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 11:05:03 +0200
Subject: [PATCH 01/39] docs: brainstorm spec for public torrent-site catalog
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Adds a design document for a curated, built-in catalog of 12 public
no-account torrent sites (TPB, 1337x, Nyaa, TorrentGalaxy, …). Ships
the parser machinery already on disk — the gap is UX. The spec covers
catalog layout, vendoring from Prowlarr-indexers, parser extensions
driven by real per-site fixture tests, a dedicated /indexers/catalog
page, and an optional onboarding step.
Co-Authored-By: Claude Opus 4.7 (1M context)
---
...4-20-public-torrent-site-catalog-design.md | 261 ++++++++++++++++++
1 file changed, 261 insertions(+)
create mode 100644 docs/superpowers/specs/2026-04-20-public-torrent-site-catalog-design.md
diff --git a/docs/superpowers/specs/2026-04-20-public-torrent-site-catalog-design.md b/docs/superpowers/specs/2026-04-20-public-torrent-site-catalog-design.md
new file mode 100644
index 0000000..67079e3
--- /dev/null
+++ b/docs/superpowers/specs/2026-04-20-public-torrent-site-catalog-design.md
@@ -0,0 +1,261 @@
+# Public torrent site catalog
+
+**Date:** 2026-04-20
+**Status:** Design, pre-implementation
+**Scope:** Ship a curated, built-in catalog of public (no-account) torrent sites so users can add them with a single click instead of copy-pasting YAML from a GitHub repo.
+
+## Problem
+
+Today, adding a public torrent site like The Pirate Bay or 1337x to Trove requires:
+
+1. Locate a Cardigann YAML definition on GitHub (typically `Prowlarr/Prowlarr-indexers`).
+2. Copy the raw YAML.
+3. On `/indexers`, click **Add**, select type **Cardigann**, paste the YAML into a textarea.
+4. Fill in name + base URL manually.
+5. Save and test.
+
+This is friction that hides an entire class of usable sites behind expert knowledge. The underlying Cardigann parser already supports search-only public sites — the gap is pure UX.
+
+## Goal
+
+Add a curated "catalog" of 12 verified public torrent sites, installable in one click from a dedicated in-app page (and as an optional step in the onboarding wizard). Installation produces an ordinary `type=cardigann` indexer row — no new storage model, no parallel driver.
+
+## Non-goals
+
+- **No login / account-based sites**. The catalog is for truly public, search-only sites. Private trackers (account + passkey + cookies) remain in the existing Cardigann / UNIT3D / RarTracker flows.
+- **No automated site-health monitoring** beyond what the existing Test button provides.
+- **No auto-failover across mirrors** at runtime. The user picks one mirror at install time.
+- **No logo hosting**. v1 ships without per-site artwork.
+- **No dynamic fetching of definitions from upstream at runtime**. Deterministic builds matter more than freshness.
+
+## Shape of the solution
+
+### Catalog content
+
+Twelve sites, all present in `Prowlarr/Prowlarr-indexers`, all search-only, all no-account:
+
+| Slug | Display name | Focus |
+|------|---|---|
+| `thepiratebay` | The Pirate Bay | general |
+| `1337x` | 1337x | general |
+| `torrentgalaxy` | TorrentGalaxy | general |
+| `limetorrents` | LimeTorrents | general |
+| `magnetdl` | MagnetDL | general |
+| `torlock` | Torlock | general |
+| `bitsearch` | BitSearch | aggregator |
+| `solidtorrents` | SolidTorrents | aggregator |
+| `nyaa` | Nyaa | anime / asian |
+| `eztv` | EZTV | TV |
+| `yts` | YTS | movies (small-size encodes) |
+| `animetosho` | AnimeTosho | anime |
+
+The list is expected to evolve; the architecture makes adding an entry a one-YAML-plus-one-registry-line operation.
+
+### Storage layout
+
+```
+backend/src/trove/indexers/
+├── catalog/
+│ ├── registry.yaml # curated metadata for the 12 entries
+│ ├── thepiratebay.yml # vendored from Prowlarr-indexers
+│ ├── 1337x.yml
+│ ├── ... (10 more)
+│ └── yts.yml
+└── cardigann.py # existing parser, extended as needed
+```
+
+`registry.yaml` is the authoritative index. One entry per site:
+
+```yaml
+- slug: thepiratebay
+ display_name: The Pirate Bay
+ description: General-purpose public torrent tracker, no account required.
+ categories: [movies, tv, music, software, games, books]
+ yaml_file: thepiratebay.yml
+ mirrors:
+ - https://thepiratebay.org
+ - https://tpb.party
+ - https://piratebay.live
+ default_mirror: https://thepiratebay.org
+ protocol: torrent
+ logo: null
+```
+
+Fields:
+
+- `slug` — stable identifier. Matches filename stem, URL path segment, frontend route.
+- `display_name` — shown in UI.
+- `description` — one-sentence sell.
+- `categories` — list of `Category` enum values, used for filtering / search-routing hints.
+- `yaml_file` — filename within `catalog/` (relative).
+- `mirrors` — list of base URLs the site is known to respond on. Must include `default_mirror`. This is **our curated list**, not a passthrough of the vendored YAML's `links:` block — upstream often lists dead or unreliable mirrors, and we control what the UI exposes to the user.
+- `default_mirror` — pre-selected in the UI dropdown. Must be an element of `mirrors`.
+- `protocol` — `torrent` for all current entries, but kept explicit for future-proofing.
+- `logo` — always `null` in v1. Reserved for future use (path relative to a static directory).
+
+No database-level representation. The catalog is source code, not data. A user who installs an entry ends up with a plain `IndexerRow` of type `cardigann`, functionally indistinguishable from a hand-added one.
+
+### Vendoring and updates
+
+Definitions are **manually vendored** — the canonical source is `Prowlarr/Prowlarr-indexers`, but the files live in our repo. Updates are explicit:
+
+- `scripts/update-catalog.py` downloads the upstream tarball, compares SHA-256 of each vendored file against upstream, prints a per-file status: `unchanged` / `upstream changed` / `not found upstream`.
+- The maintainer reviews the diff, copies updated YAMLs, runs the test suite (notably the per-site fixture tests in §Testing), and commits.
+- Cadence is manual — monthly or on user bug report.
+
+Rejected alternatives:
+
+- **Git submodule**: pulls a ~500-file repo for our 12 files, and submodules generate friction for contributors.
+- **Build-time fetch**: breaks deterministic builds (same SHA can produce different images across time).
+
+### Backend
+
+**New service: `backend/src/trove/services/catalog.py`**
+
+Module-level cache of the parsed registry plus on-demand YAML reading:
+
+```python
+@dataclass(frozen=True, slots=True)
+class CatalogEntry:
+ slug: str
+ display_name: str
+ description: str
+ categories: list[Category]
+ yaml_file: str
+ mirrors: list[str]
+ default_mirror: str
+ protocol: Protocol
+
+def load_catalog() -> dict[str, CatalogEntry]: ... # cached on first call
+def list_entries() -> list[CatalogEntry]: ...
+def get_entry(slug: str) -> CatalogEntry: ... # raises KeyError
+def read_yaml(slug: str) -> str: ... # reads file from disk
+```
+
+The cache invalidates only when the module reloads (i.e. never in production, always in tests that clear `sys.modules`).
+
+**New endpoints in `backend/src/trove/api/indexers.py`**
+
+```
+GET /api/indexers/catalog → list[CatalogEntryOut]
+POST /api/indexers/catalog/{slug} → IndexerOut (201)
+ body: { base_url: str, name: str | null }
+```
+
+The API layer exposes a separate `CatalogEntryOut` Pydantic model that mirrors `CatalogEntry`'s fields and adds `already_installed: bool`. `already_installed` is computed at request time by querying `IndexerRow.catalog_slug == slug` (see marker column below) — a single row is enough, we don't count duplicates.
+
+**Tracking which rows came from the catalog**
+
+To compute `already_installed`, we need to know which existing Cardigann indexers originated from the catalog. Options considered:
+
+- Matching by `name == display_name` is fragile (user may have renamed).
+- Matching by `base_url` within the entry's `mirrors` list is more robust but still ambiguous if two entries share a mirror (none currently do).
+- **Chosen:** add a nullable `catalog_slug: str | None` column to `IndexerRow`. Set on catalog-installed rows, `null` for hand-added rows. Exact-match lookup is O(1) and survives renames.
+
+This requires an Alembic migration adding the column; existing rows get `null`.
+
+**`POST /api/indexers/catalog/{slug}` implementation**
+
+1. `entry = catalog.get_entry(slug)` → 404 if unknown.
+2. Validate `body.base_url in entry.mirrors` → 422 otherwise. Prevents using the catalog endpoint as a generic indexer-creator with arbitrary URLs.
+3. `yaml_text = catalog.read_yaml(slug)`; parse via `load_definition_yaml`. Errors here are our bug, not the user's — 500 with a diagnostic message.
+4. Resolve `name`: prefer `body.name`, otherwise `entry.display_name`. If taken, append `-2`, `-3`, … until free (silent dedup, not a 409).
+5. Construct `IndexerRow` with `type="cardigann"`, `protocol=entry.protocol`, `base_url=body.base_url`, `definition_yaml=yaml_text`, `catalog_slug=slug`, empty credentials, enabled=True, default priority.
+6. Insert, commit, return `IndexerOut`.
+
+### Frontend
+
+**New page: `web/src/routes/indexers/catalog/+page.svelte`**
+
+Grid of tiles, one per catalog entry. Each tile shows:
+
+- Generic icon (Lucide `Database`) in v1 — no per-site logos.
+- Display name.
+- Description.
+- Category badges.
+- Mirror `` pre-set to `default_mirror`.
+- **Add** button. Disabled and relabeled **Installed** when `already_installed` is true.
+
+Layout: CSS grid, min column width ~280px, wraps to single column on narrow viewports. No pagination (12 entries fit).
+
+Interaction:
+
+- Click **Add** → POST `/api/indexers/catalog/{slug}` with chosen `base_url`, `name: null`.
+- On 201 → toast "{display_name} added", flip the tile's `already_installed`, no navigation.
+- On error → in-tile error message, keep button enabled for retry.
+
+No search/filter input in v1. Add one when the catalog exceeds ~20 entries.
+
+**Update: `web/src/routes/indexers/+page.svelte`**
+
+Add a secondary button next to **Add indexer** labeled **Browse catalog**. Routes to `/indexers/catalog`. Existing form is untouched.
+
+**Onboarding: `web/src/routes/onboarding/+page.svelte`**
+
+Insert a new step between the welcome panel and the existing "Add your first indexer" step:
+
+- Title: "Public torrent sites (optional)".
+- Body: checkbox list of the 12 catalog entries, each with display name + one-line description.
+- **Add selected** button — sequentially POSTs to the catalog endpoint for each checked entry. 12 is small enough that no batching / progress bar is needed beyond a spinner on the button.
+- **Skip** continues to the next step.
+
+### Parser extensions
+
+The existing `cardigann.py` supports `replace`, `regexp`, `append`, `prepend`. Vendoring the 12 real YAMLs will surface features the current parser does not understand. Expected additions, based on a quick scan of a handful of Prowlarr definitions:
+
+- `urldecode` / `urlencode`
+- `split` (with `delimiter` and `index` args)
+- `trim` (unless already covered by `get_text(strip=True)`)
+- `querystring` (extract a specific query-string parameter — common for `download.php?id=…`)
+
+A per-row `filters` and `remove` block (as opposed to per-field) may also appear; if so, add a pass over the extracted row after the per-field extraction.
+
+**Principle:** add only what is needed to make one of the 12 YAMLs work correctly. Do not pre-implement the full Cardigann filter library. When a filter appears that isn't implemented yet, the `_apply_filter` fallback already returns the value unchanged — silent failures would mask bugs, so add a one-time warning log the first time an unknown filter name is seen per process.
+
+Time-related filters (`dateparse`, `timeparse`) are explicitly **deferred**. The `Release` dataclass has no `added_at` field today; implementing date parsing now would only paint over a missing field. Ignored date fields are logged once as debug-level.
+
+### Testing
+
+Three layers.
+
+**1. Parser unit tests** — `backend/tests/test_cardigann_filters.py`. One test per new filter implementation, with a mock HTML row and asserted extracted value.
+
+**2. Catalog integrity** — `backend/tests/test_catalog.py`:
+
+- Registry loads, has exactly the documented slugs (spot-check subset, not hard-coded full list — allows adding entries without test churn).
+- Every entry's `yaml_file` exists in the `catalog/` directory.
+- Every entry's YAML parses without raising `IndexerError`.
+- `default_mirror` is a member of `mirrors`.
+- No duplicate slugs.
+
+**3. Per-site fixture tests** — `backend/tests/fixtures/catalog/{slug}-search.html` contains a real HTML response captured once per site, and a corresponding test runs the parser against it and asserts ≥1 valid release with title, download URL, and size populated. Fixtures intentionally decouple tests from live internet and from CI reaching blocked torrent domains. When upstream changes a YAML, the matching fixture often needs to be re-captured — this is a feature, not a bug (forces manual review).
+
+No live integration tests. Public torrent sites are regularly blocked, throttled, or down — CI cannot depend on them.
+
+### Error handling and operational model
+
+- **Test button on catalog-installed rows**: reuses the existing `/api/indexers/{id}/test` endpoint. No catalog-specific behavior. Failures surface as ordinary HTTP errors in the existing UI.
+- **Mirror goes down**: symptom is a failing Test or 0-result searches. User fix is `Edit` → change `base_url` to another mirror from the YAML's `links` block. In v1 the edit dialog does not surface the mirror list; pasting a new URL is the workflow. (A future enhancement can show a dropdown for rows with a `catalog_slug` set.)
+- **Site changes HTML**: symptom is 0-result searches on a site that used to work. No auto-detection. Remediation: run `scripts/update-catalog.py`, review the diff, pull the updated YAML, commit, ship.
+- **Catalog YAML is malformed after an update** (rare): caught by `test_catalog.py` in CI before merge. If it slips, the API returns 500 and the user sees a loud error; they can uninstall and wait for a hotfix.
+- **Slug collision when installing twice**: the name-dedup logic appends `-2`, `-3`. The `catalog_slug` column is not unique — two rows pointing at different mirrors of The Pirate Bay are legal.
+
+### Rollout
+
+Single branch, single PR. Order of commits within the branch:
+
+1. Alembic migration: add `catalog_slug` column.
+2. Vendor the 12 YAML files + write `registry.yaml`.
+3. Extend the Cardigann parser with the filters the 12 files require (driven by failing per-site fixture tests).
+4. Add `services/catalog.py` and the two API endpoints.
+5. Add the `/indexers/catalog` Svelte page + the "Browse catalog" button on `/indexers`.
+6. Add the onboarding step.
+7. Add `scripts/update-catalog.py`.
+
+Each commit is shippable on its own — the API is useful before the UI ships, and the UI works on top of an empty registry during development.
+
+## Open questions (deferred, not blockers)
+
+- Do we want a "Refresh catalog" button in the app that runs `update-catalog.py` behind the scenes? Rejected for v1 — manual maintainer workflow is safer.
+- Should we show per-site health trends on the catalog page (e.g., "3 users reported this mirror down this week")? Out of scope; depends on telemetry we don't collect.
+- Should the onboarding step default to all 12 pre-checked or all unchecked? Leaning unchecked — opt-in beats opt-out for anything that calls third-party services.
From 3bed0302f37a21c039aa5695e8f55953e7794343 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 11:12:50 +0200
Subject: [PATCH 02/39] docs: implementation plan for public torrent-site
catalog
Bite-sized task decomposition with code + test + commit steps for each
chunk: Alembic migration, catalog registry + vendoring script, five
new Cardigann parser filters (TDD), two new API endpoints, catalog
page, /indexers shortcut, onboarding step, and a parametrized fixture
test harness. References the spec committed in d676ec4.
Co-Authored-By: Claude Opus 4.7 (1M context)
---
.../2026-04-20-public-torrent-site-catalog.md | 2073 +++++++++++++++++
1 file changed, 2073 insertions(+)
create mode 100644 docs/superpowers/plans/2026-04-20-public-torrent-site-catalog.md
diff --git a/docs/superpowers/plans/2026-04-20-public-torrent-site-catalog.md b/docs/superpowers/plans/2026-04-20-public-torrent-site-catalog.md
new file mode 100644
index 0000000..1ee95a9
--- /dev/null
+++ b/docs/superpowers/plans/2026-04-20-public-torrent-site-catalog.md
@@ -0,0 +1,2073 @@
+# Public Torrent Site Catalog Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Ship a curated, built-in catalog of 12 public (no-account) torrent sites installable with one click from a dedicated `/indexers/catalog` page and as an optional step in the onboarding wizard.
+
+**Architecture:** A new `backend/src/trove/indexers/catalog/` directory holds a hand-written `registry.yaml` plus twelve vendored Cardigann `.yml` definitions. A new `services/catalog.py` module exposes the catalog, and two new endpoints (`GET /api/indexers/catalog`, `POST /api/indexers/catalog/{slug}`) create ordinary `type=cardigann` indexer rows — the only storage change is a nullable `catalog_slug` marker column. The existing Cardigann parser is extended with a handful of new filter types (driven by what the vendored YAMLs actually use).
+
+**Tech Stack:** Python 3.12, FastAPI, SQLModel, Alembic, httpx, PyYAML, BeautifulSoup4/lxml; SvelteKit 5 + TypeScript; pytest + respx for tests.
+
+**Spec:** `docs/superpowers/specs/2026-04-20-public-torrent-site-catalog-design.md`
+
+---
+
+## File Structure
+
+**New files:**
+
+- `backend/migrations/versions/0016_indexer_catalog_slug.py` — Alembic migration adding the column
+- `backend/src/trove/indexers/catalog/__init__.py` — empty marker
+- `backend/src/trove/indexers/catalog/registry.yaml` — curated metadata (authoritative)
+- `backend/src/trove/indexers/catalog/*.yml` — 12 vendored Prowlarr definitions (downloaded via script)
+- `backend/src/trove/services/catalog.py` — registry loader + YAML reader
+- `backend/src/trove/api/catalog.py` — new router with two endpoints (lives next to `indexers.py`, mounted under the same prefix)
+- `backend/tests/test_catalog.py` — registry integrity + per-YAML parse tests
+- `backend/tests/test_cardigann_filters.py` — unit tests for new parser filters
+- `backend/tests/api/test_catalog_api.py` — endpoint tests
+- `backend/tests/fixtures/catalog/` — captured HTML responses, one per site
+- `scripts/update-catalog.py` — downloads/diffs vendored YAMLs against upstream
+- `web/src/routes/indexers/catalog/+page.svelte` — tile-grid catalog page
+
+**Modified files:**
+
+- `backend/src/trove/models/indexer.py` — add `catalog_slug` field to `IndexerRow`
+- `backend/src/trove/indexers/cardigann.py` — extend `_apply_filter` with new filter types and an unknown-filter warn log
+- `backend/src/trove/main.py` — mount the new catalog router
+- `web/src/lib/api.ts` — add `CatalogEntryOut` type + `api.indexers.catalog.*` methods
+- `web/src/routes/indexers/+page.svelte` — add "Browse catalog" button
+- `web/src/routes/onboarding/+page.svelte` — insert a new "Public torrent sites" step
+- `backend/src/trove/docs/03-indexers.md` — document the catalog
+
+---
+
+## Task 1: Add `catalog_slug` column to `IndexerRow` model
+
+**Files:**
+- Modify: `backend/src/trove/models/indexer.py:12-27`
+
+- [ ] **Step 1: Add the field**
+
+Edit `backend/src/trove/models/indexer.py` — add one line inside the `IndexerRow` class, directly after `last_test_message`:
+
+```python
+ catalog_slug: str | None = Field(default=None, max_length=64, index=True)
+```
+
+- [ ] **Step 2: Verify model import still works**
+
+Run: `cd backend && uv run python -c "from trove.models.indexer import IndexerRow; print(IndexerRow.__fields__.keys())"`
+Expected: the printed list includes `catalog_slug`.
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add backend/src/trove/models/indexer.py
+git commit -m "feat: add catalog_slug field to IndexerRow"
+```
+
+---
+
+## Task 2: Alembic migration for the new column
+
+**Files:**
+- Create: `backend/migrations/versions/0016_indexer_catalog_slug.py`
+
+- [ ] **Step 1: Write the migration**
+
+Create `backend/migrations/versions/0016_indexer_catalog_slug.py`:
+
+```python
+"""indexer.catalog_slug column
+
+Revision ID: 0016
+Revises: 0015
+Create Date: 2026-04-20
+
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+revision: str = "0016"
+down_revision: str | None = "0015"
+branch_labels: str | None = None
+depends_on: str | None = None
+
+
+def upgrade() -> None:
+ with op.batch_alter_table("indexer") as batch:
+ batch.add_column(
+ sa.Column(
+ "catalog_slug",
+ sqlmodel.sql.sqltypes.AutoString(length=64),
+ nullable=True,
+ )
+ )
+ op.create_index(
+ "ix_indexer_catalog_slug", "indexer", ["catalog_slug"], unique=False
+ )
+
+
+def downgrade() -> None:
+ op.drop_index("ix_indexer_catalog_slug", table_name="indexer")
+ with op.batch_alter_table("indexer") as batch:
+ batch.drop_column("catalog_slug")
+```
+
+`op.batch_alter_table` is required on SQLite because ALTER is restricted; Trove uses SQLite in WAL mode.
+
+- [ ] **Step 2: Apply migration against a scratch DB**
+
+Run: `cd backend && rm -f /tmp/trove-mig-test.db && TROVE_CONFIG_DIR=/tmp/trove-mig-test uv run alembic upgrade head`
+Expected: last line `INFO [alembic.runtime.migration] Running upgrade 0015 -> 0016, indexer.catalog_slug column`.
+
+- [ ] **Step 3: Roll it back, then forward again**
+
+Run: `cd backend && TROVE_CONFIG_DIR=/tmp/trove-mig-test uv run alembic downgrade 0015 && TROVE_CONFIG_DIR=/tmp/trove-mig-test uv run alembic upgrade head`
+Expected: both steps complete without errors.
+
+- [ ] **Step 4: Commit**
+
+```bash
+git add backend/migrations/versions/0016_indexer_catalog_slug.py
+git commit -m "feat: migration adds indexer.catalog_slug column"
+```
+
+---
+
+## Task 3: Write `registry.yaml`
+
+**Files:**
+- Create: `backend/src/trove/indexers/catalog/__init__.py`
+- Create: `backend/src/trove/indexers/catalog/registry.yaml`
+
+- [ ] **Step 1: Create the package marker**
+
+Create `backend/src/trove/indexers/catalog/__init__.py` as an empty file:
+
+```python
+```
+
+- [ ] **Step 2: Write the registry**
+
+Create `backend/src/trove/indexers/catalog/registry.yaml`:
+
+```yaml
+# Authoritative index of public no-account torrent sites shipped with Trove.
+# Each entry references a vendored Cardigann YAML in the same directory.
+#
+# Filenames under `yaml_file:` are OUR vendored filenames — they need not
+# match upstream. `upstream_path:` records where the definition came from
+# so scripts/update-catalog.py can diff against Prowlarr-indexers.
+
+entries:
+ - slug: thepiratebay
+ display_name: The Pirate Bay
+ description: General-purpose public torrent tracker, no account required.
+ categories: [movies, tv, music, software, games, books, other]
+ yaml_file: thepiratebay.yml
+ upstream_path: definitions/v11/thepiratebay.yml
+ mirrors:
+ - https://thepiratebay.org
+ - https://tpb.party
+ - https://piratebay.live
+ default_mirror: https://thepiratebay.org
+ protocol: torrent
+
+ - slug: 1337x
+ display_name: 1337x
+ description: Large public general-purpose tracker with strong TV/movie scene coverage.
+ categories: [movies, tv, music, software, games, books, anime]
+ yaml_file: 1337x.yml
+ upstream_path: definitions/v11/1337x.yml
+ mirrors:
+ - https://1337x.to
+ - https://1337x.st
+ - https://1337x.tw
+ default_mirror: https://1337x.to
+ protocol: torrent
+
+ - slug: torrentgalaxy
+ display_name: TorrentGalaxy
+ description: General-purpose public tracker with reliable scene releases and good category filtering.
+ categories: [movies, tv, music, software, games, books, anime, other]
+ yaml_file: torrentgalaxy.yml
+ upstream_path: definitions/v11/torrentgalaxy.yml
+ mirrors:
+ - https://torrentgalaxy.to
+ - https://tgx.rs
+ default_mirror: https://torrentgalaxy.to
+ protocol: torrent
+
+ - slug: limetorrents
+ display_name: LimeTorrents
+ description: Long-running public aggregator with a wide catalogue.
+ categories: [movies, tv, music, software, games, anime, other]
+ yaml_file: limetorrents.yml
+ upstream_path: definitions/v11/limetorrents.yml
+ mirrors:
+ - https://www.limetorrents.lol
+ - https://www.limetorrents.info
+ default_mirror: https://www.limetorrents.lol
+ protocol: torrent
+
+ - slug: magnetdl
+ display_name: MagnetDL
+ description: Magnet-link aggregator, fast and light.
+ categories: [movies, tv, music, software, games, books, anime]
+ yaml_file: magnetdl.yml
+ upstream_path: definitions/v11/magnetdl.yml
+ mirrors:
+ - https://www.magnetdl.com
+ default_mirror: https://www.magnetdl.com
+ protocol: torrent
+
+ - slug: torlock
+ display_name: Torlock
+ description: General-purpose tracker focused on verified torrents.
+ categories: [movies, tv, music, software, games, books, anime]
+ yaml_file: torlock.yml
+ upstream_path: definitions/v11/torlock.yml
+ mirrors:
+ - https://www.torlock.com
+ default_mirror: https://www.torlock.com
+ protocol: torrent
+
+ - slug: bitsearch
+ display_name: BitSearch
+ description: Aggregator that searches across multiple public sites.
+ categories: [movies, tv, music, software, games, books, anime, other]
+ yaml_file: bitsearch.yml
+ upstream_path: definitions/v11/bitsearch.yml
+ mirrors:
+ - https://bitsearch.to
+ default_mirror: https://bitsearch.to
+ protocol: torrent
+
+ - slug: solidtorrents
+ display_name: SolidTorrents
+ description: Multi-source torrent search aggregator.
+ categories: [movies, tv, music, software, games, books, anime, other]
+ yaml_file: solidtorrents.yml
+ upstream_path: definitions/v11/solidtorrents.yml
+ mirrors:
+ - https://solidtorrents.to
+ default_mirror: https://solidtorrents.to
+ protocol: torrent
+
+ - slug: nyaa
+ display_name: Nyaa
+ description: The largest public anime & manga tracker.
+ categories: [anime, music, books, other]
+ yaml_file: nyaa.yml
+ upstream_path: definitions/v11/nyaasi.yml
+ mirrors:
+ - https://nyaa.si
+ default_mirror: https://nyaa.si
+ protocol: torrent
+
+ - slug: eztv
+ display_name: EZTV
+ description: TV-focused public tracker with strong scene release coverage.
+ categories: [tv]
+ yaml_file: eztv.yml
+ upstream_path: definitions/v11/eztv.yml
+ mirrors:
+ - https://eztv.re
+ - https://eztvx.to
+ default_mirror: https://eztv.re
+ protocol: torrent
+
+ - slug: yts
+ display_name: YTS
+ description: Public tracker specializing in small-size movie encodes.
+ categories: [movies]
+ yaml_file: yts.yml
+ upstream_path: definitions/v11/yts.yml
+ mirrors:
+ - https://yts.mx
+ default_mirror: https://yts.mx
+ protocol: torrent
+
+ - slug: animetosho
+ display_name: AnimeTosho
+ description: Anime mirror and long-term archive of Nyaa + Tokyo Toshokan.
+ categories: [anime]
+ yaml_file: animetosho.yml
+ upstream_path: definitions/v11/animetosho.yml
+ mirrors:
+ - https://animetosho.org
+ default_mirror: https://animetosho.org
+ protocol: torrent
+```
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add backend/src/trove/indexers/catalog/__init__.py backend/src/trove/indexers/catalog/registry.yaml
+git commit -m "feat: add catalog registry for 12 public torrent sites"
+```
+
+---
+
+## Task 4: Write `services/catalog.py`
+
+**Files:**
+- Create: `backend/src/trove/services/catalog.py`
+
+- [ ] **Step 1: Write the module**
+
+Create `backend/src/trove/services/catalog.py`:
+
+```python
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+from trove.clients.base import Protocol
+from trove.indexers.base import Category
+
+_CATALOG_DIR = Path(__file__).parent.parent / "indexers" / "catalog"
+
+
+class CatalogError(Exception):
+ """Raised when the shipped catalog is malformed. This is always our bug,
+ never user input — the registry is vendored code."""
+
+
+@dataclass(frozen=True, slots=True)
+class CatalogEntry:
+ slug: str
+ display_name: str
+ description: str
+ categories: list[Category]
+ yaml_file: str
+ upstream_path: str
+ mirrors: list[str]
+ default_mirror: str
+ protocol: Protocol
+ logo: str | None = None
+
+
+@lru_cache(maxsize=1)
+def load_catalog() -> dict[str, CatalogEntry]:
+ registry_path = _CATALOG_DIR / "registry.yaml"
+ if not registry_path.exists():
+ raise CatalogError(f"catalog registry missing at {registry_path}")
+ raw = yaml.safe_load(registry_path.read_text(encoding="utf-8")) or {}
+ entries_raw = raw.get("entries") or []
+ if not isinstance(entries_raw, list):
+ raise CatalogError("registry.yaml: `entries` must be a list")
+
+ by_slug: dict[str, CatalogEntry] = {}
+ for row in entries_raw:
+ if not isinstance(row, dict):
+ raise CatalogError("registry.yaml: each entry must be a mapping")
+ slug = row.get("slug")
+ if not slug or not isinstance(slug, str):
+ raise CatalogError("registry.yaml: entry missing `slug`")
+ if slug in by_slug:
+ raise CatalogError(f"registry.yaml: duplicate slug {slug!r}")
+
+ try:
+ categories = [Category(c) for c in row.get("categories") or []]
+ except ValueError as e:
+ raise CatalogError(f"registry.yaml: {slug}: unknown category {e}") from e
+ try:
+ protocol = Protocol(row.get("protocol") or "torrent")
+ except ValueError as e:
+ raise CatalogError(f"registry.yaml: {slug}: unknown protocol {e}") from e
+
+ mirrors = list(row.get("mirrors") or [])
+ default_mirror = row.get("default_mirror") or ""
+ if not mirrors:
+ raise CatalogError(f"registry.yaml: {slug}: at least one mirror required")
+ if default_mirror not in mirrors:
+ raise CatalogError(
+ f"registry.yaml: {slug}: default_mirror must be a member of mirrors"
+ )
+
+ by_slug[slug] = CatalogEntry(
+ slug=slug,
+ display_name=str(row.get("display_name") or slug),
+ description=str(row.get("description") or ""),
+ categories=categories,
+ yaml_file=str(row.get("yaml_file") or f"{slug}.yml"),
+ upstream_path=str(row.get("upstream_path") or ""),
+ mirrors=mirrors,
+ default_mirror=default_mirror,
+ protocol=protocol,
+ logo=row.get("logo"),
+ )
+
+ return by_slug
+
+
+def list_entries() -> list[CatalogEntry]:
+ return list(load_catalog().values())
+
+
+def get_entry(slug: str) -> CatalogEntry:
+ entries = load_catalog()
+ if slug not in entries:
+ raise KeyError(slug)
+ return entries[slug]
+
+
+def read_yaml(slug: str) -> str:
+ entry = get_entry(slug)
+ path = _CATALOG_DIR / entry.yaml_file
+ if not path.exists():
+ raise CatalogError(f"catalog: {slug}: missing yaml file at {path}")
+ return path.read_text(encoding="utf-8")
+
+
+def reset_cache_for_tests() -> None:
+ """pytest hook — the module-level cache would otherwise outlive test DBs."""
+ load_catalog.cache_clear()
+```
+
+- [ ] **Step 2: Sanity-load**
+
+Run: `cd backend && uv run python -c "from trove.services import catalog; print(len(catalog.list_entries()))"`
+Expected: `12`.
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add backend/src/trove/services/catalog.py
+git commit -m "feat: catalog service loads registry + vendored YAMLs"
+```
+
+---
+
+## Task 5: Catalog integrity test (pre-vendoring)
+
+**Files:**
+- Create: `backend/tests/test_catalog.py`
+
+- [ ] **Step 1: Write the failing test**
+
+Create `backend/tests/test_catalog.py`:
+
+```python
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from trove.services import catalog
+
+CATALOG_DIR = Path(catalog.__file__).parent.parent / "indexers" / "catalog"
+
+
+@pytest.fixture(autouse=True)
+def _reset_cache() -> None:
+ catalog.reset_cache_for_tests()
+
+
+def test_registry_loads() -> None:
+ entries = catalog.list_entries()
+ assert len(entries) >= 12
+ slugs = {e.slug for e in entries}
+ for required in ("thepiratebay", "1337x", "nyaa", "eztv", "yts"):
+ assert required in slugs, f"missing catalog entry: {required}"
+
+
+def test_default_mirror_is_in_mirrors() -> None:
+ for entry in catalog.list_entries():
+ assert entry.default_mirror in entry.mirrors, (
+ f"{entry.slug}: default_mirror {entry.default_mirror!r} "
+ f"not present in mirrors {entry.mirrors}"
+ )
+
+
+def test_every_entry_has_a_vendored_yaml_file() -> None:
+ missing: list[str] = []
+ for entry in catalog.list_entries():
+ path = CATALOG_DIR / entry.yaml_file
+ if not path.exists():
+ missing.append(f"{entry.slug} -> {entry.yaml_file}")
+ assert not missing, "missing vendored YAML files: " + ", ".join(missing)
+
+
+def test_every_vendored_yaml_parses() -> None:
+ from trove.indexers.cardigann import load_definition_yaml
+
+ failures: list[str] = []
+ for entry in catalog.list_entries():
+ try:
+ load_definition_yaml(catalog.read_yaml(entry.slug))
+ except Exception as e: # noqa: BLE001
+ failures.append(f"{entry.slug}: {type(e).__name__}: {e}")
+ assert not failures, "YAMLs failed to parse:\n " + "\n ".join(failures)
+```
+
+- [ ] **Step 2: Run — expect failure on vendored-files test**
+
+Run: `cd backend && uv run pytest tests/test_catalog.py -v`
+Expected: `test_registry_loads` and `test_default_mirror_is_in_mirrors` PASS. `test_every_entry_has_a_vendored_yaml_file` and `test_every_vendored_yaml_parses` FAIL with "missing vendored YAML files".
+
+This is the correct state before vendoring — don't fix yet.
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add backend/tests/test_catalog.py
+git commit -m "test: catalog integrity checks (pre-vendor, expected to fail)"
+```
+
+---
+
+## Task 6: Write `scripts/update-catalog.py`
+
+**Files:**
+- Create: `scripts/update-catalog.py`
+
+- [ ] **Step 1: Write the script**
+
+Create `scripts/update-catalog.py`:
+
+```python
+#!/usr/bin/env python3
+"""Vendor or diff Cardigann YAML definitions from Prowlarr-indexers.
+
+Usage:
+ scripts/update-catalog.py sync # download + overwrite vendored files
+ scripts/update-catalog.py diff # print per-file status, no writes
+
+The canonical upstream is Prowlarr/Prowlarr-indexers @ master. Slug→path
+mapping lives in backend/src/trove/indexers/catalog/registry.yaml.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import sys
+from pathlib import Path
+
+import httpx
+import yaml
+
+REPO_ROOT = Path(__file__).parent.parent
+CATALOG_DIR = REPO_ROOT / "backend" / "src" / "trove" / "indexers" / "catalog"
+REGISTRY_PATH = CATALOG_DIR / "registry.yaml"
+UPSTREAM_RAW = "https://raw.githubusercontent.com/Prowlarr/Prowlarr-indexers/master/"
+
+
+def _sha256(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+def _load_registry() -> list[dict[str, str]]:
+ raw = yaml.safe_load(REGISTRY_PATH.read_text(encoding="utf-8")) or {}
+ entries = raw.get("entries") or []
+ out: list[dict[str, str]] = []
+ for row in entries:
+ upstream = row.get("upstream_path")
+ if not upstream:
+ continue
+ out.append(
+ {
+ "slug": row["slug"],
+ "yaml_file": row["yaml_file"],
+ "upstream_path": upstream,
+ }
+ )
+ return out
+
+
+def _fetch(client: httpx.Client, upstream_path: str) -> bytes | None:
+ url = UPSTREAM_RAW + upstream_path
+ resp = client.get(url, timeout=30.0)
+ if resp.status_code == 404:
+ return None
+ resp.raise_for_status()
+ return resp.content
+
+
+def run(mode: str) -> int:
+ entries = _load_registry()
+ with httpx.Client(follow_redirects=True) as client:
+ changes = 0
+ missing = 0
+ for entry in entries:
+ slug = entry["slug"]
+ local_path = CATALOG_DIR / entry["yaml_file"]
+ upstream_bytes = _fetch(client, entry["upstream_path"])
+ if upstream_bytes is None:
+ print(f" [MISSING] {slug}: upstream {entry['upstream_path']} not found")
+ missing += 1
+ continue
+
+ local_hash = _sha256(local_path.read_bytes()) if local_path.exists() else None
+ upstream_hash = _sha256(upstream_bytes)
+
+ if local_hash == upstream_hash:
+ print(f" [unchanged] {slug}")
+ continue
+
+ changes += 1
+ if mode == "sync":
+ local_path.write_bytes(upstream_bytes)
+ state = "created" if local_hash is None else "updated"
+ print(f" [{state}] {slug}")
+ else:
+ state = "missing locally" if local_hash is None else "upstream changed"
+ print(f" [{state}] {slug}")
+
+ print(f"\n{changes} change(s), {missing} missing upstream")
+ return 1 if missing else 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("mode", choices=["sync", "diff"])
+ args = parser.parse_args()
+ return run(args.mode)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
+```
+
+- [ ] **Step 2: Mark executable**
+
+Run: `chmod +x scripts/update-catalog.py`
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add scripts/update-catalog.py
+git commit -m "feat: script to sync/diff vendored catalog YAMLs with upstream"
+```
+
+---
+
+## Task 7: Run `update-catalog.py sync` to vendor the 12 YAMLs
+
+**Files:**
+- Create: `backend/src/trove/indexers/catalog/thepiratebay.yml` (and 11 others)
+
+- [ ] **Step 1: Run the script**
+
+Run: `cd /home/masterdraco/trove && ./scripts/update-catalog.py sync`
+Expected: 12 `[created]` lines, `12 change(s), 0 missing upstream`.
+
+If any entry reports `[MISSING]`, the `upstream_path` in `registry.yaml` is wrong for that slug — look on GitHub under `Prowlarr/Prowlarr-indexers/tree/master/definitions` for the correct filename (it may live under `v10` instead of `v11`, or have a slightly different spelling), fix `registry.yaml`, re-run.
+
+- [ ] **Step 2: Spot-check one YAML**
+
+Run: `head -40 backend/src/trove/indexers/catalog/thepiratebay.yml`
+Expected: a Cardigann YAML document starting with `---` or `id:` / `name:` / `type:` fields.
+
+- [ ] **Step 3: Commit the vendored files and any fixed upstream paths**
+
+```bash
+git add backend/src/trove/indexers/catalog/*.yml backend/src/trove/indexers/catalog/registry.yaml
+git commit -m "vendor: import 12 Cardigann YAML definitions from Prowlarr-indexers"
+```
+
+---
+
+## Task 8: Verify catalog parse test now passes (or surfaces missing filters)
+
+**Files:**
+- None (investigation only)
+
+- [ ] **Step 1: Run catalog tests**
+
+Run: `cd backend && uv run pytest tests/test_catalog.py -v`
+Expected: all four tests PASS **if** the vendored YAMLs only use filters the existing parser understands.
+
+If `test_every_vendored_yaml_parses` fails, read the error — `load_definition_yaml` raises `IndexerError` with descriptive messages for structural problems. Common failure modes:
+- `has no search.paths` → upstream may have changed schema; fix in Task 15.
+- `has no search.rows.selector` → same.
+
+These structural issues are rare; most failures in this phase come from filter-parsing. The filter layer only emits warnings today, so Task 8 may pass even when unknown filters are present (silently returning the unfiltered value). Tasks 9–13 fix that.
+
+- [ ] **Step 2: Scan the vendored files for filter names in use**
+
+Run: `cd backend && uv run python -c "
+import pathlib, re
+from trove.services import catalog
+wanted = set()
+for e in catalog.list_entries():
+ text = catalog.read_yaml(e.slug)
+ for m in re.finditer(r'name:\s*(\w+)', text):
+ wanted.add(m.group(1))
+print(sorted(wanted))
+"`
+Expected: a sorted list. Note any unfamiliar names — cross-reference with Tasks 9–14 to see what's already covered.
+
+Known-covered filters: `replace`, `regexp`, `append`, `prepend`.
+Tasks 9–13 cover: `urldecode`, `urlencode`, `split`, `trim`, `querystring`.
+Anything *else* in the list is an additional filter — write a TDD task for it using the template in Tasks 9–13 (failing test in `test_cardigann_filters.py` + one branch in `_apply_filter` + passing test + commit) and slot it in between Tasks 13 and 14.
+
+No commit — this step is investigation.
+
+---
+
+## Task 9: Add `urldecode` / `urlencode` filters to Cardigann parser
+
+**Files:**
+- Create: `backend/tests/test_cardigann_filters.py`
+- Modify: `backend/src/trove/indexers/cardigann.py:256-270`
+
+- [ ] **Step 1: Write the failing tests**
+
+Create `backend/tests/test_cardigann_filters.py`:
+
+```python
+from __future__ import annotations
+
+from bs4 import BeautifulSoup
+
+from trove.indexers.cardigann import (
+ CardigannDefinition,
+ CardigannIndexer,
+ FieldSpec,
+ load_definition_yaml,
+)
+
+
+def _driver_with_field(name: str, filters: list[dict]) -> CardigannIndexer:
+ definition = CardigannDefinition(
+ site="test",
+ name="test",
+ links=["https://test.local"],
+ search_path="/",
+ search_params={},
+ rows_selector="tr",
+ fields={name: FieldSpec(selector="td", filters=filters)},
+ )
+ return CardigannIndexer(definition)
+
+
+def _apply(driver: CardigannIndexer, html: str, field: str) -> str | None:
+ row = BeautifulSoup(html, "lxml").find("tr")
+ return driver._extract_field(row, field)
+
+
+def test_urldecode() -> None:
+ drv = _driver_with_field("title", [{"name": "urldecode"}])
+ assert _apply(drv, "Hello%20World ", "title") == "Hello World"
+
+
+def test_urlencode() -> None:
+ drv = _driver_with_field("title", [{"name": "urlencode"}])
+ assert _apply(drv, "Hello World ", "title") == "Hello%20World"
+```
+
+- [ ] **Step 2: Run — expect fail**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v`
+Expected: `test_urldecode` PASSES unexpectedly (fallthrough returns value unchanged, which happens to equal the literal input for `urldecode` only when there's nothing to decode — but our input contains `%20`, so it should FAIL). `test_urlencode` FAILS — fallthrough returns unchanged.
+
+If both PASS because the fallthrough returns value unchanged and your inputs happen to match, that's still the wrong behavior: the filters are supposed to transform. Proceed to Step 3.
+
+- [ ] **Step 3: Implement the filters**
+
+Edit `backend/src/trove/indexers/cardigann.py` — extend `_apply_filter` (around line 256). Add these two branches **before** the final `return value`:
+
+```python
+ if name == "urldecode":
+ from urllib.parse import unquote
+ return unquote(value)
+ if name == "urlencode":
+ from urllib.parse import quote
+ return quote(value, safe="")
+```
+
+- [ ] **Step 4: Run — expect pass**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "urldecode or urlencode"`
+Expected: both PASS.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py
+git commit -m "feat: cardigann urldecode/urlencode filters"
+```
+
+---
+
+## Task 10: Add `split` filter
+
+**Files:**
+- Modify: `backend/tests/test_cardigann_filters.py`
+- Modify: `backend/src/trove/indexers/cardigann.py`
+
+- [ ] **Step 1: Write the failing test**
+
+Append to `backend/tests/test_cardigann_filters.py`:
+
+```python
+def test_split_by_delimiter_index() -> None:
+ drv = _driver_with_field(
+ "size",
+ [{"name": "split", "args": ["|", 1]}],
+ )
+ # "1.2 GB | 42 seeders | 3 leechers" -> split on '|', index 1 -> " 42 seeders "
+ assert _apply(drv, "1.2 GB | 42 seeders | 3 leechers ", "size").strip() == "42 seeders"
+
+
+def test_split_negative_index_returns_last() -> None:
+ drv = _driver_with_field(
+ "size",
+ [{"name": "split", "args": ["|", -1]}],
+ )
+ # index -1 -> last chunk
+ assert _apply(drv, "a|b|c ", "size") == "c"
+```
+
+- [ ] **Step 2: Run — expect fail**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py::test_split_by_delimiter_index -v`
+Expected: FAIL — returns the unsplit string.
+
+- [ ] **Step 3: Implement**
+
+Edit `backend/src/trove/indexers/cardigann.py` — add to `_apply_filter`, before the final `return value`:
+
+```python
+ if name == "split" and isinstance(args, list) and len(args) >= 2:
+ delimiter = str(args[0])
+ try:
+ index = int(args[1])
+ except (TypeError, ValueError):
+ return value
+ parts = value.split(delimiter)
+ if not parts:
+ return value
+ try:
+ return parts[index]
+ except IndexError:
+ return value
+```
+
+- [ ] **Step 4: Run — expect pass**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "split"`
+Expected: both split tests PASS.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py
+git commit -m "feat: cardigann split filter"
+```
+
+---
+
+## Task 11: Add `trim` filter
+
+**Files:**
+- Modify: `backend/tests/test_cardigann_filters.py`
+- Modify: `backend/src/trove/indexers/cardigann.py`
+
+- [ ] **Step 1: Write the failing test**
+
+Append to `backend/tests/test_cardigann_filters.py`:
+
+```python
+def test_trim() -> None:
+ drv = _driver_with_field("title", [{"name": "trim"}])
+ # get_text(" ", strip=True) strips leading/trailing whitespace already,
+ # but real use is *after* another filter (e.g. split) has reintroduced whitespace.
+ drv2 = _driver_with_field(
+ "title",
+ [{"name": "split", "args": ["|", 0]}, {"name": "trim"}],
+ )
+ assert _apply(drv2, " hello | world ", "title") == "hello"
+
+
+def test_trim_with_args_strips_specific_chars() -> None:
+ drv = _driver_with_field("title", [{"name": "trim", "args": "/"}])
+ # emulate "/path/to/thing/" -> "path/to/thing"
+ drv2 = _driver_with_field(
+ "title",
+ [{"name": "split", "args": [" | ", 0]}, {"name": "trim", "args": "/"}],
+ )
+ assert _apply(drv2, "/path/to/thing/ | rest ", "title") == "path/to/thing"
+```
+
+- [ ] **Step 2: Run — expect fail**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "trim"`
+Expected: FAIL — `test_trim` passes by accident (the text-extraction already strips), `test_trim_with_args_strips_specific_chars` FAILS.
+
+- [ ] **Step 3: Implement**
+
+Edit `backend/src/trove/indexers/cardigann.py`, add to `_apply_filter`:
+
+```python
+ if name == "trim":
+ if isinstance(args, str) and args:
+ return value.strip(args)
+ return value.strip()
+```
+
+- [ ] **Step 4: Run — expect pass**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "trim"`
+Expected: both PASS.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py
+git commit -m "feat: cardigann trim filter"
+```
+
+---
+
+## Task 12: Add `querystring` filter
+
+**Files:**
+- Modify: `backend/tests/test_cardigann_filters.py`
+- Modify: `backend/src/trove/indexers/cardigann.py`
+
+- [ ] **Step 1: Write the failing test**
+
+Append:
+
+```python
+def test_querystring_extract_param() -> None:
+ drv = _driver_with_field(
+ "infohash",
+ [{"name": "querystring", "args": "id"}],
+ )
+ assert _apply(
+ drv,
+ 'link ',
+ "infohash",
+ ) is None # spec: driver extracts from text — this case pulls text of the cell, not href.
+
+
+def test_querystring_on_href_value() -> None:
+ drv = _driver_with_field(
+ "infohash",
+ [{"name": "querystring", "args": "id"}],
+ )
+ # When combined with attribute:"href", the filter receives the URL string.
+ definition = CardigannDefinition(
+ site="t",
+ name="t",
+ links=["https://t.local"],
+ search_path="/",
+ search_params={},
+ rows_selector="tr",
+ fields={
+ "infohash": FieldSpec(
+ selector="a",
+ attribute="href",
+ filters=[{"name": "querystring", "args": "id"}],
+ )
+ },
+ )
+ drv = CardigannIndexer(definition)
+ row = BeautifulSoup(
+ 'link ',
+ "lxml",
+ ).find("tr")
+ assert drv._extract_field(row, "infohash") == "abc123"
+```
+
+- [ ] **Step 2: Run — expect fail**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "querystring"`
+Expected: `test_querystring_on_href_value` FAILS (returns the full URL unchanged).
+
+- [ ] **Step 3: Implement**
+
+Add to `_apply_filter`:
+
+```python
+ if name == "querystring" and isinstance(args, str):
+ from urllib.parse import parse_qs, urlparse
+ parsed = urlparse(value)
+ params = parse_qs(parsed.query)
+ picks = params.get(args)
+ return picks[0] if picks else value
+```
+
+- [ ] **Step 4: Run — expect pass**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "querystring"`
+Expected: both PASS.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py
+git commit -m "feat: cardigann querystring filter"
+```
+
+---
+
+## Task 13: Warn once per process on unknown filter names
+
+**Files:**
+- Modify: `backend/src/trove/indexers/cardigann.py`
+
+- [ ] **Step 1: Write the failing test**
+
+Append to `backend/tests/test_cardigann_filters.py`:
+
+```python
+def test_unknown_filter_logs_warning(caplog) -> None:
+ import logging
+ drv = _driver_with_field(
+ "title",
+ [{"name": "definitelynotarealfilter"}],
+ )
+ with caplog.at_level(logging.WARNING):
+ result = _apply(drv, "hello ", "title")
+ assert result == "hello"
+ assert any(
+ "definitelynotarealfilter" in rec.message for rec in caplog.records
+ ), "expected a warning mentioning the unknown filter name"
+```
+
+- [ ] **Step 2: Run — expect fail**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py::test_unknown_filter_logs_warning -v`
+Expected: FAIL — no warning logged.
+
+- [ ] **Step 3: Implement**
+
+Edit `backend/src/trove/indexers/cardigann.py`. Near the top of the file, below the imports, add:
+
+```python
+import logging
+
+log = logging.getLogger(__name__)
+_WARNED_FILTERS: set[str] = set()
+```
+
+Modify the final `return value` at the end of `_apply_filter` to warn on unknown names:
+
+```python
+ if name and name not in _WARNED_FILTERS:
+ _WARNED_FILTERS.add(name)
+ log.warning("cardigann: unknown filter %r — passing value through unchanged", name)
+ return value
+```
+
+- [ ] **Step 4: Run — expect pass**
+
+Run: `cd backend && uv run pytest tests/test_cardigann_filters.py::test_unknown_filter_logs_warning -v`
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py
+git commit -m "feat: cardigann warns once per unknown filter name"
+```
+
+---
+
+## Task 14: Add `CatalogEntryOut` Pydantic model
+
+**Files:**
+- Create: `backend/src/trove/api/catalog.py`
+
+- [ ] **Step 1: Scaffold the router module**
+
+Create `backend/src/trove/api/catalog.py`:
+
+```python
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel, Field
+from sqlmodel import Session, select
+
+from trove.api.deps import current_user, db_session
+from trove.api.indexers import IndexerOut, _to_out
+from trove.clients.base import Protocol
+from trove.indexers.base import Category
+from trove.indexers.cardigann import load_definition_yaml
+from trove.models.indexer import IndexerRow
+from trove.models.user import User
+from trove.services import catalog, indexer_registry
+
+router = APIRouter()
+
+
+class CatalogEntryOut(BaseModel):
+ slug: str
+ display_name: str
+ description: str
+ categories: list[Category]
+ mirrors: list[str]
+ default_mirror: str
+ protocol: Protocol
+ logo: str | None = None
+ already_installed: bool
+
+
+class CatalogInstallRequest(BaseModel):
+ base_url: str = Field(min_length=1, max_length=512)
+ name: str | None = Field(default=None, max_length=64)
+
+
+@router.get("", response_model=list[CatalogEntryOut])
+async def list_catalog(
+ session: Session = Depends(db_session),
+ _user: User = Depends(current_user),
+) -> list[CatalogEntryOut]:
+ installed_slugs = set(
+ session.exec(
+ select(IndexerRow.catalog_slug).where(IndexerRow.catalog_slug.is_not(None)) # type: ignore[attr-defined]
+ ).all()
+ )
+ out: list[CatalogEntryOut] = []
+ for entry in catalog.list_entries():
+ out.append(
+ CatalogEntryOut(
+ slug=entry.slug,
+ display_name=entry.display_name,
+ description=entry.description,
+ categories=entry.categories,
+ mirrors=entry.mirrors,
+ default_mirror=entry.default_mirror,
+ protocol=entry.protocol,
+ logo=entry.logo,
+ already_installed=entry.slug in installed_slugs,
+ )
+ )
+ return out
+```
+
+- [ ] **Step 2: Import-check**
+
+Run: `cd backend && uv run python -c "from trove.api import catalog; print(catalog.router.routes)"`
+Expected: prints one `APIRoute` (the GET handler).
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add backend/src/trove/api/catalog.py
+git commit -m "feat: scaffold catalog API router with GET /catalog"
+```
+
+---
+
+## Task 15: Implement `POST /api/indexers/catalog/{slug}`
+
+**Files:**
+- Modify: `backend/src/trove/api/catalog.py`
+
+- [ ] **Step 1: Add the endpoint**
+
+Append to `backend/src/trove/api/catalog.py`, below the GET handler:
+
+```python
+def _dedup_name(session: Session, base: str) -> str:
+ candidate = base
+ suffix = 2
+ while session.exec(select(IndexerRow).where(IndexerRow.name == candidate)).first() is not None:
+ candidate = f"{base}-{suffix}"
+ suffix += 1
+ return candidate
+
+
+@router.post("/{slug}", response_model=IndexerOut, status_code=status.HTTP_201_CREATED)
+async def install_catalog_entry(
+ slug: str,
+ payload: CatalogInstallRequest,
+ session: Session = Depends(db_session),
+ _user: User = Depends(current_user),
+) -> IndexerOut:
+ try:
+ entry = catalog.get_entry(slug)
+ except KeyError:
+ raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="unknown_slug") from None
+
+ if payload.base_url not in entry.mirrors:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail="base_url_not_in_catalog_mirrors",
+ )
+
+ try:
+ yaml_text = catalog.read_yaml(slug)
+ load_definition_yaml(yaml_text)
+ except Exception as e: # noqa: BLE001
+ raise HTTPException(
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ detail=f"catalog_yaml_broken: {e}",
+ ) from e
+
+ base_name = payload.name or entry.display_name
+ name = _dedup_name(session, base_name)
+
+ row = IndexerRow(
+ name=name,
+ type="cardigann",
+ protocol=entry.protocol.value,
+ base_url=payload.base_url,
+ credentials_cipher=indexer_registry.encrypt_credentials({}),
+ definition_yaml=yaml_text,
+ enabled=True,
+ priority=50,
+ catalog_slug=slug,
+ )
+ session.add(row)
+ session.commit()
+ session.refresh(row)
+ return _to_out(row)
+```
+
+- [ ] **Step 2: Mount the router**
+
+Modify `backend/src/trove/main.py` — find the block that includes the `indexers` router, and directly after it, include the catalog router under the catalog-specific prefix.
+
+Open the file, search for `from trove.api.indexers import router as indexers_router`, and add alongside it:
+
+```python
+from trove.api.catalog import router as catalog_router
+```
+
+Then in the `app.include_router(indexers_router, prefix="/api/indexers", ...)` call region, add:
+
+```python
+app.include_router(catalog_router, prefix="/api/indexers/catalog", tags=["catalog"])
+```
+
+- [ ] **Step 3: Sanity check — start the app headless**
+
+Run: `cd backend && uv run python -c "
+from trove.main import create_app
+app = create_app()
+paths = sorted(r.path for r in app.routes if hasattr(r, 'path'))
+print([p for p in paths if 'catalog' in p])
+"`
+Expected: prints `['/api/indexers/catalog', '/api/indexers/catalog/{slug}']`.
+
+- [ ] **Step 4: Commit**
+
+```bash
+git add backend/src/trove/api/catalog.py backend/src/trove/main.py
+git commit -m "feat: POST /api/indexers/catalog/{slug} installs a catalog entry"
+```
+
+---
+
+## Task 16: API tests for catalog endpoints
+
+**Files:**
+- Create: `backend/tests/api/test_catalog_api.py`
+
+- [ ] **Step 1: Write the tests**
+
+Create `backend/tests/api/test_catalog_api.py`:
+
+```python
+from __future__ import annotations
+
+from fastapi.testclient import TestClient
+
+
+def _login(client: TestClient) -> None:
+ client.post(
+ "/api/auth/setup",
+ json={"username": "admin", "password": "correct horse battery staple"},
+ )
+
+
+def test_list_catalog_returns_entries(client: TestClient) -> None:
+ _login(client)
+ resp = client.get("/api/indexers/catalog")
+ assert resp.status_code == 200
+ body = resp.json()
+ assert len(body) >= 12
+ slugs = {e["slug"] for e in body}
+ assert "thepiratebay" in slugs
+ tpb = next(e for e in body if e["slug"] == "thepiratebay")
+ assert tpb["already_installed"] is False
+ assert tpb["default_mirror"] in tpb["mirrors"]
+
+
+def test_install_catalog_entry_creates_indexer(client: TestClient) -> None:
+ _login(client)
+ resp = client.post(
+ "/api/indexers/catalog/thepiratebay",
+ json={"base_url": "https://thepiratebay.org", "name": None},
+ )
+ assert resp.status_code == 201
+ body = resp.json()
+ assert body["type"] == "cardigann"
+ assert body["base_url"] == "https://thepiratebay.org"
+ assert body["name"] == "The Pirate Bay"
+
+ # already_installed flips on a subsequent list
+ listing = client.get("/api/indexers/catalog").json()
+ tpb = next(e for e in listing if e["slug"] == "thepiratebay")
+ assert tpb["already_installed"] is True
+
+
+def test_install_twice_dedups_name(client: TestClient) -> None:
+ _login(client)
+ first = client.post(
+ "/api/indexers/catalog/thepiratebay",
+ json={"base_url": "https://thepiratebay.org"},
+ )
+ second = client.post(
+ "/api/indexers/catalog/thepiratebay",
+ json={"base_url": "https://tpb.party"},
+ )
+ assert first.status_code == 201
+ assert second.status_code == 201
+ assert first.json()["name"] == "The Pirate Bay"
+ assert second.json()["name"] == "The Pirate Bay-2"
+
+
+def test_install_unknown_slug_404(client: TestClient) -> None:
+ _login(client)
+ resp = client.post(
+ "/api/indexers/catalog/not-a-real-site",
+ json={"base_url": "https://example.com"},
+ )
+ assert resp.status_code == 404
+
+
+def test_install_rejects_base_url_not_in_mirrors(client: TestClient) -> None:
+ _login(client)
+ resp = client.post(
+ "/api/indexers/catalog/thepiratebay",
+ json={"base_url": "https://totally-evil-mirror.example.com"},
+ )
+ assert resp.status_code == 422
+ assert resp.json()["detail"] == "base_url_not_in_catalog_mirrors"
+```
+
+- [ ] **Step 2: Run**
+
+Run: `cd backend && uv run pytest tests/api/test_catalog_api.py -v`
+Expected: all 5 tests PASS.
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add backend/tests/api/test_catalog_api.py
+git commit -m "test: catalog API endpoints"
+```
+
+---
+
+## Task 17: Expose catalog endpoints in frontend `api.ts`
+
+**Files:**
+- Modify: `web/src/lib/api.ts`
+
+- [ ] **Step 1: Add the TypeScript type**
+
+Edit `web/src/lib/api.ts` — after the `IndexerHealthOut` type (around line 131), add:
+
+```typescript
+export type CatalogEntryOut = {
+ slug: string;
+ display_name: string;
+ description: string;
+ categories: Category[];
+ mirrors: string[];
+ default_mirror: string;
+ protocol: Protocol;
+ logo: string | null;
+ already_installed: boolean;
+};
+
+export type CatalogInstallRequest = {
+ base_url: string;
+ name?: string | null;
+};
+```
+
+- [ ] **Step 2: Add the API methods**
+
+Inside the `api.indexers` object literal (around line 483), add a `catalog` sub-object. The diff, line-for-line:
+
+```typescript
+ indexers: {
+ list: () => request("/api/indexers"),
+ health: () => request("/api/indexers/health"),
+ create: (payload: IndexerCreate) =>
+ request("/api/indexers", {
+ method: "POST",
+ body: JSON.stringify(payload)
+ }),
+ update: (id: number, payload: Partial) =>
+ request(`/api/indexers/${id}`, {
+ method: "PATCH",
+ body: JSON.stringify(payload)
+ }),
+ remove: (id: number) => request(`/api/indexers/${id}`, { method: "DELETE" }),
+ test: (id: number) =>
+ request(`/api/indexers/${id}/test`, { method: "POST" }),
+ catalog: {
+ list: () => request("/api/indexers/catalog"),
+ install: (slug: string, payload: CatalogInstallRequest) =>
+ request(`/api/indexers/catalog/${slug}`, {
+ method: "POST",
+ body: JSON.stringify(payload)
+ })
+ }
+ },
+```
+
+- [ ] **Step 3: Typecheck**
+
+Run: `cd web && pnpm check`
+Expected: no TypeScript errors.
+
+- [ ] **Step 4: Commit**
+
+```bash
+git add web/src/lib/api.ts
+git commit -m "feat(web): catalog types + api.indexers.catalog methods"
+```
+
+---
+
+## Task 18: Create `/indexers/catalog/+page.svelte`
+
+**Files:**
+- Create: `web/src/routes/indexers/catalog/+page.svelte`
+
+- [ ] **Step 1: Write the page**
+
+Create `web/src/routes/indexers/catalog/+page.svelte`:
+
+```svelte
+
+
+
+
+
+ Indexers
+
+
+
Catalog
+
+ One-click install for public, no-account torrent sites.
+
+
+
+
+ {#if loading}
+
+ Loading…
+
+ {:else if errorMsg}
+
+ {errorMsg}
+
+ {:else}
+
+ {#each entries as entry (entry.slug)}
+
+
+
+
+
+
+
{entry.display_name}
+
+ {#each entry.categories as cat (cat)}
+
+ {cat}
+
+ {/each}
+
+
+
+
{entry.description}
+
+
+ Mirror
+
+ {#each entry.mirrors as mirror (mirror)}
+ {mirror}
+ {/each}
+
+
+
+ {#if entry.already_installed}
+
+ Installed
+
+ {:else}
+
install(entry)}
+ >
+ {#if installingSlug === entry.slug}
+ Installing…
+ {:else}
+ Add
+ {/if}
+
+ {/if}
+
+ {#if perEntryError[entry.slug]}
+
{perEntryError[entry.slug]}
+ {/if}
+
+ {/each}
+
+ {/if}
+
+```
+
+- [ ] **Step 2: Typecheck**
+
+Run: `cd web && pnpm check`
+Expected: no errors.
+
+- [ ] **Step 3: Start backend + frontend, smoke-test by hand**
+
+Run (two terminals):
+- Backend: `cd backend && uv run uvicorn trove.main:app --reload`
+- Frontend: `cd web && pnpm dev`
+
+Browse to `http://localhost:5173/indexers/catalog`. Log in if prompted. Verify all 12 tiles render with mirror dropdowns and an enabled **Add** button. Click **Add** on one site. Verify the button flips to **Installed**. Visit `/indexers` — the new row appears with the correct name and URL.
+
+- [ ] **Step 4: Commit**
+
+```bash
+git add web/src/routes/indexers/catalog/+page.svelte
+git commit -m "feat(web): /indexers/catalog tile grid for catalog entries"
+```
+
+---
+
+## Task 19: Add "Browse catalog" button on `/indexers`
+
+**Files:**
+- Modify: `web/src/routes/indexers/+page.svelte:243-249`
+
+- [ ] **Step 1: Add the button**
+
+Edit `web/src/routes/indexers/+page.svelte` — replace the single header-action block:
+
+```svelte
+
+ Add indexer
+
+```
+
+with a two-button cluster:
+
+```svelte
+
+```
+
+(`Database` is already imported from `lucide-svelte` in this file; no new import required.)
+
+- [ ] **Step 2: Visual smoke-test**
+
+With both servers still running, refresh `/indexers`. Verify the new **Browse catalog** button appears next to **Add indexer** and navigates to `/indexers/catalog`.
+
+- [ ] **Step 3: Commit**
+
+```bash
+git add web/src/routes/indexers/+page.svelte
+git commit -m "feat(web): browse catalog button on /indexers"
+```
+
+---
+
+## Task 20: Add onboarding step for catalog
+
+**Files:**
+- Modify: `web/src/routes/onboarding/+page.svelte`
+
+- [ ] **Step 1: Add the new step to the Step union**
+
+Edit line 28 — change:
+
+```typescript
+ type Step = "welcome" | "client" | "indexer" | "ai" | "tmdb" | "done";
+```
+
+to:
+
+```typescript
+ type Step = "welcome" | "client" | "indexer" | "catalog" | "ai" | "tmdb" | "done";
+```
+
+- [ ] **Step 2: Add state + helpers for the step**
+
+In the `
+
+
+
+
+ Indexers
+
+
+
Catalog
+
+ One-click install for public, no-account torrent sites.
+
+
+
+
+ {#if loading}
+
+ Loading…
+
+ {:else if errorMsg}
+
+ {errorMsg}
+
+ {:else}
+
+ {#each entries as entry (entry.slug)}
+
+
+
+
+
+
+
{entry.display_name}
+
+ {#each entry.categories as cat (cat)}
+
+ {cat}
+
+ {/each}
+
+
+
+
{entry.description}
+
+
+ Mirror
+
+ {#each entry.mirrors as mirror (mirror)}
+ {mirror}
+ {/each}
+
+
+
+ {#if entry.already_installed}
+
+ Installed
+
+ {:else}
+
install(entry)}
+ >
+ {#if installingSlug === entry.slug}
+ Installing…
+ {:else}
+ Add
+ {/if}
+
+ {/if}
+
+ {#if perEntryError[entry.slug]}
+
{perEntryError[entry.slug]}
+ {/if}
+
+ {/each}
+
+ {/if}
+
From f43ac44b6881db51c1256863802b06099f1cb77d Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 13:18:44 +0200
Subject: [PATCH 26/39] feat(web): browse catalog button on /indexers
---
web/src/routes/indexers/+page.svelte | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/web/src/routes/indexers/+page.svelte b/web/src/routes/indexers/+page.svelte
index dadb8df..c8293e6 100644
--- a/web/src/routes/indexers/+page.svelte
+++ b/web/src/routes/indexers/+page.svelte
@@ -240,12 +240,20 @@
Newznab/Torznab APIs and Cardigann-style YAML trackers.
-
- Add indexer
-
+
{#if loading}
From c81fff19429e405a68ce52a2865ca7bfe6b5520c Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 13:24:11 +0200
Subject: [PATCH 27/39] refactor(web): use reactive reassignment + targeted
update on catalog install
---
web/src/routes/indexers/catalog/+page.svelte | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/web/src/routes/indexers/catalog/+page.svelte b/web/src/routes/indexers/catalog/+page.svelte
index 8de39d3..ba4adfa 100644
--- a/web/src/routes/indexers/catalog/+page.svelte
+++ b/web/src/routes/indexers/catalog/+page.svelte
@@ -30,16 +30,21 @@
async function install(entry: CatalogEntryOut) {
installingSlug = entry.slug;
- perEntryError[entry.slug] = "";
+ perEntryError = { ...perEntryError, [entry.slug]: "" };
try {
await api.indexers.catalog.install(entry.slug, {
base_url: selectedMirror[entry.slug],
name: null
});
- await load();
+ entries = entries.map((e) =>
+ e.slug === entry.slug ? { ...e, already_installed: true } : e
+ );
} catch (e) {
const err = e as { detail?: string };
- perEntryError[entry.slug] = err.detail ?? "Install failed.";
+ perEntryError = {
+ ...perEntryError,
+ [entry.slug]: err.detail ?? "Install failed."
+ };
} finally {
installingSlug = null;
}
From 23213a98b03caf22364f610a98a7d2356187d07c Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 13:28:47 +0200
Subject: [PATCH 28/39] feat(onboarding): public-sites step between indexer and
AI
Co-Authored-By: Claude Sonnet 4.6
---
web/src/routes/onboarding/+page.svelte | 142 ++++++++++++++++++++++++-
1 file changed, 138 insertions(+), 4 deletions(-)
diff --git a/web/src/routes/onboarding/+page.svelte b/web/src/routes/onboarding/+page.svelte
index 1dc3f5f..b386d7b 100644
--- a/web/src/routes/onboarding/+page.svelte
+++ b/web/src/routes/onboarding/+page.svelte
@@ -7,7 +7,8 @@
type IndexerType,
type Protocol,
type DownloadClientOut,
- type IndexerOut
+ type IndexerOut,
+ type CatalogEntryOut
} from "$lib/api";
import { CLIENT_TYPES } from "$lib/clientTypes";
import {
@@ -25,7 +26,7 @@
Trash2
} from "lucide-svelte";
- type Step = "welcome" | "client" | "indexer" | "ai" | "tmdb" | "done";
+ type Step = "welcome" | "client" | "indexer" | "catalog" | "ai" | "tmdb" | "done";
let step = $state("welcome");
let clients = $state([]);
@@ -78,6 +79,61 @@
let indexerError = $state(null);
let indexerJustSaved = $state(null);
+ // Catalog step state
+ let catalogEntries = $state([]);
+ let catalogLoading = $state(false);
+ let catalogSelected = $state>(new Set());
+ let catalogInstalling = $state(false);
+ let catalogError = $state(null);
+ let catalogLoaded = $state(false);
+
+ $effect(() => {
+ if (step === "catalog") loadCatalog();
+ });
+
+ async function loadCatalog() {
+ if (catalogLoaded || catalogLoading) return;
+ catalogLoading = true;
+ try {
+ catalogEntries = await api.indexers.catalog.list();
+ catalogLoaded = true;
+ } catch (e) {
+ const err = e as { detail?: string };
+ catalogError = err.detail ?? "Failed to load catalog.";
+ } finally {
+ catalogLoading = false;
+ }
+ }
+
+ function toggleCatalog(slug: string) {
+ const next = new Set(catalogSelected);
+ if (next.has(slug)) next.delete(slug);
+ else next.add(slug);
+ catalogSelected = next;
+ }
+
+ async function installSelectedCatalog() {
+ catalogInstalling = true;
+ catalogError = null;
+ for (const entry of catalogEntries) {
+ if (!catalogSelected.has(entry.slug) || entry.already_installed) continue;
+ try {
+ await api.indexers.catalog.install(entry.slug, {
+ base_url: entry.default_mirror,
+ name: null
+ });
+ } catch (e) {
+ const err = e as { detail?: string };
+ catalogError = `${entry.display_name}: ${err.detail ?? "install failed"}`;
+ catalogInstalling = false;
+ return;
+ }
+ }
+ catalogInstalling = false;
+ indexers = await api.indexers.list();
+ step = "ai";
+ }
+
// AI state
let aiTesting = $state(false);
let aiResult = $state<{ ok: boolean; msg: string } | null>(null);
@@ -242,6 +298,7 @@
{ key: "welcome", label: "Welcome", icon: Sparkles },
{ key: "client", label: "Download client", icon: Download },
{ key: "indexer", label: "Indexer", icon: Database },
+ { key: "catalog", label: "Public sites", icon: Database },
{ key: "ai", label: "AI", icon: Sparkles },
{ key: "tmdb", label: "Discover", icon: Sparkles },
{ key: "done", label: "Done", icon: PartyPopper }
@@ -584,7 +641,7 @@
(step = "ai")}
+ onclick={() => (step = "catalog")}
>
Continue
@@ -663,7 +720,7 @@
(step = "ai")}
+ onclick={() => (step = "catalog")}
>
{indexers.length > 0 ? "Continue" : "Skip"}
@@ -680,6 +737,83 @@
{/if}
+ {#if step === "catalog"}
+
+
+
+
+ Public torrent sites (optional)
+
+
+ Pick any public sites you want Trove to search. You can always add more later.
+
+
+
+
+ {#if catalogLoading}
+
Loading catalog…
+ {:else}
+
+ {#each catalogEntries as entry (entry.slug)}
+
+ toggleCatalog(entry.slug)}
+ />
+
+
+ {entry.display_name}
+ {#if entry.already_installed}
+
+ installed
+
+ {/if}
+
+
{entry.description}
+
+
+ {/each}
+
+ {/if}
+
+ {#if catalogError}
+
+ {catalogError}
+
+ {/if}
+
+
+ (step = "ai")}
+ >
+
+ Skip
+
+
+ {#if catalogInstalling}
+ Installing…
+ {:else}
+ Add {catalogSelected.size} site{catalogSelected.size === 1 ? "" : "s"}
+
+ {/if}
+
+
+
+ {/if}
+
{#if step === "ai"}
From 8c8d534ba507801a9b5607cf6a5913ef1ea45751 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:30:00 +0200
Subject: [PATCH 29/39] test: parametrized catalog fixture extraction harness
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Adds backend/tests/test_catalog_fixtures.py, parametrized over any
backend/tests/fixtures/catalog/-search.html files present. Zero
parametrizations for now — no fixture was captured. TPB was attempted via
tpb.party but the vendored YAML uses the apibay.org JSON API, not HTML
parsing, so captured HTML cannot be exercised by the row extractor.
Similarly, thepiratebay.org itself returns a JS-rendered shell page
(<5 KB). The harness is still wired correctly and will activate as soon
as an HTML-based site fixture is dropped in the fixtures directory.
Also adds TODO.md with a reminder to capture fixtures for the remaining
11 catalog slugs.
Co-Authored-By: Claude Sonnet 4.6
---
TODO.md | 9 +++++
backend/tests/fixtures/catalog/README.md | 16 ++++++++
backend/tests/test_catalog_fixtures.py | 50 ++++++++++++++++++++++++
3 files changed, 75 insertions(+)
create mode 100644 TODO.md
create mode 100644 backend/tests/fixtures/catalog/README.md
create mode 100644 backend/tests/test_catalog_fixtures.py
diff --git a/TODO.md b/TODO.md
new file mode 100644
index 0000000..44edd36
--- /dev/null
+++ b/TODO.md
@@ -0,0 +1,9 @@
+# TODO
+
+## Catalog fixtures
+
+- [ ] Capture `-search.html` fixtures for the 11 remaining catalog slugs.
+ Pattern: `backend/tests/fixtures/catalog/-search.html`.
+ Extraction test lives in `backend/tests/test_catalog_fixtures.py` and
+ is parametrized over whichever fixtures are present. When captured,
+ the test runs `_extract_release` against each row and asserts ≥1 result.
diff --git a/backend/tests/fixtures/catalog/README.md b/backend/tests/fixtures/catalog/README.md
new file mode 100644
index 0000000..98ff0ec
--- /dev/null
+++ b/backend/tests/fixtures/catalog/README.md
@@ -0,0 +1,16 @@
+# Catalog fixture HTML
+
+One `-search.html` file per site, containing a real search-results page captured from the site's response to a benign query (e.g. "ubuntu", "debian", "linux").
+
+## How to capture
+
+```bash
+# Example: TPB
+curl -sL 'https://thepiratebay.org/search/ubuntu/0/99/0' \
+ -H 'User-Agent: Mozilla/5.0' \
+ > thepiratebay-search.html
+```
+
+Pick a query whose results are unambiguous and stable (distro ISOs, commonly-seeded old scene releases). The fixture is committed — keep it small (<500 KB) by trimming or using a narrow query.
+
+Re-capture whenever `scripts/update-catalog.py diff` shows the upstream YAML has changed *and* the corresponding fixture test starts failing.
diff --git a/backend/tests/test_catalog_fixtures.py b/backend/tests/test_catalog_fixtures.py
new file mode 100644
index 0000000..e007f85
--- /dev/null
+++ b/backend/tests/test_catalog_fixtures.py
@@ -0,0 +1,50 @@
+"""Parse-each-fixture smoke test.
+
+For every slug in the catalog that has a corresponding
+`tests/fixtures/catalog/-search.html` file, this test:
+ - loads the vendored YAML
+ - runs the Cardigann row extractor against the fixture
+ - asserts at least one release with a non-empty title
+
+Missing fixtures are silently skipped — run `pytest -vv` to see which
+slugs are covered. Capturing real HTML fixtures is a one-time manual
+task per site; tests pass until a fixture is captured AND its extraction
+breaks.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from bs4 import BeautifulSoup
+
+from trove.indexers.cardigann import CardigannIndexer, load_definition_yaml
+from trove.services import catalog
+
+FIXTURE_DIR = Path(__file__).parent / "fixtures" / "catalog"
+
+
+def _covered_slugs() -> list[str]:
+ catalog.reset_cache_for_tests()
+ return [
+ e.slug for e in catalog.list_entries() if (FIXTURE_DIR / f"{e.slug}-search.html").exists()
+ ]
+
+
+@pytest.mark.parametrize("slug", _covered_slugs())
+def test_fixture_extracts_at_least_one_release(slug: str) -> None:
+ entry = catalog.get_entry(slug)
+ definition = load_definition_yaml(catalog.read_yaml(slug))
+ definition.name = slug
+ driver = CardigannIndexer(definition, base_url=entry.default_mirror)
+
+ html = (FIXTURE_DIR / f"{slug}-search.html").read_text(encoding="utf-8")
+ soup = BeautifulSoup(html, "lxml")
+ rows = soup.select(definition.rows_selector)
+ assert rows, f"{slug}: rows_selector {definition.rows_selector!r} matched no elements"
+
+ extracted = [driver._extract_release(r) for r in rows]
+ extracted = [r for r in extracted if r is not None]
+ assert extracted, f"{slug}: 0 releases extracted from {len(rows)} row(s)"
+ assert extracted[0].title, f"{slug}: first release has empty title"
From 4bc31bace11777e82a9c07b77298c00fabee4b7e Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:30:25 +0200
Subject: [PATCH 30/39] docs: document public-site catalog in indexers guide
Adds a "Catalog (public sites, one click)" section to 03-indexers.md
immediately before "Priority and ordering", covering how to install from
the catalog, what the onboarding wizard does, and how vendored YAMLs are
updated.
Co-Authored-By: Claude Sonnet 4.6
---
backend/src/trove/docs/03-indexers.md | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/backend/src/trove/docs/03-indexers.md b/backend/src/trove/docs/03-indexers.md
index 78ed649..c1c977a 100644
--- a/backend/src/trove/docs/03-indexers.md
+++ b/backend/src/trove/docs/03-indexers.md
@@ -100,6 +100,22 @@ For trackers that don't have a native API, Cardigann uses YAML definition files
- Custom headers beyond what httpx sends by default
- JavaScript-heavy sites that need Playwright/Puppeteer
+## Catalog (public sites, one click)
+
+For public, no-account torrent sites, Trove ships with a curated catalog of pre-configured definitions including The Pirate Bay, 1337x, TorrentGalaxy, LimeTorrents, Nyaa, EZTV, YTS, and others. Five slugs map to substitute definitions (ExtraTorrent, KickAssTorrents, TorrentDownload, TorrentProject2, Tokyo Toshokan) because the original sites have no upstream Cardigann YAML — each entry honestly declares its actual source in the description.
+
+**How to install:**
+
+1. On `/indexers`, click **Browse catalog** (next to **Add indexer**).
+2. Pick a mirror from the dropdown on the site's tile — catalog entries list multiple known mirrors for sites that have them.
+3. Click **Add**. The tile flips to **Installed** and the site appears on `/indexers` as a normal Cardigann indexer.
+
+The onboarding wizard also surfaces these sites as an optional step — pick any you want in one go.
+
+**Behind the scenes**: a catalog-installed entry is an ordinary `type=cardigann` indexer with a vendored YAML definition. The Test, Edit, and Delete buttons work the same way they do for hand-added indexers. If you ever need to override the URL or rename the entry, use **Edit** — nothing is special about catalog rows.
+
+**Updating definitions**: the shipped YAMLs track `Prowlarr/Indexers`. When a site changes its HTML, searches will start returning 0 results. Upstream usually has a fix within days. Trove refreshes the vendored files on each release; to sync sooner, a maintainer can run `scripts/update-catalog.py diff` to see what's changed upstream, then `sync` to pull.
+
## Priority and ordering
Each indexer has a **priority** field (default 50). Lower numbers run first in the fan-out, but since Trove queries all enabled indexers in parallel, priority mainly matters for tie-breaking when the same release appears from multiple sources — the higher-priority indexer's copy is kept.
From dff95b96e0e12bf0965ac929ee1c562aeb182a88 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:34:02 +0200
Subject: [PATCH 31/39] docs: temper aspirational claim about catalog refresh
automation
---
backend/src/trove/docs/03-indexers.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/src/trove/docs/03-indexers.md b/backend/src/trove/docs/03-indexers.md
index c1c977a..4f59d96 100644
--- a/backend/src/trove/docs/03-indexers.md
+++ b/backend/src/trove/docs/03-indexers.md
@@ -114,7 +114,7 @@ The onboarding wizard also surfaces these sites as an optional step — pick any
**Behind the scenes**: a catalog-installed entry is an ordinary `type=cardigann` indexer with a vendored YAML definition. The Test, Edit, and Delete buttons work the same way they do for hand-added indexers. If you ever need to override the URL or rename the entry, use **Edit** — nothing is special about catalog rows.
-**Updating definitions**: the shipped YAMLs track `Prowlarr/Indexers`. When a site changes its HTML, searches will start returning 0 results. Upstream usually has a fix within days. Trove refreshes the vendored files on each release; to sync sooner, a maintainer can run `scripts/update-catalog.py diff` to see what's changed upstream, then `sync` to pull.
+**Updating definitions**: the shipped YAMLs track `Prowlarr/Indexers`. When a site changes its HTML, searches will start returning 0 results. Upstream usually has a fix within days. A maintainer can run `scripts/update-catalog.py diff` to see what's changed upstream, then `sync` to pull — this is manual today, typically done alongside a release.
## Priority and ordering
From 3c9d514ddc7eaaf4b04c40de4fc3399a59a12934 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:52:26 +0200
Subject: [PATCH 32/39] feat(cardigann): parse settings.default into
config_defaults
Co-Authored-By: Claude Sonnet 4.6
---
backend/src/trove/indexers/cardigann.py | 177 ++++++++++++++++++++--
backend/tests/test_cardigann_templates.py | 89 +++++++++++
2 files changed, 250 insertions(+), 16 deletions(-)
create mode 100644 backend/tests/test_cardigann_templates.py
diff --git a/backend/src/trove/indexers/cardigann.py b/backend/src/trove/indexers/cardigann.py
index 389dc8d..066413d 100644
--- a/backend/src/trove/indexers/cardigann.py
+++ b/backend/src/trove/indexers/cardigann.py
@@ -66,6 +66,7 @@ class CardigannDefinition:
fields: dict[str, FieldSpec]
category_mapping: dict[str, Category] = field(default_factory=dict)
protocol: Protocol = Protocol.TORRENT
+ config_defaults: dict[str, str] = field(default_factory=dict) # from settings: block
def _coerce_field(spec_data: dict[str, Any] | str) -> FieldSpec:
@@ -113,6 +114,22 @@ def load_definition(data: dict[str, Any]) -> CardigannDefinition:
protocol_str = (data.get("type") or "").lower()
protocol = Protocol.USENET if "usenet" in protocol_str else Protocol.TORRENT
+ config_defaults: dict[str, str] = {}
+ for item in data.get("settings") or []:
+ if not isinstance(item, dict):
+ continue
+ name = item.get("name")
+ if not isinstance(name, str):
+ continue
+ default = item.get("default")
+ if default is None:
+ continue
+ # Normalize bools to Go-template casing
+ if isinstance(default, bool):
+ config_defaults[name] = "True" if default else "False"
+ else:
+ config_defaults[name] = str(default)
+
return CardigannDefinition(
site=str(data.get("site", "")),
name=str(data.get("name") or data.get("site", "")),
@@ -123,6 +140,7 @@ def load_definition(data: dict[str, Any]) -> CardigannDefinition:
fields=fields_map,
category_mapping=category_mapping,
protocol=protocol,
+ config_defaults=config_defaults,
)
@@ -133,6 +151,104 @@ def load_definition_yaml(text: str) -> CardigannDefinition:
return load_definition(data)
+# ---------------------------------------------------------------------------
+# Minimal Go-template subset evaluator for Prowlarr YAMLs.
+#
+# Supported:
+# {{ .Keywords }} -> query terms or ""
+# {{ .Query.IMDBID }}, {{ .Query.TMDBID }} -> always ""
+# {{ .Config.X }} -> config[X] or ""
+# {{ if .Keywords }}A{{ else }}B{{ end }} -> A if keywords non-empty, else B
+# {{ if and .Keywords ... }}A{{ else }}B{{ end }}-> same (conservative: keywords-present check)
+# {{ if or .Query.IMDBID .Keywords }}...{{ end }}-> keywords-present check
+# {{ range .Categories }}...{{ end }} -> "" (categories not piped through)
+# {{ join .Categories "," }} -> ""
+#
+# NOT supported: arbitrary nested ifs, function calls inside template
+# expressions, range loops over arbitrary collections.
+# Unrecognized templates pass through unchanged (never crash).
+# ---------------------------------------------------------------------------
+
+_IF_ELSE_END_RE = re.compile(
+ r"\{\{\s*if\s+(.+?)\s*\}\}(.*?)\{\{\s*else\s*\}\}(.*?)\{\{\s*end\s*\}\}",
+ re.DOTALL,
+)
+_IF_END_RE = re.compile(
+ r"\{\{\s*if\s+(.+?)\s*\}\}(.*?)\{\{\s*end\s*\}\}",
+ re.DOTALL,
+)
+_RANGE_END_RE = re.compile(
+ r"\{\{\s*range\s+.+?\s*\}\}.*?\{\{\s*end\s*\}\}",
+ re.DOTALL,
+)
+_JOIN_RE = re.compile(
+ r'\{\{\s*join\s+\.Categories\s+"[^"]*"\s*\}\}',
+)
+_KEYWORDS_RE = re.compile(r"\{\{\s*\.Keywords\s*\}\}")
+_QUERY_IMDB_RE = re.compile(r"\{\{\s*\.Query\.(IMDBID|TMDBID|TVDBID)\s*\}\}")
+_CONFIG_RE = re.compile(r"\{\{\s*\.Config\.([\w\-]+)\s*\}\}")
+
+
+def expand_template(
+ text: str,
+ *,
+ keywords: str = "",
+ config: dict[str, str] | None = None,
+) -> str:
+ """Expand a Cardigann/Go-template string.
+
+ See the module comment above for the supported subset. Anything unrecognized
+ passes through unchanged so upstream failures are visible rather than
+ silently producing wrong URLs.
+ """
+ if "{{" not in text:
+ return text
+ cfg = config or {}
+ keywords_present = bool(keywords)
+
+ # Drop range blocks entirely (categories-iter is the only real use).
+ text = _RANGE_END_RE.sub("", text)
+ # join .Categories -> empty
+ text = _JOIN_RE.sub("", text)
+
+ # Repeatedly resolve if/else/end from the innermost occurrence outward.
+ # Prowlarr chains them, so loop until stable.
+ for _ in range(10):
+ before = text
+
+ def _ifelse(m: re.Match[str]) -> str: # noqa: E306
+ return m.group(2) if keywords_present else m.group(3)
+
+ text = _IF_ELSE_END_RE.sub(_ifelse, text)
+ if text == before:
+ break
+
+ # Bare {{ if ... }}X{{ end }} without else -> X if keywords else ""
+ for _ in range(10):
+ before = text
+
+ def _if(m: re.Match[str]) -> str: # noqa: E306
+ return m.group(2) if keywords_present else ""
+
+ text = _IF_END_RE.sub(_if, text)
+ if text == before:
+ break
+
+ # Variable substitutions.
+ text = _KEYWORDS_RE.sub(lambda _: keywords, text)
+ text = _QUERY_IMDB_RE.sub("", text)
+
+ def _cfg(m: re.Match[str]) -> str:
+ return cfg.get(m.group(1), "")
+
+ text = _CONFIG_RE.sub(_cfg, text)
+
+ # .Result.* is not a request-time substitution — leave intact for
+ # field-extraction-time expansion (a separate call site).
+
+ return text
+
+
def _map_category(cat_id: int) -> Category | None:
if 2000 <= cat_id < 3000:
return Category.MOVIES
@@ -183,16 +299,19 @@ async def test_connection(self) -> IndexerHealth:
return IndexerHealth(ok=True)
async def search(self, query: SearchQuery) -> list[Release]:
+ cfg = self.definition.config_defaults
+ kw = query.terms or ""
params: dict[str, Any] = {}
for key, template in (self.definition.search_params or {}).items():
if isinstance(template, str):
- params[key] = template.replace("{{.Query.Keywords}}", query.terms)
+ params[key] = expand_template(template, keywords=kw, config=cfg)
else:
params[key] = template
if "q" not in params and "search" not in params and "query" not in params:
- params["q"] = query.terms
+ params["q"] = kw
- url = self.base_url + self.definition.search_path
+ path = expand_template(self.definition.search_path, keywords=kw, config=cfg)
+ url = self.base_url + path
try:
resp = await self._client.get(url, params=params)
except httpx.HTTPError as e:
@@ -201,28 +320,37 @@ async def search(self, query: SearchQuery) -> list[Release]:
raise IndexerError(f"{self.name}: HTTP {resp.status_code}")
soup = BeautifulSoup(resp.text, "lxml")
- rows = soup.select(self.definition.rows_selector)
+ rows_selector = expand_template(self.definition.rows_selector, keywords=kw, config=cfg)
+ rows = soup.select(rows_selector)
releases: list[Release] = []
for row in rows[: query.limit]:
- release = self._extract_release(row)
+ release = self._extract_release(row, keywords=kw, config=cfg)
if release is not None:
releases.append(release)
return releases
- def _extract_release(self, row: Tag) -> Release | None:
- title = self._extract_field(row, "title")
+ def _extract_release(
+ self,
+ row: Tag,
+ *,
+ keywords: str = "",
+ config: dict[str, str] | None = None,
+ ) -> Release | None:
+ title = self._extract_field(row, "title", keywords=keywords, config=config)
if not title:
return None
- download_url = self._extract_field(row, "download") or self._extract_field(row, "details")
+ download_url = self._extract_field(
+ row, "download", keywords=keywords, config=config
+ ) or self._extract_field(row, "details", keywords=keywords, config=config)
if download_url and not download_url.startswith(("http://", "https://", "magnet:")):
download_url = self.base_url + (
download_url if download_url.startswith("/") else f"/{download_url}"
)
- size = _parse_size(self._extract_field(row, "size"))
- infohash = self._extract_field(row, "infohash") or None
- category = self._extract_field(row, "category")
+ size = _parse_size(self._extract_field(row, "size", keywords=keywords, config=config))
+ infohash = self._extract_field(row, "infohash", keywords=keywords, config=config) or None
+ category = self._extract_field(row, "category", keywords=keywords, config=config)
return Release(
title=title,
@@ -234,7 +362,14 @@ def _extract_release(self, row: Tag) -> Release | None:
source=self.name,
)
- def _extract_field(self, row: Tag, key: str) -> str | None:
+ def _extract_field(
+ self,
+ row: Tag,
+ key: str,
+ *,
+ keywords: str = "",
+ config: dict[str, str] | None = None,
+ ) -> str | None:
spec = self.definition.fields.get(key)
if spec is None:
return None
@@ -242,14 +377,24 @@ def _extract_field(self, row: Tag, key: str) -> str | None:
return spec.text
target: Tag | None = row
- if spec.selector:
- target = row.select_one(spec.selector)
+ selector = (
+ expand_template(spec.selector, keywords=keywords, config=(config or {}))
+ if spec.selector
+ else None
+ )
+ if selector:
+ target = row.select_one(selector)
if target is None:
return None
value: str | None
- if spec.attribute:
- raw = target.get(spec.attribute)
+ attribute = (
+ expand_template(spec.attribute, keywords=keywords, config=(config or {}))
+ if spec.attribute
+ else None
+ )
+ if attribute:
+ raw = target.get(attribute)
value = (raw[0] if raw else None) if isinstance(raw, list) else raw
else:
value = target.get_text(" ", strip=True)
diff --git a/backend/tests/test_cardigann_templates.py b/backend/tests/test_cardigann_templates.py
new file mode 100644
index 0000000..1b35200
--- /dev/null
+++ b/backend/tests/test_cardigann_templates.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import yaml as _yaml
+
+from trove.indexers.cardigann import expand_template, load_definition
+
+
+def test_load_definition_parses_settings_defaults() -> None:
+ raw = _yaml.safe_load("""
+id: test
+name: test
+links: [https://test.local]
+type: public
+settings:
+ - name: sort
+ type: select
+ default: time
+ - name: disablesort
+ type: checkbox
+ default: False
+ - name: no_default_item
+ type: text
+caps:
+ categorymappings: []
+search:
+ paths:
+ - path: /
+ rows:
+ selector: tr
+ fields: {}
+""")
+ d = load_definition(raw)
+ assert d.config_defaults["sort"] == "time"
+ assert d.config_defaults["disablesort"] == "False"
+ assert "no_default_item" not in d.config_defaults
+
+
+def test_expand_keywords_substitution() -> None:
+ assert expand_template("search/{{ .Keywords }}/1/", keywords="ubuntu") == "search/ubuntu/1/"
+
+
+def test_expand_if_keywords_else() -> None:
+ tmpl = "{{ if .Keywords }}search/{{ .Keywords }}{{ else }}latest{{ end }}"
+ assert expand_template(tmpl, keywords="ubuntu") == "search/ubuntu"
+ assert expand_template(tmpl, keywords="") == "latest"
+
+
+def test_expand_config_substitution() -> None:
+ cfg = {"sort": "time", "apiurl": "example.com"}
+ assert expand_template("{{ .Config.sort }}", config=cfg) == "time"
+ assert expand_template("https://{{ .Config.apiurl }}/x", config=cfg) == "https://example.com/x"
+ assert expand_template("{{ .Config.missing }}", config=cfg) == ""
+
+
+def test_expand_query_imdb_is_empty() -> None:
+ assert expand_template("id:{{ .Query.IMDBID }}", keywords="x") == "id:"
+
+
+def test_expand_range_categories_is_empty() -> None:
+ tmpl = "path/{{ range .Categories }}:cat:{{.}}{{ end }}"
+ assert expand_template(tmpl, keywords="x") == "path/"
+
+
+def test_expand_join_categories_is_empty() -> None:
+ assert expand_template('cats={{ join .Categories "," }}', keywords="x") == "cats="
+
+
+def test_expand_complex_1337x_path() -> None:
+ # Sample pulled from 1337x.yml
+ cfg = {"sort": "time", "type": "desc", "disablesort": "False"}
+ tmpl = (
+ "{{ if and (.Keywords) (eq .Config.disablesort .False) }}sort-{{ else }}{{ end }}"
+ "{{ if .Keywords }}search/{{ .Keywords }}{{ else }}cat/Movies{{ end }}"
+ "{{ if and (.Keywords) (eq .Config.disablesort .False) }}/{{ .Config.sort }}/{{ .Config.type }}{{ else }}{{ end }}"
+ "/1/"
+ )
+ result = expand_template(tmpl, keywords="ubuntu", config=cfg)
+ # Expected: "sort-search/ubuntu/time/desc/1/"
+ assert result == "sort-search/ubuntu/time/desc/1/"
+
+
+def test_expand_no_template_passes_through() -> None:
+ assert expand_template("/") == "/"
+ assert expand_template("search/static/path") == "search/static/path"
+
+
+def test_expand_returns_unchanged_on_unknown_directive() -> None:
+ # Unknown pipeline -> untouched
+ assert "{{ weird_directive" in expand_template("{{ weird_directive }}", keywords="x")
From 58b69648ac0b02e92e3c8ed8274442562246e14e Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:52:57 +0200
Subject: [PATCH 33/39] feat(cardigann): minimal Go template expander + wire
into search path, params, selectors
Co-Authored-By: Claude Sonnet 4.6
---
backend/tests/test_cardigann_templates.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/backend/tests/test_cardigann_templates.py b/backend/tests/test_cardigann_templates.py
index 1b35200..75fa62a 100644
--- a/backend/tests/test_cardigann_templates.py
+++ b/backend/tests/test_cardigann_templates.py
@@ -87,3 +87,16 @@ def test_expand_no_template_passes_through() -> None:
def test_expand_returns_unchanged_on_unknown_directive() -> None:
# Unknown pipeline -> untouched
assert "{{ weird_directive" in expand_template("{{ weird_directive }}", keywords="x")
+
+
+def test_1337x_url_construction_uses_expanded_template() -> None:
+ from trove.indexers.cardigann import CardigannIndexer, load_definition_yaml
+ from trove.services import catalog
+
+ catalog.reset_cache_for_tests()
+ d = load_definition_yaml(catalog.read_yaml("1337x"))
+ drv = CardigannIndexer(d, base_url="https://1337x.to")
+ path = expand_template(d.search_path, keywords="ubuntu", config=d.config_defaults)
+ assert "{{" not in path, f"path still has templates: {path}"
+ assert "ubuntu" in path
+ assert path.startswith("/") or "search" in path or "sort-" in path
From 9e4854b6058db958d6a9b1bde95e362f9e7c85d5 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:53:58 +0200
Subject: [PATCH 34/39] refactor: drop TPB + YTS from catalog (JSON APIs, need
native drivers)
Co-Authored-By: Claude Sonnet 4.6
---
.../src/trove/indexers/catalog/registry.yaml | 24 --
.../trove/indexers/catalog/thepiratebay.yml | 216 ------------------
backend/src/trove/indexers/catalog/yts.yml | 140 ------------
backend/tests/api/test_catalog_api.py | 36 +--
backend/tests/test_catalog.py | 4 +-
5 files changed, 20 insertions(+), 400 deletions(-)
delete mode 100644 backend/src/trove/indexers/catalog/thepiratebay.yml
delete mode 100644 backend/src/trove/indexers/catalog/yts.yml
diff --git a/backend/src/trove/indexers/catalog/registry.yaml b/backend/src/trove/indexers/catalog/registry.yaml
index c84f22a..55a9aa0 100644
--- a/backend/src/trove/indexers/catalog/registry.yaml
+++ b/backend/src/trove/indexers/catalog/registry.yaml
@@ -6,19 +6,6 @@
# so scripts/update-catalog.py can diff against Prowlarr-indexers.
entries:
- - slug: thepiratebay
- display_name: The Pirate Bay
- description: General-purpose public torrent tracker, no account required.
- categories: [movies, tv, music, software, games, books, other]
- yaml_file: thepiratebay.yml
- upstream_path: definitions/v11/thepiratebay.yml
- mirrors:
- - https://thepiratebay.org
- - https://tpb.party
- - https://piratebay.live
- default_mirror: https://thepiratebay.org
- protocol: torrent
-
- slug: 1337x
display_name: 1337x
description: Large public general-purpose tracker with strong TV/movie scene coverage.
@@ -124,17 +111,6 @@ entries:
default_mirror: https://eztv.re
protocol: torrent
- - slug: yts
- display_name: YTS
- description: Public tracker specializing in small-size movie encodes.
- categories: [movies]
- yaml_file: yts.yml
- upstream_path: definitions/v11/yts.yml
- mirrors:
- - https://yts.mx
- default_mirror: https://yts.mx
- protocol: torrent
-
- slug: animetosho
display_name: Tokyo Toshokan
description: Public BitTorrent library for Japanese media including anime (replaces animetosho which has no upstream Cardigann definition).
diff --git a/backend/src/trove/indexers/catalog/thepiratebay.yml b/backend/src/trove/indexers/catalog/thepiratebay.yml
deleted file mode 100644
index d9ea8d4..0000000
--- a/backend/src/trove/indexers/catalog/thepiratebay.yml
+++ /dev/null
@@ -1,216 +0,0 @@
----
-id: thepiratebay
-name: The Pirate Bay
-description: "The Pirate Bay (TPB) is the galaxy’s most resilient Public BitTorrent site"
-language: en-US
-type: public
-encoding: UTF-8
-links:
- - https://thepiratebay.org/
- - https://thepiratebay.unblockninja.com/
- - https://thepiratebay.ninjaproxy1.com/
- - https://tpb.proxyninja.org/
- - https://thepiratebay.proxyninja.net/
- - https://thepiratebay.torrentbay.st/
- - https://tpb.skynetcloud.site/
- - https://piratehaven.xyz/
- - https://mirrorbay.top/
- - https://thepiratebay0.org/
- - https://thepiratebay10.xyz/
- - https://pirateproxylive.org/
- - https://thehiddenbay.com/
- - https://thepiratebay.zone/
- - https://tpb.party/
- - https://piratebayproxy.live/
- - https://piratebay.live/
- - https://piratebay.party/
- - https://thepiratebay.party/
- - https://thepiratebaye.org/
- - https://thepiratebay.cloud/
- - https://tpb-proxy.xyz/
- - https://tpb.re/
- - https://tpirbay.site/
- - https://tpirbay.top/
- - https://tpirbay.xyz/
-legacylinks:
- - https://pirate-proxy.page/
- - https://5mins.shop/
- - https://tpb.surf/
- - https://tpb.monster/
- - https://thepiratebay.host/
- - https://piratetoday.xyz/
- - https://tpb.wtf/
- - https://piratebayo3klnzokct3wt5yyxb2vpebbuyjl7m623iaxmqhsd52coid.onion.ly/
- - https://piratebayo3klnzokct3wt5yyxb2vpebbuyjl7m623iaxmqhsd52coid.tor2web.to/
- - https://piratebayo3klnzokct3wt5yyxb2vpebbuyjl7m623iaxmqhsd52coid.tor2web.link/
- - https://tpb25.ukpass.co/
- - https://tpb29.ukpass.co/
- - https://piratenow.xyz/
- - https://pirate-proxy.ink/
- - https://proxifiedpiratebay.org/
- - https://unlockedpiratebay.com/
- - https://tpb.one/
- - https://piratebayorg.net/
- - https://tpbproxy.click/
- - https://pirateproxy.live/
- - https://ukpiratebay.org/
- - https://piratebay.by/
- - https://pirate-proxy.date/
- - https://thepirateproxy.net/
- - https://thepiratebay.abcproxy.org/
- - https://tpb.proxyninja.net/
- - https://tpb31.ukpass.co/
- - https://thepiratebay10.org/
- - https://pirate-proxy.africa/
- - https://5mins.eu/
- - https://piratebay.army/
- - https://tpb-visit.me/
- - https://pirate-proxy.ong/
-
-caps:
- categorymappings:
- # Audio
- - {id: 100, cat: Audio, desc: "Audio"}
- - {id: 101, cat: Audio, desc: "Music"}
- - {id: 102, cat: Audio/Audiobook, desc: "Audio Books"}
- - {id: 103, cat: Audio, desc: "Sound Clips"}
- - {id: 104, cat: Audio/Lossless, desc: "FLAC"}
- - {id: 199, cat: Audio/Other, desc: "Audio Other"}
- # Video
- - {id: 200, cat: Movies, desc: "Video"}
- - {id: 201, cat: Movies, desc: "Movies"}
- - {id: 202, cat: Movies, desc: "Movies DVDR"}
- - {id: 203, cat: Audio/Video, desc: "Music Videos"}
- - {id: 204, cat: Movies/Other, desc: "Movie Clips"}
- - {id: 205, cat: TV, desc: "TV Shows"}
- - {id: 206, cat: TV/Other, desc: "Handheld"}
- - {id: 207, cat: Movies/HD, desc: "HD - Movies"}
- - {id: 208, cat: TV/HD, desc: "HD - TV shows"}
- - {id: 209, cat: Movies/3D, desc: "3D"}
- - {id: 210, cat: Movies/SD, desc: "CAM/TS"}
- - {id: 211, cat: Movies/UHD, desc: "UHD/4k - Movies"}
- - {id: 212, cat: TV/UHD, desc: "UHD/4k - TV shows"}
- - {id: 299, cat: Movies/Other, desc: "Video Other"}
- # Applications
- - {id: 300, cat: PC, desc: "Applications"}
- - {id: 301, cat: PC, desc: "Windows"}
- - {id: 302, cat: PC/Mac, desc: "Mac"}
- - {id: 303, cat: PC, desc: "UNIX"}
- - {id: 304, cat: PC/Mobile-Other, desc: "Handheld"}
- - {id: 305, cat: PC/Mobile-iOS, desc: "IOS (iPad/iPhone)"}
- - {id: 306, cat: PC/Mobile-Android, desc: "Android"}
- - {id: 399, cat: PC, desc: "Other OS"}
- # Games
- - {id: 400, cat: Console, desc: "Games"}
- - {id: 401, cat: PC/Games, desc: "PC"}
- - {id: 402, cat: PC/Mac, desc: "Mac"}
- - {id: 403, cat: Console/PS4, desc: "PSx"}
- - {id: 404, cat: Console/XBox, desc: "XBOX360"}
- - {id: 405, cat: Console/Wii, desc: "Wii"}
- - {id: 406, cat: Console/Other, desc: "Handheld"}
- - {id: 407, cat: Console/Other, desc: "IOS (iPad/iPhone)"}
- - {id: 408, cat: Console/Other, desc: "Android"}
- - {id: 499, cat: Console/Other, desc: "Games Other"}
- # Porn
- - {id: 500, cat: XXX, desc: "Porn"}
- - {id: 501, cat: XXX, desc: "Movies"}
- - {id: 502, cat: XXX/DVD, desc: "Movies DVDR"}
- - {id: 503, cat: XXX/ImageSet, desc: "Pictures"}
- - {id: 504, cat: XXX, desc: "Games"}
- - {id: 505, cat: XXX/x264, desc: "HD - Movies"}
- - {id: 506, cat: XXX, desc: "Movie Clips"}
- - {id: 507, cat: XXX/UHD, desc: "UHD/4k - Movies"}
- - {id: 599, cat: XXX/Other, desc: "Porn other"}
- # Other
- - {id: 600, cat: Other, desc: "Other"}
- - {id: 601, cat: Books/EBook, desc: "E-books"}
- - {id: 602, cat: Books/Comics, desc: "Comics"}
- - {id: 603, cat: Books, desc: "Pictures"}
- - {id: 604, cat: Books, desc: "Covers"}
- - {id: 605, cat: Books, desc: "Physibles"}
- - {id: 699, cat: Books/Other, desc: "Other Other"}
-
- modes:
- search: [q]
- tv-search: [q, season, ep]
- movie-search: [q]
- music-search: [q]
- book-search: [q]
-
-settings:
- - name: apiurl
- label: API URL
- type: text
- default: apibay.org
- - name: uploader
- type: text
- label: Filter by Uploader
- - name: info_uploader
- type: info
- label: About filtering by Uploader
- default: "You can filter by Uploader by entering a Case Sensitive username, or leave empty to get all results. Note: this is the username of the Uploader and not the Groupname that often show up at the end of TPB titles, eg -MeGusta."
-
-search:
- paths:
- - path: "https://{{ .Config.apiurl }}/{{ if .Keywords }}q.php?q={{ .Keywords }}&cat={{ join .Categories \",\" }}{{ else }}precompiled/data_top100_recent.json{{ end }}"
- response:
- type: json
-
- keywordsfilters:
- # remove it's #8829
- - name: re_replace
- args: ["(?i)\\bit's\\b", ""]
- # replace simplified Chinese as this confuses TPB search engine #7291
- - name: re_replace
- args: ["([\\p{IsCJKUnifiedIdeographs}\\W]+)", "."]
- - name: tolower
-
- rows:
- selector: "${{ if .Config.uploader }}:has(username:contains({{ .Config.uploader }})){{ else }}{{ end }}"
- count:
- selector: $[0].id
-
- fields:
- _id:
- selector: id
- category:
- selector: category
- title:
- selector: name
- filters:
- - name: re_replace
- args: ["- (\\w+-?\\w*)$", "-$1"]
- - name: re_replace # Season X / Season X Complete --> S0X
- args: ["(?i)\\bSeason[\\s\\.]+(\\d)([\\s\\.]+Complete)?\\b", "S0$1"]
- - name: re_replace # Season XX / Season XX Complete --> SXX
- args: ["(?i)\\bSeason[\\s\\.]+(\\d{1,2})([\\s\\.]+Complete)?\\b", "S$1"]
- details:
- text: "{{ .Config.sitelink }}description.php?id={{ .Result._id }}"
- infohash:
- selector: info_hash
- imdbid:
- selector: imdb
- date:
- # unix
- selector: added
- size:
- selector: size
- files:
- selector: num_files
- optional: true
- seeders:
- selector: seeders
- leechers:
- selector: leechers
- _username:
- selector: username
- description:
- selector: name
- filters:
- - name: prepend
- args: "Uploader: {{ .Result._username }} "
- downloadvolumefactor:
- text: 0
- uploadvolumefactor:
- text: 1
-# json engine n/a
diff --git a/backend/src/trove/indexers/catalog/yts.yml b/backend/src/trove/indexers/catalog/yts.yml
deleted file mode 100644
index 77a86d9..0000000
--- a/backend/src/trove/indexers/catalog/yts.yml
+++ /dev/null
@@ -1,140 +0,0 @@
----
-id: yts
-name: YTS
-description: "YTS is a Public torrent site specialising in HD movies of small size"
-type: public
-language: en-US
-encoding: UTF-8
-requestDelay: 2.5 # 2.5 requests per second (2 causes problems)
-links:
- # if the primary domain changes then don't forget to update the details, download and poster replace args
- - https://yts.bz/
- # official domain list are at https://yifystatus.com/ and official proxies list are at https://ytsproxies.com/
- - https://yts.ninjaproxy1.com/
- - https://yts.proxyninja.org/
- - https://yts.proxyninja.net/
- - https://yts.torrentbay.st/
- - https://yts.torrentsbay.org/
-legacylinks:
- - https://yts.am/ # redirects to .bz
- - https://yts.ag/ # redirects to .bz
- - https://yts.gg/ # redirects to .bz
- - https://yts.mx/
- - https://yts.mrunblock.bond/
- - https://yts.nocensor.cloud/
- - https://yts.unblockit.download/
- - https://yts.lt/ # redirects to .bz
- - https://yts.unblockninja.com/
-
-caps:
- categorymappings:
- # note: the API does not support searching with categories, so these are dummy ones for torznab compatibility
- # we map these newznab cats with the returned quality value in the releases routine.
- - {id: 45, cat: Movies/HD, desc: "Movies/x264/720p"}
- - {id: 44, cat: Movies/HD, desc: "Movies/x264/1080p"}
- - {id: 46, cat: Movies/UHD, desc: "Movies/x264/2160p"}
- - {id: 47, cat: Movies/3D, desc: "Movies/x264/3D"}
-
- modes:
- search: [q]
- movie-search: [q, imdbid]
-
-settings:
- - name: apiurl
- label: API URL
- type: text
- default: movies-api.accel.li
-
-search:
- paths:
- - path: "https://{{ .Config.apiurl }}/api/v2/list_movies.json"
- response:
- type: json
-
- inputs:
- query_term: "{{ if .Query.IMDBID }}{{ .Query.IMDBID }}{{ else }}{{ .Keywords }}{{ end }}"
- # without this the API sometimes returns nothing
- limit: 50
- sort_by: date_added
- order_by: desc
- keywordsfilters:
- # ignore ' (e.g. search for america's Next Top Model)
- - name: re_replace
- args: ["[^\\w]+", " "]
-
- rows:
- selector: data.movies
- attribute: torrents
- multiple: true
- # bug at YTS can return movie_count > 0 and no movie torrents #12598
- missingAttributeEqualsNoResults: true
- count:
- selector: data.movie_count
-
- fields:
- _quality:
- selector: quality
- category:
- selector: quality
- case:
- "720p": 45
- "1080p": 44
- "2160p": 46
- "3D": 47
- "*": 45
- _audio:
- selector: audio_channels
- _depth:
- selector: bit_depth
- _type:
- selector: type
- _codec:
- selector: video_codec
- year:
- selector: ..year
- title_default:
- selector: ..title
- filters:
- - name: append
- args: " ({{ .Result.year }})"
- title:
- selector: ..title_long
- optional: true
- default: "{{ .Result.title_default }}"
- filters:
- - name: replace
- args: [":", ""]
- - name: append
- args: " {{ .Result._quality }} {{ if eq .Result._type \"web\" }}WEBRip{{ else }}BRRip{{ end }} {{ if eq .Result._audio \"5.1\" }}5.1 {{ else }}{{ end }}{{ if eq .Result._depth \"10\" }}10Bit {{ else }}{{ end }}{{ .Result._codec }} -YTS"
- details:
- selector: ..url
- filters:
- - name: re_replace
- args: ["^https?:\\/\\/yts\\.(mx|lt|bz)\\/", "{{ .Config.sitelink }}"] # fix for 12494
- download:
- selector: url
- filters:
- - name: re_replace
- args: ["^https?:\\/\\/yts\\.(mx|lt|bz)\\/", "{{ .Config.sitelink }}"] # fix for 12494
- infohash:
- selector: hash
- poster:
- selector: ..large_cover_image
- filters:
- - name: re_replace
- args: ["^https?:\\/\\/yts\\.(mx|lt|bz)\\/", "{{ .Config.sitelink }}"] # fix for 12494
- imdbid:
- selector: ..imdb_code
- date:
- selector: date_uploaded_unix
- size:
- selector: size_bytes
- seeders:
- selector: seeds
- leechers:
- selector: peers
- downloadvolumefactor:
- text: 0
- uploadvolumefactor:
- text: 1
-# json api v2
diff --git a/backend/tests/api/test_catalog_api.py b/backend/tests/api/test_catalog_api.py
index 2a0fc6c..d241b0f 100644
--- a/backend/tests/api/test_catalog_api.py
+++ b/backend/tests/api/test_catalog_api.py
@@ -15,46 +15,46 @@ def test_list_catalog_returns_entries(client: TestClient) -> None:
resp = client.get("/api/indexers/catalog")
assert resp.status_code == 200
body = resp.json()
- assert len(body) >= 12
+ assert len(body) >= 10
slugs = {e["slug"] for e in body}
- assert "thepiratebay" in slugs
- tpb = next(e for e in body if e["slug"] == "thepiratebay")
- assert tpb["already_installed"] is False
- assert tpb["default_mirror"] in tpb["mirrors"]
+ assert "1337x" in slugs
+ entry = next(e for e in body if e["slug"] == "1337x")
+ assert entry["already_installed"] is False
+ assert entry["default_mirror"] in entry["mirrors"]
def test_install_catalog_entry_creates_indexer(client: TestClient) -> None:
_login(client)
resp = client.post(
- "/api/indexers/catalog/thepiratebay",
- json={"base_url": "https://thepiratebay.org", "name": None},
+ "/api/indexers/catalog/1337x",
+ json={"base_url": "https://1337x.to", "name": None},
)
assert resp.status_code == 201
body = resp.json()
assert body["type"] == "cardigann"
- assert body["base_url"] == "https://thepiratebay.org"
- assert body["name"] == "The Pirate Bay"
+ assert body["base_url"] == "https://1337x.to"
+ assert body["name"] == "1337x"
# already_installed flips on a subsequent list
listing = client.get("/api/indexers/catalog").json()
- tpb = next(e for e in listing if e["slug"] == "thepiratebay")
- assert tpb["already_installed"] is True
+ entry = next(e for e in listing if e["slug"] == "1337x")
+ assert entry["already_installed"] is True
def test_install_twice_dedups_name(client: TestClient) -> None:
_login(client)
first = client.post(
- "/api/indexers/catalog/thepiratebay",
- json={"base_url": "https://thepiratebay.org"},
+ "/api/indexers/catalog/1337x",
+ json={"base_url": "https://1337x.to"},
)
second = client.post(
- "/api/indexers/catalog/thepiratebay",
- json={"base_url": "https://tpb.party"},
+ "/api/indexers/catalog/1337x",
+ json={"base_url": "https://1337x.st"},
)
assert first.status_code == 201
assert second.status_code == 201
- assert first.json()["name"] == "The Pirate Bay"
- assert second.json()["name"] == "The Pirate Bay-2"
+ assert first.json()["name"] == "1337x"
+ assert second.json()["name"] == "1337x-2"
def test_install_unknown_slug_404(client: TestClient) -> None:
@@ -69,7 +69,7 @@ def test_install_unknown_slug_404(client: TestClient) -> None:
def test_install_rejects_base_url_not_in_mirrors(client: TestClient) -> None:
_login(client)
resp = client.post(
- "/api/indexers/catalog/thepiratebay",
+ "/api/indexers/catalog/1337x",
json={"base_url": "https://totally-evil-mirror.example.com"},
)
assert resp.status_code == 422
diff --git a/backend/tests/test_catalog.py b/backend/tests/test_catalog.py
index 6771ad6..4fbdb91 100644
--- a/backend/tests/test_catalog.py
+++ b/backend/tests/test_catalog.py
@@ -16,9 +16,9 @@ def _reset_cache() -> None:
def test_registry_loads() -> None:
entries = catalog.list_entries()
- assert len(entries) >= 12
+ assert len(entries) >= 10
slugs = {e.slug for e in entries}
- for required in ("thepiratebay", "1337x", "nyaa", "eztv", "yts"):
+ for required in ("1337x", "nyaa", "eztv", "limetorrents"):
assert required in slugs, f"missing catalog entry: {required}"
From 7e7441ec225f6008aaaafe1fc7b1ede8f14b8bdb Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:54:48 +0200
Subject: [PATCH 35/39] test: capture Nyaa search HTML fixture
Co-Authored-By: Claude Sonnet 4.6
---
.../tests/fixtures/catalog/nyaa-search.html | 375 ++++++++++++++++++
1 file changed, 375 insertions(+)
create mode 100644 backend/tests/fixtures/catalog/nyaa-search.html
diff --git a/backend/tests/fixtures/catalog/nyaa-search.html b/backend/tests/fixtures/catalog/nyaa-search.html
new file mode 100644
index 0000000..067acea
--- /dev/null
+++ b/backend/tests/fixtures/catalog/nyaa-search.html
@@ -0,0 +1,375 @@
+
+
+
+
+ ubuntu :: Nyaa
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
From 32d00c66323f7b65c8877179804ee6d7bb32bd60 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:55:10 +0200
Subject: [PATCH 36/39] docs: revise catalog list after dropping TPB and YTS
Co-Authored-By: Claude Sonnet 4.6
---
backend/src/trove/docs/03-indexers.md | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/backend/src/trove/docs/03-indexers.md b/backend/src/trove/docs/03-indexers.md
index 4f59d96..7efa80b 100644
--- a/backend/src/trove/docs/03-indexers.md
+++ b/backend/src/trove/docs/03-indexers.md
@@ -89,10 +89,11 @@ For trackers that don't have a native API, Cardigann uses YAML definition files
5. Save and test
**What works**:
-- Static `search.paths[0].path`
-- `search.rows.selector` (CSS selectors)
+- `search.paths[0].path` with Go-template expansion (`{{ .Keywords }}`, `{{ if .Keywords }}…{{ else }}…{{ end }}`, `{{ .Config.X }}`)
+- `search.rows.selector` (CSS selectors, including template-expanded ones)
- Field extraction: `title`, `download`, `size`, `infohash`, `category`
-- Basic filters: `replace`, `regexp`, `append`, `prepend`
+- Basic filters: `replace`, `regexp`, `append`, `prepend`, `trim`, `tolower`, `re_replace`
+- `settings:` block defaults populated automatically as `{{ .Config.X }}` values
**What doesn't work yet**:
- Login flows (cookie, form, POST)
@@ -102,7 +103,9 @@ For trackers that don't have a native API, Cardigann uses YAML definition files
## Catalog (public sites, one click)
-For public, no-account torrent sites, Trove ships with a curated catalog of pre-configured definitions including The Pirate Bay, 1337x, TorrentGalaxy, LimeTorrents, Nyaa, EZTV, YTS, and others. Five slugs map to substitute definitions (ExtraTorrent, KickAssTorrents, TorrentDownload, TorrentProject2, Tokyo Toshokan) because the original sites have no upstream Cardigann YAML — each entry honestly declares its actual source in the description.
+For public, no-account torrent sites, Trove ships with a curated catalog of pre-configured definitions including 1337x, TorrentGalaxy, LimeTorrents, Nyaa, EZTV, and others. Five slugs map to substitute definitions (ExtraTorrent, KickAssTorrents, TorrentDownload, TorrentProject2, Tokyo Toshokan) because the original sites have no upstream Cardigann YAML — each entry honestly declares its actual source in the description.
+
+The Pirate Bay and YTS are not in the current catalog because both return JSON responses that require a native driver rather than the HTML-scraping Cardigann parser. They are candidates for a follow-up JSON-indexer batch.
**How to install:**
From fefd8da34f00b8cfd9cf69ad34d9f308a0683668 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 16:56:21 +0200
Subject: [PATCH 37/39] style: ruff fix unused noqa directives and unused
variable in template tests
Co-Authored-By: Claude Sonnet 4.6
---
backend/src/trove/indexers/cardigann.py | 4 ++--
backend/tests/test_cardigann_templates.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/backend/src/trove/indexers/cardigann.py b/backend/src/trove/indexers/cardigann.py
index 066413d..2478676 100644
--- a/backend/src/trove/indexers/cardigann.py
+++ b/backend/src/trove/indexers/cardigann.py
@@ -216,7 +216,7 @@ def expand_template(
for _ in range(10):
before = text
- def _ifelse(m: re.Match[str]) -> str: # noqa: E306
+ def _ifelse(m: re.Match[str]) -> str:
return m.group(2) if keywords_present else m.group(3)
text = _IF_ELSE_END_RE.sub(_ifelse, text)
@@ -227,7 +227,7 @@ def _ifelse(m: re.Match[str]) -> str: # noqa: E306
for _ in range(10):
before = text
- def _if(m: re.Match[str]) -> str: # noqa: E306
+ def _if(m: re.Match[str]) -> str:
return m.group(2) if keywords_present else ""
text = _IF_END_RE.sub(_if, text)
diff --git a/backend/tests/test_cardigann_templates.py b/backend/tests/test_cardigann_templates.py
index 75fa62a..63ada02 100644
--- a/backend/tests/test_cardigann_templates.py
+++ b/backend/tests/test_cardigann_templates.py
@@ -95,7 +95,7 @@ def test_1337x_url_construction_uses_expanded_template() -> None:
catalog.reset_cache_for_tests()
d = load_definition_yaml(catalog.read_yaml("1337x"))
- drv = CardigannIndexer(d, base_url="https://1337x.to")
+ CardigannIndexer(d, base_url="https://1337x.to") # verify instantiation succeeds
path = expand_template(d.search_path, keywords="ubuntu", config=d.config_defaults)
assert "{{" not in path, f"path still has templates: {path}"
assert "ubuntu" in path
From e90b45369450104a2f4badfe3171cc42973172d2 Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 17:03:49 +0200
Subject: [PATCH 38/39] feat(cardigann): support or-expressions and inline
re_replace in templates
Fixes torrentgalaxy and bitsearch search URLs which used template
constructs the initial expander did not cover. Sanity sweep confirms
all 10 catalog sites now produce URLs without unresolved template
tokens.
Co-Authored-By: Claude Opus 4.7 (1M context)
---
backend/src/trove/indexers/cardigann.py | 41 +++++++++++++++++++++++
backend/tests/test_cardigann_templates.py | 34 +++++++++++++++++++
2 files changed, 75 insertions(+)
diff --git a/backend/src/trove/indexers/cardigann.py b/backend/src/trove/indexers/cardigann.py
index 2478676..e240b50 100644
--- a/backend/src/trove/indexers/cardigann.py
+++ b/backend/src/trove/indexers/cardigann.py
@@ -184,6 +184,16 @@ def load_definition_yaml(text: str) -> CardigannDefinition:
_JOIN_RE = re.compile(
r'\{\{\s*join\s+\.Categories\s+"[^"]*"\s*\}\}',
)
+# Go template `or` expression used as a value (not inside an if condition).
+# Example: {{ or .Query.IMDBID .Keywords }}
+# Returns the first truthy arg. We only support .Query.* (always empty) and
+# .Keywords — enough for the YAMLs in the current catalog.
+_OR_EXPR_RE = re.compile(r"\{\{\s*or\s+(.+?)\s*\}\}")
+# Go template `re_replace` function call used inline (not as a field filter).
+# Example: {{ re_replace .Config.sort "_" "" }}
+_RE_REPLACE_INLINE_RE = re.compile(
+ r'\{\{\s*re_replace\s+\.Config\.([\w\-]+)\s+"([^"]*)"\s+"([^"]*)"\s*\}\}'
+)
_KEYWORDS_RE = re.compile(r"\{\{\s*\.Keywords\s*\}\}")
_QUERY_IMDB_RE = re.compile(r"\{\{\s*\.Query\.(IMDBID|TMDBID|TVDBID)\s*\}\}")
_CONFIG_RE = re.compile(r"\{\{\s*\.Config\.([\w\-]+)\s*\}\}")
@@ -234,6 +244,37 @@ def _if(m: re.Match[str]) -> str:
if text == before:
break
+ # `or` expression as value: return first truthy arg.
+ def _or_value(m: re.Match[str]) -> str:
+ args = m.group(1).split()
+ for arg in args:
+ if arg == ".Keywords":
+ if keywords_present:
+ return keywords
+ elif arg.startswith(".Query."):
+ # .Query.IMDBID, .Query.TMDBID, etc. are always empty here.
+ continue
+ elif arg.startswith(".Config."):
+ name = arg[len(".Config.") :]
+ val = cfg.get(name, "")
+ if val:
+ return val
+ # Unknown reference → skip and try next arg.
+ return ""
+
+ text = _OR_EXPR_RE.sub(_or_value, text)
+
+ # Inline re_replace on a Config value.
+ def _inline_re_replace(m: re.Match[str]) -> str:
+ config_name, pattern, replacement = m.group(1), m.group(2), m.group(3)
+ source = cfg.get(config_name, "")
+ try:
+ return re.sub(pattern, replacement, source)
+ except re.error:
+ return source
+
+ text = _RE_REPLACE_INLINE_RE.sub(_inline_re_replace, text)
+
# Variable substitutions.
text = _KEYWORDS_RE.sub(lambda _: keywords, text)
text = _QUERY_IMDB_RE.sub("", text)
diff --git a/backend/tests/test_cardigann_templates.py b/backend/tests/test_cardigann_templates.py
index 63ada02..bb1c428 100644
--- a/backend/tests/test_cardigann_templates.py
+++ b/backend/tests/test_cardigann_templates.py
@@ -100,3 +100,37 @@ def test_1337x_url_construction_uses_expanded_template() -> None:
assert "{{" not in path, f"path still has templates: {path}"
assert "ubuntu" in path
assert path.startswith("/") or "search" in path or "sort-" in path
+
+
+def test_expand_or_expression_returns_first_truthy() -> None:
+ # Go template `or` returns first truthy arg. .Query.IMDBID is always empty,
+ # so `or .Query.IMDBID .Keywords` should return keywords.
+ tmpl = "get-posts/keywords:{{ or .Query.IMDBID .Keywords }}"
+ assert expand_template(tmpl, keywords="ubuntu") == "get-posts/keywords:ubuntu"
+
+
+def test_expand_or_expression_fallback_to_keywords() -> None:
+ tmpl = "{{ or .Query.IMDBID .Keywords }}"
+ assert expand_template(tmpl, keywords="ubuntu") == "ubuntu"
+ # When nothing is truthy, result is empty string (last arg is also empty).
+ assert expand_template(tmpl, keywords="") == ""
+
+
+def test_expand_inline_re_replace_on_config() -> None:
+ # bitsearch path uses: search{{ re_replace .Config.sort "_" "" }}?q=...
+ cfg = {"sort": "date_desc"}
+ tmpl = '{{ re_replace .Config.sort "_" "" }}'
+ assert expand_template(tmpl, keywords="x", config=cfg) == "datedesc"
+
+
+def test_expand_bitsearch_full_path() -> None:
+ cfg = {"sort": "date_desc"}
+ tmpl = '{{ if .Keywords }}search{{ re_replace .Config.sort "_" "" }}?q={{ .Keywords }}{{ else }}/{{ end }}'
+ assert expand_template(tmpl, keywords="ubuntu", config=cfg) == "searchdatedesc?q=ubuntu"
+
+
+def test_expand_torrentgalaxy_full_path() -> None:
+ tmpl = "get-posts/{{ if or .Query.IMDBID .Keywords }}keywords:{{ or .Query.IMDBID .Keywords }}{{ else }}{{ end }}{{ range .Categories }}:category:{{.}}{{end}}"
+ result = expand_template(tmpl, keywords="ubuntu")
+ assert result == "get-posts/keywords:ubuntu"
+ assert "{{" not in result
From 369a165aea55eb2c972f5d921468047e0502cfce Mon Sep 17 00:00:00 2001
From: MasterDraco
Date: Mon, 20 Apr 2026 17:20:29 +0200
Subject: [PATCH 39/39] v0.11.0: public torrent-site catalog with Go template
support
---
backend/pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 2837537..06381fb 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "trove"
-version = "0.10.4"
+version = "0.11.0"
description = "A modern replacement for FlexGet with multi-indexer search, Usenet support, and optional local AI."
readme = "../README.md"
requires-python = ">=3.12"