diff --git a/.gitignore b/.gitignore index 2e34882..3f553bb 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ session.secret # Logs *.log /logs/ + +# Worktrees +.worktrees/ diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..44edd36 --- /dev/null +++ b/TODO.md @@ -0,0 +1,9 @@ +# TODO + +## Catalog fixtures + +- [ ] Capture `-search.html` fixtures for the 11 remaining catalog slugs. + Pattern: `backend/tests/fixtures/catalog/-search.html`. + Extraction test lives in `backend/tests/test_catalog_fixtures.py` and + is parametrized over whichever fixtures are present. When captured, + the test runs `_extract_release` against each row and asserts ≥1 result. diff --git a/backend/migrations/versions/0016_indexer_catalog_slug.py b/backend/migrations/versions/0016_indexer_catalog_slug.py new file mode 100644 index 0000000..c78c8a6 --- /dev/null +++ b/backend/migrations/versions/0016_indexer_catalog_slug.py @@ -0,0 +1,38 @@ +"""indexer.catalog_slug column + +Revision ID: 0016 +Revises: 0015 +Create Date: 2026-04-20 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +import sqlmodel +from alembic import op + +revision: str = "0016" +down_revision: str | None = "0015" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + with op.batch_alter_table("indexer") as batch: + batch.add_column( + sa.Column( + "catalog_slug", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=True, + ) + ) + op.create_index( + "ix_indexer_catalog_slug", "indexer", ["catalog_slug"], unique=False + ) + + +def downgrade() -> None: + op.drop_index("ix_indexer_catalog_slug", table_name="indexer") + with op.batch_alter_table("indexer") as batch: + batch.drop_column("catalog_slug") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 2837537..06381fb 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "trove" -version = "0.10.4" +version = "0.11.0" description = "A modern replacement for FlexGet with multi-indexer search, Usenet support, and optional local AI." readme = "../README.md" requires-python = ">=3.12" diff --git a/backend/src/trove/api/catalog.py b/backend/src/trove/api/catalog.py new file mode 100644 index 0000000..a1d959f --- /dev/null +++ b/backend/src/trove/api/catalog.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, Field +from sqlmodel import Session, select + +from trove.api.deps import current_user, db_session +from trove.api.indexers import IndexerOut, _to_out +from trove.clients.base import Protocol +from trove.indexers.base import Category +from trove.indexers.cardigann import load_definition_yaml +from trove.models.indexer import IndexerRow +from trove.models.user import User +from trove.services import catalog, indexer_registry + +router = APIRouter() + + +class CatalogEntryOut(BaseModel): + slug: str + display_name: str + description: str + categories: list[Category] + mirrors: list[str] + default_mirror: str + protocol: Protocol + logo: str | None = None + already_installed: bool + + +class CatalogInstallRequest(BaseModel): + base_url: str = Field(min_length=1, max_length=512) + name: str | None = Field(default=None, max_length=64) + + +@router.get("", response_model=list[CatalogEntryOut]) +async def list_catalog( + session: Session = Depends(db_session), + _user: User = Depends(current_user), +) -> list[CatalogEntryOut]: + installed_slugs = set( + session.exec( + select(IndexerRow.catalog_slug).where( + IndexerRow.catalog_slug.is_not(None) # type: ignore[union-attr] + ) + ).all() + ) + out: list[CatalogEntryOut] = [] + for entry in catalog.list_entries(): + out.append( + CatalogEntryOut( + slug=entry.slug, + display_name=entry.display_name, + description=entry.description, + categories=entry.categories, + mirrors=entry.mirrors, + default_mirror=entry.default_mirror, + protocol=entry.protocol, + logo=entry.logo, + already_installed=entry.slug in installed_slugs, + ) + ) + return out + + +def _dedup_name(session: Session, base: str) -> str: + candidate = base + suffix = 2 + while session.exec(select(IndexerRow).where(IndexerRow.name == candidate)).first() is not None: + candidate = f"{base}-{suffix}" + suffix += 1 + return candidate + + +@router.post("/{slug}", response_model=IndexerOut, status_code=status.HTTP_201_CREATED) +async def install_catalog_entry( + slug: str, + payload: CatalogInstallRequest, + session: Session = Depends(db_session), + _user: User = Depends(current_user), +) -> IndexerOut: + try: + entry = catalog.get_entry(slug) + except KeyError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="unknown_slug") from None + + if payload.base_url not in entry.mirrors: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="base_url_not_in_catalog_mirrors", + ) + + try: + yaml_text = catalog.read_yaml(slug) + load_definition_yaml(yaml_text) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"catalog_yaml_broken: {e}", + ) from e + + base_name = payload.name or entry.display_name + name = _dedup_name(session, base_name) + + row = IndexerRow( + name=name, + type="cardigann", + protocol=entry.protocol.value, + base_url=payload.base_url, + credentials_cipher=indexer_registry.encrypt_credentials({}), + definition_yaml=yaml_text, + enabled=True, + priority=50, + catalog_slug=slug, + ) + session.add(row) + session.commit() + session.refresh(row) + return _to_out(row) diff --git a/backend/src/trove/docs/03-indexers.md b/backend/src/trove/docs/03-indexers.md index 78ed649..7efa80b 100644 --- a/backend/src/trove/docs/03-indexers.md +++ b/backend/src/trove/docs/03-indexers.md @@ -89,10 +89,11 @@ For trackers that don't have a native API, Cardigann uses YAML definition files 5. Save and test **What works**: -- Static `search.paths[0].path` -- `search.rows.selector` (CSS selectors) +- `search.paths[0].path` with Go-template expansion (`{{ .Keywords }}`, `{{ if .Keywords }}…{{ else }}…{{ end }}`, `{{ .Config.X }}`) +- `search.rows.selector` (CSS selectors, including template-expanded ones) - Field extraction: `title`, `download`, `size`, `infohash`, `category` -- Basic filters: `replace`, `regexp`, `append`, `prepend` +- Basic filters: `replace`, `regexp`, `append`, `prepend`, `trim`, `tolower`, `re_replace` +- `settings:` block defaults populated automatically as `{{ .Config.X }}` values **What doesn't work yet**: - Login flows (cookie, form, POST) @@ -100,6 +101,24 @@ For trackers that don't have a native API, Cardigann uses YAML definition files - Custom headers beyond what httpx sends by default - JavaScript-heavy sites that need Playwright/Puppeteer +## Catalog (public sites, one click) + +For public, no-account torrent sites, Trove ships with a curated catalog of pre-configured definitions including 1337x, TorrentGalaxy, LimeTorrents, Nyaa, EZTV, and others. Five slugs map to substitute definitions (ExtraTorrent, KickAssTorrents, TorrentDownload, TorrentProject2, Tokyo Toshokan) because the original sites have no upstream Cardigann YAML — each entry honestly declares its actual source in the description. + +The Pirate Bay and YTS are not in the current catalog because both return JSON responses that require a native driver rather than the HTML-scraping Cardigann parser. They are candidates for a follow-up JSON-indexer batch. + +**How to install:** + +1. On `/indexers`, click **Browse catalog** (next to **Add indexer**). +2. Pick a mirror from the dropdown on the site's tile — catalog entries list multiple known mirrors for sites that have them. +3. Click **Add**. The tile flips to **Installed** and the site appears on `/indexers` as a normal Cardigann indexer. + +The onboarding wizard also surfaces these sites as an optional step — pick any you want in one go. + +**Behind the scenes**: a catalog-installed entry is an ordinary `type=cardigann` indexer with a vendored YAML definition. The Test, Edit, and Delete buttons work the same way they do for hand-added indexers. If you ever need to override the URL or rename the entry, use **Edit** — nothing is special about catalog rows. + +**Updating definitions**: the shipped YAMLs track `Prowlarr/Indexers`. When a site changes its HTML, searches will start returning 0 results. Upstream usually has a fix within days. A maintainer can run `scripts/update-catalog.py diff` to see what's changed upstream, then `sync` to pull — this is manual today, typically done alongside a release. + ## Priority and ordering Each indexer has a **priority** field (default 50). Lower numbers run first in the fan-out, but since Trove queries all enabled indexers in parallel, priority mainly matters for tie-breaking when the same release appears from multiple sources — the higher-priority indexer's copy is kept. diff --git a/backend/src/trove/indexers/cardigann.py b/backend/src/trove/indexers/cardigann.py index a601561..e240b50 100644 --- a/backend/src/trove/indexers/cardigann.py +++ b/backend/src/trove/indexers/cardigann.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import re from dataclasses import dataclass, field from typing import Any @@ -18,6 +19,9 @@ SearchQuery, ) +log = logging.getLogger(__name__) +_WARNED_FILTERS: set[str] = set() + SIZE_RE = re.compile(r"([\d.,]+)\s*(TB|GB|MB|KB|B)", re.IGNORECASE) SIZE_MULTIPLIERS = { "B": 1, @@ -62,6 +66,7 @@ class CardigannDefinition: fields: dict[str, FieldSpec] category_mapping: dict[str, Category] = field(default_factory=dict) protocol: Protocol = Protocol.TORRENT + config_defaults: dict[str, str] = field(default_factory=dict) # from settings: block def _coerce_field(spec_data: dict[str, Any] | str) -> FieldSpec: @@ -97,13 +102,34 @@ def load_definition(data: dict[str, Any]) -> CardigannDefinition: if not isinstance(cat, dict): continue cat_id = str(cat.get("id", "")) - mapped = _map_category(int(cat.get("cat", 0)) if cat.get("cat") else 0) + raw_cat = cat.get("cat", 0) + try: + numeric_cat = int(raw_cat) + except (ValueError, TypeError): + numeric_cat = 0 + mapped = _map_category(numeric_cat) if cat_id and mapped is not None: category_mapping[cat_id] = mapped protocol_str = (data.get("type") or "").lower() protocol = Protocol.USENET if "usenet" in protocol_str else Protocol.TORRENT + config_defaults: dict[str, str] = {} + for item in data.get("settings") or []: + if not isinstance(item, dict): + continue + name = item.get("name") + if not isinstance(name, str): + continue + default = item.get("default") + if default is None: + continue + # Normalize bools to Go-template casing + if isinstance(default, bool): + config_defaults[name] = "True" if default else "False" + else: + config_defaults[name] = str(default) + return CardigannDefinition( site=str(data.get("site", "")), name=str(data.get("name") or data.get("site", "")), @@ -114,6 +140,7 @@ def load_definition(data: dict[str, Any]) -> CardigannDefinition: fields=fields_map, category_mapping=category_mapping, protocol=protocol, + config_defaults=config_defaults, ) @@ -124,6 +151,145 @@ def load_definition_yaml(text: str) -> CardigannDefinition: return load_definition(data) +# --------------------------------------------------------------------------- +# Minimal Go-template subset evaluator for Prowlarr YAMLs. +# +# Supported: +# {{ .Keywords }} -> query terms or "" +# {{ .Query.IMDBID }}, {{ .Query.TMDBID }} -> always "" +# {{ .Config.X }} -> config[X] or "" +# {{ if .Keywords }}A{{ else }}B{{ end }} -> A if keywords non-empty, else B +# {{ if and .Keywords ... }}A{{ else }}B{{ end }}-> same (conservative: keywords-present check) +# {{ if or .Query.IMDBID .Keywords }}...{{ end }}-> keywords-present check +# {{ range .Categories }}...{{ end }} -> "" (categories not piped through) +# {{ join .Categories "," }} -> "" +# +# NOT supported: arbitrary nested ifs, function calls inside template +# expressions, range loops over arbitrary collections. +# Unrecognized templates pass through unchanged (never crash). +# --------------------------------------------------------------------------- + +_IF_ELSE_END_RE = re.compile( + r"\{\{\s*if\s+(.+?)\s*\}\}(.*?)\{\{\s*else\s*\}\}(.*?)\{\{\s*end\s*\}\}", + re.DOTALL, +) +_IF_END_RE = re.compile( + r"\{\{\s*if\s+(.+?)\s*\}\}(.*?)\{\{\s*end\s*\}\}", + re.DOTALL, +) +_RANGE_END_RE = re.compile( + r"\{\{\s*range\s+.+?\s*\}\}.*?\{\{\s*end\s*\}\}", + re.DOTALL, +) +_JOIN_RE = re.compile( + r'\{\{\s*join\s+\.Categories\s+"[^"]*"\s*\}\}', +) +# Go template `or` expression used as a value (not inside an if condition). +# Example: {{ or .Query.IMDBID .Keywords }} +# Returns the first truthy arg. We only support .Query.* (always empty) and +# .Keywords — enough for the YAMLs in the current catalog. +_OR_EXPR_RE = re.compile(r"\{\{\s*or\s+(.+?)\s*\}\}") +# Go template `re_replace` function call used inline (not as a field filter). +# Example: {{ re_replace .Config.sort "_" "" }} +_RE_REPLACE_INLINE_RE = re.compile( + r'\{\{\s*re_replace\s+\.Config\.([\w\-]+)\s+"([^"]*)"\s+"([^"]*)"\s*\}\}' +) +_KEYWORDS_RE = re.compile(r"\{\{\s*\.Keywords\s*\}\}") +_QUERY_IMDB_RE = re.compile(r"\{\{\s*\.Query\.(IMDBID|TMDBID|TVDBID)\s*\}\}") +_CONFIG_RE = re.compile(r"\{\{\s*\.Config\.([\w\-]+)\s*\}\}") + + +def expand_template( + text: str, + *, + keywords: str = "", + config: dict[str, str] | None = None, +) -> str: + """Expand a Cardigann/Go-template string. + + See the module comment above for the supported subset. Anything unrecognized + passes through unchanged so upstream failures are visible rather than + silently producing wrong URLs. + """ + if "{{" not in text: + return text + cfg = config or {} + keywords_present = bool(keywords) + + # Drop range blocks entirely (categories-iter is the only real use). + text = _RANGE_END_RE.sub("", text) + # join .Categories -> empty + text = _JOIN_RE.sub("", text) + + # Repeatedly resolve if/else/end from the innermost occurrence outward. + # Prowlarr chains them, so loop until stable. + for _ in range(10): + before = text + + def _ifelse(m: re.Match[str]) -> str: + return m.group(2) if keywords_present else m.group(3) + + text = _IF_ELSE_END_RE.sub(_ifelse, text) + if text == before: + break + + # Bare {{ if ... }}X{{ end }} without else -> X if keywords else "" + for _ in range(10): + before = text + + def _if(m: re.Match[str]) -> str: + return m.group(2) if keywords_present else "" + + text = _IF_END_RE.sub(_if, text) + if text == before: + break + + # `or` expression as value: return first truthy arg. + def _or_value(m: re.Match[str]) -> str: + args = m.group(1).split() + for arg in args: + if arg == ".Keywords": + if keywords_present: + return keywords + elif arg.startswith(".Query."): + # .Query.IMDBID, .Query.TMDBID, etc. are always empty here. + continue + elif arg.startswith(".Config."): + name = arg[len(".Config.") :] + val = cfg.get(name, "") + if val: + return val + # Unknown reference → skip and try next arg. + return "" + + text = _OR_EXPR_RE.sub(_or_value, text) + + # Inline re_replace on a Config value. + def _inline_re_replace(m: re.Match[str]) -> str: + config_name, pattern, replacement = m.group(1), m.group(2), m.group(3) + source = cfg.get(config_name, "") + try: + return re.sub(pattern, replacement, source) + except re.error: + return source + + text = _RE_REPLACE_INLINE_RE.sub(_inline_re_replace, text) + + # Variable substitutions. + text = _KEYWORDS_RE.sub(lambda _: keywords, text) + text = _QUERY_IMDB_RE.sub("", text) + + def _cfg(m: re.Match[str]) -> str: + return cfg.get(m.group(1), "") + + text = _CONFIG_RE.sub(_cfg, text) + + # .Result.* is not a request-time substitution — leave intact for + # field-extraction-time expansion (a separate call site). + + return text + + def _map_category(cat_id: int) -> Category | None: if 2000 <= cat_id < 3000: return Category.MOVIES @@ -174,16 +340,19 @@ async def test_connection(self) -> IndexerHealth: return IndexerHealth(ok=True) async def search(self, query: SearchQuery) -> list[Release]: + cfg = self.definition.config_defaults + kw = query.terms or "" params: dict[str, Any] = {} for key, template in (self.definition.search_params or {}).items(): if isinstance(template, str): - params[key] = template.replace("{{.Query.Keywords}}", query.terms) + params[key] = expand_template(template, keywords=kw, config=cfg) else: params[key] = template if "q" not in params and "search" not in params and "query" not in params: - params["q"] = query.terms + params["q"] = kw - url = self.base_url + self.definition.search_path + path = expand_template(self.definition.search_path, keywords=kw, config=cfg) + url = self.base_url + path try: resp = await self._client.get(url, params=params) except httpx.HTTPError as e: @@ -192,28 +361,37 @@ async def search(self, query: SearchQuery) -> list[Release]: raise IndexerError(f"{self.name}: HTTP {resp.status_code}") soup = BeautifulSoup(resp.text, "lxml") - rows = soup.select(self.definition.rows_selector) + rows_selector = expand_template(self.definition.rows_selector, keywords=kw, config=cfg) + rows = soup.select(rows_selector) releases: list[Release] = [] for row in rows[: query.limit]: - release = self._extract_release(row) + release = self._extract_release(row, keywords=kw, config=cfg) if release is not None: releases.append(release) return releases - def _extract_release(self, row: Tag) -> Release | None: - title = self._extract_field(row, "title") + def _extract_release( + self, + row: Tag, + *, + keywords: str = "", + config: dict[str, str] | None = None, + ) -> Release | None: + title = self._extract_field(row, "title", keywords=keywords, config=config) if not title: return None - download_url = self._extract_field(row, "download") or self._extract_field(row, "details") + download_url = self._extract_field( + row, "download", keywords=keywords, config=config + ) or self._extract_field(row, "details", keywords=keywords, config=config) if download_url and not download_url.startswith(("http://", "https://", "magnet:")): download_url = self.base_url + ( download_url if download_url.startswith("/") else f"/{download_url}" ) - size = _parse_size(self._extract_field(row, "size")) - infohash = self._extract_field(row, "infohash") or None - category = self._extract_field(row, "category") + size = _parse_size(self._extract_field(row, "size", keywords=keywords, config=config)) + infohash = self._extract_field(row, "infohash", keywords=keywords, config=config) or None + category = self._extract_field(row, "category", keywords=keywords, config=config) return Release( title=title, @@ -225,7 +403,14 @@ def _extract_release(self, row: Tag) -> Release | None: source=self.name, ) - def _extract_field(self, row: Tag, key: str) -> str | None: + def _extract_field( + self, + row: Tag, + key: str, + *, + keywords: str = "", + config: dict[str, str] | None = None, + ) -> str | None: spec = self.definition.fields.get(key) if spec is None: return None @@ -233,14 +418,24 @@ def _extract_field(self, row: Tag, key: str) -> str | None: return spec.text target: Tag | None = row - if spec.selector: - target = row.select_one(spec.selector) + selector = ( + expand_template(spec.selector, keywords=keywords, config=(config or {})) + if spec.selector + else None + ) + if selector: + target = row.select_one(selector) if target is None: return None value: str | None - if spec.attribute: - raw = target.get(spec.attribute) + attribute = ( + expand_template(spec.attribute, keywords=keywords, config=(config or {})) + if spec.attribute + else None + ) + if attribute: + raw = target.get(attribute) value = (raw[0] if raw else None) if isinstance(raw, list) else raw else: value = target.get_text(" ", strip=True) @@ -267,4 +462,37 @@ def _apply_filter(self, value: str | None, flt: dict[str, Any]) -> str | None: return value + args if name == "prepend" and isinstance(args, str): return args + value + if name == "urldecode": + from urllib.parse import unquote + + return unquote(value) + if name == "split" and isinstance(args, list) and len(args) >= 2: + delimiter = str(args[0]) + try: + index = int(args[1]) + except (TypeError, ValueError): + return value + parts = value.split(delimiter) + if not parts: + return value + try: + return parts[index] + except IndexError: + return value + if name == "trim": + if isinstance(args, str) and args: + return value.strip(args) + return value.strip() + if name == "tolower": + return value.lower() + if name == "re_replace" and isinstance(args, list) and len(args) >= 2: + pattern = str(args[0]) + replacement = str(args[1]) + try: + return re.sub(pattern, replacement, value) + except re.error: + return value + if name and name not in _WARNED_FILTERS: + _WARNED_FILTERS.add(name) + log.warning("cardigann: unknown filter %r — passing value through unchanged", name) return value diff --git a/backend/src/trove/indexers/catalog/1337x.yml b/backend/src/trove/indexers/catalog/1337x.yml new file mode 100644 index 0000000..2f8bd72 --- /dev/null +++ b/backend/src/trove/indexers/catalog/1337x.yml @@ -0,0 +1,295 @@ +--- +id: 1337x +name: 1337x +description: "1337x is a Public torrent site that offers verified torrent downloads" +language: en-US +type: public +encoding: UTF-8 +requestDelay: 3 +# get status and news on domains at the official site https://1337x-status.org/ +links: + - https://1337x.to/ + - https://1337x.st/ + - https://x1337x.ws/ + - https://x1337x.eu/ + - https://x1337x.cc/ +legacylinks: + - https://1337x.is/ + - https://1337x.gd/ + - https://1337x.mrunblock.bond/ + - https://1337x.abcproxy.org/ + - https://1337x.so/ + - https://1337x.unblockit.download/ + - https://1337x.unblockninja.com/ # keyword search not working + - https://1337x.ninjaproxy1.com/ # keyword search not working + - https://1337x.proxyninja.org/ # keyword search not working + - https://1337x.proxyninja.net/ # keyword search not working + - https://1337x.torrentbay.st/ # keyword search not working + - https://1337x.torrentsbay.org/ # keyword search not working + - https://x1337x.se/ + +caps: + categorymappings: + # Anime + - {id: 28, cat: TV/Anime, desc: "Anime/Anime"} + - {id: 78, cat: TV/Anime, desc: "Anime/Dual Audio"} + - {id: 79, cat: TV/Anime, desc: "Anime/Dubbed"} + - {id: 80, cat: TV/Anime, desc: "Anime/Subbed"} + - {id: 81, cat: TV/Anime, desc: "Anime/Raw"} + # Audio + - {id: 22, cat: Audio/MP3, desc: "Music/MP3"} + - {id: 23, cat: Audio/Lossless, desc: "Music/Lossless"} + - {id: 24, cat: Audio, desc: "Music/DVD"} + - {id: 25, cat: Audio/Video, desc: "Music/Video"} + - {id: 26, cat: Audio, desc: "Music/Radio"} + - {id: 27, cat: Audio/Other, desc: "Music/Other"} + - {id: 53, cat: Audio, desc: "Music/Album"} + - {id: 58, cat: Audio, desc: "Music/Box set"} + - {id: 59, cat: Audio, desc: "Music/Discography"} + - {id: 60, cat: Audio, desc: "Music/Single"} + - {id: 68, cat: Audio, desc: "Music/Concerts"} + - {id: 69, cat: Audio, desc: "Music/AAC"} + # Movies + - {id: 1, cat: Movies/DVD, desc: "Movies/DVD"} + - {id: 2, cat: Movies/SD, desc: "Movies/Divx/Xvid"} + - {id: 3, cat: Movies, desc: "Movies/SVCD/VCD"} + - {id: 4, cat: Movies/Foreign, desc: "Movies/Dubs/Dual Audio"} + - {id: 42, cat: Movies/HD, desc: "Movies/HD"} + - {id: 54, cat: Movies/HD, desc: "Movies/h.264/x264"} + - {id: 55, cat: Movies, desc: "Movies/Mp4"} + - {id: 66, cat: Movies/3D, desc: "Movies/3D"} + - {id: 70, cat: Movies/HD, desc: "Movies/HEVC/x265"} + - {id: 73, cat: Movies, desc: "Movies/Bollywood"} + - {id: 76, cat: Movies/UHD, desc: "Movies/UHD"} + # TV + - {id: 5, cat: TV, desc: "TV/DVD"} + - {id: 6, cat: TV, desc: "TV/Divx/Xvid"} + - {id: 7, cat: TV, desc: "TV/SVCD/VCD"} + - {id: 41, cat: TV/HD, desc: "TV/HD"} + - {id: 71, cat: TV, desc: "TV/HEVC/x265"} + - {id: 74, cat: TV, desc: "TV/Cartoons"} + - {id: 75, cat: TV/SD, desc: "TV/SD"} + - {id: 9, cat: TV/Documentary, desc: "TV/Documentary"} + # Apps + - {id: 18, cat: PC, desc: "Apps/PC Software"} + - {id: 19, cat: PC/Mac, desc: "Apps/Mac"} + - {id: 20, cat: PC, desc: "Apps/Linux"} + - {id: 21, cat: PC, desc: "Apps/Other"} + - {id: 56, cat: PC/Mobile-Android, desc: "Apps/Android"} + - {id: 57, cat: PC/Mobile-iOS, desc: "Apps/iOS"} + # Games + - {id: 10, cat: PC/Games, desc: "Games/PC Game"} + - {id: 11, cat: Console/PS3, desc: "Games/PS2"} + - {id: 12, cat: Console/PSP, desc: "Games/PSP"} + - {id: 13, cat: Console/XBox, desc: "Games/Xbox"} + - {id: 14, cat: Console/XBox 360, desc: "Games/Xbox360"} + - {id: 15, cat: Console/PS3, desc: "Games/PS1"} + - {id: 16, cat: Console/Other, desc: "Games/Dreamcast"} + - {id: 17, cat: PC/Mobile-Other, desc: "Games/Other"} + - {id: 43, cat: Console/PS3, desc: "Games/PS3"} + - {id: 44, cat: Console/Wii, desc: "Games/Wii"} + - {id: 45, cat: Console/NDS, desc: "Games/DS"} + - {id: 46, cat: Console/Other, desc: "Games/GameCube"} + - {id: 72, cat: Console/3DS, desc: "Games/3DS"} + - {id: 77, cat: Console/PS4, desc: "Games/PS4"} + - {id: 82, cat: Console/Other, desc: "Games/Switch"} + # XXX + - {id: 48, cat: XXX/DVD, desc: "XXX/Video"} + - {id: 49, cat: XXX/ImageSet, desc: "XXX/Picture"} + - {id: 50, cat: XXX, desc: "XXX/Magazine"} + - {id: 51, cat: XXX, desc: "XXX/Hentai"} + - {id: 67, cat: XXX, desc: "XXX/Games"} + # Other + - {id: 33, cat: Other, desc: "Other/Emulation"} + - {id: 34, cat: Books, desc: "Other/Tutorial"} + - {id: 35, cat: Other, desc: "Other/Sounds"} + - {id: 36, cat: Books/EBook, desc: "Other/E-books"} + - {id: 37, cat: Other, desc: "Other/Images"} + - {id: 38, cat: Other, desc: "Other/Mobile Phone"} + - {id: 39, cat: Books/Comics, desc: "Other/Comics"} + - {id: 40, cat: Other/Misc, desc: "Other/Other"} + - {id: 47, cat: Other, desc: "Other/Nulled Script"} + - {id: 52, cat: Audio/Audiobook, desc: "Other/Audiobook"} + + modes: + search: [q] + tv-search: [q, season, ep] + movie-search: [q] + music-search: [q] + book-search: [q] + allowrawsearch: true + +settings: + - name: uploader + type: text + label: Filter by Uploader + - name: info_uploader + type: info + label: About filtering by Uploader + default: "You can filter by Uploader by entering a Case Sensitive username, or leave empty to get all results.
Note: this is the username of the Uploader and not the Groupname that often show up at the end of 1337x titles, eg -GalaxyRG." + - name: info_flaresolverr + type: info_flaresolverr + - name: downloadlink + type: select + label: Download link + default: "http://itorrents.org/" + options: + "http://itorrents.org/": iTorrents.org + "magnet:": magnet + - name: downloadlink2 + type: select + label: Download link (fallback) + default: "magnet:" + options: + "http://itorrents.org/": iTorrents.org + "magnet:": magnet + - name: info_download + type: info + label: About the Download links + default: As the iTorrents .torrent download link on this site is known to fail from time to time, we suggest using the magnet link as a fallback. The BTCache and Torrage services are not supported because they require additional user interaction (a captcha for BTCache and a download button on Torrage.) + - name: disablesort + type: checkbox + label: Disable sorting - 1337x prevents sorting searches during high server load, which breaks the indexer when performing a keyword search - disable if you get zero results + default: false + - name: sort + type: select + label: Sort requested from site + default: time + options: + time: created + seeders: seeders + size: size + - name: type + type: select + label: Order requested from site + default: desc + options: + desc: desc + asc: asc + +download: + # the .torrent URL and magnet URI are on the details page + selectors: + - selector: ul li a[href^="{{ .Config.downloadlink }}"] + attribute: href + - selector: ul li a[href^="{{ .Config.downloadlink2 }}"] + attribute: href + +search: + paths: + # present first page of movies tv and music results if there are no search parms supplied (20 hits per page) + - path: "{{ if and (.Keywords) (eq .Config.disablesort .False) }}sort-{{ else }}{{ end }}{{ if .Keywords }}search/{{ .Keywords }}{{ else }}cat/Movies{{ end }}{{ if and (.Keywords) (eq .Config.disablesort .False) }}/{{ .Config.sort }}/{{ .Config.type }}{{ else }}{{ end }}/1/" + - path: "{{ if and (.Keywords) (eq .Config.disablesort .False) }}sort-{{ else }}{{ end }}{{ if .Keywords }}search/{{ .Keywords }}{{ else }}cat/TV{{ end }}{{ if and (.Keywords) (eq .Config.disablesort .False)) }}/{{ .Config.sort }}/{{ .Config.type }}{{ else }}{{ end }}/{{ if .Keywords }}2{{ else }}1{{ end }}/" + - path: "{{ if and (.Keywords) (eq .Config.disablesort .False) }}sort-{{ else }}{{ end }}{{ if .Keywords }}search/{{ .Keywords }}{{ else }}cat/Music{{ end }}{{ if and (.Keywords) (eq .Config.disablesort .False) }}/{{ .Config.sort }}/{{ .Config.type }}{{ else }}{{ end }}/{{ if .Keywords }}3{{ else }}1{{ end }}/" + - path: "{{ if and (.Keywords) (eq .Config.disablesort .False) }}sort-{{ else }}{{ end }}{{ if .Keywords }}search/{{ .Keywords }}{{ else }}cat/Other{{ end }}{{ if and (.Keywords) (eq .Config.disablesort .False) }}/{{ .Config.sort }}/{{ .Config.type }}{{ else }}{{ end }}/{{ if .Keywords }}4{{ else }}1{{ end }}/" + + keywordsfilters: + - name: re_replace # daily standard S2023 > 2023 + args: ["\\bS(20\\d{2})\\b", "$1"] + + rows: + selector: "tr:has(a[href^=\"/torrent/\"]){{ if .Config.uploader }}:has(td[class^=\"coll-5\"]:contains({{ .Config.uploader }})){{ else }}{{ end }}" + + fields: + title_default: + # the movies, tv and music pages abbreviate the title + selector: td[class^="coll-1"] a[href^="/torrent/"] + title_optional: + # the movies, tv and music pages abbreviate the title so we process the href instead. #8137 + optional: true + selector: td[class^="coll-1"] a[href^="/torrent/"]:contains("...") + attribute: href + filters: + - name: urldecode + - name: split + args: ["/", 3] + title: + # title_optional can be empty so use the title_default instead #8586 + text: "{{ if .Result.title_optional }}{{ .Result.title_optional }}{{ else }}{{ .Result.title_default }}{{ end }}" + filters: + - name: re_replace + args: ["-([\\w]+(?:[\\[\\]\\(\\)\\w]+)?)$", "~$1"] + - name: replace + args: ["-", " "] + - name: re_replace + args: ["~([\\w]+(?:[\\[\\]\\(\\)\\w]+)?)$", "-$1"] + - name: replace + args: ["\u000f", ""] # get rid of unwanted character #6582 + # cleanup for Sonarr + - name: re_replace # EP 3 4 to E3-4 + args: ["(?i)\\sEP\\s(\\d{1,2})\\s(E?\\s?\\d{1,2})\\s", " E$1-$2 "] + - name: re_replace # S02E04 05 to S02E04-05 + args: ["(?i)([-_. ])S(\\d{1,2})\\s?E\\s?(\\d{1,2})\\s(E?\\s?\\d{1,2})([-_. ])", "$1S$2E$3-$4$5"] + - name: re_replace + args: ["(?i)AC3\\s?(\\d)\\s(\\d)", "AC3 $1.$2"] + - name: re_replace + args: ["(?i) DD\\s?(\\d)\\s(\\d)", " DD $1.$2"] + - name: re_replace + args: ["(?i) DDP\\s?(\\d)\\s(\\d)", " DDP $1.$2"] + - name: re_replace + args: ["(?i)\\sE\\s?AC3", " EAC3"] + - name: re_replace + args: ["(?i)WEB\\sDL", "WEB-DL"] + - name: re_replace + args: ["(?i)HDTVRIP", "HDTV"] + category_optional: + optional: true + selector: td[class^="coll-1"] a[href^="/sub/"] + attribute: href + filters: + # extract the third part + - name: split + args: ["/", 2] + category: + text: "{{ if .Result.category_optional }}{{ .Result.category_optional }}{{ else }}40{{ end }}" + details: + selector: td[class^="coll-1"] a[href^="/torrent/"] + attribute: href + download: + # .torrent link is on the details page + selector: td[class^="coll-1"] a[href^="/torrent/"] + attribute: href + # dates come in three flavours: + date_year: + # (within this year) 7am Sep. 14th + optional: true + selector: td[class^="coll-date"]:not(:contains(":")):not(:contains("'")) + filters: + - name: re_replace + args: ["st|nd|rd|th", ""] + - name: dateparse + args: "htt MMM. d" + date_years: + # (more than a year ago) Apr. 18th '11 + optional: true + selector: td[class^="coll-date"]:contains("'") + filters: + - name: replace + args: ["'", ""] + - name: re_replace + args: ["st|nd|rd|th", ""] + - name: dateparse + args: "MMM. d yy" + date_today: + # (today) 12:25am + optional: true + selector: td[class^="coll-date"]:contains(":") + filters: + - name: fuzzytime + date: + text: "{{ if or .Result.date_year .Result.date_years .Result.date_today }}{{ or .Result.date_year .Result.date_years .Result.date_today }}{{ else }}now{{ end }}" + size: + selector: td[class^="coll-4"] + seeders: + selector: td[class^="coll-2"] + leechers: + selector: td[class^="coll-3"] + _username: + selector: td[class^="coll-5"] + description: + text: "Uploader: {{ .Result._username }}" + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/indexers/catalog/__init__.py b/backend/src/trove/indexers/catalog/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/trove/indexers/catalog/animetosho.yml b/backend/src/trove/indexers/catalog/animetosho.yml new file mode 100644 index 0000000..c0a5fec --- /dev/null +++ b/backend/src/trove/indexers/catalog/animetosho.yml @@ -0,0 +1,125 @@ +--- +id: tokyotosho +name: Tokyo Toshokan +description: "Tokyo Toshokan is a Public BitTorrent Library for JAPANESE Media" +language: en-US +type: public +encoding: UTF-8 +links: + - https://www.tokyotosho.info/ +legacylinks: + - https://tokyotosho.proxyportal.fun/ + - https://tokyotosho.uk-unblock.xyz/ + - https://tokyotosho.ind-unblock.xyz/ + - https://tokyotosho.unblocked.bar/ + - https://tokyotosho.proxyportal.pw/ + - https://tokyotosho.uk-unblock.pro/ + - https://tokyotosho.unblocked.rest/ + - https://tokyotosho.unblocked.monster/ + - https://tokyotosho.mrunblock.bond/ + - https://tokyotosho.nocensor.cloud/ + +settings: + - name: cat + type: select + label: Category + default: 0 + options: + 0: All + 1: Anime + 10: Non-English + 3: Manga + 8: Drama + 2: Music + 9: "Music Video" + 7: Raws + 4: Hentai + 12: "Hentai (Anime)" + 13: "Hentai (Manga)" + 14: "Hentai (Games)" + 11: Batch + 15: JAV + 5: Other + +caps: + categorymappings: + - {id: 1, cat: TV/Anime, desc: "Anime"} + - {id: 10, cat: TV/Anime, desc: "Non-English Anime"} + - {id: 3, cat: Books, desc: "Manga"} + - {id: 8, cat: TV/Anime, desc: "Drama"} + - {id: 2, cat: Audio, desc: "Anime Music"} + - {id: 9, cat: TV/Anime, desc: "Anime Music Videos"} + - {id: 7, cat: TV/Anime, desc: "Raw Anime"} + - {id: 4, cat: XXX, desc: "Hentai"} + - {id: 12, cat: XXX, desc: "Hentai (Anime)"} + - {id: 13, cat: XXX, desc: "Hentai (Manga)"} + - {id: 14, cat: XXX, desc: "Hentai (Games)"} + - {id: 11, cat: TV/Anime, desc: "Batch"} + - {id: 15, cat: XXX, desc: "JAV"} + - {id: 5, cat: Other, desc: "Other"} + + modes: + search: [q] + tv-search: [q, season, ep] + music-search: [q] + book-search: [q] + +search: + paths: + - path: "{{ if .Keywords }}search.php{{ else }}index.php{{ end }}" + inputs: + terms: "{{ .Keywords }}" + cat: "{{ .Config.cat }}" + + rows: + selector: "table.listing tr.category_0" + after: 1 + filters: + - name: andmatch + + fields: + category: + selector: a[href*="?cat="] + attribute: href + filters: + - name: regexp + args: (\d+) + title: + selector: td.desc-top a[type="application/x-bittorrent"] + details: + selector: a[href^="details.php?id="] + attribute: href + download: + selector: td.desc-top a[type="application/x-bittorrent"] + attribute: href + magnet: + selector: a[href^="magnet:?xt="] + attribute: href + optional: true + size: + selector: td.desc-bot + filters: + - name: split + args: ["|", 1] + - name: regexp + args: "Size: (.+?) ?$" + date: + selector: td.desc-bot + filters: + - name: split + args: ["|", 2] + - name: regexp + args: "Date: (.+?) ?$" + - name: replace + args: ["UTC", "+00:00"] + - name: dateparse + args: "yyyy-MM-dd HH:mm zzz" + seeders: + selector: td.stats > span:nth-child(1) + leechers: + selector: td.stats > span:nth-child(2) + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# Engine n/a diff --git a/backend/src/trove/indexers/catalog/bitsearch.yml b/backend/src/trove/indexers/catalog/bitsearch.yml new file mode 100644 index 0000000..2921de0 --- /dev/null +++ b/backend/src/trove/indexers/catalog/bitsearch.yml @@ -0,0 +1,142 @@ +--- +id: torrentdownload +name: TorrentDownload +description: "TorrentDownload is a Public torrent meta-search engine" +language: en-US +type: public +encoding: UTF-8 +links: + - https://www.torrentdownload.info/ +legacylinks: + - https://torrentdownload.mrunblock.bond/ + - https://torrentdownload.nocensor.cloud/ + - https://torrentdownload.unblockit.download/ + +caps: + categorymappings: + - {id: Adult, cat: XXX, desc: Adult} + - {id: AdultPornHDVideo, cat: XXX, desc: "Adult Porn HD Video"} + - {id: AdultPornPictures, cat: XXX, desc: "Adult Porn Pictures"} + - {id: AdultPornVideo, cat: XXX, desc: "Adult Porn Video"} + - {id: Anime, cat: TV/Anime, desc: Anime} + - {id: AnimeEnglishtranslated, cat: TV/Anime, desc: "Anime English translated"} + - {id: AnimeAnimeOther, cat: TV/Anime, desc: "Anime Other"} + - {id: Applications, cat: PC/0day, desc: Applications} + - {id: ApplicationsAndroid, cat: PC/Mobile-Android, desc: "Applications Android"} + - {id: ApplicationsWindows, cat: PC/0day, desc: "Applications Windows"} + - {id: AudioBooks, cat: Audio/Audiobook, desc: "Books Audiobooks"} + - {id: AudioAudiobooks, cat: Audio/Audiobook, desc: "Books Audiobooks"} + - {id: AudioLossless, cat: Audio/Lossless, desc: "Audio Lossless"} + - {id: AudioMusic, cat: Audio/MP3, desc: "Audio Music"} + - {id: BooksAcademic, cat: Books, desc: "Books Academic"} + - {id: BooksComics, cat: Books/Comics, desc: "Books Comics"} + - {id: BooksEbooks, cat: Books/EBook, desc: "Books Ebooks"} + - {id: BooksEducational, cat: Books, desc: "Books Educational"} + - {id: BooksMagazines, cat: Books/Mags, desc: "Books Magazines"} + - {id: BooksFiction, cat: Books, desc: "Books Fiction"} + - {id: BooksNonfiction, cat: Books, desc: "Books Nonfiction"} + - {id: BooksTextbooks, cat: Books, desc: "Books Textbooks"} + - {id: Ebooks, cat: Books/EBook, desc: "Books Ebooks"} + - {id: Games, cat: Console, desc: Games} + - {id: GamesWindows, cat: PC/Games, desc: "Games Windows"} + - {id: Movies, cat: Movies, desc: Movies} + - {id: MoviesAction, cat: Movies, desc: Movies Action} + - {id: MoviesConcerts, cat: Movies, desc: "Movies Concerts"} + - {id: MoviesCrime, cat: Movies, desc: "Movies Crime"} + - {id: MoviesDocumentary, cat: TV/Documentary, desc: "Movies Documentary"} + - {id: MoviesDubbedMovies, cat: Movies, desc: "Movies Dubbed Movies"} + - {id: MoviesHighresMovies, cat: Movies, desc: "Movies Highres Movies"} + - {id: MoviesMusicvideos, cat: Audio/Video, desc: "Movies Musicvideos"} + - {id: MoviesThriller, cat: Movies, desc: "Movies Thriller"} + - {id: Music, cat: Audio, desc: Music} + - {id: MusicHardrock, cat: Audio, desc: "Music Hardrock"} + - {id: MusicMp, cat: Audio/MP3, desc: "Music Mp3"} + - {id: MusicFLAC, cat: Audio/Lossless, desc: "Music FLAC"} + - {id: MusicLossless, cat: Audio/Lossless, desc: "Music Lossless"} + - {id: MusicRB, cat: Audio, desc: "Music R&B"} + - {id: MusicTranceHouseDance, cat: Audio, desc: "Music Trance House Dance"} + - {id: Other, cat: Other, desc: Other} + - {id: OtherEbooks, cat: Books/EBook, desc: "Other Ebooks"} + - {id: OtherComics, cat: Other, desc: "Other Comics"} + - {id: OtherTutorials, cat: Other, desc: "Other Tutorials"} + - {id: OtherUnsorted, cat: Other, desc: "Other Unsorted"} + - {id: PicturesPicturesOther, cat: Other/Misc, desc: "Pictures Other"} + - {id: PicturesWallpapers, cat: Other/Misc, desc: "Pictures Wallpapers"} + - {id: Software, cat: PC/0day, desc: "Software"} + - {id: TV, cat: TV, desc: TV} + - {id: TVBBC, cat: TV, desc: "TV BBC"} + - {id: TVEllenDeGeneres, cat: TV, desc: "TV Ellen DeGeneres"} + - {id: TVOther, cat: TV/Other, desc: "TV Other"} + - {id: TVshows, cat: TV, desc: "TV shows"} + - {id: Television, cat: TV, desc: Television} + - {id: VideoMobile, cat: Movies, desc: "Movies Video Mobile"} + - {id: VideoMovies, cat: Movies, desc: "Video Movies"} + - {id: VideoMusic, cat: Audio/Video, desc: "Video Music"} + - {id: XXX, cat: XXX, desc: XXX} + - {id: XXXVideo, cat: XXX, desc: "XXX Video"} + - {id: XXXHDVideo, cat: XXX, desc: "XXX HD Video"} + - {id: XXXPictures, cat: XXX, desc: "XXX Pictures"} + + modes: + search: [q] + tv-search: [q, season, ep] + movie-search: [q] + music-search: [q] + book-search: [q] + +settings: + - name: sort + type: select + label: Sort requested from site (Applies only to Search with Keywords) + default: d + options: + d: created + _: seeders + +download: + selectors: + - selector: a[href^="magnet:?xt="] + attribute: href + +search: + paths: + - path: "{{ if .Keywords }}search{{ re_replace .Config.sort \"_\" \"\" }}?q={{ .Keywords }}{{ else }}/{{ end }}" + + rows: + selector: table.table2 > tbody > tr:has(span.smallish) + + fields: + category: + selector: div.tt-name > span.smallish + filters: + - name: re_replace + args: ["[^A-Za-z]+", ""] # strip everything but letters + title: + selector: div.tt-name > a[href^="/"] + details: + selector: div.tt-name > a[href^="/"] + attribute: href + download: + selector: div.tt-name > a[href^="/"] + attribute: href + date: + selector: td:nth-child(2) + filters: + - name: replace + args: ["right ", ""] + - name: replace + args: ["Last Month", "1 month ago"] + - name: replace + args: ["+", " ago"] + - name: timeago + size: + selector: td:nth-child(3) + seeders: + selector: td.tdseed + leechers: + selector: td.tdleech + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/indexers/catalog/eztv.yml b/backend/src/trove/indexers/catalog/eztv.yml new file mode 100644 index 0000000..620b98f --- /dev/null +++ b/backend/src/trove/indexers/catalog/eztv.yml @@ -0,0 +1,99 @@ +--- +id: eztv +name: EZTV +description: "EZTV is a Public torrent site for TV shows" +language: en-US +type: public +encoding: UTF-8 +links: + - https://eztvx.to/ + - https://eztv.wf/ + - https://eztv.tf/ + - https://eztv.yt/ + - https://eztv1.xyz/ +legacylinks: + - https://eztv.ag/ # redirects to .re + - https://eztv.it/ # redirects to .re + - https://eztv.ch/ # redirects to .re + - https://eztv.io/ + - https://eztv.re/ + - https://eztv.li/ + - https://eztv.mrunblock.bond/ + - https://eztv.nocensor.cloud/ + - https://eztv.unblockninja.com/ # layout=def_wlinks not working + - https://eztv.ninjaproxy1.com/ # layout=def_wlinks not working + - https://eztv.proxyninja.org/ # layout=def_wlinks not working + - https://eztv.abcproxy.org/ + - https://eztv.unblockit.download/ + +caps: + categories: + 1: TV + + modes: + search: [q] + tv-search: [q, season, ep] + +settings: [] + +search: + paths: + - path: "{{ if .Keywords }}search/{{ .Keywords }}{{ else }}home{{ end }}" + + keywordsfilters: + - name: re_replace + args: ["\\bS\\d{2,3}\\b", ""] # remove season tag without episode (search doesn't support it) + - name: trim + # fixes for site search issues - Prowlarr #1094 + - name: replace + args: ["-", ""] + - name: replace + args: [" ", "-"] + - name: replace + args: ["&", ""] + + headers: + cookie: ["sort_no=100; q_filter=all; q_filter_web=on; q_filter_reality=on; q_filter_x265=on; layout=def_wlinks"] # show 100 results for keywordless search and show links in results + + rows: + # only use latest added torrents table for keywordless search to avoid duplicates, some torrents don't have any download links so skip them + selector: "table.forum_header_border:contains('Latest') tr[name='hover'].forum_header_border:has(a.magnet), table.forum_header_border:contains('Releases') tr[name='hover'].forum_header_border:has(a.magnet)" + filters: + - name: andmatch + + fields: + category: + text: 1 + title: + selector: td:nth-child(2) a + attribute: title + filters: + - name: replace + args: ["[eztv]", ""] + - name: re_replace + args: ["\\(.*\\)$", ""] + - name: trim + details: + selector: td:nth-child(2) a + attribute: href + download: + selector: td:nth-child(3) a.magnet, td:nth-child(3) a + attribute: href + size: + selector: td:nth-child(4) + optional: true + default: 512 MB + date: + selector: td:nth-child(5) + filters: + - name: append + args: " ago" + seeders: + selector: td:nth-child(6) + leechers: + text: 0 + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/indexers/catalog/limetorrents.yml b/backend/src/trove/indexers/catalog/limetorrents.yml new file mode 100644 index 0000000..89238c7 --- /dev/null +++ b/backend/src/trove/indexers/catalog/limetorrents.yml @@ -0,0 +1,138 @@ +--- +id: limetorrents +name: LimeTorrents +description: "LimeTorrents is a Public general torrent index with mostly verified torrents" +language: en-US +type: public +encoding: UTF-8 +# changes to this indexer should also be made to limetorrentsclone +links: + - https://www.limetorrents.fun/ + - https://limetorrents.unblockninja.com/ + - https://limetorrents.ninjaproxy1.com/ + - https://limetorrents.proxyninja.org/ + - https://limetorrents.proxyninja.net/ + - https://limetorrents.torrentbay.st/ + - https://limetorrents.torrentsbay.org/ +legacylinks: + - https://limetorrents.mrunblock.bond/ + - https://limetorrents.nocensor.cloud/ + - https://limetorrents.abcproxy.org/ + - https://limetorrents.unblockit.download/ + - https://www.limetorrents.lol/ + +caps: + categorymappings: + - {id: "TV shows", cat: TV, desc: "TV shows"} + - {id: Movies, cat: Movies, desc: Movies} + - {id: Music, cat: Audio, desc: Music} + - {id: Games, cat: Console, desc: Games} + - {id: Applications, cat: PC/0day, desc: Applications} + - {id: Other, cat: Other, desc: Other} + - {id: Anime, cat: TV/Anime, desc: Anime} + - {id: E-books, cat: Books/EBook, desc: E-books} + + modes: + search: [q] + tv-search: [q, season, ep] + movie-search: [q] + music-search: [q] + book-search: [q] + +settings: + - name: downloadlink + type: select + label: Download link + default: "magnet:" + options: + "http://itorrents.org/": iTorrents.org + "magnet:": magnet + - name: downloadlink2 + type: select + label: Download link (fallback) + default: "http://itorrents.org/" + options: + "http://itorrents.org/": iTorrents.org + "magnet:": magnet + - name: info_download + type: info + label: About the Download links + default: As the .torrent download links on this site are known to fail from time to time, you can optionally set as a fallback an automatic alternate link. + - name: sort + type: select + label: Sort requested from site + default: date + options: + date: created + seeds: seeders + size: size + - name: info_category_8000 + type: info + label: About LimeTorrents Categories + default: LimeTorrents only returns category Other in its Keywordless search results page.
To pass your apps' indexer TEST you will need to include the 8000(Other) category. + +download: + # the .torrent url is on the on the details page + selectors: + - selector: a.csprite_dltorrent[href^="{{ .Config.downloadlink }}"] + attribute: href + - selector: a.csprite_dltorrent[href^="{{ .Config.downloadlink2 }}"] + attribute: href + +search: + paths: + - path: "{{ if .Keywords }}search/all/{{ .Keywords }}/{{ .Config.sort }}/1/{{ else }}/latest100{{ end }}" + keywordsfilters: + - name: re_replace + args: ["S[0-9]{2}([^E]|$)", ""] # remove season tag without episode + + rows: + selector: .table2 > tbody > tr[bgcolor] + + fields: + title: + selector: div.tt-name > a[href^="/"] + attribute: href + filters: + - name: regexp + args: "/(.+?)-torrent-\\d+\\.html" + - name: re_replace + args: ["-", " "] + category_is_tv_show: + text: "{{ .Result.title }}" + filters: + - name: regexp + args: "\\b(S\\d+(?:E\\d+)?)\\b" + category: + selector: td:nth-child(2) + optional: true + default: "{{ if .Result.category_is_tv_show }}TV shows{{ else }}Other{{ end }}" + filters: + - name: regexp + args: " in (.+?)[.]?$" + details: + selector: div.tt-name > a[href^="/"] + attribute: href + download: + selector: div.tt-name > a[href^="/"] + attribute: href + date: + selector: td:nth-child(2) + filters: + - name: split + args: ["-", 0] + - name: replace + args: ["Last Month", "1 month ago"] + - name: replace + args: ["+", " ago"] + size: + selector: td:nth-child(3) + seeders: + selector: .tdseed + leechers: + selector: .tdleech + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/indexers/catalog/magnetdl.yml b/backend/src/trove/indexers/catalog/magnetdl.yml new file mode 100644 index 0000000..d7eef68 --- /dev/null +++ b/backend/src/trove/indexers/catalog/magnetdl.yml @@ -0,0 +1,144 @@ +--- +id: extratorrent-st +name: ExtraTorrent.st +description: "ExtraTorrent.st is a Public tracker for MOVIE / TV / GENERAL magnets" +language: en-US +type: public +encoding: UTF-8 +links: + - https://extratorrent.st/ + - https://extratorrent.ninjaproxy1.com/ + - https://extratorrent.proxyninja.org/ + - https://extratorrent.proxyninja.net/ +legacylinks: + - https://extratorrent.mrunblock.bond/ + - https://extratorrent.nocensor.cloud/ + - https://extratorrent.unblockit.download/ # 502 + +caps: + categorymappings: + - {id: 3D Movies, cat: Movies/3D, desc: Movies 3D} + - {id: AAC, cat: Audio, desc: Music AAC} + - {id: "Adult / Porn", cat: XXX, desc: "Adult / Porn"} + - {id: Android, cat: PC/Mobile-Android, desc: Software Android} + - {id: Anime, cat: TV/Anime, desc: Anime} + - {id: Audio books, cat: Audio/Audiobook, desc: Books Audiobook} + - {id: Bollywood, cat: Movies, desc: Bollywood} + - {id: Comics, cat: Books/Comics, desc: Books Comics} + - {id: DVD, cat: Movies/DVD, desc: Movies DVD} + - {id: Documentary, cat: TV/Documentary, desc: Documentary} + - {id: Dubbed Movies, cat: Movies/Foreign, desc: Movies Dubbed} + - {id: Ebooks, cat: Books/EBook, desc: Books Ebook} + - {id: English-translated, cat: TV/Anime, desc: Anime English-translated} + - {id: Episodes HD, cat: TV/HD, desc: Episodes HD} + - {id: Episodes 4K UHD, cat: TV/UHD, desc: Episodes 4K UHD} + - {id: Games, cat: XXX/Other, desc: Adult Games} + - {id: Hentai, cat: XXX/Other, desc: Adult Hentai} + - {id: Highres Movies, cat: Movies/HD, desc: Movies HD} + - {id: Linux, cat: PC, desc: Software Linux} + - {id: "Live Action [Non-English]", cat: TV/Anime, desc: "Live Action [Non-English]"} + - {id: Lossless, cat: Audio/Lossless, desc: Music Lossless} + - {id: "Manga [English-translated]", cat: Books/Comics, desc: "Manga [English-translated]"} + - {id: "Manga [Raw]", cat: Books/Comics, desc: "Manga [Raw]"} + - {id: MP3, cat: Audio/MP3, desc: Music MP3} + - {id: MP4, cat: Movies, desc: Movies MP4} + - {id: Mac, cat: PC/Mac, desc: Software Mac} + - {id: Magazines, cat: XXX/Other, desc: Adult Magazines} + - {id: Movie clips, cat: Movies, desc: Movies clips} + - {id: Movies, cat: Movies, desc: Movies} + - {id: Music, cat: Audio, desc: Music} + - {id: Music videos, cat: Audio/Video, desc: Music Videos} + - {id: NDS, cat: Console/NDS, desc: Games NDS} + - {id: Other Applications, cat: PC, desc: Other Applications} + - {id: Other Games, cat: Console/Other, desc: Games Other} + - {id: Other Movies, cat: Movies/Other, desc: Movies Other} + - {id: Other Music, cat: Audio/Other, desc: Music Other} + - {id: Other, cat: Other/Misc, desc: Other} + - {id: PC Games, cat: PC/Games, desc: PC Games} + - {id: PS3, cat: Console/PS3, desc: Games PS3} + - {id: PS4, cat: Console/PS4, desc: Games PS4} + - {id: PSP, cat: Console/PSP, desc: Games PSP} + - {id: Pictures, cat: XXX/ImageSet, desc: Adult Pictures} + - {id: Radio Shows, cat: Audio/Other, desc: Music Radio} + - {id: Raw, cat: TV/Anime, desc: Anime Raw} + - {id: Season Packs, cat: TV, desc: Season Packs} + - {id: Software, cat: PC/0day, desc: Software} + - {id: Sports, cat: TV/Sport, desc: Sports} + - {id: Subs, cat: TV/Anime, desc: Anime Subs} + - {id: Switch, cat: Console/Other, desc: Games Switch} + - {id: TV, cat: TV, desc: TV} + - {id: UltraHD, cat: Movies/UHD, desc: Movies UHD} + - {id: Video, cat: XXX, desc: Adult / Porn} + - {id: Wii, cat: Console/Wii, desc: Games Wii} + - {id: Windows, cat: PC, desc: Software Windows} + - {id: Xbox360, cat: Console/XBox 360, desc: Games Xbox360} + + modes: + search: [q] + tv-search: [q, season, ep] + movie-search: [q] + music-search: [q] + book-search: [q] + +settings: + - name: info_flaresolverr + type: info_flaresolverr + +search: + paths: + # https://extratorrent.st/search/?srt=added&order=desc&search=captain&new=1&x=0&y=0 + - path: "{{ if .Keywords }}search/?srt=added&order=desc&search={{ .Keywords }}&new=1&x=0&y=0{{ else }}{{ end }}" + keywordsfilters: + - name: re_replace + args: ["[\\s]+", "."] + + rows: + selector: tr[class^="tl"]:has(a[href^="magnet:?xt="]) + filters: + - name: andmatch + + fields: + category: + selector: span.c_tor + filters: + - name: replace + args: ["in ", ""] + - name: trim + title: + selector: a[href^="/torrent/"]:not([href$="comments"]) + details: + selector: a[href^="/torrent/"] + attribute: href + download: + selector: a[href^="magnet:?xt="] + attribute: href + date: + # 8m , 13h, 2d , 3w , 1m , 1y # site uses m for both minutes and months!?!? + selector: td:nth-last-of-type(5) + filters: + - name: replace + args: ["m", " minutes"] + - name: replace + args: ["h", " hours"] + - name: replace + args: ["y", " years"] + - name: replace + args: ["d", " days"] + - name: replace + args: ["w", " weeks"] + - name: timeago + size: + selector: td:nth-last-of-type(4) + seeders: + selector: td.sy, td.sn + optional: true + default: 0 + leechers: + selector: td.ly, td.ln + optional: true + default: 0 + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/indexers/catalog/nyaa.yml b/backend/src/trove/indexers/catalog/nyaa.yml new file mode 100644 index 0000000..2be5c6d --- /dev/null +++ b/backend/src/trove/indexers/catalog/nyaa.yml @@ -0,0 +1,311 @@ +--- +id: nyaasi +name: Nyaa.si +description: "Nyaa is a Public torrent site focused on Eastern ASIAN media including anime, manga, literature and music" +language: en-US +type: public +encoding: UTF-8 +requestDelay: 2 +links: + - https://nyaa.si/ + - https://nyaa.iss.ink/ + - https://nyaa.land/ + - https://nyaa.mom/ + - https://nyaa.unblockninja.com/ # for magnets only +legacylinks: + - https://nyaa.black-mirror.xyz/ + - https://nyaa.unblocked.casa/ + - https://nyaa.proxyportal.fun/ + - https://nyaa.uk-unblock.xyz/ + - https://nyaa.ind-unblock.xyz/ + - https://nyaa.unblocked.bar/ + - https://nyaa.proxyportal.pw/ + - https://nyaa.uk-unblock.pro/ + - https://nyaa.root.yt/ + - https://nyaa.lol/ # dropped at request of owner + - https://nyaa.mrunblock.bond/ # for magnets only + - https://nyaa.nocensor.cloud/ + +caps: + categorymappings: + - {id: 1_0, cat: TV/Anime, desc: "Anime"} + - {id: 1_1, cat: TV/Anime, desc: "Anime - Anime Music Video"} + - {id: 1_2, cat: TV/Anime, desc: "Anime - English-translated"} + - {id: 1_3, cat: TV/Anime, desc: "Anime - Non-English-translated"} + - {id: 1_4, cat: TV/Anime, desc: "Anime - Raw"} + # Anime as Movies (Radarr uses t=movie): + - {id: 1_0, cat: Movies/Other, desc: "Anime"} + - {id: 1_1, cat: Movies/Other, desc: "Anime - Anime Music Video"} + - {id: 1_2, cat: Movies/Other, desc: "Anime - English-translated"} + - {id: 1_3, cat: Movies/Other, desc: "Anime - Non-English-translated"} + - {id: 1_4, cat: Movies/Other, desc: "Anime - Raw"} + - {id: 2_0, cat: Audio, desc: "Audio"} + - {id: 2_1, cat: Audio, desc: "Audio - Lossless"} + - {id: 2_2, cat: Audio, desc: "Audio - Lossy"} + - {id: 3_0, cat: Books, desc: "Literature"} + - {id: 3_1, cat: Books, desc: "Literature English-translated"} + - {id: 3_2, cat: Books, desc: "Literature - Non-English-translated"} + - {id: 3_3, cat: Books, desc: "Literature - Raw"} + - {id: 4_0, cat: TV, desc: "Live Action"} + - {id: 4_1, cat: TV, desc: "Live Action - English-translated"} + - {id: 4_2, cat: TV, desc: "Live Action - Idol/Promotional Video"} + - {id: 4_3, cat: TV, desc: "Live Action - Non-English-translated"} + - {id: 4_4, cat: TV, desc: "Live Action - Raw"} + - {id: 5_0, cat: Other, desc: "Pictures"} + - {id: 5_1, cat: Other, desc: "Pictures - Graphics"} + - {id: 5_2, cat: Other, desc: "Pictures - Photos"} + - {id: 6_0, cat: PC, desc: "Software"} + - {id: 6_1, cat: PC/ISO, desc: "Software - Applications"} + - {id: 6_2, cat: PC/Games, desc: "Software - Games"} + + modes: + search: [q] + tv-search: [q, season, ep] + movie-search: [q] + music-search: [q] + book-search: [q] + allowrawsearch: true + +settings: + - name: prefer_magnet_links + type: checkbox + label: Prefer Magnet Links + default: true + - name: sonarr_compatibility + type: checkbox + label: Improve Sonarr compatibility by trying to add Season information into Release Titles + default: false + - name: strip_s01 + type: checkbox + label: Remove first season keywords (S1/S01/Season 1), as some results do not include this for first/single season releases + default: false + - name: radarr_compatibility + type: checkbox + label: Improve Radarr compatibility by removing year information from keywords and adding it to Release Titles + default: false + - name: filter-id + type: select + label: Filter + default: 0 + options: + 0: No filter + 1: No remakes + 2: Trusted only + - name: cat-id + type: select + label: Category + default: 0_0 + options: + 0_0: "All categories" + 1_0: "Anime" + 1_1: "Anime - Anime Music Video" + 1_2: "Anime - English-translated" + 1_3: "Anime - Non-English-translated" + 1_4: "Anime - Raw" + 2_0: "Audio" + 2_1: "Audio - Lossless" + 2_2: "Audio - Lossy" + 3_0: "Literature" + 3_1: "Literature - English-translated" + 3_2: "Literature - Non-English-translated" + 3_3: "Literature - Lossy" + 4_0: "Live Action" + 4_1: "Live Action - English" + 4_2: "Live Action - Idol/PV" + 4_3: "Live Action - Non-English" + 4_4: "Live Action - Raw" + 5_0: "Pictures" + 5_1: "Pictures - Graphics" + 5_2: "Pictures - Photos" + 6_0: "Software" + 6_1: "Software - Applications" + 6_2: "Software - Games" + - name: sort + type: select + label: Sort requested from site + default: id + options: + id: created + seeders: seeders + size: size + - name: type + type: select + label: Order requested from site + default: desc + options: + desc: desc + asc: asc + +search: + paths: + - path: / + inputs: + q: "{{ .Keywords }}" + # strip 0 from start of episode number - #11019, or fetch page 2 for keywordless + - path: / + inputs: + q: "{{ if .Keywords }}{{ re_replace .Keywords \"\\b0(\\d{1})\\b\" \"$1\" }}{{ else }}{{ end }}" + p: "{{ if .Keywords }}{{ else }}2{{ end }}" + inputs: + # 0 all, 1 no remakes, 2 trusted only + f: "{{ .Config.filter-id }}" + c: "{{ .Config.cat-id }}" + s: "{{ .Config.sort }}" + o: "{{ .Config.type }}" + + keywordsfilters: + - name: re_replace + args: [" *\\b((?:19|20)\\d{2})\\b", "{{ if .Config.radarr_compatibility }}{{ else }} $1{{ end }}"] + - name: re_replace + args: ["(?i) *\\b(S(?:0|eason *)?1)\\b", "{{ if .Config.strip_s01 }}{{ else }} $1{{ end }}"] + + rows: + selector: tr.default,tr.danger,tr.success + + fields: + category: + selector: td:nth-child(1) a + attribute: href + filters: + - name: split + args: ["=", -1] + category_group_id: + selector: td:nth-child(1) a + attribute: href + filters: + - name: split + args: ["=", -1] + - name: split + args: ["_", 0] + title_default: + selector: td:nth-child(2) a:last-of-type + title_phase1: + selector: td:nth-child(2) a:last-of-type:contains("[PuyaSubs!] ") + optional: true + filters: + - name: append + args: " Spanish" + title_keyword_year: + text: "{{ .Query.Keywords }}" + filters: + - name: regexp + args: "\\b((19|20)\\d{2})\\b" + title_phase2: + text: "{{ or (.Result.title_phase1) (.Result.title_default) }}" + filters: + - name: re_replace + args: ["^(\\[.+?\\] ?)?(\\[.+?\\] ?)?(.+?)(\\[)", "$1$2$3{{ if and (.Config.radarr_compatibility) (.Result.title_keyword_year) }} {{ .Result.title_keyword_year }} $4{{ else }}$4{{ end }}"] + title_phase3: + text: "{{ .Result.title_phase2 }}" + filters: + - name: re_replace + args: ["(?i)\\b((?:S|Seasons?|EP?|Episodes?)\\s?)(\\d+)(?:\\-|[\\s~\\+àa&]+)(\\d+)\\b", "$1$2-$3"] + - name: re_replace + args: ["(?i)\\b(?:S|Seasons?)\\s?(\\d+(?:-\\d+)?)[\\s\\-]+(?:EP|Episodes?)\\s?(\\d+(?:-\\d+)?)\\b", "S$1E$2"] + - name: re_replace + args: ["(?i)\\b(?:S|Seasons?)\\s?(\\d+(?:-\\d+)?)(?:.+?)(?:EP|Episodes?)\\s?(\\d+(?:-\\d+)?)\\b(?!(?:-\\d+)?\\sS\\d+(?:-\\d+)?E\\d+(?:-\\d+)?)", "S$1E$2"] + - name: re_replace + args: ["\\b (II) - (\\d+)[\\s\\-~\\+àa&]+(\\d+)", " $1 S02 - $2-$3"] + - name: re_replace + args: ["\\b (II) - (\\d+)", " $1 S02 - $2"] + - name: re_replace + args: ["\\b (III) - (\\d+)[\\s\\-~\\+àa&]+(\\d+)", " $1 S03 - $2-$3"] + - name: re_replace + args: ["\\b (III) - (\\d+)", " $1 S03 - $2"] + - name: re_replace + args: ["(?i)\\b(\\d+)(st|nd|rd|th) Season\\b", "$0 S$1"] + - name: re_replace + args: ["(?i)_(\\d+)(st|nd|rd|th)_season_(\\d+)_", "$0S$1E$3_"] + - name: re_replace + args: ["(?i)_(\\d+)(st|nd|rd|th)_season_", "$0S$1_"] + - name: re_replace + args: ["(?i)\\b(S\\d+(?:-\\d+)?) - (\\d+)[\\s\\-~\\+àa&]+(\\d+)\\b", "$1E$2-$3"] + - name: re_replace + args: ["(?i)\\b(S\\d+(?:-\\d+)?) - (\\d+)\\b", "$1E$2"] + - name: re_replace + args: ["(?i)\\b(?:S\\s|Seasons?\\s?)(\\d+(?:-\\d+)?)\\b(?!(?:-\\d+)?\\s(?:EP|Episodes?)?\\s?(?:\\d+(?:-\\d+)?)?\\s?S\\d+(?:E\\d+(?:-\\d+)?)?)", "S$1"] + - name: re_replace + args: ["(?i)\\b(?:EP|Episodes?)\\s?(\\d+(?:-\\d+)?)\\b(?!(?:-\\d+)?\\sS\\d+(?:-\\d+)?(?:E\\d+(?:-\\d+)?)?)", "S01E$1"] + - name: re_replace + args: ["\\s+", " "] + - name: trim + title_has_season: + text: "{{ .Result.title_phase3 }}" + filters: + - name: regexp + args: "(?i)(S\\d{1,3}(E\\d+)?)" + - name: append + args: "NULL" + title_has_episode: + text: "{{ .Result.title_phase3 }}" + filters: + - name: regexp + args: "\\b(.+? - ?)(\\d+(-\\d+)?) ([\\[\\(])\\b" + - name: append + args: "NULL" + title_has_movie_ova: + text: "{{ .Result.title_phase3 }}" + filters: + - name: regexp + args: "(?i)(? Hotel.del.Luna.2019.S01.1080p... + - name: re_replace + args: ["^(.+?)([\\. ])((?:19|20)\\d{2})([\\. ])(.+)$", "{{ if and (eq .Result.category_group_id \"4\") (and (eq .Result.title_has_season \"NULL\") (eq .Result.title_has_episode \"NULL\")) }}$1$2$3$4S01$4$5{{ else }}$1$2$3$4$5{{ end }}"] + # Insert S01 before resolution for Live Action titles without year or season info + # e.g. Hotel Del Luna 720p HDTV... -> Hotel Del Luna S01 720p HDTV... + - name: re_replace + args: ["^(?!.*(?:19|20)\\d{2})(.+?)([\\. ])((?:480|720|1080|2160)[pi])(.*)$", "{{ if and (eq .Result.category_group_id \"4\") (and (eq .Result.title_has_season \"NULL\") (eq .Result.title_has_episode \"NULL\")) }}$1$2S01$2$3$4{{ else }}$1$2$3$4{{ end }}"] + title: + text: "{{ if .Config.sonarr_compatibility }}{{ .Result.title_live_action }}{{ else }}{{ .Result.title_phase2 }}{{ end }}" + details: + selector: td:nth-child(2) a:last-of-type + attribute: href + download_optional: + selector: td:nth-child(3) a[href$=".torrent"] + attribute: href + optional: true + download: + text: "{{ if .Config.prefer_magnet_links }}{{ else }}{{ .Result.download_optional }}{{ end }}" + optional: true + magnet: + selector: td:nth-child(3) a[href^="magnet:?"] + attribute: href + size: + selector: td:nth-child(4) + date: + selector: td:nth-child(5) + filters: + - name: append + args: " -00:00" # GMT + - name: dateparse + args: "yyyy-MM-dd HH:mm zzz" + seeders: + selector: td:nth-child(6):not(:empty) + optional: true + default: 0 + leechers: + selector: td:nth-child(7):not(:empty) + optional: true + default: 0 + grabs: + selector: td:nth-child(8):not(:empty) + optional: true + default: 0 + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 + description: + selector: td:nth-child(2) a:last-of-type +# engine n/a diff --git a/backend/src/trove/indexers/catalog/registry.yaml b/backend/src/trove/indexers/catalog/registry.yaml new file mode 100644 index 0000000..55a9aa0 --- /dev/null +++ b/backend/src/trove/indexers/catalog/registry.yaml @@ -0,0 +1,123 @@ +# Authoritative index of public no-account torrent sites shipped with Trove. +# Each entry references a vendored Cardigann YAML in the same directory. +# +# Filenames under `yaml_file:` are OUR vendored filenames — they need not +# match upstream. `upstream_path:` records where the definition came from +# so scripts/update-catalog.py can diff against Prowlarr-indexers. + +entries: + - slug: 1337x + display_name: 1337x + description: Large public general-purpose tracker with strong TV/movie scene coverage. + categories: [movies, tv, music, software, games, books, anime] + yaml_file: 1337x.yml + upstream_path: definitions/v11/1337x.yml + mirrors: + - https://1337x.to + - https://1337x.st + - https://1337x.tw + default_mirror: https://1337x.to + protocol: torrent + + - slug: torrentgalaxy + display_name: TorrentGalaxy + description: General-purpose public tracker with reliable scene releases and good category filtering. + categories: [movies, tv, music, software, games, books, anime, other] + yaml_file: torrentgalaxy.yml + upstream_path: definitions/v11/torrentgalaxyclone.yml + mirrors: + - https://torrentgalaxy.one + - https://torrentgalaxy.info + - https://torrentgalaxy.space + default_mirror: https://torrentgalaxy.one + protocol: torrent + + - slug: limetorrents + display_name: LimeTorrents + description: Long-running public aggregator with a wide catalogue. + categories: [movies, tv, music, software, games, anime, other] + yaml_file: limetorrents.yml + upstream_path: definitions/v11/limetorrents.yml + mirrors: + - https://www.limetorrents.lol + - https://www.limetorrents.info + default_mirror: https://www.limetorrents.lol + protocol: torrent + + - slug: magnetdl + display_name: ExtraTorrent + description: Public tracker for MOVIE / TV / GENERAL magnets (replaces magnetdl which has no upstream Cardigann definition). + categories: [movies, tv, music, software, games, books, anime] + yaml_file: magnetdl.yml + upstream_path: definitions/v11/extratorrent-st.yml + mirrors: + - https://extratorrent.st + default_mirror: https://extratorrent.st + protocol: torrent + + - slug: torlock + display_name: KickAssTorrents + description: Public KickAssTorrent clone for MOVIES / TV / GENERAL (replaces torlock which has no upstream Cardigann definition). + categories: [movies, tv, music, software, games, books, anime] + yaml_file: torlock.yml + upstream_path: definitions/v11/kickasstorrents-to.yml + mirrors: + - https://kickass.torrentbay.st + default_mirror: https://kickass.torrentbay.st + protocol: torrent + + - slug: bitsearch + display_name: TorrentDownload + description: Public torrent meta-search engine (replaces bitsearch which has no upstream Cardigann definition). + categories: [movies, tv, music, software, games, books, anime, other] + yaml_file: bitsearch.yml + upstream_path: definitions/v11/torrentdownload.yml + mirrors: + - https://www.torrentdownload.info + default_mirror: https://www.torrentdownload.info + protocol: torrent + + - slug: solidtorrents + display_name: TorrentProject2 + description: Public torrent meta-search engine (replaces solidtorrents which has no upstream Cardigann definition). + categories: [movies, tv, music, software, games, books, anime, other] + yaml_file: solidtorrents.yml + upstream_path: definitions/v11/torrentproject2.yml + mirrors: + - https://torrentproject2.net + default_mirror: https://torrentproject2.net + protocol: torrent + + - slug: nyaa + display_name: Nyaa + description: The largest public anime & manga tracker. + categories: [anime, music, books, other] + yaml_file: nyaa.yml + upstream_path: definitions/v11/nyaasi.yml + mirrors: + - https://nyaa.si + default_mirror: https://nyaa.si + protocol: torrent + + - slug: eztv + display_name: EZTV + description: TV-focused public tracker with strong scene release coverage. + categories: [tv] + yaml_file: eztv.yml + upstream_path: definitions/v11/eztv.yml + mirrors: + - https://eztv.re + - https://eztvx.to + default_mirror: https://eztv.re + protocol: torrent + + - slug: animetosho + display_name: Tokyo Toshokan + description: Public BitTorrent library for Japanese media including anime (replaces animetosho which has no upstream Cardigann definition). + categories: [anime] + yaml_file: animetosho.yml + upstream_path: definitions/v11/tokyotosho.yml + mirrors: + - https://www.tokyotosho.info + default_mirror: https://www.tokyotosho.info + protocol: torrent diff --git a/backend/src/trove/indexers/catalog/solidtorrents.yml b/backend/src/trove/indexers/catalog/solidtorrents.yml new file mode 100644 index 0000000..a27af79 --- /dev/null +++ b/backend/src/trove/indexers/catalog/solidtorrents.yml @@ -0,0 +1,117 @@ +--- +id: torrentproject2 +name: TorrentProject2 +description: "TorrentProject2 is a Public torrent meta-search engine" +language: en-US +type: public +encoding: UTF-8 +requestDelay: 2 +links: + - https://torrentproject2.net/ + - https://torrentproject2.org/ + - https://torrentproject.info/ + - https://torrentproject.biz/ + - https://torrentproject.xyz/ + - https://torrentproject.cc/ + - https://torrentproject.torrentbay.st/ +legacylinks: + - https://torrentproject2.se/ + - https://torrentproject2.com/ + +caps: + categorymappings: + - {id: Other, cat: Other, desc: Other} + + modes: + search: [q] + tv-search: [q, season, ep] + movie-search: [q] + music-search: [q] + book-search: [q] + +settings: + - name: filter-verified + type: checkbox + label: "Only include verifed content in results" + default: false + - name: sort + type: select + label: Sort requested from site + default: latest + options: + latest: "created desc" + oldest: "created asc" + seeders: seeders + size: size + - name: info_category_8000 + type: info_category_8000 + +download: + selectors: + - selector: "#download > div:nth-child(2) > div:nth-child(1) > a" + attribute: href + filters: + - name: replace + args: ["https://mylink.me.uk/?url=", ""] + - name: replace + args: ["https://mylink.cx/?url=", ""] + - name: replace + args: ["https://mylink.cloud/?url=", ""] + - name: urldecode + +search: + paths: + # browse for latest, / for keywords, 50 rows per page + - path: "{{ if .Keywords }}/{{ else }}browse{{ end }}" + - path: "{{ if .Keywords }}/{{ else }}browse{{ end }}" + inputs: + p: 1 + inputs: + t: "{{ .Keywords }}" + orderby: "{{ if .Keywords }}{{ .Config.sort }}{{ else }}{{ end }}" + safe: "{{ if and .Keywords .Config.filter-verified }}on{{ else }}{{ end }}" + + headers: + User-Agent: ["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.203"] + + rows: + selector: "#similarfiles div:has(a[href^=\"/t\"])" + + fields: + category: + # while browse has cats the search does not (atm) so we wont support cats for now. + text: Other + title: + selector: :scope > span > a + details: + selector: :scope > span > a + attribute: href + download: + selector: :scope > span > a + attribute: href + date_ago: + # 7 years ago + selector: :scope > span:nth-child(4):contains("ago") + optional: true + date_time: + # 2020-11-05 07:34:44 + selector: :scope > span:nth-child(4):contains(":") + optional: true + filters: + - name: append + args: " -07:00" # PDT + - name: dateparse + args: "yyyy-MM-dd HH:mm:ss zzz" + date: + text: "{{ if or .Result.date_ago .Result.date_time }}{{ or .Result.date_ago .Result.date_time }}{{ else }}now{{ end }}" + size: + selector: :scope > span:nth-child(5) + seeders: + selector: :scope > span:nth-child(2) + leechers: + selector: :scope > span:nth-child(3) + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/indexers/catalog/torlock.yml b/backend/src/trove/indexers/catalog/torlock.yml new file mode 100644 index 0000000..8bc9c85 --- /dev/null +++ b/backend/src/trove/indexers/catalog/torlock.yml @@ -0,0 +1,169 @@ +--- +id: kickasstorrents-to +name: kickasstorrents.to +description: "kickasstorrents.to is a Public KickAssTorrent clone for MOVIES / TV / GENERAL" +language: en-US +type: public +encoding: UTF-8 +requestDelay: 2 +links: + - https://kickass.torrentbay.st/ + - https://kickass.torrentsbay.org/ + - https://kickasstorrents.unblockninja.com/ + - https://kickasstorrents.ninjaproxy1.com/ + - https://kickasstorrents.proxyninja.org/ + - https://kickasstorrents.proxyninja.net/ +legacylinks: + - https://kat.root.yt/ + - https://kickasstorrents.abcproxy.org/ + - https://kickasstorrents.to/ + - https://kickasstorrent.cr/ # blocks Jackett UA, 'new' page broken + - https://katcr.to/ # blocks Jackett UA + - https://www.kickasstorrents.do/ # blocks Jackett UA + +caps: + categorymappings: + # category naming is inconsistent with root cat being left out on some results, hence the duplicate entries with/without root + - {id: 3DMovies, cat: Movies/3D, desc: "Movies 3D Movies"} + - {id: Adult, cat: XXX/WEB-DL, desc: "Adult Video"} + - {id: AdultGames, cat: XXX/Other, desc: "Adult Games"} + - {id: AdultHentai, cat: XXX/Other, desc: "Adult Hentai"} + - {id: AdultMagazines, cat: XXX/Other, desc: "Adult Magazines"} + - {id: AdultPictures, cat: XXX/ImageSet, desc: "Adult Pictures"} + - {id: AdultVideo, cat: XXX/WEB-DL, desc: "Adult Video"} + - {id: Android, cat: PC/Mobile-Android, desc: "Apps Android"} + - {id: Anime, cat: TV/Anime, desc: Anime} + - {id: AnimeAudioLossless, cat: Audio/Lossless, desc: "Anime Audio Lossless"} + - {id: "AnimeAudio[Lossless]", cat: Audio/Lossless, desc: "Anime Audio Lossless"} + - {id: "AnimeAudio[Lossy]", cat: Audio/MP3, desc: "Anime Audio Lossy"} + - {id: AnimeEnglish-translated, cat: TV/Anime, desc: "Anime English-translated"} + - {id: "AnimeLiveAction[English-translated]", cat: TV/Anime, desc: "Anime Live Action English-translated"} + - {id: "AnimeLiveAction[Non-English]", cat: TV/Anime, desc: "Anime Live Action Non-English"} + - {id: "AnimeLiveAction[Raw]", cat: TV/Anime, desc: "Anime Live Action Raw"} + - {id: "AnimeManga[English-translated]", cat: TV/Anime, desc: "Anime Manga English-translated"} + - {id: "AnimeManga[Raw]", cat: TV/Anime, desc: "Anime Manga Raw"} + - {id: AnimeMusicvideo, cat: Audio/Video, desc: Anime Music video} + - {id: AnimePictures, cat: Other, desc: Anime Pictures} + - {id: AnimeRaw, cat: TV/Anime, desc: Anime Raw} + - {id: AnimeSubs, cat: TV/Anime, desc: Anime Subs} + - {id: Apps, cat: PC, desc: Apps} + - {id: AppsAndroid, cat: PC/Mobile-Android, desc: "Apps Android"} + - {id: AppsLinux, cat: PC, desc: "Apps Linux"} + - {id: AppsMac, cat: PC/Mac, desc: "Apps Mac"} + - {id: AppsWindows, cat: PC/0day, desc: "Apps Windows"} + - {id: Audiobooks, cat: Audio/Audiobook, desc: "Audio books"} + - {id: Bollywood, cat: Movies, desc: "Movies Bollywood"} + - {id: Books, cat: Books, desc: Books} + - {id: BooksAudiobooks, cat: Audio/Audiobook, desc: "Audio books"} + - {id: BooksComics, cat: Books/Comics, desc: Comics} + - {id: BooksEbooks, cat: Books/EBook, desc: Ebooks} + - {id: "BooksManga[English-translated]", cat: Books/Comics, desc: "Books Manga English-translated"} + - {id: "BooksManga[Raw]", cat: Books/Comics, desc: "Books Manga Raw"} + - {id: Comics, cat: Books/Comics, desc: Comics} + - {id: DVD, cat: Movies/DVD, desc: "Movies DVD"} + - {id: Documentary, cat: Movies/Other, desc: "Movies Documentary"} + - {id: DubbedMovies, cat: Movies, desc: "Movies Dubbed"} + - {id: EBooks, cat: Books/EBook, desc: Ebooks} + - {id: English-translated, cat: TV/Anime, desc: "Anime English-translated"} + - {id: Games, cat: Console, desc: "Games"} + - {id: GamesNDS, cat: Console/NDS, desc: NDS} + - {id: GamesOtherGames, cat: PC/Games, desc: "Games Other"} + - {id: GamesPCGames, cat: PC/Games, desc: "Games PC"} + - {id: GamesPS3, cat: Console/PS3, desc: PS3} + - {id: GamesPS4, cat: Console/PS4, desc: PS4} + - {id: GamesPSP, cat: Console/PSP, desc: PSP} + - {id: GamesWii, cat: Console/Wii, desc: Wii} + - {id: GamesXbox360, cat: Console/XBox 360, desc: Xbox360} + - {id: HighresMovies, cat: Movies/HD, desc: "Movies Highres"} + - {id: Linux, cat: PC, desc: "Apps Linux"} + - {id: Lossless, cat: Audio/Lossless, desc: "Music Lossless"} + - {id: MP3, cat: Audio/MP3, desc: "Music MP3"} + - {id: MP4, cat: Movies/HD, desc: "Movies MP4"} + - {id: Mac, cat: PC/Mac, desc: "Apps Mac"} + - {id: Movieclips, cat: Other, desc: "Movie clips"} + - {id: Movies, cat: Movies, desc: Movies} + - {id: Movies3DMovies, cat: Movies/3D, desc: "Movies 3D Movies"} + - {id: MoviesBollywood, cat: Movies, desc: "Movies Bollywood"} + - {id: MoviesDVD, cat: Movies/DVD, desc: "Movies DVD"} + - {id: MoviesDocumentary, cat: Movies/Other, desc: "Movies Documentary"} + - {id: MoviesDubbedMovies, cat: Movies, desc: "Movies Dubbed"} + - {id: MoviesHighresMovies, cat: Movies/HD, desc: "Movies Highres"} + - {id: MoviesMP4, cat: Movies/HD, desc: "Movies MP4"} + - {id: MoviesMusicvideos, cat: Audio/Video, desc: "Movies Music videos"} + - {id: MoviesMovieclips, cat: Other, desc: "Movies Movie clips"} + - {id: MoviesOtherMovies, cat: Movies/Other, desc: "Movies Other"} + - {id: MoviesUltraHD, cat: Movies/UHD, desc: "Movies UltraHD"} + - {id: Music, cat: Audio, desc: Music} + - {id: MusicLossless, cat: Audio/Lossless, desc: "Music Lossless"} + - {id: MusicMP3, cat: Audio/MP3, desc: "Music MP3"} + - {id: MusicMusicvideos, cat: Audio/Video, desc: "Music videos"} + - {id: MusicOthermusic, cat: Audio/Other, desc: "Music Other"} + - {id: Musicvideos, cat: Audio/Video, desc: "Music videos"} + - {id: NDS, cat: Console/NDS, desc: NDS} + - {id: Other, cat: Other, desc: Other} + - {id: OtherGames, cat: PC/Games, desc: "Games Other"} + - {id: OtherMovies, cat: Movies/Other, desc: "Movies Other"} + - {id: Othermusic, cat: Audio/Other, desc: "Music Other"} + - {id: PCGames, cat: PC/Games, desc: "Games PC"} + - {id: PS3, cat: Console/PS3, desc: PS3} + - {id: PS4, cat: Console/PS4, desc: PS4} + - {id: PSP, cat: Console/PSP, desc: PSP} + - {id: TV, cat: TV, desc: TV} + - {id: UltraHD, cat: Movies/UHD, desc: "Movies UltraHD"} + - {id: Video, cat: XXX/WEB-DL, desc: "Adult Video"} + - {id: Wii, cat: Console/Wii, desc: Wii} + - {id: Windows, cat: PC/0day, desc: "Apps Windows"} + - {id: XXXGames, cat: XXX/Other, desc: "Adult Games"} + - {id: XXXPictures, cat: XXX/ImageSet, desc: "Adult Pictures"} + - {id: XXXVideo, cat: XXX/WEB-DL, desc: "Adult Video"} + - {id: Xbox360, cat: Console/XBox 360, desc: Xbox360} + + modes: + search: [q] + tv-search: [q, season, ep] + movie-search: [q] + music-search: [q] + book-search: [q] + +settings: + - name: info_flaresolverr + type: info_flaresolverr + +search: + paths: + # 50 rows per page, however the All page only returns 49 actual results as the 50th is a duplicate of the 49th 8-/ + - path: "{{ if .Keywords }}search/?q={{ .Keywords }}{{ else }}17/All/{{ end }}" + - path: "{{ if .Keywords }}search/?page=2&q={{ .Keywords }}{{ else }}17/All/?page=2{{ end }}" + + rows: + selector: table.data > tbody > tr:has(a[href^="magnet:?xt="]) + + fields: + category: + selector: span > strong + filters: + - name: re_replace + args: ["[>| ]+", ""] + title: + selector: a.cellMainLink + details: + selector: a.cellMainLink + attribute: href + download: + selector: a[href^="magnet:?xt="] + attribute: href + size: + selector: td:nth-child(2) + date: + selector: td.timeago + filters: + - name: timeago + seeders: + selector: td:nth-child(5) + leechers: + selector: td:nth-child(6) + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/indexers/catalog/torrentgalaxy.yml b/backend/src/trove/indexers/catalog/torrentgalaxy.yml new file mode 100644 index 0000000..3d06050 --- /dev/null +++ b/backend/src/trove/indexers/catalog/torrentgalaxy.yml @@ -0,0 +1,96 @@ +--- +id: torrentgalaxyclone +name: TorrentGalaxyClone +description: "TorrentGalaxyClone is a Public site for MOVIES / TV / GENERAL" +language: en-US +type: public +encoding: UTF-8 +# https://proxygalaxy.cc/ for health status and alternate domains +links: + - https://torrentgalaxy.one/ + - https://torrentgalaxy.info/ + - https://torrentgalaxy.space/ + +caps: + # dont forget to update the path categories in the search block + categorymappings: + - {id: Anime, cat: TV/Anime, desc: "Anime"} + - {id: Apps, cat: PC, desc: "Apps"} + - {id: Books, cat: Books, desc: "Books"} + - {id: Docus, cat: TV/Documentary, desc: "Documentaries"} + - {id: Games, cat: Console, desc: "Games"} + - {id: Movies, cat: Movies, desc: "Movies"} + - {id: Music, cat: Audio, desc: "Music"} + - {id: Other, cat: Other, desc: "Other"} + - {id: TV, cat: TV, desc: "TV"} + - {id: XXX, cat: XXX, desc: "XXX"} + # unlisted + - {id: Documentaries, cat: TV/Documentary, desc: "Docus"} + - {id: E-books, cat: Books, desc: "E-books"} + + modes: + search: [q] + tv-search: [q, season, ep, imdbid] + movie-search: [q, imdbid] + music-search: [q] + book-search: [q] + +settings: + - name: uploader + type: text + label: Filter by Uploader + - name: info_uploader + type: info + label: About filtering by Uploader + default: "You can filter by Uploader by entering a Case Sensitive username, or leave empty to get all results.
Note: this is the username of the Uploader and not the Groupname that often show up at the end of TGx titles, eg RMTeam." + +download: + selectors: + - selector: a[href^="magnet:?xt="] + attribute: href + +search: + # https://torrentgalaxy.one/get-posts/keywords:tt1890725/ + # https://torrentgalaxy.one/get-posts/keywords:andor/ + # https://torrentgalaxy.one/get-posts/category:Movies:category:TV:keywords:bodies + paths: + - path: "get-posts/{{ if or .Query.IMDBID .Keywords }}keywords:{{ or .Query.IMDBID .Keywords }}{{ else }}{{ end }}{{ range .Categories }}:category:{{.}}{{end}}" + + rows: + selector: "div.tgxtablerow{{ if .Config.uploader }}:has(a.username:contains({{ .Config.uploader }})){{ else }}{{ end }}" + + fields: + category: + selector: a[href^="/get-posts/category:"] + title: + selector: a[href^="/post-detail/"] + attribute: title + details: + selector: a[href^="/post-detail/"] + attribute: href + download: + selector: a[href^="/post-detail/"] + attribute: href + imdbid: + selector: a[href^="/get-posts/keywords:tt"] + attribute: href + size: + selector: div.tgxtablecell:nth-last-child(5) + seeders: + selector: div.tgxtablecell:nth-last-child(2) span font + leechers: + selector: div.tgxtablecell:nth-last-child(2) span font:nth-of-type(2) + date: + selector: div.tgxtablecell:nth-last-child(1) + remove: div.bighide + filters: + - name: timeago + _username: + selector: a.username + description: + text: "Uploader: {{ .Result._username }}" + downloadvolumefactor: + text: 0 + uploadvolumefactor: + text: 1 +# engine n/a diff --git a/backend/src/trove/main.py b/backend/src/trove/main.py index fe49ac3..1317487 100644 --- a/backend/src/trove/main.py +++ b/backend/src/trove/main.py @@ -34,6 +34,7 @@ from trove.api import tasks as tasks_router from trove.api import torznab as torznab_router from trove.api import watchlist as watchlist_router +from trove.api.catalog import router as catalog_router from trove.config import get_settings from trove.db import init_db from trove.logging_setup import configure_logging @@ -79,6 +80,7 @@ def create_app() -> FastAPI: app.include_router(auth_router.router, prefix="/api/auth", tags=["auth"]) app.include_router(clients_router.router, prefix="/api/clients", tags=["clients"]) app.include_router(indexers_router.router, prefix="/api/indexers", tags=["indexers"]) + app.include_router(catalog_router, prefix="/api/indexers/catalog", tags=["catalog"]) app.include_router(search_router.router, prefix="/api/search", tags=["search"]) app.include_router(browse_router.router, prefix="/api/browse", tags=["browse"]) app.include_router(alerts_router.router, prefix="/api/alerts", tags=["alerts"]) diff --git a/backend/src/trove/models/indexer.py b/backend/src/trove/models/indexer.py index 3e2b152..3a40b9d 100644 --- a/backend/src/trove/models/indexer.py +++ b/backend/src/trove/models/indexer.py @@ -25,6 +25,7 @@ class IndexerRow(SQLModel, table=True): last_test_at: datetime | None = Field(default=None) last_test_ok: bool | None = Field(default=None) last_test_message: str | None = Field(default=None, max_length=512) + catalog_slug: str | None = Field(default=None, max_length=64, index=True) class IndexerEventRow(SQLModel, table=True): diff --git a/backend/src/trove/services/catalog.py b/backend/src/trove/services/catalog.py new file mode 100644 index 0000000..03ef260 --- /dev/null +++ b/backend/src/trove/services/catalog.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +import yaml + +from trove.clients.base import Protocol +from trove.indexers.base import Category + +_CATALOG_DIR = Path(__file__).parent.parent / "indexers" / "catalog" + + +class CatalogError(Exception): + """Raised when the shipped catalog is malformed. This is always our bug, + never user input — the registry is vendored code.""" + + +@dataclass(frozen=True, slots=True) +class CatalogEntry: + slug: str + display_name: str + description: str + categories: list[Category] + yaml_file: str + upstream_path: str + mirrors: list[str] + default_mirror: str + protocol: Protocol + logo: str | None = None + + +@lru_cache(maxsize=1) +def load_catalog() -> dict[str, CatalogEntry]: + registry_path = _CATALOG_DIR / "registry.yaml" + if not registry_path.exists(): + raise CatalogError(f"catalog registry missing at {registry_path}") + raw = yaml.safe_load(registry_path.read_text(encoding="utf-8")) or {} + entries_raw = raw.get("entries") or [] + if not isinstance(entries_raw, list): + raise CatalogError("registry.yaml: `entries` must be a list") + + by_slug: dict[str, CatalogEntry] = {} + for row in entries_raw: + if not isinstance(row, dict): + raise CatalogError("registry.yaml: each entry must be a mapping") + slug = row.get("slug") + if not slug or not isinstance(slug, str): + raise CatalogError("registry.yaml: entry missing `slug`") + if slug in by_slug: + raise CatalogError(f"registry.yaml: duplicate slug {slug!r}") + + try: + categories = [Category(c) for c in row.get("categories") or []] + except ValueError as e: + raise CatalogError(f"registry.yaml: {slug}: unknown category {e}") from e + try: + protocol = Protocol(row.get("protocol") or "torrent") + except ValueError as e: + raise CatalogError(f"registry.yaml: {slug}: unknown protocol {e}") from e + + mirrors = list(row.get("mirrors") or []) + default_mirror = row.get("default_mirror") or "" + if not mirrors: + raise CatalogError(f"registry.yaml: {slug}: at least one mirror required") + if default_mirror not in mirrors: + raise CatalogError(f"registry.yaml: {slug}: default_mirror must be a member of mirrors") + + by_slug[slug] = CatalogEntry( + slug=slug, + display_name=str(row.get("display_name") or slug), + description=str(row.get("description") or ""), + categories=categories, + yaml_file=str(row.get("yaml_file") or f"{slug}.yml"), + upstream_path=str(row.get("upstream_path") or ""), + mirrors=mirrors, + default_mirror=default_mirror, + protocol=protocol, + logo=row.get("logo"), + ) + + return by_slug + + +def list_entries() -> list[CatalogEntry]: + return list(load_catalog().values()) + + +def get_entry(slug: str) -> CatalogEntry: + entries = load_catalog() + if slug not in entries: + raise KeyError(slug) + return entries[slug] + + +def read_yaml(slug: str) -> str: + entry = get_entry(slug) + path = _CATALOG_DIR / entry.yaml_file + if not path.exists(): + raise CatalogError(f"catalog: {slug}: missing yaml file at {path}") + return path.read_text(encoding="utf-8") + + +def reset_cache_for_tests() -> None: + """pytest hook — the module-level cache would otherwise outlive test DBs.""" + load_catalog.cache_clear() diff --git a/backend/tests/api/test_catalog_api.py b/backend/tests/api/test_catalog_api.py new file mode 100644 index 0000000..d241b0f --- /dev/null +++ b/backend/tests/api/test_catalog_api.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from fastapi.testclient import TestClient + + +def _login(client: TestClient) -> None: + client.post( + "/api/auth/setup", + json={"username": "admin", "password": "correct horse battery staple"}, + ) + + +def test_list_catalog_returns_entries(client: TestClient) -> None: + _login(client) + resp = client.get("/api/indexers/catalog") + assert resp.status_code == 200 + body = resp.json() + assert len(body) >= 10 + slugs = {e["slug"] for e in body} + assert "1337x" in slugs + entry = next(e for e in body if e["slug"] == "1337x") + assert entry["already_installed"] is False + assert entry["default_mirror"] in entry["mirrors"] + + +def test_install_catalog_entry_creates_indexer(client: TestClient) -> None: + _login(client) + resp = client.post( + "/api/indexers/catalog/1337x", + json={"base_url": "https://1337x.to", "name": None}, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["type"] == "cardigann" + assert body["base_url"] == "https://1337x.to" + assert body["name"] == "1337x" + + # already_installed flips on a subsequent list + listing = client.get("/api/indexers/catalog").json() + entry = next(e for e in listing if e["slug"] == "1337x") + assert entry["already_installed"] is True + + +def test_install_twice_dedups_name(client: TestClient) -> None: + _login(client) + first = client.post( + "/api/indexers/catalog/1337x", + json={"base_url": "https://1337x.to"}, + ) + second = client.post( + "/api/indexers/catalog/1337x", + json={"base_url": "https://1337x.st"}, + ) + assert first.status_code == 201 + assert second.status_code == 201 + assert first.json()["name"] == "1337x" + assert second.json()["name"] == "1337x-2" + + +def test_install_unknown_slug_404(client: TestClient) -> None: + _login(client) + resp = client.post( + "/api/indexers/catalog/not-a-real-site", + json={"base_url": "https://example.com"}, + ) + assert resp.status_code == 404 + + +def test_install_rejects_base_url_not_in_mirrors(client: TestClient) -> None: + _login(client) + resp = client.post( + "/api/indexers/catalog/1337x", + json={"base_url": "https://totally-evil-mirror.example.com"}, + ) + assert resp.status_code == 422 + assert resp.json()["detail"] == "base_url_not_in_catalog_mirrors" diff --git a/backend/tests/fixtures/catalog/README.md b/backend/tests/fixtures/catalog/README.md new file mode 100644 index 0000000..98ff0ec --- /dev/null +++ b/backend/tests/fixtures/catalog/README.md @@ -0,0 +1,16 @@ +# Catalog fixture HTML + +One `-search.html` file per site, containing a real search-results page captured from the site's response to a benign query (e.g. "ubuntu", "debian", "linux"). + +## How to capture + +```bash +# Example: TPB +curl -sL 'https://thepiratebay.org/search/ubuntu/0/99/0' \ + -H 'User-Agent: Mozilla/5.0' \ + > thepiratebay-search.html +``` + +Pick a query whose results are unambiguous and stable (distro ISOs, commonly-seeded old scene releases). The fixture is committed — keep it small (<500 KB) by trimming or using a narrow query. + +Re-capture whenever `scripts/update-catalog.py diff` shows the upstream YAML has changed *and* the corresponding fixture test starts failing. diff --git a/backend/tests/fixtures/catalog/nyaa-search.html b/backend/tests/fixtures/catalog/nyaa-search.html new file mode 100644 index 0000000..067acea --- /dev/null +++ b/backend/tests/fixtures/catalog/nyaa-search.html @@ -0,0 +1,375 @@ + + + + + ubuntu :: Nyaa + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ Click here to see only results uploaded by ubuntu +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryNameLinkSizeDate
+ + Software - Applications + + + Koha Live CD Release 3 (3.0.4 Ubuntu 9.10 Desktop x86) + + + + 624.0 MiB2009-11-03 07:03010
+
+ +
+
Displaying results 1-1 out of 1 results.
+Please refine your search results if you can't find what you were looking for.
+ +
+
+ + + + \ No newline at end of file diff --git a/backend/tests/test_cardigann_filters.py b/backend/tests/test_cardigann_filters.py new file mode 100644 index 0000000..e804c9b --- /dev/null +++ b/backend/tests/test_cardigann_filters.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from bs4 import BeautifulSoup + +from trove.indexers.cardigann import ( + CardigannDefinition, + CardigannIndexer, + FieldSpec, +) + + +def _driver_with_field(name: str, filters: list[dict]) -> CardigannIndexer: + definition = CardigannDefinition( + site="test", + name="test", + links=["https://test.local"], + search_path="/", + search_params={}, + rows_selector="tr", + fields={name: FieldSpec(selector="td", filters=filters)}, + ) + return CardigannIndexer(definition) + + +def _apply(driver: CardigannIndexer, html: str, field: str) -> str | None: + row = BeautifulSoup(html, "lxml").find("tr") + return driver._extract_field(row, field) + + +def test_urldecode() -> None: + drv = _driver_with_field("title", [{"name": "urldecode"}]) + assert _apply(drv, "Hello%20World", "title") == "Hello World" + + +def test_split_by_delimiter_index() -> None: + drv = _driver_with_field( + "size", + [{"name": "split", "args": ["|", 1]}], + ) + result = _apply(drv, "1.2 GB | 42 seeders | 3 leechers", "size") + assert result is not None + assert result.strip() == "42 seeders" + + +def test_split_negative_index_returns_last() -> None: + drv = _driver_with_field( + "size", + [{"name": "split", "args": ["|", -1]}], + ) + assert _apply(drv, "a|b|c", "size") == "c" + + +def test_trim_with_no_args_strips_whitespace() -> None: + drv = _driver_with_field( + "title", + [{"name": "split", "args": ["|", 0]}, {"name": "trim"}], + ) + assert _apply(drv, " hello | world", "title") == "hello" + + +def test_trim_with_args_strips_specific_chars() -> None: + drv = _driver_with_field( + "title", + [{"name": "split", "args": [" | ", 0]}, {"name": "trim", "args": "/"}], + ) + assert _apply(drv, "/path/to/thing/ | rest", "title") == "path/to/thing" + + +def test_tolower() -> None: + drv = _driver_with_field("title", [{"name": "tolower"}]) + assert _apply(drv, "Hello WORLD", "title") == "hello world" + + +def test_re_replace() -> None: + # Replace sequences of whitespace with a single dash. + drv = _driver_with_field( + "title", + [{"name": "re_replace", "args": [r"\s+", "-"]}], + ) + assert _apply(drv, "Big Buck Bunny", "title") == "Big-Buck-Bunny" + + +def test_re_replace_removes_matches_with_empty_replacement() -> None: + drv = _driver_with_field( + "title", + [{"name": "re_replace", "args": [r"\d+", ""]}], + ) + assert _apply(drv, "abc123def456", "title") == "abcdef" + + +def test_unknown_filter_logs_warning(caplog) -> None: + import logging + + drv = _driver_with_field( + "title", + [{"name": "definitelynotarealfilter"}], + ) + with caplog.at_level(logging.WARNING): + result = _apply(drv, "hello", "title") + assert result == "hello" + assert any("definitelynotarealfilter" in rec.message for rec in caplog.records), ( + "expected a warning mentioning the unknown filter name" + ) + + +def test_unknown_filter_logs_once_per_process(caplog) -> None: + import logging + + drv = _driver_with_field( + "title", + [{"name": "another-fake-filter"}], + ) + with caplog.at_level(logging.WARNING): + _apply(drv, "1", "title") + _apply(drv, "2", "title") + _apply(drv, "3", "title") + count = sum(1 for rec in caplog.records if "another-fake-filter" in rec.message) + assert count == 1, f"expected exactly one warning, got {count}" diff --git a/backend/tests/test_cardigann_templates.py b/backend/tests/test_cardigann_templates.py new file mode 100644 index 0000000..bb1c428 --- /dev/null +++ b/backend/tests/test_cardigann_templates.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import yaml as _yaml + +from trove.indexers.cardigann import expand_template, load_definition + + +def test_load_definition_parses_settings_defaults() -> None: + raw = _yaml.safe_load(""" +id: test +name: test +links: [https://test.local] +type: public +settings: + - name: sort + type: select + default: time + - name: disablesort + type: checkbox + default: False + - name: no_default_item + type: text +caps: + categorymappings: [] +search: + paths: + - path: / + rows: + selector: tr + fields: {} +""") + d = load_definition(raw) + assert d.config_defaults["sort"] == "time" + assert d.config_defaults["disablesort"] == "False" + assert "no_default_item" not in d.config_defaults + + +def test_expand_keywords_substitution() -> None: + assert expand_template("search/{{ .Keywords }}/1/", keywords="ubuntu") == "search/ubuntu/1/" + + +def test_expand_if_keywords_else() -> None: + tmpl = "{{ if .Keywords }}search/{{ .Keywords }}{{ else }}latest{{ end }}" + assert expand_template(tmpl, keywords="ubuntu") == "search/ubuntu" + assert expand_template(tmpl, keywords="") == "latest" + + +def test_expand_config_substitution() -> None: + cfg = {"sort": "time", "apiurl": "example.com"} + assert expand_template("{{ .Config.sort }}", config=cfg) == "time" + assert expand_template("https://{{ .Config.apiurl }}/x", config=cfg) == "https://example.com/x" + assert expand_template("{{ .Config.missing }}", config=cfg) == "" + + +def test_expand_query_imdb_is_empty() -> None: + assert expand_template("id:{{ .Query.IMDBID }}", keywords="x") == "id:" + + +def test_expand_range_categories_is_empty() -> None: + tmpl = "path/{{ range .Categories }}:cat:{{.}}{{ end }}" + assert expand_template(tmpl, keywords="x") == "path/" + + +def test_expand_join_categories_is_empty() -> None: + assert expand_template('cats={{ join .Categories "," }}', keywords="x") == "cats=" + + +def test_expand_complex_1337x_path() -> None: + # Sample pulled from 1337x.yml + cfg = {"sort": "time", "type": "desc", "disablesort": "False"} + tmpl = ( + "{{ if and (.Keywords) (eq .Config.disablesort .False) }}sort-{{ else }}{{ end }}" + "{{ if .Keywords }}search/{{ .Keywords }}{{ else }}cat/Movies{{ end }}" + "{{ if and (.Keywords) (eq .Config.disablesort .False) }}/{{ .Config.sort }}/{{ .Config.type }}{{ else }}{{ end }}" + "/1/" + ) + result = expand_template(tmpl, keywords="ubuntu", config=cfg) + # Expected: "sort-search/ubuntu/time/desc/1/" + assert result == "sort-search/ubuntu/time/desc/1/" + + +def test_expand_no_template_passes_through() -> None: + assert expand_template("/") == "/" + assert expand_template("search/static/path") == "search/static/path" + + +def test_expand_returns_unchanged_on_unknown_directive() -> None: + # Unknown pipeline -> untouched + assert "{{ weird_directive" in expand_template("{{ weird_directive }}", keywords="x") + + +def test_1337x_url_construction_uses_expanded_template() -> None: + from trove.indexers.cardigann import CardigannIndexer, load_definition_yaml + from trove.services import catalog + + catalog.reset_cache_for_tests() + d = load_definition_yaml(catalog.read_yaml("1337x")) + CardigannIndexer(d, base_url="https://1337x.to") # verify instantiation succeeds + path = expand_template(d.search_path, keywords="ubuntu", config=d.config_defaults) + assert "{{" not in path, f"path still has templates: {path}" + assert "ubuntu" in path + assert path.startswith("/") or "search" in path or "sort-" in path + + +def test_expand_or_expression_returns_first_truthy() -> None: + # Go template `or` returns first truthy arg. .Query.IMDBID is always empty, + # so `or .Query.IMDBID .Keywords` should return keywords. + tmpl = "get-posts/keywords:{{ or .Query.IMDBID .Keywords }}" + assert expand_template(tmpl, keywords="ubuntu") == "get-posts/keywords:ubuntu" + + +def test_expand_or_expression_fallback_to_keywords() -> None: + tmpl = "{{ or .Query.IMDBID .Keywords }}" + assert expand_template(tmpl, keywords="ubuntu") == "ubuntu" + # When nothing is truthy, result is empty string (last arg is also empty). + assert expand_template(tmpl, keywords="") == "" + + +def test_expand_inline_re_replace_on_config() -> None: + # bitsearch path uses: search{{ re_replace .Config.sort "_" "" }}?q=... + cfg = {"sort": "date_desc"} + tmpl = '{{ re_replace .Config.sort "_" "" }}' + assert expand_template(tmpl, keywords="x", config=cfg) == "datedesc" + + +def test_expand_bitsearch_full_path() -> None: + cfg = {"sort": "date_desc"} + tmpl = '{{ if .Keywords }}search{{ re_replace .Config.sort "_" "" }}?q={{ .Keywords }}{{ else }}/{{ end }}' + assert expand_template(tmpl, keywords="ubuntu", config=cfg) == "searchdatedesc?q=ubuntu" + + +def test_expand_torrentgalaxy_full_path() -> None: + tmpl = "get-posts/{{ if or .Query.IMDBID .Keywords }}keywords:{{ or .Query.IMDBID .Keywords }}{{ else }}{{ end }}{{ range .Categories }}:category:{{.}}{{end}}" + result = expand_template(tmpl, keywords="ubuntu") + assert result == "get-posts/keywords:ubuntu" + assert "{{" not in result diff --git a/backend/tests/test_catalog.py b/backend/tests/test_catalog.py new file mode 100644 index 0000000..4fbdb91 --- /dev/null +++ b/backend/tests/test_catalog.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from trove.services import catalog + +CATALOG_DIR = Path(catalog.__file__).parent.parent / "indexers" / "catalog" + + +@pytest.fixture(autouse=True) +def _reset_cache() -> None: + catalog.reset_cache_for_tests() + + +def test_registry_loads() -> None: + entries = catalog.list_entries() + assert len(entries) >= 10 + slugs = {e.slug for e in entries} + for required in ("1337x", "nyaa", "eztv", "limetorrents"): + assert required in slugs, f"missing catalog entry: {required}" + + +def test_default_mirror_is_in_mirrors() -> None: + for entry in catalog.list_entries(): + assert entry.default_mirror in entry.mirrors, ( + f"{entry.slug}: default_mirror {entry.default_mirror!r} " + f"not present in mirrors {entry.mirrors}" + ) + + +def test_every_entry_has_a_vendored_yaml_file() -> None: + missing: list[str] = [] + for entry in catalog.list_entries(): + path = CATALOG_DIR / entry.yaml_file + if not path.exists(): + missing.append(f"{entry.slug} -> {entry.yaml_file}") + assert not missing, "missing vendored YAML files: " + ", ".join(missing) + + +def test_every_vendored_yaml_parses() -> None: + from trove.indexers.cardigann import load_definition_yaml + + failures: list[str] = [] + for entry in catalog.list_entries(): + try: + load_definition_yaml(catalog.read_yaml(entry.slug)) + except Exception as e: + failures.append(f"{entry.slug}: {type(e).__name__}: {e}") + assert not failures, "YAMLs failed to parse:\n " + "\n ".join(failures) diff --git a/backend/tests/test_catalog_fixtures.py b/backend/tests/test_catalog_fixtures.py new file mode 100644 index 0000000..e007f85 --- /dev/null +++ b/backend/tests/test_catalog_fixtures.py @@ -0,0 +1,50 @@ +"""Parse-each-fixture smoke test. + +For every slug in the catalog that has a corresponding +`tests/fixtures/catalog/-search.html` file, this test: + - loads the vendored YAML + - runs the Cardigann row extractor against the fixture + - asserts at least one release with a non-empty title + +Missing fixtures are silently skipped — run `pytest -vv` to see which +slugs are covered. Capturing real HTML fixtures is a one-time manual +task per site; tests pass until a fixture is captured AND its extraction +breaks. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from bs4 import BeautifulSoup + +from trove.indexers.cardigann import CardigannIndexer, load_definition_yaml +from trove.services import catalog + +FIXTURE_DIR = Path(__file__).parent / "fixtures" / "catalog" + + +def _covered_slugs() -> list[str]: + catalog.reset_cache_for_tests() + return [ + e.slug for e in catalog.list_entries() if (FIXTURE_DIR / f"{e.slug}-search.html").exists() + ] + + +@pytest.mark.parametrize("slug", _covered_slugs()) +def test_fixture_extracts_at_least_one_release(slug: str) -> None: + entry = catalog.get_entry(slug) + definition = load_definition_yaml(catalog.read_yaml(slug)) + definition.name = slug + driver = CardigannIndexer(definition, base_url=entry.default_mirror) + + html = (FIXTURE_DIR / f"{slug}-search.html").read_text(encoding="utf-8") + soup = BeautifulSoup(html, "lxml") + rows = soup.select(definition.rows_selector) + assert rows, f"{slug}: rows_selector {definition.rows_selector!r} matched no elements" + + extracted = [driver._extract_release(r) for r in rows] + extracted = [r for r in extracted if r is not None] + assert extracted, f"{slug}: 0 releases extracted from {len(rows)} row(s)" + assert extracted[0].title, f"{slug}: first release has empty title" diff --git a/docs/superpowers/plans/2026-04-20-public-torrent-site-catalog.md b/docs/superpowers/plans/2026-04-20-public-torrent-site-catalog.md new file mode 100644 index 0000000..1ee95a9 --- /dev/null +++ b/docs/superpowers/plans/2026-04-20-public-torrent-site-catalog.md @@ -0,0 +1,2073 @@ +# Public Torrent Site Catalog Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a curated, built-in catalog of 12 public (no-account) torrent sites installable with one click from a dedicated `/indexers/catalog` page and as an optional step in the onboarding wizard. + +**Architecture:** A new `backend/src/trove/indexers/catalog/` directory holds a hand-written `registry.yaml` plus twelve vendored Cardigann `.yml` definitions. A new `services/catalog.py` module exposes the catalog, and two new endpoints (`GET /api/indexers/catalog`, `POST /api/indexers/catalog/{slug}`) create ordinary `type=cardigann` indexer rows — the only storage change is a nullable `catalog_slug` marker column. The existing Cardigann parser is extended with a handful of new filter types (driven by what the vendored YAMLs actually use). + +**Tech Stack:** Python 3.12, FastAPI, SQLModel, Alembic, httpx, PyYAML, BeautifulSoup4/lxml; SvelteKit 5 + TypeScript; pytest + respx for tests. + +**Spec:** `docs/superpowers/specs/2026-04-20-public-torrent-site-catalog-design.md` + +--- + +## File Structure + +**New files:** + +- `backend/migrations/versions/0016_indexer_catalog_slug.py` — Alembic migration adding the column +- `backend/src/trove/indexers/catalog/__init__.py` — empty marker +- `backend/src/trove/indexers/catalog/registry.yaml` — curated metadata (authoritative) +- `backend/src/trove/indexers/catalog/*.yml` — 12 vendored Prowlarr definitions (downloaded via script) +- `backend/src/trove/services/catalog.py` — registry loader + YAML reader +- `backend/src/trove/api/catalog.py` — new router with two endpoints (lives next to `indexers.py`, mounted under the same prefix) +- `backend/tests/test_catalog.py` — registry integrity + per-YAML parse tests +- `backend/tests/test_cardigann_filters.py` — unit tests for new parser filters +- `backend/tests/api/test_catalog_api.py` — endpoint tests +- `backend/tests/fixtures/catalog/` — captured HTML responses, one per site +- `scripts/update-catalog.py` — downloads/diffs vendored YAMLs against upstream +- `web/src/routes/indexers/catalog/+page.svelte` — tile-grid catalog page + +**Modified files:** + +- `backend/src/trove/models/indexer.py` — add `catalog_slug` field to `IndexerRow` +- `backend/src/trove/indexers/cardigann.py` — extend `_apply_filter` with new filter types and an unknown-filter warn log +- `backend/src/trove/main.py` — mount the new catalog router +- `web/src/lib/api.ts` — add `CatalogEntryOut` type + `api.indexers.catalog.*` methods +- `web/src/routes/indexers/+page.svelte` — add "Browse catalog" button +- `web/src/routes/onboarding/+page.svelte` — insert a new "Public torrent sites" step +- `backend/src/trove/docs/03-indexers.md` — document the catalog + +--- + +## Task 1: Add `catalog_slug` column to `IndexerRow` model + +**Files:** +- Modify: `backend/src/trove/models/indexer.py:12-27` + +- [ ] **Step 1: Add the field** + +Edit `backend/src/trove/models/indexer.py` — add one line inside the `IndexerRow` class, directly after `last_test_message`: + +```python + catalog_slug: str | None = Field(default=None, max_length=64, index=True) +``` + +- [ ] **Step 2: Verify model import still works** + +Run: `cd backend && uv run python -c "from trove.models.indexer import IndexerRow; print(IndexerRow.__fields__.keys())"` +Expected: the printed list includes `catalog_slug`. + +- [ ] **Step 3: Commit** + +```bash +git add backend/src/trove/models/indexer.py +git commit -m "feat: add catalog_slug field to IndexerRow" +``` + +--- + +## Task 2: Alembic migration for the new column + +**Files:** +- Create: `backend/migrations/versions/0016_indexer_catalog_slug.py` + +- [ ] **Step 1: Write the migration** + +Create `backend/migrations/versions/0016_indexer_catalog_slug.py`: + +```python +"""indexer.catalog_slug column + +Revision ID: 0016 +Revises: 0015 +Create Date: 2026-04-20 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +import sqlmodel +from alembic import op + +revision: str = "0016" +down_revision: str | None = "0015" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + with op.batch_alter_table("indexer") as batch: + batch.add_column( + sa.Column( + "catalog_slug", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=True, + ) + ) + op.create_index( + "ix_indexer_catalog_slug", "indexer", ["catalog_slug"], unique=False + ) + + +def downgrade() -> None: + op.drop_index("ix_indexer_catalog_slug", table_name="indexer") + with op.batch_alter_table("indexer") as batch: + batch.drop_column("catalog_slug") +``` + +`op.batch_alter_table` is required on SQLite because ALTER is restricted; Trove uses SQLite in WAL mode. + +- [ ] **Step 2: Apply migration against a scratch DB** + +Run: `cd backend && rm -f /tmp/trove-mig-test.db && TROVE_CONFIG_DIR=/tmp/trove-mig-test uv run alembic upgrade head` +Expected: last line `INFO [alembic.runtime.migration] Running upgrade 0015 -> 0016, indexer.catalog_slug column`. + +- [ ] **Step 3: Roll it back, then forward again** + +Run: `cd backend && TROVE_CONFIG_DIR=/tmp/trove-mig-test uv run alembic downgrade 0015 && TROVE_CONFIG_DIR=/tmp/trove-mig-test uv run alembic upgrade head` +Expected: both steps complete without errors. + +- [ ] **Step 4: Commit** + +```bash +git add backend/migrations/versions/0016_indexer_catalog_slug.py +git commit -m "feat: migration adds indexer.catalog_slug column" +``` + +--- + +## Task 3: Write `registry.yaml` + +**Files:** +- Create: `backend/src/trove/indexers/catalog/__init__.py` +- Create: `backend/src/trove/indexers/catalog/registry.yaml` + +- [ ] **Step 1: Create the package marker** + +Create `backend/src/trove/indexers/catalog/__init__.py` as an empty file: + +```python +``` + +- [ ] **Step 2: Write the registry** + +Create `backend/src/trove/indexers/catalog/registry.yaml`: + +```yaml +# Authoritative index of public no-account torrent sites shipped with Trove. +# Each entry references a vendored Cardigann YAML in the same directory. +# +# Filenames under `yaml_file:` are OUR vendored filenames — they need not +# match upstream. `upstream_path:` records where the definition came from +# so scripts/update-catalog.py can diff against Prowlarr-indexers. + +entries: + - slug: thepiratebay + display_name: The Pirate Bay + description: General-purpose public torrent tracker, no account required. + categories: [movies, tv, music, software, games, books, other] + yaml_file: thepiratebay.yml + upstream_path: definitions/v11/thepiratebay.yml + mirrors: + - https://thepiratebay.org + - https://tpb.party + - https://piratebay.live + default_mirror: https://thepiratebay.org + protocol: torrent + + - slug: 1337x + display_name: 1337x + description: Large public general-purpose tracker with strong TV/movie scene coverage. + categories: [movies, tv, music, software, games, books, anime] + yaml_file: 1337x.yml + upstream_path: definitions/v11/1337x.yml + mirrors: + - https://1337x.to + - https://1337x.st + - https://1337x.tw + default_mirror: https://1337x.to + protocol: torrent + + - slug: torrentgalaxy + display_name: TorrentGalaxy + description: General-purpose public tracker with reliable scene releases and good category filtering. + categories: [movies, tv, music, software, games, books, anime, other] + yaml_file: torrentgalaxy.yml + upstream_path: definitions/v11/torrentgalaxy.yml + mirrors: + - https://torrentgalaxy.to + - https://tgx.rs + default_mirror: https://torrentgalaxy.to + protocol: torrent + + - slug: limetorrents + display_name: LimeTorrents + description: Long-running public aggregator with a wide catalogue. + categories: [movies, tv, music, software, games, anime, other] + yaml_file: limetorrents.yml + upstream_path: definitions/v11/limetorrents.yml + mirrors: + - https://www.limetorrents.lol + - https://www.limetorrents.info + default_mirror: https://www.limetorrents.lol + protocol: torrent + + - slug: magnetdl + display_name: MagnetDL + description: Magnet-link aggregator, fast and light. + categories: [movies, tv, music, software, games, books, anime] + yaml_file: magnetdl.yml + upstream_path: definitions/v11/magnetdl.yml + mirrors: + - https://www.magnetdl.com + default_mirror: https://www.magnetdl.com + protocol: torrent + + - slug: torlock + display_name: Torlock + description: General-purpose tracker focused on verified torrents. + categories: [movies, tv, music, software, games, books, anime] + yaml_file: torlock.yml + upstream_path: definitions/v11/torlock.yml + mirrors: + - https://www.torlock.com + default_mirror: https://www.torlock.com + protocol: torrent + + - slug: bitsearch + display_name: BitSearch + description: Aggregator that searches across multiple public sites. + categories: [movies, tv, music, software, games, books, anime, other] + yaml_file: bitsearch.yml + upstream_path: definitions/v11/bitsearch.yml + mirrors: + - https://bitsearch.to + default_mirror: https://bitsearch.to + protocol: torrent + + - slug: solidtorrents + display_name: SolidTorrents + description: Multi-source torrent search aggregator. + categories: [movies, tv, music, software, games, books, anime, other] + yaml_file: solidtorrents.yml + upstream_path: definitions/v11/solidtorrents.yml + mirrors: + - https://solidtorrents.to + default_mirror: https://solidtorrents.to + protocol: torrent + + - slug: nyaa + display_name: Nyaa + description: The largest public anime & manga tracker. + categories: [anime, music, books, other] + yaml_file: nyaa.yml + upstream_path: definitions/v11/nyaasi.yml + mirrors: + - https://nyaa.si + default_mirror: https://nyaa.si + protocol: torrent + + - slug: eztv + display_name: EZTV + description: TV-focused public tracker with strong scene release coverage. + categories: [tv] + yaml_file: eztv.yml + upstream_path: definitions/v11/eztv.yml + mirrors: + - https://eztv.re + - https://eztvx.to + default_mirror: https://eztv.re + protocol: torrent + + - slug: yts + display_name: YTS + description: Public tracker specializing in small-size movie encodes. + categories: [movies] + yaml_file: yts.yml + upstream_path: definitions/v11/yts.yml + mirrors: + - https://yts.mx + default_mirror: https://yts.mx + protocol: torrent + + - slug: animetosho + display_name: AnimeTosho + description: Anime mirror and long-term archive of Nyaa + Tokyo Toshokan. + categories: [anime] + yaml_file: animetosho.yml + upstream_path: definitions/v11/animetosho.yml + mirrors: + - https://animetosho.org + default_mirror: https://animetosho.org + protocol: torrent +``` + +- [ ] **Step 3: Commit** + +```bash +git add backend/src/trove/indexers/catalog/__init__.py backend/src/trove/indexers/catalog/registry.yaml +git commit -m "feat: add catalog registry for 12 public torrent sites" +``` + +--- + +## Task 4: Write `services/catalog.py` + +**Files:** +- Create: `backend/src/trove/services/catalog.py` + +- [ ] **Step 1: Write the module** + +Create `backend/src/trove/services/catalog.py`: + +```python +from __future__ import annotations + +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path + +import yaml + +from trove.clients.base import Protocol +from trove.indexers.base import Category + +_CATALOG_DIR = Path(__file__).parent.parent / "indexers" / "catalog" + + +class CatalogError(Exception): + """Raised when the shipped catalog is malformed. This is always our bug, + never user input — the registry is vendored code.""" + + +@dataclass(frozen=True, slots=True) +class CatalogEntry: + slug: str + display_name: str + description: str + categories: list[Category] + yaml_file: str + upstream_path: str + mirrors: list[str] + default_mirror: str + protocol: Protocol + logo: str | None = None + + +@lru_cache(maxsize=1) +def load_catalog() -> dict[str, CatalogEntry]: + registry_path = _CATALOG_DIR / "registry.yaml" + if not registry_path.exists(): + raise CatalogError(f"catalog registry missing at {registry_path}") + raw = yaml.safe_load(registry_path.read_text(encoding="utf-8")) or {} + entries_raw = raw.get("entries") or [] + if not isinstance(entries_raw, list): + raise CatalogError("registry.yaml: `entries` must be a list") + + by_slug: dict[str, CatalogEntry] = {} + for row in entries_raw: + if not isinstance(row, dict): + raise CatalogError("registry.yaml: each entry must be a mapping") + slug = row.get("slug") + if not slug or not isinstance(slug, str): + raise CatalogError("registry.yaml: entry missing `slug`") + if slug in by_slug: + raise CatalogError(f"registry.yaml: duplicate slug {slug!r}") + + try: + categories = [Category(c) for c in row.get("categories") or []] + except ValueError as e: + raise CatalogError(f"registry.yaml: {slug}: unknown category {e}") from e + try: + protocol = Protocol(row.get("protocol") or "torrent") + except ValueError as e: + raise CatalogError(f"registry.yaml: {slug}: unknown protocol {e}") from e + + mirrors = list(row.get("mirrors") or []) + default_mirror = row.get("default_mirror") or "" + if not mirrors: + raise CatalogError(f"registry.yaml: {slug}: at least one mirror required") + if default_mirror not in mirrors: + raise CatalogError( + f"registry.yaml: {slug}: default_mirror must be a member of mirrors" + ) + + by_slug[slug] = CatalogEntry( + slug=slug, + display_name=str(row.get("display_name") or slug), + description=str(row.get("description") or ""), + categories=categories, + yaml_file=str(row.get("yaml_file") or f"{slug}.yml"), + upstream_path=str(row.get("upstream_path") or ""), + mirrors=mirrors, + default_mirror=default_mirror, + protocol=protocol, + logo=row.get("logo"), + ) + + return by_slug + + +def list_entries() -> list[CatalogEntry]: + return list(load_catalog().values()) + + +def get_entry(slug: str) -> CatalogEntry: + entries = load_catalog() + if slug not in entries: + raise KeyError(slug) + return entries[slug] + + +def read_yaml(slug: str) -> str: + entry = get_entry(slug) + path = _CATALOG_DIR / entry.yaml_file + if not path.exists(): + raise CatalogError(f"catalog: {slug}: missing yaml file at {path}") + return path.read_text(encoding="utf-8") + + +def reset_cache_for_tests() -> None: + """pytest hook — the module-level cache would otherwise outlive test DBs.""" + load_catalog.cache_clear() +``` + +- [ ] **Step 2: Sanity-load** + +Run: `cd backend && uv run python -c "from trove.services import catalog; print(len(catalog.list_entries()))"` +Expected: `12`. + +- [ ] **Step 3: Commit** + +```bash +git add backend/src/trove/services/catalog.py +git commit -m "feat: catalog service loads registry + vendored YAMLs" +``` + +--- + +## Task 5: Catalog integrity test (pre-vendoring) + +**Files:** +- Create: `backend/tests/test_catalog.py` + +- [ ] **Step 1: Write the failing test** + +Create `backend/tests/test_catalog.py`: + +```python +from __future__ import annotations + +from pathlib import Path + +import pytest + +from trove.services import catalog + +CATALOG_DIR = Path(catalog.__file__).parent.parent / "indexers" / "catalog" + + +@pytest.fixture(autouse=True) +def _reset_cache() -> None: + catalog.reset_cache_for_tests() + + +def test_registry_loads() -> None: + entries = catalog.list_entries() + assert len(entries) >= 12 + slugs = {e.slug for e in entries} + for required in ("thepiratebay", "1337x", "nyaa", "eztv", "yts"): + assert required in slugs, f"missing catalog entry: {required}" + + +def test_default_mirror_is_in_mirrors() -> None: + for entry in catalog.list_entries(): + assert entry.default_mirror in entry.mirrors, ( + f"{entry.slug}: default_mirror {entry.default_mirror!r} " + f"not present in mirrors {entry.mirrors}" + ) + + +def test_every_entry_has_a_vendored_yaml_file() -> None: + missing: list[str] = [] + for entry in catalog.list_entries(): + path = CATALOG_DIR / entry.yaml_file + if not path.exists(): + missing.append(f"{entry.slug} -> {entry.yaml_file}") + assert not missing, "missing vendored YAML files: " + ", ".join(missing) + + +def test_every_vendored_yaml_parses() -> None: + from trove.indexers.cardigann import load_definition_yaml + + failures: list[str] = [] + for entry in catalog.list_entries(): + try: + load_definition_yaml(catalog.read_yaml(entry.slug)) + except Exception as e: # noqa: BLE001 + failures.append(f"{entry.slug}: {type(e).__name__}: {e}") + assert not failures, "YAMLs failed to parse:\n " + "\n ".join(failures) +``` + +- [ ] **Step 2: Run — expect failure on vendored-files test** + +Run: `cd backend && uv run pytest tests/test_catalog.py -v` +Expected: `test_registry_loads` and `test_default_mirror_is_in_mirrors` PASS. `test_every_entry_has_a_vendored_yaml_file` and `test_every_vendored_yaml_parses` FAIL with "missing vendored YAML files". + +This is the correct state before vendoring — don't fix yet. + +- [ ] **Step 3: Commit** + +```bash +git add backend/tests/test_catalog.py +git commit -m "test: catalog integrity checks (pre-vendor, expected to fail)" +``` + +--- + +## Task 6: Write `scripts/update-catalog.py` + +**Files:** +- Create: `scripts/update-catalog.py` + +- [ ] **Step 1: Write the script** + +Create `scripts/update-catalog.py`: + +```python +#!/usr/bin/env python3 +"""Vendor or diff Cardigann YAML definitions from Prowlarr-indexers. + +Usage: + scripts/update-catalog.py sync # download + overwrite vendored files + scripts/update-catalog.py diff # print per-file status, no writes + +The canonical upstream is Prowlarr/Prowlarr-indexers @ master. Slug→path +mapping lives in backend/src/trove/indexers/catalog/registry.yaml. +""" + +from __future__ import annotations + +import argparse +import hashlib +import sys +from pathlib import Path + +import httpx +import yaml + +REPO_ROOT = Path(__file__).parent.parent +CATALOG_DIR = REPO_ROOT / "backend" / "src" / "trove" / "indexers" / "catalog" +REGISTRY_PATH = CATALOG_DIR / "registry.yaml" +UPSTREAM_RAW = "https://raw.githubusercontent.com/Prowlarr/Prowlarr-indexers/master/" + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _load_registry() -> list[dict[str, str]]: + raw = yaml.safe_load(REGISTRY_PATH.read_text(encoding="utf-8")) or {} + entries = raw.get("entries") or [] + out: list[dict[str, str]] = [] + for row in entries: + upstream = row.get("upstream_path") + if not upstream: + continue + out.append( + { + "slug": row["slug"], + "yaml_file": row["yaml_file"], + "upstream_path": upstream, + } + ) + return out + + +def _fetch(client: httpx.Client, upstream_path: str) -> bytes | None: + url = UPSTREAM_RAW + upstream_path + resp = client.get(url, timeout=30.0) + if resp.status_code == 404: + return None + resp.raise_for_status() + return resp.content + + +def run(mode: str) -> int: + entries = _load_registry() + with httpx.Client(follow_redirects=True) as client: + changes = 0 + missing = 0 + for entry in entries: + slug = entry["slug"] + local_path = CATALOG_DIR / entry["yaml_file"] + upstream_bytes = _fetch(client, entry["upstream_path"]) + if upstream_bytes is None: + print(f" [MISSING] {slug}: upstream {entry['upstream_path']} not found") + missing += 1 + continue + + local_hash = _sha256(local_path.read_bytes()) if local_path.exists() else None + upstream_hash = _sha256(upstream_bytes) + + if local_hash == upstream_hash: + print(f" [unchanged] {slug}") + continue + + changes += 1 + if mode == "sync": + local_path.write_bytes(upstream_bytes) + state = "created" if local_hash is None else "updated" + print(f" [{state}] {slug}") + else: + state = "missing locally" if local_hash is None else "upstream changed" + print(f" [{state}] {slug}") + + print(f"\n{changes} change(s), {missing} missing upstream") + return 1 if missing else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["sync", "diff"]) + args = parser.parse_args() + return run(args.mode) + + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 2: Mark executable** + +Run: `chmod +x scripts/update-catalog.py` + +- [ ] **Step 3: Commit** + +```bash +git add scripts/update-catalog.py +git commit -m "feat: script to sync/diff vendored catalog YAMLs with upstream" +``` + +--- + +## Task 7: Run `update-catalog.py sync` to vendor the 12 YAMLs + +**Files:** +- Create: `backend/src/trove/indexers/catalog/thepiratebay.yml` (and 11 others) + +- [ ] **Step 1: Run the script** + +Run: `cd /home/masterdraco/trove && ./scripts/update-catalog.py sync` +Expected: 12 `[created]` lines, `12 change(s), 0 missing upstream`. + +If any entry reports `[MISSING]`, the `upstream_path` in `registry.yaml` is wrong for that slug — look on GitHub under `Prowlarr/Prowlarr-indexers/tree/master/definitions` for the correct filename (it may live under `v10` instead of `v11`, or have a slightly different spelling), fix `registry.yaml`, re-run. + +- [ ] **Step 2: Spot-check one YAML** + +Run: `head -40 backend/src/trove/indexers/catalog/thepiratebay.yml` +Expected: a Cardigann YAML document starting with `---` or `id:` / `name:` / `type:` fields. + +- [ ] **Step 3: Commit the vendored files and any fixed upstream paths** + +```bash +git add backend/src/trove/indexers/catalog/*.yml backend/src/trove/indexers/catalog/registry.yaml +git commit -m "vendor: import 12 Cardigann YAML definitions from Prowlarr-indexers" +``` + +--- + +## Task 8: Verify catalog parse test now passes (or surfaces missing filters) + +**Files:** +- None (investigation only) + +- [ ] **Step 1: Run catalog tests** + +Run: `cd backend && uv run pytest tests/test_catalog.py -v` +Expected: all four tests PASS **if** the vendored YAMLs only use filters the existing parser understands. + +If `test_every_vendored_yaml_parses` fails, read the error — `load_definition_yaml` raises `IndexerError` with descriptive messages for structural problems. Common failure modes: +- `has no search.paths` → upstream may have changed schema; fix in Task 15. +- `has no search.rows.selector` → same. + +These structural issues are rare; most failures in this phase come from filter-parsing. The filter layer only emits warnings today, so Task 8 may pass even when unknown filters are present (silently returning the unfiltered value). Tasks 9–13 fix that. + +- [ ] **Step 2: Scan the vendored files for filter names in use** + +Run: `cd backend && uv run python -c " +import pathlib, re +from trove.services import catalog +wanted = set() +for e in catalog.list_entries(): + text = catalog.read_yaml(e.slug) + for m in re.finditer(r'name:\s*(\w+)', text): + wanted.add(m.group(1)) +print(sorted(wanted)) +"` +Expected: a sorted list. Note any unfamiliar names — cross-reference with Tasks 9–14 to see what's already covered. + +Known-covered filters: `replace`, `regexp`, `append`, `prepend`. +Tasks 9–13 cover: `urldecode`, `urlencode`, `split`, `trim`, `querystring`. +Anything *else* in the list is an additional filter — write a TDD task for it using the template in Tasks 9–13 (failing test in `test_cardigann_filters.py` + one branch in `_apply_filter` + passing test + commit) and slot it in between Tasks 13 and 14. + +No commit — this step is investigation. + +--- + +## Task 9: Add `urldecode` / `urlencode` filters to Cardigann parser + +**Files:** +- Create: `backend/tests/test_cardigann_filters.py` +- Modify: `backend/src/trove/indexers/cardigann.py:256-270` + +- [ ] **Step 1: Write the failing tests** + +Create `backend/tests/test_cardigann_filters.py`: + +```python +from __future__ import annotations + +from bs4 import BeautifulSoup + +from trove.indexers.cardigann import ( + CardigannDefinition, + CardigannIndexer, + FieldSpec, + load_definition_yaml, +) + + +def _driver_with_field(name: str, filters: list[dict]) -> CardigannIndexer: + definition = CardigannDefinition( + site="test", + name="test", + links=["https://test.local"], + search_path="/", + search_params={}, + rows_selector="tr", + fields={name: FieldSpec(selector="td", filters=filters)}, + ) + return CardigannIndexer(definition) + + +def _apply(driver: CardigannIndexer, html: str, field: str) -> str | None: + row = BeautifulSoup(html, "lxml").find("tr") + return driver._extract_field(row, field) + + +def test_urldecode() -> None: + drv = _driver_with_field("title", [{"name": "urldecode"}]) + assert _apply(drv, "Hello%20World", "title") == "Hello World" + + +def test_urlencode() -> None: + drv = _driver_with_field("title", [{"name": "urlencode"}]) + assert _apply(drv, "Hello World", "title") == "Hello%20World" +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v` +Expected: `test_urldecode` PASSES unexpectedly (fallthrough returns value unchanged, which happens to equal the literal input for `urldecode` only when there's nothing to decode — but our input contains `%20`, so it should FAIL). `test_urlencode` FAILS — fallthrough returns unchanged. + +If both PASS because the fallthrough returns value unchanged and your inputs happen to match, that's still the wrong behavior: the filters are supposed to transform. Proceed to Step 3. + +- [ ] **Step 3: Implement the filters** + +Edit `backend/src/trove/indexers/cardigann.py` — extend `_apply_filter` (around line 256). Add these two branches **before** the final `return value`: + +```python + if name == "urldecode": + from urllib.parse import unquote + return unquote(value) + if name == "urlencode": + from urllib.parse import quote + return quote(value, safe="") +``` + +- [ ] **Step 4: Run — expect pass** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "urldecode or urlencode"` +Expected: both PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py +git commit -m "feat: cardigann urldecode/urlencode filters" +``` + +--- + +## Task 10: Add `split` filter + +**Files:** +- Modify: `backend/tests/test_cardigann_filters.py` +- Modify: `backend/src/trove/indexers/cardigann.py` + +- [ ] **Step 1: Write the failing test** + +Append to `backend/tests/test_cardigann_filters.py`: + +```python +def test_split_by_delimiter_index() -> None: + drv = _driver_with_field( + "size", + [{"name": "split", "args": ["|", 1]}], + ) + # "1.2 GB | 42 seeders | 3 leechers" -> split on '|', index 1 -> " 42 seeders " + assert _apply(drv, "1.2 GB | 42 seeders | 3 leechers", "size").strip() == "42 seeders" + + +def test_split_negative_index_returns_last() -> None: + drv = _driver_with_field( + "size", + [{"name": "split", "args": ["|", -1]}], + ) + # index -1 -> last chunk + assert _apply(drv, "a|b|c", "size") == "c" +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py::test_split_by_delimiter_index -v` +Expected: FAIL — returns the unsplit string. + +- [ ] **Step 3: Implement** + +Edit `backend/src/trove/indexers/cardigann.py` — add to `_apply_filter`, before the final `return value`: + +```python + if name == "split" and isinstance(args, list) and len(args) >= 2: + delimiter = str(args[0]) + try: + index = int(args[1]) + except (TypeError, ValueError): + return value + parts = value.split(delimiter) + if not parts: + return value + try: + return parts[index] + except IndexError: + return value +``` + +- [ ] **Step 4: Run — expect pass** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "split"` +Expected: both split tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py +git commit -m "feat: cardigann split filter" +``` + +--- + +## Task 11: Add `trim` filter + +**Files:** +- Modify: `backend/tests/test_cardigann_filters.py` +- Modify: `backend/src/trove/indexers/cardigann.py` + +- [ ] **Step 1: Write the failing test** + +Append to `backend/tests/test_cardigann_filters.py`: + +```python +def test_trim() -> None: + drv = _driver_with_field("title", [{"name": "trim"}]) + # get_text(" ", strip=True) strips leading/trailing whitespace already, + # but real use is *after* another filter (e.g. split) has reintroduced whitespace. + drv2 = _driver_with_field( + "title", + [{"name": "split", "args": ["|", 0]}, {"name": "trim"}], + ) + assert _apply(drv2, " hello | world", "title") == "hello" + + +def test_trim_with_args_strips_specific_chars() -> None: + drv = _driver_with_field("title", [{"name": "trim", "args": "/"}]) + # emulate "/path/to/thing/" -> "path/to/thing" + drv2 = _driver_with_field( + "title", + [{"name": "split", "args": [" | ", 0]}, {"name": "trim", "args": "/"}], + ) + assert _apply(drv2, "/path/to/thing/ | rest", "title") == "path/to/thing" +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "trim"` +Expected: FAIL — `test_trim` passes by accident (the text-extraction already strips), `test_trim_with_args_strips_specific_chars` FAILS. + +- [ ] **Step 3: Implement** + +Edit `backend/src/trove/indexers/cardigann.py`, add to `_apply_filter`: + +```python + if name == "trim": + if isinstance(args, str) and args: + return value.strip(args) + return value.strip() +``` + +- [ ] **Step 4: Run — expect pass** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "trim"` +Expected: both PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py +git commit -m "feat: cardigann trim filter" +``` + +--- + +## Task 12: Add `querystring` filter + +**Files:** +- Modify: `backend/tests/test_cardigann_filters.py` +- Modify: `backend/src/trove/indexers/cardigann.py` + +- [ ] **Step 1: Write the failing test** + +Append: + +```python +def test_querystring_extract_param() -> None: + drv = _driver_with_field( + "infohash", + [{"name": "querystring", "args": "id"}], + ) + assert _apply( + drv, + 'link', + "infohash", + ) is None # spec: driver extracts from text — this case pulls text of the cell, not href. + + +def test_querystring_on_href_value() -> None: + drv = _driver_with_field( + "infohash", + [{"name": "querystring", "args": "id"}], + ) + # When combined with attribute:"href", the filter receives the URL string. + definition = CardigannDefinition( + site="t", + name="t", + links=["https://t.local"], + search_path="/", + search_params={}, + rows_selector="tr", + fields={ + "infohash": FieldSpec( + selector="a", + attribute="href", + filters=[{"name": "querystring", "args": "id"}], + ) + }, + ) + drv = CardigannIndexer(definition) + row = BeautifulSoup( + 'link', + "lxml", + ).find("tr") + assert drv._extract_field(row, "infohash") == "abc123" +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "querystring"` +Expected: `test_querystring_on_href_value` FAILS (returns the full URL unchanged). + +- [ ] **Step 3: Implement** + +Add to `_apply_filter`: + +```python + if name == "querystring" and isinstance(args, str): + from urllib.parse import parse_qs, urlparse + parsed = urlparse(value) + params = parse_qs(parsed.query) + picks = params.get(args) + return picks[0] if picks else value +``` + +- [ ] **Step 4: Run — expect pass** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py -v -k "querystring"` +Expected: both PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py +git commit -m "feat: cardigann querystring filter" +``` + +--- + +## Task 13: Warn once per process on unknown filter names + +**Files:** +- Modify: `backend/src/trove/indexers/cardigann.py` + +- [ ] **Step 1: Write the failing test** + +Append to `backend/tests/test_cardigann_filters.py`: + +```python +def test_unknown_filter_logs_warning(caplog) -> None: + import logging + drv = _driver_with_field( + "title", + [{"name": "definitelynotarealfilter"}], + ) + with caplog.at_level(logging.WARNING): + result = _apply(drv, "hello", "title") + assert result == "hello" + assert any( + "definitelynotarealfilter" in rec.message for rec in caplog.records + ), "expected a warning mentioning the unknown filter name" +``` + +- [ ] **Step 2: Run — expect fail** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py::test_unknown_filter_logs_warning -v` +Expected: FAIL — no warning logged. + +- [ ] **Step 3: Implement** + +Edit `backend/src/trove/indexers/cardigann.py`. Near the top of the file, below the imports, add: + +```python +import logging + +log = logging.getLogger(__name__) +_WARNED_FILTERS: set[str] = set() +``` + +Modify the final `return value` at the end of `_apply_filter` to warn on unknown names: + +```python + if name and name not in _WARNED_FILTERS: + _WARNED_FILTERS.add(name) + log.warning("cardigann: unknown filter %r — passing value through unchanged", name) + return value +``` + +- [ ] **Step 4: Run — expect pass** + +Run: `cd backend && uv run pytest tests/test_cardigann_filters.py::test_unknown_filter_logs_warning -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/tests/test_cardigann_filters.py backend/src/trove/indexers/cardigann.py +git commit -m "feat: cardigann warns once per unknown filter name" +``` + +--- + +## Task 14: Add `CatalogEntryOut` Pydantic model + +**Files:** +- Create: `backend/src/trove/api/catalog.py` + +- [ ] **Step 1: Scaffold the router module** + +Create `backend/src/trove/api/catalog.py`: + +```python +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, Field +from sqlmodel import Session, select + +from trove.api.deps import current_user, db_session +from trove.api.indexers import IndexerOut, _to_out +from trove.clients.base import Protocol +from trove.indexers.base import Category +from trove.indexers.cardigann import load_definition_yaml +from trove.models.indexer import IndexerRow +from trove.models.user import User +from trove.services import catalog, indexer_registry + +router = APIRouter() + + +class CatalogEntryOut(BaseModel): + slug: str + display_name: str + description: str + categories: list[Category] + mirrors: list[str] + default_mirror: str + protocol: Protocol + logo: str | None = None + already_installed: bool + + +class CatalogInstallRequest(BaseModel): + base_url: str = Field(min_length=1, max_length=512) + name: str | None = Field(default=None, max_length=64) + + +@router.get("", response_model=list[CatalogEntryOut]) +async def list_catalog( + session: Session = Depends(db_session), + _user: User = Depends(current_user), +) -> list[CatalogEntryOut]: + installed_slugs = set( + session.exec( + select(IndexerRow.catalog_slug).where(IndexerRow.catalog_slug.is_not(None)) # type: ignore[attr-defined] + ).all() + ) + out: list[CatalogEntryOut] = [] + for entry in catalog.list_entries(): + out.append( + CatalogEntryOut( + slug=entry.slug, + display_name=entry.display_name, + description=entry.description, + categories=entry.categories, + mirrors=entry.mirrors, + default_mirror=entry.default_mirror, + protocol=entry.protocol, + logo=entry.logo, + already_installed=entry.slug in installed_slugs, + ) + ) + return out +``` + +- [ ] **Step 2: Import-check** + +Run: `cd backend && uv run python -c "from trove.api import catalog; print(catalog.router.routes)"` +Expected: prints one `APIRoute` (the GET handler). + +- [ ] **Step 3: Commit** + +```bash +git add backend/src/trove/api/catalog.py +git commit -m "feat: scaffold catalog API router with GET /catalog" +``` + +--- + +## Task 15: Implement `POST /api/indexers/catalog/{slug}` + +**Files:** +- Modify: `backend/src/trove/api/catalog.py` + +- [ ] **Step 1: Add the endpoint** + +Append to `backend/src/trove/api/catalog.py`, below the GET handler: + +```python +def _dedup_name(session: Session, base: str) -> str: + candidate = base + suffix = 2 + while session.exec(select(IndexerRow).where(IndexerRow.name == candidate)).first() is not None: + candidate = f"{base}-{suffix}" + suffix += 1 + return candidate + + +@router.post("/{slug}", response_model=IndexerOut, status_code=status.HTTP_201_CREATED) +async def install_catalog_entry( + slug: str, + payload: CatalogInstallRequest, + session: Session = Depends(db_session), + _user: User = Depends(current_user), +) -> IndexerOut: + try: + entry = catalog.get_entry(slug) + except KeyError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="unknown_slug") from None + + if payload.base_url not in entry.mirrors: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="base_url_not_in_catalog_mirrors", + ) + + try: + yaml_text = catalog.read_yaml(slug) + load_definition_yaml(yaml_text) + except Exception as e: # noqa: BLE001 + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"catalog_yaml_broken: {e}", + ) from e + + base_name = payload.name or entry.display_name + name = _dedup_name(session, base_name) + + row = IndexerRow( + name=name, + type="cardigann", + protocol=entry.protocol.value, + base_url=payload.base_url, + credentials_cipher=indexer_registry.encrypt_credentials({}), + definition_yaml=yaml_text, + enabled=True, + priority=50, + catalog_slug=slug, + ) + session.add(row) + session.commit() + session.refresh(row) + return _to_out(row) +``` + +- [ ] **Step 2: Mount the router** + +Modify `backend/src/trove/main.py` — find the block that includes the `indexers` router, and directly after it, include the catalog router under the catalog-specific prefix. + +Open the file, search for `from trove.api.indexers import router as indexers_router`, and add alongside it: + +```python +from trove.api.catalog import router as catalog_router +``` + +Then in the `app.include_router(indexers_router, prefix="/api/indexers", ...)` call region, add: + +```python +app.include_router(catalog_router, prefix="/api/indexers/catalog", tags=["catalog"]) +``` + +- [ ] **Step 3: Sanity check — start the app headless** + +Run: `cd backend && uv run python -c " +from trove.main import create_app +app = create_app() +paths = sorted(r.path for r in app.routes if hasattr(r, 'path')) +print([p for p in paths if 'catalog' in p]) +"` +Expected: prints `['/api/indexers/catalog', '/api/indexers/catalog/{slug}']`. + +- [ ] **Step 4: Commit** + +```bash +git add backend/src/trove/api/catalog.py backend/src/trove/main.py +git commit -m "feat: POST /api/indexers/catalog/{slug} installs a catalog entry" +``` + +--- + +## Task 16: API tests for catalog endpoints + +**Files:** +- Create: `backend/tests/api/test_catalog_api.py` + +- [ ] **Step 1: Write the tests** + +Create `backend/tests/api/test_catalog_api.py`: + +```python +from __future__ import annotations + +from fastapi.testclient import TestClient + + +def _login(client: TestClient) -> None: + client.post( + "/api/auth/setup", + json={"username": "admin", "password": "correct horse battery staple"}, + ) + + +def test_list_catalog_returns_entries(client: TestClient) -> None: + _login(client) + resp = client.get("/api/indexers/catalog") + assert resp.status_code == 200 + body = resp.json() + assert len(body) >= 12 + slugs = {e["slug"] for e in body} + assert "thepiratebay" in slugs + tpb = next(e for e in body if e["slug"] == "thepiratebay") + assert tpb["already_installed"] is False + assert tpb["default_mirror"] in tpb["mirrors"] + + +def test_install_catalog_entry_creates_indexer(client: TestClient) -> None: + _login(client) + resp = client.post( + "/api/indexers/catalog/thepiratebay", + json={"base_url": "https://thepiratebay.org", "name": None}, + ) + assert resp.status_code == 201 + body = resp.json() + assert body["type"] == "cardigann" + assert body["base_url"] == "https://thepiratebay.org" + assert body["name"] == "The Pirate Bay" + + # already_installed flips on a subsequent list + listing = client.get("/api/indexers/catalog").json() + tpb = next(e for e in listing if e["slug"] == "thepiratebay") + assert tpb["already_installed"] is True + + +def test_install_twice_dedups_name(client: TestClient) -> None: + _login(client) + first = client.post( + "/api/indexers/catalog/thepiratebay", + json={"base_url": "https://thepiratebay.org"}, + ) + second = client.post( + "/api/indexers/catalog/thepiratebay", + json={"base_url": "https://tpb.party"}, + ) + assert first.status_code == 201 + assert second.status_code == 201 + assert first.json()["name"] == "The Pirate Bay" + assert second.json()["name"] == "The Pirate Bay-2" + + +def test_install_unknown_slug_404(client: TestClient) -> None: + _login(client) + resp = client.post( + "/api/indexers/catalog/not-a-real-site", + json={"base_url": "https://example.com"}, + ) + assert resp.status_code == 404 + + +def test_install_rejects_base_url_not_in_mirrors(client: TestClient) -> None: + _login(client) + resp = client.post( + "/api/indexers/catalog/thepiratebay", + json={"base_url": "https://totally-evil-mirror.example.com"}, + ) + assert resp.status_code == 422 + assert resp.json()["detail"] == "base_url_not_in_catalog_mirrors" +``` + +- [ ] **Step 2: Run** + +Run: `cd backend && uv run pytest tests/api/test_catalog_api.py -v` +Expected: all 5 tests PASS. + +- [ ] **Step 3: Commit** + +```bash +git add backend/tests/api/test_catalog_api.py +git commit -m "test: catalog API endpoints" +``` + +--- + +## Task 17: Expose catalog endpoints in frontend `api.ts` + +**Files:** +- Modify: `web/src/lib/api.ts` + +- [ ] **Step 1: Add the TypeScript type** + +Edit `web/src/lib/api.ts` — after the `IndexerHealthOut` type (around line 131), add: + +```typescript +export type CatalogEntryOut = { + slug: string; + display_name: string; + description: string; + categories: Category[]; + mirrors: string[]; + default_mirror: string; + protocol: Protocol; + logo: string | null; + already_installed: boolean; +}; + +export type CatalogInstallRequest = { + base_url: string; + name?: string | null; +}; +``` + +- [ ] **Step 2: Add the API methods** + +Inside the `api.indexers` object literal (around line 483), add a `catalog` sub-object. The diff, line-for-line: + +```typescript + indexers: { + list: () => request("/api/indexers"), + health: () => request("/api/indexers/health"), + create: (payload: IndexerCreate) => + request("/api/indexers", { + method: "POST", + body: JSON.stringify(payload) + }), + update: (id: number, payload: Partial) => + request(`/api/indexers/${id}`, { + method: "PATCH", + body: JSON.stringify(payload) + }), + remove: (id: number) => request(`/api/indexers/${id}`, { method: "DELETE" }), + test: (id: number) => + request(`/api/indexers/${id}/test`, { method: "POST" }), + catalog: { + list: () => request("/api/indexers/catalog"), + install: (slug: string, payload: CatalogInstallRequest) => + request(`/api/indexers/catalog/${slug}`, { + method: "POST", + body: JSON.stringify(payload) + }) + } + }, +``` + +- [ ] **Step 3: Typecheck** + +Run: `cd web && pnpm check` +Expected: no TypeScript errors. + +- [ ] **Step 4: Commit** + +```bash +git add web/src/lib/api.ts +git commit -m "feat(web): catalog types + api.indexers.catalog methods" +``` + +--- + +## Task 18: Create `/indexers/catalog/+page.svelte` + +**Files:** +- Create: `web/src/routes/indexers/catalog/+page.svelte` + +- [ ] **Step 1: Write the page** + +Create `web/src/routes/indexers/catalog/+page.svelte`: + +```svelte + + +
+
+ + Indexers + +
+

Catalog

+

+ One-click install for public, no-account torrent sites. +

+
+
+ + {#if loading} +
+ Loading… +
+ {:else if errorMsg} +
+ {errorMsg} +
+ {:else} +
+ {#each entries as entry (entry.slug)} +
+
+
+ +
+
+
{entry.display_name}
+
+ {#each entry.categories as cat (cat)} + + {cat} + + {/each} +
+
+
+

{entry.description}

+ + + + {#if entry.already_installed} + + {:else} + + {/if} + + {#if perEntryError[entry.slug]} +
{perEntryError[entry.slug]}
+ {/if} +
+ {/each} +
+ {/if} +
+``` + +- [ ] **Step 2: Typecheck** + +Run: `cd web && pnpm check` +Expected: no errors. + +- [ ] **Step 3: Start backend + frontend, smoke-test by hand** + +Run (two terminals): +- Backend: `cd backend && uv run uvicorn trove.main:app --reload` +- Frontend: `cd web && pnpm dev` + +Browse to `http://localhost:5173/indexers/catalog`. Log in if prompted. Verify all 12 tiles render with mirror dropdowns and an enabled **Add** button. Click **Add** on one site. Verify the button flips to **Installed**. Visit `/indexers` — the new row appears with the correct name and URL. + +- [ ] **Step 4: Commit** + +```bash +git add web/src/routes/indexers/catalog/+page.svelte +git commit -m "feat(web): /indexers/catalog tile grid for catalog entries" +``` + +--- + +## Task 19: Add "Browse catalog" button on `/indexers` + +**Files:** +- Modify: `web/src/routes/indexers/+page.svelte:243-249` + +- [ ] **Step 1: Add the button** + +Edit `web/src/routes/indexers/+page.svelte` — replace the single header-action block: + +```svelte + +``` + +with a two-button cluster: + +```svelte +
+ + Browse catalog + + +
+``` + +(`Database` is already imported from `lucide-svelte` in this file; no new import required.) + +- [ ] **Step 2: Visual smoke-test** + +With both servers still running, refresh `/indexers`. Verify the new **Browse catalog** button appears next to **Add indexer** and navigates to `/indexers/catalog`. + +- [ ] **Step 3: Commit** + +```bash +git add web/src/routes/indexers/+page.svelte +git commit -m "feat(web): browse catalog button on /indexers" +``` + +--- + +## Task 20: Add onboarding step for catalog + +**Files:** +- Modify: `web/src/routes/onboarding/+page.svelte` + +- [ ] **Step 1: Add the new step to the Step union** + +Edit line 28 — change: + +```typescript + type Step = "welcome" | "client" | "indexer" | "ai" | "tmdb" | "done"; +``` + +to: + +```typescript + type Step = "welcome" | "client" | "indexer" | "catalog" | "ai" | "tmdb" | "done"; +``` + +- [ ] **Step 2: Add state + helpers for the step** + +In the ` + +
+
+ + Indexers + +
+

Catalog

+

+ One-click install for public, no-account torrent sites. +

+
+
+ + {#if loading} +
+ Loading… +
+ {:else if errorMsg} +
+ {errorMsg} +
+ {:else} +
+ {#each entries as entry (entry.slug)} +
+
+
+ +
+
+
{entry.display_name}
+
+ {#each entry.categories as cat (cat)} + + {cat} + + {/each} +
+
+
+

{entry.description}

+ + + + {#if entry.already_installed} + + {:else} + + {/if} + + {#if perEntryError[entry.slug]} +
{perEntryError[entry.slug]}
+ {/if} +
+ {/each} +
+ {/if} +
diff --git a/web/src/routes/onboarding/+page.svelte b/web/src/routes/onboarding/+page.svelte index 1dc3f5f..b386d7b 100644 --- a/web/src/routes/onboarding/+page.svelte +++ b/web/src/routes/onboarding/+page.svelte @@ -7,7 +7,8 @@ type IndexerType, type Protocol, type DownloadClientOut, - type IndexerOut + type IndexerOut, + type CatalogEntryOut } from "$lib/api"; import { CLIENT_TYPES } from "$lib/clientTypes"; import { @@ -25,7 +26,7 @@ Trash2 } from "lucide-svelte"; - type Step = "welcome" | "client" | "indexer" | "ai" | "tmdb" | "done"; + type Step = "welcome" | "client" | "indexer" | "catalog" | "ai" | "tmdb" | "done"; let step = $state("welcome"); let clients = $state([]); @@ -78,6 +79,61 @@ let indexerError = $state(null); let indexerJustSaved = $state(null); + // Catalog step state + let catalogEntries = $state([]); + let catalogLoading = $state(false); + let catalogSelected = $state>(new Set()); + let catalogInstalling = $state(false); + let catalogError = $state(null); + let catalogLoaded = $state(false); + + $effect(() => { + if (step === "catalog") loadCatalog(); + }); + + async function loadCatalog() { + if (catalogLoaded || catalogLoading) return; + catalogLoading = true; + try { + catalogEntries = await api.indexers.catalog.list(); + catalogLoaded = true; + } catch (e) { + const err = e as { detail?: string }; + catalogError = err.detail ?? "Failed to load catalog."; + } finally { + catalogLoading = false; + } + } + + function toggleCatalog(slug: string) { + const next = new Set(catalogSelected); + if (next.has(slug)) next.delete(slug); + else next.add(slug); + catalogSelected = next; + } + + async function installSelectedCatalog() { + catalogInstalling = true; + catalogError = null; + for (const entry of catalogEntries) { + if (!catalogSelected.has(entry.slug) || entry.already_installed) continue; + try { + await api.indexers.catalog.install(entry.slug, { + base_url: entry.default_mirror, + name: null + }); + } catch (e) { + const err = e as { detail?: string }; + catalogError = `${entry.display_name}: ${err.detail ?? "install failed"}`; + catalogInstalling = false; + return; + } + } + catalogInstalling = false; + indexers = await api.indexers.list(); + step = "ai"; + } + // AI state let aiTesting = $state(false); let aiResult = $state<{ ok: boolean; msg: string } | null>(null); @@ -242,6 +298,7 @@ { key: "welcome", label: "Welcome", icon: Sparkles }, { key: "client", label: "Download client", icon: Download }, { key: "indexer", label: "Indexer", icon: Database }, + { key: "catalog", label: "Public sites", icon: Database }, { key: "ai", label: "AI", icon: Sparkles }, { key: "tmdb", label: "Discover", icon: Sparkles }, { key: "done", label: "Done", icon: PartyPopper } @@ -584,7 +641,7 @@ @@ -663,7 +720,7 @@ @@ -680,6 +737,83 @@ {/if} + {#if step === "catalog"} +
+
+
+

+ Public torrent sites (optional) +

+

+ Pick any public sites you want Trove to search. You can always add more later. +

+
+
+ + {#if catalogLoading} +
Loading catalog…
+ {:else} +
+ {#each catalogEntries as entry (entry.slug)} + + {/each} +
+ {/if} + + {#if catalogError} +
+ {catalogError} +
+ {/if} + +
+ + +
+
+ {/if} + {#if step === "ai"}