From 209b4fd337572ae5002d5a39ba72af392e9f47d7 Mon Sep 17 00:00:00 2001 From: Heavy Harlow <89617161+Plungis@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:42:06 -0400 Subject: [PATCH 01/82] Include torrent ID in MaM download requests --- mlm_mam/src/api.rs | 3 ++- server/src/torrent_downloader.rs | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mlm_mam/src/api.rs b/mlm_mam/src/api.rs index eb7ad66d..716fd18b 100644 --- a/mlm_mam/src/api.rs +++ b/mlm_mam/src/api.rs @@ -191,12 +191,13 @@ impl<'a> MaM<'a> { } } - pub async fn get_torrent_file(&self, dl_hash: &str) -> Result { + pub async fn get_torrent_file(&self, dl_hash: &str, mam_id: u64) -> Result { let resp = self .client .get(format!( "https://www.myanonamouse.net/tor/download.php/{dl_hash}" )) + .query(&[("tid", mam_id)]) .send() .await? .error_for_status() diff --git a/server/src/torrent_downloader.rs b/server/src/torrent_downloader.rs index b404ed12..6e1c276f 100644 --- a/server/src/torrent_downloader.rs +++ b/server/src/torrent_downloader.rs @@ -113,7 +113,7 @@ async fn grab_torrent( ); let user_info = mam.user_info().await?; - let torrent_file_bytes = get_mam_torrent_file(mam, &torrent.dl_link).await?; + let torrent_file_bytes = get_mam_torrent_file(mam, &torrent.dl_link, torrent.mam_id).await?; let torrent_file = Torrent::read_from_bytes(torrent_file_bytes.clone())?; let hash = torrent_file.info_hash(); @@ -336,9 +336,13 @@ async fn get_existing_qbit_torrent( None } -pub(crate) async fn get_mam_torrent_file(mam: &MaM<'_>, dl_link: &str) -> Result { +pub(crate) async fn get_mam_torrent_file( + mam: &MaM<'_>, + dl_link: &str, + mam_id: u64, +) -> Result { loop { - let result = mam.get_torrent_file(dl_link).await; + let result = mam.get_torrent_file(dl_link, mam_id).await; match result { Ok(v) => return Ok(v), From 6381bfe98b9322900f41699229d8676145b81c67 Mon Sep 17 00:00:00 2001 From: Heavy Harlow <89617161+Plungis@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:13:10 -0400 Subject: [PATCH 02/82] Start compatibility-first Python migration --- .gitignore | 4 + python/README.md | 33 ++++ python/pyproject.toml | 23 +++ python/src/mlm/__init__.py | 3 + python/src/mlm/__main__.py | 3 + python/src/mlm/cli.py | 46 ++++++ python/src/mlm/config.py | 115 ++++++++++++++ python/src/mlm/database.py | 99 ++++++++++++ python/src/mlm/downloader.py | 60 ++++++++ python/src/mlm/mam.py | 99 ++++++++++++ python/src/mlm/migration.py | 266 +++++++++++++++++++++++++++++++++ python/src/mlm/qbittorrent.py | 79 ++++++++++ python/src/mlm/repository.py | 119 +++++++++++++++ python/src/mlm/torrent.py | 58 +++++++ python/tests/test_clients.py | 27 ++++ python/tests/test_config.py | 30 ++++ python/tests/test_migration.py | 120 +++++++++++++++ python/tests/test_torrent.py | 12 ++ server/src/autograbber.rs | 8 +- server/src/exporter.rs | 99 ++++++++---- server/src/main.rs | 14 +- 21 files changed, 1284 insertions(+), 33 deletions(-) create mode 100644 python/README.md create mode 100644 python/pyproject.toml create mode 100644 python/src/mlm/__init__.py create mode 100644 python/src/mlm/__main__.py create mode 100644 python/src/mlm/cli.py create mode 100644 python/src/mlm/config.py create mode 100644 python/src/mlm/database.py create mode 100644 python/src/mlm/downloader.py create mode 100644 python/src/mlm/mam.py create mode 100644 python/src/mlm/migration.py create mode 100644 python/src/mlm/qbittorrent.py create mode 100644 python/src/mlm/repository.py create mode 100644 python/src/mlm/torrent.py create mode 100644 python/tests/test_clients.py create mode 100644 python/tests/test_config.py create mode 100644 python/tests/test_migration.py create mode 100644 python/tests/test_torrent.py diff --git a/.gitignore b/.gitignore index 74d73b8a..4d590352 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ docs/book config.toml data.db data.db.* +__pycache__/ +.pytest_cache/ +*.py[cod] +*.egg-info/ diff --git a/python/README.md b/python/README.md new file mode 100644 index 00000000..71bcbf6d --- /dev/null +++ b/python/README.md @@ -0,0 +1,33 @@ +# MLM Python + +This directory contains the compatibility-first Python migration of MLM. + +The first implemented slice is a lossless database migration: + +1. The legacy Rust executable exports its `native_db` database to versioned JSON. +2. The Python migrator backs up the original database. +3. Every record is inserted into SQLite with its complete canonical JSON payload. +4. Record counts and SQLite integrity are validated before the temporary database + atomically replaces the destination. + +## Migrate an existing database + +Build the legacy exporter once: + +```powershell +cargo build --release -p mlm +``` + +Then run: + +```powershell +cd python +python -m pip install -e . +mlm-python migrate ` + --source-db "$env:LOCALAPPDATA\MLM\data.db" ` + --destination "$env:LOCALAPPDATA\MLM\data.sqlite3" ` + --legacy-executable "..\target\release\mlm.exe" +``` + +The command exports from the backup copy, not the live database. You can also +pass `--export-json path\to\export.json` instead of `--legacy-executable`. diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 00000000..0e4104b1 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" + +[project] +name = "mlm-python" +version = "0.1.0a1" +description = "Python migration of Myanonamouse Library Manager" +readme = "README.md" +requires-python = ">=3.11" +license = { text = "MIT" } +authors = [{ name = "MLM contributors" }] +dependencies = ["httpx>=0.27,<1"] + +[project.scripts] +mlm-python = "mlm.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/mlm"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] diff --git a/python/src/mlm/__init__.py b/python/src/mlm/__init__.py new file mode 100644 index 00000000..355089dd --- /dev/null +++ b/python/src/mlm/__init__.py @@ -0,0 +1,3 @@ +"""Python implementation of Myanonamouse Library Manager.""" + +__version__ = "0.1.0a1" diff --git a/python/src/mlm/__main__.py b/python/src/mlm/__main__.py new file mode 100644 index 00000000..eb53e2f3 --- /dev/null +++ b/python/src/mlm/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/python/src/mlm/cli.py b/python/src/mlm/cli.py new file mode 100644 index 00000000..2bdfd3cf --- /dev/null +++ b/python/src/mlm/cli.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from .migration import MigrationError, migrate + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="mlm-python") + subparsers = parser.add_subparsers(dest="command", required=True) + migration = subparsers.add_parser( + "migrate", help="back up and migrate a legacy native_db database to SQLite" + ) + migration.add_argument("--source-db", required=True, type=Path) + migration.add_argument("--destination", required=True, type=Path) + source = migration.add_mutually_exclusive_group(required=True) + source.add_argument("--legacy-executable", type=Path) + source.add_argument("--export-json", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + result = migrate( + args.source_db, + args.destination, + export_json=args.export_json, + legacy_executable=args.legacy_executable, + ) + except MigrationError as error: + print(f"Migration failed: {error}", file=sys.stderr) + return 1 + + print(f"Migrated database: {result.destination}") + print(f"Original database backup: {result.source_backup}") + for table, count in result.counts.items(): + print(f" {table}: {count}") + print(f"Export SHA-256: {result.export_sha256}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/src/mlm/config.py b/python/src/mlm/config.py new file mode 100644 index 00000000..ecb10e0e --- /dev/null +++ b/python/src/mlm/config.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import os +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping + + +class ConfigError(ValueError): + pass + + +@dataclass(frozen=True) +class QbitConfig: + url: str + username: str = "" + password: str = "" + on_cleaned: dict[str, Any] | None = None + on_invalid_torrent: dict[str, Any] | None = None + path_mapping: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class Config: + mam_id: str + web_host: str = "0.0.0.0" + web_port: int = 3157 + min_ratio: float = 2.0 + unsat_buffer: int = 10 + wedge_buffer: int = 0 + add_torrents_stopped: bool = False + exclude_narrator_in_library_dir: bool = False + search_interval: int = 30 + link_interval: int = 10 + import_interval: int = 135 + ignore_torrents: tuple[int, ...] = () + audio_types: tuple[str, ...] = ("m4b", "m4a", "mp4", "mp3", "ogg") + ebook_types: tuple[str, ...] = ("cbz", "epub", "pdf", "mobi", "azw3", "azw", "cbr") + music_types: tuple[str, ...] = ("pdf", "mp3") + radio_types: tuple[str, ...] = ("mp3",) + qbittorrent: tuple[QbitConfig, ...] = () + search: dict[str, Any] = field(default_factory=dict) + audiobookshelf: dict[str, Any] | None = None + autograbs: tuple[dict[str, Any], ...] = () + snatchlist: tuple[dict[str, Any], ...] = () + goodreads_lists: tuple[dict[str, Any], ...] = () + notion_lists: tuple[dict[str, Any], ...] = () + tags: tuple[dict[str, Any], ...] = () + libraries: tuple[dict[str, Any], ...] = () + + +_ALIASES = { + "goodreads_interval": "import_interval", + "autograb": "autograbs", + "goodreads_list": "goodreads_lists", + "notion_list": "notion_lists", + "tag": "tags", + "library": "libraries", +} + + +def _environment_overrides(environment: Mapping[str, str]) -> dict[str, Any]: + overrides: dict[str, Any] = {} + for key, value in environment.items(): + if not key.startswith("MLM_CONF_"): + continue + name = key.removeprefix("MLM_CONF_").lower() + try: + overrides[name] = tomllib.loads(f"value = {value}")["value"] + except tomllib.TOMLDecodeError: + overrides[name] = value + return overrides + + +def load_config( + path: Path, *, environment: Mapping[str, str] | None = None +) -> Config: + try: + raw = tomllib.loads(path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as error: + raise ConfigError(f"could not read config {path}: {error}") from error + for old, new in _ALIASES.items(): + if old in raw and new not in raw: + raw[new] = raw.pop(old) + raw.update(_environment_overrides(os.environ if environment is None else environment)) + + allowed = set(Config.__dataclass_fields__) + unknown = sorted(set(raw) - allowed) + if unknown: + raise ConfigError(f"unknown configuration fields: {', '.join(unknown)}") + if "mam_id" not in raw: + raise ConfigError("missing required configuration field: mam_id") + + qbit_rows = raw.pop("qbittorrent", []) + try: + qbit = tuple(QbitConfig(**row) for row in qbit_rows) + tuple_fields = { + "ignore_torrents", + "audio_types", + "ebook_types", + "music_types", + "radio_types", + "autograbs", + "snatchlist", + "goodreads_lists", + "notion_lists", + "tags", + "libraries", + } + for name in tuple_fields & raw.keys(): + raw[name] = tuple(raw[name]) + return Config(qbittorrent=qbit, **raw) + except (TypeError, ValueError) as error: + raise ConfigError(f"invalid configuration: {error}") from error diff --git a/python/src/mlm/database.py b/python/src/mlm/database.py new file mode 100644 index 00000000..b748be01 --- /dev/null +++ b/python/src/mlm/database.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +SCHEMA_VERSION = 1 + +SCHEMA = """ +PRAGMA foreign_keys = ON; + +CREATE TABLE migration_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + payload_json TEXT NOT NULL +); + +CREATE TABLE torrents ( + id TEXT PRIMARY KEY, + mam_id INTEGER NOT NULL UNIQUE, + title_search TEXT NOT NULL, + created_at_json TEXT, + payload_json TEXT NOT NULL +); + +CREATE TABLE selected_torrents ( + mam_id INTEGER PRIMARY KEY, + hash TEXT UNIQUE, + title_search TEXT NOT NULL, + created_at_json TEXT, + payload_json TEXT NOT NULL +); + +CREATE TABLE duplicate_torrents ( + mam_id INTEGER PRIMARY KEY, + title_search TEXT NOT NULL, + created_at_json TEXT, + payload_json TEXT NOT NULL +); + +CREATE TABLE errored_torrents ( + id_json TEXT PRIMARY KEY, + created_at_json TEXT, + payload_json TEXT NOT NULL +); + +CREATE TABLE events ( + id_json TEXT PRIMARY KEY, + torrent_id TEXT, + mam_id INTEGER, + created_at_json TEXT, + payload_json TEXT NOT NULL +); + +CREATE INDEX events_torrent_id_idx ON events(torrent_id); +CREATE INDEX events_mam_id_idx ON events(mam_id); + +CREATE TABLE lists ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + payload_json TEXT NOT NULL +); + +CREATE TABLE list_items ( + guid_json TEXT PRIMARY KEY, + list_id TEXT NOT NULL, + title TEXT NOT NULL, + created_at_json TEXT, + payload_json TEXT NOT NULL +); + +CREATE INDEX list_items_list_id_idx ON list_items(list_id); +""" + +DATA_TABLES = ( + "config", + "torrents", + "selected_torrents", + "duplicate_torrents", + "errored_torrents", + "events", + "lists", + "list_items", +) + + +def connect(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(path) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + return connection + + +def initialize(connection: sqlite3.Connection) -> None: + connection.executescript(SCHEMA) diff --git a/python/src/mlm/downloader.py b/python/src/mlm/downloader.py new file mode 100644 index 00000000..341f219f --- /dev/null +++ b/python/src/mlm/downloader.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from .config import Config +from .mam import MamClient, MamRateLimitError +from .qbittorrent import QbitClient +from .repository import Repository +from .torrent import info_hash + + +@dataclass(frozen=True) +class DownloadRun: + downloaded: int = 0 + failed: int = 0 + skipped: int = 0 + + +async def _torrent_file_with_backoff( + mam: MamClient, download_hash: str, torrent_id: int +) -> bytes: + delay = 30 + while True: + try: + return await mam.get_torrent_file(download_hash, torrent_id) + except MamRateLimitError: + await asyncio.sleep(delay) + delay = min(delay * 2, 300) + + +async def grab_selected_torrents( + config: Config, + repository: Repository, + mam: MamClient, + qbit: QbitClient, +) -> DownloadRun: + downloaded = failed = skipped = 0 + for selected in repository.pending_selected(): + try: + torrent_id = int(selected["mam_id"]) + torrent_file = await _torrent_file_with_backoff( + mam, selected["dl_link"], torrent_id + ) + torrent_hash = info_hash(torrent_file) + existing = await qbit.torrents(hashes=[torrent_hash]) + if not existing: + await qbit.add_torrent( + torrent_file, + category=selected.get("category"), + tags=selected.get("tags", []), + paused=config.add_torrents_stopped, + ) + repository.record_started(selected, torrent_hash) + downloaded += 1 + except Exception as error: + repository.record_grab_error(selected, error) + failed += 1 + await asyncio.sleep(1) + return DownloadRun(downloaded=downloaded, failed=failed, skipped=skipped) diff --git a/python/src/mlm/mam.py b/python/src/mlm/mam.py new file mode 100644 index 00000000..24b57460 --- /dev/null +++ b/python/src/mlm/mam.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from typing import Any + +import httpx + + +class MamError(RuntimeError): + pass + + +class MamRateLimitError(MamError): + pass + + +class MamClient: + BASE_URL = "https://www.myanonamouse.net" + + def __init__( + self, + mam_id: str, + *, + client: httpx.AsyncClient | None = None, + timeout: float = 20, + ) -> None: + self._owns_client = client is None + self.client = client or httpx.AsyncClient( + base_url=self.BASE_URL, + headers={"User-Agent": "MLM"}, + timeout=timeout, + follow_redirects=True, + ) + self.client.cookies.set("mam_id", mam_id, domain="www.myanonamouse.net") + + async def __aenter__(self) -> MamClient: + return self + + async def __aexit__(self, *_: object) -> None: + await self.close() + + async def close(self) -> None: + if self._owns_client: + await self.client.aclose() + + @staticmethod + def _raise_for_status(response: httpx.Response) -> None: + if response.status_code == 429: + raise MamRateLimitError("Myanonamouse rate limit reached") + try: + response.raise_for_status() + except httpx.HTTPStatusError as error: + raise MamError(str(error)) from error + + async def check_mam_id(self) -> None: + response = await self.client.get("/json/checkCookie.php") + self._raise_for_status(response) + if '"Success":true' not in response.text: + raise MamError("session check failed (Success was false)") + + async def user_info(self) -> dict[str, Any]: + response = await self.client.get("/jsonLoad.php", params={"snatch_summary": "true"}) + self._raise_for_status(response) + return response.json() + + async def search(self, query: dict[str, Any]) -> dict[str, Any]: + response = await self.client.post( + "/tor/js/loadSearchJSONbasic.php", json=query + ) + self._raise_for_status(response) + result = response.json() + if isinstance(result, dict) and result.get("error"): + if result["error"] == "Nothing returned, out of 0": + return {"data": [], "found": 0} + raise MamError(str(result["error"])) + return result + + async def get_torrent_info_by_id(self, torrent_id: int) -> dict[str, Any] | None: + result = await self.search( + { + "tor": {"id": torrent_id}, + "fields": { + "description": True, + "mediaInfo": True, + "isbn": True, + "dlLink": True, + }, + } + ) + rows = result.get("data", []) + return rows[-1] if rows else None + + async def get_torrent_file(self, download_hash: str, torrent_id: int) -> bytes: + """Download a .torrent; MaM requires the numeric tid query argument.""" + response = await self.client.get( + f"/tor/download.php/{download_hash}", + params={"tid": torrent_id}, + ) + self._raise_for_status(response) + return response.content diff --git a/python/src/mlm/migration.py b/python/src/mlm/migration.py new file mode 100644 index 00000000..1a1d9d8a --- /dev/null +++ b/python/src/mlm/migration.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sqlite3 +import subprocess +import tempfile +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .database import DATA_TABLES, SCHEMA_VERSION, connect, initialize + +EXPORT_FORMAT = "mlm-native-db-export" +EXPORT_VERSION = 1 + + +class MigrationError(RuntimeError): + """Raised when a migration cannot be proven complete and consistent.""" + + +@dataclass(frozen=True) +class MigrationResult: + destination: Path + source_backup: Path + counts: dict[str, int] + export_sha256: str + + +def canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _timestamped_backup_path(source: Path) -> Path: + stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + candidate = source.with_name(f"{source.name}.{stamp}.bak") + suffix = 1 + while candidate.exists(): + candidate = source.with_name(f"{source.name}.{stamp}.{suffix}.bak") + suffix += 1 + return candidate + + +def back_up_source(source: Path) -> Path: + source = source.resolve() + if not source.is_file(): + raise MigrationError(f"source database does not exist: {source}") + backup = _timestamped_backup_path(source) + shutil.copy2(source, backup) + return backup + + +def export_legacy_database(executable: Path, database_backup: Path, output: Path) -> None: + executable = executable.resolve() + if not executable.is_file(): + raise MigrationError(f"legacy executable does not exist: {executable}") + + with tempfile.TemporaryDirectory(prefix="mlm-export-config-") as temp_dir: + config_path = Path(temp_dir) / "config.toml" + config_path.write_text('mam_id = ""\n', encoding="utf-8") + environment = os.environ.copy() + environment["MLM_DB_FILE"] = str(database_backup) + environment["MLM_CONFIG_FILE"] = str(config_path) + completed = subprocess.run( + [str(executable), "--export-db", str(output)], + env=environment, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() or "no error output" + raise MigrationError(f"legacy export failed ({completed.returncode}): {detail}") + if not output.is_file(): + raise MigrationError("legacy exporter reported success but produced no JSON file") + + +def _load_export(path: Path) -> tuple[dict[str, Any], str]: + raw = path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + try: + document = json.loads(raw) + except json.JSONDecodeError as error: + raise MigrationError(f"invalid export JSON: {error}") from error + if not isinstance(document, dict): + raise MigrationError("export root must be a JSON object") + if document.get("format") != EXPORT_FORMAT: + raise MigrationError(f"unsupported export format: {document.get('format')!r}") + if document.get("version") != EXPORT_VERSION: + raise MigrationError(f"unsupported export version: {document.get('version')!r}") + + for table in DATA_TABLES: + if not isinstance(document.get(table), list): + raise MigrationError(f"export field {table!r} must be an array") + declared = document.get("counts") + actual = {table: len(document[table]) for table in DATA_TABLES} + if declared != actual: + raise MigrationError(f"export count mismatch: declared={declared!r}, actual={actual!r}") + return document, digest + + +def _json_field(record: dict[str, Any], key: str) -> str | None: + value = record.get(key) + return None if value is None else canonical_json(value) + + +def _insert_records(connection: sqlite3.Connection, export: dict[str, Any]) -> None: + for row in export["config"]: + connection.execute( + "INSERT INTO config(key, value, payload_json) VALUES (?, ?, ?)", + (row["key"], row["value"], canonical_json(row)), + ) + for row in export["torrents"]: + connection.execute( + """INSERT INTO torrents + (id, mam_id, title_search, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?)""", + ( + row["id"], + row["mam_id"], + row["title_search"], + _json_field(row, "created_at"), + canonical_json(row), + ), + ) + for row in export["selected_torrents"]: + connection.execute( + """INSERT INTO selected_torrents + (mam_id, hash, title_search, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?)""", + ( + row["mam_id"], + row.get("hash"), + row["title_search"], + _json_field(row, "created_at"), + canonical_json(row), + ), + ) + for row in export["duplicate_torrents"]: + connection.execute( + """INSERT INTO duplicate_torrents + (mam_id, title_search, created_at_json, payload_json) + VALUES (?, ?, ?, ?)""", + ( + row["mam_id"], + row["title_search"], + _json_field(row, "created_at"), + canonical_json(row), + ), + ) + for row in export["errored_torrents"]: + connection.execute( + """INSERT INTO errored_torrents + (id_json, created_at_json, payload_json) VALUES (?, ?, ?)""", + ( + canonical_json(row["id"]), + _json_field(row, "created_at"), + canonical_json(row), + ), + ) + for row in export["events"]: + connection.execute( + """INSERT INTO events + (id_json, torrent_id, mam_id, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?)""", + ( + canonical_json(row["id"]), + row.get("torrent_id"), + row.get("mam_id"), + _json_field(row, "created_at"), + canonical_json(row), + ), + ) + for row in export["lists"]: + connection.execute( + "INSERT INTO lists(id, title, payload_json) VALUES (?, ?, ?)", + (row["id"], row["title"], canonical_json(row)), + ) + for row in export["list_items"]: + connection.execute( + """INSERT INTO list_items + (guid_json, list_id, title, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?)""", + ( + canonical_json(row["guid"]), + row["list_id"], + row["title"], + _json_field(row, "created_at"), + canonical_json(row), + ), + ) + + +def _validate(connection: sqlite3.Connection, expected: dict[str, int]) -> None: + actual = { + table: connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + for table in DATA_TABLES + } + if actual != expected: + raise MigrationError(f"SQLite count mismatch: expected={expected!r}, actual={actual!r}") + result = connection.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise MigrationError(f"SQLite integrity check failed: {result}") + + +def migrate( + source_database: Path, + destination: Path, + *, + export_json: Path | None = None, + legacy_executable: Path | None = None, +) -> MigrationResult: + source_database = source_database.resolve() + destination = destination.resolve() + if source_database == destination: + raise MigrationError("source and destination must be different files") + if destination.exists(): + raise MigrationError(f"destination already exists: {destination}") + if (export_json is None) == (legacy_executable is None): + raise MigrationError("provide exactly one of export_json or legacy_executable") + + destination.parent.mkdir(parents=True, exist_ok=True) + backup = back_up_source(source_database) + + with tempfile.TemporaryDirectory(prefix="mlm-migration-", dir=destination.parent) as temp_dir: + temp_dir_path = Path(temp_dir) + if legacy_executable is not None: + selected_export = temp_dir_path / "legacy-export.json" + export_legacy_database(legacy_executable, backup, selected_export) + else: + assert export_json is not None + selected_export = export_json.resolve() + if not selected_export.is_file(): + raise MigrationError(f"export JSON does not exist: {selected_export}") + + export, digest = _load_export(selected_export) + expected = {table: len(export[table]) for table in DATA_TABLES} + temporary_database = temp_dir_path / "data.sqlite3" + connection = connect(temporary_database) + try: + initialize(connection) + with connection: + _insert_records(connection, export) + metadata = { + "schema_version": str(SCHEMA_VERSION), + "migrated_at": datetime.now(UTC).isoformat(), + "source_database": str(source_database), + "source_backup": str(backup), + "export_sha256": digest, + "source_counts": canonical_json(expected), + } + connection.executemany( + "INSERT INTO migration_meta(key, value) VALUES (?, ?)", + metadata.items(), + ) + _validate(connection, expected) + except (KeyError, TypeError, ValueError, sqlite3.Error) as error: + raise MigrationError(f"could not import export: {error}") from error + finally: + connection.close() + os.replace(temporary_database, destination) + + return MigrationResult(destination, backup, expected, digest) diff --git a/python/src/mlm/qbittorrent.py b/python/src/mlm/qbittorrent.py new file mode 100644 index 00000000..e788d3de --- /dev/null +++ b/python/src/mlm/qbittorrent.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from typing import Iterable + +import httpx + + +class QbitError(RuntimeError): + pass + + +class QbitClient: + def __init__( + self, + url: str, + *, + client: httpx.AsyncClient | None = None, + timeout: float = 20, + ) -> None: + self._owns_client = client is None + self.client = client or httpx.AsyncClient( + base_url=url.rstrip("/"), timeout=timeout + ) + + async def __aenter__(self) -> QbitClient: + return self + + async def __aexit__(self, *_: object) -> None: + await self.close() + + async def close(self) -> None: + if self._owns_client: + await self.client.aclose() + + @staticmethod + def _check(response: httpx.Response) -> httpx.Response: + try: + response.raise_for_status() + except httpx.HTTPStatusError as error: + raise QbitError(str(error)) from error + if response.text.strip() == "Fails.": + raise QbitError("qBittorrent rejected the request") + return response + + async def login(self, username: str = "", password: str = "") -> None: + response = await self.client.post( + "/api/v2/auth/login", + data={"username": username, "password": password}, + ) + self._check(response) + + async def add_torrent( + self, + torrent_file: bytes, + *, + category: str | None = None, + tags: Iterable[str] = (), + paused: bool = False, + ) -> None: + data: dict[str, str] = {"paused": str(paused).lower()} + if category: + data["category"] = category + tags_value = ",".join(tags) + if tags_value: + data["tags"] = tags_value + response = await self.client.post( + "/api/v2/torrents/add", + data=data, + files={"torrents": ("download.torrent", torrent_file, "application/x-bittorrent")}, + ) + self._check(response) + + async def torrents(self, *, hashes: Iterable[str] = ()) -> list[dict]: + hashes_value = "|".join(hashes) + params = {"hashes": hashes_value} if hashes_value else None + response = self._check( + await self.client.get("/api/v2/torrents/info", params=params) + ) + return response.json() diff --git a/python/src/mlm/repository.py b/python/src/mlm/repository.py new file mode 100644 index 00000000..cdc9f699 --- /dev/null +++ b/python/src/mlm/repository.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import json +import sqlite3 +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from .database import connect +from .migration import canonical_json + + +class Repository: + def __init__(self, path: Path) -> None: + self.path = path + + def pending_selected(self) -> list[dict[str, Any]]: + with connect(self.path) as connection: + rows = connection.execute( + """SELECT payload_json FROM selected_torrents + WHERE json_extract(payload_json, '$.started_at') IS NULL + AND json_extract(payload_json, '$.removed_at') IS NULL + ORDER BY mam_id""" + ) + return [json.loads(row[0]) for row in rows] + + def record_started( + self, selected: dict[str, Any], torrent_hash: str, *, wedged: bool = False + ) -> None: + now = datetime.now(UTC).isoformat() + selected = dict(selected) + selected["hash"] = torrent_hash + selected["started_at"] = now + torrent = { + "id": torrent_hash, + "id_is_hash": True, + "mam_id": selected["mam_id"], + "abs_id": None, + "goodreads_id": selected.get("goodreads_id"), + "library_path": None, + "library_files": [], + "linker": None, + "category": selected.get("category"), + "selected_audio_format": None, + "selected_ebook_format": None, + "title_search": selected["title_search"], + "meta": selected.get("meta", {}), + "created_at": now, + "replaced_with": None, + "request_matadata_update": False, + "library_mismatch": None, + "client_status": None, + } + event = { + "id": str(uuid4()), + "torrent_id": torrent_hash, + "mam_id": selected["mam_id"], + "created_at": now, + "event": { + "Grabbed": { + "grabber": selected.get("grabber"), + "cost": selected.get("cost"), + "wedged": wedged, + } + }, + } + with connect(self.path) as connection: + with connection: + connection.execute( + """UPDATE selected_torrents + SET hash = ?, payload_json = ? WHERE mam_id = ?""", + (torrent_hash, canonical_json(selected), selected["mam_id"]), + ) + connection.execute( + """INSERT INTO torrents + (id, mam_id, title_search, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET payload_json=excluded.payload_json""", + ( + torrent_hash, + selected["mam_id"], + selected["title_search"], + canonical_json(now), + canonical_json(torrent), + ), + ) + connection.execute( + """INSERT INTO events + (id_json, torrent_id, mam_id, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?)""", + ( + canonical_json(event["id"]), + torrent_hash, + selected["mam_id"], + canonical_json(now), + canonical_json(event), + ), + ) + + def record_grab_error(self, selected: dict[str, Any], error: Exception) -> None: + now = datetime.now(UTC).isoformat() + identifier = {"Grabber": selected["mam_id"]} + row = { + "id": identifier, + "title": selected.get("meta", {}).get("title", selected["title_search"]), + "error": str(error), + "meta": selected.get("meta"), + "created_at": now, + } + with connect(self.path) as connection: + connection.execute( + """INSERT INTO errored_torrents + (id_json, created_at_json, payload_json) VALUES (?, ?, ?) + ON CONFLICT(id_json) DO UPDATE SET + created_at_json=excluded.created_at_json, + payload_json=excluded.payload_json""", + (canonical_json(identifier), canonical_json(now), canonical_json(row)), + ) diff --git a/python/src/mlm/torrent.py b/python/src/mlm/torrent.py new file mode 100644 index 00000000..6f979acf --- /dev/null +++ b/python/src/mlm/torrent.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import hashlib + + +class TorrentFormatError(ValueError): + pass + + +def _skip_value(data: bytes, offset: int) -> int: + if offset >= len(data): + raise TorrentFormatError("unexpected end of bencoded data") + marker = data[offset : offset + 1] + if marker == b"i": + end = data.find(b"e", offset + 1) + if end < 0: + raise TorrentFormatError("unterminated integer") + int(data[offset + 1 : end]) + return end + 1 + if marker == b"l": + offset += 1 + while data[offset : offset + 1] != b"e": + offset = _skip_value(data, offset) + return offset + 1 + if marker == b"d": + offset += 1 + while data[offset : offset + 1] != b"e": + offset = _skip_value(data, offset) + offset = _skip_value(data, offset) + return offset + 1 + if b"0" <= marker <= b"9": + colon = data.find(b":", offset) + if colon < 0: + raise TorrentFormatError("invalid byte string") + length = int(data[offset:colon]) + end = colon + 1 + length + if end > len(data): + raise TorrentFormatError("truncated byte string") + return end + raise TorrentFormatError(f"invalid bencode marker at byte {offset}") + + +def info_hash(torrent_file: bytes) -> str: + """Return the BitTorrent v1 SHA-1 info hash without re-encoding the payload.""" + if not torrent_file.startswith(b"d"): + raise TorrentFormatError("torrent root is not a dictionary") + offset = 1 + while torrent_file[offset : offset + 1] != b"e": + key_start = offset + key_end = _skip_value(torrent_file, key_start) + colon = torrent_file.find(b":", key_start) + key = torrent_file[colon + 1 : key_end] + value_start = key_end + value_end = _skip_value(torrent_file, value_start) + if key == b"info": + return hashlib.sha1(torrent_file[value_start:value_end]).hexdigest() + offset = value_end + raise TorrentFormatError("torrent has no info dictionary") diff --git a/python/tests/test_clients.py b/python/tests/test_clients.py new file mode 100644 index 00000000..eb1b982a --- /dev/null +++ b/python/tests/test_clients.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import asyncio + +import httpx + +from mlm.mam import MamClient + + +def test_torrent_download_always_includes_tid() -> None: + observed: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + observed.append(request) + return httpx.Response(200, content=b"torrent bytes") + + async def exercise() -> bytes: + async with httpx.AsyncClient( + base_url="https://www.myanonamouse.net", + transport=httpx.MockTransport(handler), + ) as http: + mam = MamClient("cookie", client=http) + return await mam.get_torrent_file("download-hash", 123456) + + assert asyncio.run(exercise()) == b"torrent bytes" + assert observed[0].url.path == "/tor/download.php/download-hash" + assert observed[0].url.params["tid"] == "123456" diff --git a/python/tests/test_config.py b/python/tests/test_config.py new file mode 100644 index 00000000..ff221266 --- /dev/null +++ b/python/tests/test_config.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from mlm.config import load_config + + +def test_loads_legacy_names_and_defaults(tmp_path: Path) -> None: + path = tmp_path / "config.toml" + path.write_text( + """ +mam_id = "secret" +goodreads_interval = 90 + +[[qbittorrent]] +url = "http://localhost:8080" + +[[autograb]] +type = "freeleech" +dry_run = true +""", + encoding="utf-8", + ) + + config = load_config(path) + + assert config.import_interval == 90 + assert config.web_port == 3157 + assert config.qbittorrent[0].url == "http://localhost:8080" + assert config.autograbs[0]["dry_run"] is True diff --git a/python/tests/test_migration.py b/python/tests/test_migration.py new file mode 100644 index 00000000..df6c9169 --- /dev/null +++ b/python/tests/test_migration.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path + +import pytest + +from mlm.migration import MigrationError, canonical_json, migrate + + +def sample_export() -> dict: + collections = { + "config": [{"key": "last-run", "value": "now"}], + "torrents": [ + { + "id": "abc", + "mam_id": 101, + "title_search": "a book", + "created_at": [1700000000, 0], + "meta": {"title": "A Book", "authors": ["Writer"]}, + } + ], + "selected_torrents": [ + { + "mam_id": 102, + "hash": None, + "title_search": "queued", + "created_at": [1700000001, 0], + "dl_link": "https://example.invalid/download?tid=102", + } + ], + "duplicate_torrents": [], + "errored_torrents": [ + { + "id": {"Grabber": 103}, + "created_at": [1700000002, 0], + "title": "Failed", + "error": "example", + "meta": None, + } + ], + "events": [ + { + "id": "event-id", + "torrent_id": "abc", + "mam_id": 101, + "created_at": [1700000003, 0], + "event": {"RemovedFromMam": None}, + } + ], + "lists": [{"id": "list-1", "title": "Reading"}], + "list_items": [ + { + "guid": ["list-1", "item-1"], + "list_id": "list-1", + "title": "Wanted", + "created_at": [1700000004, 0], + } + ], + } + return { + "format": "mlm-native-db-export", + "version": 1, + "counts": {key: len(value) for key, value in collections.items()}, + **collections, + } + + +def write_export(path: Path, document: dict) -> None: + path.write_text(json.dumps(document), encoding="utf-8") + + +def test_migration_preserves_payloads_and_counts(tmp_path: Path) -> None: + source = tmp_path / "data.db" + source.write_bytes(b"legacy database") + export_path = tmp_path / "export.json" + document = sample_export() + write_export(export_path, document) + destination = tmp_path / "data.sqlite3" + + result = migrate(source, destination, export_json=export_path) + + assert result.source_backup.read_bytes() == b"legacy database" + assert result.counts == document["counts"] + with sqlite3.connect(destination) as connection: + payload = connection.execute( + "SELECT payload_json FROM torrents WHERE id = 'abc'" + ).fetchone()[0] + assert payload == canonical_json(document["torrents"][0]) + assert connection.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + + +def test_count_mismatch_does_not_create_destination(tmp_path: Path) -> None: + source = tmp_path / "data.db" + source.write_bytes(b"legacy database") + export_path = tmp_path / "export.json" + document = sample_export() + document["counts"]["events"] = 99 + write_export(export_path, document) + destination = tmp_path / "data.sqlite3" + + with pytest.raises(MigrationError, match="export count mismatch"): + migrate(source, destination, export_json=export_path) + + assert not destination.exists() + + +def test_existing_destination_is_never_overwritten(tmp_path: Path) -> None: + source = tmp_path / "data.db" + source.write_bytes(b"legacy database") + destination = tmp_path / "data.sqlite3" + destination.write_bytes(b"keep me") + export_path = tmp_path / "export.json" + write_export(export_path, sample_export()) + + with pytest.raises(MigrationError, match="destination already exists"): + migrate(source, destination, export_json=export_path) + + assert destination.read_bytes() == b"keep me" diff --git a/python/tests/test_torrent.py b/python/tests/test_torrent.py new file mode 100644 index 00000000..8157af92 --- /dev/null +++ b/python/tests/test_torrent.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import hashlib + +from mlm.torrent import info_hash + + +def test_info_hash_uses_original_info_bytes() -> None: + info = b"d6:lengthi12e4:name4:booke" + torrent = b"d4:info" + info + b"e" + + assert info_hash(torrent) == hashlib.sha1(info).hexdigest() diff --git a/server/src/autograbber.rs b/server/src/autograbber.rs index 66b520aa..2e522126 100644 --- a/server/src/autograbber.rs +++ b/server/src/autograbber.rs @@ -789,7 +789,9 @@ async fn update_selected_torrent_meta( .map(|field| format!(" {}: {} -> {}", field.field, field.from, field.to)) .join("\n") ); - let hash = get_mam_torrent_hash(mam, &torrent.dl_link).await.ok(); + let hash = get_mam_torrent_hash(mam, &torrent.dl_link, mam_id) + .await + .ok(); let mut torrent = torrent; torrent.meta = meta; rw.upsert(torrent)?; @@ -803,8 +805,8 @@ async fn update_selected_torrent_meta( Ok(()) } -pub async fn get_mam_torrent_hash(mam: &MaM<'_>, dl_link: &str) -> Result { - let torrent_file_bytes = get_mam_torrent_file(mam, dl_link).await?; +pub async fn get_mam_torrent_hash(mam: &MaM<'_>, dl_link: &str, mam_id: u64) -> Result { + let torrent_file_bytes = get_mam_torrent_file(mam, dl_link, mam_id).await?; let torrent_file = Torrent::read_from_bytes(torrent_file_bytes.clone())?; let hash = torrent_file.info_hash(); Ok(hash) diff --git a/server/src/exporter.rs b/server/src/exporter.rs index 3b05639c..a28b93ca 100644 --- a/server/src/exporter.rs +++ b/server/src/exporter.rs @@ -1,6 +1,8 @@ use std::{ + collections::BTreeMap, fs::File, io::{BufWriter, Write as _}, + path::Path, }; use anyhow::Result; @@ -10,45 +12,88 @@ use serde::{Deserialize, Serialize}; #[allow(unused)] #[derive(Serialize, Deserialize, Debug)] struct ExportV1 { + format: &'static str, + version: u32, + counts: BTreeMap<&'static str, usize>, config: Vec, torrents: Vec, selected_torrents: Vec, duplicate_torrents: Vec, errored_torrents: Vec, + events: Vec, + lists: Vec, + list_items: Vec, } #[allow(unused)] -pub fn export_db(db: &Database<'_>) -> Result<()> { +pub fn export_db(db: &Database<'_>, output_path: &Path) -> Result<()> { let r = db.r_transaction()?; + let config = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + let torrents = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + let selected_torrents = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + let duplicate_torrents = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + let errored_torrents = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + let events = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + let lists = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + let list_items = r + .scan() + .primary()? + .all()? + .collect::, db_type::Error>>()?; + + let counts = BTreeMap::from([ + ("config", config.len()), + ("torrents", torrents.len()), + ("selected_torrents", selected_torrents.len()), + ("duplicate_torrents", duplicate_torrents.len()), + ("errored_torrents", errored_torrents.len()), + ("events", events.len()), + ("lists", lists.len()), + ("list_items", list_items.len()), + ]); let export = ExportV1 { - config: r - .scan() - .primary()? - .all()? - .collect::>()?, - torrents: r - .scan() - .primary()? - .all()? - .collect::>()?, - selected_torrents: r - .scan() - .primary()? - .all()? - .collect::>()?, - duplicate_torrents: r - .scan() - .primary()? - .all()? - .collect::>()?, - errored_torrents: r - .scan() - .primary()? - .all()? - .collect::>()?, + format: "mlm-native-db-export", + version: 1, + counts, + config, + torrents, + selected_torrents, + duplicate_torrents, + errored_torrents, + events, + lists, + list_items, }; - let file = File::create("export_v1.json")?; + let file = File::create(output_path)?; let mut writer = BufWriter::new(file); serde_json::to_writer_pretty(&mut writer, &export)?; writer.flush()?; diff --git a/server/src/main.rs b/server/src/main.rs index 48133cee..055bad6d 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -194,13 +194,21 @@ async fn app_main() -> Result<()> { let db = native_db::Builder::new().create(&mlm_db::MODELS, database_file)?; mlm_db::migrate(&db)?; - if env::args().any(|arg| arg == "--update-search-title") { + let args = env::args().collect::>(); + if let Some(index) = args.iter().position(|arg| arg == "--export-db") { + let output_path = args + .get(index + 1) + .map(PathBuf::from) + .context("--export-db requires an output JSON path")?; + export_db(&db, &output_path)?; + return Ok(()); + } + + if args.iter().any(|arg| arg == "--update-search-title") { mlm_db::update_search_title(&db)?; return Ok(()); } - // export_db(&db)?; - // return Ok(()); let db = Arc::new(db); #[cfg(target_family = "windows")] From 0f006afc2ba8e2f177c3d4972cf22cbf0947a1e9 Mon Sep 17 00:00:00 2001 From: Heavy Harlow <89617161+Plungis@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:13:39 -0400 Subject: [PATCH 03/82] Add runnable Python downloader command --- python/README.md | 12 ++++++++++++ python/src/mlm/cli.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/python/README.md b/python/README.md index 71bcbf6d..c5ad22d0 100644 --- a/python/README.md +++ b/python/README.md @@ -31,3 +31,15 @@ mlm-python migrate ` The command exports from the backup copy, not the live database. You can also pass `--export-json path\to\export.json` instead of `--legacy-executable`. + +## Process migrated pending downloads + +```powershell +mlm-python download ` + --config "$env:APPDATA\MLM\config.toml" ` + --database "$env:LOCALAPPDATA\MLM\data.sqlite3" +``` + +This command currently performs one downloader pass. It validates the MaM +cookie, logs into the first configured qBittorrent server, includes the required +`tid` on every MaM download, and records successes or errors in SQLite. diff --git a/python/src/mlm/cli.py b/python/src/mlm/cli.py index 2bdfd3cf..efac0e51 100644 --- a/python/src/mlm/cli.py +++ b/python/src/mlm/cli.py @@ -1,10 +1,16 @@ from __future__ import annotations import argparse +import asyncio import sys from pathlib import Path +from .config import ConfigError, load_config +from .downloader import grab_selected_torrents +from .mam import MamClient from .migration import MigrationError, migrate +from .qbittorrent import QbitClient +from .repository import Repository def build_parser() -> argparse.ArgumentParser: @@ -18,11 +24,40 @@ def build_parser() -> argparse.ArgumentParser: source = migration.add_mutually_exclusive_group(required=True) source.add_argument("--legacy-executable", type=Path) source.add_argument("--export-json", type=Path) + downloader = subparsers.add_parser( + "download", help="process migrated pending torrents once" + ) + downloader.add_argument("--config", required=True, type=Path) + downloader.add_argument("--database", required=True, type=Path) return parser +async def _download(config_path: Path, database_path: Path) -> int: + config = load_config(config_path) + if not config.qbittorrent: + raise ConfigError("at least one [[qbittorrent]] entry is required") + qbit_config = config.qbittorrent[0] + repository = Repository(database_path) + async with MamClient(config.mam_id) as mam, QbitClient(qbit_config.url) as qbit: + await mam.check_mam_id() + await qbit.login(qbit_config.username, qbit_config.password) + result = await grab_selected_torrents(config, repository, mam, qbit) + print( + f"Download run: {result.downloaded} downloaded, " + f"{result.failed} failed, {result.skipped} skipped" + ) + return 1 if result.failed else 0 + + def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + if args.command == "download": + try: + return asyncio.run(_download(args.config, args.database)) + except (ConfigError, OSError) as error: + print(f"Download failed: {error}", file=sys.stderr) + return 1 + try: result = migrate( args.source_db, From 2cfdc17d7006594c40486feb4a600ade74073500 Mon Sep 17 00:00:00 2001 From: Heavy Harlow <89617161+Plungis@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:26:19 -0400 Subject: [PATCH 04/82] Port autograbber and library organization --- python/src/mlm/autograbber.py | 131 +++++++++++++++++++ python/src/mlm/cleaner.py | 81 ++++++++++++ python/src/mlm/downloader.py | 55 +++++++- python/src/mlm/library.py | 237 +++++++++++++++++++++++++++++++++ python/src/mlm/mam.py | 41 +++++- python/src/mlm/qbittorrent.py | 46 +++++++ python/src/mlm/repository.py | 137 +++++++++++++++++++ python/src/mlm/search.py | 238 ++++++++++++++++++++++++++++++++++ python/tests/test_library.py | 37 ++++++ python/tests/test_search.py | 45 +++++++ 10 files changed, 1040 insertions(+), 8 deletions(-) create mode 100644 python/src/mlm/autograbber.py create mode 100644 python/src/mlm/cleaner.py create mode 100644 python/src/mlm/library.py create mode 100644 python/src/mlm/search.py create mode 100644 python/tests/test_library.py create mode 100644 python/tests/test_search.py diff --git a/python/src/mlm/autograbber.py b/python/src/mlm/autograbber.py new file mode 100644 index 00000000..7772c65f --- /dev/null +++ b/python/src/mlm/autograbber.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from .config import Config +from .mam import MamClient +from .repository import Repository +from .search import as_bool, matches_filter, normalize_title, search_pages, torrent_meta + + +def _preferred_types(config: Config, media_type: str) -> tuple[str, ...]: + if media_type in {"audiobook", "periodical_audiobook"}: + return config.audio_types + if media_type in {"ebook", "manga", "comic_book", "periodical_ebook"}: + return config.ebook_types + if media_type == "musicology": + return config.music_types + if media_type == "radio": + return config.radio_types + return () + + +def _preference(formats: list[str], preferred: tuple[str, ...]) -> int | None: + positions = [preferred.index(fmt) for fmt in formats if fmt in preferred] + return min(positions) if positions else None + + +def _cost(row: dict[str, Any], requested: str) -> str: + if as_bool(row.get("vip")): + return "Vip" + if as_bool(row.get("personal_freeleech")): + return "PersonalFreeleech" + if as_bool(row.get("free")): + return "GlobalFreeleech" + if requested == "wedge": + return "UseWedge" + if requested == "try_wedge": + return "TryWedge" + return "Ratio" + + +def _tagging(config: Config, row: dict[str, Any]) -> tuple[str | None, list[str]]: + category = None + tags: list[str] = [] + for rule in config.tags: + if matches_filter(row, rule): + category = category or rule.get("category") + tags.extend(str(tag) for tag in rule.get("tags", [])) + return category, list(dict.fromkeys(tags)) + + +async def run_autograbber( + config: Config, + repository: Repository, + mam: MamClient, + rule: dict[str, Any], + *, + index: int = 0, +) -> int: + user = await mam.user_info() + unsat = user.get("unsat", {}) + available = max(0, int(unsat.get("limit", 0)) - int(unsat.get("count", 0))) + maximum = max(0, available - int(rule.get("unsat_buffer", config.unsat_buffer))) + if rule.get("max_active_downloads") is not None: + active = sum( + selected.get("grabber") == rule.get("name", str(index)) + for selected in repository.pending_selected() + ) + maximum = min(maximum, max(0, int(rule["max_active_downloads"]) - active)) + if maximum == 0 and rule.get("cost", "free") not in {"metadata_only", "metadata_only_add"}: + return 0 + + selected_count = 0 + async for row in search_pages(mam, rule): + torrent_id = int(row.get("id", 0)) + if not torrent_id or torrent_id in config.ignore_torrents: + continue + if not matches_filter(row, rule) or repository.has_mam_id(torrent_id): + continue + requested_cost = rule.get("cost", "free") + if requested_cost == "free" and not any( + as_bool(row.get(field)) for field in ("vip", "personal_freeleech", "free", "fl_vip") + ): + continue + if requested_cost in {"metadata_only", "metadata_only_add"}: + continue + + meta = torrent_meta(row) + title_search = normalize_title(meta["title"]) + preferred = _preferred_types(config, meta["media_type"]) + preference = _preference(meta["filetypes"], preferred) + if preference is None: + continue + duplicate = False + for existing in repository.records_with_title(title_search): + old_meta = existing.get("meta", {}) + old_preference = _preference(old_meta.get("filetypes", []), preferred) + if old_preference is not None and old_preference <= preference: + duplicate = True + break + category, tags = _tagging(config, row) + candidate = { + "mam_id": torrent_id, + "goodreads_id": None, + "hash": None, + "dl_link": row.get("dl"), + "unsat_buffer": rule.get("unsat_buffer"), + "wedge_buffer": rule.get("wedge_buffer"), + "cost": _cost(row, requested_cost), + "category": rule.get("category") or category, + "tags": tags, + "title_search": title_search, + "meta": meta, + "grabber": rule.get("name", str(index)), + "created_at": datetime.now(UTC).isoformat(), + "started_at": None, + "removed_at": None, + } + if duplicate: + if not rule.get("dry_run", False): + repository.add_duplicate(candidate) + continue + if not candidate["dl_link"]: + continue + if not rule.get("dry_run", False): + repository.add_selected(candidate) + selected_count += 1 + if selected_count >= maximum: + break + return selected_count diff --git a/python/src/mlm/cleaner.py b/python/src/mlm/cleaner.py new file mode 100644 index 00000000..590c8af7 --- /dev/null +++ b/python/src/mlm/cleaner.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path + +from .config import Config +from .qbittorrent import QbitClient +from .repository import Repository + + +def _preference(config: Config, torrent: dict) -> int: + media = torrent.get("meta", {}).get("media_type") + preferred = ( + config.audio_types + if media in {"audiobook", "periodical_audiobook", "Audiobook", "PeriodicalAudiobook"} + else config.ebook_types + ) + formats = torrent.get("meta", {}).get("filetypes", []) + positions = [preferred.index(value) for value in formats if value in preferred] + return min(positions) if positions else len(preferred) + 1 + + +def remove_library_files(torrent: dict) -> None: + library_path_value = torrent.get("library_path") + if not library_path_value: + return + library_path = Path(library_path_value) + for relative in torrent.get("library_files", []): + path = library_path / relative + path.unlink(missing_ok=True) + parent = path.parent + while parent != library_path: + try: + parent.rmdir() + except OSError: + break + parent = parent.parent + remaining = list(library_path.iterdir()) if library_path.exists() else [] + if all(path.name in {"cover.jpg", "metadata.json"} for path in remaining): + for path in remaining: + path.unlink(missing_ok=True) + try: + library_path.rmdir() + except OSError: + pass + + +async def clean_superseded( + config: Config, + repository: Repository, + qbit_clients: list[tuple[dict, QbitClient]], +) -> int: + grouped: dict[str, list[dict]] = {} + for torrent in repository.library_torrents(): + grouped.setdefault(torrent["title_search"], []).append(torrent) + cleaned = 0 + for rows in grouped.values(): + if len(rows) < 2: + continue + rows.sort(key=lambda row: (_preference(config, row), -sum( + (Path(row["library_path"]) / file).stat().st_size + for file in row.get("library_files", []) + if (Path(row["library_path"]) / file).exists() + ))) + keep, *remove_rows = rows + for torrent in remove_rows: + for qbit_config, qbit in qbit_clients: + update = qbit_config.get("on_cleaned") + if not update or not torrent.get("id_is_hash"): + continue + if update.get("category"): + await qbit.set_category([torrent["id"]], update["category"]) + await qbit.add_tags([torrent["id"]], update.get("tags", [])) + remove_library_files(torrent) + torrent["replaced_with"] = [keep["id"], keep["created_at"]] + torrent["library_path"] = None + torrent["library_files"] = [] + torrent["library_mismatch"] = None + torrent["abs_id"] = None + repository.update_torrent(torrent) + cleaned += 1 + return cleaned diff --git a/python/src/mlm/downloader.py b/python/src/mlm/downloader.py index 341f219f..3629dc5d 100644 --- a/python/src/mlm/downloader.py +++ b/python/src/mlm/downloader.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from .config import Config -from .mam import MamClient, MamRateLimitError +from .mam import MamClient, MamRateLimitError, MamWedgeError from .qbittorrent import QbitClient from .repository import Repository from .torrent import info_hash @@ -36,14 +36,64 @@ async def grab_selected_torrents( qbit: QbitClient, ) -> DownloadRun: downloaded = failed = skipped = 0 + user = await mam.user_info() + unsat = user.get("unsat", {}) + available_slots = max( + 0, int(unsat.get("limit", 0)) - int(unsat.get("count", 0)) + ) + downloading_size = sum( + int(row.get("meta", {}).get("size", 0)) + for row in repository.pending_selected() + if row.get("started_at") is not None + ) + remaining_buffer = ( + float(user.get("uploaded_bytes", 0)) + - float(user.get("downloaded_bytes", 0)) + - downloading_size + ) / config.min_ratio for selected in repository.pending_selected(): try: torrent_id = int(selected["mam_id"]) + slot_buffer = int( + selected.get("unsat_buffer") + if selected.get("unsat_buffer") is not None + else config.unsat_buffer + ) + size = int(selected.get("meta", {}).get("size", 0)) + if available_slots - downloaded <= slot_buffer or remaining_buffer - size <= 0: + skipped += 1 + continue torrent_file = await _torrent_file_with_backoff( mam, selected["dl_link"], torrent_id ) torrent_hash = info_hash(torrent_file) existing = await qbit.torrents(hashes=[torrent_hash]) + wedged = False + cost = selected.get("cost") + if not existing and cost in {"UseWedge", "TryWedge"}: + wedge_buffer = int( + selected.get("wedge_buffer") + if selected.get("wedge_buffer") is not None + else config.wedge_buffer + ) + if int(user.get("wedges", 0)) <= wedge_buffer: + raise MamWedgeError( + f"fewer wedges than configured wedge buffer ({wedge_buffer})" + ) + try: + await mam.wedge_torrent(torrent_id) + wedged = True + user["wedges"] = max(0, int(user.get("wedges", 0)) - 1) + except MamWedgeError: + if cost == "UseWedge": + raise + elif not existing and cost != "Ratio": + current = await mam.get_torrent_info(torrent_hash) + if not current or not any( + current.get(field) + for field in ("free", "personal_freeleech", "fl_vip", "vip") + ): + raise RuntimeError("torrent is no longer free") if not existing: await qbit.add_torrent( torrent_file, @@ -51,8 +101,9 @@ async def grab_selected_torrents( tags=selected.get("tags", []), paused=config.add_torrents_stopped, ) - repository.record_started(selected, torrent_hash) + repository.record_started(selected, torrent_hash, wedged=wedged) downloaded += 1 + remaining_buffer -= size except Exception as error: repository.record_grab_error(selected, error) failed += 1 diff --git a/python/src/mlm/library.py b/python/src/mlm/library.py new file mode 100644 index 00000000..67d0c205 --- /dev/null +++ b/python/src/mlm/library.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import json +import os +import re +import shutil +from datetime import UTC, datetime +from pathlib import Path, PurePath +from typing import Any + +from .config import Config, QbitConfig +from .mam import MamClient +from .qbittorrent import QbitClient +from .repository import Repository +from .search import normalize_title, torrent_meta + +INVALID_FILENAME = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +DISC_PATTERN = re.compile(r"(?:CD|Disc|Disk)\s*(\d+)", re.I) + + +def sanitize_filename(value: str) -> str: + cleaned = INVALID_FILENAME.sub("_", value).strip().rstrip(". ") + return cleaned or "_" + + +def map_path(path_mapping: dict[str, str], save_path: str) -> Path: + source = Path(save_path) + matches = sorted( + ( + (Path(old), Path(new)) + for old, new in path_mapping.items() + if source == Path(old) or Path(old) in source.parents + ), + key=lambda pair: len(pair[0].parts), + reverse=True, + ) + if not matches: + return source + old, new = matches[0] + return new.joinpath(source.relative_to(old)) + + +def find_library(config: Config, torrent: dict[str, Any]) -> dict[str, Any] | None: + torrent_tags = { + tag.strip() for tag in str(torrent.get("tags", "")).split(",") if tag.strip() + } + for library in config.libraries: + by_category = "category" in library and torrent.get("category") == library["category"] + by_directory = "download_dir" in library and ( + Path(torrent.get("save_path", "")) == Path(library["download_dir"]) + or Path(library["download_dir"]) in Path(torrent.get("save_path", "")).parents + ) + if not (by_category or by_directory): + continue + if torrent_tags.intersection(library.get("deny_tags", [])): + continue + allowed = set(library.get("allow_tags", [])) + if allowed and not torrent_tags.intersection(allowed): + continue + return library + return None + + +def _series_parts(meta: dict[str, Any]) -> tuple[str, str] | None: + series_rows = meta.get("series", []) + if not series_rows: + return None + series = next( + (row for row in series_rows if row.get("entries")), + series_rows[0], + ) + name = str(series.get("name", "")).strip() + entries = series.get("entries", []) + number = str(entries[0]) if entries else "" + return name, number + + +def library_directory( + exclude_narrator: bool, library: dict[str, Any], meta: dict[str, Any] +) -> Path | None: + authors = meta.get("authors", []) + if not authors: + return None + author = sanitize_filename(str(authors[0])) + title = str(meta.get("title", "")).strip() + series = _series_parts(meta) + if series: + series_name, number = series + leaf = f"{series_name} #{number} - {title}" if number else title + relative = Path(author) / sanitize_filename(series_name) / sanitize_filename(leaf) + else: + relative = Path(author) / sanitize_filename(title) + edition = meta.get("edition") + if edition: + edition_name = edition[0] if isinstance(edition, list) else str(edition) + relative = relative.with_name(sanitize_filename(f"{relative.name}, {edition_name}")) + narrators = meta.get("narrators", []) + if narrators and not exclude_narrator: + relative = relative.with_name( + sanitize_filename(f"{relative.name} {{{narrators[0]}}}") + ) + return Path(library["library_dir"]) / relative + + +def select_format( + override: list[str] | None, preferred: tuple[str, ...], files: list[dict[str, Any]] +) -> str | None: + for extension in override or list(preferred): + suffix = "." + extension.lower().lstrip(".") + if any(str(row.get("name", "")).lower().endswith(suffix) for row in files): + return suffix + return None + + +def safe_torrent_path(name: str) -> Path: + normalized = name.replace("\\", "/") + parts = [part for part in normalized.split("/") if part not in {"", "."}] + if not parts or any(part == ".." for part in parts): + raise ValueError(f"unsafe torrent path: {name!r}") + return Path(*parts) + + +def _destination_relative(torrent_path: Path) -> Path: + parent = torrent_path.parent.name + match = DISC_PATTERN.search(parent) + return ( + Path(f"Disc {match.group(1)}") / torrent_path.name + if match + else Path(torrent_path.name) + ) + + +def _place_file(source: Path, destination: Path, method: str) -> None: + if destination.exists(): + if method.startswith("hardlink") and os.path.samefile(source, destination): + return + raise FileExistsError(f"library file already exists: {destination}") + if method == "hardlink": + os.link(source, destination) + elif method == "hardlink_or_copy": + try: + os.link(source, destination) + except OSError: + shutil.copy2(source, destination) + elif method == "hardlink_or_symlink": + try: + os.link(source, destination) + except OSError: + destination.symlink_to(source) + elif method == "copy": + shutil.copy2(source, destination) + elif method == "symlink": + destination.symlink_to(source) + elif method != "no_link": + raise ValueError(f"unknown library method: {method}") + + +async def organize_completed( + config: Config, + repository: Repository, + qbit_config: QbitConfig, + qbit: QbitClient, + mam: MamClient, +) -> int: + linked = 0 + for qbit_torrent in await qbit.torrents(): + if float(qbit_torrent.get("progress", 0)) < 1: + continue + library = find_library(config, qbit_torrent) + if library is None: + continue + torrent_hash = str(qbit_torrent["hash"]) + existing = repository.torrent(torrent_hash) + if existing and existing.get("library_path"): + continue + files = await qbit.files(torrent_hash) + audio = select_format(library.get("audio_types"), config.audio_types, files) + ebook = select_format(library.get("ebook_types"), config.ebook_types, files) + if not audio and not ebook: + continue + mam_row = await mam.get_torrent_info(torrent_hash) + if not mam_row: + continue + meta = torrent_meta(mam_row) + method = str(library.get("method", "hardlink")) + target_dir = ( + None + if method == "no_link" + else library_directory( + config.exclude_narrator_in_library_dir, library, meta + ) + ) + if method != "no_link" and target_dir is None: + continue + library_files: list[str] = [] + if target_dir is not None: + target_dir.mkdir(parents=True, exist_ok=True) + download_root = map_path(qbit_config.path_mapping, str(qbit_torrent["save_path"])) + for content in files: + torrent_path = safe_torrent_path(str(content["name"])) + lower_name = torrent_path.name.lower() + if not ((audio and lower_name.endswith(audio)) or (ebook and lower_name.endswith(ebook))): + continue + relative = _destination_relative(torrent_path) + destination = target_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + _place_file(download_root / torrent_path, destination, method) + library_files.append(str(relative)) + metadata = {"mam": mam_row, "meta": meta} + (target_dir / "metadata.json").write_text( + json.dumps(metadata, ensure_ascii=False, separators=(",", ":")), + encoding="utf-8", + ) + now = datetime.now(UTC).isoformat() + torrent = { + "id": torrent_hash, + "id_is_hash": True, + "mam_id": meta["mam_id"], + "abs_id": existing.get("abs_id") if existing else None, + "goodreads_id": existing.get("goodreads_id") if existing else None, + "library_path": str(target_dir) if target_dir else None, + "library_files": sorted(library_files), + "linker": library.get("name"), + "category": qbit_torrent.get("category") or None, + "selected_audio_format": audio.lstrip(".") if audio else None, + "selected_ebook_format": ebook.lstrip(".") if ebook else None, + "title_search": normalize_title(meta["title"]), + "meta": meta, + "created_at": existing.get("created_at", now) if existing else now, + "replaced_with": existing.get("replaced_with") if existing else None, + "request_matadata_update": False, + "library_mismatch": None, + "client_status": existing.get("client_status") if existing else None, + } + repository.record_linked(torrent, meta["mam_id"]) + linked += 1 + return linked diff --git a/python/src/mlm/mam.py b/python/src/mlm/mam.py index 24b57460..d4465fba 100644 --- a/python/src/mlm/mam.py +++ b/python/src/mlm/mam.py @@ -1,5 +1,6 @@ from __future__ import annotations +import time from typing import Any import httpx @@ -13,6 +14,10 @@ class MamRateLimitError(MamError): pass +class MamWedgeError(MamError): + pass + + class MamClient: BASE_URL = "https://www.myanonamouse.net" @@ -77,13 +82,22 @@ async def search(self, query: dict[str, Any]) -> dict[str, Any]: async def get_torrent_info_by_id(self, torrent_id: int) -> dict[str, Any] | None: result = await self.search( { + "description": True, + "mediaInfo": True, + "isbn": True, + "dlLink": True, "tor": {"id": torrent_id}, - "fields": { - "description": True, - "mediaInfo": True, - "isbn": True, - "dlLink": True, - }, + } + ) + rows = result.get("data", []) + return rows[-1] if rows else None + + async def get_torrent_info(self, torrent_hash: str) -> dict[str, Any] | None: + result = await self.search( + { + "description": True, + "isbn": True, + "tor": {"hash": torrent_hash}, } ) rows = result.get("data", []) @@ -97,3 +111,18 @@ async def get_torrent_file(self, download_hash: str, torrent_id: int) -> bytes: ) self._raise_for_status(response) return response.content + + async def wedge_torrent(self, torrent_id: int) -> None: + timestamp = int(time.time() * 1000) + response = await self.client.get( + f"/json/bonusBuy.php/{timestamp}", + params={ + "spendtype": "personalFL", + "torrentid": torrent_id, + "timestamp": timestamp, + }, + ) + self._raise_for_status(response) + result = response.json() + if not result.get("success"): + raise MamWedgeError(str(result.get("error") or "unknown wedge error")) diff --git a/python/src/mlm/qbittorrent.py b/python/src/mlm/qbittorrent.py index e788d3de..b66ea47f 100644 --- a/python/src/mlm/qbittorrent.py +++ b/python/src/mlm/qbittorrent.py @@ -77,3 +77,49 @@ async def torrents(self, *, hashes: Iterable[str] = ()) -> list[dict]: await self.client.get("/api/v2/torrents/info", params=params) ) return response.json() + + async def files(self, torrent_hash: str) -> list[dict]: + response = self._check( + await self.client.get( + "/api/v2/torrents/files", params={"hash": torrent_hash} + ) + ) + return response.json() + + async def trackers(self, torrent_hash: str) -> list[dict]: + response = self._check( + await self.client.get( + "/api/v2/torrents/trackers", params={"hash": torrent_hash} + ) + ) + return response.json() + + async def categories(self) -> dict[str, dict]: + response = self._check(await self.client.get("/api/v2/torrents/categories")) + return response.json() + + async def ensure_category(self, category: str) -> None: + if category in await self.categories(): + return + response = await self.client.post( + "/api/v2/torrents/createCategory", data={"category": category} + ) + self._check(response) + + async def set_category(self, hashes: Iterable[str], category: str) -> None: + await self.ensure_category(category) + response = await self.client.post( + "/api/v2/torrents/setCategory", + data={"hashes": "|".join(hashes), "category": category}, + ) + self._check(response) + + async def add_tags(self, hashes: Iterable[str], tags: Iterable[str]) -> None: + tags_value = ",".join(tags) + if not tags_value: + return + response = await self.client.post( + "/api/v2/torrents/addTags", + data={"hashes": "|".join(hashes), "tags": tags_value}, + ) + self._check(response) diff --git a/python/src/mlm/repository.py b/python/src/mlm/repository.py index cdc9f699..6c6acf5a 100644 --- a/python/src/mlm/repository.py +++ b/python/src/mlm/repository.py @@ -25,6 +25,143 @@ def pending_selected(self) -> list[dict[str, Any]]: ) return [json.loads(row[0]) for row in rows] + def has_mam_id(self, mam_id: int) -> bool: + with connect(self.path) as connection: + selected = connection.execute( + "SELECT 1 FROM selected_torrents WHERE mam_id = ?", (mam_id,) + ).fetchone() + library = connection.execute( + "SELECT 1 FROM torrents WHERE mam_id = ?", (mam_id,) + ).fetchone() + return selected is not None or library is not None + + def records_with_title(self, title_search: str) -> list[dict[str, Any]]: + with connect(self.path) as connection: + rows = connection.execute( + """SELECT payload_json FROM selected_torrents WHERE title_search = ? + UNION ALL + SELECT payload_json FROM torrents WHERE title_search = ?""", + (title_search, title_search), + ) + return [json.loads(row[0]) for row in rows] + + def add_selected(self, selected: dict[str, Any]) -> None: + with connect(self.path) as connection: + connection.execute( + """INSERT INTO selected_torrents + (mam_id, hash, title_search, created_at_json, payload_json) + VALUES (?, NULL, ?, ?, ?)""", + ( + selected["mam_id"], + selected["title_search"], + canonical_json(selected["created_at"]), + canonical_json(selected), + ), + ) + + def add_duplicate( + self, torrent: dict[str, Any], duplicate_of: str | None = None + ) -> None: + row = { + "mam_id": torrent["mam_id"], + "dl_link": torrent.get("dl_link"), + "title_search": torrent["title_search"], + "meta": torrent["meta"], + "created_at": datetime.now(UTC).isoformat(), + "duplicate_of": duplicate_of, + } + with connect(self.path) as connection: + connection.execute( + """INSERT INTO duplicate_torrents + (mam_id, title_search, created_at_json, payload_json) + VALUES (?, ?, ?, ?) + ON CONFLICT(mam_id) DO UPDATE SET payload_json=excluded.payload_json""", + ( + row["mam_id"], + row["title_search"], + canonical_json(row["created_at"]), + canonical_json(row), + ), + ) + + def torrent(self, torrent_id: str) -> dict[str, Any] | None: + with connect(self.path) as connection: + row = connection.execute( + "SELECT payload_json FROM torrents WHERE id = ?", (torrent_id,) + ).fetchone() + return json.loads(row[0]) if row else None + + def library_torrents(self) -> list[dict[str, Any]]: + with connect(self.path) as connection: + rows = connection.execute( + """SELECT payload_json FROM torrents + WHERE json_extract(payload_json, '$.library_path') IS NOT NULL + ORDER BY title_search""" + ) + return [json.loads(row[0]) for row in rows] + + def record_linked(self, torrent: dict[str, Any], selected_mam_id: int | None) -> None: + event = { + "id": str(uuid4()), + "torrent_id": torrent["id"], + "mam_id": torrent["mam_id"], + "created_at": datetime.now(UTC).isoformat(), + "event": { + "Linked": { + "linker": torrent.get("linker"), + "library_path": torrent.get("library_path"), + } + }, + } + with connect(self.path) as connection: + with connection: + connection.execute( + """INSERT INTO torrents + (id, mam_id, title_search, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + mam_id=excluded.mam_id, + title_search=excluded.title_search, + payload_json=excluded.payload_json""", + ( + torrent["id"], + torrent["mam_id"], + torrent["title_search"], + canonical_json(torrent["created_at"]), + canonical_json(torrent), + ), + ) + if selected_mam_id is not None: + connection.execute( + "DELETE FROM selected_torrents WHERE mam_id = ?", + (selected_mam_id,), + ) + connection.execute( + """INSERT INTO events + (id_json, torrent_id, mam_id, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?)""", + ( + canonical_json(event["id"]), + event["torrent_id"], + event["mam_id"], + canonical_json(event["created_at"]), + canonical_json(event), + ), + ) + + def update_torrent(self, torrent: dict[str, Any]) -> None: + with connect(self.path) as connection: + connection.execute( + """UPDATE torrents SET mam_id=?, title_search=?, payload_json=? + WHERE id=?""", + ( + torrent["mam_id"], + torrent["title_search"], + canonical_json(torrent), + torrent["id"], + ), + ) + def record_started( self, selected: dict[str, Any], torrent_hash: str, *, wedged: bool = False ) -> None: diff --git a/python/src/mlm/search.py b/python/src/mlm/search.py new file mode 100644 index 00000000..37ee5019 --- /dev/null +++ b/python/src/mlm/search.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import html +import re +import unicodedata +from datetime import datetime +from typing import Any, AsyncIterator + +from .mam import MamClient + +FLAG_BITS = { + "crude_language": 1 << 1, + "crude": 1 << 1, + "language": 1 << 1, + "violence": 1 << 2, + "some_explicit": 1 << 3, + "explicit": 1 << 4, + "abridged": 1 << 5, + "lgbt": 1 << 6, +} + +MEDIA_TYPE_BY_ID = { + 1: "audiobook", + 2: "ebook", + 3: "musicology", + 4: "radio", + 5: "manga", + 6: "comic_book", + 7: "periodical_ebook", + 8: "periodical_audiobook", +} + +MAIN_CATEGORY_BY_ID = {13: "audiobook", 14: "ebook", 15: "musicology", 16: "radio"} + +SIZE_UNITS = { + "b": 1, + "kb": 1_000, + "kib": 1 << 10, + "mb": 1_000_000, + "mib": 1 << 20, + "gb": 1_000_000_000, + "gib": 1 << 30, + "tb": 1_000_000_000_000, + "tib": 1 << 40, +} + + +def as_int(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def as_bool(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() not in {"", "0", "false", "no", "null"} + return bool(value) + + +def parse_size(value: Any) -> int: + if isinstance(value, (int, float)): + return int(value) + match = re.fullmatch(r"\s*([\d.]+)\s*([kmgt]?i?b)?\s*", str(value), re.I) + if not match: + raise ValueError(f"invalid size: {value!r}") + return int(float(match.group(1)) * SIZE_UNITS.get((match.group(2) or "b").lower(), 1)) + + +def normalize_title(value: str) -> str: + ascii_title = ( + unicodedata.normalize("NFKD", html.unescape(value)) + .encode("ascii", "ignore") + .decode() + .lower() + .replace(" & ", " and ") + ) + ascii_title = re.sub(r"^(the|a|an)\s+|[^\w ]", "", ascii_title) + return re.sub(r"(?i)(volume|vol\.)", "", ascii_title) + + +def _mapping_values(value: Any) -> list[str]: + if isinstance(value, dict): + return [str(item) for item in value.values()] + return [] + + +def torrent_meta(row: dict[str, Any]) -> dict[str, Any]: + media_id = as_int(row.get("mediatype")) + main_id = as_int(row.get("main_cat")) + media_type = MEDIA_TYPE_BY_ID.get(media_id) or MAIN_CATEGORY_BY_ID.get(main_id, "unknown") + series = [] + raw_series = row.get("series_info") + if isinstance(raw_series, str): + raw_series = {} + if isinstance(raw_series, dict): + for value in raw_series.values(): + if isinstance(value, list) and value: + series.append({"name": str(value[0]), "entries": value[1:2]}) + filetypes = [part.lower() for part in str(row.get("filetype", "")).split() if part] + return { + "mam_id": as_int(row.get("id")), + "vip_status": "Permanent" if as_bool(row.get("vip")) else "NotVip", + "cat": {"id": as_int(row.get("category")), "name": str(row.get("catname", ""))}, + "media_type": media_type, + "main_cat": as_int(row.get("maincat")) or None, + "categories": [as_int(value) for value in row.get("categories", [])], + "language": as_int(row.get("language")) or None, + "flags": as_int(row.get("browseflags")), + "filetypes": filetypes, + "num_files": as_int(row.get("numfiles")), + "size": parse_size(row.get("size", 0)), + "title": html.unescape(str(row.get("title", ""))), + "edition": None, + "authors": _mapping_values(row.get("author_info")), + "narrators": _mapping_values(row.get("narrator_info")), + "series": series, + "source": "Mam", + "uploaded_at": str(row.get("added", "")), + } + + +def _date(value: Any) -> datetime: + return datetime.strptime(str(value)[:10], "%Y-%m-%d") + + +def matches_filter(row: dict[str, Any], rule: dict[str, Any]) -> bool: + media_types = {str(value).lower() for value in rule.get("media_type", [])} + actual_media = MEDIA_TYPE_BY_ID.get(as_int(row.get("mediatype"))) + if media_types and actual_media not in media_types: + return False + + categories = rule.get("categories", {}) + if categories: + main = MAIN_CATEGORY_BY_ID.get(as_int(row.get("main_cat"))) + category_rule = categories.get( + {"audiobook": "audio"}.get(main, main), False + ) + if category_rule is False: + return False + if isinstance(category_rule, list): + names = {str(value).lower().replace(" ", "_") for value in category_rule} + actual = str(row.get("catname", row.get("cat", ""))).lower().replace(" ", "_") + if actual not in names: + return False + + languages = {str(value).lower() for value in rule.get("languages", [])} + if languages: + actual_language = str(row.get("lang_code", row.get("language", ""))).lower() + if actual_language not in languages: + return False + + bitfield = as_int(row.get("browseflags")) + for name, required in rule.get("flags", {}).items(): + bit = FLAG_BITS.get(name.lower().replace(" ", "_")) + if bit is None or bool(bitfield & bit) != bool(required): + return False + + size = parse_size(row.get("size", 0)) + if rule.get("min_size") and size < parse_size(rule["min_size"]): + return False + if rule.get("max_size") and size > parse_size(rule["max_size"]): + return False + if str(row.get("owner_name", "")) in rule.get("exclude_uploader", []): + return False + if rule.get("uploaded_after") and _date(row.get("added")) < _date(rule["uploaded_after"]): + return False + if rule.get("uploaded_before") and _date(row.get("added")) > _date(rule["uploaded_before"]): + return False + + comparisons = { + "seeders": "seeders", + "leechers": "leechers", + "snatched": "times_completed", + } + for label, field in comparisons.items(): + value = as_int(row.get(field)) + minimum = rule.get(f"min_{label}") + maximum = rule.get(f"max_{label}") + if minimum is not None and value < int(minimum): + return False + if maximum is not None and value > int(maximum): + return False + return True + + +def build_search_query(rule: dict[str, Any], start: int = 0) -> dict[str, Any]: + kind = rule.get("type", "new") + target = None + search_type = None + if kind == "bookmarks": + target = "bookmarks" + elif kind == "mine": + target = "mine" + elif isinstance(kind, dict) and "uploader" in kind: + target = f"u{kind['uploader']}" + if kind == "freeleech": + search_type = "fl" + elif rule.get("cost", "free") == "free": + search_type = "fl-VIP" + sort_types = { + "low_seeders": "seedersAsc", + "low_snatches": "snatchedAsc", + "oldest_first": "dateAsc", + "random": "random", + } + tor = { + "text": rule.get("query", ""), + "srchIn": rule.get("search_in", []), + "sortType": sort_types.get( + rule.get("sort_by"), "dateDesc" if kind == "new" else "" + ), + "startNumber": start, + } + if target: + tor["searchIn"] = target + if search_type: + tor["searchType"] = search_type + return {"dlLink": True, "perpage": 100, "tor": {k: v for k, v in tor.items() if v}} + + +async def search_pages( + mam: MamClient, rule: dict[str, Any] +) -> AsyncIterator[dict[str, Any]]: + kind = rule.get("type", "new") + default_pages = 50 if kind in {"bookmarks", "freeleech", "mine"} else 1 + max_pages = int(rule.get("max_pages") or default_pages) + start = 0 + for _ in range(max_pages): + result = await mam.search(build_search_query(rule, start)) + rows = result.get("data", []) + for row in rows: + if isinstance(row, dict): + yield row + start += len(rows) + found = as_int(result.get("found", len(rows))) + if not rows or start >= found: + break diff --git a/python/tests/test_library.py b/python/tests/test_library.py new file mode 100644 index 00000000..d7011ace --- /dev/null +++ b/python/tests/test_library.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from mlm.library import library_directory, map_path, safe_torrent_path, select_format + + +def test_longest_path_mapping_wins() -> None: + mapped = map_path( + {"/downloads": "/books", "/downloads/audio": "/audiobooks"}, + "/downloads/audio/new", + ) + assert mapped == Path("/audiobooks/new") + + +def test_library_directory_series_and_narrator() -> None: + result = library_directory( + False, + {"library_dir": "/library"}, + { + "authors": ["An Author"], + "title": "The Book", + "series": [{"name": "Saga", "entries": ["2"]}], + "narrators": ["A Narrator"], + "edition": None, + }, + ) + assert result == Path("/library/An Author/Saga/Saga #2 - The Book {A Narrator}") + + +def test_format_preference_and_path_traversal() -> None: + files = [{"name": "book/book.mp3"}, {"name": "book/book.m4b"}] + assert select_format(None, ("m4b", "mp3"), files) == ".m4b" + with pytest.raises(ValueError): + safe_torrent_path("../escape.mp3") diff --git a/python/tests/test_search.py b/python/tests/test_search.py new file mode 100644 index 00000000..fa1d5346 --- /dev/null +++ b/python/tests/test_search.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from mlm.search import build_search_query, matches_filter, normalize_title, parse_size + + +def test_search_payload_matches_mam_shape() -> None: + query = build_search_query( + { + "type": "bookmarks", + "cost": "free", + "query": "writer", + "search_in": ["author"], + } + ) + assert query["dlLink"] is True + assert "fields" not in query + assert query["tor"]["searchIn"] == "bookmarks" + assert query["tor"]["searchType"] == "fl-VIP" + assert query["tor"]["srchIn"] == ["author"] + + +def test_filter_size_flags_dates_and_peers() -> None: + row = { + "size": "100 MiB", + "browseflags": 1 << 5, + "added": "2025-07-06 05:40:54", + "seeders": 8, + "owner_name": "someone", + } + assert matches_filter( + row, + { + "min_size": "90 MiB", + "max_size": "110 MiB", + "flags": {"abridged": True, "explicit": False}, + "uploaded_after": "2025-07-06", + "uploaded_before": "2025-07-06", + "min_seeders": 8, + }, + ) + assert parse_size("1.5 GiB") == int(1.5 * (1 << 30)) + + +def test_normalize_title_matches_legacy_intent() -> None: + assert normalize_title("The Café & Book: Vol. 2") == "cafe and book vol 2" From c26c9294f7b1188c2267a3ff8ceeee7bcc6d0bbe Mon Sep 17 00:00:00 2001 From: Heavy Harlow <89617161+Plungis@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:33:44 -0400 Subject: [PATCH 05/82] Ship Python scheduler web UI and integrations --- .github/workflows/python.yml | 43 +++++ .github/workflows/windows-publish.yml | 81 --------- .gitignore | 1 + Dockerfile | 61 ++----- README.md | 32 +++- compose.yaml | 9 + python/README.md | 12 ++ python/pyproject.toml | 8 +- python/src/mlm/audiobookshelf.py | 117 ++++++++++++ python/src/mlm/autograbber.py | 120 +++++++------ python/src/mlm/cli.py | 19 ++ python/src/mlm/database.py | 16 ++ python/src/mlm/lists.py | 205 +++++++++++++++++++++ python/src/mlm/mam.py | 24 +++ python/src/mlm/repository.py | 124 +++++++++++++ python/src/mlm/scheduler.py | 247 ++++++++++++++++++++++++++ python/src/mlm/snatchlist.py | 95 ++++++++++ python/src/mlm/templates/base.html | 40 +++++ python/src/mlm/templates/config.html | 6 + python/src/mlm/templates/index.html | 22 +++ python/src/mlm/templates/records.html | 17 ++ python/src/mlm/web.py | 107 +++++++++++ python/tests/test_lists.py | 12 ++ python/tests/test_web.py | 22 +++ 24 files changed, 1253 insertions(+), 187 deletions(-) create mode 100644 .github/workflows/python.yml delete mode 100644 .github/workflows/windows-publish.yml create mode 100644 compose.yaml create mode 100644 python/src/mlm/audiobookshelf.py create mode 100644 python/src/mlm/lists.py create mode 100644 python/src/mlm/scheduler.py create mode 100644 python/src/mlm/snatchlist.py create mode 100644 python/src/mlm/templates/base.html create mode 100644 python/src/mlm/templates/config.html create mode 100644 python/src/mlm/templates/index.html create mode 100644 python/src/mlm/templates/records.html create mode 100644 python/src/mlm/web.py create mode 100644 python/tests/test_lists.py create mode 100644 python/tests/test_web.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 00000000..81de613c --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,43 @@ +name: Python + +on: + push: + branches: ["main", "agent/python-migration-beta"] + tags: ["v*.*.*"] + pull_request: + branches: ["main"] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: python/pyproject.toml + - name: Install + run: python -m pip install -e ./python pytest + - name: Test + run: python -m pytest -q python/tests + - name: Compile + run: python -m compileall -q python/src + + wheel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - run: python -m pip install build + - run: python -m build python + - uses: actions/upload-artifact@v4 + with: + name: python-package + path: python/dist/* diff --git a/.github/workflows/windows-publish.yml b/.github/workflows/windows-publish.yml deleted file mode 100644 index b97e2f75..00000000 --- a/.github/workflows/windows-publish.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Windows - -on: - push: - branches: [ "main", "windows" ] - # Publish semver tags as releases. - tags: [ 'v*.*.*' ] - pull_request: - branches: [ "main" ] - -permissions: - contents: write - -jobs: - build-and-upload: - name: Build and upload - runs-on: ${{ matrix.os }} - - strategy: - matrix: - include: - - build: windows-msvc - os: windows-latest - target: x86_64-pc-windows-msvc - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - - uses: shogo82148/actions-setup-perl@v1 - with: - perl-version: "5.32" - distribution: strawberry - - - name: Test - run: cargo test -p mlm - - - name: Build - run: | - cargo install cargo-packager - cargo packager --release - - - name: Prepare files - shell: bash - run: | - ls -al target - ls -al target/* - ls -al target/*/* - binary_name="mlm" - version="${GITHUB_REF#refs/tags/v}" - version="${version#refs/heads/}" - app="$binary_name-$version.exe" - installer="$binary_name-$version-x64-setup.exe" - echo "APP=$app" >> $GITHUB_ENV - echo "INSTALLER=$installer" >> $GITHUB_ENV - mv "target/release/${binary_name}_"*"-setup.exe" "$installer" - mv "target/release/$binary_name.exe" "$app" - - - name: Upload app - uses: actions/upload-artifact@v4 - with: - name: ${{ env.APP }} - path: ${{ env.APP }} - - - name: Upload installer - uses: actions/upload-artifact@v4 - with: - name: ${{ env.INSTALLER }} - path: ${{ env.INSTALLER }} - - - name: Create release - if: github.ref_type == 'tag' - uses: softprops/action-gh-release@v2 - with: - files: | - ${{ env.INSTALLER }} diff --git a/.gitignore b/.gitignore index 4d590352..17b0688a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ __pycache__/ .pytest_cache/ *.py[cod] *.egg-info/ +dist/ diff --git a/Dockerfile b/Dockerfile index 71bbc8cd..f4b8969d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,55 +1,16 @@ -# syntax=docker/dockerfile:1.3-labs +FROM python:3.13-slim -# The above line is so we can use can use heredocs in Dockerfiles. No more && and \! -# https://www.docker.com/blog/introduction-to-heredocs-in-dockerfiles/ +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + MLM_CONFIG_FILE=/config/config.toml \ + MLM_DB_FILE=/data/data.sqlite3 -FROM rust:1.91 AS build - -RUN cargo new --lib app/mlm_db -RUN cargo new --lib app/mlm_mam -RUN cargo new --lib app/mlm_parse -RUN cargo new --bin app/server - -# Capture dependencies -COPY Cargo.toml Cargo.lock /app/ -COPY mlm_db/Cargo.toml /app/mlm_db/ -COPY mlm_mam/Cargo.toml /app/mlm_mam/ -COPY mlm_parse/Cargo.toml /app/mlm_parse/ -COPY server/Cargo.toml /app/server/ - -# This step compiles only our dependencies and saves them in a layer. This is the most impactful time savings -# Note the use of --mount=type=cache. On subsequent runs, we'll have the crates already downloaded WORKDIR /app -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked cargo build --release - -# Copy our sources -COPY ./mlm_db /app/mlm_db -COPY ./mlm_mam /app/mlm_mam -COPY ./mlm_parse /app/mlm_parse -COPY ./server /app/server - -# A bit of magic here! -# * We're mounting that cache again to use during the build, otherwise it's not present and we'll have to download those again - bad! -# * EOF syntax is neat but not without its drawbacks. We need to `set -e`, otherwise a failing command is going to continue on -# * Rust here is a bit fiddly, so we'll touch the files (even though we copied over them) to force a new build -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked <=0.27,<1"] +dependencies = [ + "fastapi>=0.115,<1", + "httpx>=0.27,<1", + "jinja2>=3.1,<4", + "python-multipart>=0.0.20,<1", + "uvicorn[standard]>=0.34,<1", +] [project.scripts] mlm-python = "mlm.cli:main" diff --git a/python/src/mlm/audiobookshelf.py b/python/src/mlm/audiobookshelf.py new file mode 100644 index 00000000..2f47d5b5 --- /dev/null +++ b/python/src/mlm/audiobookshelf.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import httpx + +from .repository import Repository + + +class AudiobookshelfClient: + def __init__( + self, + url: str, + token: str, + *, + client: httpx.AsyncClient | None = None, + ) -> None: + self._owns_client = client is None + self.client = client or httpx.AsyncClient( + base_url=url.rstrip("/"), + headers={"Authorization": f"Bearer {token}", "User-Agent": "MLM"}, + timeout=30, + ) + + async def close(self) -> None: + if self._owns_client: + await self.client.aclose() + + async def find_book(self, torrent: dict[str, Any]) -> dict[str, Any] | None: + library_path = torrent.get("library_path") + authors = torrent.get("meta", {}).get("authors", []) + if not library_path or not authors: + return None + response = await self.client.get("/api/libraries") + response.raise_for_status() + libraries = [ + library + for library in response.json().get("libraries", []) + if any( + Path(library_path) == Path(folder.get("fullPath", "")) + or Path(folder.get("fullPath", "")) in Path(library_path).parents + for folder in library.get("folders", []) + ) + ] + for library in libraries: + response = await self.client.get( + f"/api/libraries/{library['id']}/search", params={"q": authors[0]} + ) + response.raise_for_status() + for author in response.json().get("authors", []): + response = await self.client.get( + f"/api/authors/{author['id']}", params={"include": "items"} + ) + response.raise_for_status() + for book in response.json().get("libraryItems", []): + if Path(book.get("path", "")) == Path(library_path): + return book + return None + + async def update_book( + self, book_id: str, mam_row: dict[str, Any], meta: dict[str, Any] + ) -> None: + title_parts = str(meta.get("title", "")).split(":", 1) + title = title_parts[0] if len(title_parts[0]) >= 4 else meta.get("title", "") + subtitle = title_parts[1].strip() if len(title_parts) > 1 and title != meta.get("title") else None + isbn_value = str(mam_row.get("isbn") or "").strip() + payload = { + "metadata": { + "title": title, + "subtitle": subtitle, + "authors": [{"name": name} for name in meta.get("authors", [])], + "series": [ + { + "name": series.get("name"), + "sequence": str(series.get("entries", [""])[0]) + if series.get("entries") + else None, + } + for series in meta.get("series", []) + ], + "narrators": meta.get("narrators", []), + "description": mam_row.get("description"), + "isbn": None if not isbn_value or isbn_value.startswith("ASIN:") else isbn_value, + "asin": isbn_value.removeprefix("ASIN:").strip() + if isbn_value.startswith("ASIN:") + else None, + "genres": [meta.get("cat", {}).get("name")] + if isinstance(meta.get("cat"), dict) + else [], + "language": meta.get("language"), + "explicit": bool(int(meta.get("flags") or 0) & (1 << 4)), + "abridged": bool(int(meta.get("flags") or 0) & (1 << 5)), + } + } + response = await self.client.patch(f"/api/items/{book_id}/media", json=payload) + response.raise_for_status() + + async def delete_book(self, book_id: str) -> None: + response = await self.client.delete(f"/api/items/{book_id}") + response.raise_for_status() + + +async def match_torrents_to_audiobookshelf( + repository: Repository, client: AudiobookshelfClient +) -> int: + matched = 0 + for torrent in repository.library_torrents(): + if torrent.get("abs_id"): + continue + book = await client.find_book(torrent) + if not book: + continue + torrent["abs_id"] = book["id"] + repository.update_torrent(torrent) + matched += 1 + return matched diff --git a/python/src/mlm/autograbber.py b/python/src/mlm/autograbber.py index 7772c65f..f8f52dd3 100644 --- a/python/src/mlm/autograbber.py +++ b/python/src/mlm/autograbber.py @@ -73,59 +73,73 @@ async def run_autograbber( selected_count = 0 async for row in search_pages(mam, rule): - torrent_id = int(row.get("id", 0)) - if not torrent_id or torrent_id in config.ignore_torrents: - continue - if not matches_filter(row, rule) or repository.has_mam_id(torrent_id): - continue - requested_cost = rule.get("cost", "free") - if requested_cost == "free" and not any( - as_bool(row.get(field)) for field in ("vip", "personal_freeleech", "free", "fl_vip") - ): - continue - if requested_cost in {"metadata_only", "metadata_only_add"}: - continue - - meta = torrent_meta(row) - title_search = normalize_title(meta["title"]) - preferred = _preferred_types(config, meta["media_type"]) - preference = _preference(meta["filetypes"], preferred) - if preference is None: - continue - duplicate = False - for existing in repository.records_with_title(title_search): - old_meta = existing.get("meta", {}) - old_preference = _preference(old_meta.get("filetypes", []), preferred) - if old_preference is not None and old_preference <= preference: - duplicate = True - break - category, tags = _tagging(config, row) - candidate = { - "mam_id": torrent_id, - "goodreads_id": None, - "hash": None, - "dl_link": row.get("dl"), - "unsat_buffer": rule.get("unsat_buffer"), - "wedge_buffer": rule.get("wedge_buffer"), - "cost": _cost(row, requested_cost), - "category": rule.get("category") or category, - "tags": tags, - "title_search": title_search, - "meta": meta, - "grabber": rule.get("name", str(index)), - "created_at": datetime.now(UTC).isoformat(), - "started_at": None, - "removed_at": None, - } - if duplicate: - if not rule.get("dry_run", False): - repository.add_duplicate(candidate) - continue - if not candidate["dl_link"]: - continue - if not rule.get("dry_run", False): - repository.add_selected(candidate) - selected_count += 1 + if await select_row(config, repository, row, rule, index=index): + selected_count += 1 if selected_count >= maximum: break return selected_count + + +async def select_row( + config: Config, + repository: Repository, + row: dict[str, Any], + rule: dict[str, Any], + *, + index: int = 0, + goodreads_id: int | None = None, +) -> bool: + torrent_id = int(row.get("id", 0)) + if not torrent_id or torrent_id in config.ignore_torrents: + return False + if not matches_filter(row, rule) or repository.has_mam_id(torrent_id): + return False + requested_cost = rule.get("cost", "free") + if requested_cost == "free" and not any( + as_bool(row.get(field)) + for field in ("vip", "personal_freeleech", "free", "fl_vip") + ): + return False + if requested_cost in {"metadata_only", "metadata_only_add"}: + return False + + meta = torrent_meta(row) + title_search = normalize_title(meta["title"]) + preferred = _preferred_types(config, meta["media_type"]) + preference = _preference(meta["filetypes"], preferred) + if preference is None: + return False + duplicate = False + for existing in repository.records_with_title(title_search): + old_meta = existing.get("meta", {}) + old_preference = _preference(old_meta.get("filetypes", []), preferred) + if old_preference is not None and old_preference <= preference: + duplicate = True + break + category, tags = _tagging(config, row) + candidate = { + "mam_id": torrent_id, + "goodreads_id": goodreads_id, + "hash": None, + "dl_link": row.get("dl"), + "unsat_buffer": rule.get("unsat_buffer"), + "wedge_buffer": rule.get("wedge_buffer"), + "cost": _cost(row, requested_cost), + "category": rule.get("category") or category, + "tags": tags, + "title_search": title_search, + "meta": meta, + "grabber": rule.get("name", str(index)), + "created_at": datetime.now(UTC).isoformat(), + "started_at": None, + "removed_at": None, + } + if duplicate: + if not rule.get("dry_run", False): + repository.add_duplicate(candidate) + return False + if not candidate["dl_link"]: + return False + if not rule.get("dry_run", False): + repository.add_selected(candidate) + return True diff --git a/python/src/mlm/cli.py b/python/src/mlm/cli.py index efac0e51..0cae3bad 100644 --- a/python/src/mlm/cli.py +++ b/python/src/mlm/cli.py @@ -29,6 +29,9 @@ def build_parser() -> argparse.ArgumentParser: ) downloader.add_argument("--config", required=True, type=Path) downloader.add_argument("--database", required=True, type=Path) + run = subparsers.add_parser("run", help="start the Python MLM service and web UI") + run.add_argument("--config", required=True, type=Path) + run.add_argument("--database", required=True, type=Path) return parser @@ -51,6 +54,22 @@ async def _download(config_path: Path, database_path: Path) -> int: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + if args.command == "run": + try: + import uvicorn + + from .web import create_app + + config = load_config(args.config) + uvicorn.run( + create_app(args.config, args.database), + host=config.web_host, + port=config.web_port, + ) + return 0 + except (ConfigError, OSError) as error: + print(f"Startup failed: {error}", file=sys.stderr) + return 1 if args.command == "download": try: return asyncio.run(_download(args.config, args.database)) diff --git a/python/src/mlm/database.py b/python/src/mlm/database.py index b748be01..ee9473d5 100644 --- a/python/src/mlm/database.py +++ b/python/src/mlm/database.py @@ -97,3 +97,19 @@ def connect(path: Path) -> sqlite3.Connection: def initialize(connection: sqlite3.Connection) -> None: connection.executescript(SCHEMA) + + +def ensure_database(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + return + connection = connect(path) + try: + initialize(connection) + connection.executemany( + "INSERT INTO migration_meta(key, value) VALUES (?, ?)", + [("schema_version", str(SCHEMA_VERSION)), ("created_fresh", "true")], + ) + connection.commit() + finally: + connection.close() diff --git a/python/src/mlm/lists.py b/python/src/mlm/lists.py new file mode 100644 index 00000000..19af9bb5 --- /dev/null +++ b/python/src/mlm/lists.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import html +import re +import xml.etree.ElementTree as ET +from datetime import UTC, datetime +from typing import Any +from urllib.parse import parse_qs, urlparse + +import httpx + +from .autograbber import select_row +from .config import Config +from .mam import MamClient +from .repository import Repository +from .search import search_pages + +SERIES_PATTERN = re.compile(r"(.*?) \(([^)]*?),? #?(\d+(?:\.\d+)?)\)$") +BOOK_LINK = re.compile(r'href=["\']((?:https://www\.goodreads\.com)?/book/show/[^"\']+)') +COVER_LINK = re.compile(r']+src=["\']([^"\']+)') + + +def _text(element: ET.Element, name: str) -> str | None: + child = next((item for item in element if item.tag.rsplit("}", 1)[-1] == name), None) + return child.text.strip() if child is not None and child.text else None + + +def goodreads_list_id(url: str) -> str: + parsed = urlparse(url) + user_id = parsed.path.rstrip("/").split("/")[-1] + shelf = parse_qs(parsed.query).get("shelf", [""])[0] + return f"{user_id}:{shelf}" + + +def _allow_media(grabs: list[dict[str, Any]], key: str) -> bool: + for grab in grabs: + categories = grab.get("categories", {}) + value = categories.get(key, True) + if value is True or (isinstance(value, list) and value): + return True + return False + + +async def run_goodreads_import( + config: Config, + repository: Repository, + mam: MamClient, + definition: dict[str, Any], + *, + client: httpx.AsyncClient | None = None, +) -> int: + owns_client = client is None + http = client or httpx.AsyncClient( + headers={"User-Agent": "Mozilla/5.0 MLM-Python"}, timeout=30 + ) + try: + response = await http.get(definition["url"]) + response.raise_for_status() + finally: + if owns_client: + await http.aclose() + root = ET.fromstring(response.content) + channel = next( + (item for item in root.iter() if item.tag.rsplit("}", 1)[-1] == "channel"), + root, + ) + title = _text(channel, "title") or definition.get("name") or "Goodreads" + list_id = goodreads_list_id(definition["url"]) + now = datetime.now(UTC).isoformat() + if not definition.get("dry_run", False): + repository.upsert_list( + {"id": list_id, "title": title, "updated_at": now, "build_date": now} + ) + selected = 0 + for xml_item in [ + item for item in channel if item.tag.rsplit("}", 1)[-1] == "item" + ]: + guid_value = _text(xml_item, "guid") or _text(xml_item, "book_id") + item_title = html.unescape(_text(xml_item, "title") or "") + if not guid_value or not item_title: + continue + series: list[list[Any]] = [] + match = SERIES_PATTERN.fullmatch(item_title) + if match: + item_title = match.group(1) + series = [[html.unescape(match.group(2)), float(match.group(3))]] + description = _text(xml_item, "description") or "" + author = html.unescape(_text(xml_item, "author_name") or "").replace(".", " ") + cover_match = COVER_LINK.search(description) + book_match = BOOK_LINK.search(description) + book_id_text = _text(xml_item, "book_id") + book_id = int(book_id_text) if book_id_text and book_id_text.isdigit() else None + guid = [list_id, guid_value] + list_item = repository.list_item(guid) or { + "guid": guid, + "list_id": list_id, + "created_at": now, + "audio_torrent": None, + "ebook_torrent": None, + "marked_done_at": None, + } + list_item.update( + { + "title": item_title, + "authors": [author] if author else [], + "series": series, + "cover_url": _text(xml_item, "book_large_image_url") + or (cover_match.group(1) if cover_match else ""), + "book_url": ( + "https://www.goodreads.com" + book_match.group(1) + if book_match and book_match.group(1).startswith("/") + else (book_match.group(1) if book_match else None) + ), + "isbn": _text(xml_item, "isbn"), + "prefer_format": definition.get("prefer_format"), + "allow_audio": _allow_media(definition.get("grab", []), "audio"), + "allow_ebook": _allow_media(definition.get("grab", []), "ebook"), + } + ) + if not definition.get("dry_run", False): + repository.upsert_list_item(list_item) + query = " ".join(filter(None, [f'"{item_title}"', f'"{author}"' if author else ""])) + for grab in definition.get("grab", []): + rule = { + **grab, + "type": "new", + "query": query, + "search_in": ["title", "author"], + "max_pages": 1, + "unsat_buffer": definition.get("unsat_buffer"), + "wedge_buffer": definition.get("wedge_buffer"), + "dry_run": definition.get("dry_run", False), + "name": definition.get("name", title), + } + async for row in search_pages(mam, rule): + if await select_row( + config, repository, row, rule, goodreads_id=book_id + ): + selected += 1 + break + else: + continue + break + return selected + + +async def run_notion_import( + config: Config, + repository: Repository, + mam: MamClient, + definition: dict[str, Any], + *, + client: httpx.AsyncClient | None = None, +) -> int: + owns_client = client is None + http = client or httpx.AsyncClient(timeout=30) + selected = 0 + cursor: str | None = None + try: + while True: + body = {"start_cursor": cursor} if cursor else {} + response = await http.post( + f"https://api.notion.com/v1/data_sources/{definition['data_source']}/query", + headers={ + "Notion-Version": "2025-09-03", + "Authorization": f"Bearer {definition['token']}", + }, + json=body, + ) + response.raise_for_status() + result = response.json() + for item in result.get("results", []): + properties = item.get("properties", {}) + for field in definition.get("mam_fields", []): + value = properties.get(field, {}) + url = value.get("url") if value.get("type") == "url" else None + if not url: + continue + try: + torrent_id = int(url.rstrip("/").split("/")[-1]) + except ValueError: + continue + if repository.has_mam_id(torrent_id): + continue + row = await mam.get_torrent_info_by_id(torrent_id) + if not row: + continue + for grab in definition.get("grab", []): + rule = { + **grab, + "unsat_buffer": definition.get("unsat_buffer"), + "wedge_buffer": definition.get("wedge_buffer"), + "dry_run": definition.get("dry_run", False), + "name": definition.get("name", "Notion"), + } + if await select_row(config, repository, row, rule): + selected += 1 + break + if not result.get("has_more") or not result.get("next_cursor"): + break + cursor = result["next_cursor"] + finally: + if owns_client: + await http.aclose() + return selected diff --git a/python/src/mlm/mam.py b/python/src/mlm/mam.py index d4465fba..9b151dc6 100644 --- a/python/src/mlm/mam.py +++ b/python/src/mlm/mam.py @@ -126,3 +126,27 @@ async def wedge_torrent(self, torrent_id: int) -> None: result = response.json() if not result.get("success"): raise MamWedgeError(str(result.get("error") or "unknown wedge error")) + + async def snatchlist( + self, kind: str, page: int, cache_timestamp: int + ) -> dict[str, Any]: + user = await self.user_info() + kinds = { + "unsat": "unsat", + "inact_unsat": "inactUnsat", + "seed_unsat": "seedUnsat", + "seed_sat": "sSat", + "inact_sat": "inactSat", + "uploads_active": "upAct", + } + response = await self.client.get( + "https://cdn.myanonamouse.net/json/loadUserDetailsTorrents.php", + params={ + "uid": user["uid"], + "iteration": page, + "type": kinds[kind], + "cacheTime": cache_timestamp, + }, + ) + self._raise_for_status(response) + return response.json() diff --git a/python/src/mlm/repository.py b/python/src/mlm/repository.py index 6c6acf5a..ff2c6052 100644 --- a/python/src/mlm/repository.py +++ b/python/src/mlm/repository.py @@ -9,6 +9,7 @@ from .database import connect from .migration import canonical_json +from .search import normalize_title class Repository: @@ -162,6 +163,129 @@ def update_torrent(self, torrent: dict[str, Any]) -> None: ), ) + def upsert_list(self, row: dict[str, Any]) -> None: + with connect(self.path) as connection: + connection.execute( + """INSERT INTO lists(id, title, payload_json) VALUES (?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + title=excluded.title, payload_json=excluded.payload_json""", + (row["id"], row["title"], canonical_json(row)), + ) + + def list_item(self, guid: list[str]) -> dict[str, Any] | None: + with connect(self.path) as connection: + row = connection.execute( + "SELECT payload_json FROM list_items WHERE guid_json = ?", + (canonical_json(guid),), + ).fetchone() + return json.loads(row[0]) if row else None + + def upsert_list_item(self, row: dict[str, Any]) -> None: + with connect(self.path) as connection: + connection.execute( + """INSERT INTO list_items + (guid_json, list_id, title, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(guid_json) DO UPDATE SET + title=excluded.title, payload_json=excluded.payload_json""", + ( + canonical_json(row["guid"]), + row["list_id"], + row["title"], + canonical_json(row["created_at"]), + canonical_json(row), + ), + ) + + def table_rows(self, table: str, *, limit: int = 500) -> list[dict[str, Any]]: + allowed = { + "torrents", + "selected_torrents", + "duplicate_torrents", + "errored_torrents", + "events", + "lists", + "list_items", + } + if table not in allowed: + raise ValueError(f"unsupported table: {table}") + order = "created_at_json DESC" if table not in {"lists"} else "title" + with connect(self.path) as connection: + rows = connection.execute( + f"SELECT payload_json FROM {table} ORDER BY {order} LIMIT ?", + (limit,), + ) + return [json.loads(row[0]) for row in rows] + + def counts(self) -> dict[str, int]: + tables = ( + "torrents", + "selected_torrents", + "duplicate_torrents", + "errored_torrents", + "events", + "lists", + "list_items", + ) + with connect(self.path) as connection: + return { + table: connection.execute( + f"SELECT COUNT(*) FROM {table}" + ).fetchone()[0] + for table in tables + } + + def delete_selected(self, mam_id: int) -> None: + with connect(self.path) as connection: + connection.execute( + "DELETE FROM selected_torrents WHERE mam_id = ?", (mam_id,) + ) + + def delete_error(self, id_json: Any) -> None: + with connect(self.path) as connection: + connection.execute( + "DELETE FROM errored_torrents WHERE id_json = ?", + (canonical_json(id_json),), + ) + + def add_metadata_torrent(self, meta: dict[str, Any], linker: str | None) -> str: + torrent_id = str(uuid4()) + now = datetime.now(UTC).isoformat() + row = { + "id": torrent_id, + "id_is_hash": False, + "mam_id": meta["mam_id"], + "abs_id": None, + "goodreads_id": None, + "library_path": None, + "library_files": [], + "linker": linker, + "category": None, + "selected_audio_format": None, + "selected_ebook_format": None, + "title_search": normalize_title(meta["title"]), + "meta": meta, + "created_at": now, + "replaced_with": None, + "request_matadata_update": False, + "library_mismatch": None, + "client_status": None, + } + with connect(self.path) as connection: + connection.execute( + """INSERT INTO torrents + (id, mam_id, title_search, created_at_json, payload_json) + VALUES (?, ?, ?, ?, ?)""", + ( + row["id"], + row["mam_id"], + row["title_search"], + canonical_json(now), + canonical_json(row), + ), + ) + return torrent_id + def record_started( self, selected: dict[str, Any], torrent_hash: str, *, wedged: bool = False ) -> None: diff --git a/python/src/mlm/scheduler.py b/python/src/mlm/scheduler.py new file mode 100644 index 00000000..5954c679 --- /dev/null +++ b/python/src/mlm/scheduler.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import asyncio +import contextlib +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from typing import Awaitable, Callable + +from .audiobookshelf import AudiobookshelfClient, match_torrents_to_audiobookshelf +from .autograbber import run_autograbber +from .cleaner import clean_superseded +from .config import Config +from .downloader import grab_selected_torrents +from .library import organize_completed +from .lists import run_goodreads_import, run_notion_import +from .mam import MamClient +from .qbittorrent import QbitClient +from .repository import Repository +from .snatchlist import run_snatchlist_search + + +@dataclass +class JobStatus: + last_run: str | None = None + last_error: str | None = None + running: bool = False + + +@dataclass +class ServiceState: + config: Config + repository: Repository + mam: MamClient + jobs: dict[str, JobStatus] = field(default_factory=dict) + tasks: list[asyncio.Task] = field(default_factory=list) + stop_event: asyncio.Event = field(default_factory=asyncio.Event) + + async def run_job(self, name: str, job: Callable[[], Awaitable[object]]) -> None: + status = self.jobs.setdefault(name, JobStatus()) + if status.running: + return + status.running = True + status.last_run = datetime.now(UTC).isoformat() + status.last_error = None + try: + await job() + except asyncio.CancelledError: + raise + except Exception as error: + status.last_error = f"{type(error).__name__}: {error}" + finally: + status.running = False + + async def _periodic( + self, + name: str, + interval_minutes: int, + job: Callable[[], Awaitable[object]], + ) -> None: + while not self.stop_event.is_set(): + await self.run_job(name, job) + try: + await asyncio.wait_for( + self.stop_event.wait(), timeout=max(1, interval_minutes * 60) + ) + except TimeoutError: + pass + + async def _qbit(self, index: int) -> tuple[QbitClient, object]: + qbit_config = self.config.qbittorrent[index] + qbit = QbitClient(qbit_config.url) + await qbit.login(qbit_config.username, qbit_config.password) + return qbit, qbit_config + + async def downloader(self) -> None: + if not self.config.qbittorrent: + return + qbit, _ = await self._qbit(0) + try: + await grab_selected_torrents( + self.config, self.repository, self.mam, qbit + ) + finally: + await qbit.close() + + async def organizer(self, index: int) -> None: + qbit, qbit_config = await self._qbit(index) + try: + await organize_completed( + self.config, self.repository, qbit_config, qbit, self.mam + ) + finally: + await qbit.close() + + async def cleaner(self) -> None: + clients: list[tuple[dict, QbitClient]] = [] + try: + for index in range(len(self.config.qbittorrent)): + qbit, qbit_config = await self._qbit(index) + clients.append((asdict(qbit_config), qbit)) + await clean_superseded(self.config, self.repository, clients) + finally: + for _, qbit in clients: + await qbit.close() + + async def audiobookshelf(self) -> None: + definition = self.config.audiobookshelf + if not definition: + return + client = AudiobookshelfClient(definition["url"], definition["token"]) + try: + await match_torrents_to_audiobookshelf(self.repository, client) + finally: + await client.close() + + def start(self) -> None: + for index, rule in enumerate(self.config.autograbs): + interval = int(rule.get("search_interval") or self.config.search_interval) + name = f"autograb:{index}" + self.tasks.append( + asyncio.create_task( + self._periodic( + name, + interval, + lambda rule=rule, index=index: run_autograbber( + self.config, + self.repository, + self.mam, + rule, + index=index, + ), + ), + name=name, + ) + ) + for index, rule in enumerate(self.config.goodreads_lists): + interval = int(rule.get("search_interval") or self.config.import_interval) + name = f"goodreads:{index}" + self.tasks.append( + asyncio.create_task( + self._periodic( + name, + interval, + lambda rule=rule: run_goodreads_import( + self.config, self.repository, self.mam, rule + ), + ), + name=name, + ) + ) + for index, rule in enumerate(self.config.notion_lists): + interval = int(rule.get("search_interval") or self.config.import_interval) + name = f"notion:{index}" + self.tasks.append( + asyncio.create_task( + self._periodic( + name, + interval, + lambda rule=rule: run_notion_import( + self.config, self.repository, self.mam, rule + ), + ), + name=name, + ) + ) + for index, rule in enumerate(self.config.snatchlist): + interval = int(rule.get("search_interval") or self.config.search_interval) + name = f"snatchlist:{index}" + self.tasks.append( + asyncio.create_task( + self._periodic( + name, + interval, + lambda rule=rule: run_snatchlist_search( + self.config, self.repository, self.mam, rule + ), + ), + name=name, + ) + ) + self.tasks.append( + asyncio.create_task( + self._periodic("downloader", 1, self.downloader), name="downloader" + ) + ) + if self.config.audiobookshelf: + self.tasks.append( + asyncio.create_task( + self._periodic( + "audiobookshelf", + int(self.config.audiobookshelf.get("interval", 10)), + self.audiobookshelf, + ), + name="audiobookshelf", + ) + ) + for index in range(len(self.config.qbittorrent)): + self.tasks.append( + asyncio.create_task( + self._periodic( + f"organizer:{index}", + self.config.link_interval, + lambda index=index: self.organizer(index), + ), + name=f"organizer:{index}", + ) + ) + self.tasks.append( + asyncio.create_task( + self._periodic("cleaner", self.config.link_interval, self.cleaner), + name="cleaner", + ) + ) + + async def close(self) -> None: + self.stop_event.set() + for task in self.tasks: + task.cancel() + for task in self.tasks: + with contextlib.suppress(asyncio.CancelledError): + await task + await self.mam.close() + + async def trigger(self, name: str) -> None: + if name == "downloader": + await self.run_job(name, self.downloader) + elif name == "cleaner": + await self.run_job(name, self.cleaner) + elif name == "organizer": + for index in range(len(self.config.qbittorrent)): + await self.run_job( + f"organizer:{index}", lambda index=index: self.organizer(index) + ) + elif name == "autograb": + for index, rule in enumerate(self.config.autograbs): + await self.run_job( + f"autograb:{index}", + lambda rule=rule, index=index: run_autograbber( + self.config, + self.repository, + self.mam, + rule, + index=index, + ), + ) + else: + raise ValueError(f"unknown job: {name}") diff --git a/python/src/mlm/snatchlist.py b/python/src/mlm/snatchlist.py new file mode 100644 index 00000000..d83c4fc6 --- /dev/null +++ b/python/src/mlm/snatchlist.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import time +from typing import Any + +from .config import Config +from .mam import MamClient +from .repository import Repository +from .search import as_bool, as_int, matches_filter, parse_size + + +def user_torrent_meta(row: dict[str, Any]) -> dict[str, Any]: + authors = [ + str(item.get("name", "")) + for item in sorted(row.get("author", []), key=lambda item: as_int(item.get("id"))) + ] + narrators = [ + str(item.get("name", "")) + for item in sorted(row.get("narrator", []), key=lambda item: as_int(item.get("id"))) + ] + series = [ + {"name": item.get("name", ""), "entries": [item.get("number", "")]} + for item in sorted(row.get("series", []), key=lambda item: as_int(item.get("id"))) + ] + category = as_int(row.get("category")) + return { + "mam_id": as_int(row.get("id")), + "vip_status": "Permanent" if as_bool(row.get("vip")) else "NotVip", + "cat": {"id": category, "name": row.get("catname", "")}, + "media_type": "unknown", + "main_cat": None, + "categories": [as_int(item.get("id")) for item in row.get("categories", [])], + "language": None, + "flags": as_int(row.get("browseFlags")), + "filetypes": [ + str(item.get("name", "")).lower() for item in row.get("fileTypes", []) + ], + "num_files": 0, + "size": parse_size(row.get("size", 0)), + "title": str(row.get("title", "")), + "edition": None, + "authors": authors, + "narrators": narrators, + "series": series, + "source": "Mam", + "uploaded_at": "1970-01-01T00:00:00Z", + } + + +def _search_compatible_row(row: dict[str, Any]) -> dict[str, Any]: + return { + **row, + "browseflags": row.get("browseFlags"), + "owner_name": row.get("uploaderName"), + "personal_freeleech": row.get("personalFree"), + } + + +async def run_snatchlist_search( + config: Config, + repository: Repository, + mam: MamClient, + definition: dict[str, Any], +) -> int: + if definition.get("languages"): + raise ValueError("language filtering is not supported for snatchlists") + if definition.get("uploaded_after") or definition.get("uploaded_before"): + raise ValueError("upload date filtering is not supported for snatchlists") + cost = definition.get("cost", "metadata_only") + if cost not in {"metadata_only", "metadata_only_add"}: + raise ValueError("snatchlists only support metadata_only costs") + + added = 0 + timestamp = int(time.time()) + max_pages = int(definition.get("max_pages", 100)) + for page in range(max_pages): + result = await mam.snatchlist(definition["type"], page, timestamp) + rows = result.get("rows", []) + for row in rows: + torrent_id = as_int(row.get("id")) + if ( + not torrent_id + or torrent_id in config.ignore_torrents + or repository.has_mam_id(torrent_id) + or not matches_filter(_search_compatible_row(row), definition) + ): + continue + if cost == "metadata_only_add" and not definition.get("dry_run", False): + repository.add_metadata_torrent( + user_torrent_meta(row), row.get("uploaderName") or None + ) + added += 1 + if len(rows) != 250: + break + return added diff --git a/python/src/mlm/templates/base.html b/python/src/mlm/templates/base.html new file mode 100644 index 00000000..5cc1de66 --- /dev/null +++ b/python/src/mlm/templates/base.html @@ -0,0 +1,40 @@ + + + + + + {{ title }} · MLM Python + + + + +
{% block content %}{% endblock %}
+ + diff --git a/python/src/mlm/templates/config.html b/python/src/mlm/templates/config.html new file mode 100644 index 00000000..076dbcca --- /dev/null +++ b/python/src/mlm/templates/config.html @@ -0,0 +1,6 @@ +{% extends "base.html" %} +{% block content %} +

Configuration

+

Secrets are redacted. Edit the TOML file and restart MLM to apply changes.

+
{{ config | tojson(indent=2) }}
+{% endblock %} diff --git a/python/src/mlm/templates/index.html b/python/src/mlm/templates/index.html new file mode 100644 index 00000000..bbc905d8 --- /dev/null +++ b/python/src/mlm/templates/index.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} +{% block content %} +

Dashboard

+
+{% for table,count in counts.items() %} + +
{{ table.replace("_"," ").title() }}
{{ count }}
+ +{% endfor %} +
+

Run now

+{% for name in ["autograb","downloader","organizer","cleaner"] %} +
+{% endfor %} +

Jobs

+ +{% for name,status in jobs.items() %} + + +{% endfor %} +
JobRunningLast runError
{{ name }}{{ status.running }}{{ status.last_run or "—" }}{{ status.last_error or "" }}
+{% endblock %} diff --git a/python/src/mlm/templates/records.html b/python/src/mlm/templates/records.html new file mode 100644 index 00000000..3bbd3e64 --- /dev/null +++ b/python/src/mlm/templates/records.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} +{% block content %} +

{{ title }}

+{% if table == "selected_torrents" %}{% endif %} +{% for row in rows %} + + + + {% if table == "selected_torrents" %}{% endif %} + +{% else %}{% endfor %} +
SummaryRecord
{{ row.title or row.meta.title or row.id or row.mam_id or row.guid }}
{{ row | tojson(indent=2) }}
 +
+ +
+
No records.
+{% endblock %} diff --git a/python/src/mlm/web.py b/python/src/mlm/web.py new file mode 100644 index 00000000..63288da9 --- /dev/null +++ b/python/src/mlm/web.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from dataclasses import asdict +from pathlib import Path +from typing import AsyncIterator + +from fastapi import FastAPI, Form, HTTPException, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates + +from .config import Config, load_config +from .database import ensure_database +from .mam import MamClient +from .repository import Repository +from .scheduler import ServiceState + +PACKAGE_DIR = Path(__file__).resolve().parent +templates = Jinja2Templates(directory=PACKAGE_DIR / "templates") + + +def _redacted_config(config: Config) -> dict: + value = asdict(config) + value["mam_id"] = "***" if config.mam_id else "" + if value.get("audiobookshelf"): + value["audiobookshelf"]["token"] = "***" + for row in value.get("qbittorrent", []): + if row.get("password"): + row["password"] = "***" + for row in value.get("notion_lists", []): + if row.get("token"): + row["token"] = "***" + return value + + +def create_app(config_path: Path, database_path: Path) -> FastAPI: + ensure_database(database_path) + config = load_config(config_path) + repository = Repository(database_path) + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + mam = MamClient(config.mam_id) + state = ServiceState(config, repository, mam) + app.state.services = state + try: + await mam.check_mam_id() + state.start() + yield + finally: + await state.close() + + app = FastAPI(title="Myanonamouse Library Manager", lifespan=lifespan) + + def context(request: Request, **values: object) -> dict: + return { + "request": request, + "counts": repository.counts(), + "jobs": app.state.services.jobs if hasattr(app.state, "services") else {}, + **values, + } + + @app.get("/", response_class=HTMLResponse) + async def index(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request, "index.html", context(request, title="Dashboard") + ) + + @app.get("/records/{table}", response_class=HTMLResponse) + async def records(request: Request, table: str) -> HTMLResponse: + try: + rows = repository.table_rows(table) + except ValueError as error: + raise HTTPException(404, str(error)) from error + return templates.TemplateResponse( + request, + "records.html", + context(request, title=table.replace("_", " ").title(), table=table, rows=rows), + ) + + @app.get("/config", response_class=HTMLResponse) + async def show_config(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request, + "config.html", + context(request, title="Configuration", config=_redacted_config(config)), + ) + + @app.post("/actions/{name}") + async def action(name: str) -> RedirectResponse: + try: + asyncio.create_task(app.state.services.trigger(name)) + except ValueError as error: + raise HTTPException(404, str(error)) from error + return RedirectResponse("/", status_code=303) + + @app.post("/selected/remove") + async def remove_selected(mam_id: int = Form(...)) -> RedirectResponse: + repository.delete_selected(mam_id) + return RedirectResponse("/records/selected_torrents", status_code=303) + + @app.get("/health") + async def health() -> dict: + return {"status": "ok", "counts": repository.counts()} + + return app diff --git a/python/tests/test_lists.py b/python/tests/test_lists.py new file mode 100644 index 00000000..af5870a7 --- /dev/null +++ b/python/tests/test_lists.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from mlm.lists import goodreads_list_id + + +def test_goodreads_list_id_legacy_shape() -> None: + assert ( + goodreads_list_id( + "https://www.goodreads.com/review/list_rss/12345?shelf=want-to-read" + ) + == "12345:want-to-read" + ) diff --git a/python/tests/test_web.py b/python/tests/test_web.py new file mode 100644 index 00000000..40eb2388 --- /dev/null +++ b/python/tests/test_web.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi.testclient import TestClient + +from mlm.web import create_app + + +def test_dashboard_and_health_on_fresh_database(tmp_path: Path) -> None: + config = tmp_path / "config.toml" + config.write_text('mam_id = ""\n', encoding="utf-8") + app = create_app(config, tmp_path / "data.sqlite3") + client = TestClient(app) + + health = client.get("/health") + dashboard = client.get("/") + + assert health.status_code == 200 + assert health.json()["status"] == "ok" + assert dashboard.status_code == 200 + assert "MLM Python" in dashboard.text From b121fac22b38cc797c007b201923f8ca75442c47 Mon Sep 17 00:00:00 2001 From: Heavy Harlow <89617161+Plungis@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:37:16 -0400 Subject: [PATCH 06/82] Harden Python runtime and add manual search --- .github/workflows/python.yml | 6 +- python/pyproject.toml | 10 +++ python/src/mlm/audiobookshelf.py | 10 ++- python/src/mlm/autograbber.py | 5 +- python/src/mlm/cleaner.py | 23 ++++-- python/src/mlm/config.py | 11 +-- python/src/mlm/downloader.py | 11 +-- python/src/mlm/library.py | 28 +++++-- python/src/mlm/lists.py | 16 ++-- python/src/mlm/mam.py | 12 +-- python/src/mlm/migration.py | 24 ++++-- python/src/mlm/qbittorrent.py | 13 ++- python/src/mlm/repository.py | 119 +++++++++++++-------------- python/src/mlm/scheduler.py | 12 +-- python/src/mlm/search.py | 35 +++++--- python/src/mlm/snatchlist.py | 12 ++- python/src/mlm/templates/base.html | 1 + python/src/mlm/templates/search.html | 15 ++++ python/src/mlm/web.py | 55 +++++++++++-- python/tests/test_downloader.py | 80 ++++++++++++++++++ 20 files changed, 356 insertions(+), 142 deletions(-) create mode 100644 python/src/mlm/templates/search.html create mode 100644 python/tests/test_downloader.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 81de613c..f4c4b3ec 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -22,7 +22,11 @@ jobs: cache: pip cache-dependency-path: python/pyproject.toml - name: Install - run: python -m pip install -e ./python pytest + run: python -m pip install -e "./python[dev]" + - name: Lint + run: | + ruff check python/src python/tests + ruff format --check python/src python/tests - name: Test run: python -m pytest -q python/tests - name: Compile diff --git a/python/pyproject.toml b/python/pyproject.toml index 1a49096e..b6c3c86d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -18,6 +18,9 @@ dependencies = [ "uvicorn[standard]>=0.34,<1", ] +[project.optional-dependencies] +dev = ["pytest>=8,<10", "ruff>=0.16,<1"] + [project.scripts] mlm-python = "mlm.cli:main" @@ -27,3 +30,10 @@ packages = ["src/mlm"] [tool.pytest.ini_options] pythonpath = ["src"] testpaths = ["tests"] + +[tool.ruff] +target-version = "py311" +line-length = 88 + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM", "DTZ", "FURB", "PYI"] diff --git a/python/src/mlm/audiobookshelf.py b/python/src/mlm/audiobookshelf.py index 2f47d5b5..64ddfd48 100644 --- a/python/src/mlm/audiobookshelf.py +++ b/python/src/mlm/audiobookshelf.py @@ -63,7 +63,11 @@ async def update_book( ) -> None: title_parts = str(meta.get("title", "")).split(":", 1) title = title_parts[0] if len(title_parts[0]) >= 4 else meta.get("title", "") - subtitle = title_parts[1].strip() if len(title_parts) > 1 and title != meta.get("title") else None + subtitle = ( + title_parts[1].strip() + if len(title_parts) > 1 and title != meta.get("title") + else None + ) isbn_value = str(mam_row.get("isbn") or "").strip() payload = { "metadata": { @@ -81,7 +85,9 @@ async def update_book( ], "narrators": meta.get("narrators", []), "description": mam_row.get("description"), - "isbn": None if not isbn_value or isbn_value.startswith("ASIN:") else isbn_value, + "isbn": None + if not isbn_value or isbn_value.startswith("ASIN:") + else isbn_value, "asin": isbn_value.removeprefix("ASIN:").strip() if isbn_value.startswith("ASIN:") else None, diff --git a/python/src/mlm/autograbber.py b/python/src/mlm/autograbber.py index f8f52dd3..0daa82df 100644 --- a/python/src/mlm/autograbber.py +++ b/python/src/mlm/autograbber.py @@ -68,7 +68,10 @@ async def run_autograbber( for selected in repository.pending_selected() ) maximum = min(maximum, max(0, int(rule["max_active_downloads"]) - active)) - if maximum == 0 and rule.get("cost", "free") not in {"metadata_only", "metadata_only_add"}: + if maximum == 0 and rule.get("cost", "free") not in { + "metadata_only", + "metadata_only_add", + }: return 0 selected_count = 0 diff --git a/python/src/mlm/cleaner.py b/python/src/mlm/cleaner.py index 590c8af7..d91fd959 100644 --- a/python/src/mlm/cleaner.py +++ b/python/src/mlm/cleaner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib from pathlib import Path from .config import Config @@ -11,7 +12,8 @@ def _preference(config: Config, torrent: dict) -> int: media = torrent.get("meta", {}).get("media_type") preferred = ( config.audio_types - if media in {"audiobook", "periodical_audiobook", "Audiobook", "PeriodicalAudiobook"} + if media + in {"audiobook", "periodical_audiobook", "Audiobook", "PeriodicalAudiobook"} else config.ebook_types ) formats = torrent.get("meta", {}).get("filetypes", []) @@ -38,10 +40,8 @@ def remove_library_files(torrent: dict) -> None: if all(path.name in {"cover.jpg", "metadata.json"} for path in remaining): for path in remaining: path.unlink(missing_ok=True) - try: + with contextlib.suppress(OSError): library_path.rmdir() - except OSError: - pass async def clean_superseded( @@ -56,11 +56,16 @@ async def clean_superseded( for rows in grouped.values(): if len(rows) < 2: continue - rows.sort(key=lambda row: (_preference(config, row), -sum( - (Path(row["library_path"]) / file).stat().st_size - for file in row.get("library_files", []) - if (Path(row["library_path"]) / file).exists() - ))) + rows.sort( + key=lambda row: ( + _preference(config, row), + -sum( + (Path(row["library_path"]) / file).stat().st_size + for file in row.get("library_files", []) + if (Path(row["library_path"]) / file).exists() + ), + ) + ) keep, *remove_rows = rows for torrent in remove_rows: for qbit_config, qbit in qbit_clients: diff --git a/python/src/mlm/config.py b/python/src/mlm/config.py index ecb10e0e..d5409b50 100644 --- a/python/src/mlm/config.py +++ b/python/src/mlm/config.py @@ -2,9 +2,10 @@ import os import tomllib +from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Mapping +from typing import Any class ConfigError(ValueError): @@ -73,9 +74,7 @@ def _environment_overrides(environment: Mapping[str, str]) -> dict[str, Any]: return overrides -def load_config( - path: Path, *, environment: Mapping[str, str] | None = None -) -> Config: +def load_config(path: Path, *, environment: Mapping[str, str] | None = None) -> Config: try: raw = tomllib.loads(path.read_text(encoding="utf-8")) except (OSError, tomllib.TOMLDecodeError) as error: @@ -83,7 +82,9 @@ def load_config( for old, new in _ALIASES.items(): if old in raw and new not in raw: raw[new] = raw.pop(old) - raw.update(_environment_overrides(os.environ if environment is None else environment)) + raw.update( + _environment_overrides(os.environ if environment is None else environment) + ) allowed = set(Config.__dataclass_fields__) unknown = sorted(set(raw) - allowed) diff --git a/python/src/mlm/downloader.py b/python/src/mlm/downloader.py index 3629dc5d..ff589d20 100644 --- a/python/src/mlm/downloader.py +++ b/python/src/mlm/downloader.py @@ -38,9 +38,7 @@ async def grab_selected_torrents( downloaded = failed = skipped = 0 user = await mam.user_info() unsat = user.get("unsat", {}) - available_slots = max( - 0, int(unsat.get("limit", 0)) - int(unsat.get("count", 0)) - ) + available_slots = max(0, int(unsat.get("limit", 0)) - int(unsat.get("count", 0))) downloading_size = sum( int(row.get("meta", {}).get("size", 0)) for row in repository.pending_selected() @@ -60,7 +58,10 @@ async def grab_selected_torrents( else config.unsat_buffer ) size = int(selected.get("meta", {}).get("size", 0)) - if available_slots - downloaded <= slot_buffer or remaining_buffer - size <= 0: + if ( + available_slots - downloaded <= slot_buffer + or remaining_buffer - size <= 0 + ): skipped += 1 continue torrent_file = await _torrent_file_with_backoff( @@ -104,7 +105,7 @@ async def grab_selected_torrents( repository.record_started(selected, torrent_hash, wedged=wedged) downloaded += 1 remaining_buffer -= size - except Exception as error: + except Exception as error: # noqa: BLE001 - isolate failures per torrent repository.record_grab_error(selected, error) failed += 1 await asyncio.sleep(1) diff --git a/python/src/mlm/library.py b/python/src/mlm/library.py index 67d0c205..8c38cc9e 100644 --- a/python/src/mlm/library.py +++ b/python/src/mlm/library.py @@ -5,7 +5,7 @@ import re import shutil from datetime import UTC, datetime -from pathlib import Path, PurePath +from pathlib import Path from typing import Any from .config import Config, QbitConfig @@ -15,7 +15,7 @@ from .search import normalize_title, torrent_meta INVALID_FILENAME = re.compile(r'[<>:"/\\|?*\x00-\x1f]') -DISC_PATTERN = re.compile(r"(?:CD|Disc|Disk)\s*(\d+)", re.I) +DISC_PATTERN = re.compile(r"(?:CD|Disc|Disk)\s*(\d+)", re.IGNORECASE) def sanitize_filename(value: str) -> str: @@ -45,10 +45,13 @@ def find_library(config: Config, torrent: dict[str, Any]) -> dict[str, Any] | No tag.strip() for tag in str(torrent.get("tags", "")).split(",") if tag.strip() } for library in config.libraries: - by_category = "category" in library and torrent.get("category") == library["category"] + by_category = ( + "category" in library and torrent.get("category") == library["category"] + ) by_directory = "download_dir" in library and ( Path(torrent.get("save_path", "")) == Path(library["download_dir"]) - or Path(library["download_dir"]) in Path(torrent.get("save_path", "")).parents + or Path(library["download_dir"]) + in Path(torrent.get("save_path", "")).parents ) if not (by_category or by_directory): continue @@ -87,13 +90,17 @@ def library_directory( if series: series_name, number = series leaf = f"{series_name} #{number} - {title}" if number else title - relative = Path(author) / sanitize_filename(series_name) / sanitize_filename(leaf) + relative = ( + Path(author) / sanitize_filename(series_name) / sanitize_filename(leaf) + ) else: relative = Path(author) / sanitize_filename(title) edition = meta.get("edition") if edition: edition_name = edition[0] if isinstance(edition, list) else str(edition) - relative = relative.with_name(sanitize_filename(f"{relative.name}, {edition_name}")) + relative = relative.with_name( + sanitize_filename(f"{relative.name}, {edition_name}") + ) narrators = meta.get("narrators", []) if narrators and not exclude_narrator: relative = relative.with_name( @@ -195,11 +202,16 @@ async def organize_completed( library_files: list[str] = [] if target_dir is not None: target_dir.mkdir(parents=True, exist_ok=True) - download_root = map_path(qbit_config.path_mapping, str(qbit_torrent["save_path"])) + download_root = map_path( + qbit_config.path_mapping, str(qbit_torrent["save_path"]) + ) for content in files: torrent_path = safe_torrent_path(str(content["name"])) lower_name = torrent_path.name.lower() - if not ((audio and lower_name.endswith(audio)) or (ebook and lower_name.endswith(ebook))): + if not ( + (audio and lower_name.endswith(audio)) + or (ebook and lower_name.endswith(ebook)) + ): continue relative = _destination_relative(torrent_path) destination = target_dir / relative diff --git a/python/src/mlm/lists.py b/python/src/mlm/lists.py index 19af9bb5..610cfcbd 100644 --- a/python/src/mlm/lists.py +++ b/python/src/mlm/lists.py @@ -16,12 +16,16 @@ from .search import search_pages SERIES_PATTERN = re.compile(r"(.*?) \(([^)]*?),? #?(\d+(?:\.\d+)?)\)$") -BOOK_LINK = re.compile(r'href=["\']((?:https://www\.goodreads\.com)?/book/show/[^"\']+)') +BOOK_LINK = re.compile( + r'href=["\']((?:https://www\.goodreads\.com)?/book/show/[^"\']+)' +) COVER_LINK = re.compile(r']+src=["\']([^"\']+)') def _text(element: ET.Element, name: str) -> str | None: - child = next((item for item in element if item.tag.rsplit("}", 1)[-1] == name), None) + child = next( + (item for item in element if item.tag.rsplit("}", 1)[-1] == name), None + ) return child.text.strip() if child is not None and child.text else None @@ -72,9 +76,7 @@ async def run_goodreads_import( {"id": list_id, "title": title, "updated_at": now, "build_date": now} ) selected = 0 - for xml_item in [ - item for item in channel if item.tag.rsplit("}", 1)[-1] == "item" - ]: + for xml_item in [item for item in channel if item.tag.rsplit("}", 1)[-1] == "item"]: guid_value = _text(xml_item, "guid") or _text(xml_item, "book_id") item_title = html.unescape(_text(xml_item, "title") or "") if not guid_value or not item_title: @@ -119,7 +121,9 @@ async def run_goodreads_import( ) if not definition.get("dry_run", False): repository.upsert_list_item(list_item) - query = " ".join(filter(None, [f'"{item_title}"', f'"{author}"' if author else ""])) + query = " ".join( + filter(None, [f'"{item_title}"', f'"{author}"' if author else ""]) + ) for grab in definition.get("grab", []): rule = { **grab, diff --git a/python/src/mlm/mam.py b/python/src/mlm/mam.py index 9b151dc6..bdc4f82b 100644 --- a/python/src/mlm/mam.py +++ b/python/src/mlm/mam.py @@ -1,7 +1,7 @@ from __future__ import annotations import time -from typing import Any +from typing import Any, Self import httpx @@ -37,7 +37,7 @@ def __init__( ) self.client.cookies.set("mam_id", mam_id, domain="www.myanonamouse.net") - async def __aenter__(self) -> MamClient: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *_: object) -> None: @@ -63,14 +63,14 @@ async def check_mam_id(self) -> None: raise MamError("session check failed (Success was false)") async def user_info(self) -> dict[str, Any]: - response = await self.client.get("/jsonLoad.php", params={"snatch_summary": "true"}) + response = await self.client.get( + "/jsonLoad.php", params={"snatch_summary": "true"} + ) self._raise_for_status(response) return response.json() async def search(self, query: dict[str, Any]) -> dict[str, Any]: - response = await self.client.post( - "/tor/js/loadSearchJSONbasic.php", json=query - ) + response = await self.client.post("/tor/js/loadSearchJSONbasic.php", json=query) self._raise_for_status(response) result = response.json() if isinstance(result, dict) and result.get("error"): diff --git a/python/src/mlm/migration.py b/python/src/mlm/migration.py index 1a1d9d8a..7cf87d09 100644 --- a/python/src/mlm/migration.py +++ b/python/src/mlm/migration.py @@ -53,7 +53,9 @@ def back_up_source(source: Path) -> Path: return backup -def export_legacy_database(executable: Path, database_backup: Path, output: Path) -> None: +def export_legacy_database( + executable: Path, database_backup: Path, output: Path +) -> None: executable = executable.resolve() if not executable.is_file(): raise MigrationError(f"legacy executable does not exist: {executable}") @@ -72,10 +74,14 @@ def export_legacy_database(executable: Path, database_backup: Path, output: Path check=False, ) if completed.returncode != 0: - detail = completed.stderr.strip() or completed.stdout.strip() or "no error output" + detail = ( + completed.stderr.strip() or completed.stdout.strip() or "no error output" + ) raise MigrationError(f"legacy export failed ({completed.returncode}): {detail}") if not output.is_file(): - raise MigrationError("legacy exporter reported success but produced no JSON file") + raise MigrationError( + "legacy exporter reported success but produced no JSON file" + ) def _load_export(path: Path) -> tuple[dict[str, Any], str]: @@ -98,7 +104,9 @@ def _load_export(path: Path) -> tuple[dict[str, Any], str]: declared = document.get("counts") actual = {table: len(document[table]) for table in DATA_TABLES} if declared != actual: - raise MigrationError(f"export count mismatch: declared={declared!r}, actual={actual!r}") + raise MigrationError( + f"export count mismatch: declared={declared!r}, actual={actual!r}" + ) return document, digest @@ -200,7 +208,9 @@ def _validate(connection: sqlite3.Connection, expected: dict[str, int]) -> None: for table in DATA_TABLES } if actual != expected: - raise MigrationError(f"SQLite count mismatch: expected={expected!r}, actual={actual!r}") + raise MigrationError( + f"SQLite count mismatch: expected={expected!r}, actual={actual!r}" + ) result = connection.execute("PRAGMA integrity_check").fetchone()[0] if result != "ok": raise MigrationError(f"SQLite integrity check failed: {result}") @@ -225,7 +235,9 @@ def migrate( destination.parent.mkdir(parents=True, exist_ok=True) backup = back_up_source(source_database) - with tempfile.TemporaryDirectory(prefix="mlm-migration-", dir=destination.parent) as temp_dir: + with tempfile.TemporaryDirectory( + prefix="mlm-migration-", dir=destination.parent + ) as temp_dir: temp_dir_path = Path(temp_dir) if legacy_executable is not None: selected_export = temp_dir_path / "legacy-export.json" diff --git a/python/src/mlm/qbittorrent.py b/python/src/mlm/qbittorrent.py index b66ea47f..dcbd586e 100644 --- a/python/src/mlm/qbittorrent.py +++ b/python/src/mlm/qbittorrent.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import Iterable +from collections.abc import Iterable +from typing import Self import httpx @@ -22,7 +23,7 @@ def __init__( base_url=url.rstrip("/"), timeout=timeout ) - async def __aenter__(self) -> QbitClient: + async def __aenter__(self) -> Self: return self async def __aexit__(self, *_: object) -> None: @@ -66,7 +67,13 @@ async def add_torrent( response = await self.client.post( "/api/v2/torrents/add", data=data, - files={"torrents": ("download.torrent", torrent_file, "application/x-bittorrent")}, + files={ + "torrents": ( + "download.torrent", + torrent_file, + "application/x-bittorrent", + ) + }, ) self._check(response) diff --git a/python/src/mlm/repository.py b/python/src/mlm/repository.py index ff2c6052..b4425f4d 100644 --- a/python/src/mlm/repository.py +++ b/python/src/mlm/repository.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import sqlite3 from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -76,7 +75,8 @@ def add_duplicate( """INSERT INTO duplicate_torrents (mam_id, title_search, created_at_json, payload_json) VALUES (?, ?, ?, ?) - ON CONFLICT(mam_id) DO UPDATE SET payload_json=excluded.payload_json""", + ON CONFLICT(mam_id) DO UPDATE SET + payload_json=excluded.payload_json""", ( row["mam_id"], row["title_search"], @@ -101,7 +101,9 @@ def library_torrents(self) -> list[dict[str, Any]]: ) return [json.loads(row[0]) for row in rows] - def record_linked(self, torrent: dict[str, Any], selected_mam_id: int | None) -> None: + def record_linked( + self, torrent: dict[str, Any], selected_mam_id: int | None + ) -> None: event = { "id": str(uuid4()), "torrent_id": torrent["id"], @@ -114,41 +116,40 @@ def record_linked(self, torrent: dict[str, Any], selected_mam_id: int | None) -> } }, } - with connect(self.path) as connection: - with connection: - connection.execute( - """INSERT INTO torrents + with connect(self.path) as connection, connection: + connection.execute( + """INSERT INTO torrents (id, mam_id, title_search, created_at_json, payload_json) VALUES (?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET mam_id=excluded.mam_id, title_search=excluded.title_search, payload_json=excluded.payload_json""", - ( - torrent["id"], - torrent["mam_id"], - torrent["title_search"], - canonical_json(torrent["created_at"]), - canonical_json(torrent), - ), - ) - if selected_mam_id is not None: - connection.execute( - "DELETE FROM selected_torrents WHERE mam_id = ?", - (selected_mam_id,), - ) + ( + torrent["id"], + torrent["mam_id"], + torrent["title_search"], + canonical_json(torrent["created_at"]), + canonical_json(torrent), + ), + ) + if selected_mam_id is not None: connection.execute( - """INSERT INTO events + "DELETE FROM selected_torrents WHERE mam_id = ?", + (selected_mam_id,), + ) + connection.execute( + """INSERT INTO events (id_json, torrent_id, mam_id, created_at_json, payload_json) VALUES (?, ?, ?, ?, ?)""", - ( - canonical_json(event["id"]), - event["torrent_id"], - event["mam_id"], - canonical_json(event["created_at"]), - canonical_json(event), - ), - ) + ( + canonical_json(event["id"]), + event["torrent_id"], + event["mam_id"], + canonical_json(event["created_at"]), + canonical_json(event), + ), + ) def update_torrent(self, torrent: dict[str, Any]) -> None: with connect(self.path) as connection: @@ -209,7 +210,7 @@ def table_rows(self, table: str, *, limit: int = 500) -> list[dict[str, Any]]: } if table not in allowed: raise ValueError(f"unsupported table: {table}") - order = "created_at_json DESC" if table not in {"lists"} else "title" + order = "created_at_json DESC" if table != "lists" else "title" with connect(self.path) as connection: rows = connection.execute( f"SELECT payload_json FROM {table} ORDER BY {order} LIMIT ?", @@ -229,9 +230,7 @@ def counts(self) -> dict[str, int]: ) with connect(self.path) as connection: return { - table: connection.execute( - f"SELECT COUNT(*) FROM {table}" - ).fetchone()[0] + table: connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] for table in tables } @@ -326,38 +325,38 @@ def record_started( } }, } - with connect(self.path) as connection: - with connection: - connection.execute( - """UPDATE selected_torrents + with connect(self.path) as connection, connection: + connection.execute( + """UPDATE selected_torrents SET hash = ?, payload_json = ? WHERE mam_id = ?""", - (torrent_hash, canonical_json(selected), selected["mam_id"]), - ) - connection.execute( - """INSERT INTO torrents + (torrent_hash, canonical_json(selected), selected["mam_id"]), + ) + connection.execute( + """INSERT INTO torrents (id, mam_id, title_search, created_at_json, payload_json) VALUES (?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET payload_json=excluded.payload_json""", - ( - torrent_hash, - selected["mam_id"], - selected["title_search"], - canonical_json(now), - canonical_json(torrent), - ), - ) - connection.execute( - """INSERT INTO events + ON CONFLICT(id) DO UPDATE SET + payload_json=excluded.payload_json""", + ( + torrent_hash, + selected["mam_id"], + selected["title_search"], + canonical_json(now), + canonical_json(torrent), + ), + ) + connection.execute( + """INSERT INTO events (id_json, torrent_id, mam_id, created_at_json, payload_json) VALUES (?, ?, ?, ?, ?)""", - ( - canonical_json(event["id"]), - torrent_hash, - selected["mam_id"], - canonical_json(now), - canonical_json(event), - ), - ) + ( + canonical_json(event["id"]), + torrent_hash, + selected["mam_id"], + canonical_json(now), + canonical_json(event), + ), + ) def record_grab_error(self, selected: dict[str, Any], error: Exception) -> None: now = datetime.now(UTC).isoformat() diff --git a/python/src/mlm/scheduler.py b/python/src/mlm/scheduler.py index 5954c679..3a74c74f 100644 --- a/python/src/mlm/scheduler.py +++ b/python/src/mlm/scheduler.py @@ -2,9 +2,9 @@ import asyncio import contextlib +from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field from datetime import UTC, datetime -from typing import Awaitable, Callable from .audiobookshelf import AudiobookshelfClient, match_torrents_to_audiobookshelf from .autograbber import run_autograbber @@ -46,7 +46,7 @@ async def run_job(self, name: str, job: Callable[[], Awaitable[object]]) -> None await job() except asyncio.CancelledError: raise - except Exception as error: + except Exception as error: # noqa: BLE001 - jobs must not stop the scheduler status.last_error = f"{type(error).__name__}: {error}" finally: status.running = False @@ -59,12 +59,10 @@ async def _periodic( ) -> None: while not self.stop_event.is_set(): await self.run_job(name, job) - try: + with contextlib.suppress(TimeoutError): await asyncio.wait_for( self.stop_event.wait(), timeout=max(1, interval_minutes * 60) ) - except TimeoutError: - pass async def _qbit(self, index: int) -> tuple[QbitClient, object]: qbit_config = self.config.qbittorrent[index] @@ -77,9 +75,7 @@ async def downloader(self) -> None: return qbit, _ = await self._qbit(0) try: - await grab_selected_torrents( - self.config, self.repository, self.mam, qbit - ) + await grab_selected_torrents(self.config, self.repository, self.mam, qbit) finally: await qbit.close() diff --git a/python/src/mlm/search.py b/python/src/mlm/search.py index 37ee5019..f2ba339c 100644 --- a/python/src/mlm/search.py +++ b/python/src/mlm/search.py @@ -3,8 +3,9 @@ import html import re import unicodedata -from datetime import datetime -from typing import Any, AsyncIterator +from collections.abc import AsyncIterator +from datetime import date +from typing import Any from .mam import MamClient @@ -61,10 +62,12 @@ def as_bool(value: Any) -> bool: def parse_size(value: Any) -> int: if isinstance(value, (int, float)): return int(value) - match = re.fullmatch(r"\s*([\d.]+)\s*([kmgt]?i?b)?\s*", str(value), re.I) + match = re.fullmatch(r"\s*([\d.]+)\s*([kmgt]?i?b)?\s*", str(value), re.IGNORECASE) if not match: raise ValueError(f"invalid size: {value!r}") - return int(float(match.group(1)) * SIZE_UNITS.get((match.group(2) or "b").lower(), 1)) + return int( + float(match.group(1)) * SIZE_UNITS.get((match.group(2) or "b").lower(), 1) + ) def normalize_title(value: str) -> str: @@ -88,7 +91,9 @@ def _mapping_values(value: Any) -> list[str]: def torrent_meta(row: dict[str, Any]) -> dict[str, Any]: media_id = as_int(row.get("mediatype")) main_id = as_int(row.get("main_cat")) - media_type = MEDIA_TYPE_BY_ID.get(media_id) or MAIN_CATEGORY_BY_ID.get(main_id, "unknown") + media_type = MEDIA_TYPE_BY_ID.get(media_id) or MAIN_CATEGORY_BY_ID.get( + main_id, "unknown" + ) series = [] raw_series = row.get("series_info") if isinstance(raw_series, str): @@ -120,8 +125,8 @@ def torrent_meta(row: dict[str, Any]) -> dict[str, Any]: } -def _date(value: Any) -> datetime: - return datetime.strptime(str(value)[:10], "%Y-%m-%d") +def _date(value: Any) -> date: + return date.fromisoformat(str(value)[:10]) def matches_filter(row: dict[str, Any], rule: dict[str, Any]) -> bool: @@ -133,14 +138,14 @@ def matches_filter(row: dict[str, Any], rule: dict[str, Any]) -> bool: categories = rule.get("categories", {}) if categories: main = MAIN_CATEGORY_BY_ID.get(as_int(row.get("main_cat"))) - category_rule = categories.get( - {"audiobook": "audio"}.get(main, main), False - ) + category_rule = categories.get({"audiobook": "audio"}.get(main, main), False) if category_rule is False: return False if isinstance(category_rule, list): names = {str(value).lower().replace(" ", "_") for value in category_rule} - actual = str(row.get("catname", row.get("cat", ""))).lower().replace(" ", "_") + actual = ( + str(row.get("catname", row.get("cat", ""))).lower().replace(" ", "_") + ) if actual not in names: return False @@ -163,9 +168,13 @@ def matches_filter(row: dict[str, Any], rule: dict[str, Any]) -> bool: return False if str(row.get("owner_name", "")) in rule.get("exclude_uploader", []): return False - if rule.get("uploaded_after") and _date(row.get("added")) < _date(rule["uploaded_after"]): + if rule.get("uploaded_after") and _date(row.get("added")) < _date( + rule["uploaded_after"] + ): return False - if rule.get("uploaded_before") and _date(row.get("added")) > _date(rule["uploaded_before"]): + if rule.get("uploaded_before") and _date(row.get("added")) > _date( + rule["uploaded_before"] + ): return False comparisons = { diff --git a/python/src/mlm/snatchlist.py b/python/src/mlm/snatchlist.py index d83c4fc6..0de47531 100644 --- a/python/src/mlm/snatchlist.py +++ b/python/src/mlm/snatchlist.py @@ -12,15 +12,21 @@ def user_torrent_meta(row: dict[str, Any]) -> dict[str, Any]: authors = [ str(item.get("name", "")) - for item in sorted(row.get("author", []), key=lambda item: as_int(item.get("id"))) + for item in sorted( + row.get("author", []), key=lambda item: as_int(item.get("id")) + ) ] narrators = [ str(item.get("name", "")) - for item in sorted(row.get("narrator", []), key=lambda item: as_int(item.get("id"))) + for item in sorted( + row.get("narrator", []), key=lambda item: as_int(item.get("id")) + ) ] series = [ {"name": item.get("name", ""), "entries": [item.get("number", "")]} - for item in sorted(row.get("series", []), key=lambda item: as_int(item.get("id"))) + for item in sorted( + row.get("series", []), key=lambda item: as_int(item.get("id")) + ) ] category = as_int(row.get("category")) return { diff --git a/python/src/mlm/templates/base.html b/python/src/mlm/templates/base.html index 5cc1de66..9aa45445 100644 --- a/python/src/mlm/templates/base.html +++ b/python/src/mlm/templates/base.html @@ -27,6 +27,7 @@