diff --git a/README.md b/README.md index 471d617..e13561b 100644 --- a/README.md +++ b/README.md @@ -183,20 +183,21 @@ export MAM_ID="..." ## Configuration Reference -| Option | Description | Default | -| ----------------------- | --------------------------- | ----------- | -| `hardcover_api_keys` | List of hardcover api keys | required | -| `qbittorrent` | qBittorrent connection | required | -| `calibre_db_path` | Path to Calibre metadata.db | required | -| `calibredb_executable` | Path to calibredb | `calibredb` | -| `mam_id` | MyAnonamouse session cookie | required | -| `matcher_threshold` | Fuzzy match sensitivity | `0.7` | -| `lang_codes` | Allowed languages | `["ENG"]` | -| `schedule` | Cron expression | `0 * * * *` | -| `redact_sensitive_data` | Hide secrets in logs | `true` | -| `apprise` | Notifications via [Apprise](https://appriseit.com/getting-started/configuration/) | `None` | -| `filetypes` | List of filetypes to support | `["epub", "mobi", "azw3", "azw"]` | -| `blacklisted_torrent_ids`| List of MaM torrent id's to blacklist | `[]` | +| Option | Description | Default | +| ------------------------- | ------------------------------------- | ----------- | +| `hardcover_api_keys` | List of hardcover api keys | required | +| `qbittorrent` | qBittorrent connection | required | +| `calibre_db_path` | Path to Calibre metadata.db | required | +| `calibredb_executable` | Path to calibredb | `calibredb` | +| `mam_id` | MyAnonamouse session cookie | required | +| `matcher_threshold` | Fuzzy match sensitivity | `0.7` | +| `lang_codes` | Allowed languages | `["ENG"]` | +| `schedule` | Cron expression | `0 * * * *` | +| `redact_sensitive_data` | Hide secrets in logs | `true` | +| `apprise` | Notifications via [Apprise](https://appriseit.com/getting-started/configuration/) | `None` | +| `filetypes` | List of filetypes to support | `["epub", "mobi", "azw3", "azw"]` | +| `blacklisted_torrent_ids` | List of MaM torrent id's to blacklist | `[]` | +| `torrent_timeout_seconds` | Seconds before a torrent is considered to have timed out | `1800` | ### qBittorrent diff --git a/nixosModule.nix b/nixosModule.nix index 8ecbaff..03b7a50 100644 --- a/nixosModule.nix +++ b/nixosModule.nix @@ -185,6 +185,11 @@ in { type = lib.types.listOf lib.types.int; default = []; }; + + torrent_timeout_seconds = lib.mkOption { + description = "Number of seconds before a torrent download is considered timed out"; + default = 1800; # 30 minutes + }; }; }; }; diff --git a/src/murid/__init__.py b/src/murid/__init__.py index a39a56a..f9399d3 100644 --- a/src/murid/__init__.py +++ b/src/murid/__init__.py @@ -10,6 +10,7 @@ from .domain.torrent import Torrent, TorrentMetadata from .domain.torrent_selector import TorrentSelector from .notifications.apprise import AppriseHandler, init_apprise, send_test_notification +from .services.retry_service import RetryService from .services.service_factory import ServiceFactory from .services.sync_service import SyncService from .services.torrent_discovery import TorrentDiscoveryService @@ -35,6 +36,7 @@ "MAMError", "MyAnonamouse", "MyAnonamouseQuery", + "RetryService", "send_test_notification", "ServiceFactory", "SyncService", diff --git a/src/murid/clients/myanonamouse.py b/src/murid/clients/myanonamouse.py index 7e35d32..bc248d1 100644 --- a/src/murid/clients/myanonamouse.py +++ b/src/murid/clients/myanonamouse.py @@ -23,6 +23,7 @@ class MyAnonamouseQuery: categories: list[int] | None = None search_fields: list[str] | None = None main_categories: list[int] | None = None + id: int | None = None class MAMError(Exception): @@ -73,12 +74,13 @@ def search(self, query: MyAnonamouseQuery) -> set[Torrent]: "searchIn": "torrents", "sortType": "default", "startNumber": str(0), + "id": str(query.id) if query.id is not None else "", }, "dlLink": "true", "isbn": "true", } - logger.debug("Searching MyAnonamouse for %s", query.text) + logger.debug("Searching MyAnonamouse for %s", query.text if query.text else str(query.id)) try: response = self.session.post( self.SEARCH_URL, diff --git a/src/murid/clients/torrent_clients/qbittorrent.py b/src/murid/clients/torrent_clients/qbittorrent.py index 85db4ea..c4124b0 100644 --- a/src/murid/clients/torrent_clients/qbittorrent.py +++ b/src/murid/clients/torrent_clients/qbittorrent.py @@ -96,3 +96,14 @@ def get_completed_path(self, torrent_id: str) -> str | None: def add_tag(self, torrent_id: str, tag: str) -> None: """Add a tag to a torrent by its ID.""" self.client.torrents_add_tags(tags=tag, torrent_hashes=torrent_id) + + def remove_tag(self, torrent_id: str, tag: str) -> None: + """Remove a tag from a torrent by its ID.""" + self.client.torrents_remove_tags(tags=tag, torrent_hashes=torrent_id) + + def get_torrents_with_tag(self, tag: str) -> dict: + """Get a list of torrent IDs and MaM IDs that have the specified tag.""" + torrents = [ + t for t in self.client.torrents_info(tags=tag) if tag in (t.tags or "").split(", ") + ] + return torrents diff --git a/src/murid/config/config.py b/src/murid/config/config.py index af8110a..bf87d9f 100644 --- a/src/murid/config/config.py +++ b/src/murid/config/config.py @@ -51,6 +51,7 @@ def __repr__(self) -> str: "azw", ], "blacklisted_torrent_ids": [], + "torrent_timeout_seconds": 1800, } @@ -154,6 +155,7 @@ def validate(self) -> None: self._ensure_cron(self.get("schedule")) self._ensure_filetypes(self.get("filetypes")) self._ensure_blacklisted_torrent_ids(self.get("blacklisted_torrent_ids")) + self._ensure_type(self.get("torrent_timeout_seconds"), int, "torrent_timeout_seconds") self._check_extra_keys(self._config) diff --git a/src/murid/services/retry_service.py b/src/murid/services/retry_service.py new file mode 100644 index 0000000..d15e387 --- /dev/null +++ b/src/murid/services/retry_service.py @@ -0,0 +1,74 @@ +"""This module defines the RetryService class.""" + +import logging +import re +from collections.abc import Iterator + +from ..clients.myanonamouse import MAMError, MyAnonamouse, MyAnonamouseQuery +from ..clients.torrent_clients import TorrentClient +from ..domain.book import Book +from .torrent_import import TorrentImportService + +logger = logging.getLogger("murid") + + +class RetryService: + """Service responsible for retrying torrents that previously timed out during import.""" + + def __init__( + self, + torrent_client: TorrentClient, + import_service: TorrentImportService, + myanonamouse: MyAnonamouse, + ) -> None: + """Initialize the RetryService with the necessary dependencies.""" + self.torrent_client = torrent_client + self.import_service = import_service + self.myanonamouse = myanonamouse + + def retry_torrents(self) -> None: + """Retry torrents that previously timed out during import.""" + for torrent_id, mam_id in self.fetch_previous_torrents(): + book = self.get_book_by_mam_id(mam_id) + if book: + if self.import_service.process_torrent(torrent_id, book): + self.torrent_client.remove_tag(torrent_id, "murid_timeout") + else: + logger.warning("Could not import %s, will retry later", book) + + def fetch_previous_torrents(self) -> Iterator[tuple[str, int]]: + """Fetch torrents that previously timed out during import.""" + previous_torrents = self.torrent_client.get_torrents_with_tag("murid_timeout") + if not previous_torrents: + logger.debug("No previously timed out torrents found with tag 'murid_timeout'") + return + logger.debug( + "Found %d previously timed out torrents with tag 'murid_timeout'", + len(previous_torrents), + ) + for torrent in previous_torrents: + match = re.search(r"MID=(\d+)", torrent.comment or "") + if match: + mam_id = int(match.group(1)) + yield (torrent.hash, mam_id) + + def get_book_by_mam_id(self, mam_id: int) -> Book | None: + """Fetch the book information from MyAnonamouse using the provided mam_id.""" + try: + result = self.myanonamouse.search(MyAnonamouseQuery(text="", id=mam_id)) + if not result: + logger.warning("No torrent found on MyAnonamouse for ID %d", mam_id) + return None + + if len(result) > 1: + logger.warning( + "Multiple torrents found on MyAnonamouse for ID %d, expected only one", mam_id + ) + return None + + torrent = next(iter(result)) + return torrent.book + + except MAMError as e: + logger.error("Error searching MyAnonamouse for ID %d: %s", mam_id, e) + return None diff --git a/src/murid/services/service_factory.py b/src/murid/services/service_factory.py index abae62d..3725311 100644 --- a/src/murid/services/service_factory.py +++ b/src/murid/services/service_factory.py @@ -14,6 +14,7 @@ from ..domain.book_matcher import BookMatcher from ..domain.torrent_selector import TorrentSelector from ..notifications.apprise import init_apprise as apprise +from .retry_service import RetryService from .sync_service import SyncService from .torrent_discovery import TorrentDiscoveryService from .torrent_import import TorrentImportConfig, TorrentImportService @@ -91,8 +92,10 @@ def myanonamouse(self): def qbittorrent(self): """Create a Qbittorrent instance using the configuration for qBittorrent.""" - config = self.config.copy() # Make a copy to avoid modifying the original config - qbittorrent_config = config["qbittorrent"] + # config = self.config.copy() # Make a copy to avoid modifying the original config + qbittorrent_config = self.config[ + "qbittorrent" + ].copy() # Make a copy to avoid modifying the original config mapping = qbittorrent_config.pop("mapping", None) category = qbittorrent_config.pop("category", "murid") qbittorrent_config["VERIFY_WEBUI_CERTIFICATE"] = qbittorrent_config.pop("verify_cert", True) @@ -126,6 +129,7 @@ def torrent_import(self): matcher=self.matcher(), notify=self.notifier(), dry_run=self.args.dry_run, + timeout=self.config["torrent_timeout_seconds"], ) ) @@ -144,3 +148,11 @@ def torrent_selector(self): wanted_filetypes=set(self.config["filetypes"]), blacklist=set(self.config["blacklisted_torrent_ids"]), ) + + def retry_service(self): + """Create a RetryService instance.""" + return RetryService( + self.qbittorrent(), + self.torrent_import(), + self.myanonamouse(), + ) diff --git a/src/murid/services/sync_service.py b/src/murid/services/sync_service.py index 166a8c5..0bce89e 100644 --- a/src/murid/services/sync_service.py +++ b/src/murid/services/sync_service.py @@ -46,6 +46,9 @@ def start_scheduler(self) -> None: def run(self) -> None: """Run the synchronization process.""" + retry_service = self.factory.retry_service() + retry_service.retry_torrents() + calibre = self.factory.calibre() calibre_books = self.fetch_calibre_books(calibre) diff --git a/src/murid/services/torrent_import.py b/src/murid/services/torrent_import.py index 5012d8a..6b4abcf 100644 --- a/src/murid/services/torrent_import.py +++ b/src/murid/services/torrent_import.py @@ -24,6 +24,7 @@ class TorrentImportConfig: matcher: BookMatcher notify: Callable[[str, str, apprise.NotifyType], None] dry_run: bool = False + timeout: int = 1800 # 30 minutes class TorrentImportService: @@ -36,6 +37,7 @@ def __init__(self, config: TorrentImportConfig) -> None: self.calibre = config.calibre self.dry_run = config.dry_run self.matcher = config.matcher + self.timeout = config.timeout def import_torrents(self, torrent_files: Iterable[tuple[bytes, Book]]) -> None: """Import the given torrent files into the torrent client and track their completion.""" @@ -63,6 +65,7 @@ def import_torrents(self, torrent_files: Iterable[tuple[bytes, Book]]) -> None: logger.info("Tracking %d torrents for completion", len(pending)) + start_time = time.time() while pending: completed = [] @@ -77,11 +80,19 @@ def import_torrents(self, torrent_files: Iterable[tuple[bytes, Book]]) -> None: logger.debug("%d torrents still active", len(pending)) time.sleep(self.torrent_client.poll_interval) + if self._is_timeout(start_time, pending): + break + def process_torrent(self, torrent_id: str, book: Book) -> bool: """Check if the torrent with the given ID is completed and import it into Calibre.""" path = self.torrent_client.get_completed_path(torrent_id) if not path: return False + + if self.dry_run: + logger.info("Dry run enabled, not importing %s into Calibre", book) + return True + try: time.sleep(0.5) # Give the torrent client a moment to move the files path = self.torrent_client.get_completed_path(torrent_id) @@ -96,9 +107,31 @@ def process_torrent(self, torrent_id: str, book: Book) -> bool: "\nManual intervention is likely required.", book, ) - self.torrent_client.add_tag(torrent_id, "import_failed") + self.torrent_client.add_tag(torrent_id, "murid_import_failed") except CalibreError: logger.error("Error importing %s into Calibre", book) - self.torrent_client.add_tag(torrent_id, "import_failed") + self.torrent_client.add_tag(torrent_id, "murid_import_failed") return True + + def _is_timeout(self, start_time: float, pending: dict) -> bool: + """Check if the timeout has been reached since the given start time.""" + if time.time() - start_time > self.timeout: + logger.warning( + "Timeout reached while waiting for torrent to complete. " + "The following torrents did not complete:\n%s", + "\n".join(f"- {book} (torrent ID: {tid})" for tid, book in pending.items()), + ) + for tid, book in pending.items(): + self.notify( + title="Murid - Torrent download timeout", + body=( + f"The torrent for {book} did not complete within " + f"{self.timeout} seconds. If completed it will be imported " + "in the next run, otherwise you may need to investigate the " + "torrent client for issues." + ), + ) + self.torrent_client.add_tag(tid, "murid_timeout") + return True + return False diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 79783d4..0df16f8 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -457,3 +457,8 @@ def test_blacklisted_torrent_ids_list_items_must_be_int(build_config): def test_blacklisted_torrent_ids_can_be_set(build_config): config = build_config(blacklisted_torrent_ids=[123, 456]) assert config.get("blacklisted_torrent_ids") == [123, 456] + + +def test_torrent_timeout_seconds_must_be_int(build_config): + with pytest.raises(ConfigError, match="Config item 'torrent_timeout_seconds' must be an int"): + build_config(torrent_timeout_seconds="not-an-int") diff --git a/tests/services/test_retry_service.py b/tests/services/test_retry_service.py new file mode 100644 index 0000000..a87e6dc --- /dev/null +++ b/tests/services/test_retry_service.py @@ -0,0 +1,183 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from murid.clients.myanonamouse import MAMError +from murid.domain.book import Book +from murid.domain.torrent import Torrent, TorrentMetadata +from murid.services.retry_service import RetryService + + +@pytest.fixture +def torrent_client(): + return Mock() + + +@pytest.fixture +def import_service(): + return Mock() + + +@pytest.fixture +def mam(): + return Mock() + + +@pytest.fixture +def service(torrent_client, import_service, mam): + return RetryService( + torrent_client=torrent_client, + import_service=import_service, + myanonamouse=mam, + ) + + +@pytest.fixture +def book(): + return Book( + title="Dune", + authors=["Frank Herbert"], + id=1, + isbn=[], + source="test", + ) + + +def test_fetch_previous_torrents_empty(service, torrent_client): + torrent_client.get_torrents_with_tag.return_value = [] + + assert list(service.fetch_previous_torrents()) == [] + + +def test_fetch_previous_torrents_valid_mid(service, torrent_client): + torrent_client.get_torrents_with_tag.return_value = [ + SimpleNamespace( + hash="torrent1", + comment="something MID=123 something", + ) + ] + + assert list(service.fetch_previous_torrents()) == [("torrent1", 123)] + + +def test_fetch_previous_torrents_ignores_missing_mid( + service, + torrent_client, +): + torrent_client.get_torrents_with_tag.return_value = [ + SimpleNamespace( + hash="torrent1", + comment="no mam id here", + ) + ] + + assert list(service.fetch_previous_torrents()) == [] + + +def test_fetch_previous_torrents_mixed(service, torrent_client): + torrent_client.get_torrents_with_tag.return_value = [ + SimpleNamespace(hash="a", comment="MID=111"), + SimpleNamespace(hash="b", comment="nothing"), + SimpleNamespace(hash="c", comment="MID=222"), + ] + + assert list(service.fetch_previous_torrents()) == [ + ("a", 111), + ("c", 222), + ] + + +def make_torrent(book): + return Torrent( + book=book, + metadata=TorrentMetadata( + category=1, + size=100, + seeders=1, + leechers=0, + freeleech=False, + vip=False, + ), + ) + + +def test_get_book_by_mam_id_success(service, mam, book): + mam.search.return_value = [make_torrent(book)] + + result = service.get_book_by_mam_id(123) + + assert result == book + + +def test_get_book_by_mam_id_no_results(service, mam): + mam.search.return_value = [] + + assert service.get_book_by_mam_id(123) is None + + +def test_get_book_by_mam_id_multiple_results(service, mam, book): + mam.search.return_value = [ + make_torrent(book), + make_torrent(book), + ] + + assert service.get_book_by_mam_id(123) is None + + +def test_get_book_by_mam_id_mam_error(service, mam): + mam.search.side_effect = MAMError("boom") + + assert service.get_book_by_mam_id(123) is None + + +def test_retry_torrents_success( + service, + import_service, + torrent_client, + book, +): + service.fetch_previous_torrents = Mock(return_value=[("torrent1", 123)]) + + service.get_book_by_mam_id = Mock(return_value=book) + + import_service.process_torrent.return_value = True + + service.retry_torrents() + + torrent_client.remove_tag.assert_called_once_with( + "torrent1", + "murid_timeout", + ) + + +def test_retry_torrents_import_failed( + service, + import_service, + torrent_client, + book, +): + service.fetch_previous_torrents = Mock(return_value=[("torrent1", 123)]) + + service.get_book_by_mam_id = Mock(return_value=book) + + import_service.process_torrent.return_value = False + + service.retry_torrents() + + torrent_client.remove_tag.assert_not_called() + + +def test_retry_torrents_book_not_found( + service, + import_service, + torrent_client, +): + service.fetch_previous_torrents = Mock(return_value=[("torrent1", 123)]) + + service.get_book_by_mam_id = Mock(return_value=None) + + service.retry_torrents() + + import_service.process_torrent.assert_not_called() + torrent_client.remove_tag.assert_not_called() diff --git a/tests/services/test_sync_service.py b/tests/services/test_sync_service.py index b921123..d22db3f 100644 --- a/tests/services/test_sync_service.py +++ b/tests/services/test_sync_service.py @@ -24,6 +24,7 @@ def factory(): matcher=Mock(), torrent_discovery=Mock(), torrent_import=Mock(), + retry_service=Mock(), ) @@ -98,11 +99,14 @@ def test_run_orchestrates_pipeline(factory): torrent_import = Mock() + retry_service = Mock() + factory.calibre.return_value = calibre factory.hardcover.return_value = [hardcover_client] factory.matcher.return_value = matcher factory.torrent_discovery.return_value = torrent_discovery factory.torrent_import.return_value = torrent_import + factory.retry_service.return_value = retry_service service.run() @@ -111,6 +115,7 @@ def test_run_orchestrates_pipeline(factory): factory.matcher.assert_called_once() factory.torrent_discovery.assert_called_once() factory.torrent_import.assert_called_once() + factory.retry_service.assert_called_once() torrent_import.import_torrents.assert_called_once() diff --git a/tests/services/test_torrent_import.py b/tests/services/test_torrent_import.py index 0890da4..a3c689a 100644 --- a/tests/services/test_torrent_import.py +++ b/tests/services/test_torrent_import.py @@ -1,3 +1,4 @@ +import time from unittest.mock import MagicMock import pytest @@ -102,7 +103,7 @@ def test_calibre_error_adds_tag(service, torrent_client, calibre, book): service.process_torrent("abc123", book) - torrent_client.add_tag.assert_called_once_with("abc123", "import_failed") + torrent_client.add_tag.assert_called_once_with("abc123", "murid_import_failed") def test_success_triggers_notification(service, torrent_client, calibre, notify, book): @@ -123,7 +124,7 @@ def test_failed_verification_adds_tag(service, torrent_client, calibre, book): service.process_torrent("abc123", book) - torrent_client.add_tag.assert_called_once_with("abc123", "import_failed") + torrent_client.add_tag.assert_called_once_with("abc123", "murid_import_failed") def test_process_torrent_no_retry(service, torrent_client, calibre, book): @@ -134,3 +135,84 @@ def test_process_torrent_no_retry(service, torrent_client, calibre, book): assert result is False assert torrent_client.get_completed_path.call_count == 1 calibre.add_book.assert_not_called() + + +def test_is_timeout_returns_false_before_timeout(service): + service.timeout = 1800 + + result = service._is_timeout( + start_time=time.time(), + pending={"abc": "book"}, + ) + + assert result is False + + +def test_is_timeout_returns_true_after_timeout( + service, + torrent_client, + notify, +): + service.timeout = 1 + + pending = { + "abc123": "Dune", + } + + result = service._is_timeout( + start_time=time.time() - 10, + pending=pending, + ) + + assert result is True + + notify.assert_called_once() + + torrent_client.add_tag.assert_called_once_with( + "abc123", + "murid_timeout", + ) + + +def test_is_timeout_tags_all_pending_torrents( + service, + torrent_client, + notify, +): + service.timeout = 1 + + pending = { + "a": "Book A", + "b": "Book B", + "c": "Book C", + } + + service._is_timeout( + start_time=time.time() - 10, + pending=pending, + ) + + assert torrent_client.add_tag.call_count == 3 + + torrent_client.add_tag.assert_any_call("a", "murid_timeout") + torrent_client.add_tag.assert_any_call("b", "murid_timeout") + torrent_client.add_tag.assert_any_call("c", "murid_timeout") + + assert notify.call_count == 3 + + +def test_timeout_notification_contains_timeout_value( + service, + torrent_client, + notify, +): + service.timeout = 123 + + service._is_timeout( + start_time=time.time() - 1000, + pending={"abc": "Dune"}, + ) + + _, kwargs = notify.call_args + + assert "123 seconds" in kwargs["body"]