retry torrents that have timed out

README.md

@@ -183,20 +183,21 @@ export MAM_ID="..."
 
 ## Configuration Reference
 
-| Option                  | Description                 | Default     |
-| ----------------------- | --------------------------- | ----------- |
-| `hardcover_api_keys`    | List of hardcover api keys  | required    |
-| `qbittorrent`           | qBittorrent connection      | required    |
-| `calibre_db_path`       | Path to Calibre metadata.db | required    |
-| `calibredb_executable`  | Path to calibredb           | `calibredb` |
-| `mam_id`                | MyAnonamouse session cookie | required    |
-| `matcher_threshold`     | Fuzzy match sensitivity     | `0.7`       |
-| `lang_codes`            | Allowed languages           | `["ENG"]`   |
-| `schedule`              | Cron expression             | `0 * * * *` |
-| `redact_sensitive_data` | Hide secrets in logs        | `true`      |
-| `apprise`               | Notifications via [Apprise](https://appriseit.com/getting-started/configuration/)   | `None`      |
-| `filetypes`             | List of filetypes to support | `["epub", "mobi", "azw3", "azw"]` |
-| `blacklisted_torrent_ids`| List of MaM torrent id's to blacklist | `[]` |
+| Option                    | Description                           | Default     |
+| ------------------------- | ------------------------------------- | ----------- |
+| `hardcover_api_keys`      | List of hardcover api keys            | required    |
+| `qbittorrent`             | qBittorrent connection                | required    |
+| `calibre_db_path`         | Path to Calibre metadata.db           | required    |
+| `calibredb_executable`    | Path to calibredb                     | `calibredb` |
+| `mam_id`                  | MyAnonamouse session cookie           | required    |
+| `matcher_threshold`       | Fuzzy match sensitivity               | `0.7`       |
+| `lang_codes`              | Allowed languages                     | `["ENG"]`   |
+| `schedule`                | Cron expression                       | `0 * * * *` |
+| `redact_sensitive_data`   | Hide secrets in logs                  | `true`      |
+| `apprise`                 | Notifications via [Apprise](https://appriseit.com/getting-started/configuration/)   | `None`      |
+| `filetypes`               | List of filetypes to support          | `["epub", "mobi", "azw3", "azw"]` |
+| `blacklisted_torrent_ids` | List of MaM torrent id's to blacklist | `[]` |
+| `torrent_timeout_seconds` | Seconds before a torrent is considered to have timed out | `1800` |
 
 ### qBittorrent
 

nixosModule.nix

@@ -185,6 +185,11 @@ in {
               type = lib.types.listOf lib.types.int;
               default = [];
             };
+
+            torrent_timeout_seconds = lib.mkOption {
+              description = "Number of seconds before a torrent download is considered timed out";
+              default = 1800; # 30 minutes
+            };
           };
         };
       };

src/murid/__init__.py

@@ -10,6 +10,7 @@
 from .domain.torrent import Torrent, TorrentMetadata
 from .domain.torrent_selector import TorrentSelector
 from .notifications.apprise import AppriseHandler, init_apprise, send_test_notification
+from .services.retry_service import RetryService
 from .services.service_factory import ServiceFactory
 from .services.sync_service import SyncService
 from .services.torrent_discovery import TorrentDiscoveryService
@@ -35,6 +36,7 @@
     "MAMError",
     "MyAnonamouse",
     "MyAnonamouseQuery",
+    "RetryService",
     "send_test_notification",
     "ServiceFactory",
     "SyncService",

src/murid/clients/myanonamouse.py

@@ -23,6 +23,7 @@ class MyAnonamouseQuery:
     categories: list[int] | None = None
     search_fields: list[str] | None = None
     main_categories: list[int] | None = None
+    id: int | None = None
 
 
 class MAMError(Exception):
@@ -73,12 +74,13 @@ def search(self, query: MyAnonamouseQuery) -> set[Torrent]:
                 "searchIn": "torrents",
                 "sortType": "default",
                 "startNumber": str(0),
+                "id": str(query.id) if query.id is not None else "",
             },
             "dlLink": "true",
             "isbn": "true",
         }
 
-        logger.debug("Searching MyAnonamouse for %s", query.text)
+        logger.debug("Searching MyAnonamouse for %s", query.text if query.text else str(query.id))
         try:
             response = self.session.post(
                 self.SEARCH_URL,

src/murid/clients/torrent_clients/qbittorrent.py

@@ -96,3 +96,14 @@ def get_completed_path(self, torrent_id: str) -> str | None:
     def add_tag(self, torrent_id: str, tag: str) -> None:
         """Add a tag to a torrent by its ID."""
         self.client.torrents_add_tags(tags=tag, torrent_hashes=torrent_id)
+
+    def remove_tag(self, torrent_id: str, tag: str) -> None:
+        """Remove a tag from a torrent by its ID."""
+        self.client.torrents_remove_tags(tags=tag, torrent_hashes=torrent_id)
+
+    def get_torrents_with_tag(self, tag: str) -> dict:
+        """Get a list of torrent IDs and MaM IDs that have the specified tag."""
+        torrents = [
+            t for t in self.client.torrents_info(tags=tag) if tag in (t.tags or "").split(", ")
+        ]
+        return torrents

src/murid/config/config.py

@@ -51,6 +51,7 @@ def __repr__(self) -> str:
         "azw",
     ],
     "blacklisted_torrent_ids": [],
+    "torrent_timeout_seconds": 1800,
 }
 
 
@@ -154,6 +155,7 @@ def validate(self) -> None:
         self._ensure_cron(self.get("schedule"))
         self._ensure_filetypes(self.get("filetypes"))
         self._ensure_blacklisted_torrent_ids(self.get("blacklisted_torrent_ids"))
+        self._ensure_type(self.get("torrent_timeout_seconds"), int, "torrent_timeout_seconds")
 
         self._check_extra_keys(self._config)
 

src/murid/services/retry_service.py

@@ -0,0 +1,74 @@
+"""This module defines the RetryService class."""
+
+import logging
+import re
+from collections.abc import Iterator
+
+from ..clients.myanonamouse import MAMError, MyAnonamouse, MyAnonamouseQuery
+from ..clients.torrent_clients import TorrentClient
+from ..domain.book import Book
+from .torrent_import import TorrentImportService
+
+logger = logging.getLogger("murid")
+
+
+class RetryService:
+    """Service responsible for retrying torrents that previously timed out during import."""
+
+    def __init__(
+        self,
+        torrent_client: TorrentClient,
+        import_service: TorrentImportService,
+        myanonamouse: MyAnonamouse,
+    ) -> None:
+        """Initialize the RetryService with the necessary dependencies."""
+        self.torrent_client = torrent_client
+        self.import_service = import_service
+        self.myanonamouse = myanonamouse
+
+    def retry_torrents(self) -> None:
+        """Retry torrents that previously timed out during import."""
+        for torrent_id, mam_id in self.fetch_previous_torrents():
+            book = self.get_book_by_mam_id(mam_id)
+            if book:
+                if self.import_service.process_torrent(torrent_id, book):
+                    self.torrent_client.remove_tag(torrent_id, "murid_timeout")
+                else:
+                    logger.warning("Could not import %s, will retry later", book)
+
+    def fetch_previous_torrents(self) -> Iterator[tuple[str, int]]:
+        """Fetch torrents that previously timed out during import."""
+        previous_torrents = self.torrent_client.get_torrents_with_tag("murid_timeout")
+        if not previous_torrents:
+            logger.debug("No previously timed out torrents found with tag 'murid_timeout'")
+            return
+        logger.debug(
+            "Found %d previously timed out torrents with tag 'murid_timeout'",
+            len(previous_torrents),
+        )
+        for torrent in previous_torrents:
+            match = re.search(r"MID=(\d+)", torrent.comment or "")
+            if match:
+                mam_id = int(match.group(1))
+                yield (torrent.hash, mam_id)
+
+    def get_book_by_mam_id(self, mam_id: int) -> Book | None:
+        """Fetch the book information from MyAnonamouse using the provided mam_id."""
+        try:
+            result = self.myanonamouse.search(MyAnonamouseQuery(text="", id=mam_id))
+            if not result:
+                logger.warning("No torrent found on MyAnonamouse for ID %d", mam_id)
+                return None
+
+            if len(result) > 1:
+                logger.warning(
+                    "Multiple torrents found on MyAnonamouse for ID %d, expected only one", mam_id
+                )
+                return None
+
+            torrent = next(iter(result))
+            return torrent.book
+
+        except MAMError as e:
+            logger.error("Error searching MyAnonamouse for ID %d: %s", mam_id, e)
+            return None

src/murid/services/service_factory.py

@@ -14,6 +14,7 @@
 from ..domain.book_matcher import BookMatcher
 from ..domain.torrent_selector import TorrentSelector
 from ..notifications.apprise import init_apprise as apprise
+from .retry_service import RetryService
 from .sync_service import SyncService
 from .torrent_discovery import TorrentDiscoveryService
 from .torrent_import import TorrentImportConfig, TorrentImportService
@@ -91,8 +92,10 @@ def myanonamouse(self):
 
     def qbittorrent(self):
         """Create a Qbittorrent instance using the configuration for qBittorrent."""
-        config = self.config.copy()  # Make a copy to avoid modifying the original config
-        qbittorrent_config = config["qbittorrent"]
+        # config = self.config.copy()  # Make a copy to avoid modifying the original config
+        qbittorrent_config = self.config[
+            "qbittorrent"
+        ].copy()  # Make a copy to avoid modifying the original config
         mapping = qbittorrent_config.pop("mapping", None)
         category = qbittorrent_config.pop("category", "murid")
         qbittorrent_config["VERIFY_WEBUI_CERTIFICATE"] = qbittorrent_config.pop("verify_cert", True)
@@ -126,6 +129,7 @@ def torrent_import(self):
                 matcher=self.matcher(),
                 notify=self.notifier(),
                 dry_run=self.args.dry_run,
+                timeout=self.config["torrent_timeout_seconds"],
             )
         )
 
@@ -144,3 +148,11 @@ def torrent_selector(self):
             wanted_filetypes=set(self.config["filetypes"]),
             blacklist=set(self.config["blacklisted_torrent_ids"]),
         )
+
+    def retry_service(self):
+        """Create a RetryService instance."""
+        return RetryService(
+            self.qbittorrent(),
+            self.torrent_import(),
+            self.myanonamouse(),
+        )

src/murid/services/sync_service.py

@@ -46,6 +46,9 @@ def start_scheduler(self) -> None:
 
     def run(self) -> None:
         """Run the synchronization process."""
+        retry_service = self.factory.retry_service()
+        retry_service.retry_torrents()
+
         calibre = self.factory.calibre()
         calibre_books = self.fetch_calibre_books(calibre)
 

src/murid/services/torrent_import.py

@@ -24,6 +24,7 @@ class TorrentImportConfig:
     matcher: BookMatcher
     notify: Callable[[str, str, apprise.NotifyType], None]
     dry_run: bool = False
+    timeout: int = 1800  # 30 minutes
 
 
 class TorrentImportService:
@@ -36,6 +37,7 @@ def __init__(self, config: TorrentImportConfig) -> None:
         self.calibre = config.calibre
         self.dry_run = config.dry_run
         self.matcher = config.matcher
+        self.timeout = config.timeout
 
     def import_torrents(self, torrent_files: Iterable[tuple[bytes, Book]]) -> None:
         """Import the given torrent files into the torrent client and track their completion."""
@@ -63,6 +65,7 @@ def import_torrents(self, torrent_files: Iterable[tuple[bytes, Book]]) -> None:
 
         logger.info("Tracking %d torrents for completion", len(pending))
 
+        start_time = time.time()
         while pending:
             completed = []
 
@@ -77,11 +80,19 @@ def import_torrents(self, torrent_files: Iterable[tuple[bytes, Book]]) -> None:
                 logger.debug("%d torrents still active", len(pending))
                 time.sleep(self.torrent_client.poll_interval)
 
+            if self._is_timeout(start_time, pending):
+                break
+
     def process_torrent(self, torrent_id: str, book: Book) -> bool:
         """Check if the torrent with the given ID is completed and import it into Calibre."""
         path = self.torrent_client.get_completed_path(torrent_id)
         if not path:
             return False
+
+        if self.dry_run:
+            logger.info("Dry run enabled, not importing %s into Calibre", book)
+            return True
+
         try:
             time.sleep(0.5)  # Give the torrent client a moment to move the files
             path = self.torrent_client.get_completed_path(torrent_id)
@@ -96,9 +107,31 @@ def process_torrent(self, torrent_id: str, book: Book) -> bool:
                     "\nManual intervention is likely required.",
                     book,
                 )
-                self.torrent_client.add_tag(torrent_id, "import_failed")
+                self.torrent_client.add_tag(torrent_id, "murid_import_failed")
         except CalibreError:
             logger.error("Error importing %s into Calibre", book)
-            self.torrent_client.add_tag(torrent_id, "import_failed")
+            self.torrent_client.add_tag(torrent_id, "murid_import_failed")
 
         return True
+
+    def _is_timeout(self, start_time: float, pending: dict) -> bool:
+        """Check if the timeout has been reached since the given start time."""
+        if time.time() - start_time > self.timeout:
+            logger.warning(
+                "Timeout reached while waiting for torrent to complete. "
+                "The following torrents did not complete:\n%s",
+                "\n".join(f"- {book} (torrent ID: {tid})" for tid, book in pending.items()),
+            )
+            for tid, book in pending.items():
+                self.notify(
+                    title="Murid - Torrent download timeout",
+                    body=(
+                        f"The torrent for {book} did not complete within "
+                        f"{self.timeout} seconds. If completed it will be imported "
+                        "in the next run, otherwise you may need to investigate the "
+                        "torrent client for issues."
+                    ),
+                )
+                self.torrent_client.add_tag(tid, "murid_timeout")
+            return True
+        return False

tests/config/test_config.py

@@ -457,3 +457,8 @@ def test_blacklisted_torrent_ids_list_items_must_be_int(build_config):
 def test_blacklisted_torrent_ids_can_be_set(build_config):
     config = build_config(blacklisted_torrent_ids=[123, 456])
     assert config.get("blacklisted_torrent_ids") == [123, 456]
+
+
+def test_torrent_timeout_seconds_must_be_int(build_config):
+    with pytest.raises(ConfigError, match="Config item 'torrent_timeout_seconds' must be an int"):
+        build_config(torrent_timeout_seconds="not-an-int")