diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be96bc79db..e178509aae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,7 @@ jobs: tests/bazarr/test_rar_extraction_production_config.py \ tests/bazarr/test_provider_hub_auto_install_optin.py \ tests/bazarr/test_provider_hub_archive_select.py \ + tests/bazarr/test_provider_hub_archive_candidates.py \ tests/bazarr/test_archive_member_matching.py \ tests/bazarr/test_provider_hub_worker_timeout.py \ tests/bazarr/test_arr_instances_schema.py \ @@ -203,6 +204,7 @@ jobs: tests/bazarr/test_processing_sync_substep.py \ tests/bazarr/test_mass_download_path_mapping.py \ tests/bazarr/test_processing_postprocessing.py \ + tests/bazarr/test_upload_background_sync.py \ tests/bazarr/test_sync_progress_report.py \ tests/bazarr/test_subsync_engines.py \ tests/bazarr/test_autosubsync_diagnostics.py \ @@ -276,6 +278,11 @@ jobs: tests/bazarr/test_jellyfin_client.py \ tests/bazarr/test_jellyfin_operations.py \ tests/bazarr/test_logging_filters.py \ + tests/bazarr/test_openrouter_provider_routing.py \ + tests/bazarr/test_openrouter_model_migration.py \ + tests/bazarr/test_openrouter_translator_partial.py \ + tests/bazarr/test_openrouter_translator.py \ + tests/bazarr/test_translator_model_migration.py \ tests/bazarr/test_provider_hub_install_serialization.py \ tests/bazarr/test_provider_hub_language_fallback.py \ tests/bazarr/test_provider_hub_release_scoring.py \ @@ -289,12 +296,14 @@ jobs: tests/bazarr/test_sonarr_sync_episodes.py \ tests/bazarr/test_sql_profiler.py \ tests/bazarr/test_subtitle_content_sync_outputs.py \ + tests/bazarr/test_subtitle_destination_path.py \ tests/bazarr/test_subtitle_download_api.py \ tests/bazarr/test_subtitles_pool.py \ tests/bazarr/test_swagger_static.py \ tests/bazarr/test_supervisor_proxy.py \ tests/bazarr/test_supervisor_stages.py \ tests/bazarr/test_tracemalloc_dumper.py \ + tests/bazarr/test_translate_job_media_name.py \ tests/bazarr/test_translator_auth.py \ tests/bazarr/test_upgrade_embedded_history.py \ tests/subliminal_patch/test_animetosho.py \ @@ -328,6 +337,7 @@ jobs: tests/bazarr/test_arr_pg_cutover_migration.py \ tests/bazarr/test_editor_instance_scope.py \ tests/bazarr/test_history_blacklist_scope.py \ + tests/bazarr/test_history_owner_resolution.py \ tests/bazarr/test_local_id_actions.py \ tests/bazarr/test_navigation_local_ids.py \ tests/bazarr/test_provider_blacklist_instance_scope.py \ @@ -350,6 +360,7 @@ jobs: tests/bazarr/test_indexer_owner_scope.py \ tests/bazarr/test_indexer_path_mapping_owner.py \ tests/bazarr/test_provider_hub_worker_reaper.py \ + tests/bazarr/test_provider_hub_worker_errors.py \ tests/bazarr/test_job_module_identity.py \ tests/bazarr/test_sonarr_sync_memory.py \ tests/bazarr/test_anilist_index.py \ diff --git a/README.md b/README.md index 39ae42e709..eb44187df7 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ Bazarr+ solves it with a **Provider Priority toggle** in Settings > Providers. W Upstream has Google Translate, Gemini, and Lingarr. Bazarr+ adds **OpenRouter** as a fourth translator engine, giving access to 300+ LLMs (Claude, Gemini, GPT, LLaMA, Grok, and more) plus any custom model ID from openrouter.ai. It runs as a separate microservice with an async job queue supporting 1-5 concurrent jobs and 1-8 parallel batches. Features include: - **Translate from the subtitle action menu**: click (...) on a missing subtitle row, pick an existing source subtitle to translate from - **Batch translation** for entire series/movie libraries from the Wanted pages -- **Dedicated settings page** with 4 zones: engine picker, connection config, model tuning (temperature, reasoning mode, parallel batches), and a live status panel showing queue stats, job progress, token usage, cost, and speed +- **Dedicated settings page** with 4 zones: engine picker, connection config, model tuning (temperature, reasoning mode, provider routing, parallel batches), and a live status panel showing queue stats, job progress, token usage, cost, and speed - **Model details** fetched live from the OpenRouter API with per-million token pricing, per-episode/movie cost estimates, context length, and prompt caching indicators - **AES-256-GCM encryption** for API keys in transit between Bazarr and the translator service, with a Test Connection button that validates encryption and API key status before saving - **Auto disk scan** triggers Sonarr/Radarr to rescan after translation completes @@ -455,7 +455,7 @@ The OpenSubtitles.org plugin no longer uses environment variables. Configure its 1. Go to **Settings** > **AI Translator** 2. Select **"AI Subtitle Translator"** as the translator engine 3. Enter your **OpenRouter API Key** (get one at [openrouter.ai/keys](https://openrouter.ai/keys)) -4. Choose your preferred **AI Model** (Google: Gemini 2.5 Flash Lite Preview 09-2025 recommended) +4. Choose your preferred **AI Model** (`google/gemini-2.5-flash-lite` recommended) 5. Save and test with a manual translation diff --git a/ai-subtitle-translator b/ai-subtitle-translator index 1ef643b7a9..16b32b3df3 160000 --- a/ai-subtitle-translator +++ b/ai-subtitle-translator @@ -1 +1 @@ -Subproject commit 1ef643b7a93a977879230526655a2a61817e4b6a +Subproject commit 16b32b3df34476634112f358ecbfeb24393f64b0 diff --git a/bazarr/api/subtitles/content.py b/bazarr/api/subtitles/content.py index 591a46ded2..88292f8c97 100644 --- a/bazarr/api/subtitles/content.py +++ b/bazarr/api/subtitles/content.py @@ -21,6 +21,7 @@ from subtitles.indexer.movies import store_subtitles_movie from subtitles.indexer.series import store_subtitles from subtitles.processing import ProcessSubtitlesResult +from subtitles.tools.subsync_engines import subtitle_mutation, subtitle_write_locks from utilities.helper import get_target_folder from utilities.path_mappings import path_mappings @@ -813,13 +814,6 @@ def _save_subtitle_content(media_type, media_id, language_code, arr_instance_id= subtitle_path, metadata = result - # Optimistic locking via ETag (optional but recommended) - if_match = request.headers.get('If-Match') - if if_match: - current_etag = generate_etag(subtitle_path) - if if_match.strip('"') != current_etag: - return 'Subtitle file has been modified since last read', 412 - data = request.get_json() if not data or 'content' not in data: return 'Request body must include "content" field', 400 @@ -838,8 +832,19 @@ def _save_subtitle_content(media_type, media_id, language_code, arr_instance_id= if len(encoded) > MAX_FILE_SIZE: return f'Content too large ({len(encoded)} bytes, max {MAX_FILE_SIZE})', 413 + video_path = path_mappings.path_replace_instance( + metadata['mediaPath'], metadata.get('arrInstanceId', arr_instance_id), media_type) try: - _write_bytes_atomically(subtitle_path, encoded) + with subtitle_write_locks(video_path, subtitle_path): + if not os.path.isfile(subtitle_path): + return 'Subtitle file or directory not found', 404 + if_match = request.headers.get('If-Match') + if if_match and if_match.strip('"') != generate_etag(subtitle_path): + return 'Subtitle file has been modified since last read', 412 + with subtitle_mutation(video_path, subtitle_path): + _write_bytes_atomically(subtitle_path, encoded) + _apply_subtitle_chmod(subtitle_path) + new_etag = generate_etag(subtitle_path) except FileNotFoundError: return 'Subtitle file or directory not found', 404 except PermissionError: @@ -849,11 +854,7 @@ def _save_subtitle_content(media_type, media_id, language_code, arr_instance_id= return 'No space left on device', 507 raise - _apply_subtitle_chmod(subtitle_path) - _refresh_media_subtitles(media_type, media_id, metadata) - - new_etag = generate_etag(subtitle_path) response = make_response('', 204) response.headers['ETag'] = f'"{new_etag}"' return response @@ -900,15 +901,20 @@ def promote_sync_subtitle(media_type, media_id, target_language, source_language if os.path.realpath(source_path) == os.path.realpath(target_path): return 'Source and target subtitles are the same file', 400 + video_path = path_mappings.path_replace_instance( + metadata['mediaPath'], metadata.get('arrInstanceId', arr_instance_id), media_type) try: - file_size = os.path.getsize(source_path) - if file_size > MAX_FILE_SIZE: - return f'Subtitle file too large ({file_size} bytes, max {MAX_FILE_SIZE})', 413 - - with open(source_path, 'rb') as source_file: - data = source_file.read() - - _write_bytes_atomically(target_path, data) + with subtitle_write_locks(video_path, source_path, target_path): + file_size = os.path.getsize(source_path) + if file_size > MAX_FILE_SIZE: + return f'Subtitle file too large ({file_size} bytes, max {MAX_FILE_SIZE})', 413 + if not os.path.isfile(target_path): + return 'Subtitle file or directory not found', 404 + with open(source_path, 'rb') as source_file: + data = source_file.read() + with subtitle_mutation(video_path, target_path): + _write_bytes_atomically(target_path, data) + _apply_subtitle_chmod(target_path) except FileNotFoundError: return 'Subtitle file or directory not found', 404 except PermissionError: @@ -918,16 +924,17 @@ def promote_sync_subtitle(media_type, media_id, target_language, source_language return 'No space left on device', 507 raise - _apply_subtitle_chmod(target_path) - _log_promoted_sync_history( - media_type=media_type, - media_id=media_id, - target_language=target_language, - source_language=source_language, - target_path=target_path, - metadata=metadata, - ) - _refresh_media_subtitles(media_type, media_id, metadata) + try: + _refresh_media_subtitles(media_type, media_id, metadata) + finally: + _log_promoted_sync_history( + media_type=media_type, + media_id=media_id, + target_language=target_language, + source_language=source_language, + target_path=target_path, + metadata=metadata, + ) return { 'sourceLanguage': source_language, @@ -1096,24 +1103,20 @@ def _create_subtitle(media_type, media_id, arr_instance_id=None): subtitle_path.startswith(target_folder_real + os.sep)): return 'Invalid subtitle path', 400 - # Check for existing file - if os.path.isfile(subtitle_path): - return 'Subtitle file already exists', 409 - # Encode content encoded = content.encode('utf-8') if len(encoded) > MAX_FILE_SIZE: return f'Content too large ({len(encoded)} bytes, max {MAX_FILE_SIZE})', 413 - # Write atomically + # Check and create under the same mutation lock as replacements. try: - fd, tmp_path = tempfile.mkstemp(dir=target_folder) - try: - os.write(fd, encoded) - finally: - os.close(fd) + with subtitle_write_locks(video_path, subtitle_path): + if os.path.isfile(subtitle_path): + return 'Subtitle file already exists', 409 + with subtitle_mutation(video_path, subtitle_path): + _write_bytes_atomically(subtitle_path, encoded) + _apply_subtitle_chmod(subtitle_path) - os.replace(tmp_path, subtitle_path) except FileNotFoundError: return 'Target directory not found', 404 except PermissionError: @@ -1123,14 +1126,6 @@ def _create_subtitle(media_type, media_id, arr_instance_id=None): return 'No space left on device', 507 raise - # Apply chmod if configured - if settings.general.chmod_enabled: - try: - chmod_value = int(settings.general.chmod, 8) - os.chmod(subtitle_path, chmod_value) - except Exception: - pass - # Force re-scan subtitles from disk using the media (video) path if media_type == 'episode': store_subtitles(row.path, video_path, use_cache=False, arr_instance_id=arr_instance_id) diff --git a/bazarr/api/translator/translator.py b/bazarr/api/translator/translator.py index 62f3f42fcc..acd81828cf 100644 --- a/bazarr/api/translator/translator.py +++ b/bazarr/api/translator/translator.py @@ -8,6 +8,7 @@ from app.config import settings from app.jobs_queue import jobs_queue from subtitles.tools.translate.services.auth import get_translator_auth_headers +from subtitles.tools.translate.services.openrouter_translator import build_provider_config from ..utils import authenticate api_ns_translator = Namespace('Translator', description='AI Subtitle Translator service operations') @@ -124,6 +125,7 @@ def post(self): "apiKey": api_key, "model": settings.translator.openrouter_model, "temperature": settings.translator.openrouter_temperature, + "provider": build_provider_config(), } } diff --git a/bazarr/app/config.py b/bazarr/app/config.py index ce3aa0a034..ef0669caf4 100644 --- a/bazarr/app/config.py +++ b/bazarr/app/config.py @@ -238,12 +238,17 @@ def check_parser_binary(value): Validator('translator.lingarr_url', must_exist=True, default='http://lingarr:9876', is_type_of=str), Validator('translator.openrouter_url', must_exist=True, default='http://subtitle-translator:8765', is_type_of=str), Validator('translator.openrouter_api_key', must_exist=True, default='', is_type_of=str, cast=str), - Validator('translator.openrouter_model', must_exist=True, default='google/gemini-2.5-flash-preview-05-20', is_type_of=str), + Validator('translator.openrouter_model', must_exist=True, default='google/gemini-2.5-flash-lite', is_type_of=str), Validator('translator.openrouter_temperature', must_exist=True, default=0.3, is_type_of=float), Validator('translator.openrouter_max_concurrent', must_exist=True, default=2, is_type_of=int, gte=1, lte=10), Validator('translator.openrouter_reasoning', must_exist=True, default='disabled', is_type_of=str, is_in=['disabled', 'low', 'medium', 'high']), Validator('translator.openrouter_parallel_batches', must_exist=True, default=4, is_type_of=int, gte=1, lte=8), + # Which OpenRouter provider serves the model: throughput (the sidecar's historical default), + # nitro/floor (OpenRouter's slug shortcuts, which also unlock the priority/flex tiers), + # price, latency, or OpenRouter's own load balancing. + Validator('translator.openrouter_provider_routing', must_exist=True, default='throughput', is_type_of=str, + is_in=['throughput', 'nitro', 'price', 'floor', 'latency', 'default']), Validator('translator.openrouter_encryption_key', must_exist=True, default='', is_type_of=str, cast=str), Validator('translator.lingarr_token', must_exist=True, default='', is_type_of=str, cast=str), @@ -727,6 +732,26 @@ def write_config(): _force_first_save_migration = False +# OpenRouter retired these ids, including Bazarr's default and documented recommendation. +# Users who never chose a model would otherwise get a model-not-found error. +RETIRED_OPENROUTER_MODELS = { + 'google/gemini-2.5-flash-preview-05-20': 'google/gemini-2.5-flash', + 'google/gemini-2.5-flash-lite-preview-06-17': 'google/gemini-2.5-flash-lite', + 'google/gemini-2.5-flash-lite-preview-09-2025': 'google/gemini-2.5-flash-lite', + 'google/gemini-3-pro-preview': 'google/gemini-3.1-pro-preview', +} + + +def migrate_retired_openrouter_model(settings) -> bool: + current_model = settings.translator.openrouter_model.strip() + replacement = RETIRED_OPENROUTER_MODELS.get(current_model) + if replacement is None: + return False + settings.translator.openrouter_model = replacement + logging.warning(f'Replaced retired OpenRouter model {current_model} with {replacement}') # noqa: G004 + return True + + base_url = settings.general.base_url.rstrip('/') array_keys = ['excluded_tags', @@ -790,6 +815,8 @@ def write_config(): settings.translator.gemini_keys = [legacy_key] del settings.translator.gemini_key +migrate_retired_openrouter_model(settings) + # save updated settings to file write_config() diff --git a/bazarr/app/requirements.py b/bazarr/app/requirements.py index 105b1bf49b..65773c7f4f 100644 --- a/bazarr/app/requirements.py +++ b/bazarr/app/requirements.py @@ -6,7 +6,6 @@ import logging import os from pathlib import Path -import re import subprocess import sys @@ -95,21 +94,21 @@ ) RUNTIME_REQUIREMENTS = { - "setuptools": ("setuptools", ">=82.0.1"), - "aiohttp": ("aiohttp", ">=3.13.5"), - "alembic": ("alembic", "==1.18.4"), + "setuptools": ("setuptools", ">=84.0.0"), + "aiohttp": ("aiohttp", ">=3.14.3"), + "alembic": ("alembic", "==1.19.1"), "apprise": ("apprise", "==1.13.0"), - "apscheduler": ("apscheduler", "==3.11.2"), + "apscheduler": ("apscheduler", "==3.11.3"), "babelfish": ("babelfish", "==0.6.1"), - "bs4": ("beautifulsoup4", "==4.14.3"), - "cachetools": ("cachetools", ">=7.1.1"), + "bs4": ("beautifulsoup4", "==4.15.0"), + "cachetools": ("cachetools", ">=7.1.4"), "certifi": ("certifi", "==2026.7.22"), - "chardet": ("chardet", "==7.4.3"), + "chardet": ("chardet", "==7.6.0"), "charset_normalizer": ("charset-normalizer", "==3.5.1"), - "click_option_group": ("click-option-group", ">=0.5.6"), - "cloudscraper": ("cloudscraper", "<=1.2.58"), - "cryptography": ("cryptography", ">=48.0.0"), - "dateutil": ("python-dateutil", "==2.9.0"), + "click_option_group": ("click-option-group", ">=0.5.9"), + "cloudscraper": ("cloudscraper", "<=1.2.71"), + "cryptography": ("cryptography", ">=50.0.1"), + "dateutil": ("python-dateutil", "==2.9.0.post0"), "deathbycaptcha": ("deathbycaptcha-official", "==4.7.1"), "deep_translator": ("deep-translator", "==1.11.4"), "dns": ("dnspython", "==2.8.0"), @@ -120,32 +119,32 @@ "fcache": ("fcache", "==0.6.0"), "fese": ("fese", "==0.3.0"), "ffmpeg": ("ffmpeg-python", "==0.2.0"), - "ffsubsync": ("ffsubsync", "==0.4.31"), + "ffsubsync": ("ffsubsync", "==0.5.0"), "autosubsync": ("autosubsync", "==1.0.1"), "filetype": ("filetype", "==1.2.0"), "flask": ("Flask", "==3.1.3"), "flask_compress": ("Flask-Compress", "==1.24"), - "flask_cors": ("flask-cors", "==6.0.2"), + "flask_cors": ("flask-cors", "==6.0.5"), "flask_migrate": ("Flask-Migrate", "==4.1.0"), "flask_restx": ("flask-restx", "==1.3.2"), "flask_socketio": ("Flask-SocketIO", "==5.6.1"), "flask_sqlalchemy": ("flask_sqlalchemy", "==3.1.1"), "ftfy": ("ftfy", "==6.3.1"), - "guess_language": ("guess_language-spirit", "==0.5.3"), - "guessit": ("guessit", "==3.8.0"), + "guess_language": ("guess_language-spirit", "==0.5.4"), + "guessit": ("guessit", "==4.4.0"), "itsdangerous": ("itsdangerous", "==2.2.0"), - "jwt": ("PyJWT", ">=2.12.1"), + "jwt": ("PyJWT", ">=2.13.0"), "json_tricks": ("json_tricks", "==3.17.3"), - "knowit": ("knowit", "==0.5.11"), - "lxml": ("lxml", ">=6.1.0"), + "knowit": ("knowit", "==0.6.1"), + "lxml": ("lxml", ">=6.1.1"), "msgpack": ("msgpack", "==1.2.1"), # signalrcore over-pins ==1.1.2; we install signalrcore --no-deps - "numpy": ("numpy", ">=2.0.0,<2.4.0"), - "PIL": ("Pillow", ">=12.2.0"), - "plexapi": ("plexapi", ">=4.16.1"), + "numpy": ("numpy", ">=2.5.2,<2.6.0"), + "PIL": ("Pillow", ">=12.3.0"), + "plexapi": ("plexapi", ">=4.18.2"), "py7zr": ("py7zr", "==1.1.3"), "pycountry": ("pycountry", "==26.2.16"), "pysrt": ("pysrt", "==1.1.2"), - "pysubs2": ("pysubs2", "==1.8.0"), + "pysubs2": ("pysubs2", "==1.8.1"), "python_anticaptcha": ("python-anticaptcha", "==2.0.0"), "rarfile": ("rarfile", "==4.5"), "requests": ("requests", "==2.34.2"), @@ -153,7 +152,7 @@ "semver": ("semver", "==3.0.4"), "signalrcore": ("signalrcore", "==1.0.2"), "six": ("six", "==1.17.0"), - "sqlalchemy": ("sqlalchemy", "==2.0.49"), + "sqlalchemy": ("sqlalchemy", "==2.0.52"), "srt": ("srt", "==3.5.3"), "subliminal": ("subliminal", "==2.6.0"), "textdistance": ("textdistance", "==4.6.3"), @@ -170,8 +169,8 @@ } WINDOWS_RUNTIME_REQUIREMENTS = { - "win32api": ("pywin32", ">=311"), - "win32con": ("pywin32", ">=311"), + "win32api": ("pywin32", ">=312"), + "win32con": ("pywin32", ">=312"), } REPO_ROOT = Path(__file__).resolve().parents[2] @@ -199,30 +198,32 @@ def is_virtualenv(): return base_prefix != real_prefix -def _version_tuple(version): - parts = re.findall(r"\d+", version.split("+", 1)[0].split("-", 1)[0]) - return tuple(int(part) for part in parts) - - def _satisfies_spec(installed_version, spec): - installed = _version_tuple(installed_version) - for item in spec.split(","): - item = item.strip() - if item.startswith("=="): - if installed != _version_tuple(item[2:]): - return False - elif item.startswith(">="): - if installed < _version_tuple(item[2:]): - return False - elif item.startswith("<="): - if installed > _version_tuple(item[2:]): - return False - elif item.startswith("<"): - if installed >= _version_tuple(item[1:]): - return False - else: - raise ValueError(f"Unsupported requirement specifier: {item}") - return True + # Keep startup and --no-update usable before dependencies are installed. + try: + from packaging.specifiers import SpecifierSet + from packaging.version import InvalidVersion, Version + except ModuleNotFoundError as error: + if error.name not in ("packaging", "packaging.specifiers", "packaging.version"): + raise + try: + from pip._vendor.packaging.specifiers import SpecifierSet + from pip._vendor.packaging.version import InvalidVersion, Version + except ModuleNotFoundError as error: + if error.name not in ("pip", "pip._vendor", "pip._vendor.packaging", + "pip._vendor.packaging.specifiers", "pip._vendor.packaging.version"): + raise + logging.warning("BAZARR cannot verify runtime versions because packaging and pip's parser are unavailable.") + return False + + requirement = SpecifierSet(spec) + try: + installed = Version(installed_version) + except InvalidVersion: + return False + # Already-installed prereleases remain eligible, but must satisfy the full + # version bounds. Do not rely on packaging's changing prerelease defaults. + return requirement.contains(installed, prereleases=True) def _module_origin(module): diff --git a/bazarr/provider_hub/protocol.py b/bazarr/provider_hub/protocol.py index 6640f386f4..dc5bce1ad9 100644 --- a/bazarr/provider_hub/protocol.py +++ b/bazarr/provider_hub/protocol.py @@ -5,12 +5,15 @@ import hashlib import logging import os +import re +import stat from typing import Any from subzero.language import Language from subliminal.video import Episode, Movie from subliminal_patch.core import SUBTITLE_EXTENSIONS +from subliminal_patch.exceptions import SubtitleCandidateRejected from subliminal_patch.subtitle import Subtitle logger = logging.getLogger(__name__) @@ -320,6 +323,8 @@ def _format_from_member(name: str | None, content: bytes | None = None) -> str | def worker_download_to_content( subtitle: HubWorkerSubtitle, payload: dict[str, Any], select_member_cb=None ) -> bool: + if not isinstance(payload, dict): + raise WorkerProtocolError("download must return an object") if payload.get("empty"): subtitle.content = b"" return True @@ -371,6 +376,11 @@ def _guard_archive_members(archive) -> None: raise WorkerProtocolError("download.archive_b64 contains too many members") total = 0 for info in infos: + is_link = getattr(info, "is_symlink", False) + if callable(is_link): + is_link = is_link() + if is_link or stat.S_ISLNK(getattr(info, "external_attr", 0) >> 16): + raise WorkerProtocolError("download.archive_b64 contains a symbolic link") # zip/rar expose file_size; py7zr exposes uncompressed. size = int(getattr(info, "file_size", 0) or getattr(info, "uncompressed", 0) or 0) if size > _MAX_MEMBER_BYTES: @@ -378,6 +388,40 @@ def _guard_archive_members(archive) -> None: total += size if total > _MAX_ARCHIVE_TOTAL_BYTES: raise WorkerProtocolError("download.archive_b64 decompresses past the size limit") + for name in archive.namelist(): + _validate_archive_member_name(name, disk_extracted=isinstance(archive, _SevenZipArchive)) + + +def _validate_archive_member_name(name, disk_extracted=False): + if not isinstance(name, str) or not name: + raise WorkerProtocolError("archive member must be a non-empty string") + normalized = name.replace("\\", "/") + if (normalized.startswith("/") or ".." in normalized.split("/") + or re.match(r"^[A-Za-z]:", normalized) or "\0" in normalized + or (disk_extracted and ":" in normalized)): + raise WorkerProtocolError("archive member has an unsafe path") + + +def _archive_rejection(subtitle, archive, reason): + """Describe ordinary mismatch without retaining paths, URLs or archive bytes.""" + def safe_label(value, limit): + return re.sub(r"[^a-zA-Z0-9_. ()-]", "_", str(value or ""))[:limit] + + members = [] + for name in archive.namelist()[:8]: + basename = name.replace("\\", "/").rsplit("/", 1)[-1] + basename = basename.split("?", 1)[0].split("#", 1)[0] + members.append(safe_label(basename, 80)) + identifier = str(getattr(subtitle, "worker_id", "")) + language = getattr(subtitle, "language", None) + return SubtitleCandidateRejected( + f"{reason}; provider={safe_label(getattr(subtitle, 'provider_name', ''), 48)} " + f"result_sha256={hashlib.sha256(identifier.encode()).hexdigest()[:16]} " + f"season={safe_label(getattr(subtitle, 'season', None), 12)} " + f"episode={safe_label(getattr(subtitle, 'episode', None), 12)} " + f"language={safe_label(getattr(language, 'basename', None), 24)} " + f"archive={type(archive).__name__} members={members!r}" + ) def _list_archive_members(archive): @@ -516,50 +560,81 @@ def _worker_archive_to_content( if archive is None: raise WorkerProtocolError("download.archive_b64 is not a zip, rar, or 7z archive") _guard_archive_members(archive) + for flag in ("select_member", "first_subtitle"): + if flag in payload and not isinstance(payload[flag], bool): + raise WorkerProtocolError(f"download.{flag} must be a boolean") + episode = payload.get("episode") + episodes = episode if isinstance(episode, (list, tuple)) else [episode] + if any(value is not None and (type(value) is not int or value < 0) for value in episodes): + raise WorkerProtocolError("download.episode must contain non-negative integers") + season = payload.get("season") + if season is not None and (type(season) is not int or season < 0): + raise WorkerProtocolError("download.season must be a non-negative integer") + member = payload.get("member") + if member is not None: + _validate_archive_member_name(member, disk_extracted=isinstance(archive, _SevenZipArchive)) def _episode_pick(): forced = bool(getattr(getattr(subtitle, "language", None), "forced", False)) - episode = payload.get("episode") - if isinstance(episode, (list, tuple)): - episode = episode[0] if episode else None + context = getattr(subtitle, "_requested_archive_context", None) + if context is None: + context_season = getattr(subtitle, "season", None) + if context_season is None: + context_season = season + episode = payload.get("episode") + if isinstance(episode, (list, tuple)): + episode = episode[0] if episode else None + else: + # An absent requested number is intentional, including movie searches. + context_season = context.get("season") + requested_episodes = (context.get("episode"), context.get("absolute_episode")) + episode = tuple(value for value in requested_episodes if type(value) is int and value >= 0) or None picked = get_subtitle_from_archive( archive, forced=forced, episode=episode, + season=context_season, episode_title=payload.get("episode_title"), get_first_subtitle=bool(payload.get("first_subtitle")), extensions=ARCHIVE_MEMBER_EXTENSIONS, + match_episode_context=True, + log_member_names=False, ) if picked is None: - raise WorkerProtocolError("download.archive_b64 has no usable subtitle member") + raise _archive_rejection(subtitle, archive, "No usable subtitle member") return picked - member = payload.get("member") - if member: + if member is not None: if member not in set(archive.namelist()): - raise WorkerProtocolError(f"download.member is not in the archive: {member}") + raise _archive_rejection(subtitle, archive, "Pinned subtitle member is absent") content = fix_line_ending(archive.read(member)) chosen = member - elif payload.get("select_member") and select_member_cb is not None: + elif payload.get("select_member"): # Host lists the members; the worker language-pins one (tri-state pin/defer/reject). # This is the only way to language-select rar/7z, which the stdlib-only worker - # cannot list. "defer" => safe episode pick (single-language); "reject" => fail loud. - result = select_member_cb(_list_archive_members(archive)) or {} + # cannot list. "defer" => episode pick; "reject" => skip this candidate. + if select_member_cb is None: + raise WorkerProtocolError("download.select_member requires a selector") + offered_members = tuple(_list_archive_members(archive)) + result = select_member_cb(list(offered_members)) + if not isinstance(result, dict): + raise WorkerProtocolError("select_archive_member must return an object") decision = result.get("decision") + if decision in ("defer", "reject") and result.get("member") is not None: + raise WorkerProtocolError("select_archive_member may only name a member when pinning") if decision == "pin": chosen = result.get("member") - if chosen not in set(archive.namelist()): - raise WorkerProtocolError( - f"select_archive_member returned a member not in the archive: {chosen!r}" - ) + _validate_archive_member_name(chosen, disk_extracted=isinstance(archive, _SevenZipArchive)) + if chosen not in offered_members: + raise WorkerProtocolError("select_archive_member pinned a member outside the offered subtitles") content = fix_line_ending(archive.read(chosen)) elif decision == "defer": content = _episode_pick() chosen = payload.get("filename") or "" + elif decision == "reject": + raise _archive_rejection(subtitle, archive, "No matching subtitle language") else: - raise WorkerProtocolError( - "select_archive_member rejected the archive (no language match)" - ) + raise WorkerProtocolError("select_archive_member returned an invalid decision") else: content = _episode_pick() chosen = payload.get("filename") or "" diff --git a/bazarr/provider_hub/registry.py b/bazarr/provider_hub/registry.py index a82e79d80e..f77fb4fd4c 100644 --- a/bazarr/provider_hub/registry.py +++ b/bazarr/provider_hub/registry.py @@ -139,10 +139,18 @@ def list_subtitles(self, video, languages): "languages": [language_to_payload(item) for item in languages], } result = self._worker().request("search", request, timeout=timeout) - return [ - candidate_from_worker(self.provider_name, item) - for item in result.payload.get("candidates", []) - ] + subtitles = [] + for item in result.payload.get("candidates", []): + subtitle = candidate_from_worker(self.provider_name, item) + # Request context belongs to this candidate, separate from provider + # display metadata and opaque payloads used for scoring and download. + subtitle._requested_archive_context = { + "season": request["video"]["season"], + "episode": request["video"]["episode"], + "absolute_episode": request["video"]["absolute_episode"], + } + subtitles.append(subtitle) + return subtitles def download_subtitle(self, subtitle): timeout = self._request_timeout() @@ -155,16 +163,22 @@ def download_subtitle(self, subtitle): result = self._worker().request("download", request, timeout=timeout) def _select_member_cb(members): + context = getattr(subtitle, "_requested_archive_context", {}) + selector_payload = { + "provider": self.provider_name, + "provider_payload": subtitle.provider_payload, + "language": language_to_payload(subtitle.language), + "members": members, + "season": context.get("season", getattr(subtitle, "season", None)), + "episode": context.get("episode", getattr(subtitle, "episode", None)), + "config": self.config, + } + if "absolute_episode" in context: + selector_payload["absolute_episode"] = context["absolute_episode"] + elif hasattr(subtitle, "absolute_episode"): + selector_payload["absolute_episode"] = subtitle.absolute_episode response = self._worker().select_archive_member( - { - "provider": self.provider_name, - "provider_payload": subtitle.provider_payload, - "language": language_to_payload(subtitle.language), - "members": members, - "season": getattr(subtitle, "season", None), - "episode": getattr(subtitle, "episode", None), - "config": self.config, - }, + selector_payload, timeout=timeout, ) return response.payload diff --git a/bazarr/provider_hub/worker.py b/bazarr/provider_hub/worker.py index 1b281536c6..30736615f1 100644 --- a/bazarr/provider_hub/worker.py +++ b/bazarr/provider_hub/worker.py @@ -53,6 +53,53 @@ def _json_default(obj): class WorkerError(RuntimeError): """Raised when a provider worker fails or returns an error.""" + def __init__(self, message, *, remote_class_name=None, code=None, retryable=False): + super().__init__(message) + self.remote_class_name = remote_class_name + self.code = code + self.retryable = retryable + + +def _raise_worker_error(payload): + if not isinstance(payload, dict): + raise WorkerError("worker request failed") + + remote_name = payload.get("class_name") + remote_name = remote_name if isinstance(remote_name, str) else None + code = payload.get("code") + code = code if isinstance(code, str) else None + retryable = payload.get("retryable", False) + retryable = retryable if isinstance(retryable, bool) else False + message = payload.get("message") + message = message if isinstance(message, str) and message else code or "worker request failed" + error = WorkerError(message, remote_class_name=remote_name, code=code, retryable=retryable) + if code != "provider": + raise error + + # Keep host exception imports out of worker startup. Only these fixed + # semantic names can cross the boundary as host provider exceptions. + from subliminal.exceptions import ( + AuthenticationError, + ConfigurationError, + DownloadLimitExceeded, + ServiceUnavailable, + ) + from subliminal_patch.exceptions import APIThrottled, TooManyRequests + + exception_type = { + "DownloadLimitExceeded": DownloadLimitExceeded, + "TooManyRequests": TooManyRequests, + "RateLimited": TooManyRequests, + "ServiceUnavailable": ServiceUnavailable, + "APIThrottled": APIThrottled, + "AuthenticationError": AuthenticationError, + "AuthenticationRequired": AuthenticationError, + "ConfigurationError": ConfigurationError, + }.get(remote_name) + if exception_type is None: + raise error + raise exception_type(message) from error + @dataclass class WorkerResult: @@ -460,15 +507,15 @@ def request(self, op: str, payload: dict[str, Any] | None = None, timeout: float except json.JSONDecodeError as error: raise WorkerError("worker returned malformed JSON") from error + if not isinstance(response, dict): + raise WorkerError("worker response must be an object") if response.get("abi") != WORKER_ABI_VERSION: raise WorkerError("worker returned unsupported ABI") if response.get("id") != request_id: raise WorkerError("worker returned mismatched request id") if not response.get("ok", False): - error = response.get("error") or {} - message = error.get("message") or error.get("code") or "worker request failed" - raise WorkerError(str(message)) + _raise_worker_error(response.get("error")) payload = response.get("payload") or {} events = response.get("events") or [] diff --git a/bazarr/provider_hub/worker_runner.py b/bazarr/provider_hub/worker_runner.py index 1bafe5c4b6..b78e953b93 100644 --- a/bazarr/provider_hub/worker_runner.py +++ b/bazarr/provider_hub/worker_runner.py @@ -78,25 +78,26 @@ def _handle(provider, op, payload): if op == "select_archive_member": selector = getattr(provider, "select_archive_member", None) if selector is None: - # Provider asked the host to list members (select_member) but exposes no - # selector: reject so the host fails loud rather than risk a wrong member. - return {"member": None, "decision": "reject"} + raise ValueError("select_archive_member is not implemented") provider_payload = dict(payload.get("provider_payload") or {}) - # The host forwards the requested season/episode at the top level of the op payload. + # The host forwards relative/absolute episode context at the top level of the op payload. # Surface them on provider_payload (host context is authoritative) so a selector can # disambiguate season-pack members even when the search payload didn't carry them. - for key in ("season", "episode"): - if payload.get(key) is not None: - provider_payload[key] = payload.get(key) + # An explicit null clears stale context; an absent key preserves legacy payloads. + for key in ("season", "episode", "absolute_episode"): + if key in payload: + provider_payload[key] = payload[key] result = selector( provider_payload=provider_payload, language=payload.get("language") or {}, members=payload.get("members") or [], config=payload.get("config") or {}, - ) or {} + ) + if not isinstance(result, dict): + raise ValueError("select_archive_member must return an object") decision = result.get("decision") if decision not in ("pin", "defer", "reject"): - decision = "reject" + raise ValueError("select_archive_member returned an invalid decision") return {"member": result.get("member"), "decision": decision} raise ValueError(f"unsupported worker op: {op}") diff --git a/bazarr/radarr/history.py b/bazarr/radarr/history.py index 17f67c0c70..a05fb6e38e 100644 --- a/bazarr/radarr/history.py +++ b/bazarr/radarr/history.py @@ -65,10 +65,11 @@ def history_log_movie(action, radarr_id, result, fake_provider=None, fake_score= upgradedFromId=upgraded_from_id, ) - # Stamp owner + local ref (movie_id -> the local PK) from the movie row so - # history is instance-owned (the INC4 pattern for history). Guarded; NULL - # for an unresolved row or pre-backfill install. No-op for the default - # instance beyond populating movie_id. + if arr_instance_id is not None: + values['arr_instance_id'] = arr_instance_id + + # Resolve the local ref when the movie is available. A missing movie must + # not discard an owner supplied by the caller. if radarr_id is not None: mv = _resolve_movie_owner_row(radarr_id, arr_instance_id) if mv is not None: @@ -78,5 +79,11 @@ def history_log_movie(action, radarr_id, result, fake_provider=None, fake_score= if mv.id is not None: values['movie_id'] = mv.id + if values.get('arr_instance_id') is None: + logging.warning( + 'BAZARR skipping movie history insert: unable to resolve owning instance ' + 'for radarrId %s', radarr_id) + return + database.execute(insert(TableHistoryMovie).values(**values)) event_stream(type='movie-history') diff --git a/bazarr/sonarr/history.py b/bazarr/sonarr/history.py index 49343f5965..439337f5c7 100644 --- a/bazarr/sonarr/history.py +++ b/bazarr/sonarr/history.py @@ -68,11 +68,11 @@ def history_log(action, sonarr_series_id, sonarr_episode_id, result, fake_provid upgradedFromId=upgraded_from_id, ) - # Stamp the owning instance + local refs (episode_id/series_id -> the local - # PKs) from the episode row so history is instance-owned, the same way INC4 - # stamps media. Guarded: a row that can't be resolved (or a pre-backfill - # install) leaves the columns NULL. No-op for the default single instance - # beyond populating the local refs. + if arr_instance_id is not None: + values['arr_instance_id'] = arr_instance_id + + # Resolve local refs when the episode is available. A missing episode must + # not discard an owner supplied by the caller. if sonarr_episode_id is not None: ep = _resolve_episode_owner_row(sonarr_episode_id, arr_instance_id) if ep is not None: @@ -84,5 +84,11 @@ def history_log(action, sonarr_series_id, sonarr_episode_id, result, fake_provid if ep.series_id is not None: values['series_id'] = ep.series_id + if values.get('arr_instance_id') is None: + logging.warning( + 'BAZARR skipping episode history insert: unable to resolve owning instance ' + 'for sonarrEpisodeId %s', sonarr_episode_id) + return + database.execute(insert(TableHistory).values(**values)) event_stream(type='episode-history') diff --git a/bazarr/subtitles/download.py b/bazarr/subtitles/download.py index 26b93110ff..8e3a3fd28b 100644 --- a/bazarr/subtitles/download.py +++ b/bazarr/subtitles/download.py @@ -4,11 +4,13 @@ import os import sys import logging +from functools import partial import subliminal import ast from subzero.language import Language from subliminal_patch.core import save_subtitles +from subtitles.tools.subsync_engines import subtitle_write_locks, subtitle_mutation, write_subtitle_file from subliminal_patch.core_persistent import download_best_subtitles from app.config import settings @@ -149,21 +151,29 @@ def _on_provider(provider_name): fld = get_target_folder(path) chmod = int(settings.general.chmod, 8) if not sys.platform.startswith( 'win') and settings.general.chmod_enabled else None - if is_upgrade and previous_subtitles_to_delete: - try: - # delete previously downloaded subtitles in case of an upgrade to prevent edge loop - # issue. - os.remove(previous_subtitles_to_delete) - except (OSError, FileNotFoundError): - pass - saved_subtitles = save_subtitles(video.original_path, subtitles, - single=settings.general.single_language, - tags=None, # fixme - directory=fld, - chmod=chmod, - formats=subtitle_formats, - path_decoder=force_unicode - ) + with subtitle_write_locks(path, os.path.join(fld or os.path.dirname(path), '.destination'), + previous_subtitles_to_delete or path): + written_paths = [] + saved_subtitles = save_subtitles(video.original_path, subtitles, + single=settings.general.single_language, + tags=None, # fixme + directory=fld, + chmod=chmod, + formats=subtitle_formats, + path_decoder=force_unicode, + write_subtitle=partial(write_subtitle_file, path, written_paths=written_paths) + ) + if is_upgrade and previous_subtitles_to_delete and written_paths and ( + os.path.normcase(os.path.realpath(previous_subtitles_to_delete)) not in + {os.path.normcase(os.path.realpath(written)) for written in written_paths}): + try: + with subtitle_mutation(path, previous_subtitles_to_delete): + os.remove(previous_subtitles_to_delete) + except OSError: + logging.exception('BAZARR unable to remove superseded subtitle: %s', + previous_subtitles_to_delete) + saved_subtitles = [saved for saved in saved_subtitles if saved.storage_path in written_paths] + except Exception as e: logging.exception( f'BAZARR Error saving Subtitles file to disk for this file {path}: {repr(e)}') # noqa: G004 diff --git a/bazarr/subtitles/indexer/movies.py b/bazarr/subtitles/indexer/movies.py index 3ae8820071..500604d843 100644 --- a/bazarr/subtitles/indexer/movies.py +++ b/bazarr/subtitles/indexer/movies.py @@ -13,7 +13,8 @@ get_audio_profile_languages, database, update, select from languages.get_languages import alpha2_from_alpha3, get_language_set from app.config import settings -from utilities.helper import get_subtitle_destination_folder +from utilities.helper import get_subtitle_destination_folder, get_target_folder +from subtitles.tools.subsync_engines import SyncOutputOwnerIndex, subtitle_write_locks from utilities.path_mappings import path_mappings from utilities.video_analyzer import embedded_subs_reader from app.event_handler import event_stream @@ -31,7 +32,7 @@ gc.enable() -def store_subtitles_movie(original_path, reversed_path, use_cache=True, arr_instance_id=None): +def store_subtitles_movie(original_path, reversed_path, use_cache=True, arr_instance_id=None, ownership_index=None): logging.debug(f'BAZARR started subtitles indexing for this file: {reversed_path}') # noqa: G004 actual_subtitles = [] # The owning instance decides everything below: which per-instance @@ -102,87 +103,91 @@ def _prr(p): f"BAZARR error when trying to analyze this {os.path.splitext(reversed_path)[1]} file: " # noqa: G004 f"{reversed_path}") - try: - dest_folder = get_subtitle_destination_folder() - core.CUSTOM_PATHS = [dest_folder] if dest_folder else [] - - # get previously indexed subtitles that haven't changed: - item = database.execute( - scoped(select(TableMovies.subtitles) - .where(TableMovies.path == original_path), - TableMovies.arr_instance_id, owner_instance_id))\ - .first() - if not item: - previously_indexed_subtitles_to_exclude = [] - else: - previously_indexed_subtitles = ast.literal_eval(item.subtitles) if item.subtitles else [] - previously_indexed_subtitles_to_exclude = [x for x in previously_indexed_subtitles - if len(x) == 3 and - x[1] and - os.path.isfile(_pr(x[1])) and - os.stat(_pr(x[1])).st_size == x[2]] - - subtitles = search_external_subtitles(reversed_path, languages=get_language_set(), - only_one=settings.general.single_language) - full_dest_folder_path = os.path.dirname(reversed_path) - if dest_folder: - if settings.general.subfolder == "absolute": - full_dest_folder_path = dest_folder - elif settings.general.subfolder == "relative": - full_dest_folder_path = os.path.join(os.path.dirname(reversed_path), dest_folder) - subtitles = add_sync_engine_outputs(full_dest_folder_path, subtitles) - subtitles = add_combined_outputs(full_dest_folder_path, subtitles, - video_filename=os.path.basename(reversed_path)) - subtitles = guess_external_subtitles(full_dest_folder_path, subtitles, "movie", - previously_indexed_subtitles_to_exclude) - except Exception as e: - logging.exception(f"BAZARR unable to index external subtitles for this file {reversed_path}: {repr(e)}") # noqa: G004 - else: - for subtitle, language in subtitles.items(): - valid_language = False - if language: - if hasattr(language, 'alpha3'): - valid_language = alpha2_from_alpha3(language.alpha3) + destination = os.path.join(get_target_folder(reversed_path, create=False) or os.path.dirname(reversed_path), + '.destination') + with subtitle_write_locks(reversed_path, destination): + try: + dest_folder = get_subtitle_destination_folder() + core.CUSTOM_PATHS = [dest_folder] if dest_folder else [] + + # get previously indexed subtitles that haven't changed: + item = database.execute( + scoped(select(TableMovies.subtitles) + .where(TableMovies.path == original_path), + TableMovies.arr_instance_id, owner_instance_id))\ + .first() + if not item: + previously_indexed_subtitles_to_exclude = [] else: - logging.debug(f"Skipping subtitles because we are unable to define language: {subtitle}") # noqa: G004 - continue + previously_indexed_subtitles = ast.literal_eval(item.subtitles) if item.subtitles else [] + previously_indexed_subtitles_to_exclude = [x for x in previously_indexed_subtitles + if len(x) == 3 and + x[1] and + os.path.isfile(_pr(x[1])) and + os.stat(_pr(x[1])).st_size == x[2]] + + subtitles = search_external_subtitles(reversed_path, languages=get_language_set(), + only_one=settings.general.single_language) + full_dest_folder_path = os.path.dirname(reversed_path) + if dest_folder: + if settings.general.subfolder == "absolute": + full_dest_folder_path = dest_folder + elif settings.general.subfolder == "relative": + full_dest_folder_path = os.path.join(os.path.dirname(reversed_path), dest_folder) + subtitles = add_sync_engine_outputs(full_dest_folder_path, subtitles, + video_path=reversed_path, ownership_index=ownership_index) + subtitles = add_combined_outputs(full_dest_folder_path, subtitles, + video_filename=os.path.basename(reversed_path)) + subtitles = guess_external_subtitles(full_dest_folder_path, subtitles, "movie", + previously_indexed_subtitles_to_exclude) + except Exception as e: + logging.exception(f"BAZARR unable to index external subtitles for this file {reversed_path}: {repr(e)}") # noqa: G004 + else: + for subtitle, language in subtitles.items(): + valid_language = False + if language: + if hasattr(language, 'alpha3'): + valid_language = alpha2_from_alpha3(language.alpha3) + else: + logging.debug(f"Skipping subtitles because we are unable to define language: {subtitle}") # noqa: G004 + continue - if not valid_language: - logging.debug(f'{language.alpha3} is an unsupported language code.') # noqa: G004 - continue + if not valid_language: + logging.debug(f'{language.alpha3} is an unsupported language code.') # noqa: G004 + continue - subtitle_path = get_external_subtitles_path(reversed_path, subtitle) + subtitle_path = get_external_subtitles_path(reversed_path, subtitle) - try: - subtitle_size = os.stat(subtitle_path).st_size - except FileNotFoundError: - logging.debug(f"BAZARR skipping missing subtitle file: {subtitle_path}") # noqa: G004 - continue - - custom = CustomLanguage.found_external(subtitle, subtitle_path) - - if custom is not None: - actual_subtitles.append([custom, _prr(subtitle_path), - subtitle_size]) - - elif str(language.basename) != 'und': - if language.forced: - language_str = f'{language}:forced' - elif language.hi: - language_str = f'{language}:hi' - else: - language_str = str(language) - language_str = subtitle_language_with_sync_modifier(language_str, subtitle) - language_str = subtitle_language_with_combined_modifier(language_str, subtitle) - logging.debug(f"BAZARR external subtitles detected: {language_str}") # noqa: G004 - actual_subtitles.append([language_str, _prr(subtitle_path), - subtitle_size]) + try: + subtitle_size = os.stat(subtitle_path).st_size + except FileNotFoundError: + logging.debug(f"BAZARR skipping missing subtitle file: {subtitle_path}") # noqa: G004 + continue - database.execute( - scoped(update(TableMovies) - .values(subtitles=str(actual_subtitles)) - .where(TableMovies.path == original_path), - TableMovies.arr_instance_id, owner_instance_id)) + custom = CustomLanguage.found_external(subtitle, subtitle_path) + + if custom is not None: + actual_subtitles.append([custom, _prr(subtitle_path), + subtitle_size]) + + elif str(language.basename) != 'und': + if language.forced: + language_str = f'{language}:forced' + elif language.hi: + language_str = f'{language}:hi' + else: + language_str = str(language) + language_str = subtitle_language_with_sync_modifier(language_str, subtitle) + language_str = subtitle_language_with_combined_modifier(language_str, subtitle) + logging.debug(f"BAZARR external subtitles detected: {language_str}") # noqa: G004 + actual_subtitles.append([language_str, _prr(subtitle_path), + subtitle_size]) + + database.execute( + scoped(update(TableMovies) + .values(subtitles=str(actual_subtitles)) + .where(TableMovies.path == original_path), + TableMovies.arr_instance_id, owner_instance_id)) matching_movies = database.execute( scoped(select(TableMovies.radarrId, TableMovies.arr_instance_id) .where(TableMovies.path == original_path), @@ -437,12 +442,13 @@ def movies_full_scan_subtitles(job_id=None, use_cache=None, wait_for_completion= .all() jobs_queue.update_job_progress(job_id=job_id, progress_max=len(movies), progress_message='Indexing') + ownership_index = SyncOutputOwnerIndex() for i, movie in enumerate(movies, start=1): jobs_queue.update_job_progress(job_id=job_id, progress_value=i, progress_message=movie.title) store_subtitles_movie(movie.path, path_mappings.path_replace_instance(movie.path, movie.arr_instance_id, 'movie'), - use_cache=use_cache, arr_instance_id=movie.arr_instance_id) + use_cache=use_cache, arr_instance_id=movie.arr_instance_id, ownership_index=ownership_index) logging.info('BAZARR All existing movie subtitles indexed from disk.') @@ -460,8 +466,9 @@ def movies_scan_subtitles(no, arr_instance_id=None): TableMovies.arr_instance_id, arr_instance_id)) \ .all() + ownership_index = SyncOutputOwnerIndex() for movie in movies: store_subtitles_movie(movie.path, path_mappings.path_replace_instance(movie.path, movie.arr_instance_id, 'movie'), - use_cache=False, arr_instance_id=movie.arr_instance_id) + use_cache=False, arr_instance_id=movie.arr_instance_id, ownership_index=ownership_index) diff --git a/bazarr/subtitles/indexer/series.py b/bazarr/subtitles/indexer/series.py index 6566213962..f98d2323d6 100644 --- a/bazarr/subtitles/indexer/series.py +++ b/bazarr/subtitles/indexer/series.py @@ -12,7 +12,8 @@ get_audio_profile_languages, database, update, select from languages.get_languages import alpha2_from_alpha3, get_language_set from app.config import settings -from utilities.helper import get_subtitle_destination_folder +from utilities.helper import get_subtitle_destination_folder, get_target_folder +from subtitles.tools.subsync_engines import SyncOutputOwnerIndex, subtitle_write_locks from utilities.path_mappings import path_mappings from utilities.video_analyzer import embedded_subs_reader from app.event_handler import event_stream @@ -30,7 +31,7 @@ gc.enable() -def store_subtitles(original_path, reversed_path, use_cache=True, arr_instance_id=None): +def store_subtitles(original_path, reversed_path, use_cache=True, arr_instance_id=None, ownership_index=None): logging.debug(f'BAZARR started subtitles indexing for this file: {reversed_path}') # noqa: G004 actual_subtitles = [] # The owning instance decides everything below: which per-instance @@ -102,86 +103,90 @@ def _prr(p): "BAZARR error when trying to analyze this %s file: %s" % (os.path.splitext(reversed_path)[1], # noqa: G002 reversed_path)) pass - try: - dest_folder = get_subtitle_destination_folder() - core.CUSTOM_PATHS = [dest_folder] if dest_folder else [] - - # get previously indexed subtitles that haven't changed: - item = database.execute( - scoped(select(TableEpisodes.subtitles) - .where(TableEpisodes.path == original_path), - TableEpisodes.arr_instance_id, owner_instance_id)) \ - .first() - if not item: - previously_indexed_subtitles_to_exclude = [] - else: - previously_indexed_subtitles = ast.literal_eval(item.subtitles) if item.subtitles else [] - previously_indexed_subtitles_to_exclude = [x for x in previously_indexed_subtitles - if len(x) == 3 and - x[1] and - os.path.isfile(_pr(x[1])) and - os.stat(_pr(x[1])).st_size == x[2]] - - subtitles = search_external_subtitles(reversed_path, languages=get_language_set(), - only_one=settings.general.single_language) - full_dest_folder_path = os.path.dirname(reversed_path) - if dest_folder: - if settings.general.subfolder == "absolute": - full_dest_folder_path = dest_folder - elif settings.general.subfolder == "relative": - full_dest_folder_path = os.path.join(os.path.dirname(reversed_path), dest_folder) - subtitles = add_sync_engine_outputs(full_dest_folder_path, subtitles) - subtitles = add_combined_outputs(full_dest_folder_path, subtitles, - video_filename=os.path.basename(reversed_path)) - subtitles = guess_external_subtitles(full_dest_folder_path, subtitles, "series", - previously_indexed_subtitles_to_exclude) - except Exception as e: - logging.exception(f"BAZARR unable to index external subtitles for this file {reversed_path}: {repr(e)}") # noqa: G004 - else: - for subtitle, language in subtitles.items(): - valid_language = False - if language: - if hasattr(language, 'alpha3'): - valid_language = alpha2_from_alpha3(language.alpha3) + destination = os.path.join(get_target_folder(reversed_path, create=False) or os.path.dirname(reversed_path), + '.destination') + with subtitle_write_locks(reversed_path, destination): + try: + dest_folder = get_subtitle_destination_folder() + core.CUSTOM_PATHS = [dest_folder] if dest_folder else [] + + # get previously indexed subtitles that haven't changed: + item = database.execute( + scoped(select(TableEpisodes.subtitles) + .where(TableEpisodes.path == original_path), + TableEpisodes.arr_instance_id, owner_instance_id)) \ + .first() + if not item: + previously_indexed_subtitles_to_exclude = [] else: - logging.debug(f"Skipping subtitles because we are unable to define language: {subtitle}") # noqa: G004 - continue + previously_indexed_subtitles = ast.literal_eval(item.subtitles) if item.subtitles else [] + previously_indexed_subtitles_to_exclude = [x for x in previously_indexed_subtitles + if len(x) == 3 and + x[1] and + os.path.isfile(_pr(x[1])) and + os.stat(_pr(x[1])).st_size == x[2]] + + subtitles = search_external_subtitles(reversed_path, languages=get_language_set(), + only_one=settings.general.single_language) + full_dest_folder_path = os.path.dirname(reversed_path) + if dest_folder: + if settings.general.subfolder == "absolute": + full_dest_folder_path = dest_folder + elif settings.general.subfolder == "relative": + full_dest_folder_path = os.path.join(os.path.dirname(reversed_path), dest_folder) + subtitles = add_sync_engine_outputs(full_dest_folder_path, subtitles, + video_path=reversed_path, ownership_index=ownership_index) + subtitles = add_combined_outputs(full_dest_folder_path, subtitles, + video_filename=os.path.basename(reversed_path)) + subtitles = guess_external_subtitles(full_dest_folder_path, subtitles, "series", + previously_indexed_subtitles_to_exclude) + except Exception as e: + logging.exception(f"BAZARR unable to index external subtitles for this file {reversed_path}: {repr(e)}") # noqa: G004 + else: + for subtitle, language in subtitles.items(): + valid_language = False + if language: + if hasattr(language, 'alpha3'): + valid_language = alpha2_from_alpha3(language.alpha3) + else: + logging.debug(f"Skipping subtitles because we are unable to define language: {subtitle}") # noqa: G004 + continue - if not valid_language: - logging.debug(f'{language.alpha3} is an unsupported language code.') # noqa: G004 - continue + if not valid_language: + logging.debug(f'{language.alpha3} is an unsupported language code.') # noqa: G004 + continue - subtitle_path = get_external_subtitles_path(reversed_path, subtitle) + subtitle_path = get_external_subtitles_path(reversed_path, subtitle) - try: - subtitle_size = os.stat(subtitle_path).st_size - except FileNotFoundError: - logging.debug(f"BAZARR skipping missing subtitle file: {subtitle_path}") # noqa: G004 - continue - - custom = CustomLanguage.found_external(subtitle, subtitle_path) - if custom is not None: - actual_subtitles.append([custom, _prr(subtitle_path), - subtitle_size]) - - elif str(language.basename) != 'und': - if language.forced: - language_str = f'{language}:forced' - elif language.hi: - language_str = f'{language}:hi' - else: - language_str = str(language) - language_str = subtitle_language_with_sync_modifier(language_str, subtitle) - language_str = subtitle_language_with_combined_modifier(language_str, subtitle) - logging.debug(f"BAZARR external subtitles detected: {language_str}") # noqa: G004 - actual_subtitles.append([language_str, _prr(subtitle_path), - subtitle_size]) + try: + subtitle_size = os.stat(subtitle_path).st_size + except FileNotFoundError: + logging.debug(f"BAZARR skipping missing subtitle file: {subtitle_path}") # noqa: G004 + continue - database.execute( - scoped(update(TableEpisodes) - .values(subtitles=str(actual_subtitles)) - .where(TableEpisodes.path == original_path), - TableEpisodes.arr_instance_id, owner_instance_id)) + custom = CustomLanguage.found_external(subtitle, subtitle_path) + if custom is not None: + actual_subtitles.append([custom, _prr(subtitle_path), + subtitle_size]) + + elif str(language.basename) != 'und': + if language.forced: + language_str = f'{language}:forced' + elif language.hi: + language_str = f'{language}:hi' + else: + language_str = str(language) + language_str = subtitle_language_with_sync_modifier(language_str, subtitle) + language_str = subtitle_language_with_combined_modifier(language_str, subtitle) + logging.debug(f"BAZARR external subtitles detected: {language_str}") # noqa: G004 + actual_subtitles.append([language_str, _prr(subtitle_path), + subtitle_size]) + + database.execute( + scoped(update(TableEpisodes) + .values(subtitles=str(actual_subtitles)) + .where(TableEpisodes.path == original_path), + TableEpisodes.arr_instance_id, owner_instance_id)) matching_episodes = database.execute( scoped(select(TableEpisodes.sonarrEpisodeId, TableEpisodes.sonarrSeriesId, TableEpisodes.arr_instance_id) @@ -458,6 +463,7 @@ def series_full_scan_subtitles(job_id=None, use_cache=None, wait_for_completion= ).all() jobs_queue.update_job_progress(job_id=job_id, progress_max=len(episodes), progress_message='Indexing') + ownership_index = SyncOutputOwnerIndex() for i, episode in enumerate(episodes, start=1): jobs_queue.update_job_progress( job_id=job_id, progress_value=i, @@ -465,7 +471,7 @@ def series_full_scan_subtitles(job_id=None, use_cache=None, wait_for_completion= store_subtitles(episode.path, path_mappings.path_replace_instance(episode.path, episode.arr_instance_id, 'series'), - use_cache=use_cache, arr_instance_id=episode.arr_instance_id) + use_cache=use_cache, arr_instance_id=episode.arr_instance_id, ownership_index=ownership_index) logging.info('BAZARR All existing episode subtitles indexed from disk.') @@ -483,8 +489,9 @@ def series_scan_subtitles(no, arr_instance_id=None): TableEpisodes.arr_instance_id, arr_instance_id))\ .all() + ownership_index = SyncOutputOwnerIndex() for episode in episodes: store_subtitles(episode.path, path_mappings.path_replace_instance(episode.path, episode.arr_instance_id, 'series'), - use_cache=False, arr_instance_id=episode.arr_instance_id) + use_cache=False, arr_instance_id=episode.arr_instance_id, ownership_index=ownership_index) diff --git a/bazarr/subtitles/indexer/utils.py b/bazarr/subtitles/indexer/utils.py index 120ca33c22..07c256e1e8 100644 --- a/bazarr/subtitles/indexer/utils.py +++ b/bazarr/subtitles/indexer/utils.py @@ -2,6 +2,7 @@ import os import logging +import unicodedata from guess_language import guess_language from subliminal_patch import core @@ -12,7 +13,7 @@ from app.config import settings from utilities.path_mappings import path_mappings from languages.custom_lang import CustomLanguage -from subtitles.tools.subsync_engines import SYNC_ENGINES, sync_engine_from_output_path +from subtitles.tools.subsync_engines import SYNC_ENGINES, sync_engine_from_output_path, sync_output_owner_is_unique import re as _re_combine @@ -46,6 +47,23 @@ def get_external_subtitles_path(file, subtitle): return path +def get_subtitle_destination_path(file, subtitle): + """Where a new external subtitle for ``file`` gets written. + + get_external_subtitles_path() answers only for files that already exist, which is + right for indexing and wrong for a first-time write: in the absolute and relative + subfolder modes it returned None and a translation crashed on save. An existing file + in either location is still preferred so a re-run overwrites what the user already + sees; otherwise the configured folder is used and created, the way downloads do it. + """ + existing = get_external_subtitles_path(file, subtitle) + if existing: + return existing + from utilities.helper import get_target_folder + folder = get_target_folder(file) or os.path.dirname(file) + return os.path.join(folder, subtitle) + + def normalize_subtitle_language_variant(language, forced=False, hi=False): language_text = str(language) parts = language_text.split(':') @@ -75,14 +93,18 @@ def sync_engine_from_subtitle_name(subtitle): return sync_engine_from_output_path(subtitle) -def _language_code_from_sync_engine_output(subtitle): - filename = os.path.basename(subtitle).lower() +def _normalized_video_stem(stem): + return unicodedata.normalize('NFC', stem.lower()) + + +def _language_code_from_sync_engine_output(subtitle, video_filename=None): + filename = os.path.basename(subtitle) stem, extension = os.path.splitext(filename) - if extension not in core.SUBTITLE_EXTENSIONS: + if extension.lower() not in core.SUBTITLE_EXTENSIONS: return None parts = stem.split('.') - if len(parts) < 3 or parts[-1] not in SYNC_ENGINES: + if len(parts) < 3 or parts[-1].lower() not in SYNC_ENGINES: return None parts = parts[:-1] @@ -93,40 +115,86 @@ def _language_code_from_sync_engine_output(subtitle): # combined output is a known limitation: it is left on disk but not indexed # as a tracked variant. Overwrite-mode sync of a combined file works in # place (it keeps the Movie.en.combined-hu.srt name) and is unaffected. - if parts and _COMBINED_MODIFIER_PATTERN.match(parts[-1]): + if parts and _COMBINED_MODIFIER_PATTERN.match(parts[-1].lower()): return None variants = [] - if parts and parts[-1] in ['hi', 'sdh', 'cc']: + if parts and parts[-1].lower() in ['hi', 'sdh', 'cc']: variants.append('hi') parts = parts[:-1] - if parts and parts[-1] == 'forced': + if parts and parts[-1].lower() == 'forced': variants.append('forced') parts = parts[:-1] if not parts: return None - language = parts[-1].replace('_', '-') + if video_filename is not None: + video_stem = os.path.splitext(video_filename)[0] + if _normalized_video_stem('.'.join(parts[:-1])) != _normalized_video_stem(video_stem): + return None + + language = parts[-1].lower().replace('_', '-') if not language: return None return ':'.join([language] + variants) -def add_sync_engine_outputs(dest_folder, subtitles): +def add_sync_engine_outputs(dest_folder, subtitles, video_filename=None, video_path=None, ownership_index=None): + """Add generated outputs belonging to the video, regardless of save preferences.""" if not os.path.isdir(dest_folder): return subtitles + video_stem = None + media_stems = set() + if video_path is not None: + video_filename = os.path.basename(video_path) + if video_filename is not None: + video_stem = _normalized_video_stem(os.path.splitext(video_filename)[0]) + media_folder = (os.path.dirname(video_path) or '.') if video_path is not None else dest_folder + with os.scandir(media_folder) as media_files: + media_stems = {_normalized_video_stem(os.path.splitext(entry.name)[0]) + for entry in media_files + if entry.is_file() and entry.name.lower().endswith(core.VIDEO_EXTENSIONS)} + + ownership = {} for subtitle in os.listdir(dest_folder): - if subtitle in subtitles or not sync_engine_from_subtitle_name(subtitle): + if not sync_engine_from_subtitle_name(subtitle): continue subtitle_path = os.path.join(dest_folder, subtitle) if not os.path.isfile(subtitle_path): continue - language_code = _language_code_from_sync_engine_output(subtitle) + if video_path is not None: + stem, extension = os.path.splitext(subtitle) + source_path = os.path.join(dest_folder, stem.rsplit('.', 1)[0] + extension) + if source_path not in ownership: + ownership[source_path] = sync_output_owner_is_unique(video_path, source_path, + ownership_index=ownership_index) + if not ownership[source_path]: + subtitles.pop(subtitle, None) + continue + if subtitle in subtitles: + continue + + if video_stem is not None: + stem, extension = os.path.splitext(subtitle) + if extension.lower() not in core.SUBTITLE_EXTENSIONS: + continue + owner_stem = _normalized_video_stem(stem.rsplit('.', 1)[0]) + if owner_stem == video_stem: + # The complete basename owns an untagged output. Detect its + # language normally, even if the save preference has changed. + subtitles[subtitle] = None + continue + if owner_stem in media_stems: + # Movie.en.mkv owns Movie.en.ffsubsync.srt when it exists. + # Without that sibling, it can be a tagged output of Movie.mkv. + continue + + language_code = _language_code_from_sync_engine_output(subtitle, video_filename=video_filename) if not language_code: - logging.debug("BAZARR skipping generated sync subtitle with unknown language: %s", subtitle_path) + logging.debug("BAZARR skipping unrelated or unrecognized generated sync subtitle: %s", subtitle_path) continue try: diff --git a/bazarr/subtitles/manual.py b/bazarr/subtitles/manual.py index 781c10125d..177df32209 100644 --- a/bazarr/subtitles/manual.py +++ b/bazarr/subtitles/manual.py @@ -4,10 +4,12 @@ import os import sys import logging +from functools import partial import subliminal from subzero.language import Language from subliminal_patch.core import save_subtitles +from subtitles.tools.subsync_engines import subtitle_write_locks, write_subtitle_file from subliminal_patch.core_persistent import list_all_subtitles, download_subtitles from subliminal_patch.score import compute_score, DEFAULT_SCORES @@ -196,13 +198,18 @@ def manual_download_subtitle(path, audio_language, hi, forced, subtitle, provide try: chmod = int(settings.general.chmod, 8) if not sys.platform.startswith( 'win') and settings.general.chmod_enabled else None - saved_subtitles = save_subtitles(video.original_path, [subtitle], - single=settings.general.single_language, - tags=None, # fixme - directory=get_target_folder(path), - chmod=chmod, - formats=(subtitle.format,), - path_decoder=force_unicode) + with subtitle_write_locks(path, os.path.join(get_target_folder(path) or os.path.dirname(path), '.destination')): + written_paths = [] + saved_subtitles = save_subtitles(video.original_path, [subtitle], + single=settings.general.single_language, + tags=None, # fixme + directory=get_target_folder(path), + chmod=chmod, + formats=(subtitle.format,), + path_decoder=force_unicode, + write_subtitle=partial(write_subtitle_file, path, written_paths=written_paths)) + saved_subtitles = [saved for saved in saved_subtitles if saved.storage_path in written_paths] + except Exception as e: logging.exception(f'BAZARR Error saving Subtitles file to disk for this file {path}: {repr(e)}') # noqa: G004 return 'Error saving Subtitles file to disk' diff --git a/bazarr/subtitles/post_processing.py b/bazarr/subtitles/post_processing.py index 2391340bcb..4054d737b8 100644 --- a/bazarr/subtitles/post_processing.py +++ b/bazarr/subtitles/post_processing.py @@ -6,9 +6,23 @@ import subprocess from locale import getpreferredencoding +from utilities.helper import get_target_folder +from subtitles.tools.subsync_engines import subtitle_write_locks, subtitle_mutation -def postprocessing(command, path): +def postprocessing(command, path, subtitle_path=None): + # Configured commands can mutate subtitles in place. This is the one boundary + # that must hold this media's mutation locks while the external command runs. + destination = os.path.join(get_target_folder(path, create=False) or os.path.dirname(path), '.destination') + with subtitle_write_locks(path, path, destination, subtitle_path or path) as states: + watched_paths = {watched for state in states.values() for watched in state.revisions} + if subtitle_path: + watched_paths.add(subtitle_path) + with subtitle_mutation(path, *watched_paths): + return _postprocessing_locked(command, path) + + +def _postprocessing_locked(command, path): try: encoding = getpreferredencoding() if os.name == 'nt': diff --git a/bazarr/subtitles/processing.py b/bazarr/subtitles/processing.py index a4384fa8ab..71845cba6f 100644 --- a/bazarr/subtitles/processing.py +++ b/bazarr/subtitles/processing.py @@ -71,15 +71,6 @@ def _trigger_auto_translation(downloaded_lang, subtitle_path, video_path, media_ if forced: return - min_score = settings.translator.min_source_score - if source_score_percent is not None and source_score_percent < min_score: - logging.info( - 'BAZARR auto-translate skipped: source score %.1f%% ' - 'below threshold %.1f%% for %s', - source_score_percent, min_score, video_path, - ) - return - # Fetch the episode/movie row that postprocess_subtitles dereferences # for plex/jellyfin refresh (sonarrSeriesId, imdbId, season, episode, # tvdbId for episodes; imdbId, tmdbId for movies). Passing None here @@ -140,6 +131,23 @@ def _trigger_auto_translation(downloaded_lang, subtitle_path, video_path, media_ if not profile: return + translation_targets = [ + item for item in profile.get('items', []) + if item.get('translate_from') == downloaded_lang + and item.get('language') != downloaded_lang + ] + if not translation_targets: + return + + min_score = settings.translator.min_source_score + if source_score_percent is not None and source_score_percent < min_score: + logging.info( + 'BAZARR auto-translate skipped: source score %.1f%% ' + 'below threshold %.1f%% for %s', + source_score_percent, min_score, video_path, + ) + return + # Hoisted out of the loop: check_missing_languages is independent of the # profile item being considered, so calling it once per profile item # (potentially N database/indexer queries) is wasteful. Compute the @@ -160,14 +168,8 @@ def _trigger_auto_translation(downloaded_lang, subtitle_path, video_path, media_ else: missing_codes.add(code2) - for item in profile.get('items', []): + for item in translation_targets: target_lang = item.get('language') - translate_from = item.get('translate_from') - - if not translate_from or translate_from != downloaded_lang: - continue - if target_lang == downloaded_lang: - continue target_code = profile_item_language_code(item) if target_code not in missing_codes: @@ -331,7 +333,7 @@ def process_subtitle(subtitle, media_type, audio_language, path, max_score, is_u if not use_pp_threshold or (use_pp_threshold and percent_score < pp_threshold): logging.debug(f"BAZARR Using post-processing command: {command}") # noqa: G004 - postprocessing(command, path) + postprocessing(command, path, subtitle_path=downloaded_path) set_chmod(subtitles_path=downloaded_path) else: logging.debug(f"BAZARR post-processing skipped because subtitles score isn't below this " # noqa: G004 diff --git a/bazarr/subtitles/sync.py b/bazarr/subtitles/sync.py index 26c923be64..f84cedf045 100644 --- a/bazarr/subtitles/sync.py +++ b/bazarr/subtitles/sync.py @@ -5,7 +5,7 @@ import gc from app.config import settings -from app.jobs_queue import jobs_queue +from app.jobs_queue import jobs_queue, JobCancelled from subtitles.tools.subsyncer import SubSyncer from subtitles.tools.subsync_engines import ( DEFAULT_ENABLED_ENGINES, @@ -18,8 +18,13 @@ REASON_MISSING_ENGINE, REASON_OUTPUT_EXISTS, REASON_RESULT_REJECTED, + REASON_SOURCE_CHANGED, + REASON_DESTINATION_CHANGED, + REASON_DESTINATION_AMBIGUOUS, is_sync_engine_output, normalize_enabled_engines, + release_subtitle_publication, + release_unqueued_subtitle_publication, ) @@ -38,6 +43,9 @@ "when it cannot confidently align a file. {message}"), REASON_RESULT_REJECTED: "{label} result rejected: {message}", REASON_ENGINE_FAILED: "{label} failed: {message}", + REASON_SOURCE_CHANGED: "The uploaded subtitle was replaced or deleted. Sync was skipped.", + REASON_DESTINATION_CHANGED: "The synchronized subtitle destination was changed or deleted. Sync was skipped.", + REASON_DESTINATION_AMBIGUOUS: "The synchronized subtitle destination has no unique media owner. Sync was skipped.", } _ENGINE_MESSAGE_LIMIT = 160 @@ -251,103 +259,135 @@ def sync_subtitles(video_path, callback=None, track_job_progress=True, arr_instance_id=None, - owns_job_progress=True): - # The audio-sync settings resolve against the owning instance (#227); a None - # owner / unset override yields the global value, so legacy paths are - # unchanged. The use_subsync gate is evaluated inline via the module-level - # helper so NO non-signature local is created before add_job_from_function - # below, which re-passes the frame's locals as kwargs on re-invocation. - if (not _resolve_subsync_overrides( - arr_instance_id, bool(sonarr_episode_id), enabled_engines, max_offset_seconds)[0] - and not force_sync): - logging.debug('BAZARR automatic syncing is disabled in settings. Skipping sync routine.') - return False + owns_job_progress=True, + source_version=None, + on_success=None): + try: + # The audio-sync settings resolve against the owning instance (#227); a None + # owner / unset override yields the global value, so legacy paths are + # unchanged. The use_subsync gate is evaluated inline via the module-level + # helper so NO non-signature local is created before add_job_from_function + # below, which re-passes the frame's locals as kwargs on re-invocation. + if (not _resolve_subsync_overrides( + arr_instance_id, bool(sonarr_episode_id), enabled_engines, max_offset_seconds)[0] + and not force_sync): + logging.debug('BAZARR automatic syncing is disabled in settings. Skipping sync routine.') + release_subtitle_publication(source_version) + return False - if is_sync_engine_output(srt_path): - logging.debug('BAZARR generated sync output cannot be synchronized again. Skipping: %s', srt_path) - _report_progress(job_id, track_job_progress, owns_job_progress, 'Sync skipped', - value='max', name=f"Skipped sync for {srt_path}") - return False + if is_sync_engine_output(srt_path): + logging.debug('BAZARR generated sync output cannot be synchronized again. Skipping: %s', srt_path) + _report_progress(job_id, track_job_progress, owns_job_progress, 'Sync skipped', + value='max', name=f"Skipped sync for {srt_path}") + release_subtitle_publication(source_version) + return False - if not job_id and track_job_progress: - jobs_queue.add_job_from_function( - f"Syncing {srt_path}", - is_progress=True, - progress_max=_sync_progress_total(enabled_engines), - ) - return False + if not job_id and track_job_progress: + if not jobs_queue.add_job_from_function( + f"Syncing {srt_path}", + is_progress=True, + progress_max=_sync_progress_total(enabled_engines), + ): + release_unqueued_subtitle_publication(source_version, jobs_queue) + return False - # Past the enqueue point it is safe to bind locals. Resolve the per-instance - # overrides for the real run (the use_subsync gate already passed above). - (_, use_subsync_threshold, subsync_threshold, - enabled_engines, max_offset_seconds) = _resolve_subsync_overrides( - arr_instance_id, bool(sonarr_episode_id), enabled_engines, max_offset_seconds) - progress_total = _sync_progress_total(enabled_engines) + except BaseException: + if job_id: + release_subtitle_publication(source_version) + else: + release_unqueued_subtitle_publication(source_version, jobs_queue) + raise - def report(message, value=None, total=None, name=None): - _report_progress(job_id, track_job_progress, owns_job_progress, message, value, total, name) + try: + # Past the enqueue point it is safe to bind locals. Resolve the per-instance + # overrides for the real run (the use_subsync gate already passed above). + (_, use_subsync_threshold, subsync_threshold, + enabled_engines, max_offset_seconds) = _resolve_subsync_overrides( + arr_instance_id, bool(sonarr_episode_id), enabled_engines, max_offset_seconds) + progress_total = _sync_progress_total(enabled_engines) - report('Preparing synchronization', value=0, total=progress_total, name=f"Syncing {srt_path}") + def report(message, value=None, total=None, name=None): + _report_progress(job_id, track_job_progress, owns_job_progress, message, value, total, name) - def update_progress(message, value, total): - report(message, value=value, total=total) + report('Preparing synchronization', value=0, total=progress_total, name=f"Syncing {srt_path}") - if forced: - logging.debug('BAZARR cannot sync forced subtitles. Skipping sync routine.') - report('Sync skipped', value='max', name=f"Skipped sync for {srt_path}") - return False + def update_progress(message, value, total): + report(message, value=value, total=total) - logging.debug(f'BAZARR automatic syncing is enabled in settings. We\'ll try to sync this ' # noqa: G004 - f'subtitles: {srt_path}.') - if not use_subsync_threshold or (use_subsync_threshold and percent_score <= float(subsync_threshold)): - subsync = SubSyncer() - sync_kwargs = { - 'video_path': video_path, - 'srt_path': srt_path, - 'srt_lang': srt_lang, - 'forced': forced, - 'hi': hi, - 'max_offset_seconds': max_offset_seconds, - 'no_fix_framerate': no_fix_framerate, - 'gss': gss, - 'reference': reference, - 'sonarr_series_id': sonarr_series_id, - 'sonarr_episode_id': sonarr_episode_id, - 'radarr_id': radarr_id, - 'job_id': job_id, - 'force_sync': force_sync, - 'output_mode': output_mode, - 'enabled_engines': enabled_engines, - 'progress_callback': update_progress if track_job_progress else None, - 'arr_instance_id': arr_instance_id, - } - sync_result = None - try: - sync_result = subsync.sync(**sync_kwargs) - if sync_result and sync_result.success: - if callback: - callback() - elif getattr(sync_result, 'output_mode', None) == OUTPUT_MODE_KEEP_ALL: - _index_keep_all_outputs( - video_path, - sonarr_series_id=sonarr_series_id, - sonarr_episode_id=sonarr_episode_id, - radarr_id=radarr_id, - arr_instance_id=arr_instance_id, - ) - except Exception: - logging.exception(f'BAZARR an unhandled exception occurs during the synchronization process for this ' # noqa: G004 - f'subtitle file: {srt_path}') + if forced: + logging.debug('BAZARR cannot sync forced subtitles. Skipping sync routine.') + report('Sync skipped', value='max', name=f"Skipped sync for {srt_path}") + release_subtitle_publication(source_version) return False - else: - return bool(sync_result and sync_result.success) - finally: - report(_sync_outcome_message(sync_result), value='max', - name=_sync_complete_job_name(srt_path, sync_result)) - del subsync - gc.collect() - - logging.debug(f"BAZARR subsync skipped because subtitles score isn't below this " # noqa: G004 - f"threshold value: {subsync_threshold}%") - report('Sync skipped', value='max', name=f"Skipped sync for {srt_path}") - return False + + logging.debug(f'BAZARR automatic syncing is enabled in settings. We\'ll try to sync this ' # noqa: G004 + f'subtitles: {srt_path}.') + if not use_subsync_threshold or (use_subsync_threshold and percent_score <= float(subsync_threshold)): + subsync = SubSyncer() + sync_kwargs = { + 'video_path': video_path, + 'srt_path': srt_path, + 'srt_lang': srt_lang, + 'forced': forced, + 'hi': hi, + 'max_offset_seconds': max_offset_seconds, + 'no_fix_framerate': no_fix_framerate, + 'gss': gss, + 'reference': reference, + 'sonarr_series_id': sonarr_series_id, + 'sonarr_episode_id': sonarr_episode_id, + 'radarr_id': radarr_id, + 'job_id': job_id, + 'force_sync': force_sync, + 'output_mode': output_mode, + 'enabled_engines': enabled_engines, + 'progress_callback': update_progress if track_job_progress else None, + 'arr_instance_id': arr_instance_id, + } + sync_result = None + if source_version is not None: + sync_kwargs['source_version'] = source_version + try: + sync_result = subsync.sync(**sync_kwargs) + if sync_result and sync_result.success: + if callback and source_version is None: + callback() + elif not callback and getattr(sync_result, 'output_mode', None) == OUTPUT_MODE_KEEP_ALL: + _index_keep_all_outputs( + video_path, + sonarr_series_id=sonarr_series_id, + sonarr_episode_id=sonarr_episode_id, + radarr_id=radarr_id, + arr_instance_id=arr_instance_id, + ) + except JobCancelled: + raise + except Exception: + logging.exception(f'BAZARR an unhandled exception occurs during the synchronization process for this ' # noqa: G004 + f'subtitle file: {srt_path}') + return False + else: + return bool(sync_result and sync_result.success) + finally: + try: + try: + if callback and source_version is not None: + callback() + finally: + if on_success and (sync_result or getattr(subsync, 'sync_result', None)) and ( + sync_result or subsync.sync_result).success: + on_success() + report(_sync_outcome_message(sync_result), value='max', + name=_sync_complete_job_name(srt_path, sync_result)) + finally: + release_subtitle_publication(source_version) + del subsync + gc.collect() + + logging.debug(f"BAZARR subsync skipped because subtitles score isn't below this " # noqa: G004 + f"threshold value: {subsync_threshold}%") + report('Sync skipped', value='max', name=f"Skipped sync for {srt_path}") + release_subtitle_publication(source_version) + return False + finally: + release_subtitle_publication(source_version) diff --git a/bazarr/subtitles/tools/combine/main.py b/bazarr/subtitles/tools/combine/main.py index 7c6a7cfb55..1a7b4163ad 100644 --- a/bazarr/subtitles/tools/combine/main.py +++ b/bazarr/subtitles/tools/combine/main.py @@ -3,7 +3,9 @@ import logging import os import re +import stat from dataclasses import dataclass +from subtitles.tools.subsync_engines import staged_subtitle_write, subtitle_mutation, sync_output_owner_is_unique from .composer import compose from .naming import compose_combined_filename, external_subtitles_dir @@ -84,6 +86,8 @@ def try_combine_for_video(video_path, media_type, sonarr_series_id=None, # inside this video's external-subtitles directory. Re-assert it here so a # crafted code can never steer makedirs/open/remove outside that folder. safe_root = os.path.realpath(external_subtitles_dir(video_path)) + if os.path.islink(out_path) or not sync_output_owner_is_unique(video_path, out_path): + return CombineResult(status="skipped", reason="combined output has no unique regular-file owner") out_path = os.path.realpath(out_path) if os.path.commonpath([safe_root, out_path]) != safe_root: return CombineResult( @@ -92,36 +96,20 @@ def try_combine_for_video(video_path, media_type, sonarr_series_id=None, ) try: - content = compose( - primary_path=sources.primary, - secondary_paths=sources.secondaries, - format=rule["format"], - ) - except Exception as e: - logging.exception("BAZARR combine compose failed for %s", video_path) - return CombineResult(status="failed", error=str(e)) - - try: - # Create the destination directory (mirrors how Bazarr's - # get_target_folder makedirs the configured subtitle subfolder) so a - # first-time combine into a not-yet-created absolute/relative folder - # succeeds instead of failing on open(). out_dir = os.path.dirname(out_path) if out_dir: os.makedirs(out_dir, exist_ok=True) - with open(out_path, "wb") as fh: - fh.write(content) - except OSError as e: - logging.exception("BAZARR combine write failed for %s", out_path) + with staged_subtitle_write( + video_path, out_path, source_paths=(sources.primary, *sources.secondaries), + after_publish=lambda: _remove_stale_combined_siblings(out_path, video_path)) as temporary: + content = compose(primary_path=sources.primary, secondary_paths=sources.secondaries, + format=rule["format"]) + with open(temporary, "wb") as fh: + fh.write(content) + except Exception as e: + logging.exception("BAZARR combine could not publish %s", out_path) return CombineResult(status="failed", error=str(e)) - # A combined output is one logical subtitle. Drop any sibling combined - # file in a different subtitle format (left over when the profile format - # changes or a rebuild switches format) before re-indexing, so the same - # combined language is not indexed twice and the editor does not load a - # stale positioned ASS as overlapping cues. - _remove_stale_combined_siblings(out_path, video_path) - _post_write(out_path, video_path, media_type, sonarr_episode_id, radarr_id) @@ -173,19 +161,22 @@ def _remove_stale_combined_siblings(out_path, video_path): crafted output path can never steer os.remove outside that folder.""" safe_dir = os.path.realpath(external_subtitles_dir(video_path)) out_real = os.path.realpath(out_path) - if os.path.dirname(out_real) != safe_dir: + if os.path.dirname(out_real) != safe_dir or not sync_output_owner_is_unique(video_path, out_path): return root, _ext = os.path.splitext(out_real) for ext in _COMBINED_OUTPUT_EXTS: - sibling = os.path.realpath(root + ext) + sibling = root + ext if sibling == out_real: continue try: if os.path.commonpath([safe_dir, sibling]) != safe_dir: continue - if os.path.isfile(sibling): - os.remove(sibling) + if stat.S_ISREG(os.lstat(sibling).st_mode): + with subtitle_mutation(video_path, sibling): + os.remove(sibling) logging.info("BAZARR combine removed stale sibling %s", sibling) + except FileNotFoundError: + continue except OSError: logging.exception( "BAZARR combine could not remove stale sibling %s", sibling) diff --git a/bazarr/subtitles/tools/delete.py b/bazarr/subtitles/tools/delete.py index b4fb77099e..b572960fdf 100644 --- a/bazarr/subtitles/tools/delete.py +++ b/bazarr/subtitles/tools/delete.py @@ -15,6 +15,8 @@ from subtitles.indexer.series import store_subtitles from subtitles.indexer.movies import store_subtitles_movie from subtitles.processing import ProcessSubtitlesResult +from subtitles.tools.subsync_engines import (subtitle_write_lock, subtitle_write_locks, + quarantine_sync_outputs_after_mutation) from sonarr.history import history_log from radarr.history import history_log_movie from sonarr.notify import notify_sonarr @@ -23,6 +25,21 @@ from jellyfin.operations import jellyfin_refresh_item +def _delete_subtitle_file(media_path, subtitle_path): + with subtitle_write_locks(media_path, subtitle_path): + state = subtitle_write_lock(media_path, os.path.dirname(subtitle_path)) + try: + os.remove(subtitle_path) + except OSError as exc: + if isinstance(exc, FileNotFoundError): + state.changed(subtitle_path) + logging.exception('BAZARR cannot delete subtitles file: %s', subtitle_path) + return False + state.changed(subtitle_path) + quarantine_sync_outputs_after_mutation(media_path, subtitle_path) + return True + + def delete_subtitles(media_type, language, forced, hi, media_path, subtitles_path, sonarr_series_id=None, sonarr_episode_id=None, radarr_id=None, arr_instance_id=None): if not subtitles_path: @@ -81,64 +98,56 @@ def prr(p): hearing_impaired=None) if media_type == 'series': - try: - os.remove(pr(subtitles_path)) - except OSError: - logging.exception(f'BAZARR cannot delete subtitles file: {subtitles_path}') # noqa: G004 - store_subtitles(prr(media_path), media_path, arr_instance_id=arr_instance_id) + removed = _delete_subtitle_file(media_path, pr(subtitles_path)) + store_subtitles(prr(media_path), media_path, arr_instance_id=arr_instance_id) + if not removed: return False - else: - history_log(0, sonarr_series_id, sonarr_episode_id, result, arr_instance_id=arr_instance_id) - store_subtitles(prr(media_path), media_path, arr_instance_id=arr_instance_id) - # Route the rescan at the OWNING instance's Sonarr (#156); None - # owner = default server (legacy single-instance), unchanged. - notify_sonarr(sonarr_series_id, - arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)) - event_stream(type='series', action='update', payload=sonarr_series_id) - event_stream(type='episode-wanted', action='update', payload=sonarr_episode_id) - - if settings.general.use_plex and settings.plex.update_series_library: - plex_refresh_item(metadata.imdbId, is_movie=False, season=metadata.season, - episode=metadata.episode) - if settings.general.use_jellyfin and settings.jellyfin.update_series_library: - jellyfin_refresh_item(metadata.imdbId, is_movie=False, season=metadata.season, - episode=metadata.episode, tvdb_id=metadata.tvdbId) - - # Call external webhook after all processing is complete - call_external_webhook( - subtitle_path=subtitles_path, - media_path=media_path, - language=language_log, - media_type=media_type - ) - - return True + history_log(0, sonarr_series_id, sonarr_episode_id, result, arr_instance_id=arr_instance_id) + # Route the rescan at the OWNING instance's Sonarr (#156); None + # owner = default server (legacy single-instance), unchanged. + notify_sonarr(sonarr_series_id, + arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)) + event_stream(type='series', action='update', payload=sonarr_series_id) + event_stream(type='episode-wanted', action='update', payload=sonarr_episode_id) + + if settings.general.use_plex and settings.plex.update_series_library: + plex_refresh_item(metadata.imdbId, is_movie=False, season=metadata.season, + episode=metadata.episode) + if settings.general.use_jellyfin and settings.jellyfin.update_series_library: + jellyfin_refresh_item(metadata.imdbId, is_movie=False, season=metadata.season, + episode=metadata.episode, tvdb_id=metadata.tvdbId) + + # Call external webhook after all processing is complete + call_external_webhook( + subtitle_path=subtitles_path, + media_path=media_path, + language=language_log, + media_type=media_type + ) + + return True else: - try: - os.remove(pr(subtitles_path)) - except OSError: - logging.exception(f'BAZARR cannot delete subtitles file: {subtitles_path}') # noqa: G004 - store_subtitles_movie(prr(media_path), media_path, arr_instance_id=arr_instance_id) + removed = _delete_subtitle_file(media_path, pr(subtitles_path)) + store_subtitles_movie(prr(media_path), media_path, arr_instance_id=arr_instance_id) + if not removed: return False - else: - history_log_movie(0, radarr_id, result, arr_instance_id=arr_instance_id) - store_subtitles_movie(prr(media_path), media_path, arr_instance_id=arr_instance_id) - notify_radarr(radarr_id, - arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)) - event_stream(type='movie-wanted', action='update', payload=radarr_id) - - if settings.general.use_plex and settings.plex.update_movie_library: - plex_refresh_item(metadata.imdbId, is_movie=True) - if settings.general.use_jellyfin and settings.jellyfin.update_movie_library: - jellyfin_refresh_item(metadata.imdbId, is_movie=True, - tmdb_id=metadata.tmdbId) - - # Call external webhook after all processing is complete - call_external_webhook( - subtitle_path=subtitles_path, - media_path=media_path, - language=language_log, - media_type=media_type - ) - - return True + history_log_movie(0, radarr_id, result, arr_instance_id=arr_instance_id) + notify_radarr(radarr_id, + arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)) + event_stream(type='movie-wanted', action='update', payload=radarr_id) + + if settings.general.use_plex and settings.plex.update_movie_library: + plex_refresh_item(metadata.imdbId, is_movie=True) + if settings.general.use_jellyfin and settings.jellyfin.update_movie_library: + jellyfin_refresh_item(metadata.imdbId, is_movie=True, + tmdb_id=metadata.tmdbId) + + # Call external webhook after all processing is complete + call_external_webhook( + subtitle_path=subtitles_path, + media_path=media_path, + language=language_log, + media_type=media_type + ) + + return True diff --git a/bazarr/subtitles/tools/mods.py b/bazarr/subtitles/tools/mods.py index 1f50053e16..ab505ff940 100644 --- a/bazarr/subtitles/tools/mods.py +++ b/bazarr/subtitles/tools/mods.py @@ -12,7 +12,9 @@ from app.jobs_queue import jobs_queue from languages.custom_lang import CustomLanguage from languages.get_languages import alpha3_from_alpha2 -from subtitles.indexer.utils import get_external_subtitles_path +from subtitles.indexer.utils import get_subtitle_destination_path +from utilities.helper import get_target_folder +from subtitles.tools.subsync_engines import subtitle_write_locks, subtitle_mutation def has_remove_hi(mods): @@ -145,6 +147,12 @@ def apply_subtitle_mods(language, subtitle_path, mods, video_path, def subtitles_apply_mods(language, subtitle_path, mods, video_path, arr_instance_id=None): + destination = os.path.join(get_target_folder(video_path, create=False) or os.path.dirname(video_path), '.destination') + with subtitle_write_locks(video_path, subtitle_path, destination): + return _apply_mods_locked(language, subtitle_path, mods, video_path, arr_instance_id) + + +def _apply_mods_locked(language, subtitle_path, mods, video_path, arr_instance_id): # The mod list is user-chosen here, so only the keep-lyrics preference is # instance-relevant: resolve it against the media's owning instance (#227). # A None owner keeps the legacy global-only behaviour (single-instance). @@ -180,18 +188,19 @@ def subtitles_apply_mods(language, subtitle_path, mods, video_path, arr_instance # get the real modded subtitles path taking into account if the user set up Bazarr to store external # subtitles in a custom folder or relative folder - modded_subtitles_path = get_external_subtitles_path( + modded_subtitles_path = get_subtitle_destination_path( file=video_path, subtitle=os.path.basename(modded_subtitles_path_if_alongside_video) ) else: modded_subtitles_path = subtitle_path - if os.path.exists(subtitle_path): - os.remove(subtitle_path) + with subtitle_mutation(video_path, subtitle_path, modded_subtitles_path): + if os.path.exists(subtitle_path): + os.remove(subtitle_path) - if os.path.exists(modded_subtitles_path): - os.remove(modded_subtitles_path) + if os.path.exists(modded_subtitles_path): + os.remove(modded_subtitles_path) - with open(modded_subtitles_path, 'wb') as f: - f.write(content) + with open(modded_subtitles_path, 'wb') as f: + f.write(content) diff --git a/bazarr/subtitles/tools/subsync_engines.py b/bazarr/subtitles/tools/subsync_engines.py index 3f69eb01f1..3f05fb6c12 100644 --- a/bazarr/subtitles/tools/subsync_engines.py +++ b/bazarr/subtitles/tools/subsync_engines.py @@ -2,9 +2,337 @@ import logging import os +import stat +import shutil import tempfile +import unicodedata +import uuid from dataclasses import dataclass, field +from contextlib import contextmanager, ExitStack, nullcontext from pathlib import Path +from threading import Lock, RLock +from weakref import WeakValueDictionary + + +_subtitle_write_locks = WeakValueDictionary() +_subtitle_write_locks_guard = Lock() + + +class _SubtitleWriteState: + def __init__(self): + self.lock = RLock() + self.revisions = {} + + def __enter__(self): + self.lock.acquire() + return self + + def __exit__(self, *exc): + self.lock.release() + + def changed(self, path): + key = os.path.normcase(os.path.realpath(path)) + self.revisions[key] = self.revisions.get(key, 0) + 1 + + def revision(self, path): + return self.revisions.setdefault(os.path.normcase(os.path.realpath(path)), 0) + + +def subtitle_write_lock(video_path, subtitle_directory): + """Coordinate one media destination, including its subtitle language variants. + + The saver can change HI tags and format extensions. The target directory and + video stem identify their shared destination without predicting those names. + Weak references release idle locks once all participating operations finish. + """ + key = (os.path.normcase(os.path.realpath(subtitle_directory)), + os.path.normcase(os.path.splitext(os.path.basename(video_path))[0])) + with _subtitle_write_locks_guard: + lock = _subtitle_write_locks.get(key) + if lock is None: + lock = _SubtitleWriteState() + _subtitle_write_locks[key] = lock + return lock + + +def subtitle_source_version(path): + """Identify the saved file a queued upload sync is allowed to replace.""" + try: + stat = os.stat(path) + except OSError: + return None + return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns) + + +class SubtitlePublication: + """Hold a media coordinator while a queued sync watches exact destinations.""" + + def __init__(self, video_path, source_path, source_version): + self.state = subtitle_write_lock(video_path, os.path.dirname(source_path)) + self.source_path = source_path + self.source_version = source_version + with self.state: + self.source_revision = self.state.revision(source_path) + self.owns_destinations = sync_output_owner_is_unique(video_path, source_path) + self.destinations = { + str(path): (self.state.revision(path), subtitle_source_version(path)) + for path in (engine_output_path(source_path, engine) for engine in SYNC_ENGINES) + } + + def source_unchanged(self): + return (self.state is not None and self.source_version is not None + and self.state.revision(self.source_path) == self.source_revision + and subtitle_source_version(self.source_path) == self.source_version) + + def destination_owned(self, path): + return self.owns_destinations and not os.path.islink(path) + + def destination_unchanged(self, path): + expected = self.destinations.get(str(path)) + return (self.state is not None and self.destination_owned(path) and expected is not None + and expected == (self.state.revision(path), subtitle_source_version(path))) + + def release(self): + self.state = None + + +def source_is_unchanged(path, version): + if isinstance(version, SubtitlePublication): + return version.source_unchanged() + return subtitle_source_version(path) == version + + +def release_subtitle_publication(version): + if isinstance(version, SubtitlePublication): + version.release() + + +def release_unqueued_subtitle_publication(version, queue): + if not isinstance(version, SubtitlePublication): + return + # Duplicate submission can pass the very same snapshot as the queued job. + # Its coordinator belongs to that job until the running call finishes. + with queue._queue_lock: + if any(job.kwargs.get('source_version') is version + for job in (*queue.jobs_pending_queue, *queue.jobs_running_queue)): + return + release_subtitle_publication(version) + + +@contextmanager +def subtitle_write_locks(video_path, *paths): + """Acquire participating subtitle directories in stable order.""" + directories = sorted({os.path.normcase(os.path.realpath(os.path.dirname(path))) for path in (video_path, *paths)}) + with ExitStack() as stack: + states = {directory: stack.enter_context(subtitle_write_lock(video_path, directory)) + for directory in directories} + yield states + + +@contextmanager +def subtitle_mutation(video_path, *paths, invalidate_outputs=True): + """Coordinate exact local writes and invalidate pending publications. + + Network and engine work stays outside. Indexers share this coordinator for + their external-file scan and database update. + """ + with subtitle_write_locks(video_path, *paths) as states: + versions = {path: subtitle_source_version(path) for path in paths} + try: + yield + finally: + for path, version in versions.items(): + if subtitle_source_version(path) != version: + directory = os.path.normcase(os.path.realpath(os.path.dirname(path))) + states[directory].changed(path) + if invalidate_outputs: + quarantine_sync_outputs_after_mutation(video_path, path) + + +@contextmanager +def staged_subtitle_write(video_path, destination, before_publish=None, allow_empty=False, + source_paths=(), after_publish=None): + """Compute privately, then publish only while source and destination are current.""" + with subtitle_write_locks(video_path, destination, *source_paths) as states: + def version(path): + directory = os.path.normcase(os.path.realpath(os.path.dirname(path))) + return (states[directory].revision(path), subtitle_source_version(path)) + + destination_version = version(destination) + source_versions = {path: version(path) for path in source_paths} + temporary = os.path.join(os.path.dirname(destination), f'.bazarr-write-{uuid.uuid4().hex}{Path(destination).suffix}') + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) + os.close(fd) + try: + yield temporary + if allow_empty and os.path.isfile(temporary) and os.path.getsize(temporary) == 0: + return + if not os.path.isfile(temporary) or os.path.getsize(temporary) == 0: + raise OSError('Subtitle writer did not produce a nonempty file') + with subtitle_write_locks(video_path, destination, *source_paths): + with subtitle_mutation(video_path, destination): + if before_publish: + before_publish() + if destination_version != version(destination): + raise SubtitleDestinationChanged('Subtitle changed during processing') + if any(expected[1] is None or expected != version(path) for path, expected in source_versions.items()): + raise SubtitleSourceChanged('Source subtitle changed during processing') + if os.path.isfile(destination): + shutil.copymode(destination, temporary) + os.replace(temporary, destination) + if after_publish: + after_publish() + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def write_subtitle_file(video_path, destination, content, written_paths=None): + """Write one saver output atomically and record that exact successful path.""" + with staged_subtitle_write(video_path, destination, + after_publish=(lambda: written_paths.append(destination)) + if written_paths is not None else None) as temporary: + with open(temporary, 'wb') as handle: + handle.write(content) + + +def _normalized_media_stem(path): + return unicodedata.normalize('NFC', os.path.splitext(os.path.basename(path))[0].lower()) + + +class SyncOutputOwnerIndex: + """Lazily map library owners once for one sequential subtitle scan. + + Only indexers share this snapshot. File mutations and queued publications + use fresh ownership checks so a completed scan cannot authorize a later write. + """ + + def __init__(self): + self._loaded = False + self._owners = {} + + @staticmethod + def _load_owners(): + from app.database import database, select, TableEpisodes, TableMovies + from app.config import settings + from utilities.path_mappings import path_mappings + + subfolder = settings.general.subfolder + custom_folder = settings.general.subfolder_custom + absolute_folder = (os.path.normcase(os.path.realpath(custom_folder)) + if subfolder == 'absolute' else None) + owners = {} + for table, media_type in ((TableEpisodes, 'episode'), (TableMovies, 'movie')): + for row in database.execute(select(table.path, table.arr_instance_id)).all(): + if not row.path: + continue + mapped = os.path.normcase(os.path.realpath(path_mappings.path_replace_instance( + row.path, row.arr_instance_id, media_type))) + folders = {os.path.dirname(mapped)} + if subfolder == 'absolute': + folders.add(absolute_folder) + elif subfolder == 'relative': + folders.add(os.path.normcase(os.path.realpath(os.path.join( + os.path.dirname(mapped), custom_folder)))) + stem = _normalized_media_stem(mapped) + for folder in folders: + owners.setdefault((folder, stem), set()).add(mapped) + return owners + + def is_unique(self, video, directory, conflicting_stems): + if not self._loaded: + # A failed build leaves an empty index, never a partial ownership + # claim or repeated full-library retries during the same scan. + self._loaded = True + self._owners = self._load_owners() + owned = False + expected_owner = {video} + for stem in conflicting_stems: + owners = self._owners.get((directory, stem)) + if owners: + if owners != expected_owner: + return False + owned = True + return owned + + +def sync_output_owner_is_unique(video_path, source_path, ownership_index=None): + """Prove ownership before moving or replacing an exact generated variant.""" + from subliminal_patch.core import VIDEO_EXTENSIONS + + video = os.path.normcase(os.path.realpath(video_path)) + directory = os.path.normcase(os.path.realpath(os.path.dirname(source_path))) + media_directory = os.path.dirname(video) + + stem = _normalized_media_stem + if stem(source_path) != stem(video) and not stem(source_path).startswith(stem(video) + '.'): + return False + conflicting_stems = {stem(video), stem(source_path)} + try: + with os.scandir(media_directory) as entries: + local_videos = {os.path.normcase(os.path.realpath(entry.path)) for entry in entries + if entry.is_file() and entry.name.lower().endswith(VIDEO_EXTENSIONS)} + if any(path != video and stem(path) in conflicting_stems for path in local_videos): + return False + if directory == media_directory: + return video in local_videos + + # Separate media directories can share a custom subtitle folder. Keep + # all mapped owners, amortizing that lookup only within an explicit scan. + if ownership_index is None: + ownership_index = SyncOutputOwnerIndex() + return ownership_index.is_unique(video, directory, conflicting_stems) + except (OSError, ValueError): + return False + except Exception: + logging.exception('BAZARR unable to verify generated subtitle ownership') + return False + + +def quarantine_sync_outputs(video_path, source_path): + """Preserve proven generated siblings under names no subtitle scan accepts.""" + if is_sync_engine_output(source_path) or not sync_output_owner_is_unique(video_path, source_path): + return + from subtitles.indexer.utils import add_sync_engine_outputs + + directory = os.path.dirname(source_path) + with subtitle_write_lock(video_path, directory) as state: + owned = add_sync_engine_outputs(directory, {}, video_path=video_path) + for engine in SYNC_ENGINES: + output = engine_output_path(source_path, engine) + if output.name not in owned: + continue + try: + if not stat.S_ISREG(output.lstat().st_mode): + continue + except FileNotFoundError: + continue + fd, quarantine = tempfile.mkstemp(prefix=f'.bazarr-sync-obsolete-{output.name[:40]}-', suffix='.bak', dir=directory) + os.close(fd) + try: + os.replace(output, quarantine) + except BaseException: + os.unlink(quarantine) + raise + state.changed(output) + + +def quarantine_sync_outputs_after_mutation(video_path, source_path): + """Report cleanup failure separately from an already completed file change.""" + try: + quarantine_sync_outputs(video_path, source_path) + except OSError as exc: + logging.error( + 'BAZARR subtitle change completed for %s, but obsolete generated subtitle cleanup failed (%s). ' + 'Previous generated files may remain available.', source_path, type(exc).__name__) + + +class SubtitleSourceChanged(Exception): + """The upload was replaced or deleted while its sync was queued or running.""" + + +class SubtitleDestinationChanged(Exception): + """A user changed or deleted this engine's destination after enqueueing.""" SYNC_ENGINES = ('ffsubsync', 'autosubsync', 'alass') @@ -46,6 +374,9 @@ REASON_ENGINE_DECLINED = 'engine_declined' REASON_RESULT_REJECTED = 'result_rejected' REASON_ENGINE_FAILED = 'engine_failed' +REASON_SOURCE_CHANGED = 'source_changed' +REASON_DESTINATION_CHANGED = 'destination_changed' +REASON_DESTINATION_AMBIGUOUS = 'destination_ambiguous' ENGINE_LABELS = { 'ffsubsync': 'FFsubsync', @@ -409,9 +740,10 @@ def _existing_keep_all_output_is_current(self, srt_path, output_path, engine): return output_stat.st_size > 0 and output_stat.st_mtime_ns >= source_stat.st_mtime_ns - def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync=False): + def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync=False, + source_version=None, before_publish=None, publication_lock=None, after_publish=None): output_mode = normalize_output_mode(output_mode) - result = SyncRunResult(source_path=srt_path, output_mode=output_mode) + result = self.result = SyncRunResult(source_path=srt_path, output_mode=output_mode) if is_sync_engine_output(srt_path): result.results.append(SyncEngineResult( @@ -423,11 +755,24 @@ def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync return result for engine in normalize_enabled_engines(enabled_engines): + if source_version is not None and not source_is_unchanged(srt_path, source_version): + result.results.append(SyncEngineResult( + engine=engine, status=RESULT_SKIPPED, reason=REASON_SOURCE_CHANGED, + message='The uploaded subtitle was replaced or deleted.', + )) + break final_engine_output_path = engine_output_path(srt_path, engine) - output_path = ( - final_engine_output_path if output_mode == OUTPUT_MODE_KEEP_ALL - else temporary_engine_output_path(srt_path, engine) - ) + output_path = temporary_engine_output_path(srt_path, engine) + + if (output_mode == OUTPUT_MODE_KEEP_ALL and isinstance(source_version, SubtitlePublication) + and not source_version.destination_unchanged(final_engine_output_path)): + result.results.append(SyncEngineResult( + engine=engine, status=RESULT_SKIPPED, + reason=(REASON_DESTINATION_CHANGED if source_version.destination_owned(final_engine_output_path) + else REASON_DESTINATION_AMBIGUOUS), + message='The synchronized subtitle destination changed or its owner is ambiguous.', + )) + continue if self.failure_store.should_skip(srt_path, engine) and not force_sync: result.results.append(SyncEngineResult( @@ -462,10 +807,26 @@ def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync generated_path = str(output_path) final_output_path = output_path - if output_mode == OUTPUT_MODE_OVERWRITE: - os.replace(str(output_path), srt_path) - final_output_path = Path(srt_path) - generated_path = None + with publication_lock or nullcontext() as state: + if before_publish: + before_publish() + if source_version is not None and not source_is_unchanged(srt_path, source_version): + raise SubtitleSourceChanged() + if (output_mode == OUTPUT_MODE_KEEP_ALL and isinstance(source_version, SubtitlePublication) + and not source_version.destination_unchanged(final_engine_output_path)): + raise SubtitleDestinationChanged() + if output_mode == OUTPUT_MODE_OVERWRITE: + os.replace(str(output_path), srt_path) + final_output_path = Path(srt_path) + generated_path = None + else: + os.replace(str(output_path), str(final_engine_output_path)) + final_output_path = final_engine_output_path + generated_path = str(final_engine_output_path) + if hasattr(state, 'changed'): + state.changed(final_output_path) + if after_publish: + after_publish() self.failure_store.record_success(srt_path, engine) result.results.append(SyncEngineResult( @@ -479,6 +840,24 @@ def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync if output_mode == OUTPUT_MODE_OVERWRITE: break + except SubtitleSourceChanged: + if output_path.is_file(): + output_path.unlink() + result.results.append(SyncEngineResult( + engine=engine, status=RESULT_SKIPPED, reason=REASON_SOURCE_CHANGED, + message='The uploaded subtitle was replaced or deleted.', + )) + break + except SubtitleDestinationChanged: + if output_path.is_file(): + output_path.unlink() + result.results.append(SyncEngineResult( + engine=engine, status=RESULT_SKIPPED, + reason=(REASON_DESTINATION_CHANGED if source_version.destination_owned(final_engine_output_path) + else REASON_DESTINATION_AMBIGUOUS), + message='The synchronized subtitle destination changed or its owner is ambiguous.', + )) + continue except MissingSyncEngineError as exc: logging.warning('BAZARR %s sync engine skipped: %s', engine, exc) result.results.append(SyncEngineResult( @@ -519,6 +898,11 @@ def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync )) except Exception as exc: + if before_publish: + from app.jobs_queue import JobCancelled + if isinstance(exc, JobCancelled): + self._discard_engine_output(srt_path, engine, output_path, exc, record=False) + raise logging.exception('BAZARR %s sync engine failed for %s', engine, srt_path) self._discard_engine_output(srt_path, engine, output_path, exc) result.results.append(SyncEngineResult( diff --git a/bazarr/subtitles/tools/subsyncer.py b/bazarr/subtitles/tools/subsyncer.py index 69029f056a..8ada6648d1 100644 --- a/bazarr/subtitles/tools/subsyncer.py +++ b/bazarr/subtitles/tools/subsyncer.py @@ -22,6 +22,10 @@ normalize_enabled_engines, normalize_output_mode, validate_engine_result, + subtitle_write_lock, + SubtitlePublication, + subtitle_source_version, + quarantine_sync_outputs_after_mutation, ) from languages.get_languages import audio_language_from_name, language_from_alpha2 from utilities.path_mappings import path_mappings @@ -523,7 +527,7 @@ def _log_sync_history(self, success_result, output_mode, srt_lang, hi, forced, def sync(self, video_path, srt_path, srt_lang, hi, forced, max_offset_seconds, no_fix_framerate, gss, reference=None, sonarr_series_id=None, sonarr_episode_id=None, radarr_id=None, progress_callback=None, job_id=None, force_sync=False, output_mode=None, - enabled_engines=None, write_history=True, arr_instance_id=None): + enabled_engines=None, write_history=True, arr_instance_id=None, source_version=None): self.reference = video_path self.srtin = srt_path self.progress_callback = progress_callback @@ -586,13 +590,33 @@ def execute_engine(engine, output_path): return raw_result runner = SubsyncEngineRunner() - self.sync_result = runner.run( - srt_path=self.srtin, - output_mode=output_mode, - enabled_engines=enabled_engines, - execute_engine=execute_engine, - force_sync=force_sync, - ) + publication_lock = subtitle_write_lock(video_path, os.path.dirname(srt_path)) + publication = source_version + if publication is None: + with publication_lock: + publication = SubtitlePublication(video_path, srt_path, subtitle_source_version(srt_path)) + + def publish(): + self._report_progress('Saving synchronized subtitle', None, None) + + try: + self.sync_result = runner.run( + srt_path=self.srtin, + output_mode=output_mode, + enabled_engines=enabled_engines, + execute_engine=execute_engine, + force_sync=force_sync, + source_version=publication, + before_publish=publish, + publication_lock=publication_lock, + after_publish=(lambda: quarantine_sync_outputs_after_mutation(video_path, srt_path)) + if output_mode == OUTPUT_MODE_OVERWRITE and source_version is None else None, + ) + finally: + # Preserve successful destinations even if a later engine is cancelled. + self.sync_result = getattr(runner, 'result', self.sync_result) + if source_version is None: + publication.release() if settings.subsync.debug: return self.sync_result diff --git a/bazarr/subtitles/tools/translate/core/translator_utils.py b/bazarr/subtitles/tools/translate/core/translator_utils.py index 86d5341df6..0732294326 100644 --- a/bazarr/subtitles/tools/translate/core/translator_utils.py +++ b/bazarr/subtitles/tools/translate/core/translator_utils.py @@ -111,12 +111,38 @@ def add_translator_info(dest_srt_file, info): f.write(srt.compose(subtitles)) -def get_description(media_type, radarr_id, sonarr_series_id): +# The callers do not agree on how to spell this: the manual translate endpoint +# sends "movie" while the batch and mass-operation paths send "movies". Both mean +# the same thing, and matching only one of them silently produced an empty title +# and an empty prompt description for every movie translated from its own page. +MOVIE_MEDIA_TYPES = ('movie', 'movies') + + +def is_movie_media_type(media_type) -> bool: + return str(media_type or '').strip().lower() in MOVIE_MEDIA_TYPES + + +def _scope(statement, table, arr_instance_id): + """Narrow a lookup to the owning arr instance (#156). + + ``radarrId`` and ``sonarrSeriesId`` are unique only together with + ``arr_instance_id``, so an unscoped query can return a sibling instance's + media. Callers that genuinely have no owner (older paths, and the tests that + seed a single instance) pass None and keep the previous behaviour. + """ + if arr_instance_id is None: + return statement + return statement.where(table.arr_instance_id == arr_instance_id) + + +def get_description(media_type, radarr_id, sonarr_series_id, arr_instance_id=None): try: - if media_type == 'movies': + if is_movie_media_type(media_type): movie = database.execute( - select(TableMovies.title, TableMovies.imdbId, TableMovies.year, TableMovies.overview) - .where(TableMovies.radarrId == radarr_id) + _scope( + select(TableMovies.title, TableMovies.imdbId, TableMovies.year, TableMovies.overview) + .where(TableMovies.radarrId == radarr_id), + TableMovies, arr_instance_id) ).first() if movie: @@ -128,8 +154,10 @@ def get_description(media_type, radarr_id, sonarr_series_id): else: series = database.execute( - select(TableShows.title, TableShows.imdbId, TableShows.year, TableShows.overview) - .where(TableShows.sonarrSeriesId == sonarr_series_id) + _scope( + select(TableShows.title, TableShows.imdbId, TableShows.year, TableShows.overview) + .where(TableShows.sonarrSeriesId == sonarr_series_id), + TableShows, arr_instance_id) ).first() if series: @@ -147,15 +175,17 @@ def get_title( media_type: str, radarr_id: Union[int, None] = None, sonarr_series_id: Union[int, None] = None, - sonarr_episode_id: Union[int, None] = None + sonarr_episode_id: Union[int, None] = None, + arr_instance_id: Union[int, None] = None ) -> str: try: - if media_type == "movies": + if is_movie_media_type(media_type): if radarr_id is None: return "" movie_row = database.execute( - select(TableMovies.title).where(TableMovies.radarrId == radarr_id) + _scope(select(TableMovies.title).where(TableMovies.radarrId == radarr_id), + TableMovies, arr_instance_id) ).first() if movie_row is None: @@ -176,7 +206,8 @@ def get_title( return "" series_row = database.execute( - select(TableShows.title).where(TableShows.sonarrSeriesId == sonarr_series_id) + _scope(select(TableShows.title).where(TableShows.sonarrSeriesId == sonarr_series_id), + TableShows, arr_instance_id) ).first() if series_row is None: @@ -193,8 +224,10 @@ def get_title( # If episode ID is provided, get episode details and format as "Series - S##E## - Episode Title" if sonarr_episode_id is not None: episode_row = database.execute( - select(TableEpisodes.season, TableEpisodes.episode, TableEpisodes.title) - .where(TableEpisodes.sonarrEpisodeId == sonarr_episode_id) + _scope( + select(TableEpisodes.season, TableEpisodes.episode, TableEpisodes.title) + .where(TableEpisodes.sonarrEpisodeId == sonarr_episode_id), + TableEpisodes, arr_instance_id) ).first() if episode_row is not None: diff --git a/bazarr/subtitles/tools/translate/main.py b/bazarr/subtitles/tools/translate/main.py index ce5a423e47..4a22a4b797 100644 --- a/bazarr/subtitles/tools/translate/main.py +++ b/bazarr/subtitles/tools/translate/main.py @@ -12,7 +12,7 @@ from languages.get_languages import alpha3_from_alpha2 from app.config import settings from app.jobs_queue import jobs_queue -from subtitles.indexer.utils import get_external_subtitles_path +from subtitles.indexer.utils import get_subtitle_destination_path def translate_subtitles_file(video_path, source_srt_file, from_lang, to_lang, forced, hi, @@ -25,7 +25,8 @@ def translate_subtitles_file(video_path, source_srt_file, from_lang, to_lang, fo jobs_queue.add_job_from_function( (lambda t: f'Translating {t} ({from_lang.upper()} to {to_lang.upper()})' if t else f'Translating {from_lang.upper()} to {to_lang.upper()}')( - get_title(media_type, radarr_id, sonarr_series_id, sonarr_episode_id)), + get_title(media_type, radarr_id, sonarr_series_id, sonarr_episode_id, + arr_instance_id)), is_progress=True) return @@ -46,7 +47,9 @@ def translate_subtitles_file(video_path, source_srt_file, from_lang, to_lang, fo hi_tag=hi ) - dest_srt_file = get_external_subtitles_path( + # Resolved as a write target, not a lookup: the file does not exist yet on a + # first translation, and the lookup helper answered None for custom folders. + dest_srt_file = get_subtitle_destination_path( file=video_path, subtitle=os.path.basename(dest_srt_file_if_alongside_video) ) @@ -68,7 +71,8 @@ def translate_subtitles_file(video_path, source_srt_file, from_lang, to_lang, fo hi=hi, sonarr_series_id=sonarr_series_id, sonarr_episode_id=sonarr_episode_id, - radarr_id=radarr_id + radarr_id=radarr_id, + arr_instance_id=arr_instance_id ) logging.debug(f'Created translator instance: {translator.__class__.__name__}') # noqa: G004 @@ -105,11 +109,12 @@ def translate_subtitles_file(video_path, source_srt_file, from_lang, to_lang, fo logging.exception("BAZARR combine-after-translate failed for %s", video_path) # Get current job name (which batch.py already set with title) and mark as done + completion_label = 'Partially translated' if getattr(translator, 'partial_error', None) else 'Translated' current_name = jobs_queue.get_job_name(job_id) if current_name and 'Translating' in current_name: - done_name = current_name.replace('Translating', 'Translated') + done_name = current_name.replace('Translating', completion_label) else: - done_name = f'Translated {from_lang.upper()} \u2192 {to_lang.upper()} using {translator_label}' + done_name = f'{completion_label} {from_lang.upper()} \u2192 {to_lang.upper()} using {translator_label}' jobs_queue.update_job_name(job_id=job_id, new_job_name=done_name) return result diff --git a/bazarr/subtitles/tools/translate/services/gemini_translator.py b/bazarr/subtitles/tools/translate/services/gemini_translator.py index 7a2e7ada94..b653c16cc2 100644 --- a/bazarr/subtitles/tools/translate/services/gemini_translator.py +++ b/bazarr/subtitles/tools/translate/services/gemini_translator.py @@ -12,6 +12,7 @@ import srt import pysubs2 +from subtitles.tools.subsync_engines import staged_subtitle_write import requests import unicodedata as ud from collections import Counter @@ -23,7 +24,7 @@ from radarr.history import history_log_movie from utilities.path_mappings import path_mappings # noqa: F401 from subtitles.processing import ProcessSubtitlesResult # noqa: F401 -from app.jobs_queue import jobs_queue +from app.jobs_queue import JobCancelled, jobs_queue from languages.get_languages import alpha3_from_alpha2, language_from_alpha2, language_from_alpha3 # noqa: F401 from ..core.translator_utils import add_translator_info, get_description, create_process_result @@ -45,13 +46,17 @@ class SubtitleObject(typing.TypedDict): class GeminiTranslatorService: def __init__(self, source_srt_file, dest_srt_file, to_lang, media_type, sonarr_series_id, sonarr_episode_id, - radarr_id, forced, hi, video_path, from_lang, orig_to_lang, **kwargs): + radarr_id, forced, hi, video_path, from_lang, orig_to_lang, + arr_instance_id=None, **kwargs): self.source_srt_file = source_srt_file self.dest_srt_file = dest_srt_file self.to_lang = to_lang self.media_type = media_type self.sonarr_series_id = sonarr_series_id self.radarr_id = radarr_id + # The owning arr instance (#156): radarrId and sonarrSeriesId are only + # unique together with it, so every media lookup below carries it. + self.arr_instance_id = arr_instance_id self.from_lang = from_lang self.video_path = video_path self.forced = forced @@ -95,23 +100,31 @@ def translate(self, job_id): self.gemini_api_key = self.current_api_key self.target_language = language_from_alpha3(self.to_lang) self.input_file = self.source_srt_file - self.output_file = self.dest_srt_file self.model_name = settings.translator.gemini_model self.batch_size = self._get_batch_size() - self.description = get_description(self.media_type, self.radarr_id, self.sonarr_series_id) + self.description = get_description(self.media_type, self.radarr_id, self.sonarr_series_id, + arr_instance_id=self.arr_instance_id) if self.input_file: self.progress_file = os.path.join(os.path.dirname(self.input_file), f".{os.path.basename(self.input_file)}.progress") - self._check_saved_progress() - try: - self._translate_with_gemini() - add_translator_info(self.dest_srt_file, f"# Subtitles translated with {settings.translator.gemini_model} # ") + with staged_subtitle_write(self.video_path, self.dest_srt_file, + source_paths=(self.source_srt_file,), + before_publish=lambda: jobs_queue.update_job_progress(job_id=job_id)) as temporary: + self.output_file = temporary + self._check_saved_progress() + self._translate_with_gemini() + add_translator_info(temporary, f"# Subtitles translated with {settings.translator.gemini_model} # ") + except JobCancelled: + raise except Exception as e: jobs_queue.update_job_progress(job_id=job_id, progress_message=f'Gemini translation error: {str(e)}') raise + except JobCancelled: + self._clear_progress() + raise except Exception as e: logger.error(f'BAZARR encountered an error translating with Gemini: {str(e)}') # noqa: G004 raise @@ -177,6 +190,8 @@ def _check_saved_progress(self): if saved_line > 1 and self.start_line == 1: os.remove(self.output_file) + except JobCancelled: + raise except Exception as e: jobs_queue.update_job_progress(job_id=self.job_id, progress_message=f"Error reading progress file: {e}") @@ -192,7 +207,7 @@ def _save_progress(self, line): jobs_queue.update_job_progress(job_id=self.job_id, progress_message=f"Failed to save progress: {e}") def _clear_progress(self): - """Clear the progress file on successful completion""" + """Remove saved progress after completion, failure, or cancellation.""" if self.progress_file and os.path.exists(self.progress_file): try: os.remove(self.progress_file) @@ -362,6 +377,7 @@ def _process_batch( } try: + jobs_queue.update_job_progress(job_id=self.job_id) response = requests.request("POST", url, headers=headers, data=payload) response.raise_for_status() # Raise an exception for bad status codes @@ -398,6 +414,8 @@ def clean_json_string(json_string): return self.current_progress + except JobCancelled: + raise except Exception as e: response = getattr(e, "response", None) if self._is_rate_limited_response(response): @@ -538,15 +556,20 @@ def _translate_with_gemini(self): # Clear progress file on successful completion self._clear_progress() + except JobCancelled: + raise except Exception as e: logger.error(f'BAZARR encountered an error translating with Gemini: {str(e)}') # noqa: G004 jobs_queue.update_job_progress(job_id=self.job_id, progress_value=total, progress_message=f'Gemini translation failed: {str(e)}') - self._clear_progress() - if self.output_file and os.path.exists(self.output_file): - try: - if os.path.getsize(self.output_file) == 0: - os.remove(self.output_file) - except OSError: - pass - raise e + raise + finally: + try: + self._clear_progress() + finally: + if self.output_file and os.path.exists(self.output_file): + try: + if os.path.getsize(self.output_file) == 0: + os.remove(self.output_file) + except OSError: + pass diff --git a/bazarr/subtitles/tools/translate/services/google_translator.py b/bazarr/subtitles/tools/translate/services/google_translator.py index ec5ecd7515..99e97e82a7 100644 --- a/bazarr/subtitles/tools/translate/services/google_translator.py +++ b/bazarr/subtitles/tools/translate/services/google_translator.py @@ -2,6 +2,7 @@ import logging import pysubs2 +from subtitles.tools.subsync_engines import staged_subtitle_write from retry.api import retry from app.config import settings # noqa: F401 @@ -23,7 +24,7 @@ class GoogleTranslatorService: def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, media_type, video_path, orig_to_lang, forced, hi, sonarr_series_id, sonarr_episode_id, - radarr_id): + radarr_id, arr_instance_id=None): self.source_srt_file = source_srt_file self.dest_srt_file = dest_srt_file self.lang_obj = lang_obj @@ -37,6 +38,9 @@ def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, self.sonarr_series_id = sonarr_series_id self.sonarr_episode_id = sonarr_episode_id self.radarr_id = radarr_id + # The owning arr instance (#156): radarrId and sonarrSeriesId are only + # unique together with it, so every media lookup below carries it. + self.arr_instance_id = arr_instance_id self.language_code_convert_dict = { 'he': 'iw', 'zh': 'zh-CN', @@ -45,67 +49,71 @@ def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, def translate(self, job_id): try: - subs = pysubs2.load(self.source_srt_file, encoding='utf-8') - subs.remove_miscellaneous_events() - lines_list = [x.plaintext for x in subs] - lines_list_len = len(lines_list) + with staged_subtitle_write(self.video_path, self.dest_srt_file, + source_paths=(self.source_srt_file,), + before_publish=lambda: jobs_queue.update_job_progress(job_id=job_id), + allow_empty=True) as temporary: + subs = pysubs2.load(self.source_srt_file, encoding='utf-8') + subs.remove_miscellaneous_events() + lines_list = [x.plaintext for x in subs] + lines_list_len = len(lines_list) + + jobs_queue.update_job_progress(job_id=job_id, progress_max=lines_list_len, + progress_message=self.source_srt_file) + + translated_lines = [] + logger.debug(f'starting translation for {self.source_srt_file}') # noqa: G004 + + def translate_line(line_id, subtitle_line): + try: + translated_text = self._translate_text(subtitle_line, job_id) + translated_lines.append({'id': line_id, 'line': translated_text}) + except TranslationNotFound: + logger.debug(f'Unable to translate line {subtitle_line}') # noqa: G004 + translated_lines.append({'id': line_id, 'line': subtitle_line}) + finally: + jobs_queue.update_job_progress(job_id=job_id, progress_value=len(translated_lines)) + + logger.debug(f'BAZARR is sending {lines_list_len} blocks to Google Translate') # noqa: G004 + pool = ThreadPoolExecutor(max_workers=10) + futures = [] + for i, line in enumerate(lines_list): + future = pool.submit(translate_line, i, line) + futures.append(future) + pool.shutdown(wait=True) + for future in futures: + try: + future.result() + except Exception as e: + logger.error(f"Error in translation task: {e}") # noqa: G004 + + for i, line in enumerate(translated_lines): + lines_list[line['id']] = line['line'] + + logger.debug(f'BAZARR saving translated subtitles to {self.dest_srt_file}') # noqa: G004 + for i, line in enumerate(subs): + try: + if lines_list[i]: + line.plaintext = lines_list[i] + else: + # we assume that there was nothing to translate if Google returns None. ex.: "♪♪" + continue + except IndexError: + logger.error(f'BAZARR is unable to translate malformed subtitles: {self.source_srt_file}') # noqa: G004 + jobs_queue.update_job_progress(job_id=job_id, + progress_message=f'Translation failed: Unable to translate ' + f'malformed subtitles for {self.source_srt_file}') + raise - jobs_queue.update_job_progress(job_id=job_id, progress_max=lines_list_len, - progress_message=self.source_srt_file) - - translated_lines = [] - logger.debug(f'starting translation for {self.source_srt_file}') # noqa: G004 - - def translate_line(line_id, subtitle_line): - try: - translated_text = self._translate_text(subtitle_line, job_id) - translated_lines.append({'id': line_id, 'line': translated_text}) - except TranslationNotFound: - logger.debug(f'Unable to translate line {subtitle_line}') # noqa: G004 - translated_lines.append({'id': line_id, 'line': subtitle_line}) - finally: - jobs_queue.update_job_progress(job_id=job_id, progress_value=len(translated_lines)) - - logger.debug(f'BAZARR is sending {lines_list_len} blocks to Google Translate') # noqa: G004 - pool = ThreadPoolExecutor(max_workers=10) - futures = [] - for i, line in enumerate(lines_list): - future = pool.submit(translate_line, i, line) - futures.append(future) - pool.shutdown(wait=True) - for future in futures: - try: - future.result() - except Exception as e: - logger.error(f"Error in translation task: {e}") # noqa: G004 - - for i, line in enumerate(translated_lines): - lines_list[line['id']] = line['line'] - - logger.debug(f'BAZARR saving translated subtitles to {self.dest_srt_file}') # noqa: G004 - for i, line in enumerate(subs): try: - if lines_list[i]: - line.plaintext = lines_list[i] - else: - # we assume that there was nothing to translate if Google returns None. ex.: "♪♪" - continue - except IndexError: - logger.error(f'BAZARR is unable to translate malformed subtitles: {self.source_srt_file}') # noqa: G004 + subs.save(temporary) + add_translator_info(temporary, f"# Subtitles translated with Google Translate # ") # noqa: F541 + except OSError: + logger.error(f'BAZARR is unable to save translated subtitles to {self.dest_srt_file}') # noqa: G004 jobs_queue.update_job_progress(job_id=job_id, - progress_message=f'Translation failed: Unable to translate ' - f'malformed subtitles for {self.source_srt_file}') - raise - - try: - subs.save(self.dest_srt_file) - add_translator_info(self.dest_srt_file, f"# Subtitles translated with Google Translate # ") # noqa: F541 - except OSError: - logger.error(f'BAZARR is unable to save translated subtitles to {self.dest_srt_file}') # noqa: G004 - jobs_queue.update_job_progress(job_id=job_id, - progress_message=f'Translation failed: Unable to save translated ' - f'subtitles to {self.dest_srt_file}') - raise OSError + progress_message=f'Translation failed: Unable to save translated ' + f'subtitles to {self.dest_srt_file}') + raise OSError message = f"{language_from_alpha2(self.from_lang)} subtitles translated to {language_from_alpha3(self.to_lang)}." result = create_process_result(message, self.video_path, self.orig_to_lang, self.forced, self.hi, self.dest_srt_file, self.media_type) diff --git a/bazarr/subtitles/tools/translate/services/lingarr_translator.py b/bazarr/subtitles/tools/translate/services/lingarr_translator.py index 0c98f9428f..88a14b6346 100644 --- a/bazarr/subtitles/tools/translate/services/lingarr_translator.py +++ b/bazarr/subtitles/tools/translate/services/lingarr_translator.py @@ -2,6 +2,7 @@ import logging import pysubs2 +from subtitles.tools.subsync_engines import staged_subtitle_write import requests from retry.api import retry @@ -30,7 +31,7 @@ class LingarrAuthError(Exception): class LingarrTranslatorService: def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, media_type, video_path, orig_to_lang, forced, hi, sonarr_series_id, sonarr_episode_id, - radarr_id): + radarr_id, arr_instance_id=None): self.source_srt_file = source_srt_file self.dest_srt_file = dest_srt_file self.lang_obj = lang_obj @@ -44,6 +45,9 @@ def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, self.sonarr_series_id = sonarr_series_id self.sonarr_episode_id = sonarr_episode_id self.radarr_id = radarr_id + # The owning arr instance (#156): radarrId and sonarrSeriesId are only + # unique together with it, so every media lookup below carries it. + self.arr_instance_id = arr_instance_id self.language_code_convert_dict = { 'zh': 'zh-CN', 'zt': 'zh-TW', @@ -52,44 +56,48 @@ def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, def translate(self, job_id=None): try: - jobs_queue.update_job_progress(job_id=job_id, progress_max=1, progress_message=self.source_srt_file) - - subs = pysubs2.load(self.source_srt_file, encoding='utf-8') - lines_list = [x.plaintext for x in subs] - lines_list_len = len(lines_list) - - if lines_list_len == 0: - logger.debug('No lines to translate in subtitle file') - return self.dest_srt_file - - logger.debug(f'Starting translation for {self.source_srt_file}') # noqa: G004 - translated_lines = self._translate_content(lines_list, job_id=job_id) - - if translated_lines is None: - logger.error(f'Translation failed for {self.source_srt_file}') # noqa: G004 - jobs_queue.update_job_progress(job_id=job_id, - progress_message=f'Translation failed for {self.source_srt_file}') - raise RuntimeError(f'Translation failed for {self.source_srt_file}') - - logger.debug(f'BAZARR saving Lingarr translated subtitles to {self.dest_srt_file}') # noqa: G004 - translation_map = {} - for item in translated_lines: - if isinstance(item, dict) and 'position' in item and 'line' in item: - translation_map[item['position']] = item['line'] - - for i, line in enumerate(subs): - if i in translation_map and translation_map[i]: - line.text = translation_map[i] - - try: - subs.save(self.dest_srt_file) - add_translator_info(self.dest_srt_file, f"# Subtitles translated with Lingarr # ") # noqa: F541 - except OSError: - logger.error(f'BAZARR is unable to save translated subtitles to {self.dest_srt_file}') # noqa: G004 - jobs_queue.update_job_progress(job_id=job_id, - progress_message=f'Translation failed: Unable to save translated ' - f'subtitles to {self.dest_srt_file}') - raise OSError + with staged_subtitle_write(self.video_path, self.dest_srt_file, + source_paths=(self.source_srt_file,), + before_publish=lambda: jobs_queue.update_job_progress(job_id=job_id), + allow_empty=True) as temporary: + jobs_queue.update_job_progress(job_id=job_id, progress_max=1, progress_message=self.source_srt_file) + + subs = pysubs2.load(self.source_srt_file, encoding='utf-8') + lines_list = [x.plaintext for x in subs] + lines_list_len = len(lines_list) + + if lines_list_len == 0: + logger.debug('No lines to translate in subtitle file') + return self.dest_srt_file + + logger.debug(f'Starting translation for {self.source_srt_file}') # noqa: G004 + translated_lines = self._translate_content(lines_list, job_id=job_id) + + if translated_lines is None: + logger.error(f'Translation failed for {self.source_srt_file}') # noqa: G004 + jobs_queue.update_job_progress(job_id=job_id, + progress_message=f'Translation failed for {self.source_srt_file}') + raise RuntimeError(f'Translation failed for {self.source_srt_file}') + + logger.debug(f'BAZARR saving Lingarr translated subtitles to {self.dest_srt_file}') # noqa: G004 + translation_map = {} + for item in translated_lines: + if isinstance(item, dict) and 'position' in item and 'line' in item: + translation_map[item['position']] = item['line'] + + for i, line in enumerate(subs): + if i in translation_map and translation_map[i]: + line.text = translation_map[i] + + try: + subs.save(temporary) + add_translator_info(temporary, f"# Subtitles translated with Lingarr # ") # noqa: F541 + except OSError: + logger.error(f'BAZARR is unable to save translated subtitles to {self.dest_srt_file}') # noqa: G004 + jobs_queue.update_job_progress(job_id=job_id, + progress_message=f'Translation failed: Unable to save translated ' + f'subtitles to {self.dest_srt_file}') + raise OSError message = (f"{language_from_alpha2(self.from_lang)} subtitles translated to " f"{language_from_alpha3(self.to_lang)} using Lingarr.") @@ -139,7 +147,8 @@ def _translate_content(self, lines_list, job_id): media_type=self.media_type, radarr_id=self.radarr_id, sonarr_series_id=self.sonarr_series_id, - sonarr_episode_id=self.sonarr_episode_id + sonarr_episode_id=self.sonarr_episode_id, + arr_instance_id=self.arr_instance_id ) if self.media_type == 'episode': diff --git a/bazarr/subtitles/tools/translate/services/openrouter_translator.py b/bazarr/subtitles/tools/translate/services/openrouter_translator.py index f58b3dbc01..a56940d511 100644 --- a/bazarr/subtitles/tools/translate/services/openrouter_translator.py +++ b/bazarr/subtitles/tools/translate/services/openrouter_translator.py @@ -1,8 +1,10 @@ # coding=utf-8 +import re import time import logging import pysubs2 +from subtitles.tools.subsync_engines import staged_subtitle_write, SubtitleDestinationChanged import requests from typing import Optional, List, Dict, Any @@ -14,13 +16,99 @@ from radarr.history import history_log_movie from sonarr.history import history_log from app.event_handler import show_progress, hide_progress, show_message -from app.jobs_queue import jobs_queue +from app.jobs_queue import jobs_queue, JobCancelled from ..core.translator_utils import add_translator_info, create_process_result, get_title from .auth import get_translator_auth_headers logger = logging.getLogger(__name__) +PROVIDER_ROUTING_VALUES = ('throughput', 'nitro', 'price', 'floor', 'latency', 'default') +DEFAULT_PROVIDER_ROUTING = 'throughput' +# Sidecars before this version forward provider.sort to OpenRouter verbatim, which +# rejects nitro, floor and default; they get the plain sort each value stands for. +ROUTING_SHORTCUTS_MIN_SIDECAR = (1, 3, 4) +ROUTING_PLAIN_SORT = {'nitro': 'throughput', 'floor': 'price', 'default': 'throughput'} +SIDECAR_VERSION_CACHE_SECONDS = 300 +_sidecar_version_cache = {} + +POLL_HARD_CAP_SECONDS = 12 * 3600 +POLL_UNREACHABLE_LIMIT_SECONDS = 600 +POLL_INTERVAL_SECONDS = 2 + + +def _typed_routing_suffix(model_id): + """'floor' or 'nitro' when the model id ends with that OpenRouter shortcut, else None.""" + for suffix in ('floor', 'nitro'): + if str(model_id or '').endswith(f':{suffix}'): + return suffix + return None + + +def reset_sidecar_version_cache(): + _sidecar_version_cache.clear() + + +def _parse_version(text): + """'1.3.4', '1.3.4-rc1' or 'v1.3.4' -> (1, 3, 4); None when it does not start with digits.""" + numbers = re.match(r'v?(\d+)(?:\.(\d+))?(?:\.(\d+))?', str(text or '')) + if not numbers: + return None + return tuple(int(part or 0) for part in numbers.groups()) + + +def sidecar_version(base_url): + """The AI Subtitle Translator version behind ``base_url``, cached per URL. + + Returns a version tuple, or None when the health endpoint is unreachable or + does not report a version. The probe is cheap and unauthenticated, and it is + cached so a job of many batches asks once. + """ + base_url = (base_url or '').rstrip('/') + cached = _sidecar_version_cache.get(base_url) + if cached and cached[0] > time.monotonic(): + return cached[1] + version = None + try: + response = requests.get(f"{base_url}/health", timeout=5) + if response.status_code == 200: + version = _parse_version(response.json().get('version')) + except (requests.exceptions.RequestException, ValueError, AttributeError) as e: + logger.debug("Could not read the AI Subtitle Translator version from %s: %s", base_url, e) + _sidecar_version_cache[base_url] = (time.monotonic() + SIDECAR_VERSION_CACHE_SECONDS, version) + return version + + +def build_provider_config(): + """The OpenRouter provider routing the sidecar applies to every request of a job. + + Left unset the sidecar sorts providers by throughput, which is the fastest and + often not the cheapest endpoint; the setting lets the user pick price, latency, + the ``:nitro``/``:floor`` shortcuts, or OpenRouter's own load balancing. A + sidecar older than 1.3.4 (or one whose version cannot be read) does not know + the shortcuts and would hand them to OpenRouter as an invalid sort, so it gets + the plain sort each of them stands for. + """ + routing = getattr(settings.translator, 'openrouter_provider_routing', DEFAULT_PROVIDER_ROUTING) + if routing not in PROVIDER_ROUTING_VALUES: + logger.warning("Unknown OpenRouter provider routing '%s', using %s", routing, DEFAULT_PROVIDER_ROUTING) + routing = DEFAULT_PROVIDER_ROUTING + typed = _typed_routing_suffix(getattr(settings.translator, 'openrouter_model', '')) + if typed: + # The slug already says how to route. A sidecar from 1.3.4 on drops the sort + # for a typed shortcut anyway; an older one forwards both, so the sort has to + # agree with the slug rather than with the setting. + return {'sort': ROUTING_PLAIN_SORT[typed]} + if routing in ROUTING_PLAIN_SORT: + version = sidecar_version(settings.translator.openrouter_url) + if version is None or version < ROUTING_SHORTCUTS_MIN_SIDECAR: + plain = ROUTING_PLAIN_SORT[routing] + logger.warning( + "AI Subtitle Translator %s does not support the '%s' provider routing (needs 1.3.4), sending %s", + '.'.join(map(str, version)) if version else 'of unknown version', routing, plain) + routing = plain + return {'sort': routing} + class OpenRouterTranslatorService: """ @@ -30,7 +118,7 @@ class OpenRouterTranslatorService: def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, media_type, video_path, orig_to_lang, forced, hi, sonarr_series_id, sonarr_episode_id, - radarr_id): + radarr_id, arr_instance_id=None): self.source_srt_file = source_srt_file self.dest_srt_file = dest_srt_file self.lang_obj = lang_obj @@ -44,6 +132,10 @@ def __init__(self, source_srt_file, dest_srt_file, lang_obj, to_lang, from_lang, self.sonarr_series_id = sonarr_series_id self.sonarr_episode_id = sonarr_episode_id self.radarr_id = radarr_id + # The owning arr instance (#156): radarrId and sonarrSeriesId are only + # unique together with it, so every media lookup below carries it. + self.arr_instance_id = arr_instance_id + self.partial_error = None self.language_code_convert_dict = { 'he': 'iw', 'zh': 'zh-CN', @@ -78,45 +170,53 @@ def _get_api_key_value(self): return api_key def translate(self, job_id=None): + self.partial_error = None try: - subs = pysubs2.load(self.source_srt_file, encoding='utf-8') - lines_list: List[str] = [x.plaintext for x in subs] - lines_list_len = len(lines_list) - - if lines_list_len == 0: - logger.debug('No lines to translate in subtitle file') - return self.dest_srt_file - - logger.debug(f'Starting AI translation for {self.source_srt_file}') # noqa: G004 - - # Submit job and poll for completion - translated_lines = self._submit_and_poll(lines_list, bazarr_job_id=job_id) - - if translated_lines is None: - logger.error(f'Translation failed for {self.source_srt_file}') # noqa: G004 - show_message(f'Translation failed for {self.source_srt_file}') - return False - - # Process results - logger.debug(f'BAZARR saving AI translated subtitles to {self.dest_srt_file}') # noqa: G004 - translation_map = {} - for item in translated_lines: - if isinstance(item, dict) and 'position' in item and 'line' in item: - translation_map[item['position']] = item['line'] - - for i, line in enumerate(subs): - if i in translation_map and translation_map[i]: - line.text = translation_map[i] - - try: - subs.save(self.dest_srt_file) - add_translator_info(self.dest_srt_file, "# Subtitles translated with AI Subtitle Translator #") - except OSError: - logger.error(f'BAZARR is unable to save translated subtitles to {self.dest_srt_file}') # noqa: G004 - show_message(f'Translation failed: Unable to save translated subtitles to {self.dest_srt_file}') - raise OSError - - message = f"{language_from_alpha2(self.from_lang)} subtitles translated to {language_from_alpha3(self.to_lang)} using AI Subtitle Translator." + with staged_subtitle_write(self.video_path, self.dest_srt_file, + source_paths=(self.source_srt_file,), + before_publish=lambda: jobs_queue.update_job_progress(job_id=job_id), + allow_empty=True) as temporary: + subs = pysubs2.load(self.source_srt_file, encoding='utf-8') + lines_list: List[str] = [x.plaintext for x in subs] + lines_list_len = len(lines_list) + + if lines_list_len == 0: + logger.debug('No lines to translate in subtitle file') + return False + + logger.debug(f'Starting AI translation for {self.source_srt_file}') # noqa: G004 + + # Submit job and poll for completion + translated_lines = self._submit_and_poll(lines_list, bazarr_job_id=job_id) + + if translated_lines is None: + logger.error(f'Translation failed for {self.source_srt_file}') # noqa: G004 + show_message(f'Translation failed for {self.source_srt_file}') + return False + + # Process results + logger.debug(f'BAZARR saving AI translated subtitles to {self.dest_srt_file}') # noqa: G004 + translation_map = {} + for item in translated_lines: + if isinstance(item, dict) and 'position' in item and 'line' in item: + translation_map[item['position']] = item['line'] + + missing_lines = sum(bool(source.strip()) and not translation_map.get(i, '').strip() + for i, source in enumerate(lines_list)) + if missing_lines and not self.partial_error: + self._mark_partial(f'No translated text was returned for {missing_lines} of {lines_list_len} cues.') + + for i, line in enumerate(subs): + if i in translation_map and translation_map[i].strip(): + line.text = translation_map[i] + + subs.save(temporary) + translated = 'partially translated' if self.partial_error else 'translated' + add_translator_info(temporary, f"# Subtitles {translated} with AI Subtitle Translator #") + + message = f"{language_from_alpha2(self.from_lang)} subtitles {translated} to {language_from_alpha3(self.to_lang)} using AI Subtitle Translator." + if self.partial_error: + message += f' Some lines may remain in the source language. {self.partial_error}' result = create_process_result(message, self.video_path, self.orig_to_lang, self.forced, self.hi, self.dest_srt_file, self.media_type) if self.media_type == 'episode': @@ -131,6 +231,8 @@ def translate(self, job_id=None): return self.dest_srt_file + except (JobCancelled, SubtitleDestinationChanged): + raise except Exception as e: logger.error(f'BAZARR encountered an error during AI translation: {str(e)}') # noqa: G004 show_message(f'AI translation failed: {str(e)}') @@ -168,7 +270,8 @@ def _submit_and_poll(self, lines_list: List[str], bazarr_job_id=None) -> Optiona media_type=self.media_type, radarr_id=self.radarr_id, sonarr_series_id=self.sonarr_series_id, - sonarr_episode_id=self.sonarr_episode_id + sonarr_episode_id=self.sonarr_episode_id, + arr_instance_id=self.arr_instance_id ) api_media_type = "Episode" if self.media_type == 'episode' else "Movie" @@ -189,6 +292,7 @@ def _submit_and_poll(self, lines_list: List[str], bazarr_job_id=None) -> Optiona "maxConcurrentJobs": settings.translator.openrouter_max_concurrent, "parallelBatches": settings.translator.openrouter_parallel_batches, "reasoning": self._build_reasoning_config(), + "provider": build_provider_config(), } } @@ -229,13 +333,38 @@ def _submit_and_poll(self, lines_list: List[str], bazarr_job_id=None) -> Optiona logger.error(f'AI Subtitle Translator error: {str(e)}') # noqa: G004 return None + def _mark_partial(self, detail): + self.partial_error = ' '.join(str(detail).split())[:500] or 'Some translation batches failed.' + logger.warning("Translation partially completed: %s", self.partial_error) + show_message('Translation is partial. Some lines may remain in the source language. ' + f'{self.partial_error}') + def _poll_job(self, base_url: str, job_id: str, total_lines: int, bazarr_job_id=None) -> Optional[Any]: - """Poll job status until completion""" - poll_interval = 2 # seconds - max_wait_time = 1800 # 30 minutes - elapsed = 0 + """Poll until a terminal status, subject to reachability and safety limits. + + The sidecar owns request timeouts and retries, so there is no normal total-time cap. + A slow model with reasoning enabled and a shrunk batch size can take over half an hour. + The old 30-minute cap discarded a translation that the sidecar finished successfully. + A 12-hour hard cap remains as a safety net. + """ + self.partial_error = None + started_at = time.monotonic() + last_reachable_at = started_at + + while True: + now = time.monotonic() + if now - started_at >= POLL_HARD_CAP_SECONDS: + reason = "reached the 12-hour polling hard cap" + user_message = "Translation stopped after 12 hours" + break + + unreachable_seconds = now - last_reachable_at + if unreachable_seconds >= POLL_UNREACHABLE_LIMIT_SECONDS: + unreachable_minutes = int(unreachable_seconds // 60) + reason = f"status endpoint unreachable for {unreachable_minutes} minutes" + user_message = f"Translation service unreachable for {unreachable_minutes} minutes" + break - while elapsed < max_wait_time: try: status_response = requests.get( f"{base_url}/api/v1/jobs/{job_id}", @@ -245,10 +374,10 @@ def _poll_job(self, base_url: str, job_id: str, total_lines: int, bazarr_job_id= if status_response.status_code != 200: logger.error(f"Error getting job status: {status_response.status_code}") # noqa: G004 - time.sleep(poll_interval) - elapsed += poll_interval + time.sleep(POLL_INTERVAL_SECONDS) continue + last_reachable_at = time.monotonic() job_status = status_response.json() status = job_status.get("status") progress = job_status.get("progress", 0) @@ -275,15 +404,12 @@ def _poll_job(self, base_url: str, job_id: str, total_lines: int, bazarr_job_id= if status == "completed": hide_progress(id=f'translate_progress_{self.dest_srt_file}') - result = job_status.get("result") - if result: - # Handle structured response with "lines" key from AI Subtitle Translator - # The service returns {"lines": [...], "model_used": ..., "tokens_used": ...} - if isinstance(result, dict) and "lines" in result: - logger.debug(f'Extracted {len(result["lines"])} lines from structured result') # noqa: G004 - return result["lines"] - # Fallback for direct list response - return result + lines = self._validated_result_lines(job_status.get("result"), total_lines) + # An empty list is not a translation: saving it would write every source + # line under the target name and record a success in History. + if lines: + logger.debug(f'Extracted {len(lines)} lines from job result') # noqa: G004 + return lines logger.error("Job completed but no result returned") return None @@ -296,7 +422,11 @@ def _poll_job(self, base_url: str, job_id: str, total_lines: int, bazarr_job_id= elif status == "partial": hide_progress(id=f'translate_progress_{self.dest_srt_file}') - error = job_status.get("error", message or "Partial translation") + error = job_status.get("error") or message or "Partial translation" + lines = self._validated_result_lines(job_status.get("result"), total_lines) + if lines is not None: + self._mark_partial(error) + return lines logger.error(f"Translation partially failed: {error}") # noqa: G004 show_message(f"Translation failed (partial): {error}") return None @@ -307,20 +437,37 @@ def _poll_job(self, base_url: str, job_id: str, total_lines: int, bazarr_job_id= return None # Still processing or queued - time.sleep(poll_interval) - elapsed += poll_interval + time.sleep(POLL_INTERVAL_SECONDS) except requests.exceptions.RequestException as e: logger.warning(f"Error polling job status: {e}") # noqa: G004 - time.sleep(poll_interval) - elapsed += poll_interval + time.sleep(POLL_INTERVAL_SECONDS) - # Timeout hide_progress(id=f'translate_progress_{self.dest_srt_file}') - logger.error("Translation job timed out") - show_message("Translation timed out after 30 minutes") + logger.error(f"Translation job {job_id} {reason}") # noqa: G004 + show_message(user_message) return None + @staticmethod + def _validated_result_lines(result, total_lines): + if isinstance(result, dict): + result = result.get('lines') + if not isinstance(result, list) or not result: + return None + positions = set() + has_translation = False + for item in result: + if not isinstance(item, dict): + return None + position = item.get('position') + line = item.get('line') + if (type(position) is not int or not 0 <= position < total_lines + or position in positions or not isinstance(line, str)): + return None + positions.add(position) + has_translation = has_translation or bool(line.strip()) + return result if has_translation else None + @retry(exceptions=(TooManyRequests, RequestError, requests.exceptions.RequestException), tries=3, delay=1, backoff=2, jitter=(0, 1)) def _translate_sync(self, lines_list: List[str], payload: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: """Fallback synchronous translation (Lingarr-compatible)""" @@ -334,20 +481,11 @@ def _translate_sync(self, lines_list: List[str], payload: Dict[str, Any]) -> Opt ) if response.status_code == 200: - translated_batch = response.json() - if isinstance(translated_batch, list): - for item in translated_batch: - if not isinstance(item, dict) or 'position' not in item or 'line' not in item: - logger.error(f'Invalid response format: {item}') # noqa: G004 - return None - return translated_batch - else: - logger.error(f'Unexpected response format: {translated_batch}') # noqa: G004 - return None + return self._validated_result_lines(response.json(), len(lines_list)) elif response.status_code == 429: raise TooManyRequests("Rate limit exceeded") elif response.status_code >= 500: raise RequestError(f"Server error: {response.status_code}") else: logger.error(f'API error: {response.status_code} - {response.text}') # noqa: G004 - return None \ No newline at end of file + return None diff --git a/bazarr/subtitles/upload.py b/bazarr/subtitles/upload.py index 3cbd7f40ab..3e1bc0152d 100644 --- a/bazarr/subtitles/upload.py +++ b/bazarr/subtitles/upload.py @@ -4,6 +4,7 @@ import os import sys import logging +from functools import partial from subzero.language import Language from subliminal_patch.core import save_subtitles @@ -28,16 +29,59 @@ from app.event_handler import event_stream from app.notifier import send_notifications from app.notifier import send_notifications_movie -from subtitles.indexer.series import store_subtitles -from subtitles.indexer.movies import store_subtitles_movie from subtitles.processing import ProcessSubtitlesResult +from subtitles.tools.subsync_engines import (SubtitlePublication, write_subtitle_file, + subtitle_source_version, subtitle_write_locks) -from .sync import sync_subtitles +from .sync import sync_subtitles, _index_keep_all_outputs from .post_processing import postprocessing from plex.operations import plex_set_movie_added_date_now, plex_set_episode_added_date_now, plex_refresh_item from jellyfin.operations import jellyfin_refresh_item +def _refresh_uploaded_subtitles(video_path, subtitle_path, sonarr_series_id=None, sonarr_episode_id=None, + radarr_id=None, arr_instance_id=None): + _index_keep_all_outputs(video_path, sonarr_series_id=sonarr_series_id, + sonarr_episode_id=sonarr_episode_id, radarr_id=radarr_id, + arr_instance_id=arr_instance_id) + + +def _notify_upload(consumer, callback, *args, **kwargs): + try: + callback(*args, **kwargs) + except Exception as exc: + logging.warning('BAZARR upload notification failed for %s (%s)', consumer, type(exc).__name__) + + +def _refresh_upload_consumers(media_type, metadata, arr_instance_id): + callbacks = [] + if media_type == 'series': + callbacks.append(('Sonarr', lambda: notify_sonarr( + metadata.sonarrSeriesId, + arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)))) + if settings.general.use_plex and settings.plex.update_series_library: + callbacks.append(('Plex', lambda: plex_refresh_item( + metadata.imdbId, is_movie=False, season=metadata.season, episode=metadata.episode))) + if settings.general.use_jellyfin and settings.jellyfin.update_series_library: + callbacks.append(('Jellyfin', lambda: jellyfin_refresh_item( + metadata.imdbId, is_movie=False, season=metadata.season, episode=metadata.episode, + tvdb_id=metadata.tvdbId))) + else: + callbacks.append(('Radarr', lambda: notify_radarr( + metadata.radarrId, + arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)))) + if settings.general.use_plex and settings.plex.update_movie_library: + callbacks.append(('Plex', lambda: plex_refresh_item(metadata.imdbId, is_movie=True))) + if settings.general.use_jellyfin and settings.jellyfin.update_movie_library: + callbacks.append(('Jellyfin', lambda: jellyfin_refresh_item( + metadata.imdbId, is_movie=True, tmdb_id=metadata.tmdbId))) + for consumer, callback in callbacks: + try: + callback() + except Exception as exc: + logging.warning('BAZARR upload refresh failed for %s (%s)', consumer, type(exc).__name__) + + def manual_upload_subtitle(path, language, forced, hi, media_type, subtitle, filename, audio_language, job_id=None, sonarrSeriesId=None, sonarrEpisodeId=None, radarrId=None, arr_instance_id=None): if not job_id: @@ -87,6 +131,8 @@ def manual_upload_subtitle(path, language, forced, hi, media_type, subtitle, fil .first() if episode_metadata: + sonarrSeriesId = episode_metadata.sonarrSeriesId + sonarrEpisodeId = episode_metadata.sonarrEpisodeId use_original_format = bool(get_profiles_list(episode_metadata.profileId)["originalFormat"]) else: return @@ -99,6 +145,7 @@ def manual_upload_subtitle(path, language, forced, hi, media_type, subtitle, fil .first() if movie_metadata: + radarrId = movie_metadata.radarrId use_original_format = bool(get_profiles_list(movie_metadata.profileId)["originalFormat"]) else: return @@ -133,14 +180,22 @@ def manual_upload_subtitle(path, language, forced, hi, media_type, subtitle, fil try: # ensure that formats must be a tuple of strings sub_format = (sub.format,) if isinstance(sub.format, str) else sub.format - saved_subtitles = save_subtitles(path, - [sub], - single=single, - tags=None, # fixme - directory=get_target_folder(path), - chmod=chmod, - formats=sub_format if use_original_format else ("srt",), - path_decoder=force_unicode) + subtitle_directory = get_target_folder(path) + with subtitle_write_locks(path, os.path.join(subtitle_directory or os.path.dirname(path), '.destination')): + written_paths = [] + saved_subtitles = save_subtitles(path, + [sub], + single=single, + tags=None, # fixme + directory=subtitle_directory, + chmod=chmod, + formats=sub_format if use_original_format else ("srt",), + path_decoder=force_unicode, + write_subtitle=partial(write_subtitle_file, path, written_paths=written_paths)) + saved_subtitles = [saved for saved in saved_subtitles if saved.storage_path in written_paths] + source_version = subtitle_source_version(saved_subtitles[0].storage_path) if saved_subtitles else None + source_publication = (SubtitlePublication(path, saved_subtitles[0].storage_path, source_version) + if source_version is not None else None) except Exception as e: logging.exception(f'BAZARR Error saving Subtitles file to disk for this file {path}: {repr(e)}') # noqa: G004 return @@ -185,14 +240,20 @@ def manual_upload_subtitle(path, language, forced, hi, media_type, subtitle, fil uploaded_language_code3, audio_language['name'], audio_language['code2'], audio_language['code3'], 100, "1", "manual", "user", "unknown", sonarrSeriesId, sonarrEpisodeId or radarrId,) - postprocessing(command, path) - set_chmod(subtitles_path=subtitle_path) + with subtitle_write_locks(path, subtitle_path): + if subtitle_source_version(subtitle_path) == source_version: + postprocessing(command, path, subtitle_path=subtitle_path) + set_chmod(subtitles_path=subtitle_path) + source_version = subtitle_source_version(subtitle_path) + source_publication.release() + source_publication = SubtitlePublication(path, subtitle_path, source_version) + + refresh_subtitles = partial(_refresh_uploaded_subtitles, path, subtitle_path, sonarr_series_id=sonarrSeriesId, + sonarr_episode_id=sonarrEpisodeId, radarr_id=radarrId, + arr_instance_id=arr_instance_id) + refresh_subtitles() if media_type == 'series': - sync_subtitles(video_path=path, srt_path=subtitle_path, srt_lang=uploaded_language_code2, percent_score=100, - sonarr_series_id=episode_metadata.sonarrSeriesId, forced=forced, hi=hi, - sonarr_episode_id=episode_metadata.sonarrEpisodeId, job_id=job_id, - arr_instance_id=arr_instance_id) # Reverse-map through the owning instance's path_mappings (#156); None # owner => global mapping (the default/single-instance path), unchanged. reversed_path = path_mappings.path_replace_reverse_instance(path, arr_instance_id, "series") @@ -200,21 +261,14 @@ def manual_upload_subtitle(path, language, forced, hi, media_type, subtitle, fil subtitle_path, arr_instance_id, "series") # Route the rescan at the OWNING instance's server (#156). None owner = # default server (legacy single-instance), unchanged. - notify_sonarr(episode_metadata.sonarrSeriesId, - arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)) event_stream(type='series', action='update', payload=episode_metadata.sonarrSeriesId) event_stream(type='episode-wanted', action='delete', payload=episode_metadata.sonarrEpisodeId) else: - sync_subtitles(video_path=path, srt_path=subtitle_path, srt_lang=uploaded_language_code2, percent_score=100, - radarr_id=movie_metadata.radarrId, forced=forced, hi=hi, job_id=job_id, - arr_instance_id=arr_instance_id) # Reverse-map through the owning instance's path_mappings (#156); None # owner => global mapping (the default/single-instance path), unchanged. reversed_path = path_mappings.path_replace_reverse_instance(path, arr_instance_id, "movie") reversed_subtitles_path = path_mappings.path_replace_reverse_instance( subtitle_path, arr_instance_id, "movie") - notify_radarr(movie_metadata.radarrId, - arr_client=client_for_instance(database, arr_instance_id, enabled_only=False)) event_stream(type='movie', action='update', payload=movie_metadata.radarrId) event_stream(type='movie-wanted', action='delete', payload=movie_metadata.radarrId) @@ -240,32 +294,27 @@ def manual_upload_subtitle(path, language, forced, hi, media_type, subtitle, fil history_log(4, sonarrSeriesId, sonarrEpisodeId, result, fake_provider=provider, fake_score=MAX_SCORES['episode'], arr_instance_id=arr_instance_id) if not settings.general.dont_notify_manual_actions: - send_notifications(sonarrSeriesId, sonarrEpisodeId, result.message, + _notify_upload("user", send_notifications, sonarrSeriesId, sonarrEpisodeId, result.message, arr_instance_id=arr_instance_id) - store_subtitles(result.path, path, arr_instance_id=arr_instance_id) if settings.general.use_plex: - if settings.plex.update_series_library: - plex_refresh_item(episode_metadata.imdbId, is_movie=False, - season=episode_metadata.season, episode=episode_metadata.episode) if settings.plex.set_episode_added: - plex_set_episode_added_date_now(episode_metadata) - if settings.general.use_jellyfin and settings.jellyfin.update_series_library: - jellyfin_refresh_item(episode_metadata.imdbId, is_movie=False, - season=episode_metadata.season, episode=episode_metadata.episode, - tvdb_id=episode_metadata.tvdbId) + _notify_upload("Plex added date", plex_set_episode_added_date_now, episode_metadata) else: history_log_movie(4, radarrId, result, fake_provider=provider, fake_score=MAX_SCORES['movie'], arr_instance_id=arr_instance_id) if not settings.general.dont_notify_manual_actions: - send_notifications_movie(radarrId, result.message, arr_instance_id=arr_instance_id) - store_subtitles_movie(result.path, path, arr_instance_id=arr_instance_id) + _notify_upload("user", send_notifications_movie, radarrId, result.message, arr_instance_id=arr_instance_id) if settings.general.use_plex: - if settings.plex.update_movie_library: - plex_refresh_item(movie_metadata.imdbId, is_movie=True) if settings.plex.set_movie_added: - plex_set_movie_added_date_now(movie_metadata) - if settings.general.use_jellyfin and settings.jellyfin.update_movie_library: - jellyfin_refresh_item(movie_metadata.imdbId, is_movie=True, - tmdb_id=movie_metadata.tmdbId) - + _notify_upload("Plex added date", plex_set_movie_added_date_now, movie_metadata) + + refresh_consumers = partial(_refresh_upload_consumers, media_type, + episode_metadata if media_type == 'series' else movie_metadata, arr_instance_id) + refresh_consumers() + if source_publication is not None: + sync_subtitles(video_path=path, srt_path=subtitle_path, srt_lang=uploaded_language_code2, + percent_score=100, forced=forced, hi=hi, sonarr_series_id=sonarrSeriesId, + sonarr_episode_id=sonarrEpisodeId, radarr_id=radarrId, + arr_instance_id=arr_instance_id, callback=refresh_subtitles, + source_version=source_publication, on_success=refresh_consumers) return '', 204 diff --git a/custom_libs/subliminal_patch/core.py b/custom_libs/subliminal_patch/core.py index 3829e6b199..c24757daa6 100644 --- a/custom_libs/subliminal_patch/core.py +++ b/custom_libs/subliminal_patch/core.py @@ -23,7 +23,7 @@ from concurrent.futures import as_completed from .extensions import provider_registry -from .exceptions import APIThrottled, MustGetBlacklisted +from .exceptions import APIThrottled, MustGetBlacklisted, SubtitleCandidateRejected from .score import compute_score, MAX_SCORES from subliminal.video import VIDEO_EXTENSIONS, Video, Episode, Movie from subliminal.core import guessit, ProviderPool, ThreadPoolExecutor, check_video @@ -709,6 +709,10 @@ def download_subtitle(self, subtitle): self.post_download_hook(subtitle) break + except SubtitleCandidateRejected as e: + logger.warning('Subtitle candidate rejected: %s', e) + return False + except (requests.ConnectionError, requests.exceptions.ProxyError, requests.exceptions.SSLError, @@ -871,10 +875,10 @@ def download_best_subtitles(self, subtitles, video, languages, min_score=0, hear subtitle.score = score downloaded_subtitles.append(subtitle) - # stop if only one subtitle is requested - if only_one: - logger.debug('Only one subtitle downloaded') - break + # stop after one successful download, not one attempted candidate + if only_one: + logger.debug('Only one subtitle downloaded') + break # --- WHISPER FALLBACK PRECONDITIONS --- # 1. No regular provider results with at least minimum score @@ -1467,7 +1471,7 @@ def get_subtitle_path(video_path, language=None, extension='.srt', forced_tag=Fa def save_subtitles(file_path, subtitles, single=False, directory=None, chmod=None, formats=("srt",), - tags=None, path_decoder=None, debug_mods=False): + tags=None, path_decoder=None, debug_mods=False, write_subtitle=None): """Save subtitles on filesystem. Subtitles are saved in the order of the list. If a subtitle with a language has already been saved, other subtitles @@ -1482,6 +1486,7 @@ def save_subtitles(file_path, subtitles, single=False, directory=None, chmod=Non :type subtitles: list of :class:`~subliminal.subtitle.Subtitle` :param bool single: save a single subtitle, default is to save one subtitle per language. :param str directory: path to directory where to save the subtitles, default is next to the video. + :param write_subtitle: optional callback receiving each exact destination and encoded content :return: the saved subtitles :rtype: list of :class:`~subliminal.subtitle.Subtitle` @@ -1539,11 +1544,13 @@ def save_subtitles(file_path, subtitles, single=False, directory=None, chmod=Non logger.debug(u"Saving %r to %r", subtitle, subtitle_path) content = subtitle.get_modified_content(format=format, debug=debug_mods) if content: - if os.path.exists(subtitle_path): - os.remove(subtitle_path) - - with open(subtitle_path, 'wb') as f: - f.write(content) + if write_subtitle is not None: + write_subtitle(subtitle_path, content) + else: + if os.path.exists(subtitle_path): + os.remove(subtitle_path) + with open(subtitle_path, 'wb') as f: + f.write(content) subtitle.storage_path = subtitle_path else: logger.error(u"Something went wrong when getting modified subtitle for %s", subtitle) diff --git a/custom_libs/subliminal_patch/exceptions.py b/custom_libs/subliminal_patch/exceptions.py index 0714f23080..8567114a10 100644 --- a/custom_libs/subliminal_patch/exceptions.py +++ b/custom_libs/subliminal_patch/exceptions.py @@ -15,6 +15,13 @@ def __init__(self, *args, retry_after=None, **kwargs): self.retry_after = retry_after +class SubtitleCandidateRejected(ProviderError): + """A valid response has no usable subtitle for this candidate. + + Messages must contain only bounded, non-sensitive diagnostic context. + """ + + class ForbiddenError(ProviderError): pass diff --git a/custom_libs/subliminal_patch/providers/utils.py b/custom_libs/subliminal_patch/providers/utils.py index 4857e818c0..136d3e8789 100644 --- a/custom_libs/subliminal_patch/providers/utils.py +++ b/custom_libs/subliminal_patch/providers/utils.py @@ -113,51 +113,69 @@ def wrapper(self, subtitle): return decorator +def _contains_number(guessed, requested): + guessed_numbers = guessed if isinstance(guessed, (list, tuple)) else (guessed,) + requested_numbers = requested if isinstance(requested, (list, tuple)) else (requested,) + return any(number in guessed_numbers for number in requested_numbers) + + def _get_matching_sub( - sub_names, forced=False, episode=None, episode_title=None, **kwargs + sub_names, forced=False, episode=None, episode_title=None, log_member_names=True, + match_episode_context=False, season=None, **kwargs ): - guess_options = {"single_value": True} - if episode is not None: + guess_options = {} if match_episode_context else {"single_value": True} + if episode is not None or (match_episode_context and season is not None): guess_options["type"] = "episode" # type: ignore matching_subs = [] + movie_selection = episode is None and episode_title is None for sub_name in sub_names: if not forced and os.path.splitext(sub_name.lower())[0].endswith("forced"): - logger.debug("Ignoring forced subtitle: %s", sub_name) + if log_member_names: + logger.debug("Ignoring forced subtitle: %s", sub_name) + continue + + guess = {} + if not movie_selection or (match_episode_context and season is not None): + guess = guessit(sub_name, options=guess_options) + if (match_episode_context and season is not None and guess.get("season") is not None + and not _contains_number(guess["season"], season)): continue - # If it's a movie then get the first subtitle - if episode is None and episode_title is None: - logger.debug("Movie subtitle found: %s", sub_name) + # Without episode context, get the first member that passed the season check. + if movie_selection: + if log_member_names: + logger.debug("Movie subtitle found: %s", sub_name) matching_subs.append(_MatchingSub(sub_name, 2, "Movie subtitle")) break - guess = guessit(sub_name, options=guess_options) - matched_episode_num = guess.get("episode") - if not matched_episode_num: + if not matched_episode_num and log_member_names: logger.debug("No episode number found in file: %s", sub_name) if episode_title is not None: - from_name = _analize_sub_name(sub_name, episode_title) + from_name = _analize_sub_name(sub_name, episode_title, log_member_names) if from_name is not None: matching_subs.append(from_name) - if episode == matched_episode_num: - logger.debug("Episode matched from number: %s", sub_name) + if (_contains_number(matched_episode_num, episode) if match_episode_context + else episode == matched_episode_num): + if log_member_names: + logger.debug("Episode matched from number: %s", sub_name) matching_subs.append(_MatchingSub(sub_name, 2, "Episode number matched")) if matching_subs: matching_subs.sort(key=lambda x: x.priority, reverse=True) - logger.debug("Matches: %s", matching_subs) + if log_member_names: + logger.debug("Matches: %s", matching_subs) return matching_subs[0].file else: logger.debug("Nothing matched") return None -def _analize_sub_name(sub_name: str, title_: str): +def _analize_sub_name(sub_name: str, title_: str, log_member_names=True): titles = re.split(r"[\s_\.\+]?[.-][\s_\.\+]?", os.path.splitext(sub_name)[0]) for title in titles: @@ -165,9 +183,10 @@ def _analize_sub_name(sub_name: str, title_: str): ratio = SequenceMatcher(None, title.lower(), title_.lower()).ratio() if ratio > 0.85: - logger.debug( - "Episode title matched: '%s' -> '%s' [%s]", title, sub_name, ratio - ) + if log_member_names: + logger.debug( + "Episode title matched: '%s' -> '%s' [%s]", title, sub_name, ratio + ) # Avoid false positives with short titles if len(title_) > 4 and ratio >= 0.98: @@ -175,7 +194,8 @@ def _analize_sub_name(sub_name: str, title_: str): return _MatchingSub(sub_name, 1, "Normal title ratio") - logger.debug("No episode title matched from file: %s", sub_name) + if log_member_names: + logger.debug("No episode title matched from file: %s", sub_name) return None @@ -185,34 +205,57 @@ def get_subtitle_from_archive( episode=None, get_first_subtitle=False, extensions=DEFAULT_ARCHIVE_EXTENSIONS, + match_episode_context=False, + log_member_names=True, + season=None, **kwargs, ): - "Get subtitle from Rarfile/Zipfile object. Return None if nothing is found." + """Return subtitle bytes, or None if no member matches. + + The host can match all included episodes and reject explicit season mismatches + while supplying sanitized diagnostics. Defaults preserve provider behavior. + An explicit first-subtitle request remains authoritative. + """ subs_in_archive = list_subtitle_members(archive, extensions) if not subs_in_archive: # Name the members: without them a user whose download failed here has no # way to tell an archive full of the wrong extensions from a broken one. - logger.warning( - "No subtitles found in archive. Members: %s", _archive_member_names(archive) - ) + if log_member_names: + logger.warning( + "No subtitles found in archive. Members: %s", _archive_member_names(archive) + ) return None - logger.debug("Subtitles in archive: %s", subs_in_archive) + if log_member_names: + logger.debug("Subtitles in archive: %s", subs_in_archive) if len(subs_in_archive) == 1 or get_first_subtitle: - logger.debug("Getting first subtitle in archive: %s", subs_in_archive) + if match_episode_context and (episode is not None or season is not None) and not get_first_subtitle: + guess = guessit( + subs_in_archive[0], options={"type": "episode"} + ) + for key, requested in (("season", season), ("episode", episode)): + if requested is not None and guess.get(key) is not None and not _contains_number(guess[key], requested): + return None + if log_member_names: + logger.debug("Getting first subtitle in archive: %s", subs_in_archive) return fix_line_ending(archive.read(subs_in_archive[0])) - matching_sub = _get_matching_sub(subs_in_archive, forced, episode, **kwargs) + matching_sub = _get_matching_sub( + subs_in_archive, forced, episode, log_member_names=log_member_names, + match_episode_context=match_episode_context, season=season, **kwargs + ) if matching_sub is not None: - logger.info("Using %s from archive", matching_sub) + if log_member_names: + logger.info("Using %s from archive", matching_sub) return fix_line_ending(archive.read(matching_sub)) - logger.warning( - "No subtitle in archive matched this episode. Members: %s", subs_in_archive - ) + if log_member_names: + logger.warning( + "No subtitle in archive matched this episode. Members: %s", subs_in_archive + ) return None diff --git a/docs/release-notes/v2.6.2-clockwork.md b/docs/release-notes/v2.6.2-clockwork.md new file mode 100644 index 0000000000..8efc5e3a9c --- /dev/null +++ b/docs/release-notes/v2.6.2-clockwork.md @@ -0,0 +1,141 @@ +# Bazarr+ v2.6.2 (Clockwork) + +Codename: **Clockwork** + +A stability patch for subtitle uploads, searches, disk scans and translation. Uploaded subtitles appear before automatic sync finishes. Downloads keep their server ownership, sync outputs stay with the correct video, and useful partial OpenRouter translations are saved with an honest incomplete status. OpenRouter routing choices now stay consistent with the model field, and Provider Hub distinguishes a bad subtitle candidate from a provider-wide failure. + +--- + +## Headline: Keep useful subtitle work and continue past unusable candidates + +### Partial translations are recoverable + +The OpenRouter translator service can return useful lines even when some batches fail. Bazarr previously discarded that partial result. It now saves valid translated lines, retains the source text where translation is missing, and marks the result as partially translated in history, job status and the subtitle footer. An empty or malformed result still fails. Saving uses a temporary file, so an unsuccessful write or footer update preserves any existing subtitle. + +The host also waits for long-running OpenRouter translation jobs while the service remains reachable, with a 12-hour overall limit and a 10-minute limit without a successful status response. This does not repair a translator service's own model or response-parsing errors. + +The separately deployed translator service reference advances to v1.3.4. The included service updates tolerate more malformed model replies, let batch sizes recover after failures, and report how many lines were actually translated. They align the prompt with JSON object mode, retry incomplete replies at the minimum batch size, and account for usage from failed or retried batches. Version 1.3.4 adds provider routing. Updating Bazarr alone does not replace a running translator service. + +### Uploads stay visible while sync runs + +A successful subtitle upload previously waited for automatic synchronization before appearing in the media's subtitle list. It now appears as soon as the upload is finalized, and synchronization runs as a separate progress job. Its completion refreshes the list, including saved outputs from multiple engines. A failed, skipped or cancelled sync leaves the uploaded subtitle available. + +Queued and running sync work also checks whether its source was replaced or deleted. An older sync result cannot overwrite a newer upload or restore a subtitle the user removed. Existing synchronization settings, instance ownership and upload history are retained. + +If older synchronized outputs cannot be moved aside after a new subtitle is saved, Bazarr reports the cleanup failure separately and keeps the completed save available. Those older outputs may remain listed until cleanup succeeds. + +### OpenRouter routing matches the saved settings + +Settings now offer fastest, cheapest, lowest-latency and OpenRouter-default routing, with separate `:nitro` and `:floor` shortcut choices. Typing either shortcut into the model field moves it into the routing selector when editing finishes and keeps the plain model ID for model details. Stacked routing shortcuts adopt the last one; other model variants remain unchanged. Lookalike text such as `:floorplan` is not rewritten. + +When the settings Save bar is visible, Enter, Ctrl+S and Cmd+S now include the focused field's latest edit before saving, including fields that apply changes when focus leaves them. Previously a keyboard save could retain the earlier value even though the field showed the new text. Ctrl+S and Cmd+S still require a pending settings change before starting a save. + +Full routing support requires translator service v1.3.4. Older or unrecognized service versions receive compatible plain routing choices where possible. Routing can affect provider selection and cost when a model has multiple providers; it does not guarantee a particular price or completion time. + +### An unusable archive does not disable the provider + +A valid catalog-provider archive can contain no subtitle for the requested episode or language. That now rejects the candidate and lets the search continue, including when only one successful subtitle is requested. Automatic archive selection handles filenames covering multiple episodes and relative or absolute episode numbers. It checks the requested season when filenames provide one. Selector callbacks are limited to the offered subtitle files. Corrupt archives, invalid selectors and unsafe archive paths still follow the provider-failure path. + +Provider workers also retain recognized authentication, quota, rate-limit and service errors. These reach Bazarr's matching pause policy instead of all becoming a generic worker error. OpenSubtitles.com download-quota errors use the existing six-hour pause; this change does not claim to know the account's exact quota-reset time. + +--- + +## Other Improvements & Fixes + +- Dropping files into an open movie or series upload dialog no longer opens a second dialog. This also prevents extra dialogs while an archive is being expanded. Existing file selections are retained, and closing once dismisses the upload dialog. +- Disk scans associate saved sync-engine outputs with their actual video. Case differences, Unicode normalization, language tags and sibling videos with dotted names are handled without duplicating or assigning another video's subtitles. Existing language-tagged files remain discoverable after single-language naming is enabled. +- History preserves an explicitly supplied Sonarr or Radarr owner when a media lookup fails, avoiding the reported non-null ownership error. +- Global search shows the owning instance name when multiple Sonarr instances or multiple Radarr instances are configured for that media kind. +- Movie translation jobs now show the movie title, and Gemini receives its overview. Translations started from a media page use the owning Sonarr or Radarr instance for title and overview lookups, so matching upstream IDs on another server do not supply the wrong context. +- Cancelling a Gemini translation no longer retries the request. Cancellation at the initial or final progress update also removes temporary progress files, while the existing destination subtitle is preserved. +- Translator status cards remain visible when idle. Available counts show zero, and unavailable counts keep a placeholder while loading or reconnecting. +- Automatic translation checks eligibility before logging that a source score is too low, including when automatic translation is disabled. +- Translation and subtitle modifications retain an existing file's directory. New files use the configured subtitle folder, including relative and absolute custom folders. +- Four retired OpenRouter model IDs are replaced in saved settings only when one of those known IDs is selected. New installations default to `google/gemini-2.5-flash-lite`; other custom selections are preserved. The model menu and translation guide are updated too. Grok menu suggestions are refreshed separately; saved Grok model IDs are not automatically migrated. +- Startup dependency checks now agree with the versions selected in `requirements.txt`, preventing unnecessary repeated repair-and-restart attempts on otherwise compliant source installations. + +### Provider catalog updates + +Provider bundles have a separate release channel from the application. At preparation time, catalog `beta` includes OpenSubtitles.com 0.1.9 regional-language matching, anonymous OpenSubtitles.org 0.1.12 episode search and Anubis clearance, Titlovi 0.1.8 season-pack eligibility, and SubDL 0.1.4 error classification. Promotion to catalog `main` is still pending. Installing this application version alone does not replace those bundles. SubDL distinguishes documented exhausted-quota responses from other HTTP 429 rate limits and ordinary service failures. + +Anonymous OpenSubtitles.org and OpenSubtitles.com regional-language searches were verified through search, download-link creation and nonempty subtitle streams. Titlovi's season-pack correction passed local tests, but live download verification remains limited by the configured account's disabled API access. SubDL movie search and download passed; its episode control returned zero results with both the prior and updated versions, so episode downloads remain unverified. Current Subsource catalog searches and downloads passed; the reported failures came from the retired built-in implementation. + +--- + +## CI / Docker + +The regression suites run in the standard Python 3.12, 3.13 and 3.14 matrix, including native PostgreSQL checks. Coverage includes upload visibility and synchronization races, settings submission, translation routing, provider error handling, ownership and runtime dependency checks. Release acceptance checks the combined production image, startup, the served UI and release-tour navigation, with provider canaries recorded separately. Frontend builds use Node 24.20.0 LTS, satisfying the installed dependencies' runtime requirements. Existing CI and Docker build workflows read this shared version pin. No Docker base-image change is included. + +--- + +## Dependency Updates + +- Raise the cryptography security floor to 50.0.1 in both installation and runtime checks. Existing encrypted settings and ciphertext compatibility were verified. +- Update the frontend's transitive Browserslist dependency from 4.28.4 to 4.28.7 with its required metadata packages. +- Update GuessIt from 3.8.0 to 4.4.0 for subtitle filename parsing, and align its startup version check. +- Update SQLAlchemy from 2.0.51 to 2.0.52, and align its startup version check. +- Update Mantine from 9.5.2 to 9.6.0, including its react-dropzone 20 upload dependency. +- Align the remaining stale runtime declarations with the dependencies already selected by the installation requirements. Version checks also distinguish prereleases from the required final and post-release versions. + +--- + +## Database Migrations + +No schema changes. There are no new database migrations in this patch. Existing library and history data are preserved. + +--- + +## Included Pull Requests + +- LavX/bazarr#407: Update GuessIt and its runtime version check. +- LavX/bazarr#409: Update Mantine and verify upload compatibility. +- LavX/bazarr#410: Update SQLAlchemy and its runtime version check. +- LavX/bazarr#411: Preserve explicit history ownership when media lookup fails. +- LavX/bazarr#412: Check automatic-translation eligibility before source-score logging. +- LavX/bazarr#413: Show instance names in global search. +- LavX/bazarr#414: Preserve Provider Hub semantic errors across the worker boundary. +- LavX/bazarr#415: Scope sync-engine output discovery to the owning video. +- LavX/bazarr#416: Preserve partial translations, wait for long jobs, and respect subtitle destinations. +- LavX/bazarr#417: Update the Browserslist security dependency. +- LavX/bazarr#418: Continue past unusable provider archive candidates. +- LavX/bazarr#419: Align the cryptography security floor and verify encrypted-setting compatibility. +- LavX/bazarr#420: Align startup dependency declarations and correct version comparisons. +- LavX/bazarr#421: Update the translator service reference to v1.3.1. +- LavX/bazarr#422: Update the translator service reference to v1.3.2. +- LavX/bazarr#423: Update the translator service reference to v1.3.4. +- LavX/bazarr#424: Keep nested subtitle drops in one upload dialog. +- LavX/bazarr#425: Configure OpenRouter provider routing and compatibility with older services. +- LavX/bazarr#426: Normalize routing shortcuts and commit edited settings before keyboard saves. +- LavX/bazarr#427: Name movie translation jobs and scope media context to the owning server. +- LavX/bazarr#428: Show uploaded subtitles immediately and synchronize them in a separate job. +- LavX/bazarr#429: Protect subtitle changes during background sync, stop cancelled Gemini requests, keep translator status cards visible, and prepare this release. + +Catalog changes: [OpenSubtitles.com](https://github.com/LavX/bazarr-provider-catalog/pull/109), [Titlovi](https://github.com/LavX/bazarr-provider-catalog/pull/110), [SubDL](https://github.com/LavX/bazarr-provider-catalog/pull/111), and [OpenSubtitles.org](https://github.com/LavX/bazarr-provider-catalog/pull/112). + +--- + +## Upgrade / Migration Notes + +- Update and restart normally. Back up your configuration and database before upgrading. +- Review subtitles marked partially translated: some cues may remain in the source language. A saved partial result is not a complete translation. +- Update the separately deployed translator service to v1.3.4 for the parsing, retry, usage-accounting and routing changes. Updating Bazarr alone does not replace a running translator service. +- Review Provider Routing in Settings > AI Translator. Fastest remains the default; choosing Cheapest or `:floor` can select different providers. Existing model variants remain valid, and editing a trailing `:nitro` or `:floor` moves that choice into the routing selector. +- If you used one of the retired OpenRouter model IDs, check the replacement in Settings > AI Translator. Other custom model IDs are retained. +- Run Scan Disk to refresh existing sync-output entries after upgrading. Keep all engine outputs still intentionally saves one file per engine, and this fix does not delete files. Unrelated videos with identical basenames remain ambiguous in a shared absolute subtitle folder; use separate subtitle folders for those files. +- Update provider bundles separately through Subtitle Hub when their catalog updates are available. OpenSubtitles.org's scraper supports anonymous downloads; OpenSubtitles.com and Titlovi have their own account requirements. +- SQLite and PostgreSQL remain supported. Native Windows dependency execution was not part of this release's test-machine checks. + +--- + +## Docker + +```bash +docker pull ghcr.io/lavx/bazarr:2.6.2 +docker pull ghcr.io/lavx/bazarr:latest +``` + +After upgrading, confirm that the UI loads and `/api/system/status` reports `2.6.2`. + +--- + +**Full Changelog:** https://github.com/LavX/bazarr/compare/v2.6.1...v2.6.2 diff --git a/frontend/.nvmrc b/frontend/.nvmrc index 6fa8dec4cd..b7179aed06 100644 --- a/frontend/.nvmrc +++ b/frontend/.nvmrc @@ -1 +1 @@ -22.13.0 +24.20.0 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e74d488b8e..a8bf4b3ad9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -13,12 +13,12 @@ "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", "@fontsource-variable/geist": "^5.2.9", - "@mantine/core": "^9.5.2", - "@mantine/dropzone": "^9.5.2", - "@mantine/form": "^9.5.2", - "@mantine/hooks": "^9.5.2", - "@mantine/modals": "^9.5.2", - "@mantine/notifications": "^9.5.2", + "@mantine/core": "^9.6.0", + "@mantine/dropzone": "^9.6.0", + "@mantine/form": "^9.6.0", + "@mantine/hooks": "^9.6.0", + "@mantine/modals": "^9.6.0", + "@mantine/notifications": "^9.6.0", "@tanstack/react-query": "^5.101.0", "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.14.3", @@ -2131,32 +2131,32 @@ } }, "node_modules/@floating-ui/core": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz", - "integrity": "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", "license": "MIT", "dependencies": { - "@floating-ui/utils": "^0.2.11" + "@floating-ui/utils": "^0.2.12" } }, "node_modules/@floating-ui/dom": { - "version": "1.7.6", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz", - "integrity": "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==", + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", "license": "MIT", "dependencies": { - "@floating-ui/core": "^1.7.5", - "@floating-ui/utils": "^0.2.11" + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" } }, "node_modules/@floating-ui/react": { - "version": "0.27.19", - "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.27.19.tgz", - "integrity": "sha512-31B8h5mm8YxotlE7/AU/PhNAl8eWxAmjL/v2QOxroDNkTFLk3Uu82u63N3b6TXa4EGJeeZLVcd/9AlNlVqzeog==", + "version": "0.27.20", + "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.27.20.tgz", + "integrity": "sha512-CMqMy7OaXl9W0eq1Uy7L7i2Y/anPvHmFmESd2CEw0t5YvZhcVCeo4MBevAmswRllX7Y2dEidA4ozGPunLSTQpw==", "license": "MIT", "dependencies": { - "@floating-ui/react-dom": "^2.1.8", - "@floating-ui/utils": "^0.2.11", + "@floating-ui/react-dom": "^2.1.9", + "@floating-ui/utils": "^0.2.12", "tabbable": "^6.0.0" }, "peerDependencies": { @@ -2165,12 +2165,12 @@ } }, "node_modules/@floating-ui/react-dom": { - "version": "2.1.8", - "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz", - "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==", + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", "license": "MIT", "dependencies": { - "@floating-ui/dom": "^1.7.6" + "@floating-ui/dom": "^1.8.0" }, "peerDependencies": { "react": ">=16.8.0", @@ -2178,9 +2178,9 @@ } }, "node_modules/@floating-ui/utils": { - "version": "0.2.11", - "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz", - "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==", + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", "license": "MIT" }, "node_modules/@fontsource-variable/geist": { @@ -3147,42 +3147,45 @@ } }, "node_modules/@mantine/core": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/@mantine/core/-/core-9.5.2.tgz", - "integrity": "sha512-yhnR+XVGmy7S66abvj/tF9SxzNJ+k60w+UZlC/vGOztrU2Cxpbr95gtrBrin4fpWyiE80qzVC2AY0WzaHPufWw==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@mantine/core/-/core-9.6.0.tgz", + "integrity": "sha512-WpdtSv9q2k4RrAVUxBlABMyEul4vv94EkjoVuQglBBC7jVmLhBBgI64HltE6di/l6zvj9D7k1vgkbVjyxQsn1g==", "license": "MIT", "dependencies": { - "@floating-ui/react": "^0.27.19", + "@floating-ui/react": "^0.27.20", "clsx": "^2.1.1", "react-number-format": "^5.4.5", "react-remove-scroll": "^2.7.2", "type-fest": "^5.8.0" }, "peerDependencies": { - "@mantine/hooks": "9.5.2", + "@mantine/hooks": "9.6.0", "react": "^19.2.0", "react-dom": "^19.2.0" } }, "node_modules/@mantine/dropzone": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/@mantine/dropzone/-/dropzone-9.5.2.tgz", - "integrity": "sha512-GKDWcJAMj+SU6QtI0lpvt6Y23atBVrgWAdUYyGSEP5lDomVa/dbarsy180hSdXR0JxeWUNpG9gbcgDxPSs1n0A==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@mantine/dropzone/-/dropzone-9.6.0.tgz", + "integrity": "sha512-XvUx2Mxl2wKMUB1HoRPc+b6h5WDJ1ZPfYt4g6xDzNI4m8mnyKXmPJSgmEJD1RWzNA1CtxTX6rqRaiOUar4hbAw==", "license": "MIT", "dependencies": { - "react-dropzone": "15.0.0" + "react-dropzone": "20.1.1" + }, + "engines": { + "node": ">=22" }, "peerDependencies": { - "@mantine/core": "9.5.2", - "@mantine/hooks": "9.5.2", + "@mantine/core": "9.6.0", + "@mantine/hooks": "9.6.0", "react": "^19.2.0", "react-dom": "^19.2.0" } }, "node_modules/@mantine/form": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/@mantine/form/-/form-9.5.2.tgz", - "integrity": "sha512-9zWyAvEIn5QGfDa7Mxf+iCeEnUHtIam2avY8Csx6PsHqWmRSl9eYfQFrofv6FcR6eAbVUzQHzL/gpqZ3BzPt9g==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@mantine/form/-/form-9.6.0.tgz", + "integrity": "sha512-4TbT/DiixG5uWjgm/HNQlS3Ac0Ov35WVcAW6HQ7Q4+NGuj8mED4gBXHzqVywNi7PhshivdNMox3zADvHIOyUdA==", "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", @@ -3194,46 +3197,46 @@ } }, "node_modules/@mantine/hooks": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/@mantine/hooks/-/hooks-9.5.2.tgz", - "integrity": "sha512-CsANdaF07VRhcvDCupIvAPtIqU1NxYSc+bhFCCasDHpgJOUQxmqeMOGCCGSjtOOEIs+E+pt2ZbbdfnygDYPybA==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@mantine/hooks/-/hooks-9.6.0.tgz", + "integrity": "sha512-r+1RWxc0F2XK9exKIk3Cxc17eyDTEtRqvomeQMCqI4S2Z3A78ObAGuNS8WXn8aqDGN66MJlO+WV7oCgQn2iQfQ==", "license": "MIT", "peerDependencies": { "react": "^19.2.0" } }, "node_modules/@mantine/modals": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/@mantine/modals/-/modals-9.5.2.tgz", - "integrity": "sha512-mB+A9uk6U/1+e9khNCWrblE4cvrYA3H5eKp7iuz1KE472HXWp+A4h50Wb2WcN0FDIEU7+bTHWa167LUA/5jXwA==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@mantine/modals/-/modals-9.6.0.tgz", + "integrity": "sha512-uNNwdYkk5ZqsPKi6zVe+bHq09RCM/HRbzC+UFGXmmOKq2AgOuCoLPNimvVV3y6GQ+jcPFFlkR4D7wbv4eQKPAQ==", "license": "MIT", "peerDependencies": { - "@mantine/core": "9.5.2", - "@mantine/hooks": "9.5.2", + "@mantine/core": "9.6.0", + "@mantine/hooks": "9.6.0", "react": "^19.2.0", "react-dom": "^19.2.0" } }, "node_modules/@mantine/notifications": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/@mantine/notifications/-/notifications-9.5.2.tgz", - "integrity": "sha512-2DQW2i6BlTGI7Vqp1a8IN03Q2CI/nMd7joJu7RGMTDhnnD0rhAhQtu/NwGdmq5gl4vFCpzzwHtH359Do2MlRGg==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@mantine/notifications/-/notifications-9.6.0.tgz", + "integrity": "sha512-1sTE7EaGpr6LX7ykE8DOwpTNxi3U7AHF7Ve/oBd1VEe2k/5RmEYLTWpexxf7WwNpEmHCi99MvG7qUMQJyqc9Fg==", "license": "MIT", "dependencies": { - "@mantine/store": "9.5.2", + "@mantine/store": "9.6.0", "react-transition-group": "4.4.5" }, "peerDependencies": { - "@mantine/core": "9.5.2", - "@mantine/hooks": "9.5.2", + "@mantine/core": "9.6.0", + "@mantine/hooks": "9.6.0", "react": "^19.2.0", "react-dom": "^19.2.0" } }, "node_modules/@mantine/store": { - "version": "9.5.2", - "resolved": "https://registry.npmjs.org/@mantine/store/-/store-9.5.2.tgz", - "integrity": "sha512-vpOS9QwqaJ1KOPr8MdQXYXFDIPkDg1KCr5O8FOGVZIGfsei1vvQsvvmMGGEKGmEE/4FCYwEfhCUWaAVw2psVzw==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@mantine/store/-/store-9.6.0.tgz", + "integrity": "sha512-nYt2tfn1/e6CK35a5TWlPbOL3Xwf7OzBmBElV0l+Ae6mswVaMip1WHm0mCaHAbTKgsOnvL/tooDTVxBBrmm87Q==", "license": "MIT", "peerDependencies": { "react": "^19.2.0" @@ -5768,12 +5771,12 @@ } }, "node_modules/attr-accept": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.5.tgz", - "integrity": "sha512-0bDNnY/u6pPwHDMoF0FieU354oBi0a8rD9FcsLwzcGWbc8KS8KPIi7y+s13OlVY+gMWc/9xEMUgNE6Qm8ZllYQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-4.0.0.tgz", + "integrity": "sha512-hmCnJClmeKNKlsBHgbM8yLZRiQZ4/20UXbLJb6OUT16eWcM5/xNZerr80a/zCYob768KIGq++aLrQNTuwPsIOQ==", "license": "MIT", "engines": { - "node": ">=4" + "node": ">= 22" } }, "node_modules/available-typed-arrays": { @@ -5877,9 +5880,9 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.10.38", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.38.tgz", - "integrity": "sha512-31/02mVB4yuQU6adKk5SlY6m+mxDwUq5KZkyYgnLrrKl7TEm1+3PyDtDBz2kOv/wxZz41GHsvV1A/u6RmiyBvw==", + "version": "2.10.44", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.44.tgz", + "integrity": "sha512-T3ghW+sl/ZJ8w1v/yQx3qvJ9040DWoLBz8JT/CILbAKcFyG9b2MRe75v6W5uXjv6uH1lumK2Kv46y2zSkcej0Q==", "dev": true, "license": "Apache-2.0", "bin": { @@ -5925,9 +5928,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.4.tgz", - "integrity": "sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw==", + "version": "4.28.7", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.7.tgz", + "integrity": "sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==", "dev": true, "funding": [ { @@ -5945,10 +5948,10 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.10.38", - "caniuse-lite": "^1.0.30001799", - "electron-to-chromium": "^1.5.376", - "node-releases": "^2.0.48", + "baseline-browser-mapping": "^2.10.44", + "caniuse-lite": "^1.0.30001806", + "electron-to-chromium": "^1.5.393", + "node-releases": "^2.0.51", "update-browserslist-db": "^1.2.3" }, "bin": { @@ -6035,9 +6038,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001799", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001799.tgz", - "integrity": "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw==", + "version": "1.0.30001806", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz", + "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==", "dev": true, "funding": [ { @@ -6815,9 +6818,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.377", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.377.tgz", - "integrity": "sha512-cH1jZgJHoezfTnKfKwnScpHywTFVnJUNITDPREFdhNjiuD502+QFpG0Qk7G8jhsV/f+CEAFlIrzP1fT+IMb92g==", + "version": "1.5.393", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.393.tgz", + "integrity": "sha512-kiDJdIUawuEIcp9XoICKp1iTYDEbgguIPq526N1Q7jIQDeQ3CqoMx71025PI/7E48Ddtw2HuWsVjY7afEgNxmg==", "dev": true, "license": "ISC" }, @@ -7482,15 +7485,12 @@ } }, "node_modules/file-selector": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-2.1.2.tgz", - "integrity": "sha512-QgXo+mXTe8ljeqUFaX3QVHc5osSItJ/Km+xpocx0aSqWGMSCf6qYs/VnzZgS864Pjn5iceMRFigeAV7AfTlaig==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-5.0.1.tgz", + "integrity": "sha512-v0g/PTeuQgvKCBrVRsfVudvwXlRHSWHEQkVgKawgCGHkEpKA1clp3Om5jvEVhz8G9W/mOYjJH9FhkH4C888PgQ==", "license": "MIT", - "dependencies": { - "tslib": "^2.7.0" - }, "engines": { - "node": ">= 12" + "node": ">= 22" } }, "node_modules/filelist": { @@ -10393,9 +10393,9 @@ "optional": true }, "node_modules/node-releases": { - "version": "2.0.48", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.48.tgz", - "integrity": "sha512-1uz8041X6LoI6ZSdZacM9lVY28vuzDlSKitnpbSNK0RfKoIJkX29NBPVEFXhnuSuEOA9Ww0xnPJ+ILWbGAv8DA==", + "version": "2.0.51", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz", + "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==", "dev": true, "license": "MIT", "engines": { @@ -11095,20 +11095,25 @@ } }, "node_modules/react-dropzone": { - "version": "15.0.0", - "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-15.0.0.tgz", - "integrity": "sha512-lGjYV/EoqEjEWPnmiSvH4v5IoIAwQM2W4Z1C0Q/Pw2xD0eVzKPS359BQTUMum+1fa0kH2nrKjuavmTPOGhpLPg==", + "version": "20.1.1", + "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-20.1.1.tgz", + "integrity": "sha512-2cilRFP8bsjDOHpV0sJ6XY8pzJhmz4/cQ6s9yeckOACWYDR+n4MGGtnJh3Rycq/9SLhDWugqDx1z5mtfmOOZhw==", "license": "MIT", "dependencies": { - "attr-accept": "^2.2.4", - "file-selector": "^2.1.0", - "prop-types": "^15.8.1" + "attr-accept": "^4.0.0", + "file-selector": "^5.0.0" }, "engines": { - "node": ">= 10.13" + "node": ">= 22" }, "peerDependencies": { - "react": ">= 16.8 || 18.0.0" + "@types/react": "*", + "react": ">= 18" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } } }, "node_modules/react-is": { diff --git a/frontend/package.json b/frontend/package.json index 0a25ec18eb..c3e6334bb7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,12 +17,12 @@ "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", "@fontsource-variable/geist": "^5.2.9", - "@mantine/core": "^9.5.2", - "@mantine/dropzone": "^9.5.2", - "@mantine/form": "^9.5.2", - "@mantine/hooks": "^9.5.2", - "@mantine/modals": "^9.5.2", - "@mantine/notifications": "^9.5.2", + "@mantine/core": "^9.6.0", + "@mantine/dropzone": "^9.6.0", + "@mantine/form": "^9.6.0", + "@mantine/hooks": "^9.6.0", + "@mantine/modals": "^9.6.0", + "@mantine/notifications": "^9.6.0", "@tanstack/react-query": "^5.101.0", "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.14.3", diff --git a/frontend/src/components/Search.test.tsx b/frontend/src/components/Search.test.tsx index 27f3d465d2..8fc783fd83 100644 --- a/frontend/src/components/Search.test.tsx +++ b/frontend/src/components/Search.test.tsx @@ -1,17 +1,180 @@ +/* eslint-disable camelcase -- API fixtures use the server's field names. */ + +import { createMemoryRouter, RouterProvider } from "react-router"; +import { MantineProvider } from "@mantine/core"; +import { QueryClientProvider } from "@tanstack/react-query"; +import userEvent from "@testing-library/user-event"; import { http, HttpResponse } from "msw"; -import { describe, it } from "vitest"; -import { Search } from "@/components/index"; -import { customRender } from "@/tests"; +import { describe, expect, it } from "vitest"; +import queryClient from "@/apis/queries"; +import type { ArrInstance, ArrKind } from "@/apis/raw/arrInstances"; +import Search from "@/components/Search"; +import { rawRender, screen, within } from "@/tests"; import server from "@/tests/mocks/node"; +function instance( + id: number, + kind: ArrKind, + name: string, + isDefault: boolean, +): ArrInstance { + return { + id, + kind, + name, + display_name: name, + stable_key: `${kind}-${id}`, + enabled: true, + is_default: isDefault, + ip: "localhost", + port: kind === "sonarr" ? 8989 : 7878, + base_url: "/", + ssl: false, + verify_ssl: true, + http_timeout: 60, + api_key_set: true, + }; +} + +const instances = [ + instance(1, "sonarr", "HD Series", true), + instance(2, "sonarr", "4K Series", false), + instance(3, "radarr", "HD Movies", true), + instance(4, "radarr", "4K Movies", false), +]; + +const results: ItemSearchResult[] = [ + { + id: 101, + sonarrSeriesId: 7, + arr_instance_id: 1, + title: "Café", + year: "2020", + poster: null, + }, + { + id: 102, + sonarrSeriesId: 7, + arr_instance_id: 2, + title: "Café", + year: "2020", + poster: null, + }, + { + id: 201, + radarrId: 7, + arr_instance_id: 3, + title: "Amélie", + year: "2001", + poster: null, + }, + { + id: 202, + radarrId: 7, + arr_instance_id: 4, + title: "Amélie", + year: "2001", + poster: null, + }, +]; + +function renderSearch( + configuredInstances = instances, + searchResults = results, +) { + server.use( + http.get("/api/system/searches", () => HttpResponse.json(searchResults)), + http.get("/api/system/arr-instances", () => + HttpResponse.json(configuredInstances), + ), + ); + + const router = createMemoryRouter([ + { path: "/", element: }, + { path: "/movies/:id", element:

Movie details

}, + { path: "/series/:id", element:

Series details

}, + ]); + + rawRender( + + + + + , + ); + + return { router }; +} + describe("Search Bar", () => { it("should render the closed empty state", () => { - server.use( - http.get("/api/system/searches", () => { - return HttpResponse.json([]); - }), + renderSearch([], []); + + expect(screen.getByPlaceholderText("Search")).toHaveValue(""); + expect(screen.queryByRole("listbox")).not.toBeInTheDocument(); + }); + + it.each([ + { query: "CAFE", owner: "HD Series", path: "/series/101" }, + { query: "CAFE", owner: "4K Series", path: "/series/102" }, + { query: "amelie", owner: "HD Movies", path: "/movies/201" }, + { query: "amelie", owner: "4K Movies", path: "/movies/202" }, + ])( + "selects the duplicate owned by $owner using its local ID", + async ({ query, owner, path }) => { + const user = userEvent.setup(); + const { router } = renderSearch(); + + await user.type(screen.getByPlaceholderText("Search"), query); + + const option = await screen.findByRole("option", { + name: new RegExp(owner), + }); + expect(screen.getAllByRole("option")).toHaveLength(2); + await user.click(option); + + expect(router.state.location.pathname).toBe(path); + }, + ); + + it.each([ + { ids: [1, 3], seriesBadge: false, movieBadge: false }, + { ids: [1, 3, 4], seriesBadge: false, movieBadge: true }, + { ids: [1, 2, 3], seriesBadge: true, movieBadge: false }, + ])( + "only labels kinds with multiple configured instances: $ids", + async ({ ids, seriesBadge, movieBadge }) => { + const user = userEvent.setup(); + renderSearch( + instances.filter((item) => ids.includes(item.id)), + [results[0], results[2]], + ); + + await user.type(screen.getByPlaceholderText("Search"), "a"); + + const series = await screen.findByRole("option", { name: /Café/ }); + const movie = await screen.findByRole("option", { name: /Amélie/ }); + expect(within(series).queryByText("HD Series") !== null).toBe( + seriesBadge, + ); + expect(within(movie).queryByText("HD Movies") !== null).toBe(movieBadge); + }, + ); + + it("filters by title without matching the instance name", async () => { + const user = userEvent.setup(); + renderSearch( + [instances[2], instance(4, "radarr", "Amelie 4K", false)], + [results[2], { ...results[3], title: "Inception", year: "2010" }], ); - customRender(); + await user.type(screen.getByPlaceholderText("Search"), "amelie"); + + expect( + await screen.findByRole("option", { name: /Amélie/ }), + ).toBeInTheDocument(); + expect( + screen.queryByRole("option", { name: /Inception/ }), + ).not.toBeInTheDocument(); }); }); diff --git a/frontend/src/components/Search.tsx b/frontend/src/components/Search.tsx index 4036137fa3..48d2629737 100644 --- a/frontend/src/components/Search.tsx +++ b/frontend/src/components/Search.tsx @@ -7,12 +7,15 @@ import { Image, OptionsFilter, Select, + Stack, Text, } from "@mantine/core"; import { useMediaQuery } from "@mantine/hooks"; import { faSearch } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { useServerSearch } from "@/apis/hooks"; +import { useArrInstanceLabels } from "@/apis/hooks/arrInstances"; +import { InstanceBadge } from "@/components/bazarr"; import { useDebouncedValue } from "@/utilities"; type SearchResultItem = { @@ -21,6 +24,7 @@ type SearchResultItem = { link: string; poster: string | null; type: string; + instanceId?: number; }; function useSearch(query: string) { @@ -64,6 +68,7 @@ function useSearch(query: string) { label: label, type: type, link: link, + instanceId: v.arr_instance_id, }; }) ?? [], [data], @@ -95,6 +100,8 @@ const Search: FunctionComponent = () => { const [query, setQuery] = useState(""); const results = useSearch(query); + const sonarrInstances = useArrInstanceLabels("sonarr"); + const radarrInstances = useArrInstanceLabels("radarr"); const isMobile = useMediaQuery(`(max-width: ${em(750)})`); @@ -120,13 +127,24 @@ const Search: FunctionComponent = () => { }} renderOption={(input) => { const result = results.find((r) => r.value === input.option.value); + const instanceLabels = + result?.type === "show" ? sonarrInstances : radarrInstances; return ( - - {result?.label} - + + + {result?.label} + + {instanceLabels.multiInstance && ( + + )} + ); }} diff --git a/frontend/src/components/TranslatorStatus.tsx b/frontend/src/components/TranslatorStatus.tsx index d350e02c97..b1a77bb790 100644 --- a/frontend/src/components/TranslatorStatus.tsx +++ b/frontend/src/components/TranslatorStatus.tsx @@ -215,7 +215,7 @@ const JobRow: FunctionComponent = ({ job }) => { interface StatCardProps { label: string; - value: number; + value: number | undefined; color: string; } @@ -230,8 +230,13 @@ const StatCard: FunctionComponent = ({ className={classes.statCard} style={{ borderLeftColor: `var(--bz-stat-${color})` }} > - - {value} + + {value ?? "-"} + + + + + + ); + // Show loading state on first load if (statusLoading && !status) { return ( - - - - + + + + + + {queueStats} + ); } @@ -293,26 +327,30 @@ export const TranslatorStatusPanel: FunctionComponent< : "The translation service isn't responding. Check that it's running at the configured URL, then hit retry."; return ( - + + {errorMessage} + + + + {queueStats} + ); } @@ -350,42 +388,7 @@ export const TranslatorStatusPanel: FunctionComponent< {/* Queue Stats */} - {status && - (status.queue.processing > 0 || - status.queue.completed > 0 || - status.queue.failed > 0 || - (status.bazarr_queue?.pending ?? 0) > 0) && ( - - {(status.bazarr_queue?.pending ?? 0) > 0 && ( - - )} - {status.queue.processing > 0 && ( - - )} - {status.queue.completed > 0 && ( - - )} - {status.queue.failed > 0 && ( - - )} - - )} + {queueStats} {/* Jobs Table */} diff --git a/frontend/src/components/forms/MovieUploadForm.tsx b/frontend/src/components/forms/MovieUploadForm.tsx index aeb215e5f0..cebad43259 100644 --- a/frontend/src/components/forms/MovieUploadForm.tsx +++ b/frontend/src/components/forms/MovieUploadForm.tsx @@ -393,6 +393,7 @@ const MovieUploadForm: FunctionComponent = ({ > void addFiles(dropped)} loading={processing} multiple diff --git a/frontend/src/components/forms/SeriesUploadForm.tsx b/frontend/src/components/forms/SeriesUploadForm.tsx index 0c02cd4baa..8de96e68f6 100644 --- a/frontend/src/components/forms/SeriesUploadForm.tsx +++ b/frontend/src/components/forms/SeriesUploadForm.tsx @@ -476,6 +476,7 @@ const SeriesUploadForm: FunctionComponent = ({ > void addFiles(dropped)} loading={processing} multiple diff --git a/frontend/src/components/forms/__tests__/FullPageUploadFlow.test.tsx b/frontend/src/components/forms/__tests__/FullPageUploadFlow.test.tsx new file mode 100644 index 0000000000..076662c76f --- /dev/null +++ b/frontend/src/components/forms/__tests__/FullPageUploadFlow.test.tsx @@ -0,0 +1,395 @@ +/* eslint-disable camelcase */ +import { StrictMode, useRef } from "react"; +import { MantineProvider } from "@mantine/core"; +import { QueryClientProvider } from "@tanstack/react-query"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import queryClient from "@/apis/queries"; +import { QueryKeys } from "@/apis/queries/keys"; +import api from "@/apis/raw"; +import { ArchiveExtractResponse } from "@/apis/raw/subtitles"; +import { MovieUploadModal } from "@/components/forms/MovieUploadForm"; +import { SeriesUploadModal } from "@/components/forms/SeriesUploadForm"; +import { FullPageDropzone } from "@/components/inputs/FullPageDropzone"; +import { ModalsProvider, useModals } from "@/modules/modals"; +import { act, fireEvent, rawRender, screen, waitFor, within } from "@/tests"; + +const baseItem = { + id: 560, + arr_instance_id: 7, + title: "Upload fixture", + path: "/media/Upload fixture", + profileId: 1, + fanart: "", + overview: "", + imdbId: "tt0000001", + alternativeTitles: [], + poster: "", + year: "2026", + monitored: true, + tags: [], + audio_language: [], +}; + +const movie: Item.Movie = { + ...baseItem, + radarrId: 42, + subtitles: [], + missing_subtitles: [], +}; + +const series: Item.Series = { + ...baseItem, + sonarrSeriesId: 42, + episodeFileCount: 4, + episodeMissingCount: 4, + ended: false, + lastAired: "2026-01-01", + seriesType: "Standard", + tvdbId: 1234, +}; + +const episodes: Item.Episode[] = [1, 2, 3, 4].map((episode) => ({ + id: 900 + episode, + series_id: series.id, + sonarrSeriesId: series.sonarrSeriesId, + sonarrEpisodeId: 1900 + episode, + arr_instance_id: series.arr_instance_id, + season: 1, + episode, + title: `Episode ${episode}`, + path: `/media/S01E0${episode}.mkv`, + monitored: true, + subtitles: [], + missing_subtitles: [], + audio_language: [], +})); + +const profile: Language.Profile = { + profileId: 1, + name: "Upload languages", + cutoff: null, + originalFormat: false, + tag: undefined, + mustContain: [], + mustNotContain: [], + items: ["en", "hu"].map((language, index) => ({ + id: index, + language, + forced: "False", + hi: "False", + audio_exclude: "False", + audio_only_include: "False", + translate_from: null, + })), +}; + +type MediaKind = "movie" | "series"; + +function UploadPage({ kind }: { kind: MediaKind }) { + const modals = useModals(); + const openRef = useRef<(() => void) | null>(null); + const onDrop = (files: File[]) => { + if (kind === "movie") { + modals.openContextModal(MovieUploadModal, { movie, files }); + } else { + modals.openContextModal(SeriesUploadModal, { series, files }); + } + }; + return ( +
+ + {modals.modals.length} + +
+ ); +} + +function renderUpload(kind: MediaKind) { + // jsdom has no layout. Mantine's documented test environment prevents its + // floating dropdowns from hiding zero-size references; browser checks cover + // portal placement and visibility with the production provider. + return rawRender( + + + + + + + + + , + ); +} + +function subtitle(episode: number) { + return new File( + ["1\n00:00:01,000 --> 00:00:02,000\nSubtitle fixture\n"], + `S01E0${episode}.srt`, + ); +} + +function dropFiles(target: HTMLElement, files: File[]) { + fireEvent.drop(target, { + dataTransfer: { + files, + items: files.map((file) => ({ + kind: "file", + type: file.type, + getAsFile: () => file, + })), + types: ["Files"], + }, + }); +} + +function fileRow(name: string) { + return screen.getByRole("row", { name: new RegExp(name) }); +} + +beforeEach(() => { + // Detail pages already have these queries loaded before opening the modal. + queryClient.setQueryData( + [QueryKeys.System, QueryKeys.Languages, false], + [ + { code2: "en", code3: "eng", name: "English", enabled: true }, + { code2: "hu", code3: "hun", name: "Hungarian", enabled: true }, + ], + ); + queryClient.setQueryData( + [QueryKeys.System, QueryKeys.LanguagesProfiles], + [profile], + ); + vi.spyOn(api.episodes, "bySeriesId").mockResolvedValue(episodes); + vi.spyOn(api.subtitles, "info").mockImplementation(async (names) => + names.map((filename) => ({ + filename, + season: 1, + episode: Number(/E0(\d)/.exec(filename)?.[1]), + })), + ); + vi.spyOn(api.movies, "uploadSubtitles").mockResolvedValue(undefined); + vi.spyOn(api.episodes, "uploadSubtitles").mockResolvedValue(undefined); +}); + +afterEach(() => vi.restoreAllMocks()); + +const overlayText = + "Drop subtitle files or a .zip / .rar / .7z archive to upload"; + +function dragFiles( + target: HTMLElement, + type: "dragEnter" | "dragLeave", + files: File[], +) { + fireEvent[type](target, { dataTransfer: { files, types: ["Files"] } }); +} + +async function openFromWindow(kind: MediaKind) { + renderUpload(kind); + const page = screen.getByTestId("page-drop-target"); + dragFiles(page, "dragEnter", [subtitle(1)]); + expect(screen.getByText(overlayText)).toBeInTheDocument(); + dropFiles(page, [subtitle(1)]); + const dialog = await screen.findByRole("dialog"); + await waitFor(() => + expect( + within(fileRow("S01E01.srt")).getByDisplayValue("English"), + ).toBeInTheDocument(), + ); + expect(screen.getByTestId("modal-count")).toHaveTextContent("1"); + expect(screen.queryByText(overlayText)).not.toBeInTheDocument(); + return dialog; +} + +// Mantine 9.5 does not expose accessible names for these native controls. +function nativeControl( + scope: HTMLElement, + selector: string, +): T { + // eslint-disable-next-line testing-library/no-node-access + const control = scope.querySelector(selector); + expect(control).not.toBeNull(); + return control!; +} + +async function closeOnce( + user: ReturnType, + dialog: HTMLElement, +) { + await user.click( + nativeControl(dialog, ".mantine-Modal-close"), + ); + await waitFor(() => + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(), + ); + expect(screen.getByTestId("modal-count")).toHaveTextContent("0"); +} + +describe.each(["movie", "series"])( + "%s full-page upload flow", + (kind) => { + it("adds repeated nested drops once, retains metadata and closes once after dragging from outside", async () => { + const user = userEvent.setup(); + const dialog = await openFromWindow(kind); + const initial = fileRow("S01E01.srt"); + await user.click(within(initial).getByDisplayValue("English")); + await user.click( + await screen.findByRole("option", { name: "Hungarian" }), + ); + await user.click(within(initial).getByDisplayValue("Normal")); + await user.click(await screen.findByRole("option", { name: "Forced" })); + if (kind === "series") { + await user.click(await screen.findByDisplayValue("(1x1) Episode 1")); + await user.click( + await screen.findByRole("option", { name: "(1x2) Episode 2" }), + ); + } + const dropTarget = within(dialog).getByText( + /Attach as many files as you like/, + ); + const page = screen.getByTestId("page-drop-target"); + dragFiles(page, "dragEnter", [subtitle(2)]); + expect(screen.getByText(overlayText)).toBeInTheDocument(); + dragFiles(dropTarget, "dragEnter", [subtitle(2)]); + dragFiles(page, "dragLeave", [subtitle(2)]); + dropFiles(dropTarget, [subtitle(2), subtitle(3)]); + await waitFor(() => expect(fileRow("S01E03.srt")).toBeInTheDocument()); + expect(screen.queryByText(overlayText)).not.toBeInTheDocument(); + expect(screen.getByTestId("modal-count")).toHaveTextContent("1"); + dropFiles(dropTarget, [subtitle(4)]); + await waitFor(() => expect(fileRow("S01E04.srt")).toBeInTheDocument()); + expect(within(dialog).getAllByRole("row")).toHaveLength(5); + expect( + within(fileRow("S01E01.srt")).getByDisplayValue("Hungarian"), + ).toBeInTheDocument(); + expect( + within(fileRow("S01E01.srt")).getByDisplayValue("Forced"), + ).toBeInTheDocument(); + if (kind === "series") { + expect( + within(fileRow("S01E01.srt")).getByDisplayValue("(1x2) Episode 2"), + ).toBeInTheDocument(); + } + expect(api.movies.uploadSubtitles).not.toHaveBeenCalled(); + expect(api.episodes.uploadSubtitles).not.toHaveBeenCalled(); + await closeOnce(user, dialog); + dropFiles(page, [subtitle(2)]); + const reopened = await screen.findByRole("dialog"); + await waitFor(() => expect(fileRow("S01E02.srt")).toBeInTheDocument()); + expect(within(reopened).getAllByRole("row")).toHaveLength(2); + await closeOnce(user, reopened); + }); + + it("preserves the toolbar picker and a nested picker batch without adding a second modal", async () => { + const user = userEvent.setup(); + renderUpload(kind); + const input = nativeControl( + screen.getByTestId("page-drop-target"), + "input[type=file]", + ); + const clicked = vi.fn(); + input.addEventListener("click", clicked); + await user.click( + screen.getByRole("button", { name: "Upload subtitles" }), + ); + expect(clicked).toHaveBeenCalledOnce(); + await user.upload(input, [subtitle(1)]); + const dialog = await screen.findByRole("dialog"); + await waitFor(() => expect(fileRow("S01E01.srt")).toBeInTheDocument()); + const nestedInput = nativeControl( + dialog, + "input[type=file]", + ); + await user.upload(nestedInput, [subtitle(2), subtitle(3)]); + await waitFor(() => expect(fileRow("S01E03.srt")).toBeInTheDocument()); + expect(within(dialog).getAllByRole("row")).toHaveLength(4); + expect(screen.getByTestId("modal-count")).toHaveTextContent("1"); + await closeOnce(user, dialog); + }); + + it("does not open another modal when a nested drop occurs during archive extraction", async () => { + const user = userEvent.setup(); + let finish: ((value: ArchiveExtractResponse) => void) | undefined; + vi.spyOn(api.subtitles, "extractArchive").mockImplementation( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const dialog = await openFromWindow(kind); + const dropTarget = within(dialog).getByText( + /Attach as many files as you like/, + ); + const archive = new File(["synthetic archive"], "subtitles.zip"); + // The picker starts extraction without relying on the nested drag fix. + await user.upload( + nativeControl(dialog, "input[type=file]"), + [archive], + ); + await waitFor(() => + expect(api.subtitles.extractArchive).toHaveBeenCalledOnce(), + ); + expect( + within(dialog).getByRole("button", { name: /^Upload$/ }), + ).toBeDisabled(); + const loadingOverlay = nativeControl( + dialog, + ".mantine-LoadingOverlay-root", + ); + dragFiles(loadingOverlay, "dragEnter", [subtitle(4)]); + expect(screen.getByText(overlayText)).toBeInTheDocument(); + dropFiles(loadingOverlay, [subtitle(4)]); + await act(async () => { + await Promise.resolve(); + }); + expect(screen.queryByText(overlayText)).not.toBeInTheDocument(); + expect(screen.queryByText("S01E04.srt")).not.toBeInTheDocument(); + expect(screen.getByTestId("modal-count")).toHaveTextContent("1"); + await act(async () => + finish!({ + files: [{ name: "S01E02.srt", content: btoa("Subtitle fixture") }], + count: 1, + }), + ); + await waitFor(() => expect(fileRow("S01E02.srt")).toBeInTheDocument()); + expect( + within(dialog).getByRole("button", { name: /^Upload$/ }), + ).toBeEnabled(); + expect(screen.queryByText(overlayText)).not.toBeInTheDocument(); + dropFiles(dropTarget, [subtitle(3)]); + await waitFor(() => expect(fileRow("S01E03.srt")).toBeInTheDocument()); + expect(within(dialog).getAllByRole("row")).toHaveLength(4); + await closeOnce(user, dialog); + }); + + it("still dispatches an outside drop once when the document prevents its browser default", async () => { + const user = userEvent.setup(); + await openFromWindow(kind); + let defaultPreventedAtDocument = false; + const observe = (event: Event) => { + event.preventDefault(); + defaultPreventedAtDocument = event.defaultPrevented; + }; + document.addEventListener("drop", observe); + try { + // Preventing document navigation does not claim this outside upload. + dropFiles(screen.getByTestId("page-drop-target"), [subtitle(2)]); + expect(defaultPreventedAtDocument).toBe(true); + expect(screen.getByTestId("modal-count")).toHaveTextContent("2"); + await user.click( + nativeControl( + screen.getByRole("dialog"), + ".mantine-Modal-close", + ), + ); + await waitFor(() => + expect(screen.getByTestId("modal-count")).toHaveTextContent("1"), + ); + await closeOnce(user, screen.getByRole("dialog")); + } finally { + document.removeEventListener("drop", observe); + } + }); + }, +); diff --git a/frontend/src/components/forms/__tests__/UploadFormDropzone.test.tsx b/frontend/src/components/forms/__tests__/UploadFormDropzone.test.tsx new file mode 100644 index 0000000000..0f7df41087 --- /dev/null +++ b/frontend/src/components/forms/__tests__/UploadFormDropzone.test.tsx @@ -0,0 +1,343 @@ +/* eslint-disable camelcase */ +import { StrictMode } from "react"; +import { MantineProvider } from "@mantine/core"; +import { QueryClientProvider } from "@tanstack/react-query"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import queryClient from "@/apis/queries"; +import { QueryKeys } from "@/apis/queries/keys"; +import api from "@/apis/raw"; +import { ArchiveExtractResponse } from "@/apis/raw/subtitles"; +import { MovieUploadModal } from "@/components/forms/MovieUploadForm"; +import { SeriesUploadModal } from "@/components/forms/SeriesUploadForm"; +import { ModalsProvider, useModals } from "@/modules/modals"; +import { fireEvent, rawRender, screen, waitFor, within } from "@/tests"; + +const baseItem = { + id: 560, + arr_instance_id: 7, + title: "Upload fixture", + path: "/media/Upload fixture", + profileId: 1, + fanart: "", + overview: "", + imdbId: "tt0000001", + alternativeTitles: [], + poster: "", + year: "2026", + monitored: true, + tags: [], + audio_language: [], +}; + +const movie: Item.Movie = { + ...baseItem, + radarrId: 42, + subtitles: [], + missing_subtitles: [], +}; + +const series: Item.Series = { + ...baseItem, + sonarrSeriesId: 42, + episodeFileCount: 4, + episodeMissingCount: 4, + ended: false, + lastAired: "2026-01-01", + seriesType: "Standard", + tvdbId: 1234, +}; + +const episodes: Item.Episode[] = [1, 2, 3, 4].map((episode) => ({ + id: 900 + episode, + series_id: series.id, + sonarrSeriesId: series.sonarrSeriesId, + sonarrEpisodeId: 1900 + episode, + arr_instance_id: series.arr_instance_id, + season: 1, + episode, + title: `Episode ${episode}`, + path: `/media/S01E0${episode}.mkv`, + monitored: true, + subtitles: [], + missing_subtitles: [], + audio_language: [], +})); + +const profile: Language.Profile = { + profileId: 1, + name: "Upload languages", + cutoff: null, + originalFormat: false, + tag: undefined, + mustContain: [], + mustNotContain: [], + items: ["en", "hu"].map((language, index) => ({ + id: index, + language, + forced: "False", + hi: "False", + audio_exclude: "False", + audio_only_include: "False", + translate_from: null, + })), +}; + +type MediaKind = "movie" | "series"; + +function OpenUpload({ kind, file }: { kind: MediaKind; file: File }) { + const modals = useModals(); + return ( + + ); +} + +function renderUpload(kind: MediaKind, file: File) { + // jsdom has no layout. Mantine's documented test environment prevents its + // floating dropdowns from hiding zero-size references; browser checks cover + // portal placement and visibility with the production provider. + return rawRender( + + + + + + + + + , + ); +} + +function subtitle(episode: number) { + return new File( + ["1\n00:00:01,000 --> 00:00:02,000\nSubtitle fixture\n"], + `S01E0${episode}.srt`, + ); +} + +function dropFiles(target: HTMLElement, files: File[]) { + fireEvent.drop(target, { + dataTransfer: { + files, + items: files.map((file) => ({ + kind: "file", + type: file.type, + getAsFile: () => file, + })), + types: ["Files"], + }, + }); +} + +function fileRow(name: string) { + return screen.getByRole("row", { name: new RegExp(name) }); +} + +beforeEach(() => { + // Detail pages already have these queries loaded before opening the modal. + queryClient.setQueryData( + [QueryKeys.System, QueryKeys.Languages, false], + [ + { code2: "en", code3: "eng", name: "English", enabled: true }, + { code2: "hu", code3: "hun", name: "Hungarian", enabled: true }, + ], + ); + queryClient.setQueryData( + [QueryKeys.System, QueryKeys.LanguagesProfiles], + [profile], + ); + vi.spyOn(api.episodes, "bySeriesId").mockResolvedValue(episodes); + vi.spyOn(api.subtitles, "info").mockImplementation(async (names) => + names.map((filename) => ({ + filename, + season: 1, + episode: Number(/E0(\d)/.exec(filename)?.[1]), + })), + ); + vi.spyOn(api.movies, "uploadSubtitles").mockResolvedValue(undefined); + vi.spyOn(api.episodes, "uploadSubtitles").mockResolvedValue(undefined); +}); + +afterEach(() => vi.restoreAllMocks()); + +describe.each(["movie", "series"])("%s upload Dropzone", (kind) => { + it.each(["picker", "drop"])( + "adds an entire %s batch without changing existing metadata", + async (method) => { + const user = userEvent.setup(); + const initial = subtitle(1); + renderUpload(kind, initial); + await user.click(screen.getByRole("button", { name: "Open upload" })); + const dialog = await screen.findByRole("dialog"); + await waitFor(() => { + expect( + within(fileRow(initial.name)).getByDisplayValue("English"), + ).toBeInTheDocument(); + }); + await user.click( + within(fileRow(initial.name)).getByDisplayValue("English"), + ); + await user.click( + await screen.findByRole("option", { name: "Hungarian" }), + ); + await user.click( + within(fileRow(initial.name)).getByDisplayValue("Normal"), + ); + await user.click(await screen.findByRole("option", { name: "Forced" })); + if (kind === "series") { + await user.click(await screen.findByDisplayValue("(1x1) Episode 1")); + await user.click( + await screen.findByRole("option", { name: "(1x2) Episode 2" }), + ); + } + + const incoming = [subtitle(2), subtitle(3), subtitle(4)]; + Object.defineProperty(incoming[0], "webkitRelativePath", { + value: `subtitles/${incoming[0].name}`, + }); + if (method === "picker") { + await user.upload( + within(dialog).getByLabelText("file upload"), + incoming, + ); + } else { + dropFiles( + within(dialog).getByText(/Attach as many files as you like/), + incoming, + ); + } + + for (const file of incoming) { + await waitFor(() => { + expect( + within(fileRow(file.name)).getByDisplayValue("English"), + ).toBeInTheDocument(); + expect( + within(fileRow(file.name)).getByDisplayValue("Normal"), + ).toBeInTheDocument(); + }); + } + expect(within(dialog).getAllByRole("row")).toHaveLength(5); + expect( + within(fileRow(initial.name)).getByDisplayValue("Hungarian"), + ).toBeInTheDocument(); + expect( + within(fileRow(initial.name)).getByDisplayValue("Forced"), + ).toBeInTheDocument(); + if (kind === "series") { + await waitFor(() => { + expect( + within(fileRow(initial.name)).getByDisplayValue("(1x2) Episode 2"), + ).toBeInTheDocument(); + expect( + within(fileRow(incoming[2].name)).getByDisplayValue( + "(1x4) Episode 4", + ), + ).toBeInTheDocument(); + }); + } + expect(api.movies.uploadSubtitles).not.toHaveBeenCalled(); + expect(api.episodes.uploadSubtitles).not.toHaveBeenCalled(); + + await user.click( + within(dialog).getByRole("button", { name: /^Upload$/ }), + ); + const upload = + kind === "movie" + ? api.movies.uploadSubtitles + : api.episodes.uploadSubtitles; + await waitFor(() => expect(upload).toHaveBeenCalledTimes(4)); + const form = { file: initial, language: "hu", forced: true, hi: false }; + if (kind === "movie") { + expect(api.movies.uploadSubtitles).toHaveBeenCalledWith( + movie.radarrId, + form, + 7, + ); + } else { + expect(api.episodes.uploadSubtitles).toHaveBeenCalledWith( + series.sonarrSeriesId, + 1902, + form, + 7, + ); + } + incoming.forEach((file, index) => { + const addedForm = { file, language: "en", forced: false, hi: false }; + if (kind === "movie") { + expect(api.movies.uploadSubtitles).toHaveBeenCalledWith( + movie.radarrId, + addedForm, + 7, + ); + } else { + expect(api.episodes.uploadSubtitles).toHaveBeenCalledWith( + series.sonarrSeriesId, + 1902 + index, + addedForm, + 7, + ); + } + }); + await waitFor(() => + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(), + ); + }, + ); + + it("keeps a mixed archive batch pending until extracted rows are available", async () => { + const user = userEvent.setup(); + let finishExtraction: (result: ArchiveExtractResponse) => void = () => { + throw new Error("Archive request has not started"); + }; + vi.spyOn(api.subtitles, "extractArchive").mockImplementation( + () => + new Promise((resolve) => { + finishExtraction = resolve; + }), + ); + renderUpload(kind, subtitle(1)); + await user.click(screen.getByRole("button", { name: "Open upload" })); + const dialog = await screen.findByRole("dialog"); + const archive = new File(["synthetic archive"], "subtitles.ZIP"); + dropFiles(within(dialog).getByText(/Attach as many files as you like/), [ + subtitle(2), + archive, + ]); + await waitFor(() => + expect(api.subtitles.extractArchive).toHaveBeenCalledWith(archive), + ); + expect( + within(dialog).getByRole("button", { name: /^Upload$/ }), + ).toBeDisabled(); + finishExtraction({ + files: [{ name: "S01E03.srt", content: btoa("Subtitle fixture") }], + count: 1, + }); + await waitFor(() => { + expect(fileRow("S01E02.srt")).toBeInTheDocument(); + expect(fileRow("S01E03.srt")).toBeInTheDocument(); + expect( + within(dialog).getByRole("button", { name: /^Upload$/ }), + ).toBeEnabled(); + }); + expect(within(dialog).getAllByRole("row")).toHaveLength(4); + expect(screen.queryByText(archive.name)).not.toBeInTheDocument(); + expect(api.movies.uploadSubtitles).not.toHaveBeenCalled(); + expect(api.episodes.uploadSubtitles).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/components/inputs/FullPageDropzone.tsx b/frontend/src/components/inputs/FullPageDropzone.tsx index 9edb7ccefe..0cfea8cd49 100644 --- a/frontend/src/components/inputs/FullPageDropzone.tsx +++ b/frontend/src/components/inputs/FullPageDropzone.tsx @@ -97,6 +97,19 @@ export const FullPageDropzone: FunctionComponent = ({ event.preventDefault(); depth.current = 0; setVisible(false); + // Nested upload areas own their files even while disabled or loading. + // Observe their drop completion to clear the page overlay first. + if ( + event + .composedPath() + .some( + (target) => + target instanceof Element && + target.hasAttribute("data-full-page-dropzone-ignore"), + ) + ) { + return; + } const files = Array.from(event.dataTransfer?.files ?? []); if (files.length > 0) { onDrop(files); diff --git a/frontend/src/data/whatsNew.ts b/frontend/src/data/whatsNew.ts index 7e0cd96871..6579df673c 100644 --- a/frontend/src/data/whatsNew.ts +++ b/frontend/src/data/whatsNew.ts @@ -32,7 +32,7 @@ export interface WhatsNewSlide { * cutting a release. Kept as an explicit token so the wizard never has to parse the * fork's `version + YYMMDD` runtime string. */ -export const latestWhatsNewVersion = "2.6.1"; +export const latestWhatsNewVersion = "2.6.2"; // v2.6.0 feature slides; v2.6.1 (a patch on the same line) leads with its // fix and keeps the whole Clockwork tour behind it. @@ -87,6 +87,116 @@ const clockworkSlides: WhatsNewSlide[] = [ ]; export const whatsNew: Record = { + "2.6.2": [ + { + title: "Uploaded subtitles appear before sync finishes", + body: "A saved upload is listed immediately while automatic sync runs as a separate job. Failed or cancelled sync leaves the uploaded subtitle available, and older sync work cannot replace a newer upload.", + icon: faDownload, + cta: { label: "Open your series", to: "/series" }, + }, + { + title: "One upload dialog for each batch", + body: "Dropping files into an open upload dialog keeps the existing selection without opening another dialog, including while an archive is expanding. Closing once dismisses it.", + icon: faFileZipper, + cta: { label: "Open your movies", to: "/movies" }, + }, + { + title: "Choose how OpenRouter routes your translations", + body: "Pick fastest, cheapest, lowest latency or OpenRouter's default. Typed :nitro and :floor shortcuts move into the routing selector when editing finishes, keeping model details and the saved choice consistent. Full shortcut support needs translator service 1.3.4.", + icon: faSliders, + cta: { label: "Open Translator settings", to: "/settings/translator" }, + }, + { + title: "Keyboard saves keep your latest settings edit", + body: "When the Save bar is visible, Enter and Ctrl+S or Cmd+S now include the latest text in the focused field, including fields that finish editing when focus moves away.", + icon: faSliders, + cta: { label: "Open Translator settings", to: "/settings/translator" }, + }, + { + title: "History keeps the right server", + body: "Downloads keep their Sonarr or Radarr owner even if the media lookup fails, so a missing lookup no longer prevents the history entry from being saved.", + icon: faClockRotateLeft, + cta: { label: "Open History", to: "/history/series" }, + }, + { + title: "Disk scans keep sync outputs with their video", + body: "Keep all still stores one file per sync engine. Disk scans now keep those outputs with the owning video, including existing language-tagged files after a single-language naming change.", + icon: faSliders, + cta: { label: "Open Subtitles settings", to: "/settings/subtitles" }, + }, + { + title: "Movie translation jobs show the movie title", + body: "Movie translations now show the movie's name in the job list, and Gemini receives its overview. Starting a translation from a media page also keeps title and overview lookups with that media's Sonarr or Radarr server.", + icon: faWandMagicSparkles, + cta: { label: "Open your movies", to: "/movies" }, + }, + { + title: "Recover useful OpenRouter translation results", + body: "Usable partial OpenRouter translations are saved and marked incomplete. Missing translations retain their source text, so review incomplete files. Empty or invalid results still fail, and failed saves preserve the previous subtitle.", + icon: faWandMagicSparkles, + cta: { label: "Open Translator settings", to: "/settings/translator" }, + }, + { + title: "Long OpenRouter translation jobs can finish", + body: "OpenRouter translation jobs can keep running for up to 12 hours. Ten minutes without a successful status response still stops the wait.", + icon: faClockRotateLeft, + cta: { label: "Open Translator settings", to: "/settings/translator" }, + }, + { + title: "Translation and edits respect subtitle folders", + body: "Existing subtitles stay in their current location. New translated or modified subtitles use your configured subtitle folder, including relative and absolute custom folders.", + icon: faFileZipper, + cta: { label: "Open Subtitles settings", to: "/settings/subtitles" }, + }, + { + title: "Cancelled Gemini jobs stop retrying", + body: "Cancelling a Gemini translation no longer retries the request. The existing subtitle stays in place, and temporary progress files are cleaned up.", + icon: faWandMagicSparkles, + cta: { label: "Open Translator settings", to: "/settings/translator" }, + }, + { + title: "Updated translation model choices", + body: "Four known retired Gemini IDs in OpenRouter settings migrate to replacements. Other saved IDs stay unchanged. Removed Grok menu suggestions are not migrated automatically.", + icon: faWandMagicSparkles, + cta: { label: "Open Translator settings", to: "/settings/translator" }, + }, + { + title: "One unusable archive no longer stops the search", + body: "If a catalog-provider archive has no suitable subtitle, Bazarr tries the next candidate without disabling that provider. The search also continues when only one successful subtitle is requested.", + icon: faDownload, + cta: { label: "Open Subtitle Hub", to: "/subtitle-hub" }, + }, + { + title: "Provider errors keep their meaning", + body: "Provider Hub now preserves quota, login, rate-limit and service errors across its workers, so Bazarr applies the matching pause instead of treating them all as a generic failure.", + icon: faStore, + cta: { label: "Open Subtitle Hub", to: "/subtitle-hub" }, + }, + { + title: "See which server owns a search result", + body: "When you have multiple Sonarr or multiple Radarr instances, global search shows the owning instance beside that media type\u2019s results.", + icon: faServer, + cta: { label: "Open your series", to: "/series" }, + }, + { + title: "Fewer misleading translation messages", + body: "Automatic translation checks whether it is enabled and eligible before reporting that a source score is too low.", + icon: faWandMagicSparkles, + cta: { label: "Open Translator settings", to: "/settings/translator" }, + }, + { + title: "Dependency checks match installed versions", + body: "Source installs no longer request repeated repairs for dependencies that already satisfy the installation requirements. This patch also updates filename parsing, database access, shared UI components and security dependencies.", + icon: faShieldHalved, + cta: { label: "Open System Status", to: "/system/status" }, + }, + { + title: "Provider fixes are delivered separately", + body: "Reviewed OpenSubtitles.org, OpenSubtitles.com, Titlovi and SubDL updates are in catalog beta. Stable availability follows promotion to main; update the bundles in Subtitle Hub when available.", + icon: faStore, + cta: { label: "Open Subtitle Hub", to: "/subtitle-hub" }, + }, + ], "2.6.1": [ { title: "History shows your events again", diff --git a/frontend/src/pages/Settings/Translator/AIModelSelector.test.tsx b/frontend/src/pages/Settings/Translator/AIModelSelector.test.tsx new file mode 100644 index 0000000000..b996c9527f --- /dev/null +++ b/frontend/src/pages/Settings/Translator/AIModelSelector.test.tsx @@ -0,0 +1,234 @@ +import { FunctionComponent } from "react"; +import { useForm } from "@mantine/form"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; +import { + FormContext, + type FormValues, + useFormActions, +} from "@/pages/Settings/utilities/FormValues"; +import { SettingsProvider } from "@/pages/Settings/utilities/SettingsProvider"; +import { customRender, screen, waitFor } from "@/tests"; +import AIModelSelector from "./AIModelSelector"; + +const MODEL_KEY = "settings-translator-openrouter_model"; +const ROUTING_KEY = "settings-translator-openrouter_provider_routing"; + +// Renders the selector with a real settings form behind it and exposes the +// staged values so a test can assert what the form would submit. +// Stands in for the Provider Routing selector, which lives elsewhere on the +// page: clicking it changes the routing setting the way that selector would. +function RoutingSetter() { + const { setValue } = useFormActions(); + + return ( + + ); +} + +const Harness: FunctionComponent<{ onValues: (v: LooseObject) => void }> = ({ + onValues, +}) => { + const mantineForm = useForm({ + initialValues: { settings: {}, hooks: {} }, + }); + onValues(mantineForm.values.settings); + return ( + + + + + + + ); +}; + +// Returns the live staged settings the form would submit, refreshed on render. +function mountSelector() { + const stagedValues: LooseObject = {}; + customRender( + { + Object.keys(stagedValues).forEach((k) => delete stagedValues[k]); + Object.assign(stagedValues, v); + }} + />, + ); + return stagedValues; +} + +describe("AIModelSelector routing adoption", () => { + it("does not adopt half a word while the user is still typing", async () => { + // ":floor" is an exact suffix in the middle of typing ":floorplan", so + // adopting on every keystroke would eat the model id. + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.type(input, "some/model:floorplan"); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("some/model:floorplan"); + }); + expect(stagedValues[ROUTING_KEY]).toBeUndefined(); + }); + + it("keeps a typo intact rather than adopting the prefix", async () => { + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.type(input, "some/model:floorr"); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("some/model:floorr"); + }); + expect(stagedValues[ROUTING_KEY]).toBeUndefined(); + }); + + it("adopts a suffix typed one character at a time once the field is left", async () => { + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.type(input, "z-ai/glm-5.3-flash:nitro"); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("z-ai/glm-5.3-flash"); + expect(stagedValues[ROUTING_KEY]).toBe("nitro"); + }); + }); + + it("moves a typed :floor into the provider routing setting", async () => { + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.paste("z-ai/glm-5.3-flash:floor"); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("z-ai/glm-5.3-flash"); + expect(stagedValues[ROUTING_KEY]).toBe("floor"); + }); + }); + + it("moves a typed :nitro into the provider routing setting", async () => { + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.paste("deepseek/deepseek-v4-flash:nitro"); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("deepseek/deepseek-v4-flash"); + expect(stagedValues[ROUTING_KEY]).toBe("nitro"); + }); + }); + + it("leaves another variant on the model and does not touch the routing", async () => { + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.paste("deepseek/deepseek-chat:thinking"); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("deepseek/deepseek-chat:thinking"); + }); + expect(stagedValues[ROUTING_KEY]).toBeUndefined(); + }); + + it("trims a pasted model id even when it carries no routing suffix", async () => { + // The details lookup trims, so an untrimmed id would show a valid card + // while the saved setting, and the request built from it, keep the spaces. + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.paste(" z-ai/glm-5.3-flash "); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("z-ai/glm-5.3-flash"); + }); + expect(stagedValues[ROUTING_KEY]).toBeUndefined(); + }); + + it("takes every stacked routing suffix off the model id", async () => { + const user = userEvent.setup(); + const stagedValues = mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.paste("z-ai/glm-5.3-flash:nitro:floor"); + await user.tab(); + + await waitFor(() => { + expect(stagedValues[MODEL_KEY]).toBe("z-ai/glm-5.3-flash"); + expect(stagedValues[ROUTING_KEY]).toBe("floor"); + }); + }); + + it("drops the notice once the routing is changed by hand", async () => { + const user = userEvent.setup(); + mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.paste("z-ai/glm-5.3-flash:floor"); + await user.tab(); + expect( + await screen.findByText(/moved .*:floor.* into provider routing/i), + ).toBeInTheDocument(); + + // The routing selector lives elsewhere on the page; changing the setting is + // what the notice has to react to. + await user.click( + screen.getByRole("button", { name: "Set routing to throughput" }), + ); + + await waitFor(() => { + expect( + screen.queryByText(/moved .*into provider routing/i), + ).not.toBeInTheDocument(); + }); + }); + + it("tells the user the routing was adopted", async () => { + const user = userEvent.setup(); + mountSelector(); + + const input = screen.getByRole("combobox"); + await user.clear(input); + await user.paste("z-ai/glm-5.3-flash:floor"); + await user.tab(); + + expect( + await screen.findByText(/moved .*:floor.* into provider routing/i), + ).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/pages/Settings/Translator/AIModelSelector.tsx b/frontend/src/pages/Settings/Translator/AIModelSelector.tsx index 5720e82a8e..73ba6d8375 100644 --- a/frontend/src/pages/Settings/Translator/AIModelSelector.tsx +++ b/frontend/src/pages/Settings/Translator/AIModelSelector.tsx @@ -1,24 +1,77 @@ -import { FunctionComponent } from "react"; -import { Autocomplete } from "@mantine/core"; -import { useBaseInput } from "@/pages/Settings/utilities/hooks"; +import { FunctionComponent, useCallback, useState } from "react"; +import { Autocomplete, Text as MantineText } from "@mantine/core"; +import { useFormActions } from "@/pages/Settings/utilities/FormValues"; +import { + useBaseInput, + useSettingValue, +} from "@/pages/Settings/utilities/hooks"; import { aiTranslatorModelOptions } from "./options"; +import { routingLabel, splitRoutingSuffix } from "./routing"; const modelData = aiTranslatorModelOptions.map((o) => o.value); +const ROUTING_KEY = "settings-translator-openrouter_provider_routing"; + const AIModelSelector: FunctionComponent = () => { const { value, update } = useBaseInput<{ settingKey: string }, string>({ settingKey: "settings-translator-openrouter_model", }); + const { setValue } = useFormActions(); + const [adopted, setAdopted] = useState(null); + // The selector that owns the routing lives elsewhere on the page, so the + // notice below follows the setting rather than asserting what it once was. + const routing = useSettingValue(ROUTING_KEY); + + // OpenRouter's :nitro and :floor are routing shortcuts rather than models, so + // a model id carrying one is split: the model keeps its own id, and the + // routing moves to the selector that owns it. Leaving both in place would + // send a slug and a sort that disagree, and would stop the model details + // lookup from resolving the id. + // + // This waits for the field to be left rather than running on each keystroke. + // Mid-word the text can be an exact suffix of something longer, so adopting + // as the user types would turn "some/model:floorplan" into "some/modelplan". + const adopt = useCallback(() => { + const raw = value ?? ""; + const { modelId, routing } = splitRoutingSuffix(raw); + // The details lookup works on the normalised id, so the setting has to hold + // the same thing. Otherwise a padded id shows a valid model card while the + // translation request goes out with the padding still on the slug. + if (modelId !== raw) { + update(modelId); + } + if (routing) { + setValue(routing, ROUTING_KEY); + setAdopted(routing); + } + }, [value, update, setValue]); + + const onChange = useCallback( + (raw: string) => { + update(raw); + setAdopted(null); + }, + [update], + ); return ( - update(val)} - placeholder="Select or type any model ID..." - limit={30} - /> + <> + + {adopted !== null && adopted === routing && ( + + Moved :{adopted} into Provider Routing, now set to{" "} + {routingLabel(adopted)}. + + )} + ); }; diff --git a/frontend/src/pages/Settings/Translator/ModelDetails.tsx b/frontend/src/pages/Settings/Translator/ModelDetails.tsx index fb5224addb..a5bae25cfa 100644 --- a/frontend/src/pages/Settings/Translator/ModelDetails.tsx +++ b/frontend/src/pages/Settings/Translator/ModelDetails.tsx @@ -1,6 +1,7 @@ import { FunctionComponent } from "react"; import { Badge, Box, Group, SimpleGrid, Text } from "@mantine/core"; import { useQuery } from "@tanstack/react-query"; +import { splitRoutingSuffix } from "./routing"; // Average total token usage per translation (calibrated from real data) // Episode ~90K tokens, Movie ~180K tokens @@ -44,7 +45,11 @@ function useOpenRouterModelDetails(modelId: string) { queryFn: async () => { const response = await fetch("https://openrouter.ai/api/v1/models"); const data = await response.json(); - const found = data.data?.find((m: OpenRouterModel) => m.id === modelId); + // :nitro and :floor are routing shortcuts rather than models, so they + // never appear in OpenRouter's model list. A saved config can still carry + // one, and the details belong to the model underneath it. + const { modelId: lookupId } = splitRoutingSuffix(modelId); + const found = data.data?.find((m: OpenRouterModel) => m.id === lookupId); return (found as OpenRouterModel) || null; }, enabled: !!modelId, diff --git a/frontend/src/pages/Settings/Translator/Translator.test.tsx b/frontend/src/pages/Settings/Translator/Translator.test.tsx index 776a098662..f83ea7bfc5 100644 --- a/frontend/src/pages/Settings/Translator/Translator.test.tsx +++ b/frontend/src/pages/Settings/Translator/Translator.test.tsx @@ -5,6 +5,7 @@ import { aiTranslatorReasoningOptions, translatorOption, } from "./options"; +import { aiTranslatorProviderRoutingOptions } from "./routing"; describe("Translator options", () => { it("exports all required option arrays", () => { @@ -45,4 +46,23 @@ describe("Translator options", () => { expect(values).toContain("disabled"); expect(values).toContain("high"); }); + + it("provider routing options match the backend setting values", () => { + const values = aiTranslatorProviderRoutingOptions.map((o) => o.value); + expect(values).toEqual([ + "throughput", + "nitro", + "price", + "floor", + "latency", + "default", + ]); + }); + + it("provider routing marks throughput as the default", () => { + const throughput = aiTranslatorProviderRoutingOptions.find( + (o) => o.value === "throughput", + ); + expect(throughput?.label).toContain("Default"); + }); }); diff --git a/frontend/src/pages/Settings/Translator/index.tsx b/frontend/src/pages/Settings/Translator/index.tsx index dfbd2bca09..1b5e89d6f7 100644 --- a/frontend/src/pages/Settings/Translator/index.tsx +++ b/frontend/src/pages/Settings/Translator/index.tsx @@ -42,6 +42,7 @@ import { aiTranslatorParallelBatchesOptions, aiTranslatorReasoningOptions, } from "./options"; +import { aiTranslatorProviderRoutingOptions } from "./routing"; const engineOptions = [ { value: "openrouter", label: "OpenRouter" }, @@ -488,7 +489,7 @@ const SettingsTranslatorView: FunctionComponent = () => { - +
{
+ + + [] = [ }, { label: "google/gemini-2.5-flash", value: "google/gemini-2.5-flash" }, { - label: "google/gemini-2.5-flash-lite-preview-09-2025", - value: "google/gemini-2.5-flash-lite-preview-09-2025", + label: "google/gemini-2.5-flash-lite", + value: "google/gemini-2.5-flash-lite", }, { label: "google/gemini-2.5-pro", value: "google/gemini-2.5-pro" }, { label: "google/gemini-3-flash-preview", value: "google/gemini-3-flash-preview", }, - { - label: "google/gemini-3-pro-preview", - value: "google/gemini-3-pro-preview", - }, { label: "google/gemini-3.1-flash-lite-preview", value: "google/gemini-3.1-flash-lite-preview", @@ -62,8 +58,8 @@ export const aiTranslatorModelOptions: SelectorOption[] = [ { label: "openrouter/auto", value: "openrouter/auto" }, { label: "openrouter/free", value: "openrouter/free" }, { label: "qwen/qwen3.5-plus-02-15", value: "qwen/qwen3.5-plus-02-15" }, - { label: "x-ai/grok-4-fast", value: "x-ai/grok-4-fast" }, - { label: "x-ai/grok-4.20-beta", value: "x-ai/grok-4.20-beta" }, + { label: "x-ai/grok-4.20", value: "x-ai/grok-4.20" }, + { label: "x-ai/grok-4.6", value: "x-ai/grok-4.6" }, { label: "z-ai/glm-4.7-flash", value: "z-ai/glm-4.7-flash" }, ]; diff --git a/frontend/src/pages/Settings/Translator/routing.test.ts b/frontend/src/pages/Settings/Translator/routing.test.ts new file mode 100644 index 0000000000..8e0b9fdd81 --- /dev/null +++ b/frontend/src/pages/Settings/Translator/routing.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { + aiTranslatorProviderRoutingOptions, + routingLabel, + splitRoutingSuffix, +} from "./routing"; + +describe("splitRoutingSuffix", () => { + it("leaves a plain model id alone", () => { + expect(splitRoutingSuffix("z-ai/glm-5.3-flash")).toEqual({ + modelId: "z-ai/glm-5.3-flash", + routing: null, + }); + }); + + it("takes :floor off the model id", () => { + expect(splitRoutingSuffix("z-ai/glm-5.3-flash:floor")).toEqual({ + modelId: "z-ai/glm-5.3-flash", + routing: "floor", + }); + }); + + it("takes :nitro off the model id", () => { + expect(splitRoutingSuffix("deepseek/deepseek-v4-flash:nitro")).toEqual({ + modelId: "deepseek/deepseek-v4-flash", + routing: "nitro", + }); + }); + + it("keeps other variants, which are part of the model", () => { + for (const id of [ + "deepseek/deepseek-chat:thinking", + "liquid/lfm-2.5-2.6b:free", + "z-ai/glm-5.3-flash:batch", + "openai/gpt-5.2:online", + "anthropic/claude-haiku-4.5:extended", + "x-ai/grok-4.1:exacto", + ]) { + expect(splitRoutingSuffix(id)).toEqual({ modelId: id, routing: null }); + } + }); + + it("takes the routing suffix off a model that also carries another variant", () => { + expect(splitRoutingSuffix("liquid/lfm-2.5-2.6b:free:nitro")).toEqual({ + modelId: "liquid/lfm-2.5-2.6b:free", + routing: "nitro", + }); + }); + + it("is case insensitive and tolerates surrounding whitespace", () => { + expect(splitRoutingSuffix(" z-ai/glm-5.3-flash:FLOOR ")).toEqual({ + modelId: "z-ai/glm-5.3-flash", + routing: "floor", + }); + }); + + it("takes every stacked routing suffix off and keeps the last one typed", () => { + // Leaving one behind would contradict the selector: the backend reads the + // suffix off the model id and lets it win. + expect(splitRoutingSuffix("z-ai/glm-5.3-flash:nitro:floor")).toEqual({ + modelId: "z-ai/glm-5.3-flash", + routing: "floor", + }); + expect(splitRoutingSuffix("z-ai/glm-5.3-flash:floor:nitro")).toEqual({ + modelId: "z-ai/glm-5.3-flash", + routing: "nitro", + }); + }); + + it("keeps a non-routing variant that sits under stacked routing suffixes", () => { + expect(splitRoutingSuffix("liquid/lfm-2.5-2.6b:free:nitro:floor")).toEqual({ + modelId: "liquid/lfm-2.5-2.6b:free", + routing: "floor", + }); + }); + + it("does not mistake a lookalike for a routing suffix", () => { + for (const id of [ + "some/model:floorplan", + "some/model:nitrox", + "some/floor", + ]) { + expect(splitRoutingSuffix(id).routing).toBeNull(); + } + }); + + it("handles an empty value", () => { + expect(splitRoutingSuffix("")).toEqual({ modelId: "", routing: null }); + }); +}); + +describe("routingLabel", () => { + it("names every routing value from the option list", () => { + for (const option of aiTranslatorProviderRoutingOptions) { + expect(routingLabel(option.value)).toBe(option.label); + } + }); + + it("falls back to the raw value it does not know", () => { + expect(routingLabel("cheapest")).toBe("cheapest"); + }); +}); diff --git a/frontend/src/pages/Settings/Translator/routing.ts b/frontend/src/pages/Settings/Translator/routing.ts new file mode 100644 index 0000000000..496a34ceb9 --- /dev/null +++ b/frontend/src/pages/Settings/Translator/routing.ts @@ -0,0 +1,60 @@ +import { SelectorOption } from "@/components/inputs/Selector"; + +// Which OpenRouter provider serves the model. Mirrors the backend validator for +// translator.openrouter_provider_routing; the sidecar turns nitro and floor into +// the OpenRouter slug shortcuts and the rest into provider.sort. +export const aiTranslatorProviderRoutingOptions: SelectorOption[] = [ + { label: "Fastest (Default)", value: "throughput" }, + { label: "Fastest + priority tier (:nitro)", value: "nitro" }, + { label: "Cheapest", value: "price" }, + { label: "Cheapest + flex tier (:floor)", value: "floor" }, + { label: "Lowest latency", value: "latency" }, + { label: "OpenRouter default (load balanced)", value: "default" }, +]; + +// The two OpenRouter shortcuts that can be appended to a model id. Every other +// variant, :free, :thinking, :online, :extended, :exacto and :batch among them, +// names a different model rather than a way to route to the same one. +const ROUTING_SUFFIXES = ["nitro", "floor"] as const; + +export type RoutingSuffix = (typeof ROUTING_SUFFIXES)[number]; + +export interface SplitModelId { + modelId: string; + routing: RoutingSuffix | null; +} + +// Separates the routing shortcut typed into a model id from the model itself, so +// the id can be looked up on OpenRouter and the routing can be shown in the +// selector that owns it. +// +// Stacked shortcuts are all removed and the last one typed wins. Leaving one on +// the id would contradict the selector, because both the Bazarr backend and the +// sidecar read a shortcut off the model id and let it beat the configured sort. +export function splitRoutingSuffix(rawModelId: string): SplitModelId { + let modelId = (rawModelId ?? "").trim(); + let routing: RoutingSuffix | null = null; + + for (;;) { + const colon = modelId.lastIndexOf(":"); + if (colon <= 0) { + return { modelId, routing }; + } + + const tail = modelId.slice(colon + 1).toLowerCase(); + const found = ROUTING_SUFFIXES.find((suffix) => suffix === tail); + if (!found) { + return { modelId, routing }; + } + + routing = routing ?? found; + modelId = modelId.slice(0, colon); + } +} + +export function routingLabel(value: string): string { + return ( + aiTranslatorProviderRoutingOptions.find((o) => o.value === value)?.label ?? + value + ); +} diff --git a/frontend/src/pages/Settings/components/Layout.test.tsx b/frontend/src/pages/Settings/components/Layout.test.tsx index cf18fca2d2..b6ba177c13 100644 --- a/frontend/src/pages/Settings/components/Layout.test.tsx +++ b/frontend/src/pages/Settings/components/Layout.test.tsx @@ -1,10 +1,27 @@ import { Text } from "@mantine/core"; import userEvent from "@testing-library/user-event"; +import { http, HttpResponse } from "msw"; import { describe, expect, it } from "vitest"; import { useFormActions } from "@/pages/Settings/utilities/FormValues"; import { customRender, screen, waitFor } from "@/tests"; +import server from "@/tests/mocks/node"; import Layout from "./Layout"; +// An input that only stages its value when it is left, the way the AI Model +// field commits an OpenRouter routing shortcut. +function CommitOnBlurInput() { + const { setValue } = useFormActions(); + + return ( + + setValue(e.currentTarget.value, "settings-general-instance_name") + } + /> + ); +} + function StageChangeButton() { const { setValue } = useFormActions(); @@ -82,4 +99,70 @@ describe("Settings layout", () => { }); expect(screen.getByLabelText("1 unsaved change")).toHaveTextContent("1"); }); + + it("commits the focused field before a keyboard save submits", async () => { + const submitted: LooseObject[] = []; + server.use( + http.post("/api/system/settings", async ({ request }) => { + const form = await request.formData(); + submitted.push(Object.fromEntries(form.entries())); + return HttpResponse.json({}); + }), + ); + + const user = userEvent.setup(); + customRender( + + + + , + ); + + // Something has to be staged, or the shortcut does nothing at all. + await user.click(screen.getByRole("button", { name: "Stage change" })); + + const input = screen.getByLabelText("Commits on blur"); + await user.click(input); + await user.keyboard("typed-while-focused"); + await user.keyboard("{Control>}s{/Control}"); + + await waitFor(() => { + expect(submitted).toHaveLength(1); + }); + expect(submitted[0]["settings-general-instance_name"]).toBe( + "typed-while-focused", + ); + }); + + it("commits the focused field when Enter submits the form", async () => { + const submitted: LooseObject[] = []; + server.use( + http.post("/api/system/settings", async ({ request }) => { + const form = await request.formData(); + submitted.push(Object.fromEntries(form.entries())); + return HttpResponse.json({}); + }), + ); + + const user = userEvent.setup(); + customRender( + + + + , + ); + + await user.click(screen.getByRole("button", { name: "Stage change" })); + + const input = screen.getByLabelText("Commits on blur"); + await user.click(input); + await user.keyboard("typed-then-enter{Enter}"); + + await waitFor(() => { + expect(submitted).toHaveLength(1); + }); + expect(submitted[0]["settings-general-instance_name"]).toBe( + "typed-then-enter", + ); + }); }); diff --git a/frontend/src/pages/Settings/components/Layout.tsx b/frontend/src/pages/Settings/components/Layout.tsx index 9d9519cfda..7514014ad9 100644 --- a/frontend/src/pages/Settings/components/Layout.tsx +++ b/frontend/src/pages/Settings/components/Layout.tsx @@ -1,9 +1,11 @@ import { + FormEvent, FunctionComponent, ReactNode, useCallback, useEffect, useMemo, + useRef, } from "react"; import { Badge, @@ -51,6 +53,9 @@ const Layout: FunctionComponent = (props) => { }, }); + const formRef = useRef(form); + formRef.current = form; + useOnValueChange(isRefetching, (value) => { if (!value) { form.reset(); @@ -92,24 +97,48 @@ const Layout: FunctionComponent = (props) => { useDocumentTitle(`${name} - ${useInstanceName()} (Settings)`); + // Some inputs only stage their value when they are left, so the focused field + // is blurred first and the submit waits a tick for that change to land. + // Without it a save keeps the value the field held before the user's last + // edit. Every route into a save goes through here: the Save button, Enter in + // a field, and the keyboard shortcut below. + const commitAndSubmit = useCallback(() => { + const focused = document.activeElement; + if (focused instanceof HTMLElement) { + focused.blur(); + } + + // formRef, not form: a blur handler restages values, and the closure this + // was created in would otherwise submit the ones captured before it ran. + window.setTimeout(() => formRef.current.onSubmit(submit)(), 0); + }, [submit]); + + const onFormSubmit = useCallback( + (e: FormEvent) => { + e.preventDefault(); + commitAndSubmit(); + }, + [commitAndSubmit], + ); + // Ctrl+S / Cmd+S keyboard shortcut useEffect(() => { const handler = (e: KeyboardEvent) => { if ((e.ctrlKey || e.metaKey) && e.key === "s") { e.preventDefault(); if (totalStagedCount > 0) { - form.onSubmit(submit)(); + commitAndSubmit(); } } }; document.addEventListener("keydown", handler); return () => document.removeEventListener("keydown", handler); - }, [form, submit, totalStagedCount]); + }, [commitAndSubmit, totalStagedCount]); return ( -
+ =0.5.9 # support must not be picked up unattended. 1.2.71 is the newest release on # PyPI today, so this resolves to the latest version. cloudscraper<=1.2.71 -cryptography>=49.0.0 +cryptography>=50.0.1 deathbycaptcha-official==4.7.1 deep-translator==1.11.4 dnspython==2.8.0 @@ -40,7 +40,7 @@ Flask-SocketIO==5.6.1 flask_sqlalchemy==3.1.1 ftfy==6.3.1 guess_language-spirit==0.5.4 -guessit==3.8.0 +guessit==4.4.0 itsdangerous==2.2.0 json_tricks==3.17.3 knowit==0.6.1 @@ -66,7 +66,7 @@ semver==3.0.4 # which we pin above at 1.2.1. Keeping it out of the normal resolve avoids the # ResolutionImpossible conflict and keeps Dependabot off the transitive pin. six==1.17.0 -sqlalchemy==2.0.51 +sqlalchemy==2.0.52 srt==3.5.3 subliminal[rar]==2.6.0 textdistance==4.6.3 diff --git a/site/guides/ai-translation.html b/site/guides/ai-translation.html index c7482bc4d4..46380c7dd4 100644 --- a/site/guides/ai-translation.html +++ b/site/guides/ai-translation.html @@ -170,10 +170,11 @@

Choosing a model

  • Quality vs. speed. Larger models (Claude Opus 4.6, GPT-4.1) produce better translations but cost more. Mid-range models (Gemini 2.5 Flash, Claude Sonnet 4.5) hit a good balance. Lightweight models (Gemini 2.5 Flash Lite, Claude Haiku 4.5) are fast and cheap.
  • Free models. OpenRouter offers free tiers for select models (look for the "free" badge). Great for testing the workflow or large batches where perfect quality is not critical.
  • Context window. Subtitle files can be large. Models with bigger context windows (200k+ tokens) handle full-length movies in fewer chunks, producing more consistent translations.
  • +
  • Provider routing. Most models are served by several OpenRouter providers at different prices and speeds. The Provider Routing selector picks between them: Fastest (the default) takes the highest-throughput provider, Cheapest the lowest price. The :nitro and :floor variants also make OpenRouter's priority and flex tiers eligible, and OpenRouter default leaves their load balancing on. Typing :floor or :nitro straight after a model ID works too: Bazarr+ moves the shortcut into the selector and leaves the plain model ID in the field, so the two can never disagree and the model details still load. Routing only changes anything when a model is served by more than one provider, and around half of OpenRouter's catalogue is served by exactly one. OpenRouter ignores a suffix it does not recognise, so a typo such as :floorr fails silently rather than erroring. The two shortcut options and OpenRouter default need AI Subtitle Translator 1.3.4 or newer; on an older sidecar Bazarr+ sends the plain sort each of them stands for (fastest or cheapest) and logs a warning.
  • - Recommendation: Start with google/gemini-2.5-flash-lite-preview-09-2025. It handles subtitle translation well and costs fractions of a cent per episode. Switch to Claude Sonnet 4.5 if you need higher quality for difficult content. + Recommendation: Start with google/gemini-2.5-flash-lite. It handles subtitle translation well and costs fractions of a cent per episode. Switch to Claude Sonnet 4.5 if you need higher quality for difficult content.

    Batch translation from Missing pages

    @@ -216,7 +217,7 @@

    Cost estimates

  • Premium models (Claude Opus 4.6, GPT-4.1): ~$0.15 to $0.50
  • -

    These are ballpark figures. Actual costs vary with subtitle density and the length of each line. OpenRouter shows your usage and spending on their dashboard in real time.

    +

    These are ballpark figures. Actual costs vary with subtitle density, the length of each line, and the provider that serves the request: the same model can cost twice as much from a fast provider as from the cheapest one, so switch Provider Routing to Cheapest when price matters more than speed. OpenRouter shows your usage and spending on their dashboard in real time.

    Watch your spending. Batch-translating hundreds of episodes with a premium model can add up. Set a spending limit on your OpenRouter account if you want a safety net. The setting is under Account > Credits on their site. diff --git a/site/index.html b/site/index.html index e8b4867c67..fbdf402ed8 100644 --- a/site/index.html +++ b/site/index.html @@ -36,7 +36,7 @@ "name": "Bazarr+", "applicationCategory": "MultimediaApplication", "operatingSystem": "Docker, Linux, Windows, macOS", - "softwareVersion": "2.6.1", + "softwareVersion": "2.6.2", "license": "https://www.gnu.org/licenses/gpl-3.0.html", "url": "https://lavx.github.io/bazarr/", "codeRepository": "https://github.com/LavX/bazarr", @@ -597,12 +597,14 @@

    Roadmap

  • Shipped

    The v2 line so far -

    Eleven releases on the v2 line so far, most recently - v2.6 “Clockwork”: a correctness pass across the whole surface, - subtitle downloads including whole-season bundles, sync results held to the offset you - configured, and databases from upstream Bazarr adopted instead of refused. Before it, - v2.5 “Murmuration” brought many Sonarr and Radarr instances - managed as one, per-instance subtitle settings and archive uploads.

    +

    v2.6.2 “Clockwork” keeps subtitle history and sync outputs + with the right media, saves useful partial OpenRouter translations with an incomplete + status, lists uploads before background sync finishes, and adds explicit OpenRouter + routing choices. Searches continue past unusable archives from catalog providers. The v2.6 line also + brought whole-season subtitle downloads, stricter sync validation, and adoption of + upstream Bazarr databases. Before it, v2.5 “Murmuration” + brought many Sonarr and Radarr instances managed as one, per-instance subtitle settings + and archive uploads.

    Every release, with full notes

  • diff --git a/tests/bazarr/fixtures/cryptography49_fernet.json b/tests/bazarr/fixtures/cryptography49_fernet.json new file mode 100644 index 0000000000..b41f8b8c35 --- /dev/null +++ b/tests/bazarr/fixtures/cryptography49_fernet.json @@ -0,0 +1,18 @@ +{ + "source_version": "49.0.0", + "master_key": "synthetic-master-key-for-version-compatibility-only", + "cases": [ + { + "plaintext": "synthetic-provider-token", + "ciphertext": "enc:v1:gAAAAABqm1uaEus9uqCEc7V5GDwndiAh3HvmiIbEEuVCLpudK4OzvCXIt_jOGT-flGv-rNjFnj06Za8vLGX9SM6UNK9POzfHSc012E_Vd2wA_-zEQqis2PQ=" + }, + { + "plaintext": "synthetic-translator-token", + "ciphertext": "enc:v1:gAAAAABqm1uaHS6Ro_bVCFQEdgdCn2veRLwb7Bj4wcK5cJi1z5VPRpY1A-UO1u1qeddWggCYNKMH743Bw25kNLRStuSrDAv_yW90ktfxZKl7ahXKmJwEkZc=" + }, + { + "plaintext": "synthetic-unicode-árvíztűrÅ‘", + "ciphertext": "enc:v1:gAAAAABqm1uaQlgBNKC4euTO9bTpCe-MJ0qtKVdFu8O56EwIXx9BtLH-YGq_nQcNZi53xqMy8ft1Cyi2V8SDqqm2SoTgCxLPzcPLW3FZa9uzZXSA2HWlyJk=" + } + ] +} diff --git a/tests/bazarr/test_archive_member_matching.py b/tests/bazarr/test_archive_member_matching.py index ba12c32236..8eddacc1d6 100644 --- a/tests/bazarr/test_archive_member_matching.py +++ b/tests/bazarr/test_archive_member_matching.py @@ -26,6 +26,7 @@ import provider_hub.protocol as proto from subliminal_patch.core import SUBTITLE_EXTENSIONS +from subliminal_patch.exceptions import SubtitleCandidateRejected from subliminal_patch.providers import utils from subliminal_patch.providers.utils import get_archive_from_bytes @@ -173,8 +174,8 @@ def test_failed_extraction_logs_the_member_names(caplog): assert "Trailer.mkv" in logged -def test_archive_without_any_subtitle_member_still_fails_loudly(): - with pytest.raises(proto.WorkerProtocolError): +def test_archive_without_any_subtitle_member_rejects_candidate(): + with pytest.raises(SubtitleCandidateRejected, match="No usable subtitle member"): _download(_zip([("poster.jpg", b"binary")])) diff --git a/tests/bazarr/test_auto_translation_profile.py b/tests/bazarr/test_auto_translation_profile.py index 61e731b8cb..46f392af44 100644 --- a/tests/bazarr/test_auto_translation_profile.py +++ b/tests/bazarr/test_auto_translation_profile.py @@ -1,14 +1,18 @@ # coding=utf-8 +import logging from types import SimpleNamespace from unittest.mock import Mock, patch +import pytest + def _missing_language(alpha3, hi=False, forced=False): return SimpleNamespace(alpha3=alpha3, hi=hi, forced=forced) -def test_downloaded_series_subtitle_queues_episode_translation(): +@pytest.mark.parametrize('source_score_percent', [90, 100, None]) +def test_downloaded_series_subtitle_queues_episode_translation(source_score_percent): from subtitles.processing import _trigger_auto_translation mock_database = Mock() @@ -30,7 +34,7 @@ def test_downloaded_series_subtitle_queues_episode_translation(): patch('subtitles.processing.alpha2_from_alpha3', return_value='hu'), patch('subtitles.tools.translate.main.translate_subtitles_file') as mock_translate, ): - mock_settings.translator.min_source_score = 0 + mock_settings.translator.min_source_score = 90 mock_profiles.return_value = { 'items': [ { @@ -50,13 +54,125 @@ def test_downloaded_series_subtitle_queues_episode_translation(): media_type='series', series_id=10, episode_id=20, - source_score_percent=100, + source_score_percent=source_score_percent, ) mock_translate.assert_called_once() assert mock_translate.call_args.kwargs['media_type'] == 'episode' +@pytest.mark.parametrize('profile', [ + None, + {'items': []}, + {'items': [ + {'language': 'en', 'translate_from': None}, + {'language': 'hr', 'translate_from': None}, + ]}, + {'items': [{'language': 'hu', 'translate_from': 'fr'}]}, + {'items': [{'language': 'en', 'translate_from': 'en'}]}, +], ids=['no-profile', 'empty-profile', 'disabled', 'different-source', 'same-language']) +def test_downloaded_subtitle_without_matching_translation_target_stays_silent(profile, caplog): + from subtitles.processing import _trigger_auto_translation + + mock_database = Mock() + mock_database.execute.return_value.first.return_value = SimpleNamespace( + imdbId='tt2', tmdbId='2', profileId=1, + ) + + with ( + caplog.at_level(logging.INFO), + patch('subtitles.processing.settings') as mock_settings, + patch('app.database.get_profiles_list', return_value=profile), + patch('app.database.database', mock_database), + patch('subtitles.download.check_missing_languages') as mock_missing, + patch('subtitles.tools.translate.main.translate_subtitles_file') as mock_translate, + ): + mock_settings.translator.min_source_score = 90 + + _trigger_auto_translation( + downloaded_lang='en', + subtitle_path='/subs/source.en.srt', + video_path='/video/movie.mkv', + media_type='movie', + radarr_id=30, + source_score_percent=80, + ) + + assert not [record for record in caplog.records if 'auto-translate' in record.getMessage()] + assert not [record for record in caplog.records if record.levelno >= logging.ERROR] + mock_missing.assert_not_called() + mock_translate.assert_not_called() + + +def test_low_score_subtitle_logs_once_for_multiple_matching_translation_targets(caplog): + from subtitles.processing import _trigger_auto_translation + + mock_database = Mock() + mock_database.execute.return_value.first.return_value = SimpleNamespace( + imdbId='tt2', tmdbId='2', profileId=1, + ) + + with ( + caplog.at_level(logging.INFO), + patch('subtitles.processing.settings') as mock_settings, + patch('app.database.get_profiles_list') as mock_profiles, + patch('app.database.database', mock_database), + patch('subtitles.download.check_missing_languages') as mock_missing, + patch('subtitles.tools.translate.main.translate_subtitles_file') as mock_translate, + ): + mock_settings.translator.min_source_score = 90 + mock_profiles.return_value = { + 'items': [ + {'language': 'hu', 'translate_from': 'en'}, + {'language': 'hr', 'translate_from': 'en'}, + ] + } + + _trigger_auto_translation( + downloaded_lang='en', + subtitle_path='/subs/source.en.srt', + video_path='/video/movie.mkv', + media_type='movie', + radarr_id=30, + source_score_percent=80, + ) + + score_logs = [record for record in caplog.records if 'below threshold' in record.getMessage()] + assert len(score_logs) == 1 + assert score_logs[0].levelno == logging.INFO + assert score_logs[0].args == (80, 90, '/video/movie.mkv') + mock_missing.assert_not_called() + mock_translate.assert_not_called() + + +@pytest.mark.parametrize('source_score_percent', [80, 100]) +def test_forced_source_subtitle_stays_silent_and_does_not_translate(source_score_percent, caplog): + from subtitles.processing import _trigger_auto_translation + + with ( + caplog.at_level(logging.INFO), + patch('subtitles.processing.settings') as mock_settings, + patch('app.database.database') as mock_database, + patch('subtitles.tools.translate.main.translate_subtitles_file') as mock_translate, + ): + mock_settings.translator.min_source_score = 90 + + _trigger_auto_translation( + downloaded_lang='en', + subtitle_path='/subs/source.en.forced.srt', + video_path='/video/movie.mkv', + media_type='movie', + radarr_id=30, + source_score_percent=source_score_percent, + forced=True, + ) + + assert not [record for record in caplog.records if 'auto-translate' in record.getMessage()] + assert not [record for record in caplog.records if record.levelno >= logging.ERROR] + mock_database.execute.assert_not_called() + mock_translate.assert_not_called() + + def test_downloaded_subtitle_does_not_translate_for_mismatched_target_variant(): from subtitles.processing import _trigger_auto_translation diff --git a/tests/bazarr/test_combine_main.py b/tests/bazarr/test_combine_main.py index eb68cdf054..a1ac6007d7 100644 --- a/tests/bazarr/test_combine_main.py +++ b/tests/bazarr/test_combine_main.py @@ -153,7 +153,7 @@ def test_failed_compose_returns_failed( mock_resolve.return_value = make_sources( tmp_path, "en_hu_sibling_en.srt", "en_hu_sibling_hu.srt" ) - mock_filename.return_value = str(tmp_path / "out.srt") + mock_filename.return_value = str(tmp_path / "Movie.en.combined-hu.srt") mock_compose.side_effect = ValueError("bad SRT") result = try_combine_for_video( diff --git a/tests/bazarr/test_dependency_loading.py b/tests/bazarr/test_dependency_loading.py index 9522696563..f5457bd969 100644 --- a/tests/bazarr/test_dependency_loading.py +++ b/tests/bazarr/test_dependency_loading.py @@ -1,9 +1,15 @@ import subprocess import sys +import re +import json from pathlib import Path +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name import subliminal import yaml +import pytest def test_pytest_conftest_imports_without_pkg_resources_deprecation_warning(): @@ -234,6 +240,7 @@ def test_startup_requirements_probe_uses_security_patched_dependency_versions(): requirements = (repo_root / "requirements.txt").read_text() expected_versions = { + "cryptography": ("cryptography", ">=50.0.1"), "dynaconf": ("dynaconf", "==3.3.5"), "urllib3": ("urllib3", "==2.7.0"), } @@ -243,6 +250,102 @@ def test_startup_requirements_probe_uses_security_patched_dependency_versions(): assert f"{distribution}{spec}" in requirements +def test_startup_declarations_match_installation_requirements(): + from app.requirements import RUNTIME_REQUIREMENTS, WINDOWS_RUNTIME_REQUIREMENTS + + repo_root = Path(__file__).resolve().parents[2] + installed = {} + for raw_line in (repo_root / "requirements.txt").read_text().splitlines(): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + # pip's per-requirement options are not part of the version or marker. + requirement = Requirement(re.split(r"\s+--", line, maxsplit=1)[0]) + installed[canonicalize_name(requirement.name)] = requirement + + declarations = {**RUNTIME_REQUIREMENTS, **WINDOWS_RUNTIME_REQUIREMENTS} + separate_installs = {distribution for distribution, _ in declarations.values() + if canonicalize_name(distribution) not in installed} + # signalrcore is installed separately with --no-deps to retain safe msgpack. + assert separate_installs == {"signalrcore"} + mismatches = {} + for module, (distribution, spec) in declarations.items(): + if distribution == "signalrcore": + continue + requirement = installed[canonicalize_name(distribution)] + if SpecifierSet(spec) != requirement.specifier: + mismatches[module] = (spec, str(requirement.specifier)) + if module in WINDOWS_RUNTIME_REQUIREMENTS: + assert requirement.marker is not None + assert requirement.marker.evaluate({"platform_system": "Windows"}) + assert not requirement.marker.evaluate({"platform_system": "Linux"}) + else: + assert requirement.marker is None + assert mismatches == {} + + +def test_complete_compliant_runtime_does_not_request_install_or_restart(monkeypatch, tmp_path): + from app import requirements + + monkeypatch.setenv("PLEXAPI_CONFIG_PATH", str(tmp_path / "plexapi.ini")) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + repairs, restarts = [], [] + monkeypatch.setattr(requirements, "install_requirements", lambda missing: repairs.append(missing) or True) + monkeypatch.setattr(requirements, "restart_after_requirements_install", lambda: restarts.append(True)) + + assert requirements.missing_runtime_requirements() == [] + assert requirements.ensure_requirements() is False + assert requirements.ensure_requirements() is False + assert repairs == restarts == [] + + +@pytest.mark.parametrize("module,distribution,unsupported", [ + ("alembic", "alembic", "1.18.4"), + ("cloudscraper", "cloudscraper", "1.2.72"), + ("numpy", "numpy", "2.6.0"), + ("PIL", "Pillow", "12.2.0"), + ("dateutil", "python-dateutil", "2.9.0rc0"), + ("dateutil", "python-dateutil", "2.9.0.dev0"), + ("dateutil", "python-dateutil", "2.9.0"), + ("dateutil", "python-dateutil", "2.9.0.post1"), + ("dateutil", "python-dateutil", "invalid"), +]) +def test_complete_runtime_repairs_an_unsupported_version_once(monkeypatch, tmp_path, + module, distribution, unsupported): + from app import requirements + + monkeypatch.setenv("PLEXAPI_CONFIG_PATH", str(tmp_path / "plexapi.ini")) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + actual_version = requirements.metadata.version + installed_version = [unsupported] + + def version(name): + if canonicalize_name(name) == canonicalize_name(distribution): + return installed_version[0] + return actual_version(name) + + monkeypatch.setattr(requirements.metadata, "version", version) + repairs, restarts = [], [] + + def install(missing): + repairs.append(missing) + installed_version[0] = actual_version(distribution) + return True + + monkeypatch.setattr(requirements, "install_requirements", install) + monkeypatch.setattr(requirements, "restart_after_requirements_install", lambda: restarts.append(True)) + + assert requirements.missing_runtime_requirements() == [module] + assert requirements.ensure_requirements(no_update=True) is False + assert repairs == restarts == [] + assert requirements.ensure_requirements() is True + assert repairs == [[module]] + assert restarts == [True] + assert requirements.missing_runtime_requirements() == [] + assert requirements.ensure_requirements() is False + assert len(repairs) == len(restarts) == 1 + + def test_startup_requirements_probe_rejects_wrong_pinned_versions(monkeypatch): from app import requirements @@ -263,6 +366,154 @@ def test_startup_requirements_probe_supports_upper_bound_specs(): assert not _satisfies_spec("1.2.59", "<=1.2.58") +VERSION_BOUNDARIES = [ + ("2.9.0rc0", "==2.9.0.post0", False), + ("2.9.0.dev0", "==2.9.0.post0", False), + ("2.9.0", "==2.9.0.post0", False), + ("2.9.0.post0", "==2.9.0.post0", True), + ("2.9.0.post1", "==2.9.0.post0", False), + ("2.9.0.post0+vendor.1", "==2.9.0.post0", True), + ("2.9.0.post0", "==2.9.0", False), + ("50.0.1rc1", ">=50.0.1", False), + ("50.0.1.dev1", ">=50.0.1", False), + ("50.0.1", ">=50.0.1", True), + ("50.0.1.post0", ">=50.0.1", True), + ("50.0.1+vendor.1", ">=50.0.1", True), + ("50.0.2rc1", ">=50.0.1", True), + ("50.0.2.dev1", ">=50.0.1", True), + ("1.0", "==1.0.0", True), + ("v1.0.0", "==1.0", True), + ("1.0+vendor.1", "==1.0", True), + ("1.0+vendor.2", "==1.0+vendor.1", False), + ("1.0+vendor.1", "==1.0+vendor.1", True), + ("1!1.0", ">=2.0", True), + ("2.0", ">=1!1.0", False), + ("2.5.1", ">=2.5.2,<2.6.0", False), + ("2.5.2", ">=2.5.2,<2.6.0", True), + ("2.5.9rc1", ">=2.5.2,<2.6.0", True), + ("2.6.0rc1", ">=2.5.2,<2.6.0", False), + ("2.6.0.dev1", ">=2.5.2,<2.6.0", False), + ("2.6.0", ">=2.5.2,<2.6.0", False), + ("1.2.71", "<=1.2.71", True), + ("1.2.71.post0", "<=1.2.71", False), + ("1.2.71rc1", "<=1.2.71", True), + ("1.0rc1", ">=1.0rc1,<1.0", False), + ("1.0rc1", "==1.0rc1", True), + ("1.0rc2", "==1.0rc1", False), + ("unknown", ">=1.0", False), + ("unknown", "<2.0", False), + ("1.0-broken", "==1.0", False), +] + + +@pytest.mark.parametrize("installed,spec,expected", VERSION_BOUNDARIES) +def test_installed_versions_follow_pep440_boundaries(installed, spec, expected): + from app.requirements import _satisfies_spec + + assert _satisfies_spec(installed, spec) is expected + + +@pytest.mark.parametrize("backend", ["packaging", "pip_vendored", "unavailable"]) +def test_version_parser_bootstraps_with_available_production_dependencies(backend): + repo_root = Path(__file__).resolve().parents[2] + script = r''' +import importlib.abc +import json +import sys + +backend, cases = sys.argv[1], json.loads(sys.argv[2]) +class ParserBlocker(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if backend != "packaging" and (fullname == "packaging" or fullname.startswith("packaging.")): + raise ModuleNotFoundError(name=fullname) + if backend == "unavailable" and (fullname == "pip" or fullname.startswith("pip.")): + raise ModuleNotFoundError(name=fullname) + +sys.meta_path.insert(0, ParserBlocker()) +from app import requirements +assert requirements.ensure_requirements(no_update=True) is False +assert "packaging.specifiers" not in sys.modules +assert "pip._vendor.packaging.specifiers" not in sys.modules +if backend == "unavailable": + assert requirements._satisfies_spec("1.0", "==1.0") is False + requirements.RUNTIME_IMPORTS = ("types",) + requirements.RUNTIME_REQUIREMENTS = {"types": ("fixture", "==1.0")} + requirements.metadata.version = lambda distribution: "1.0" + requirements.importlib.util.find_spec = lambda name: None + def forbidden_restart(): + raise AssertionError("restart without pip") + requirements.restart_after_requirements_install = forbidden_restart + assert requirements.ensure_requirements() is True +else: + for installed, spec, expected in cases: + assert requirements._satisfies_spec(installed, spec) is expected, (installed, spec) + parser = "packaging.specifiers" if backend == "packaging" else "pip._vendor.packaging.specifiers" + assert parser in sys.modules + if backend == "pip_vendored": + assert "packaging" not in sys.modules +''' + result = subprocess.run( + [sys.executable, "-B", "-c", "import sys; sys.path.insert(0, " + + repr(str(repo_root / "bazarr")) + ");\n" + script, backend, json.dumps(VERSION_BOUNDARIES)], + cwd=repo_root, capture_output=True, text=True, check=False, + ) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("backend", ["packaging", "pip_vendored"]) +def test_parser_import_does_not_hide_unrelated_missing_dependencies(monkeypatch, backend): + import builtins + from app.requirements import _satisfies_spec + + original_import = builtins.__import__ + + def broken_parser(name, *args, **kwargs): + if name == "packaging.specifiers": + if backend == "pip_vendored": + raise ModuleNotFoundError(name="packaging") + raise ModuleNotFoundError(name="unrelated_parser_dependency") + if name == "pip._vendor.packaging.specifiers": + raise ModuleNotFoundError(name="unrelated_parser_dependency") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", broken_parser) + with pytest.raises(ModuleNotFoundError) as raised: + _satisfies_spec("1.0", "==1.0") + assert raised.value.name == "unrelated_parser_dependency" + + +def test_invalid_requirement_declarations_remain_errors(): + from app.requirements import _satisfies_spec + + with pytest.raises(ValueError): + _satisfies_spec("1.0", "not-a-specifier") + + +@pytest.mark.parametrize('installed', ['48.0.0', '49.0.0', '50.0.0', '50.0.1', '50.0.2']) +def test_cryptography_security_floor_repairs_old_installs_once(monkeypatch, installed): + from app import requirements + + monkeypatch.setattr(requirements, 'RUNTIME_IMPORTS', ('cryptography',)) + installed_version = [installed] + monkeypatch.setattr(requirements.metadata, 'version', lambda distribution: installed_version[0]) + repairs, restarts = [], [] + + def install(missing): + repairs.append(missing) + installed_version[0] = '50.0.1' + return True + + monkeypatch.setattr(requirements, 'install_requirements', install) + monkeypatch.setattr(requirements, 'restart_after_requirements_install', lambda: restarts.append(True)) + needs_repair = installed in ['48.0.0', '49.0.0', '50.0.0'] + assert requirements.missing_runtime_requirements() == (['cryptography'] if needs_repair else []) + assert requirements.ensure_requirements() is needs_repair + assert repairs == ([['cryptography']] if needs_repair else []) + assert restarts == ([True] if needs_repair else []) + assert requirements.ensure_requirements() is False + assert len(repairs) == len(restarts) == int(needs_repair) + + def test_startup_requirements_probe_rejects_removed_vendor_origins(monkeypatch): from app import requirements diff --git a/tests/bazarr/test_gemini_translator.py b/tests/bazarr/test_gemini_translator.py index b7396fee0f..b6ffbad132 100644 --- a/tests/bazarr/test_gemini_translator.py +++ b/tests/bazarr/test_gemini_translator.py @@ -1,4 +1,9 @@ import time +import json +import sys +from pathlib import Path +from threading import Event, Thread +from types import ModuleType, SimpleNamespace import pytest @@ -160,3 +165,167 @@ def _fail_after_output_file_created(*args, **kwargs): service._translate_with_gemini() assert not output_file.exists() + + +@pytest.fixture +def queued_gemini(tmp_path, monkeypatch): + from app import jobs_queue as queue_module + + monkeypatch.setattr(queue_module.JobsQueue, "_flush_progress_loop", lambda self: None) + monkeypatch.setattr(queue_module, "event_stream", lambda **kwargs: None) + queue = queue_module.JobsQueue() + monkeypatch.setattr(gemini_translator, "jobs_queue", queue) + source = tmp_path / "Video.en.srt" + source.write_text("1\n00:00:01,000 --> 00:00:02,000\nSource text\n", encoding="utf-8") + destination = tmp_path / "Video.hu.srt" + destination.write_text("Previous destination", encoding="utf-8") + video = tmp_path / "Video.mkv" + video.touch() + service = gemini_translator.GeminiTranslatorService( + str(source), str(destination), "hun", "movie", None, None, 7, False, False, + str(video), "en", "hu", arr_instance_id=2) + monkeypatch.setattr(service, "_get_configured_api_keys", lambda: ["fixture-one", "fixture-two"]) + monkeypatch.setattr(service, "_get_batch_size", lambda: 1) + monkeypatch.setattr(gemini_translator, "get_description", lambda *args, **kwargs: "") + monkeypatch.setattr(gemini_translator, "language_from_alpha2", lambda value: "English") + monkeypatch.setattr(gemini_translator, "language_from_alpha3", lambda value: "Hungarian") + monkeypatch.setattr(gemini_translator, "create_process_result", lambda *args, **kwargs: None) + history = [] + monkeypatch.setattr(gemini_translator, "history_log_movie", lambda **kwargs: history.append(kwargs)) + dispatch = ModuleType("gemini_cancellation_fixture") + dispatch.translate = service.translate + monkeypatch.setitem(sys.modules, dispatch.__name__, dispatch) + queue.feed_jobs_pending_queue("Translation", dispatch.__name__, "translate", is_progress=True) + job = queue.jobs_pending_queue.popleft() + queue.jobs_running_queue.append(job) + worker = Thread(target=queue._run_job, args=(job,)) + release = Event() + fixture = SimpleNamespace(queue=queue, job=job, service=service, source=source, destination=destination, + worker=worker, entered=Event(), release=release, calls=[], history=history) + yield fixture + release.set() + if worker.ident: + worker.join(3) + assert not worker.is_alive() + + +def _gemini_response(status=200): + response = gemini_translator.requests.Response() + response.status_code = status + response.headers["Retry-After"] = "7" + content = ({"candidates": [{"content": {"parts": [{"text": json.dumps([ + {"index": "0", "content": "Translated text"}])}]}}]} if status == 200 else + {"error": {"status": "RESOURCE_EXHAUSTED"}}) + response._content = json.dumps(content).encode() + return response + + +def _record_cancellation_state(fixture, record_property): + record_property("transport_calls", len(fixture.calls)) + record_property("progress_files_remaining", len(list(fixture.source.parent.glob("*.progress")))) + record_property("temporary_files_remaining", len(list(fixture.source.parent.glob(".bazarr-write-*")))) + record_property("job_cancelled", fixture.job.cancelled) + assert fixture.job.cancelled + assert fixture.job.status == "completed" + assert fixture.job.progress_message == "Cancelled by user" + assert fixture.destination.read_text() == "Previous destination" + assert "Source text" in fixture.source.read_text() + assert not fixture.history + assert not list(fixture.source.parent.glob(".bazarr-write-*")) + assert not list(fixture.source.parent.glob("*.progress")) + + +@pytest.mark.parametrize("response_kind", ["success", "transient-error", "rate-limit"]) +def test_cancelled_response_does_not_retry_translation(queued_gemini, monkeypatch, record_property, response_kind): + fixture = queued_gemini + + def transport(*args, **kwargs): + fixture.calls.append(True) + fixture.entered.set() + assert fixture.release.wait(3) + if response_kind == "transient-error": + raise gemini_translator.requests.ConnectionError("synthetic transport failure") + return _gemini_response(429 if response_kind == "rate-limit" else 200) + + monkeypatch.setattr(gemini_translator.requests, "request", transport) + fixture.worker.start() + assert fixture.entered.wait(3) + assert fixture.queue.cancel_running_job(fixture.job.job_id) + fixture.release.set() + fixture.worker.join(3) + assert not fixture.worker.is_alive() + _record_cancellation_state(fixture, record_property) + assert len(fixture.calls) == 1 + + +@pytest.mark.parametrize("phase", ["initial-progress", "final-progress"]) +def test_cancellation_at_progress_boundary_clears_progress(queued_gemini, monkeypatch, record_property, phase): + fixture = queued_gemini + original = fixture.queue.update_job_progress + + def progress(*args, **kwargs): + if ((phase == "initial-progress" and "progress_max" in kwargs) or + (phase == "final-progress" and "progress_value" in kwargs + and kwargs.get("progress_message") == str(fixture.source))): + assert Path(fixture.service.progress_file).is_file() + assert fixture.queue.cancel_running_job(fixture.job.job_id) + return original(*args, **kwargs) + + monkeypatch.setattr(fixture.queue, "update_job_progress", progress) + monkeypatch.setattr(gemini_translator.requests, "request", + lambda *args, **kwargs: fixture.calls.append(True) or _gemini_response()) + fixture.worker.start() + fixture.worker.join(3) + assert not fixture.worker.is_alive() + _record_cancellation_state(fixture, record_property) + assert len(fixture.calls) == (0 if phase == "initial-progress" else 1) + + +def test_already_cancelled_batch_does_not_send_request(queued_gemini, monkeypatch, record_property): + from app.jobs_queue import JobCancelled + + fixture = queued_gemini + service = fixture.service + service.job_id = fixture.job.job_id + service.current_api_key = "fixture" + service.target_language = "Hungarian" + assert fixture.queue.cancel_running_job(fixture.job.job_id) + monkeypatch.setattr(gemini_translator.requests, "request", + lambda *args, **kwargs: fixture.calls.append(True) or _gemini_response()) + batch = [{"index": "0", "content": "Source text"}] + translated = list(gemini_translator.srt.parse(fixture.source.read_text())) + with pytest.raises(JobCancelled): + service._process_batch(batch, translated, 1) + record_property("transport_calls", len(fixture.calls)) + assert not fixture.calls + + +@pytest.mark.parametrize("failure", ["transient", "rate-limit", "exhausted"]) +def test_ordinary_request_retries_keep_existing_behavior(queued_gemini, monkeypatch, failure): + fixture = queued_gemini + keys = [] + + def transport(*args, **kwargs): + fixture.calls.append(True) + keys.append(fixture.service.current_api_key) + if failure == "exhausted" or (failure == "transient" and len(fixture.calls) == 1): + raise gemini_translator.requests.ConnectionError("synthetic transport failure") + return _gemini_response(429 if failure == "rate-limit" and len(fixture.calls) == 1 else 200) + + monkeypatch.setattr(gemini_translator.requests, "request", transport) + fixture.worker.start() + fixture.worker.join(3) + assert not fixture.worker.is_alive() + assert len(fixture.calls) == (4 if failure == "exhausted" else 2) + assert not fixture.job.cancelled + assert fixture.job.status == ("failed" if failure == "exhausted" else "completed") + assert len(fixture.history) == (0 if failure == "exhausted" else 1) + if failure == "exhausted": + assert fixture.destination.read_text() == "Previous destination" + else: + assert "Translated text" in fixture.destination.read_text() + assert "Source text" in fixture.source.read_text() + assert not list(fixture.source.parent.glob("*.progress")) + assert not list(fixture.source.parent.glob(".bazarr-write-*")) + if failure == "rate-limit": + assert keys[0] != keys[1] diff --git a/tests/bazarr/test_history_owner_resolution.py b/tests/bazarr/test_history_owner_resolution.py new file mode 100644 index 0000000000..9d2ebb10cb --- /dev/null +++ b/tests/bazarr/test_history_owner_resolution.py @@ -0,0 +1,227 @@ +# coding=utf-8 +"""History ownership must survive missing media rows (LavX/bazarr#404).""" + +import logging +from datetime import datetime +from functools import partial +from types import SimpleNamespace + +import pytest +from sqlalchemy import MetaData, create_engine, event, select +from sqlalchemy.orm import sessionmaker + + +@pytest.fixture +def history_session(): + from app.database import Base + + # The cutover migration requires an owner, while the ORM still allows NULL. + # Clone the metadata so this fixture cannot change other tests' schemas. + metadata = MetaData() + for table in Base.metadata.sorted_tables: + table.to_metadata(metadata) + for name in ('table_history', 'table_history_movie'): + metadata.tables[name].c.arr_instance_id.nullable = False + + engine = create_engine('sqlite:///:memory:') + + @event.listens_for(engine, 'connect') + def enable_foreign_keys(connection, connection_record): + connection.execute('PRAGMA foreign_keys=ON') + + metadata.create_all(engine) + session = sessionmaker(bind=engine)() + try: + yield session + finally: + session.close() + engine.dispose() + + +@pytest.fixture(params=['episode', 'movie']) +def history(request, history_session, monkeypatch): + from app.database import TableHistory, TableHistoryMovie + from radarr import history as movie_history + from sonarr import history as episode_history + + if request.param == 'episode': + case = SimpleNamespace( + module=episode_history, + log=partial(episode_history.history_log, sonarr_series_id=10), + table=TableHistory, + upstream_arg='sonarr_episode_id', + upstream_column='sonarrEpisodeId', + local_refs={'episode_id': 601, 'series_id': 501}, + resolver='_resolve_episode_owner_row', + score_out_of=360, + instance_kind='sonarr', + ) + else: + case = SimpleNamespace( + module=movie_history, + log=movie_history.history_log_movie, + table=TableHistoryMovie, + upstream_arg='radarr_id', + upstream_column='radarrId', + local_refs={'movie_id': 601}, + resolver='_resolve_movie_owner_row', + score_out_of=180, + instance_kind='radarr', + ) + case.kind = request.param + case.events = [] + monkeypatch.setattr(case.module, 'database', history_session) + monkeypatch.setattr(case.module, 'event_stream', lambda **event: case.events.append(event)) + return case + + +@pytest.fixture +def download_result(): + return SimpleNamespace( + message='Subtitle downloaded', + path='/media/video.mkv', + language_code='en', + provider='provider-a', + score=75, + subs_id='subtitle-123', + subs_path='/media/video.en.srt', + matched=['title', 'year'], + not_matched=['release_group'], + ) + + +def add_media(history, session, owner, local_id=601): + from app.database import TableEpisodes, TableMovies, TableShows + + if history.kind == 'episode': + session.add(TableShows( + id=local_id - 100, sonarrSeriesId=10, arr_instance_id=owner, + path=f'/media/series-{local_id}', title='Series', + )) + session.flush() + session.add(TableEpisodes( + id=local_id, series_id=local_id - 100, + sonarrSeriesId=10, sonarrEpisodeId=101, arr_instance_id=owner, + path=f'/media/video-{local_id}.mkv', title='Episode', season=1, episode=1, + )) + else: + session.add(TableMovies( + id=local_id, radarrId=101, arr_instance_id=owner, + path=f'/media/video-{local_id}.mkv', title='Movie', tmdbId='123', + )) + session.flush() + + +def log_history(history, result, media_id=101, **kwargs): + return history.log(action=1, result=result, **{history.upstream_arg: media_id}, **kwargs) + + +def test_explicit_owner_survives_missing_media(history, history_session, download_result): + # A matching upstream ID in another instance must not supply local refs. + add_media(history, history_session, owner=7, local_id=701) + + log_history(history, download_result, arr_instance_id=4) + history_session.commit() + + row = history_session.execute(select(history.table)).scalar_one() + assert row.arr_instance_id == 4 + assert getattr(row, history.upstream_column) == 101 + assert all(getattr(row, ref) is None for ref in history.local_refs) + assert history.events == [{'type': f'{history.kind}-history'}] + + +def test_explicit_owner_without_media_id_skips_lookup(history, history_session, download_result, monkeypatch): + def unexpected_lookup(*args): + pytest.fail('History without an upstream media ID must not query its owner') + + monkeypatch.setattr(history.module, history.resolver, unexpected_lookup) + + log_history(history, download_result, media_id=None, arr_instance_id=4) + history_session.commit() + + row = history_session.execute(select(history.table)).scalar_one() + assert row.arr_instance_id == 4 + assert getattr(row, history.upstream_column) is None + assert all(getattr(row, ref) is None for ref in history.local_refs) + assert history.events == [{'type': f'{history.kind}-history'}] + + +def test_resolved_media_supplies_owner_and_local_refs(history, history_session, download_result): + add_media(history, history_session, owner=4) + + log_history(history, download_result) + history_session.commit() + + row = history_session.execute(select(history.table)).scalar_one() + assert row.arr_instance_id == 4 + assert {ref: getattr(row, ref) for ref in history.local_refs} == history.local_refs + assert row.provider == 'provider-a' + assert row.score == 75 + assert row.score_out_of == history.score_out_of + assert history.events == [{'type': f'{history.kind}-history'}] + + +@pytest.mark.parametrize('media_state', ['missing', 'no-upstream-id', 'unowned']) +def test_missing_owner_skips_history_and_event(history, history_session, download_result, caplog, media_state): + from app.database import TableArrInstances + + # Even a configured default cannot establish ownership of missing media. + history_session.add(TableArrInstances( + id=4, kind=history.instance_kind, stable_key='default', name='Default', + port=8989, enabled=1, is_default=1, + )) + if media_state == 'unowned': + add_media(history, history_session, owner=None) + history_session.flush() + media_id = None if media_state == 'no-upstream-id' else 101 + + with caplog.at_level(logging.WARNING): + result = log_history(history, download_result, media_id=media_id) + history_session.commit() + + assert result is None + assert history_session.execute(select(history.table)).scalars().all() == [] + assert history.events == [] + assert any( + record.levelno == logging.WARNING + and history.kind in record.getMessage() + and 'owning instance' in record.getMessage() + and str(media_id) in record.getMessage() + for record in caplog.records + ) + + +def test_explicit_owner_keeps_history_fields_and_scoped_refs(history, history_session, download_result): + add_media(history, history_session, owner=7, local_id=701) + add_media(history, history_session, owner=4) + history_session.add(history.table( + id=9, arr_instance_id=4, action=1, description='Previous download', + timestamp=datetime(2026, 9, 1), + )) + history_session.flush() + + before = datetime.now() + log_history(history, download_result, arr_instance_id=4, + fake_provider='override-provider', fake_score=80, upgraded_from_id=9) + history_session.commit() + + row = history_session.execute(select(history.table).where(history.table.id != 9)).scalar_one() + assert row.arr_instance_id == 4 + assert {ref: getattr(row, ref) for ref in history.local_refs} == history.local_refs + assert getattr(row, history.upstream_column) == 101 + if history.kind == 'episode': + assert row.sonarrSeriesId == 10 + assert row.action == 1 + assert row.description == 'Subtitle downloaded' + assert row.video_path == '/media/video.mkv' + assert row.language == 'en' + assert row.provider == 'override-provider' + assert row.score == 80 + assert row.score_out_of == history.score_out_of + assert row.subs_id == 'subtitle-123' + assert row.subtitles_path == '/media/video.en.srt' + assert row.matched == "['title', 'year']" + assert row.not_matched == "['release_group']" + assert row.upgradedFromId == 9 + assert before <= row.timestamp <= datetime.now() + assert history.events == [{'type': f'{history.kind}-history'}] diff --git a/tests/bazarr/test_indexer_owner_scope.py b/tests/bazarr/test_indexer_owner_scope.py index 84cf98e7fa..81a310de3f 100644 --- a/tests/bazarr/test_indexer_owner_scope.py +++ b/tests/bazarr/test_indexer_owner_scope.py @@ -16,9 +16,12 @@ Callers know the owner. They must be able to say so. """ +import ast +import unicodedata + import pytest -from sqlalchemy import select +from sqlalchemy import select, update from app.database import TableEpisodes, TableMovies, TableShows @@ -35,7 +38,7 @@ def stub_indexing(monkeypatch): for mod in (se, mv): monkeypatch.setattr(mod.os.path, 'exists', lambda p: True) monkeypatch.setattr(mod, 'search_external_subtitles', lambda *a, **kw: {}) - monkeypatch.setattr(mod, 'add_sync_engine_outputs', lambda folder, subs: subs) + monkeypatch.setattr(mod, 'add_sync_engine_outputs', lambda folder, subs, **kw: subs) monkeypatch.setattr(mod, 'add_combined_outputs', lambda folder, subs, **kw: subs) monkeypatch.setattr(mod, 'guess_external_subtitles', lambda *a, **kw: {}) monkeypatch.setattr(mod, 'event_stream', lambda *a, **kw: None) @@ -73,6 +76,306 @@ def _subs(session, table, row_id): return session.execute(select(table.subtitles).where(table.id == row_id)).scalar() +@pytest.mark.parametrize('media_type', ['series', 'movie']) +@pytest.mark.parametrize('subfolder', ['current', 'relative', 'absolute']) +@pytest.mark.parametrize('naming,single_language', [ + ('original', False), ('case', False), ('unicode', False), + ('single-language', True), ('single-language', False), + ('legacy-tagged', True), ('legacy-tagged', False), + ('legacy-case', True), ('legacy-unicode', True), +]) +def test_sync_outputs_belong_to_the_indexed_video(schema_session, monkeypatch, tmp_path, + media_type, subfolder, naming, single_language): + from app import database as database_module + from subzero.language import Language + from subtitles.tools.subsync_engines import SyncOutputOwnerIndex + import subtitles.indexer.movies as mv + import subtitles.indexer.series as se + + module = se if media_type == 'series' else mv + table = TableEpisodes if media_type == 'series' else TableMovies + store = se.store_subtitles if media_type == 'series' else mv.store_subtitles_movie + stems = (['Show.S01E01', 'Show.S01E02', 'Show.S01E01.Extended'] if media_type == 'series' + else ['Movie (2020)', 'Other (2021)', 'Movie (2020).Extended']) + if naming in ['unicode', 'legacy-unicode']: + stems = ['\u00c9pisode.S01E01', '\u00c9pisode.S01E02', '\u00c9pisode.S01E01.Extended'] + elif naming == 'single-language': + stems = ['Movie', 'Movie.en', 'Movie.en.Extended', 'Movie.hi', 'Movie.forced'] + media_folder = tmp_path / 'media' + media_folder.mkdir() + subtitle_folder = {'current': media_folder, 'relative': media_folder / 'subs', + 'absolute': tmp_path / 'shared-subs'}[subfolder] + subtitle_folder.mkdir(exist_ok=True) + custom_folder = 'subs' if subfolder == 'relative' else str(subtitle_folder) + monkeypatch.setattr(module.settings.general, 'use_embedded_subs', False) + monkeypatch.setattr(module.settings.general, 'single_language', single_language) + monkeypatch.setattr(module.settings.general, 'subfolder', subfolder) + monkeypatch.setattr(module.settings.general, 'subfolder_custom', custom_folder) + monkeypatch.setattr(module.core, 'CUSTOM_PATHS', []) + monkeypatch.setattr(module, 'database', schema_session) + monkeypatch.setattr(database_module, 'database', schema_session) + monkeypatch.setattr(module, 'get_language_set', lambda: {Language.fromietf('en')}) + monkeypatch.setattr(module, 'alpha2_from_alpha3', lambda code: Language(code).alpha2) + monkeypatch.setattr(module.path_mappings, 'path_replace_instance', lambda p, *a: p) + monkeypatch.setattr(module.path_mappings, 'path_replace_reverse_instance', lambda p, *a: p) + monkeypatch.setattr(module, 'event_stream', lambda *a, **kw: None) + missing = 'list_missing_subtitles' if media_type == 'series' else 'list_missing_subtitles_movies' + monkeypatch.setattr(module, missing, lambda *a, **kw: None) + + if media_type == 'series': + schema_session.add(TableShows(id=1, arr_instance_id=1, sonarrSeriesId=1, title='Show', + path=str(media_folder), profileId=None)) + schema_session.flush() + + suffixes = {'en.srt': 'en', 'hu.ffsubsync.srt': 'hu:sync-ffsubsync', + 'en.hi.alass.ass': 'en:hi:sync-alass', + 'fr.forced.autosubsync.vtt': 'fr:forced:sync-autosubsync'} + if naming == 'single-language': + suffixes = {f'{engine}.srt': f'en:sync-{engine}' + for engine in ['ffsubsync', 'alass', 'autosubsync']} + suffixes.update({'hu.ffsubsync.srt': 'hu:sync-ffsubsync', + 'en.hi.alass.ass': 'en:hi:sync-alass'}) + elif naming.startswith('legacy-'): + suffixes['en.ffsubsync.srt'] = 'en:sync-ffsubsync' + expected = {} + for row_id, stem in enumerate(stems, 1): + video = media_folder / f'{stem}.mkv' + video.write_bytes(b'video') + if media_type == 'series': + row = TableEpisodes(id=row_id, arr_instance_id=1, series_id=1, sonarrSeriesId=1, + sonarrEpisodeId=row_id, title=stem, path=str(video), season=1, + episode=row_id, subtitles='[]') + else: + row = TableMovies(id=row_id, arr_instance_id=1, radarrId=row_id, title=stem, + path=str(video), tmdbId=str(row_id), subtitles='[]') + schema_session.add(row) + expected[row_id] = [] + subtitle_stem = stem.lower() if naming in ['case', 'legacy-case'] else stem + if naming in ['unicode', 'legacy-unicode']: + subtitle_stem = unicodedata.normalize('NFD', stem.lower()) + for suffix, language in suffixes.items(): + subtitle = subtitle_folder / f'{subtitle_stem}.{suffix}' + subtitle.write_text('1\n00:00:00,000 --> 00:00:01,000\n' + 'This is an English subtitle. We are going to the house together. ' + 'There is a friend waiting for us in the garden. ' + 'Please tell everyone that we will arrive before dinner.\n', encoding='utf-8') + expected[row_id].append([language, str(subtitle), subtitle.stat().st_size]) + schema_session.flush() + contaminated = [entry for entries in expected.values() for entry in entries] + schema_session.execute(update(table).values(subtitles=str(contaminated))) + schema_session.commit() + + first_scan = {} + for scan in range(3): + ownership_index = SyncOutputOwnerIndex() + if scan == 2: + schema_session.execute(update(table).values(subtitles='[]')) + schema_session.commit() + for row_id, stem in enumerate(stems, 1): + video = str(media_folder / f'{stem}.mkv') + actual = store(video, video, arr_instance_id=1, ownership_index=ownership_index) + assert len(actual) == len(expected[row_id]) + assert {path: (set(language.split(':')), size) for language, path, size in actual} == { + path: (set(language.split(':')), size) for language, path, size in expected[row_id]} + assert ast.literal_eval(_subs(schema_session, table, row_id)) == actual + if scan == 0: + first_scan[row_id] = sorted(actual) + else: + assert sorted(actual) == first_scan[row_id] + + +@pytest.mark.parametrize('media_type', ['series', 'movie']) +@pytest.mark.parametrize('subfolder', ['relative', 'absolute']) +@pytest.mark.parametrize('library_size', [8, 32]) +def test_full_scan_amortizes_sync_output_ownership(schema_session, monkeypatch, tmp_path, + media_type, subfolder, library_size): + from app import database as database_module + from subzero.language import Language + import subtitles.indexer.movies as mv + import subtitles.indexer.series as se + + module = se if media_type == 'series' else mv + table = TableEpisodes if media_type == 'series' else TableMovies + full_scan = se.series_full_scan_subtitles if media_type == 'series' else mv.movies_full_scan_subtitles + mapping_calls = [] + owner_queries = [] + execute = schema_session.execute + + def counted_execute(statement, *args, **kwargs): + if getattr(statement, 'is_select', False): + columns = list(statement.selected_columns) + if [column.name for column in columns] == ['path', 'arr_instance_id']: + owner_queries.append(columns[0].table.name) + return execute(statement, *args, **kwargs) + + def mapped(path, instance, kind): + mapping_calls.append((instance, kind)) + assert instance == 7 + assert kind in ('episode', 'series', 'movie') + return str(tmp_path / 'media' / path.removeprefix('/remote/')) + + monkeypatch.setattr(schema_session, 'execute', counted_execute) + monkeypatch.setattr(module, 'database', schema_session) + monkeypatch.setattr(database_module, 'database', schema_session) + monkeypatch.setattr(module.path_mappings, 'path_replace_instance', mapped) + monkeypatch.setattr(module.path_mappings, 'path_replace_reverse_instance', lambda path, *a: path) + monkeypatch.setattr(module.settings.general, 'use_embedded_subs', False) + monkeypatch.setattr(module.settings.general, 'single_language', False) + monkeypatch.setattr(module.settings.general, 'subfolder', subfolder) + monkeypatch.setattr(module.settings.general, 'subfolder_custom', + '../subs' if subfolder == 'relative' else str(tmp_path / 'subs')) + monkeypatch.setattr(module.core, 'CUSTOM_PATHS', []) + monkeypatch.setattr(module, 'get_language_set', lambda: {Language.fromietf('en')}) + monkeypatch.setattr(module, 'alpha2_from_alpha3', lambda code: Language(code).alpha2) + monkeypatch.setattr(module, 'event_stream', lambda **kw: None) + missing = 'list_missing_subtitles' if media_type == 'series' else 'list_missing_subtitles_movies' + monkeypatch.setattr(module, missing, lambda **kw: None) + monkeypatch.setattr(module.jobs_queue, 'update_job_progress', lambda **kw: None) + monkeypatch.setattr(module.jobs_queue, 'update_job_name', lambda **kw: None) + if media_type == 'series': + schema_session.add(TableShows(id=1, arr_instance_id=7, sonarrSeriesId=10, + title='Show', path='/remote', profileId=None)) + schema_session.flush() + + expected = {} + for row_id in range(1, library_size + 1): + filename = f'Video{row_id:04}.mkv' + remote = f'/remote/{row_id}/{filename}' + video = tmp_path / 'media' / str(row_id) / filename + video.parent.mkdir(parents=True) + video.touch() + folder = tmp_path / 'media' / 'subs' if subfolder == 'relative' else tmp_path / 'subs' + folder.mkdir(exist_ok=True) + output = folder / f'Video{row_id:04}.en.ffsubsync.srt' + output.write_text('1\n00:00:00,000 --> 00:00:01,000\nAn English subtitle.\n') + indexed_path = video.parent / '..' / 'subs' / output.name if subfolder == 'relative' else output + expected[row_id] = [['en:sync-ffsubsync', str(indexed_path), output.stat().st_size]] + if media_type == 'series': + row = TableEpisodes(id=row_id, arr_instance_id=7, series_id=1, sonarrSeriesId=10, + sonarrEpisodeId=row_id, title=filename, path=remote, season=1, + episode=row_id, subtitles='[]') + else: + row = TableMovies(id=row_id, arr_instance_id=7, radarrId=row_id, title=filename, + path=remote, tmdbId=str(row_id), subtitles='[]') + schema_session.add(row) + schema_session.commit() + + full_scan(job_id=1, use_cache=False) + + counts = {'library_size': library_size, 'owner_queries': len(owner_queries), + 'path_mappings': len(mapping_calls)} + print(f'Ownership scan counts: {counts}') + for row_id, subtitles in expected.items(): + assert ast.literal_eval(_subs(schema_session, table, row_id)) == subtitles + assert len(owner_queries) == 2, counts + assert len(mapping_calls) <= 8 * library_size, counts + + # A new scan must see another media type claiming the same generated name. + conflict = tmp_path / 'media' / 'conflict' / 'Video0001.en.mkv' + conflict.parent.mkdir() + conflict.touch() + if media_type == 'series': + row = TableMovies(id=1, arr_instance_id=7, radarrId=1, title='Conflict', tmdbId='1', + path='/remote/conflict/Video0001.en.mkv', subtitles='[]') + else: + schema_session.add(TableShows(id=1, arr_instance_id=7, sonarrSeriesId=10, + title='Show', path='/remote', profileId=None)) + schema_session.flush() + row = TableEpisodes(id=1, arr_instance_id=7, series_id=1, sonarrSeriesId=10, + sonarrEpisodeId=1, title='Conflict', season=1, episode=1, + path='/remote/conflict/Video0001.en.mkv', subtitles='[]') + schema_session.add(row) + schema_session.commit() + owner_queries.clear() + full_scan(job_id=2, use_cache=False) + assert len(owner_queries) == 2, 'independent scans must rebuild their ownership snapshot' + assert ast.literal_eval(_subs(schema_session, table, 1)) == [] + for row_id in range(2, library_size + 1): + assert ast.literal_eval(_subs(schema_session, table, row_id)) == expected[row_id] + + +@pytest.mark.parametrize('subfolder', ['relative', 'absolute']) +@pytest.mark.parametrize('collision', [False, True]) +def test_scan_owner_index_preserves_instance_mapping_and_normalized_conflicts( + schema_session, monkeypatch, tmp_path, subfolder, collision): + from app import database as database_module + from app.config import settings + from subtitles.tools.subsync_engines import SyncOutputOwnerIndex, sync_output_owner_is_unique + from utilities.path_mappings import path_mappings + + video = tmp_path / 'one' / '\u00c9pisode.mkv' + other = tmp_path / 'two' / 'e\u0301PISODE.en.mkv' + folder = tmp_path / 'subs' + for path in (video, other): + path.parent.mkdir() + path.touch() + folder.mkdir() + source = folder / 'e\u0301pisode.en.srt' + source.touch() + monkeypatch.setattr(settings.general, 'subfolder', subfolder) + monkeypatch.setattr(settings.general, 'subfolder_custom', '../subs' if subfolder == 'relative' else str(folder)) + monkeypatch.setattr(database_module, 'database', schema_session) + for owner in (7, 8): + schema_session.add(TableMovies(id=owner, arr_instance_id=owner, radarrId=1, + title='Mapped video', path='/same/remote.mkv', tmdbId=str(owner))) + schema_session.commit() + mappings = [] + + def mapped(path, owner, media_type): + assert (path, media_type) == ('/same/remote.mkv', 'movie') + mappings.append(owner) + return str(other if collision and owner == 8 else video) + + monkeypatch.setattr(path_mappings, 'path_replace_instance', mapped) + index = SyncOutputOwnerIndex() + for _ in range(2): + assert sync_output_owner_is_unique(str(video), str(source), ownership_index=index) is (not collision) + assert sorted(mappings) == [7, 8] + + # Standalone mutation checks must not inherit the earlier scan snapshot. + monkeypatch.setattr(path_mappings, 'path_replace_instance', lambda path, owner, kind: str(other if owner == 8 else video)) + assert sync_output_owner_is_unique(str(video), str(source)) is False + + +def test_scan_owner_index_fails_closed_after_incomplete_library_read(schema_session, monkeypatch, tmp_path): + from app import database as database_module + from app.config import settings + from subtitles.tools.subsync_engines import SyncOutputOwnerIndex, sync_output_owner_is_unique + from utilities.path_mappings import path_mappings + + video = tmp_path / 'Video.mkv' + video.touch() + folder = tmp_path / 'subs' + folder.mkdir() + source = folder / 'Video.en.srt' + source.touch() + monkeypatch.setattr(settings.general, 'subfolder', 'relative') + monkeypatch.setattr(settings.general, 'subfolder_custom', 'subs') + monkeypatch.setattr(database_module, 'database', schema_session) + monkeypatch.setattr(path_mappings, 'path_replace_instance', lambda path, *args: path) + schema_session.add(TableShows(id=1, arr_instance_id=7, sonarrSeriesId=10, title='Show', path=str(tmp_path))) + schema_session.flush() + schema_session.add(TableEpisodes(id=1, arr_instance_id=7, series_id=1, sonarrSeriesId=10, + sonarrEpisodeId=1, title='Video', path=str(video), season=1, episode=1)) + schema_session.commit() + execute = schema_session.execute + queries = [] + + def fail_movie_read(statement, *args, **kwargs): + queries.append(statement) + if list(statement.selected_columns)[0].table.name == 'table_movies': + raise RuntimeError('controlled owner lookup failure') + return execute(statement, *args, **kwargs) + + monkeypatch.setattr(schema_session, 'execute', fail_movie_read) + index = SyncOutputOwnerIndex() + assert sync_output_owner_is_unique(str(video), str(source), ownership_index=index) is False + assert len(queries) == 2 + monkeypatch.setattr(schema_session, 'execute', execute) + assert sync_output_owner_is_unique(str(video), str(source), ownership_index=index) is False + assert sync_output_owner_is_unique(str(video), str(source), ownership_index=SyncOutputOwnerIndex()) is True + + def test_only_the_owning_episode_row_is_indexed(two_series_rows): se, session = two_series_rows diff --git a/tests/bazarr/test_openrouter_model_migration.py b/tests/bazarr/test_openrouter_model_migration.py new file mode 100644 index 0000000000..5e951c8cc4 --- /dev/null +++ b/tests/bazarr/test_openrouter_model_migration.py @@ -0,0 +1,37 @@ +import logging +from types import SimpleNamespace + +import pytest + + +@pytest.mark.parametrize("retired,replacement", [ + ("google/gemini-2.5-flash-preview-05-20", "google/gemini-2.5-flash"), + ("google/gemini-2.5-flash-lite-preview-06-17", "google/gemini-2.5-flash-lite"), + ("google/gemini-2.5-flash-lite-preview-09-2025", "google/gemini-2.5-flash-lite"), + ("google/gemini-3-pro-preview", "google/gemini-3.1-pro-preview"), + (" google/gemini-2.5-flash-preview-05-20 ", "google/gemini-2.5-flash"), +]) +def test_migrate_retired_openrouter_model_rewrites_retired_id(retired, replacement, caplog): + from app import config + + settings = SimpleNamespace(translator=SimpleNamespace(openrouter_model=retired)) + + with caplog.at_level(logging.WARNING): + assert config.migrate_retired_openrouter_model(settings) is True + + assert settings.translator.openrouter_model == replacement + assert any( + record.levelno == logging.WARNING and retired.strip() in record.message and replacement in record.message + for record in caplog.records + ) + assert config.migrate_retired_openrouter_model(settings) is False + + +@pytest.mark.parametrize("model", ["google/gemini-2.5-flash-lite", "custom/model", " custom/model ", "", " "]) +def test_migrate_retired_openrouter_model_leaves_other_values_unchanged(model): + from app import config + + settings = SimpleNamespace(translator=SimpleNamespace(openrouter_model=model)) + + assert config.migrate_retired_openrouter_model(settings) is False + assert settings.translator.openrouter_model == model diff --git a/tests/bazarr/test_openrouter_provider_routing.py b/tests/bazarr/test_openrouter_provider_routing.py new file mode 100644 index 0000000000..e8266b5ed2 --- /dev/null +++ b/tests/bazarr/test_openrouter_provider_routing.py @@ -0,0 +1,201 @@ +# coding=utf-8 +"""OpenRouter provider routing reaches the AI Subtitle Translator as ``config.provider``. + +The sidecar sorts OpenRouter providers by throughput unless it is told otherwise, +so a user who wants the cheapest provider (OpenRouter's ``:floor``) or the +priority tier (``:nitro``) needs a setting that Bazarr forwards on every request: +the async translate job and the manual content endpoint alike. +""" +from types import SimpleNamespace + +import pytest +from flask import Flask + +from app import config +from subtitles.tools.translate.services import openrouter_translator + +ROUTING_VALUES = ['throughput', 'nitro', 'price', 'floor', 'latency', 'default'] + + +def _routing_validator(): + return next( + v for v in config.validators + if v.names == ('translator.openrouter_provider_routing',) + ) + + +def test_setting_defaults_to_throughput_and_pins_the_allowed_values(): + validator = _routing_validator() + assert validator.default == 'throughput' + assert list(validator.operations.get('is_in')) == ROUTING_VALUES + assert config.settings.translator.openrouter_provider_routing == 'throughput' + + +def _sidecar_health(monkeypatch, version): + """Answer the sidecar's /health probe with the given version (None = unreachable).""" + openrouter_translator.reset_sidecar_version_cache() + calls = [] + + def _get(url, *args, **kwargs): + calls.append(url) + if version is None: + raise openrouter_translator.requests.exceptions.ConnectionError('down') + return SimpleNamespace(status_code=200, json=lambda: {'status': 'healthy', 'version': version}) + + monkeypatch.setattr(openrouter_translator.requests, 'get', _get) + return calls + + +@pytest.mark.parametrize('value', ROUTING_VALUES) +def test_build_provider_config_forwards_the_setting(monkeypatch, value): + _sidecar_health(monkeypatch, '1.3.4') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', value) + assert openrouter_translator.build_provider_config() == {'sort': value} + + +def test_build_provider_config_falls_back_to_throughput_for_unknown_values(monkeypatch): + _sidecar_health(monkeypatch, '1.3.4') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', 'cheapest') + assert openrouter_translator.build_provider_config() == {'sort': 'throughput'} + + +@pytest.mark.parametrize('value, expected', [ + ('floor', 'price'), + ('nitro', 'throughput'), + ('default', 'throughput'), + ('price', 'price'), + ('latency', 'latency'), +]) +def test_older_sidecars_get_the_plain_sort(monkeypatch, value, expected): + # Before 1.3.4 the sidecar forwards whatever it gets as provider.sort, and + # OpenRouter rejects nitro/floor/default there. + _sidecar_health(monkeypatch, '1.3.3') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', value) + assert openrouter_translator.build_provider_config() == {'sort': expected} + + +def test_unreachable_sidecar_gets_the_plain_sort(monkeypatch): + _sidecar_health(monkeypatch, None) + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', 'floor') + assert openrouter_translator.build_provider_config() == {'sort': 'price'} + + +def test_sidecar_version_is_probed_once_per_url(monkeypatch): + calls = _sidecar_health(monkeypatch, '1.3.4') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_url', 'http://sidecar:8765/') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', 'floor') + assert openrouter_translator.build_provider_config() == {'sort': 'floor'} + assert openrouter_translator.build_provider_config() == {'sort': 'floor'} + assert calls == ['http://sidecar:8765/health'] + + +@pytest.mark.parametrize('version', ['1.3.3', '1.3.4', None]) +@pytest.mark.parametrize('suffix, setting, expected', [ + ('floor', 'throughput', 'price'), + ('nitro', 'floor', 'throughput'), + ('floor', 'floor', 'price'), +]) +def test_typed_routing_suffix_aligns_the_sort(monkeypatch, version, suffix, setting, expected): + # The slug says how to route; the sort Bazarr sends must never contradict it, + # whatever the setting says and whatever the sidecar version is. + _sidecar_health(monkeypatch, version) + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_model', f'deepseek/deepseek-v4-flash:{suffix}') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', setting) + assert openrouter_translator.build_provider_config() == {'sort': expected} + + +def test_other_model_variants_do_not_touch_the_sort(monkeypatch): + _sidecar_health(monkeypatch, '1.3.4') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_model', 'deepseek/deepseek-chat:thinking') + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', 'floor') + assert openrouter_translator.build_provider_config() == {'sort': 'floor'} + + +@pytest.mark.parametrize('version', ['1.3.4', '1.4.0', '2.0.0', '1.3.10', '1.3.4-rc1']) +def test_versions_from_1_3_4_support_the_shortcuts(monkeypatch, version): + _sidecar_health(monkeypatch, version) + monkeypatch.setattr(openrouter_translator.settings.translator, 'openrouter_provider_routing', 'nitro') + assert openrouter_translator.build_provider_config() == {'sort': 'nitro'} + + +def _build_service(): + return openrouter_translator.OpenRouterTranslatorService( + source_srt_file='input.srt', + dest_srt_file='output.srt', + lang_obj=None, + to_lang='hun', + from_lang='en', + media_type='movie', + video_path='/tmp/video.mkv', + orig_to_lang='hu', + forced=False, + hi=False, + sonarr_series_id=None, + sonarr_episode_id=None, + radarr_id=7, + ) + + +def _translator_settings(monkeypatch, routing): + # Every key below exists in the settings box already (the validators give them + # defaults), so a plain setattr restores cleanly. Never patch a settings box with + # create=True: the restore deletes the key and later tests lose it. + for key, value in { + 'openrouter_url': 'http://sidecar:8765', + 'openrouter_api_key': 'sk-or-key', + 'openrouter_encryption_key': '', + 'openrouter_model': 'deepseek/deepseek-v4-flash', + 'openrouter_temperature': 0.3, + 'openrouter_max_concurrent': 2, + 'openrouter_parallel_batches': 4, + 'openrouter_reasoning': 'disabled', + 'openrouter_provider_routing': routing, + }.items(): + monkeypatch.setattr(openrouter_translator.settings.translator, key, value) + + +def test_translate_job_sends_the_routing_in_the_config(mocker, monkeypatch): + _translator_settings(monkeypatch, 'floor') + _sidecar_health(monkeypatch, '1.3.4') + mocker.patch.object(openrouter_translator, 'get_title', return_value='Some Movie') + mocker.patch.object(openrouter_translator, 'language_from_alpha2', lambda code: code) + mocker.patch.object(openrouter_translator, 'language_from_alpha3', lambda code: code) + mocker.patch.object(openrouter_translator, 'get_translator_auth_headers', return_value={}) + post = mocker.patch.object( + openrouter_translator.requests, + 'post', + return_value=SimpleNamespace(status_code=200, json=lambda: {'jobId': 'job-1'}), + ) + service = _build_service() + mocker.patch.object(service, '_poll_job', return_value=[{'index': 0, 'content': 'Szia'}]) + + result = service._submit_and_poll(['Hi']) + + assert result == [{'index': 0, 'content': 'Szia'}] + payload = post.call_args.kwargs['json'] + assert payload['config']['provider'] == {'sort': 'floor'} + assert payload['config']['model'] == 'deepseek/deepseek-v4-flash' + + +def test_content_endpoint_sends_the_routing_in_the_config(mocker, monkeypatch): + from api.translator import translator as api_mod + + _translator_settings(monkeypatch, 'nitro') + _sidecar_health(monkeypatch, '1.3.4') + mocker.patch.object(api_mod, 'get_translator_auth_headers', return_value={}) + post = mocker.patch.object( + api_mod.requests, + 'post', + return_value=SimpleNamespace(status_code=200, json=lambda: {'jobId': 'job-2'}), + ) + + app = Flask(__name__) + with app.test_request_context( + '/api/translator/jobs', method='POST', json={'lines': ['Hi'], 'targetLanguage': 'hu'} + ): + body, status = api_mod.TranslatorJobs.post.__wrapped__(api_mod.TranslatorJobs()) + + assert status == 200 + assert body == {'jobId': 'job-2'} + payload = post.call_args.kwargs['json'] + assert payload['config']['provider'] == {'sort': 'nitro'} diff --git a/tests/bazarr/test_openrouter_translator.py b/tests/bazarr/test_openrouter_translator.py new file mode 100644 index 0000000000..ba5ea751a4 --- /dev/null +++ b/tests/bazarr/test_openrouter_translator.py @@ -0,0 +1,245 @@ +from pathlib import Path +from types import SimpleNamespace + +import pysubs2 +import pytest + +from subtitles.tools.translate.services import openrouter_translator as module + + +@pytest.fixture +def service(tmp_path, monkeypatch): + source = tmp_path / 'source.en.srt' + source.write_text('1\n00:00:01,000 --> 00:00:02,000\nOne\n\n' + '2\n00:00:03,000 --> 00:00:04,000\nTwo\n\n' + '3\n00:00:05,000 --> 00:00:06,000\nThree\n', encoding='utf-8') + monkeypatch.setattr(module.settings.translator, 'translator_info', False) + monkeypatch.setattr(module, 'get_title', lambda **kwargs: 'Example') + monkeypatch.setattr(module, 'get_translator_auth_headers', lambda: {}) + monkeypatch.setattr(module, 'language_from_alpha2', lambda code: {'en': 'English', 'hu': 'Hungarian'}[code]) + monkeypatch.setattr(module, 'language_from_alpha3', lambda code: 'Hungarian') + monkeypatch.setattr(module.requests, 'post', lambda *args, **kwargs: SimpleNamespace( + status_code=200, json=lambda: {'jobId': 'fixture-job'})) + monkeypatch.setattr(module, 'show_progress', lambda **kwargs: None) + monkeypatch.setattr(module, 'hide_progress', lambda **kwargs: None) + monkeypatch.setattr(module.jobs_queue, 'update_job_progress', lambda **kwargs: None) + translator = module.OpenRouterTranslatorService( + source_srt_file=str(source), dest_srt_file=str(tmp_path / 'output.hu.srt'), + lang_obj=None, to_lang='hun', from_lang='en', media_type='episode', + video_path=str(tmp_path / 'video.mkv'), orig_to_lang='hu', forced=False, hi=False, + sonarr_series_id=1, sonarr_episode_id=2, radarr_id=None, + ) + monkeypatch.setattr(translator, '_get_api_key_value', lambda: '') + return translator + + +@pytest.fixture +def events(monkeypatch): + captured = SimpleNamespace(messages=[], history=[]) + monkeypatch.setattr(module, 'show_message', captured.messages.append) + monkeypatch.setattr(module, 'history_log', lambda **kwargs: captured.history.append(kwargs)) + monkeypatch.setattr(module, 'history_log_movie', lambda **kwargs: captured.history.append(kwargs)) + return captured + + +def _response(monkeypatch, status, result=None, **fields): + payload = {'status': status, 'result': result, **fields} + monkeypatch.setattr(module.requests, 'get', lambda *args, **kwargs: SimpleNamespace( + status_code=200, json=lambda: payload)) + + +@pytest.mark.parametrize('structured', [True, False]) +@pytest.mark.parametrize('status', ['partial', 'completed']) +def test_poll_preserves_usable_lines(service, events, monkeypatch, status, structured): + lines = [{'position': 2, 'line': 'Három'}, {'position': 0, 'line': 'Egy'}] + _response(monkeypatch, status, {'lines': lines} if structured else lines, + error='One batch failed') + + assert service._poll_job('http://fixture', 'job', 3) == lines + if status == 'partial': + assert any('partial' in message.lower() for message in events.messages) + + +@pytest.mark.parametrize('status', ['partial', 'completed']) +@pytest.mark.parametrize('result', [None, [], {}, {'lines': []}, {'lines': None}, + {'lines': 'invalid'}, 'invalid']) +def test_poll_rejects_missing_or_empty_lines(service, events, monkeypatch, status, result): + _response(monkeypatch, status, result) + assert service._poll_job('http://fixture', 'job', 3) is None + + +@pytest.mark.parametrize('status', ['failed', 'cancelled']) +def test_poll_does_not_return_failed_output(service, events, monkeypatch, status): + _response(monkeypatch, status, [{'position': 0, 'line': 'Egy'}], error='Unavailable') + assert service._poll_job('http://fixture', 'job', 3) is None + + +@pytest.mark.parametrize('media_type', ['episode', 'movie']) +def test_partial_translation_writes_ordered_cues_and_records_warning(service, events, monkeypatch, media_type): + service.media_type = media_type + service.radarr_id = 3 if media_type == 'movie' else None + source = Path(service.source_srt_file).read_bytes() + _response(monkeypatch, 'partial', {'lines': [ + {'position': 2, 'line': 'Három'}, {'position': 0, 'line': 'Egy\nMásik'}, + ]}, error='One batch failed: malformed response') + + assert service.translate(job_id='bazarr-job') == service.dest_srt_file + + output = pysubs2.load(service.dest_srt_file) + assert [cue.plaintext for cue in output] == ['Egy\nMásik', 'Two', 'Három'] + assert [(cue.start, cue.end) for cue in output] == [(1000, 2000), (3000, 4000), (5000, 6000)] + assert Path(service.source_srt_file).read_bytes() == source + assert len(events.history) == 1 + message = events.history[0]['result'].message + assert 'partial' in message.lower() + assert 'source language' in message.lower() + assert 'malformed response' in message + assert any('partial' in message.lower() for message in events.messages) + + +@pytest.mark.parametrize('lines', [ + [], [{'position': 0}], [{'position': -1, 'line': 'Invalid'}], + [{'position': 3, 'line': 'Invalid'}], [{'position': True, 'line': 'Invalid'}], + [{'position': '0', 'line': 'Invalid'}], [{'position': 0, 'line': None}], + [{'position': 0, 'line': ''}], [{'position': 0, 'line': ' '}], + [{'position': 0, 'line': 'Valid'}, {'position': 0, 'line': 'Duplicate'}], + [{'position': 0, 'line': 'Valid'}, {'position': 1, 'line': ['Invalid']}], +]) +@pytest.mark.parametrize('status', ['partial', 'completed']) +def test_invalid_results_do_not_overwrite_files(service, events, monkeypatch, status, lines): + destination = Path(service.dest_srt_file) + destination.write_bytes(b'existing subtitle') + source = Path(service.source_srt_file).read_bytes() + _response(monkeypatch, status, {'lines': lines}) + + assert service.translate() is False + + assert destination.read_bytes() == b'existing subtitle' + assert Path(service.source_srt_file).read_bytes() == source + assert events.history == [] + + +def test_partial_detail_is_bounded_and_cleared_for_next_operation(service, events, monkeypatch): + lines = [{'position': 0, 'line': 'Egy'}] + _response(monkeypatch, 'partial', lines, error='Failure detail ' + 'x' * 2000) + assert service.translate() == service.dest_srt_file + assert len(events.history[-1]['result'].message) < 1000 + + complete_lines = [{'position': i, 'line': line} for i, line in enumerate(['Egy', 'KettÅ‘', 'Három'])] + _response(monkeypatch, 'completed', complete_lines) + assert service.translate() == service.dest_srt_file + assert 'partial' not in events.history[-1]['result'].message.lower() + assert not service.partial_error + + _response(monkeypatch, 'partial', lines, error='Failure detail') + assert service.translate() == service.dest_srt_file + _response(monkeypatch, 'failed', error='Unavailable') + assert service.translate() is False + assert not service.partial_error + + +def test_partial_without_error_detail_still_marks_the_saved_output(service, events, monkeypatch): + monkeypatch.setattr(module.settings.translator, 'translator_info', True) + _response(monkeypatch, 'partial', [{'position': 0, 'line': 'Egy'}], error=' ') + + assert service.translate() == service.dest_srt_file + assert 'partially translated' in pysubs2.load(service.dest_srt_file)[-1].plaintext.lower() + assert 'partial' in events.history[-1]['result'].message.lower() + + +def test_empty_source_does_not_report_a_saved_translation(service, events): + Path(service.source_srt_file).write_text('', encoding='utf-8') + assert service.translate() is False + assert not Path(service.dest_srt_file).exists() + assert events.history == [] + + +@pytest.mark.parametrize('valid', [True, False]) +def test_sync_fallback_uses_the_same_result_validation(service, events, monkeypatch, valid): + lines = [{'position': 0, 'line': 'Egy'}, {'position': 1, 'line': ''}] + if not valid: + lines[0]['position'] = 9 + responses = iter([ + SimpleNamespace(status_code=404), + SimpleNamespace(status_code=200, json=lambda: lines), + ]) + monkeypatch.setattr(module.requests, 'post', lambda *args, **kwargs: next(responses)) + + result = service.translate() + if valid: + assert result == service.dest_srt_file + assert [cue.plaintext for cue in pysubs2.load(result)] == ['Egy', 'Two', 'Three'] + assert 'partial' in events.history[-1]['result'].message.lower() + else: + assert result is False + assert not Path(service.dest_srt_file).exists() + assert events.history == [] + + +@pytest.mark.parametrize('status', ['partial', 'completed']) +def test_save_failure_preserves_destination_and_cleans_temporary_file(service, events, monkeypatch, status): + destination = Path(service.dest_srt_file) + destination.write_bytes(b'existing subtitle') + before = set(destination.parent.iterdir()) + _response(monkeypatch, status, [{'position': 0, 'line': 'Egy'}]) + + def fail_save(subtitles, filename, *args, **kwargs): + Path(filename).write_bytes(b'incomplete write') + raise OSError('write failed') + + monkeypatch.setattr(pysubs2.SSAFile, 'save', fail_save) + + assert service.translate() is False + assert destination.read_bytes() == b'existing subtitle' + assert set(destination.parent.iterdir()) == before + assert events.history == [] + + +@pytest.mark.parametrize('synchronous', [False, True]) +def test_missing_or_blank_cues_are_reported_as_partial(service, events, monkeypatch, synchronous): + lines = [{'position': 0, 'line': 'Egy'}, {'position': 1, 'line': ' '}] + monkeypatch.setattr(module.settings.translator, 'translator_info', True) + if synchronous: + responses = iter([SimpleNamespace(status_code=404), + SimpleNamespace(status_code=200, json=lambda: lines)]) + monkeypatch.setattr(module.requests, 'post', lambda *args, **kwargs: next(responses)) + else: + _response(monkeypatch, 'completed', {'lines': lines}) + + assert service.translate() == service.dest_srt_file + output = pysubs2.load(service.dest_srt_file) + assert [cue.plaintext for cue in output[:3]] == ['Egy', 'Two', 'Three'] + assert 'partially translated' in output[-1].plaintext.lower() + assert 'partial' in events.history[-1]['result'].message.lower() + assert any('partial' in message.lower() for message in events.messages) + + +def test_long_destination_filename_retains_atomic_saving_and_permissions(service, events, monkeypatch): + destination = Path(service.dest_srt_file).with_name('x' * 220 + '.hu.srt') + destination.write_bytes(b'existing subtitle') + destination.chmod(0o640) + service.dest_srt_file = str(destination) + before = set(destination.parent.iterdir()) + lines = [{'position': i, 'line': line} for i, line in enumerate(['Egy', 'KettÅ‘', 'Három'])] + _response(monkeypatch, 'completed', lines) + + assert service.translate() == str(destination) + assert [cue.plaintext for cue in pysubs2.load(destination)] == ['Egy', 'KettÅ‘', 'Három'] + assert destination.stat().st_mode & 0o777 == 0o640 + assert set(destination.parent.iterdir()) == before + assert 'partial' not in events.history[-1]['result'].message.lower() + + +def test_existing_staging_file_is_never_overwritten_or_removed(service, events, monkeypatch): + destination = Path(service.dest_srt_file) + destination.write_bytes(b'existing subtitle') + staging = destination.with_name('.bazarr-write-fixture-collision.srt') + staging.write_bytes(b'other operation') + from subtitles.tools import subsync_engines + monkeypatch.setattr(subsync_engines.uuid, 'uuid4', lambda: SimpleNamespace(hex='fixture-collision')) + _response(monkeypatch, 'completed', [{'position': 0, 'line': 'Egy'}]) + + assert service.translate() is False + assert staging.read_bytes() == b'other operation' + assert destination.read_bytes() == b'existing subtitle' + assert not events.history diff --git a/tests/bazarr/test_openrouter_translator_partial.py b/tests/bazarr/test_openrouter_translator_partial.py new file mode 100644 index 0000000000..a720fa5491 --- /dev/null +++ b/tests/bazarr/test_openrouter_translator_partial.py @@ -0,0 +1,250 @@ +import logging +from pathlib import Path + +import pytest + +from subtitles.tools.translate.services import openrouter_translator + + +def _build_service(source_srt_file="input.srt", dest_srt_file="output.srt", media_type="episode"): + return openrouter_translator.OpenRouterTranslatorService( + source_srt_file=source_srt_file, + dest_srt_file=dest_srt_file, + lang_obj=None, + to_lang="hun", + from_lang="en", + media_type=media_type, + video_path="/tmp/video.mkv", + orig_to_lang="hu", + forced=False, + hi=False, + sonarr_series_id=1, + sonarr_episode_id=2, + radarr_id=3, + ) + + +@pytest.fixture(autouse=True) +def _mock_dependencies(mocker): + mocker.patch.object(openrouter_translator.requests, "get", return_value=mocker.Mock(status_code=200)) + mocker.patch.object(openrouter_translator.time, "sleep") + for name in ( + "show_progress", "hide_progress", "show_message", "jobs_queue", "history_log", "history_log_movie", + "add_translator_info", "create_process_result", + ): + mocker.patch.object(openrouter_translator, name) + + +@pytest.mark.parametrize("structured", [True, False]) +def test_poll_job_keeps_partial_lines_and_bounded_error(structured, caplog): + service = _build_service() + lines = [{"position": 0, "line": "Szia"}, {"position": 1, "line": "Source text"}] + error = "1 of 58 batches failed: " + "Failed to parse JSON. " * 100 + detail = error.strip()[:500] + result = {"lines": lines, "model_used": "google/gemini-2.5-flash-lite", "tokens_used": 100} if structured else lines + openrouter_translator.requests.get.return_value.json.return_value = { + "status": "partial", "result": result, "error": error, + } + + with caplog.at_level(logging.WARNING, logger=openrouter_translator.__name__): + assert service._poll_job("http://translator", "job-1", 2, bazarr_job_id="bazarr-job-1") == lines + + assert service.partial_error == detail + openrouter_translator.hide_progress.assert_called_once_with(id="translate_progress_output.srt") + openrouter_translator.show_message.assert_called_once_with( + f"Translation is partial. Some lines may remain in the source language. {detail}" + ) + assert (openrouter_translator.__name__, logging.WARNING, + f"Translation partially completed: {detail}") in caplog.record_tuples + openrouter_translator.time.sleep.assert_not_called() + + +@pytest.mark.parametrize("result", [None, {}, [], {"lines": []}, {"lines": "invalid"}, "invalid", 42]) +def test_poll_job_rejects_partial_without_lines(result, caplog): + service = _build_service() + error = "No lines translated" + openrouter_translator.requests.get.return_value.json.return_value = { + "status": "partial", "result": result, "error": error, + } + + assert service._poll_job("http://translator", "job-1", 1) is None + openrouter_translator.hide_progress.assert_called_once_with(id="translate_progress_output.srt") + openrouter_translator.show_message.assert_called_once_with(f"Translation failed (partial): {error}") + assert (openrouter_translator.__name__, logging.ERROR, + f"Translation partially failed: {error}") in caplog.record_tuples + + +@pytest.mark.parametrize("structured", [True, False]) +def test_poll_job_returns_completed_lines(structured): + service = _build_service() + lines = [{"position": 0, "line": "Szia"}] + result = {"lines": lines, "model_used": "google/gemini-2.5-flash-lite", "tokens_used": 100} if structured else lines + openrouter_translator.requests.get.return_value.json.return_value = {"status": "completed", "result": result} + + assert service._poll_job("http://translator", "job-1", 1) == lines + assert service.partial_error is None + openrouter_translator.hide_progress.assert_called_once_with(id="translate_progress_output.srt") + openrouter_translator.show_message.assert_not_called() + + +@pytest.mark.parametrize("result", [None, [], {"lines": []}, {"lines": "invalid"}, {"model_used": "model"}, "invalid", 42]) +def test_poll_job_rejects_completed_without_lines(result): + service = _build_service() + openrouter_translator.requests.get.return_value.json.return_value = {"status": "completed", "result": result} + + assert service._poll_job("http://translator", "job-1", 1) is None + + +@pytest.mark.parametrize("media_type", ["episode", "movie"]) +def test_translate_saves_partial_subtitles_and_marks_history(tmp_path, mocker, media_type): + source = tmp_path / "input.srt" + destination = tmp_path / "output.srt" + source.write_text( + "1\n00:00:00,000 --> 00:00:02,000\nHello\n\n" + "2\n00:00:03,000 --> 00:00:05,000\nSource text\n\n", + encoding="utf-8", + ) + service = _build_service(str(source), str(destination), media_type) + error = "1 of 2 batches failed: " + "Failed to parse JSON. " * 15 + + def _partial_result(lines_list, bazarr_job_id=None): + assert lines_list == ["Hello", "Source text"] + assert bazarr_job_id == "bazarr-job-1" + service._mark_partial(error) + return [{"position": 0, "line": "Szia"}, {"position": 1, "line": "Source text"}] + + mocker.patch.object(service, "_submit_and_poll", side_effect=_partial_result) + mocker.patch.object(openrouter_translator, "language_from_alpha2", return_value="English") + mocker.patch.object(openrouter_translator, "language_from_alpha3", return_value="Hungarian") + + assert service.translate(job_id="bazarr-job-1") == str(destination) + assert destination.read_text(encoding="utf-8") == ( + "1\n00:00:00,000 --> 00:00:02,000\nSzia\n\n" + "2\n00:00:03,000 --> 00:00:05,000\nSource text\n\n" + ) + openrouter_translator.create_process_result.assert_called_once() + history_message = openrouter_translator.create_process_result.call_args.args[0] + assert "partially translated" in history_message + assert history_message.endswith(error.strip()[:500]) + openrouter_translator.add_translator_info.assert_called_once() + staged, footer = openrouter_translator.add_translator_info.call_args.args + assert Path(staged).parent == destination.parent + assert Path(staged).name.startswith(".bazarr-write-") + assert footer == "# Subtitles partially translated with AI Subtitle Translator #" + result = openrouter_translator.create_process_result.return_value + if media_type == "episode": + openrouter_translator.history_log.assert_called_once_with( + action=6, sonarr_series_id=1, sonarr_episode_id=2, result=result, + ) + openrouter_translator.history_log_movie.assert_not_called() + else: + openrouter_translator.history_log_movie.assert_called_once_with(action=6, radarr_id=3, result=result) + openrouter_translator.history_log.assert_not_called() + + +class _FakeClock: + def __init__(self): + self.elapsed = 0 + + def monotonic(self): + return 1000 + self.elapsed + + def advance(self, seconds): + self.elapsed += seconds + + +@pytest.fixture +def poll_clock(mocker, _mock_dependencies): + clock = _FakeClock() + mocker.patch.object(openrouter_translator.time, "monotonic", side_effect=clock.monotonic) + openrouter_translator.time.sleep.side_effect = clock.advance + return clock + + +@pytest.mark.parametrize("status", ["processing", "queued"]) +def test_poll_job_completes_after_more_than_thirty_minutes(poll_clock, mocker, status): + service = _build_service() + lines = [{"position": 0, "line": "Szia"}] + response = mocker.Mock(status_code=200) + + def _get_status(*args, **kwargs): + if poll_clock.elapsed < 40 * 60: + response.json.return_value = {"status": status} + else: + response.json.return_value = {"status": "completed", "result": {"lines": lines}} + return response + + openrouter_translator.requests.get.side_effect = _get_status + + assert service._poll_job("http://translator", "job-1", 1) == lines + assert poll_clock.elapsed == 40 * 60 + openrouter_translator.hide_progress.assert_called_once_with(id="translate_progress_output.srt") + openrouter_translator.show_message.assert_not_called() + + +@pytest.mark.parametrize("failure", ["http_500", "request_exception"]) +def test_poll_job_stops_when_status_endpoint_is_unreachable(poll_clock, mocker, caplog, failure): + service = _build_service() + response = mocker.Mock(status_code=500) + + def _get_status(*args, **kwargs): + poll_clock.advance(8) + if failure == "request_exception": + raise openrouter_translator.requests.exceptions.RequestException("Service unavailable") + return response + + openrouter_translator.requests.get.side_effect = _get_status + + assert service._poll_job("http://translator", "job-1", 1) is None + assert poll_clock.elapsed == 10 * 60 + openrouter_translator.hide_progress.assert_called_once_with(id="translate_progress_output.srt") + openrouter_translator.show_message.assert_called_once() + assert any( + record.levelno == logging.ERROR + and "job-1" in record.message + and "unreachable for 10 minutes" in record.message + for record in caplog.records + ) + + +@pytest.mark.parametrize("failure", ["http_500", "request_exception"]) +def test_poll_job_resets_unreachable_clock_after_http_200(poll_clock, mocker, failure): + service = _build_service() + lines = [{"position": 0, "line": "Szia"}] + response = mocker.Mock(status_code=200) + failed_response = mocker.Mock(status_code=500) + + def _get_status(*args, **kwargs): + if poll_clock.elapsed == 8 * 60: + response.json.return_value = {"status": "processing"} + return response + if poll_clock.elapsed >= 16 * 60: + response.json.return_value = {"status": "completed", "result": {"lines": lines}} + return response + if failure == "request_exception": + raise openrouter_translator.requests.exceptions.RequestException("Service unavailable") + return failed_response + + openrouter_translator.requests.get.side_effect = _get_status + + assert service._poll_job("http://translator", "job-1", 1) == lines + assert poll_clock.elapsed == 16 * 60 + openrouter_translator.hide_progress.assert_called_once_with(id="translate_progress_output.srt") + openrouter_translator.show_message.assert_not_called() + + +def test_poll_job_stops_at_twelve_hour_hard_cap(poll_clock, caplog): + service = _build_service() + openrouter_translator.requests.get.return_value.json.return_value = {"status": "processing"} + openrouter_translator.time.sleep.side_effect = lambda seconds: poll_clock.advance(5 * 60) + + assert service._poll_job("http://translator", "job-1", 1) is None + assert poll_clock.elapsed == 12 * 3600 + openrouter_translator.hide_progress.assert_called_once_with(id="translate_progress_output.srt") + openrouter_translator.show_message.assert_called_once() + assert any( + record.levelno == logging.ERROR + and "job-1" in record.message + and "hard cap" in record.message + for record in caplog.records + ) diff --git a/tests/bazarr/test_provider_hub.py b/tests/bazarr/test_provider_hub.py index 945d8470ad..8c41173ba8 100644 --- a/tests/bazarr/test_provider_hub.py +++ b/tests/bazarr/test_provider_hub.py @@ -445,6 +445,7 @@ def test_worker_download_auto_selects_single_archive_member(): def test_worker_download_rejects_missing_member_and_bad_archive(): from provider_hub.protocol import WorkerProtocolError, worker_download_to_content + from subliminal_patch.exceptions import SubtitleCandidateRejected candidate = _hub_candidate("sub-bad") buffer = io.BytesIO() @@ -452,7 +453,7 @@ def test_worker_download_rejects_missing_member_and_bad_archive(): archive.writestr("Show.S01E01.srt", b"1\n00:00:01,000 --> 00:00:02,000\nHi\n") archive_bytes = buffer.getvalue() - with pytest.raises(WorkerProtocolError): + with pytest.raises(SubtitleCandidateRejected, match="Pinned subtitle member is absent"): worker_download_to_content( candidate, { diff --git a/tests/bazarr/test_provider_hub_archive_candidates.py b/tests/bazarr/test_provider_hub_archive_candidates.py new file mode 100644 index 0000000000..84a30c01d4 --- /dev/null +++ b/tests/bazarr/test_provider_hub_archive_candidates.py @@ -0,0 +1,922 @@ +"""Archive mismatch rejects one download without disabling its provider.""" +import base64 +import hashlib +import io +import logging +import stat +from types import SimpleNamespace +import zipfile + +import pytest +from subzero.language import Language + +from provider_hub import protocol, registry, worker_runner +from subliminal_patch import core, core_persistent +from subliminal_patch.providers.utils import get_archive_from_bytes, get_subtitle_from_archive + + +SRT = b"1\n00:00:01,000 --> 00:00:02,000\nFixture\n" + + +def archive_payload(names, **extra): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + members = names.items() if isinstance(names, dict) else ((name, SRT) for name in names) + for name, content in members: + archive.writestr(name, content) + body = buffer.getvalue() + return {"archive_b64": base64.b64encode(body).decode("ascii"), + "archive_sha256": hashlib.sha256(body).hexdigest(), "episode": 1, **extra} + + +def formatted_archive_payload(kind, members, **extra): + if kind == "zip": + return archive_payload(members, **extra) + buffer = io.BytesIO() + if kind == "rar": + import struct + import zlib + + def block(block_type, flags, body): + header = struct.pack(" 00:00:02,000\nx\n" @@ -85,7 +89,7 @@ def test_select_member_pin_unknown_member_raises(): def test_select_member_reject_raises(): body = _zip(["show.eng.srt", "show.fre.srt"]) - with pytest.raises(proto.WorkerProtocolError): + with pytest.raises(SubtitleCandidateRejected, match="No matching subtitle language"): _run(body, _archive_payload(body, select_member=True), lambda members: {"member": None, "decision": "reject"}) @@ -134,25 +138,254 @@ def select_archive_member(self, provider_payload, language, members, config): assert seen["payload"]["url"] == "x" +@pytest.mark.parametrize("context,expected", [ + ({}, {"season": 99, "episode": 99}), + ({"season": None}, {"season": None, "episode": 99}), + ({"episode": None}, {"season": 99, "episode": None}), + ({"season": None, "episode": None}, {"season": None, "episode": None}), + ({"season": 3}, {"season": 3, "episode": 99}), + ({"episode": 1}, {"season": 99, "episode": 1}), + ({"season": 3, "episode": None}, {"season": 3, "episode": None}), + ({"season": None, "episode": 1}, {"season": None, "episode": 1}), + ({"season": 3, "episode": 1}, {"season": 3, "episode": 1}), +]) +def test_selector_context_distinguishes_absent_and_null_without_mutating_payload(context, expected): + from provider_hub import worker_runner + + opaque = {"season": 99, "episode": 99, "id": "stored-candidate"} + + class Provider: + def select_archive_member(self, provider_payload, **kwargs): + assert provider_payload == {**expected, "id": "stored-candidate"} + provider_payload["id"] = "selector-local" + return {"decision": "defer"} + + result = worker_runner._handle(Provider(), "select_archive_member", { + "provider_payload": opaque, **context, + }) + + assert result == {"decision": "defer", "member": None} + assert opaque == {"season": 99, "episode": 99, "id": "stored-candidate"} + + +@pytest.mark.parametrize("context,expected", [({}, 99), ({"absolute_episode": None}, None), + ({"absolute_episode": 0}, 0), ({"absolute_episode": 49}, 49)]) +def test_selector_absolute_context_preserves_presence_and_payload_copy(context, expected): + from provider_hub import worker_runner + + opaque = {"absolute_episode": 99, "id": "stored-candidate"} + + class Provider: + def select_archive_member(self, provider_payload, **kwargs): + assert provider_payload["absolute_episode"] == expected + provider_payload["absolute_episode"] = 50 + return {"decision": "defer"} + + assert worker_runner._handle(Provider(), "select_archive_member", { + "provider_payload": opaque, **context, + }) == {"decision": "defer", "member": None} + assert opaque == {"absolute_episode": 99, "id": "stored-candidate"} + + +@pytest.fixture +def selector_worker(tmp_path): + from provider_hub.worker import ProviderWorkerClient, worker_command + + # The bundle uses only stdlib and the actual isolated worker entry point. + (tmp_path / "provider.py").write_text(''' +class Provider: + def search(self, video, languages, config): + return [{ + "id": "fixture", "language": languages[0], + "provider_payload": {"archive": config["archive"], **config["opaque"]}, + "display": {"season": 99, "episode": 99}, + }] + + def download(self, provider_payload, language, config): + return provider_payload["archive"] + + def select_archive_member(self, provider_payload, language, members, config): + if config.get("absolute_selector"): + number = provider_payload.get("absolute_episode") + if number is None: + number = provider_payload.get("episode") + member = str(number) + ".en.srt" + if member not in members: + return {"decision": "reject"} + decision = config["decision"] + return {"decision": decision, "member": member if decision == "pin" else None} + # A provider may use these fields to accept, pin or defer an archive. + # Incorrect context must be observable even when the host would defer. + context = {key: provider_payload.get(key) for key in ("season", "episode")} + if context != config["requested"]: + return {"decision": "pin", "member": "Stale.S99E99.srt"} + decision = config["decision"] + return {"decision": decision, "member": members[0] if decision == "pin" else None} +''', encoding="utf-8") + runner = Path(__file__).parents[2] / "bazarr" / "provider_hub" / "worker_runner.py" + client = ProviderWorkerClient( + worker_command(sys.executable, runner), cwd=tmp_path, + env={"BAZARR_PROVIDER_HUB_BUNDLE": str(tmp_path), + "BAZARR_PROVIDER_HUB_MANIFEST": json.dumps({"entry_module": "provider", "entry_class": "Provider"})}, + ) + try: + yield client + finally: + process = client.process + client.stop() + if process is not None: + assert process.poll() is not None + + +@pytest.mark.parametrize("context,expected", [ + ({}, {"season": 99, "episode": 99}), + ({"season": None}, {"season": None, "episode": 99}), + ({"episode": None}, {"season": 99, "episode": None}), +]) +def test_real_worker_preserves_legacy_context_only_for_absent_keys(selector_worker, context, expected): + response = selector_worker.select_archive_member({ + "provider_payload": {"season": 99, "episode": 99}, + "members": ["Requested.srt", "Stale.S99E99.srt"], + "config": {"requested": expected, "decision": "pin"}, + **context, + }, timeout=5) + assert response.payload == {"decision": "pin", "member": "Requested.srt"} + + +@pytest.mark.parametrize("context,member", [({}, "99.en.srt"), ({"absolute_episode": None}, "1.en.srt"), + ({"absolute_episode": 0}, "0.en.srt"), + ({"absolute_episode": 49}, "49.en.srt")]) +def test_real_worker_absolute_context_preserves_absent_null_and_zero(selector_worker, context, member): + response = selector_worker.select_archive_member({ + "provider_payload": {"episode": 1, "absolute_episode": 99}, + "members": ["99.en.srt", "1.en.srt", "0.en.srt", "49.en.srt"], + "config": {"absolute_selector": True, "decision": "pin"}, **context, + }, timeout=5) + assert response.payload == {"decision": "pin", "member": member} + + +@pytest.mark.parametrize("absolute,member", [(49, "49.en.srt"), (0, "0.en.srt"), (None, "1.en.srt")]) +@pytest.mark.parametrize("opaque_absolute", [{}, {"absolute_episode": None}, {"absolute_episode": 99}], + ids=["missing", "null", "stale"]) +@pytest.mark.parametrize("decision", ["pin", "defer", "reject"]) +def test_registry_forwards_absolute_context_to_real_selector( + selector_worker, absolute, member, opaque_absolute, decision): + from provider_hub.registry import HubProxyProvider + from subzero.language import Language + from subliminal_patch.core import Episode + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(member, _SRT) + archive.writestr("99.en.srt", _SRT.replace(b"\nx\n", b"\nwrong member\n")) + payload = _archive_payload(buffer.getvalue(), select_member=True, episode=99) + opaque = {"episode": 99, **opaque_absolute} + provider = HubProxyProvider(worker_client=selector_worker, archive=payload, opaque=opaque, + absolute_selector=True, decision=decision) + provider.provider_name = "fixture" + video = Episode("/fixtures/Show.mkv", "Show", 3, 1) + video.absolute_episode = absolute + subtitle = provider.list_subtitles(video, {Language("eng")})[0] + # Selection uses the frozen search request, never a later video mutation. + video.absolute_episode = 50 + subtitle.absolute_episode = 50 + stored = {"archive": payload, **opaque} + for _ in range(2): + if decision == "reject": + with pytest.raises(SubtitleCandidateRejected): + provider.download_subtitle(subtitle) + else: + assert provider.download_subtitle(subtitle) is True + assert subtitle.content == _SRT + assert subtitle.provider_payload == stored + + +@pytest.mark.parametrize("attributes,member", [({}, "99.en.srt"), ({"absolute_episode": None}, "1.en.srt"), + ({"absolute_episode": 0}, "0.en.srt"), + ({"absolute_episode": 49}, "49.en.srt")]) +def test_legacy_registry_selector_only_forwards_available_absolute_context(selector_worker, attributes, member): + from provider_hub.registry import HubProxyProvider + from subzero.language import Language + from subliminal_patch.core import Episode + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + for name in ["99.en.srt", "1.en.srt", "0.en.srt", "49.en.srt"]: + archive.writestr(name, _SRT if name == member else _SRT.replace(b"\nx\n", b"\nwrong member\n")) + payload = _archive_payload(buffer.getvalue(), select_member=True) + opaque = {"episode": 1, "absolute_episode": 99} + provider = HubProxyProvider(worker_client=selector_worker, archive=payload, opaque=opaque, + absolute_selector=True, decision="pin") + provider.provider_name = "fixture" + subtitle = provider.list_subtitles(Episode("/fixtures/Show.mkv", "Show", 3, 1), {Language("eng")})[0] + del subtitle._requested_archive_context + assert not hasattr(subtitle, "absolute_episode") + for key, value in attributes.items(): + setattr(subtitle, key, value) + subtitle.season, subtitle.episode = 3, 1 + assert provider.download_subtitle(subtitle) is True + assert subtitle.content == _SRT + assert subtitle.provider_payload == {"archive": payload, **opaque} + + +@pytest.mark.parametrize("kind,season,episode,member", [ + ("movie", None, None, "Film.srt"), + ("season_only", 3, None, "Show.S03E01.srt"), + ("unknown_season", None, 1, "Show.S04E01.srt"), +]) +@pytest.mark.parametrize("opaque", [{}, {"season": None, "episode": None}, {"season": 99, "episode": 99}], + ids=["missing", "null", "stale"]) +@pytest.mark.parametrize("decision", ["pin", "defer", "reject"]) +def test_registry_selector_preserves_empty_context_through_real_worker( + selector_worker, kind, season, episode, member, opaque, decision): + from provider_hub.registry import HubProxyProvider + from subzero.language import Language + from subliminal_patch.core import Episode, Movie + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr(member, _SRT) + archive.writestr("Stale.S99E99.srt", _SRT.replace(b"\nx\n", b"\nwrong member\n")) + payload = _archive_payload(buffer.getvalue(), select_member=True, season=99, episode=99) + provider = HubProxyProvider(worker_client=selector_worker, archive=payload, opaque=opaque, + requested={"season": season, "episode": episode}, decision=decision) + provider.provider_name = "fixture" + video = (Movie("/fixtures/Film.mkv", "Film", year=2024) if kind == "movie" else + Episode("/fixtures/Show.mkv", "Show", season, episode)) + subtitle = provider.list_subtitles(video, {Language("eng")})[0] + stored = {"archive": payload, **opaque} + assert subtitle.provider_payload == stored + + # Repeat the download to expose mutation of the stored opaque payload. + for _ in range(2): + if decision == "reject": + with pytest.raises(SubtitleCandidateRejected, match="No matching subtitle language"): + provider.download_subtitle(subtitle) + else: + assert provider.download_subtitle(subtitle) is True + assert subtitle.content == _SRT + assert subtitle.provider_payload == stored + + def test_worker_runner_select_archive_member_rejects_when_unimplemented(): from provider_hub import worker_runner class P: pass - out = worker_runner._handle(P(), "select_archive_member", {"members": ["a.srt"]}) - assert out == {"member": None, "decision": "reject"} + with pytest.raises(ValueError, match="not implemented"): + worker_runner._handle(P(), "select_archive_member", {"members": ["a.srt"]}) -def test_worker_runner_select_archive_member_coerces_bad_decision(): +def test_worker_runner_select_archive_member_rejects_bad_decision(): from provider_hub import worker_runner class P: def select_archive_member(self, provider_payload, language, members, config): return {"member": None, "decision": "weird"} - out = worker_runner._handle(P(), "select_archive_member", {"members": ["a.srt"]}) - assert out["decision"] == "reject" + with pytest.raises(ValueError, match="invalid decision"): + worker_runner._handle(P(), "select_archive_member", {"members": ["a.srt"]}) def test_select_member_callback_receives_listed_members(): diff --git a/tests/bazarr/test_provider_hub_worker_errors.py b/tests/bazarr/test_provider_hub_worker_errors.py new file mode 100644 index 0000000000..9e37b778a2 --- /dev/null +++ b/tests/bazarr/test_provider_hub_worker_errors.py @@ -0,0 +1,217 @@ +import datetime +import json +from pathlib import Path +import sys + +import pytest + +from provider_hub.worker import ProviderWorkerClient, WorkerError, worker_command +from subliminal.exceptions import ( + AuthenticationError, + ConfigurationError, + DownloadLimitExceeded, + ServiceUnavailable, +) +from subliminal_patch.exceptions import APIThrottled, TooManyRequests + + +@pytest.fixture +def error_worker(tmp_path): + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "provider.py").write_text( + """ +class ErrorProvider: + def search(self, video, languages, config): + raise type(config["failure"], (RuntimeError,), {})(config["message"]) + + def download(self, provider_payload, language, config): + raise type(config["failure"], (RuntimeError,), {})(config["message"]) +""", + encoding="utf-8", + ) + runner = Path(__file__).parents[2] / "bazarr/provider_hub/worker_runner.py" + client = ProviderWorkerClient( + worker_command(sys.executable, runner), + cwd=bundle, + env={ + "BAZARR_PROVIDER_HUB_BUNDLE": str(bundle), + "BAZARR_PROVIDER_HUB_MANIFEST": json.dumps( + {"entry_module": "provider", "entry_class": "ErrorProvider"} + ), + }, + ) + yield client + client.stop() + + +@pytest.fixture +def throttle_state(monkeypatch, tmp_path): + from app.get_args import args + from types import SimpleNamespace + + for name in ("config", "db", "cache", "log", "backup"): + (tmp_path / name).mkdir() + monkeypatch.setattr(args, "config_dir", str(tmp_path)) + from app import get_providers + + monkeypatch.setattr(get_providers, "tp", {}) + monkeypatch.setattr(get_providers, "throttle_count", {}) + names = ["opensubtitlescom", "examplehub"] + monkeypatch.setattr( + get_providers, "settings", + SimpleNamespace(general=SimpleNamespace(enabled_providers=names)), + ) + monkeypatch.setattr(get_providers, "provider_registry", SimpleNamespace(names=lambda: names)) + monkeypatch.setattr(get_providers, "event_stream", lambda **kwargs: None) + return get_providers + + +@pytest.mark.parametrize("operation", ["search", "download"]) +@pytest.mark.parametrize( + "remote_name,provider,host_type,seconds", + [ + ("DownloadLimitExceeded", "opensubtitlescom", DownloadLimitExceeded, 21600), + ("DownloadLimitExceeded", "examplehub", DownloadLimitExceeded, 10800), + ("RateLimited", "opensubtitlescom", TooManyRequests, 60), + ("TooManyRequests", "examplehub", TooManyRequests, 3600), + ("ServiceUnavailable", "examplehub", ServiceUnavailable, 1200), + ("APIThrottled", "examplehub", APIThrottled, 600), + ("AuthenticationRequired", "opensubtitlescom", AuthenticationError, 43200), + ("AuthenticationError", "examplehub", AuthenticationError, 43200), + ("ConfigurationError", "examplehub", ConfigurationError, 43200), + ("RuntimeError", "opensubtitlescom", WorkerError, 600), + ], +) +def test_worker_failure_reaches_persisted_provider_throttle( + error_worker, throttle_state, operation, remote_name, provider, host_type, seconds, +): + # The message deliberately looks like quota exhaustion even for unknown errors. + # Classification must come from the fixed semantic names, never message parsing. + with pytest.raises(Exception) as raised: + error_worker.request(operation, { + "config": {"failure": remote_name, "message": "download limit exceeded"}, + }, timeout=5) + + # Counted transient errors throttle on their fifth failure. Keep the real + # counter and persistence path while avoiding four retry sleeps in this test. + before = datetime.datetime.now() + throttle_state.throttle_count[provider] = { + "count": 4, "time": before + datetime.timedelta(seconds=120), + } + throttle_state.provider_throttle(provider, raised.value) + after = datetime.datetime.now() + + reason, until, _description = throttle_state.get_throttled_providers()[provider] + expected_delay = datetime.timedelta(seconds=seconds) + assert before + expected_delay <= until <= after + expected_delay + assert reason == host_type.__name__ + assert type(raised.value) is host_type + remote_error = raised.value if host_type is WorkerError else raised.value.__cause__ + assert isinstance(remote_error, WorkerError) + assert remote_error.remote_class_name == remote_name + assert remote_error.code == "provider" + assert remote_error.retryable is False + assert str(raised.value) == "download limit exceeded" + assert error_worker.request("health", timeout=5).payload == {"initialized": True} + + +@pytest.fixture +def response_worker(tmp_path): + clients = [] + + def start(response): + script = tmp_path / f"response-{len(clients)}.py" + script.write_text( + "import json, sys\n" + f"response = json.loads({json.dumps(response)!r})\n" + "for line in sys.stdin:\n" + " request = json.loads(line)\n" + " if isinstance(response, dict):\n" + " response.setdefault('abi', request['abi'])\n" + " response.setdefault('id', request['id'])\n" + " print(json.dumps(response), flush=True)\n", + encoding="utf-8", + ) + client = ProviderWorkerClient(worker_command(sys.executable, script)) + clients.append(client) + return client + + yield start + for client in clients: + client.stop() + + +@pytest.mark.parametrize("error", [None, [], ["DownloadLimitExceeded"], "failure", 5]) +def test_malformed_error_metadata_stays_worker_error(response_worker, error): + with pytest.raises(WorkerError): + response_worker({"ok": False, "error": error}).request("search", timeout=5) + + +@pytest.mark.parametrize( + "remote_name,code", + [ + ("builtins.PermissionError", "provider"), + ("SystemExit", "provider"), + ("DownloadLimitExceeded", "transport"), + ("DownloadLimitExceeded", None), + ("DownloadLimitExceeded", []), + (["DownloadLimitExceeded"], "provider"), + ({"class": "DownloadLimitExceeded"}, "provider"), + ], +) +def test_unknown_or_invalid_semantic_metadata_stays_worker_error(response_worker, remote_name, code): + with pytest.raises(WorkerError, match="failure") as raised: + response_worker({"ok": False, "error": { + "message": "failure", "class_name": remote_name, "code": code, "retryable": True, + }}).request("search", timeout=5) + assert type(raised.value) is WorkerError + assert raised.value.remote_class_name == (remote_name if isinstance(remote_name, str) else None) + assert raised.value.code == (code if isinstance(code, str) else None) + assert raised.value.retryable is True + + +@pytest.mark.parametrize("retryable", [True, False, "false", 1, None, []]) +def test_retryable_metadata_accepts_only_booleans(response_worker, retryable): + with pytest.raises(DownloadLimitExceeded) as raised: + response_worker({"ok": False, "error": { + "class_name": "DownloadLimitExceeded", "code": "provider", "message": "quota", + "retryable": retryable, + }}).request("download", timeout=5) + assert raised.value.__cause__.retryable is (retryable if isinstance(retryable, bool) else False) + + +@pytest.mark.parametrize( + "response,diagnostic", + [ + ([], "response must be an object"), + ({"abi": "unsupported", "ok": False, "error": {"class_name": "DownloadLimitExceeded"}}, "unsupported ABI"), + ({"id": "wrong-id", "ok": False, "error": {"class_name": "DownloadLimitExceeded"}}, "mismatched request id"), + ], +) +def test_invalid_envelope_is_not_promoted_to_provider_error(response_worker, response, diagnostic): + with pytest.raises(WorkerError, match=diagnostic): + response_worker(response).request("search", timeout=5) + + +@pytest.mark.parametrize( + "output,diagnostic", + [("not-json", "malformed JSON"), (None, "closed stdout")], +) +def test_worker_transport_failure_keeps_generic_error(output, diagnostic): + script = "import sys; sys.stdin.readline(); " + script += "sys.exit(1)" if output is None else f"print({output!r}, flush=True)" + client = ProviderWorkerClient([sys.executable, "-I", "-B", "-c", script]) + try: + with pytest.raises(WorkerError, match=diagnostic) as raised: + client.request("download", timeout=5) + assert raised.value.remote_class_name is None + assert raised.value.code is None + assert raised.value.retryable is False + finally: + client.stop() + + +def test_invalid_success_payload_keeps_generic_error(response_worker): + with pytest.raises(WorkerError, match="payload must be an object"): + response_worker({"ok": True, "payload": ["candidate"]}).request("search", timeout=5) diff --git a/tests/bazarr/test_release_type_mismatch.py b/tests/bazarr/test_release_type_mismatch.py index c030a9810b..491c79a8b3 100644 --- a/tests/bazarr/test_release_type_mismatch.py +++ b/tests/bazarr/test_release_type_mismatch.py @@ -1013,8 +1013,11 @@ class _Written: storage_path = str(written_file) matches = set() - search.monkeypatch.setattr(search.module, "save_subtitles", - lambda *args, **kwargs: [_Written()]) + def save(*args, **kwargs): + kwargs['write_subtitle'](str(written_file), b'1\n') + return [_Written()] + + search.monkeypatch.setattr(search.module, "save_subtitles", save) search.monkeypatch.setattr(search.module, "process_subtitle", lambda **kwargs: None) search.run() diff --git a/tests/bazarr/test_secret_store_crypto.py b/tests/bazarr/test_secret_store_crypto.py index eb9dbcd968..0685590b5d 100644 --- a/tests/bazarr/test_secret_store_crypto.py +++ b/tests/bazarr/test_secret_store_crypto.py @@ -6,6 +6,8 @@ covers just the crypto primitives. """ from unittest.mock import MagicMock +import json +from pathlib import Path import pytest @@ -18,6 +20,23 @@ ) +_OLD_CRYPTOGRAPHY = json.loads( + (Path(__file__).with_name('fixtures') / 'cryptography49_fernet.json').read_text(encoding='utf-8') +) + + +@pytest.mark.parametrize('case', _OLD_CRYPTOGRAPHY['cases'], ids=['provider', 'translator', 'unicode']) +def test_ciphertext_from_cryptography_49_remains_readable(case): + """Persisted synthetic fixtures were encrypted by the actual 49.0.0 wheel.""" + key = _OLD_CRYPTOGRAPHY['master_key'] + assert decrypt_secret(case['ciphertext'], master_key=key) == case['plaintext'] + assert encrypt_secret(case['ciphertext'], master_key=key) == case['ciphertext'] + with pytest.raises(ValueError, match='tampered|master key'): + decrypt_secret(case['ciphertext'], master_key='synthetic-wrong-master-key') + with pytest.raises(ValueError, match='tampered|master key'): + decrypt_secret(case['ciphertext'][:-2] + 'AA', master_key=key) + + @pytest.fixture def settings_obj(): """Minimal stand-in for the dynaconf settings root with a writable diff --git a/tests/bazarr/test_subsync_engines.py b/tests/bazarr/test_subsync_engines.py index e314af4730..fe0e6f8942 100644 --- a/tests/bazarr/test_subsync_engines.py +++ b/tests/bazarr/test_subsync_engines.py @@ -412,7 +412,7 @@ def test_successful_sync_without_history_does_not_log_error(monkeypatch, tmp_pat _write(subtitle, 'original') class FakeRunner: - def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync=False): + def run(self, srt_path, output_mode, enabled_engines, execute_engine, force_sync=False, **source_options): result = SyncRunResult(source_path=srt_path, output_mode=output_mode) result.results = [ SyncEngineResult( @@ -663,6 +663,7 @@ def test_subsyncer_reports_engine_progress(monkeypatch, tmp_path): subtitle = tmp_path / 'Movie.hu.srt' _write(subtitle, 'original') + (tmp_path / 'Movie.mkv').touch() progress = [] def fake_engine(self, output_path, **kwargs): @@ -692,6 +693,7 @@ def fake_engine(self, output_path, **kwargs): ('Preparing synchronization', 0, 1), ('Running FFsubsync (1/1)', 0, 1), ('Finished FFsubsync (1/1)', 1, 1), + ('Saving synchronized subtitle', None, None), ] @@ -756,6 +758,135 @@ def test_add_sync_engine_outputs_indexes_generated_files(tmp_path): ) == 'hu:sync-alass' +@pytest.mark.parametrize('engine', ['ffsubsync', 'autosubsync', 'ALASS']) +@pytest.mark.parametrize('extension', ['srt', 'sub', 'smi', 'txt', 'ssa', 'ASS', 'mpl', 'vtt']) +@pytest.mark.parametrize('suffix, basename, hi, forced', [ + ('en', 'en', False, False), + ('en.HI', 'en', True, False), + ('hu.sdh', 'hu', True, False), + ('fr.cc', 'fr', True, False), + ('pt_BR.forced', 'pt-BR', False, True), + ('pb.forced.hi', 'pt-BR', True, True), +]) +def test_sync_output_owner_filter_preserves_language_and_format(tmp_path, engine, extension, + suffix, basename, hi, forced): + from subtitles.indexer.utils import add_sync_engine_outputs + + filename = f'Movie.Release.{suffix}.{engine}.{extension}' + _write(tmp_path / filename, 'subtitle') + _write(tmp_path / f'Movie.Release.Extended.{suffix}.{engine}.{extension}', 'sibling') + _write(tmp_path / f'Movie.{suffix}.{engine}.{extension}', 'shorter stem') + + result = add_sync_engine_outputs(str(tmp_path), {}, video_filename='Movie.Release.mkv') + + assert set(result) == {filename} + language = result[filename] + assert language.basename == basename + assert language.hi is hi + assert language.forced is forced + + +def test_sync_output_owner_filter_preserves_existing_entries_and_is_idempotent(tmp_path): + from subzero.language import Language + from subtitles.indexer.utils import add_sync_engine_outputs + + original = Language.fromietf('hu') + existing = Language.fromietf('de') + subtitles = {'Movie.hu.srt': original, 'Movie.en.alass.srt': existing, + 'Existing.en.srt': Language.fromietf('en')} + for filename in ['Movie.hu.srt', 'Movie.en.alass.srt', 'Movie.en.ffsubsync.srt', + 'Other.en.alass.srt', 'Movie.en.unknown.srt', 'Movie.en.alass.mkv', + 'Movie.invalid.alass.srt', 'Movie.en.combined-hu.alass.srt']: + _write(tmp_path / filename, 'subtitle') + (tmp_path / 'Movie.fr.alass.srt').mkdir() + + result = add_sync_engine_outputs(str(tmp_path), subtitles, video_filename='Movie.mkv') + assert result is subtitles + assert set(result) == {'Movie.hu.srt', 'Movie.en.alass.srt', 'Existing.en.srt', + 'Movie.en.ffsubsync.srt'} + assert result['Movie.hu.srt'] is original + assert result['Movie.en.alass.srt'] is existing + first_scan = result.copy() + + assert add_sync_engine_outputs(str(tmp_path), result, video_filename='Movie.mkv') == first_scan + assert add_sync_engine_outputs(str(tmp_path / 'missing'), result, + video_filename='Movie.mkv') == first_scan + + +@pytest.mark.parametrize('video,subtitle', [ + ('Movie.mkv', 'movie.en.ffsubsync.srt'), + ('\u00c9pisode.mkv', 'E\u0301PISODE.en.ffsubsync.srt'), + ('E\u0301pisode.mkv', '\u00e9pisode.en.ffsubsync.srt'), +]) +def test_sync_output_owner_uses_case_and_unicode_normalization(tmp_path, video, subtitle): + from subtitles.indexer.utils import add_sync_engine_outputs + + _write(tmp_path / subtitle, 'subtitle') + _write(tmp_path / subtitle.replace('.en.', '.Extended.en.'), 'sibling') + assert set(add_sync_engine_outputs(str(tmp_path), {}, video_filename=video)) == {subtitle} + + +@pytest.mark.parametrize('stem', ['Movie', 'Movie.en', 'Movie.hi', 'Movie.forced']) +def test_sync_output_keeps_complete_actual_owner_stem(tmp_path, stem): + from subtitles.indexer.utils import add_sync_engine_outputs + + for sibling in ['Movie', 'Movie.en', 'Movie.hi', 'Movie.forced', 'Movie.en.Extended']: + _write(tmp_path / f'{sibling}.mkv', 'video') + _write(tmp_path / f'{sibling}.ffsubsync.srt', 'subtitle') + result = add_sync_engine_outputs(str(tmp_path), {}, video_filename=f'{stem}.mkv') + assert result == {f'{stem}.ffsubsync.srt': None} + + +@pytest.mark.parametrize('video_stem,subtitle_stem,sibling_stem', [ + ('Movie', 'movie', 'MOVIE.EN'), + ('\u00c9pisode', 'E\u0301PISODE', '\u00e9pisode.en'), + ('E\u0301pisode', '\u00e9pisode', 'E\u0301PISODE.EN'), +]) +@pytest.mark.parametrize('sibling_kind', ['media', 'directory', 'non-media', 'absent']) +def test_tagged_sync_output_defers_only_to_an_actual_media_owner( + tmp_path, monkeypatch, video_stem, subtitle_stem, sibling_stem, sibling_kind): + from subtitles.indexer.utils import add_sync_engine_outputs + + media_folder = tmp_path / 'media' + media_folder.mkdir() + subtitle_folder = tmp_path / 'subtitles' + subtitle_folder.mkdir() + video = media_folder / f'{video_stem}.mkv' + _write(video, 'video') + _write(media_folder / f'{sibling_stem}.Extended.mkv', 'prefix sibling') + sibling = media_folder / f'{sibling_stem}.MP4' + if sibling_kind == 'media': + _write(sibling, 'video') + elif sibling_kind == 'directory': + sibling.mkdir() + elif sibling_kind == 'non-media': + _write(media_folder / f'{sibling_stem}.txt', 'not a video') + subtitle = f'{subtitle_stem}.en.ffsubsync.srt' + _write(subtitle_folder / subtitle, 'subtitle') + + from types import SimpleNamespace + from unittest.mock import Mock + from app import database as db_module + from app.config import settings + from utilities.path_mappings import path_mappings + monkeypatch.setattr(settings.general, 'subfolder', 'absolute') + monkeypatch.setattr(settings.general, 'subfolder_custom', str(subtitle_folder)) + rows = [SimpleNamespace(path=str(path), arr_instance_id=None) for path in media_folder.iterdir() + if path.is_file() and path.suffix.lower() in ('.mkv', '.mp4')] + db = Mock() + db.execute.return_value.all.return_value = rows + monkeypatch.setattr(db_module, 'database', db) + monkeypatch.setattr(path_mappings, 'path_replace_instance', lambda path, *args: path) + result = add_sync_engine_outputs(str(subtitle_folder), {}, video_path=str(video)) + + if sibling_kind == 'media': + assert result == {} + assert add_sync_engine_outputs(str(subtitle_folder), {}, video_path=str(sibling)) == {subtitle: None} + else: + assert set(result) == {subtitle} + assert result[subtitle].basename == 'en' + + def test_keep_all_job_name_does_not_claim_original_was_overwritten(): from subtitles.sync import _sync_complete_job_name from subtitles.tools.subsync_engines import ( diff --git a/tests/bazarr/test_subtitle_content_sync_outputs.py b/tests/bazarr/test_subtitle_content_sync_outputs.py index 8907126879..f57ed2c480 100644 --- a/tests/bazarr/test_subtitle_content_sync_outputs.py +++ b/tests/bazarr/test_subtitle_content_sync_outputs.py @@ -7,6 +7,7 @@ from types import SimpleNamespace from flask import Flask +import pytest import app.database # noqa: F401 @@ -223,6 +224,72 @@ def fake_resolve(media_type, media_id, language_code, arr_instance_id=None): assert events == [{'type': 'movie', 'payload': 1203}] +@pytest.mark.parametrize('media_type', ['movie', 'episode']) +@pytest.mark.parametrize('failure', ['index', 'event', 'history']) +def test_promotion_preserves_confirmation_and_refresh_attempts_on_failure(schema_session, tmp_path, monkeypatch, + media_type, failure): + from app.database import TableHistory, TableHistoryMovie, select + from radarr import history as movie_history + from sonarr import history as episode_history + + content = _real_module('api.subtitles.content') + video = tmp_path / 'Video.mkv' + original = tmp_path / 'Video.en.srt' + generated = tmp_path / 'Video.en.alass.srt' + video.touch() + original.write_text('original subtitle') + generated.write_text('selected subtitle') + metadata = {'mediaPath': str(video), 'mediaId': 10, 'mediaUpstreamId': 20, + 'episodeId': 30, 'episodeUpstreamId': 40, 'arrInstanceId': 7} + media_id = 40 if media_type == 'episode' else 20 + table = TableHistory if media_type == 'episode' else TableHistoryMovie + refreshed = [] + + monkeypatch.setattr(content, 'resolve_subtitle_path', + lambda kind, item_id, lang, **kw: (str(generated if ':sync-' in lang else original), metadata)) + monkeypatch.setattr(content, 'database', schema_session) + for module in (movie_history, episode_history): + monkeypatch.setattr(module, 'database', schema_session) + monkeypatch.setattr(module, 'event_stream', lambda **kw: None) + for name in ('path_replace_reverse', 'path_replace_reverse_movie'): + monkeypatch.setattr(content.path_mappings, name, lambda path: path) + monkeypatch.setattr(content.path_mappings, 'path_replace_instance', lambda path, *a: path) + monkeypatch.setattr(content, '_active_sync_job_for_path', lambda path: None) + + def refresh_step(step): + refreshed.append(step) + if step == failure: + raise RuntimeError('controlled refresh failure') + + for name in ('store_subtitles', 'store_subtitles_movie'): + monkeypatch.setattr(content, name, lambda *a, **kw: refresh_step('index')) + monkeypatch.setattr(content, 'event_stream', lambda **kw: refresh_step('event')) + if failure == 'history': + execute = schema_session.execute + + def fail_history_insert(statement, *args, **kwargs): + if getattr(statement, 'is_insert', False): + raise RuntimeError('controlled refresh failure') + return execute(statement, *args, **kwargs) + + monkeypatch.setattr(schema_session, 'execute', fail_history_insert) + + with pytest.raises(RuntimeError, match='controlled refresh failure'): + content.promote_sync_subtitle(media_type, media_id, 'en', 'en:sync-alass', arr_instance_id=7) + + assert original.read_text() == 'selected subtitle' + rows = schema_session.execute(select(table)).scalars().all() + if failure == 'history': + assert refreshed == (['index', 'event', 'event'] if media_type == 'episode' else ['index', 'event']) + assert rows == [] + return + assert len(rows) == 1, 'a completed publication lost its action-5 confirmation' + assert (rows[0].action, rows[0].arr_instance_id, rows[0].subtitles_path) == (5, 7, str(original)) + status, code = content.get_subtitle_sync_status(media_type, media_id, 'en', arr_instance_id=7) + assert code == 200 + assert status['confirmed'] is True + + def test_promote_sync_output_rejects_different_base_language(monkeypatch): content = _real_module('api.subtitles.content') diff --git a/tests/bazarr/test_subtitle_destination_path.py b/tests/bazarr/test_subtitle_destination_path.py new file mode 100644 index 0000000000..e01a45ca41 --- /dev/null +++ b/tests/bazarr/test_subtitle_destination_path.py @@ -0,0 +1,206 @@ +# coding=utf-8 +"""A new external subtitle has to land in the configured subtitle folder. + +get_external_subtitles_path() is a lookup: it returns a path only for a file that +already exists. translate and mods used it to build the path of the file they were +about to WRITE, so in the absolute and relative subfolder modes a first-time +translation got None and crashed on save. get_subtitle_destination_path() keeps the +lookup's preference for an existing file and otherwise resolves the configured folder. +""" +import os +from unittest.mock import MagicMock, patch + +import pytest + + +@pytest.fixture +def subtitle_folder(monkeypatch): + from app import config as app_config + from subtitles.indexer import utils as indexer_utils + + # Startup modules write the config to disk when first imported. If that first + # import happens while these values are patched, the patched folder would be + # persisted into the local test config and break every later run. + monkeypatch.setattr(app_config, "write_config", lambda: None) + + def _configure(mode, custom=""): + monkeypatch.setattr(indexer_utils.settings.general, "subfolder", mode) + monkeypatch.setattr(indexer_utils.settings.general, "subfolder_custom", custom) + + return _configure + + +def test_current_mode_writes_next_to_the_video(tmp_path, subtitle_folder): + from subtitles.indexer.utils import get_subtitle_destination_path + + subtitle_folder("current") + video = tmp_path / "video.mkv" + + assert get_subtitle_destination_path(str(video), "video.hu.srt") == str(tmp_path / "video.hu.srt") + + +def test_absolute_mode_resolves_and_creates_the_custom_folder(tmp_path, subtitle_folder): + from subtitles.indexer.utils import get_subtitle_destination_path + + custom = tmp_path / "subs" + subtitle_folder("absolute", str(custom)) + video = tmp_path / "video.mkv" + + destination = get_subtitle_destination_path(str(video), "video.hu.srt") + + assert destination == str(custom / "video.hu.srt") + assert custom.is_dir(), "the write target folder has to exist, the way downloads create it" + + +def test_relative_mode_resolves_under_the_video_folder(tmp_path, subtitle_folder): + from subtitles.indexer.utils import get_subtitle_destination_path + + subtitle_folder("relative", "Subs") + video = tmp_path / "video.mkv" + + assert get_subtitle_destination_path(str(video), "video.hu.srt") == str(tmp_path / "Subs" / "video.hu.srt") + + +def test_an_existing_file_next_to_the_video_keeps_precedence(tmp_path, subtitle_folder): + from subtitles.indexer.utils import get_subtitle_destination_path + + subtitle_folder("absolute", str(tmp_path / "subs")) + video = tmp_path / "video.mkv" + existing = tmp_path / "video.hu.srt" + existing.write_text("1\n00:00:00,000 --> 00:00:01,000\nSzia\n", encoding="utf-8") + + assert get_subtitle_destination_path(str(video), "video.hu.srt") == str(existing) + + +def test_translate_passes_a_custom_folder_destination_to_the_translator(tmp_path, monkeypatch, subtitle_folder): + from subtitles.tools.translate import main as translate_main + from subzero.language import Language + + custom = tmp_path / "subs" + subtitle_folder("absolute", str(custom)) + video = tmp_path / "video.mkv" + + monkeypatch.setattr(translate_main, "validate_translation_params", lambda *a, **kw: None) + monkeypatch.setattr(translate_main, "convert_language_codes", + lambda to_lang, forced, hi: (Language("hun"), to_lang)) + monkeypatch.setattr(translate_main, "get_subtitle_path", lambda *a, **kw: str(tmp_path / "video.hu.srt")) + monkeypatch.setattr(translate_main, "alpha3_from_alpha2", lambda code: "hun") + + seen = {} + translator = MagicMock() + translator.translate.return_value = True + + def _create(kind, **kwargs): + seen.update(kwargs) + return translator + + monkeypatch.setattr(translate_main.TranslatorFactory, "create_translator", staticmethod(_create)) + + with ( + patch("api.subtitles.subtitles.postprocess_subtitles", lambda *a, **kw: None), + patch("subtitles.tools.combine.main.try_combine_for_video", lambda **kw: None), + ): + translate_main.translate_subtitles_file( + video_path=str(video), source_srt_file=str(tmp_path / "video.en.srt"), + from_lang="en", to_lang="hu", forced=False, hi=False, + media_type="movies", sonarr_series_id=None, sonarr_episode_id=None, + radarr_id=9, metadata={}, job_id="job-1") + + assert seen["dest_srt_file"] == os.path.join(str(custom), "video.hu.srt"), ( + "a first-time translation in absolute subfolder mode must target the custom folder, not None") + + +@pytest.mark.parametrize("mode", ["absolute", "relative"]) +@pytest.mark.parametrize("status", ["partial", "completed"]) +@pytest.mark.parametrize("failure", [None, "footer", "replace"]) +def test_translation_saves_atomic_results_in_configured_folder( + tmp_path, monkeypatch, subtitle_folder, mode, status, failure): + import sys + from types import SimpleNamespace + + import pysubs2 + + from languages import get_languages + from subtitles.tools.translate import main as translate_main + from subtitles.tools.translate.services import openrouter_translator as service_module + + monkeypatch.setattr(get_languages, "languages_dict", [ + {"code2": "en", "code3": "eng", "code3b": "eng", "name": "English"}, + {"code2": "hu", "code3": "hun", "code3b": "hun", "name": "Hungarian"}, + ], raising=False) + video = tmp_path / "video.mkv" + video.touch() + source = tmp_path / "video.en.srt" + source.write_text("1\n00:00:01,000 --> 00:00:02,000\nOne\n\n" + "2\n00:00:03,000 --> 00:00:04,000\nTwo\n", encoding="utf-8") + original_source = source.read_bytes() + custom = tmp_path / "Subs" + destination = custom / "video.hu.srt" + subtitle_folder(mode, str(custom) if mode == "absolute" else "Subs") + if failure: + custom.mkdir() + destination.write_bytes(b"existing subtitle") + + monkeypatch.setattr(service_module.settings.translator, "translator_info", True) + monkeypatch.setattr(service_module.settings.translator, "translator_type", "openrouter") + monkeypatch.setattr(service_module, "get_title", lambda **kwargs: "Example") + monkeypatch.setattr(service_module, "get_translator_auth_headers", lambda: {}) + monkeypatch.setattr(service_module.OpenRouterTranslatorService, "_get_api_key_value", lambda self: "") + monkeypatch.setattr(service_module.requests, "post", lambda *args, **kwargs: SimpleNamespace( + status_code=200, json=lambda: {"jobId": "fixture-job"})) + lines = [{"position": 0, "line": "Egy"}] + if status == "completed": + lines.append({"position": 1, "line": "KettÅ‘"}) + monkeypatch.setattr(service_module.requests, "get", lambda *args, **kwargs: SimpleNamespace( + status_code=200, json=lambda: {"status": status, "result": {"lines": lines}, + "error": "Some batches failed" if failure else None})) + for name in ("show_progress", "hide_progress", "show_message"): + monkeypatch.setattr(service_module, name, lambda *args, **kwargs: None) + history = [] + monkeypatch.setattr(service_module, "history_log_movie", lambda **kwargs: history.append(kwargs)) + monkeypatch.setattr(service_module.jobs_queue, "update_job_progress", lambda **kwargs: None) + names = [] + monkeypatch.setattr(translate_main.jobs_queue, "get_job_name", lambda job_id: "Translating Example") + monkeypatch.setattr(translate_main.jobs_queue, "update_job_name", + lambda **kwargs: names.append(kwargs["new_job_name"])) + monkeypatch.setattr(translate_main.TranslatorFactory, "create_translator", + staticmethod(lambda kind, **kwargs: service_module.OpenRouterTranslatorService(**kwargs))) + postprocessed = [] + + def fail_write(*args, **kwargs): + raise OSError("synthetic write failure") + + if failure == "footer": + monkeypatch.setattr(service_module, "add_translator_info", fail_write) + elif failure == "replace": + from subtitles.tools import subsync_engines + monkeypatch.setattr(subsync_engines.os, "replace", fail_write) + + monkeypatch.setitem(sys.modules, "api.subtitles.subtitles", SimpleNamespace( + postprocess_subtitles=lambda *args, **kwargs: postprocessed.append(args[0]))) + monkeypatch.setitem(sys.modules, "subtitles.tools.combine.main", SimpleNamespace( + try_combine_for_video=lambda **kwargs: None)) + kwargs = dict(video_path=str(video), source_srt_file=str(source), from_lang="en", to_lang="hu", + forced=False, hi=False, media_type="movies", sonarr_series_id=None, + sonarr_episode_id=None, radarr_id=9, metadata={}, job_id="fixture-job", arr_instance_id=3) + if failure: + with pytest.raises(RuntimeError, match="failed translation result"): + translate_main.translate_subtitles_file(**kwargs) + else: + assert translate_main.translate_subtitles_file(**kwargs) == str(destination) + + assert source.read_bytes() == original_source + assert not list(custom.glob(".bazarr-translate-*")) + if failure: + assert destination.read_bytes() == b"existing subtitle" + assert not history + assert not postprocessed + assert names[-1].startswith("Failed") + else: + output = pysubs2.load(destination) + assert [cue.plaintext for cue in output[:2]] == (["Egy", "Two"] if status == "partial" else ["Egy", "KettÅ‘"]) + assert [(cue.start, cue.end) for cue in output[:2]] == [(1000, 2000), (3000, 4000)] + assert ("partially translated" in output[-1].plaintext.lower()) == (status == "partial") + assert ("partial" in history[0]["result"].message.lower()) == (status == "partial") + assert names[-1].startswith("Partially translated" if status == "partial" else "Translated") + assert postprocessed == [str(destination)] diff --git a/tests/bazarr/test_threading_followup.py b/tests/bazarr/test_threading_followup.py index d1ad9f2da4..a9780f078a 100644 --- a/tests/bazarr/test_threading_followup.py +++ b/tests/bazarr/test_threading_followup.py @@ -303,7 +303,7 @@ def __str__(self): return "en" monkeypatch.setattr(si, "search_external_subtitles", lambda *a, **k: {sub_local: _Lang()}) - monkeypatch.setattr(si, "add_sync_engine_outputs", lambda f, s: s) + monkeypatch.setattr(si, "add_sync_engine_outputs", lambda f, s, **k: s) monkeypatch.setattr(si, "add_combined_outputs", lambda f, s, **k: s) monkeypatch.setattr(si, "guess_external_subtitles", lambda f, s, t, e: s) monkeypatch.setattr(si, "get_external_subtitles_path", lambda r, s: s) diff --git a/tests/bazarr/test_translate_job_media_name.py b/tests/bazarr/test_translate_job_media_name.py new file mode 100644 index 0000000000..5298637e11 --- /dev/null +++ b/tests/bazarr/test_translate_job_media_name.py @@ -0,0 +1,275 @@ +# coding=utf-8 +"""Translation job labels have to name the media, movies included. + +``translate_subtitles_file`` and the AI translator both build their label from +``get_title``, and the LLM prompt gets its context from ``get_description``. +Both branch on ``media_type``, and the callers do not agree on how to spell it: +the manual translate endpoint sends "movie" while the batch paths send +"movies". Only the plural matched, so a movie translated from its detail page +fell through to the series branch, found no ``sonarr_series_id`` and produced an +empty title. The job then read "Translating HU to EN" with no film in it, the +Translation Jobs table showed only the language pair, and the prompt lost the +movie's overview. + +Both lookups also have to be scoped to the owning instance (#156). ``radarrId`` +and ``sonarrSeriesId`` are unique only together with ``arr_instance_id`` +(``ux_table_movies_instance_upstream_id``), so an unscoped query can hand back a +sibling instance's media and label the job, and prompt the provider, with the +wrong film. +""" +import pytest + +from app.database import TableEpisodes, TableMovies, TableShows +from subtitles.tools.translate.core import translator_utils + + +@pytest.fixture +def library(schema_session, monkeypatch): + """Two Radarr/Sonarr instances that reuse the same upstream ids.""" + monkeypatch.setattr(translator_utils, "database", schema_session) + schema_session.add( + TableMovies( + arr_instance_id=1, + radarrId=7, + tmdbId="12345", + title="The Quiet Earth", + year="1985", + imdbId="tt0089869", + overview="The last man alive.", + path="/movies/quiet.mkv", + tags="[]", + monitored="True", + ) + ) + schema_session.add( + TableShows( + arr_instance_id=1, + sonarrSeriesId=3, + tvdbId=999, + title="Dark Matter", + year="2024", + imdbId="tt1234567", + overview="A man wakes in another life.", + path="/tv/dark-matter", + tags="[]", + monitored="True", + seriesType="standard", + ) + ) + schema_session.add( + TableEpisodes( + arr_instance_id=1, + sonarrEpisodeId=42, + sonarrSeriesId=3, + season=2, + episode=5, + title="Superposition", + path="/tv/dark-matter/s02e05.mkv", + monitored="True", + ) + ) + # The 4K instance reuses the same upstream ids for different media. + schema_session.add( + TableMovies( + arr_instance_id=2, + radarrId=7, + tmdbId="67890", + title="Sorcerer", + year="1977", + imdbId="tt0076740", + overview="Four men drive nitroglycerin through the jungle.", + path="/movies-4k/sorcerer.mkv", + tags="[]", + monitored="True", + ) + ) + schema_session.add( + TableShows( + arr_instance_id=2, + sonarrSeriesId=3, + tvdbId=1000, + title="Severance", + year="2022", + imdbId="tt11280740", + overview="Work you cannot remember.", + path="/tv-4k/severance", + tags="[]", + monitored="True", + seriesType="standard", + ) + ) + schema_session.add( + TableEpisodes( + arr_instance_id=2, + sonarrEpisodeId=42, + sonarrSeriesId=3, + season=1, + episode=9, + title="The We We Are", + path="/tv-4k/severance/s01e09.mkv", + monitored="True", + ) + ) + schema_session.commit() + return schema_session + + +@pytest.mark.parametrize("media_type", ["movie", "movies"]) +def test_movie_title_is_found_however_the_caller_spells_the_media_type(library, media_type): + assert translator_utils.get_title(media_type, radarr_id=7) == "The Quiet Earth" + + +@pytest.mark.parametrize("media_type", ["movie", "movies"]) +def test_movie_description_is_found_however_the_caller_spells_the_media_type(library, media_type): + description = translator_utils.get_description(media_type, radarr_id=7, sonarr_series_id=None) + assert "The Quiet Earth" in description + assert "The last man alive." in description + + +def test_unknown_movie_still_gives_an_empty_title(library): + assert translator_utils.get_title("movie", radarr_id=999) == "" + + +def test_episode_title_still_carries_the_series_and_the_episode_number(library): + title = translator_utils.get_title( + "episode", sonarr_series_id=3, sonarr_episode_id=42 + ) + assert title == "Dark Matter - S02E05 - Superposition" + + +def test_series_description_is_unchanged(library): + description = translator_utils.get_description( + "episode", radarr_id=None, sonarr_series_id=3 + ) + assert "Dark Matter" in description + + +@pytest.mark.parametrize( + "instance, expected", [(1, "The Quiet Earth"), (2, "Sorcerer")] +) +def test_movie_title_comes_from_the_owning_instance(library, instance, expected): + # radarrId 7 exists on both instances; only the pair identifies the film. + assert ( + translator_utils.get_title("movie", radarr_id=7, arr_instance_id=instance) + == expected + ) + + +@pytest.mark.parametrize( + "instance, expected", [(1, "The last man alive."), (2, "nitroglycerin")] +) +def test_movie_description_comes_from_the_owning_instance(library, instance, expected): + description = translator_utils.get_description( + "movie", radarr_id=7, sonarr_series_id=None, arr_instance_id=instance + ) + assert expected in description + + +@pytest.mark.parametrize( + "instance, expected", + [(1, "Dark Matter - S02E05 - Superposition"), (2, "Severance - S01E09 - The We We Are")], +) +def test_episode_title_comes_from_the_owning_instance(library, instance, expected): + assert ( + translator_utils.get_title( + "episode", sonarr_series_id=3, sonarr_episode_id=42, arr_instance_id=instance + ) + == expected + ) + + +def test_series_description_comes_from_the_owning_instance(library): + description = translator_utils.get_description( + "episode", radarr_id=None, sonarr_series_id=3, arr_instance_id=2 + ) + assert "Severance" in description + + +def test_an_instance_that_owns_nothing_gives_an_empty_title(library): + assert translator_utils.get_title("movie", radarr_id=7, arr_instance_id=99) == "" + + +# --- the owner has to survive the trip from the endpoint to the lookup -------- + + +@pytest.mark.parametrize( + "module_name, service_name, helper", + [ + ("openrouter_translator", "OpenRouterTranslatorService", "get_title"), + ("lingarr_translator", "LingarrTranslatorService", "get_title"), + ("gemini_translator", "GeminiTranslatorService", "get_description"), + ], +) +def test_translator_services_pass_the_owner_to_the_lookup( + monkeypatch, module_name, service_name, helper +): + import importlib + + module = importlib.import_module( + f"subtitles.tools.translate.services.{module_name}" + ) + seen = {} + + def _capture(*args, **kwargs): + seen.update(kwargs) + seen["positional"] = args + return "" + + monkeypatch.setattr(module, helper, _capture) + + service = getattr(module, service_name)( + source_srt_file="in.srt", + dest_srt_file="out.srt", + lang_obj=None, + to_lang="hun", + from_lang="en", + media_type="movie", + video_path="/movies/x.mkv", + orig_to_lang="hu", + forced=False, + hi=False, + sonarr_series_id=None, + sonarr_episode_id=None, + radarr_id=7, + arr_instance_id=2, + ) + + assert service.arr_instance_id == 2 + + +def test_translate_subtitles_file_labels_the_job_with_the_owning_instance(monkeypatch): + from subtitles.tools.translate import main as translate_main + + seen = {} + + def _capture(*args, **kwargs): + seen["args"] = args + seen["kwargs"] = kwargs + return "The Quiet Earth" + + monkeypatch.setattr(translate_main, "get_title", _capture) + monkeypatch.setattr( + translate_main.jobs_queue, + "add_job_from_function", + lambda *a, **k: seen.setdefault("label", a[0]), + ) + + translate_main.translate_subtitles_file( + video_path="/movies/x.mkv", + source_srt_file="in.srt", + from_lang="en", + to_lang="hu", + forced=False, + hi=False, + media_type="movie", + sonarr_series_id=None, + sonarr_episode_id=None, + radarr_id=7, + metadata={}, + job_id=None, + arr_instance_id=2, + ) + + assert seen["label"] == "Translating The Quiet Earth (EN to HU)" + passed = list(seen["args"]) + list(seen["kwargs"].values()) + assert 2 in passed diff --git a/tests/bazarr/test_translate_owner_threading.py b/tests/bazarr/test_translate_owner_threading.py index 8bcaaa1f2a..8c81e14a01 100644 --- a/tests/bazarr/test_translate_owner_threading.py +++ b/tests/bazarr/test_translate_owner_threading.py @@ -27,12 +27,13 @@ def translate_harness(monkeypatch, tmp_path): lambda to_lang, forced, hi: (Language('nld'), to_lang)) monkeypatch.setattr(translate_main, 'get_subtitle_path', lambda *a, **kw: str(tmp_path / 'video.nl.srt')) - monkeypatch.setattr(translate_main, 'get_external_subtitles_path', + monkeypatch.setattr(translate_main, 'get_subtitle_destination_path', lambda file, subtitle: str(tmp_path / subtitle)) monkeypatch.setattr(translate_main, 'alpha3_from_alpha2', lambda code: 'nld') translator = MagicMock() translator.translate.return_value = True + translator.partial_error = None monkeypatch.setattr(translate_main.TranslatorFactory, 'create_translator', staticmethod(lambda *a, **kw: translator)) return translate_main @@ -80,3 +81,27 @@ def _postprocess(subtitles_path, video_path, media_type, metadata, id, radarr_id=9, metadata={}, job_id='job-1') assert seen['arr_instance_id'] is None + + +@pytest.mark.parametrize('partial', [False, True]) +@pytest.mark.parametrize('current_name', [None, 'Translating Example']) +def test_translation_job_label_preserves_partial_status(translate_harness, monkeypatch, partial, current_name): + translate_main = translate_harness + translator = translate_main.TranslatorFactory.create_translator('openrouter') + translator.partial_error = 'Some batches failed' if partial else None + names = [] + monkeypatch.setattr(translate_main.jobs_queue, 'get_job_name', lambda job_id: current_name) + monkeypatch.setattr(translate_main.jobs_queue, 'update_job_name', + lambda **kwargs: names.append(kwargs['new_job_name'])) + + with ( + patch('api.subtitles.subtitles.postprocess_subtitles', lambda *a, **kw: None), + patch('subtitles.tools.combine.main.try_combine_for_video', lambda **kw: None), + ): + translate_main.translate_subtitles_file( + video_path='/local/tv/s/e.mkv', source_srt_file='/local/tv/s/e.en.srt', + from_lang='en', to_lang='nl', forced=False, hi=False, + media_type='episode', sonarr_series_id=1, sonarr_episode_id=42, + radarr_id=None, metadata={}, job_id='job-1', arr_instance_id=3) + + assert names[-1].startswith('Partially translated' if partial else 'Translated') diff --git a/tests/bazarr/test_translator_model_migration.py b/tests/bazarr/test_translator_model_migration.py new file mode 100644 index 0000000000..8df7f0695b --- /dev/null +++ b/tests/bazarr/test_translator_model_migration.py @@ -0,0 +1,60 @@ +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import yaml + + +@pytest.mark.parametrize('configured, expected', [ + (None, 'google/gemini-2.5-flash-lite'), + ('google/gemini-2.5-flash-preview-05-20', 'google/gemini-2.5-flash'), + ('google/gemini-2.5-flash-lite-preview-06-17', 'google/gemini-2.5-flash-lite'), + ('google/gemini-2.5-flash-lite-preview-09-2025', 'google/gemini-2.5-flash-lite'), + ('google/gemini-3-pro-preview', 'google/gemini-3.1-pro-preview'), + ('google/gemini-2.5-flash', 'google/gemini-2.5-flash'), + ('google/gemini-2.5-flash-lite', 'google/gemini-2.5-flash-lite'), + ('google/gemini-3.1-pro-preview', 'google/gemini-3.1-pro-preview'), + ('x-ai/grok-4-fast', 'x-ai/grok-4-fast'), + ('x-ai/grok-4.20-beta', 'x-ai/grok-4.20-beta'), + ('custom/my-model', 'custom/my-model'), + ('google/gemini-2.5-flash-preview-custom', 'google/gemini-2.5-flash-preview-custom'), +]) +def test_startup_persists_only_known_model_replacements(tmp_path, configured, expected): + config_dir = tmp_path / 'install' + (config_dir / 'config').mkdir(parents=True) + config_file = config_dir / 'config' / 'config.yaml' + translator = {'openrouter_temperature': 0.42, 'translator_type': 'openrouter'} + if configured is not None: + translator['openrouter_model'] = configured + config_file.write_text(yaml.safe_dump({'translator': translator}), encoding='utf-8') + repo = Path(__file__).resolve().parents[2] + script = ''' +import json +import os +import sys +sys.path[:0] = [os.path.join(sys.argv[1], 'custom_libs'), os.path.join(sys.argv[1], 'bazarr')] +os.environ['NO_CLI'] = 'true' +from app.get_args import args +args.config_dir = sys.argv[2] +args.no_tasks = True +from app.config import settings +print(json.dumps({'model': settings.translator.openrouter_model, + 'temperature': settings.translator.openrouter_temperature})) +''' + env = {**os.environ, 'PYTHONDONTWRITEBYTECODE': '1', 'XDG_CACHE_HOME': str(tmp_path / 'cache')} + first_contents = None + for _startup in range(2): + process = subprocess.run([sys.executable, '-c', script, str(repo), str(config_dir)], + cwd=repo, env=env, capture_output=True, text=True, timeout=30) + assert process.returncode == 0, process.stderr + assert json.loads(process.stdout) == {'model': expected, 'temperature': 0.42} + persisted = yaml.safe_load(config_file.read_text(encoding='utf-8')) + assert persisted['translator']['openrouter_model'] == expected + assert persisted['translator']['openrouter_temperature'] == 0.42 + if first_contents is None: + first_contents = config_file.read_bytes() + else: + assert config_file.read_bytes() == first_contents diff --git a/tests/bazarr/test_upload_background_sync.py b/tests/bazarr/test_upload_background_sync.py new file mode 100644 index 0000000000..b7ec83766c --- /dev/null +++ b/tests/bazarr/test_upload_background_sync.py @@ -0,0 +1,1738 @@ +"""Manual uploads remain visible while their separate sync job is running.""" + +from io import BytesIO +import ast +from pathlib import Path +from threading import Event, Thread +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def upload_flow(monkeypatch, tmp_path): + from app import jobs_queue as queue_module + from app.config import settings + from arr_instances import resolution + from languages import get_languages + from subtitles import manual, processing, sync, upload + from subtitles.indexer import movies, series + from subtitles.tools import delete, subsyncer + from subtitles.tools.subsync_engines import InMemorySubsyncFailureStore, SubsyncEngineRunner + + monkeypatch.setattr(queue_module.JobsQueue, "_flush_progress_loop", lambda self: None) + queue = queue_module.JobsQueue() + events, indexes, history, notifications, sync_history, engine_calls = [], [], [], [], [], [] + real_indexers = {"series": series.store_subtitles, "movie": movies.store_subtitles_movie} + controls = SimpleNamespace(fail=False) + engine_started, release_engine = Event(), Event() + video = tmp_path / "Video.mkv" + video.touch() + metadata = SimpleNamespace(sonarrSeriesId=10, sonarrEpisodeId=20, radarrId=30, + profileId=1, imdbId="tt1", tvdbId=1, tmdbId=1, + season=1, episode=1) + db = Mock() + db.execute.return_value.first.return_value = metadata + monkeypatch.setattr(get_languages, "languages_dict", [ + {"code2": "en", "code3": "eng", "code3b": "eng", "name": "English"}, + ], raising=False) + monkeypatch.setattr(upload, "database", db) + monkeypatch.setattr(delete, "database", db) + monkeypatch.setattr(upload, "get_profiles_list", lambda _: {"originalFormat": False}) + monkeypatch.setattr(upload, "get_audio_profile_languages", lambda _: []) + monkeypatch.setattr(upload, "client_for_instance", lambda *a, **k: None) + monkeypatch.setattr(delete, "client_for_instance", lambda *a, **k: None) + monkeypatch.setattr(delete, "call_external_webhook", lambda **k: None) + monkeypatch.setattr(upload, "get_target_folder", lambda _: None) + monkeypatch.setattr(manual, "clear_mismatch_after_manual_save", lambda *a: None) + monkeypatch.setattr(processing, "_postprocessing_config", lambda *a: (False, "", False, 0)) + monkeypatch.setattr(resolution, "_subtitle_settings_cache", {7: {}, 8: {}}) + monkeypatch.setattr(upload.path_mappings, "path_replace_reverse_instance", lambda path, *a: path) + monkeypatch.setattr(upload.path_mappings, "path_replace_instance", lambda path, *a: path) + monkeypatch.setattr("utilities.media_ids.local_episode_id", lambda episode_id, owner: 2000 + owner) + for name, value in {"single_language": False, "chmod_enabled": False, "utf8_encode": True, + "dont_notify_manual_actions": False, "use_plex": False, "use_jellyfin": False}.items(): + monkeypatch.setattr(settings.general, name, value) + for name, value in {"use_subsync": True, "use_subsync_threshold": False, + "use_subsync_movie_threshold": False, "enabled_engines": ["ffsubsync"], + "output_mode": "overwrite", "debug": False}.items(): + monkeypatch.setattr(settings.subsync, name, value) + + def event(**kwargs): + if kwargs.get("type") in ("movie", "series", "episode"): + assert indexes, "media events must follow the first subtitle index" + events.append(kwargs) + + def index(mapped_path, path, arr_instance_id=None): + indexes.append((arr_instance_id, {p.name: p.read_text() for p in Path(path).parent.glob("*.srt")})) + + for module in (queue_module, upload, sync): + monkeypatch.setattr(module, "jobs_queue", queue) + for module in (queue_module, upload, delete): + monkeypatch.setattr(module, "event_stream", event) + monkeypatch.setattr("app.event_handler.event_stream", event) + monkeypatch.setattr(series, "store_subtitles", index) + monkeypatch.setattr(movies, "store_subtitles_movie", index) + monkeypatch.setattr(delete, "store_subtitles", index) + monkeypatch.setattr(delete, "store_subtitles_movie", index) + for name in ("history_log", "history_log_movie"): + monkeypatch.setattr(upload, name, lambda *a, **k: history.append((a, k))) + monkeypatch.setattr(delete, name, lambda *a, **k: None) + for name in ("notify_sonarr", "notify_radarr", "send_notifications", "send_notifications_movie"): + monkeypatch.setattr(upload, name, lambda *a, **k: notifications.append((a, k))) + for name in ("notify_sonarr", "notify_radarr"): + monkeypatch.setattr(delete, name, lambda *a, **k: None) + monkeypatch.setattr(subsyncer.SubSyncer, "_log_sync_history", lambda *a, **k: sync_history.append(k)) + monkeypatch.setattr(subsyncer, "SubsyncEngineRunner", lambda: SubsyncEngineRunner(InMemorySubsyncFailureStore())) + + def engine(self, output_path, **kwargs): + engine_calls.append(kwargs) + source = Path(self.srtin).read_text() + engine_started.set() + assert release_engine.wait(5), "test did not release sync engine" + output_path.write_text("1\n00:00:03,000 --> 00:00:04,000\nSynced " + source) + if controls.fail: + raise RuntimeError("controlled sync failure") + return {"offset_seconds": 0, "framerate_scale_factor": 1} + + monkeypatch.setattr(subsyncer.SubSyncer, "_run_ffsubsync_engine", engine) + + def submit(media_type, content="Uploaded", owner=7, **kwargs): + return upload.manual_upload_subtitle( + path=str(video), language="en", forced=kwargs.pop("forced", False), + hi=kwargs.pop("hi", False), media_type=media_type, + subtitle=BytesIO(f"1\n00:00:01,000 --> 00:00:02,000\n{content}\n".encode()), + filename="upload.srt", audio_language=[], + sonarrSeriesId=kwargs.pop("sonarrSeriesId", 10 if media_type == "series" else None), + sonarrEpisodeId=20 if media_type == "series" else None, + radarrId=30 if media_type == "movie" else None, + arr_instance_id=owner, **kwargs) + + threads = [] + + def start(job_id=None): + job = next(job for job in queue.jobs_pending_queue if job_id is None or job.job_id == job_id) + queue.jobs_pending_queue.remove(job) + queue.jobs_running_queue.append(job) + thread = Thread(target=queue._run_job, args=(job,)) + threads.append(thread) + thread.start() + return job, thread + + def remove(media_type): + return delete.delete_subtitles(media_type=media_type, language="en", forced=False, hi=False, + media_path=str(video), subtitles_path=str(video.with_suffix(".en.srt")), + sonarr_series_id=10, sonarr_episode_id=20, radarr_id=30, arr_instance_id=7) + + yield SimpleNamespace(submit=submit, start=start, queue=queue, events=events, indexes=indexes, + history=history, notifications=notifications, engine_started=engine_started, + release_engine=release_engine, video=video, remove=remove, controls=controls, + sync_history=sync_history, engine_calls=engine_calls, real_indexers=real_indexers) + release_engine.set() + for thread in threads: + thread.join(5) + assert not thread.is_alive() + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +@pytest.mark.parametrize("failure", ["settings", "enqueue"]) +def test_saved_upload_refreshes_consumers_when_sync_setup_fails(upload_flow, monkeypatch, media_type, failure): + from app.config import settings + from subtitles import sync, upload + + flow = upload_flow + refreshes = [] + monkeypatch.setattr(settings.general, 'use_plex', True) + monkeypatch.setattr(settings.general, 'use_jellyfin', True) + for service in (settings.plex, settings.jellyfin): + monkeypatch.setattr(service, 'update_movie_library', True) + monkeypatch.setattr(service, 'update_series_library', True) + monkeypatch.setattr(settings.plex, 'set_episode_added', False) + monkeypatch.setattr(settings.plex, 'set_movie_added', False) + + def consumer(name): + def refresh(*args, **kwargs): + assert flow.indexes, 'consumers must see an indexed source publication' + refreshes.append((name, args, kwargs)) + return refresh + + for name in ('notify_sonarr', 'notify_radarr', 'plex_refresh_item', 'jellyfin_refresh_item'): + monkeypatch.setattr(upload, name, consumer(name)) + + def fail(*args, **kwargs): + raise RuntimeError('controlled sync setup failure') + + if failure == 'settings': + monkeypatch.setattr(sync, '_resolve_subsync_overrides', fail) + else: + monkeypatch.setattr(flow.queue, 'add_job_from_function', fail) + + with pytest.raises(RuntimeError, match='controlled sync setup failure'): + flow.submit(media_type, job_id='existing-upload-job') + + assert 'Uploaded' in flow.video.with_suffix('.en.srt').read_text() + assert flow.indexes + assert len(flow.history) == 1 + assert not flow.queue.jobs_pending_queue + assert [name for name, args, kwargs in refreshes] == [ + 'notify_sonarr' if media_type == 'series' else 'notify_radarr', + 'plex_refresh_item', 'jellyfin_refresh_item'] + assert refreshes[0][1] == (10 if media_type == 'series' else 30,) + assert refreshes[1][2]['is_movie'] == (media_type == 'movie') + assert refreshes[2][2]['is_movie'] == (media_type == 'movie') + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +def test_saved_upload_is_visible_and_upload_completes_before_sync(upload_flow, media_type): + flow = upload_flow + upload_id = flow.submit(media_type) + upload_job, upload_thread = flow.start() + upload_thread.join(1) + + assert (flow.video.parent / "Video.en.srt").is_file() + assert upload_job.status != "failed" + assert flow.indexes, "the saved subtitle is still hidden while sync is pending/running" + assert upload_job.status == "completed" + assert upload_job.job_returned_value == ("", 204) + assert len(flow.history) == 1 + assert len(flow.queue.jobs_pending_queue) == 1 + sync_job, sync_thread = flow.start() + assert flow.engine_started.wait(2) + assert sync_job.job_id != upload_id + assert sync_job.is_progress + assert sync_job.status == "running" + assert "Uploaded" in flow.indexes[-1][1]["Video.en.srt"] + flow.release_engine.set() + sync_thread.join(3) + assert sync_job.status == "completed" + assert "Synced" in flow.indexes[-1][1]["Video.en.srt"] + assert len(flow.history) == 1 + assert sum("arr_client" in kwargs for _, kwargs in flow.notifications) == 2 + assert sum("arr_instance_id" in kwargs for _, kwargs in flow.notifications) == 1 + assert len(flow.sync_history) == 1 + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +@pytest.mark.parametrize("output_mode", ["overwrite", "keep_all"]) +@pytest.mark.parametrize("running", [False, True], ids=["queued", "running"]) +@pytest.mark.parametrize("change", ["upload", "delete"]) +def test_stale_sync_does_not_replace_a_new_upload_or_restore_a_deleted_file( + upload_flow, monkeypatch, media_type, output_mode, running, change): + from app.config import settings + + monkeypatch.setattr(settings.subsync, "output_mode", output_mode) + flow = upload_flow + flow.submit(media_type) + job, thread = flow.start() + thread.join(2) + assert job.status == "completed" + sync_id = flow.queue.jobs_pending_queue[0].job_id + if running: + sync_job, sync_thread = flow.start(sync_id) + assert flow.engine_started.wait(2) + + source = flow.video.parent / "Video.en.srt" + if change == "upload": + second_upload_id = flow.submit(media_type, content="New upload") + second_job, second_thread = flow.start(second_upload_id) + second_thread.join(2) + assert second_job.status == "completed" + assert "New upload" in source.read_text() + else: + source.unlink() + + flow.release_engine.set() + if not running: + sync_job, sync_thread = flow.start(sync_id) + sync_thread.join(2) + assert sync_job.status == "completed" + if not running: + assert not flow.engine_started.is_set(), "outdated queued sync should skip its engines" + assert not (flow.video.parent / "Video.en.ffsubsync.srt").exists() + if change == "upload": + assert "New upload" in source.read_text() + assert "Synced" not in source.read_text() + latest_sync, latest_thread = flow.start() + latest_thread.join(2) + assert latest_sync.status == "completed" + output = source if output_mode == "overwrite" else source.with_name("Video.en.ffsubsync.srt") + assert "Synced" in output.read_text() + assert "New upload" in output.read_text() + else: + assert not source.exists() + assert not list(flow.video.parent.glob(".bazarr-sync-*")) + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +def test_delete_during_sync_publication_cannot_be_undone(upload_flow, monkeypatch, media_type): + from subtitles.tools import subsync_engines + + flow = upload_flow + flow.submit(media_type) + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + publishing, publish = Event(), Event() + real_replace = subsync_engines.os.replace + + def paused_replace(source, destination): + publishing.set() + assert publish.wait(5) + return real_replace(source, destination) + + monkeypatch.setattr(subsync_engines.os, "replace", paused_replace) + flow.release_engine.set() + sync_job, sync_thread = flow.start() + assert publishing.wait(2) + deletion = Thread(target=flow.remove, args=(media_type,)) + deletion.start() + try: + deletion.join(0.1) + assert deletion.is_alive(), "deletion must serialize with the checked output publication" + finally: + publish.set() + sync_thread.join(2) + deletion.join(2) + assert not deletion.is_alive() + assert not flow.video.with_suffix(".en.srt").exists() + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +@pytest.mark.parametrize("outcome", ["failure", "cancel", "disabled", "forced", "threshold"]) +def test_optional_sync_failure_or_skip_keeps_the_successful_upload(upload_flow, monkeypatch, media_type, outcome): + from app.config import settings + + flow = upload_flow + if outcome == "disabled": + monkeypatch.setattr(settings.subsync, "use_subsync", False) + if outcome == "threshold": + threshold = "subsync_threshold" if media_type == "series" else "subsync_movie_threshold" + monkeypatch.setattr(settings.subsync, "use_" + threshold, True) + monkeypatch.setattr(settings.subsync, threshold, 99) + flow.controls.fail = outcome == "failure" + flow.submit(media_type, forced=outcome == "forced") + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + before = flow.indexes[-1] + if outcome == "disabled": + assert not flow.queue.jobs_pending_queue + else: + sync_job, sync_thread = flow.start() + if outcome in ("cancel", "failure"): + assert flow.engine_started.wait(2) + if outcome == "cancel": + assert flow.queue.cancel_running_job(sync_job.job_id) + flow.release_engine.set() + sync_thread.join(2) + assert sync_job.status == "completed" + if outcome == "cancel": + assert sync_job.progress_message == "Cancelled by user" + assert sync_job.job_returned_value is not True + assert flow.indexes[-1] == before + assert len(flow.history) == 1 + assert len(flow.notifications) == 2 + assert not flow.sync_history + assert not list(flow.video.parent.glob(".bazarr-sync-*")) + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +def test_upload_preserves_postprocessing_hi_and_notification_policy(upload_flow, monkeypatch, media_type): + from app.config import settings + from subtitles import processing, upload + + flow = upload_flow + monkeypatch.setattr(settings.general, "dont_notify_manual_actions", True) + monkeypatch.setattr(processing, "_postprocessing_config", lambda kind, owner: (True, "process", False, 0)) + monkeypatch.setattr(upload, "pp_replace", lambda *args: "process") + monkeypatch.setattr(upload, "set_chmod", lambda **kwargs: None) + + def postprocess(command, video_path, subtitle_path=None): + assert command == "process" + flow.video.with_suffix(".en.hi.srt").write_text("1\n00:00:01,000 --> 00:00:02,000\nProcessed\n") + + monkeypatch.setattr(upload, "postprocessing", postprocess) + flow.submit(media_type, hi=True) + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + assert "Processed" in flow.indexes[-1][1]["Video.en.hi.srt"] + flow.release_engine.set() + sync_job, sync_thread = flow.start() + sync_thread.join(2) + assert sync_job.status == "completed" + assert "Synced" in flow.indexes[-1][1]["Video.en.hi.srt"] + assert "Processed" in flow.indexes[-1][1]["Video.en.hi.srt"] + assert sum("arr_client" in kwargs for _, kwargs in flow.notifications) == 2 + assert not any("arr_instance_id" in kwargs for _, kwargs in flow.notifications) + assert flow.sync_history[0]["hi"] is True + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +def test_real_index_and_settings_stay_with_the_upload_owner(upload_flow, schema_session, monkeypatch, media_type): + from app.config import settings + from app.database import TableEpisodes, TableMovies, TableShows, select + from arr_instances import resolution + from subzero.language import Language + from subtitles import upload + from subtitles.indexer import movies, series + from subtitles.tools import subsyncer + + flow = upload_flow + module = series if media_type == "series" else movies + table = TableEpisodes if media_type == "series" else TableMovies + for owner in (7, 8): + if media_type == "series": + schema_session.add(TableShows(id=owner, arr_instance_id=owner, sonarrSeriesId=10, + title="Show", path=str(flow.video.parent), profileId=None)) + schema_session.flush() + row = TableEpisodes(id=owner, arr_instance_id=owner, series_id=owner, + sonarrSeriesId=10, sonarrEpisodeId=20, title="Episode", path=str(flow.video), + season=1, episode=1, subtitles="[]") + else: + row = TableMovies(id=owner, arr_instance_id=owner, radarrId=30, title="Movie", + path=str(flow.video), tmdbId=str(owner), profileId=None, subtitles="[]") + schema_session.add(row) + schema_session.commit() + monkeypatch.setattr(upload, "database", schema_session) + monkeypatch.setattr(module, "database", schema_session) + monkeypatch.setattr(settings.general, "use_embedded_subs", False) + monkeypatch.setattr(settings.general, "subfolder", "current") + monkeypatch.setattr(module, "get_language_set", lambda: {Language.fromietf("en")}) + monkeypatch.setattr(module.core, "CUSTOM_PATHS", []) + missing = "list_missing_subtitles" if media_type == "series" else "list_missing_subtitles_movies" + monkeypatch.setattr(module, missing, lambda *a, **k: None) + monkeypatch.setattr(module, "event_stream", lambda **kwargs: None) + real_index = flow.real_indexers[media_type] + + def index(*args, **kwargs): + result = real_index(*args, **kwargs) + flow.indexes.append((kwargs["arr_instance_id"], result)) + return result + + monkeypatch.setattr(module, "store_subtitles" if media_type == "series" else "store_subtitles_movie", index) + resolution._subtitle_settings_cache[7] = {"subsync": {"use_subsync": False}} + resolution._subtitle_settings_cache[8] = {"subsync": { + "use_subsync": True, "enabled_engines": ["alass"], "max_offset_seconds": 17, + "use_subsync_threshold": True, "subsync_threshold": 100, + "use_subsync_movie_threshold": True, "subsync_movie_threshold": 100, + }} + monkeypatch.setattr(settings.subsync, "use_subsync", False) + monkeypatch.setattr(settings.subsync, "output_mode", "keep_all") + external_calls = [] + + def external(self, engine, output_path, video_path): + external_calls.append(engine) + output_path.write_text("1\n00:00:03,000 --> 00:00:04,000\nSynced\n") + return {"offset_seconds": 16, "framerate_scale_factor": 1} + + monkeypatch.setattr(subsyncer.SubSyncer, "_run_external_engine", external) + flow.submit(media_type, owner=8) + + def run_next(): + job = flow.queue.jobs_pending_queue.popleft() + flow.queue.jobs_running_queue.append(job) + assert flow.queue._run_job(job) + return job + + upload_job = run_next() + assert upload_job.job_returned_value == ("", 204) + before = ast.literal_eval(schema_session.execute(select(table.subtitles).where(table.id == 8)).scalar()) + assert len(before) == 1 and before[0][0] == "en" + assert schema_session.execute(select(table.subtitles).where(table.id == 7)).scalar() == "[]" + sync_job = run_next() + assert sync_job.job_returned_value is True + after = ast.literal_eval(schema_session.execute(select(table.subtitles).where(table.id == 8)).scalar()) + assert {row[0] for row in after} == {"en", "en:sync-alass"} + assert schema_session.execute(select(table.subtitles).where(table.id == 7)).scalar() == "[]" + assert external_calls == ["alass"] + assert all(row[0] == 8 for row in flow.indexes) + assert flow.history[0][1]["arr_instance_id"] == 8 + assert flow.sync_history[0]["arr_instance_id"] == 8 + + +def test_unrelated_upload_finishes_while_other_media_postprocessing_is_blocked(upload_flow, monkeypatch): + from subtitles import processing, upload + + flow = upload_flow + blocked, unblock, second_entered = Event(), Event(), Event() + + def config(kind, owner): + if owner == 8: + second_entered.set() + return (owner == 7, 'controlled', False, 0) + + def postprocess(command, video_path, subtitle_path=None): + blocked.set() + assert unblock.wait(5), 'probe failed to release postprocessing' + + monkeypatch.setattr(processing, '_postprocessing_config', config) + monkeypatch.setattr(upload, 'pp_replace', lambda *args: 'controlled') + monkeypatch.setattr(upload, 'postprocessing', postprocess) + monkeypatch.setattr(upload, 'set_chmod', lambda **kwargs: None) + flow.submit('movie', owner=7) + first_job, first_thread = flow.start() + assert blocked.wait(2) + other_video = flow.video.with_name('Unrelated.mkv') + other_video.touch() + second_id = upload.manual_upload_subtitle( + path=str(other_video), language='en', forced=False, hi=False, media_type='movie', + subtitle=BytesIO(b'1\n00:00:01,000 --> 00:00:02,000\nOther media\n'), + filename='other.srt', audio_language=[], radarrId=31, arr_instance_id=8) + second_job, second_thread = flow.start(second_id) + try: + assert second_entered.wait(2), 'second upload did not start' + second_thread.join(0.25) + completed_before_release = second_job.status == 'completed' + saved_before_release = other_video.with_suffix('.en.srt').exists() + finally: + unblock.set() + first_thread.join(2) + second_thread.join(2) + assert not first_thread.is_alive() and not second_thread.is_alive() + assert first_job.status == second_job.status == 'completed' + assert completed_before_release and saved_before_release, 'unrelated upload was blocked by another media postprocessing lock' + + +@pytest.mark.parametrize("output_mode", ["overwrite", "keep_all"]) +def test_cancel_while_finished_engine_waits_for_publication_preserves_upload(upload_flow, monkeypatch, output_mode): + from app.config import settings + monkeypatch.setattr(settings.subsync, "output_mode", output_mode) + from subtitles.tools import subsync_engines, subsyncer + + flow = upload_flow + flow.submit('movie') + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == 'completed' + original = flow.video.with_suffix('.en.srt').read_bytes() + finished_checkpoint = Event() + real_report = subsyncer.SubSyncer._report_progress + + def report(self, message, value, total): + result = real_report(self, message, value, total) + if message.startswith('Finished '): + finished_checkpoint.set() + return result + + monkeypatch.setattr(subsyncer.SubSyncer, '_report_progress', report) + with subsync_engines.subtitle_write_lock(str(flow.video), str(flow.video.parent)): + flow.release_engine.set() + sync_job, sync_thread = flow.start() + assert finished_checkpoint.wait(2), 'engine did not reach publication boundary' + assert flow.video.with_suffix('.en.srt').read_bytes() == original + assert flow.queue.cancel_running_job(sync_job.job_id) + sync_thread.join(2) + assert not sync_thread.is_alive() + assert sync_job.progress_message == 'Cancelled by user' + assert flow.video.with_suffix('.en.srt').read_bytes() == original + assert not flow.video.with_suffix('.en.ffsubsync.srt').exists() + assert not list(flow.video.parent.glob('.bazarr-sync-*')) + assert not flow.sync_history + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +def test_sync_reindex_cannot_restore_a_listing_after_successful_delete(upload_flow, monkeypatch, media_type): + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + from sqlalchemy.sql.dml import Update + from app.config import settings + from app.database import Base, TableEpisodes, TableMovies, TableShows, select + from subzero.language import Language + from subtitles import upload + from subtitles.indexer import movies, series + from subtitles.tools import delete + + flow = upload_flow + module = series if media_type == "series" else movies + table = TableEpisodes if media_type == "series" else TableMovies + engine = create_engine(f"sqlite:///{flow.video.parent / 'index.sqlite'}") + Base.metadata.create_all(engine) + Session = sessionmaker(bind=engine) + with Session.begin() as session: + if media_type == "series": + session.add(TableShows(id=7, arr_instance_id=7, sonarrSeriesId=10, + title="Show", path=str(flow.video.parent), profileId=None)) + session.flush() + session.add(TableEpisodes(id=7, arr_instance_id=7, series_id=7, sonarrSeriesId=10, + sonarrEpisodeId=20, title="Episode", path=str(flow.video), + season=1, episode=1, subtitles="[]")) + else: + session.add(TableMovies(id=7, arr_instance_id=7, radarrId=30, title="Movie", + path=str(flow.video), tmdbId="1", profileId=None, subtitles="[]")) + before_commit, commit = Event(), Event() + controls = {"armed": False} + + class Database: + def execute(self, statement, *args, **kwargs): + if controls["armed"] and isinstance(statement, Update): + controls["armed"] = False + before_commit.set() + assert commit.wait(5) + with Session.begin() as session: + result = session.execute(statement, *args, **kwargs) + return result.freeze()() if statement.is_select else result + + db = Database() + for item in (upload, delete, module): + monkeypatch.setattr(item, "database", db) + monkeypatch.setattr(settings.general, "use_embedded_subs", False) + monkeypatch.setattr(settings.general, "subfolder", "current") + monkeypatch.setattr(module, "get_language_set", lambda: {Language.fromietf("en")}) + missing = "list_missing_subtitles" if media_type == "series" else "list_missing_subtitles_movies" + monkeypatch.setattr(module, missing, lambda *a, **k: None) + monkeypatch.setattr(module, "event_stream", lambda **k: None) + + def index(*args, **kwargs): + result = flow.real_indexers[media_type](*args, **kwargs) + flow.indexes.append((kwargs["arr_instance_id"], result)) + return result + + index_name = "store_subtitles" if media_type == "series" else "store_subtitles_movie" + monkeypatch.setattr(module, index_name, index) + monkeypatch.setattr(delete, index_name, index) + deletion = None + try: + flow.submit(media_type) + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + assert len(ast.literal_eval(db.execute(select(table.subtitles)).scalar())) == 1 + controls["armed"] = True + flow.release_engine.set() + sync_job, sync_thread = flow.start() + assert before_commit.wait(2) + deleted = [] + deletion = Thread(target=lambda: deleted.append(flow.remove(media_type))) + deletion.start() + deletion.join(0.2) + commit.set() + sync_thread.join(2) + deletion.join(2) + assert not deletion.is_alive() and not sync_thread.is_alive() + assert sync_job.status == "completed" + assert deleted == [True] + assert not flow.video.with_suffix(".en.srt").exists() + assert ast.literal_eval(db.execute(select(table.subtitles)).scalar()) == [] + finally: + commit.set() + if deletion: + deletion.join(2) + engine.dispose() + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +@pytest.mark.parametrize("output_mode", ["overwrite", "keep_all"]) +def test_review_sync_publication_refreshes_external_consumers(upload_flow, monkeypatch, media_type, output_mode): + from app.config import settings + from subtitles import upload + + flow = upload_flow + monkeypatch.setattr(settings.subsync, "output_mode", output_mode) + monkeypatch.setattr(settings.general, "use_plex", True) + monkeypatch.setattr(settings.general, "use_jellyfin", True) + for name in ("update_series_library", "update_movie_library"): + monkeypatch.setattr(settings.plex, name, True) + monkeypatch.setattr(settings.jellyfin, name, True) + monkeypatch.setattr(settings.plex, "set_movie_added", False) + monkeypatch.setattr(settings.plex, "set_episode_added", False) + refreshes = [] + + def refresh(consumer, *args, **kwargs): + refreshes.append((consumer, {path.name: path.read_text() for path in flow.video.parent.glob("*.srt")})) + + for name in ("notify_sonarr", "notify_radarr", "plex_refresh_item", "jellyfin_refresh_item"): + monkeypatch.setattr(upload, name, lambda *args, _consumer=name, **kwargs: refresh(_consumer, *args, **kwargs)) + flow.submit(media_type) + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + assert len(refreshes) == 3 + flow.release_engine.set() + sync_job, sync_thread = flow.start() + sync_thread.join(3) + assert sync_job.status == "completed" + assert len(refreshes) == 6, "external consumers were not refreshed after sync publication" + expected = "Video.en.ffsubsync.srt" if output_mode == "keep_all" else "Video.en.srt" + assert all("Synced" in files[expected] for _, files in refreshes[3:]) + assert len(flow.history) == 1 + assert len(flow.notifications) == 1, "sync publication repeated the upload notification" + + +def test_review_upload_sync_uses_resolved_series_id(upload_flow): + flow = upload_flow + flow.submit("series", sonarrSeriesId=999) + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + sync_job = flow.queue.jobs_pending_queue[0] + assert sync_job.kwargs["sonarr_series_id"] == 10, "unvalidated request ID reached synchronization" + assert sync_job.kwargs["callback"].keywords["sonarr_series_id"] == 10 + assert sync_job.kwargs["sonarr_episode_id"] == 20 + assert sync_job.kwargs["arr_instance_id"] == 7 + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +@pytest.mark.parametrize("outcome", ["success", "failure", "cancel", "disabled"]) +def test_review_replacement_upload_does_not_list_stale_generated_outputs(upload_flow, monkeypatch, media_type, outcome): + from app.config import settings + + flow = upload_flow + monkeypatch.setattr(settings.subsync, "output_mode", "keep_all") + old_output = flow.video.with_suffix(".en.ffsubsync.srt") + old_bytes = b"1\n00:00:01,000 --> 00:00:02,000\nOld generated output\n" + old_output.write_bytes(old_bytes) + if outcome == "disabled": + monkeypatch.setattr(settings.subsync, "use_subsync", False) + flow.submit(media_type, content="Replacement upload") + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + assert "Replacement upload" in flow.indexes[-1][1]["Video.en.srt"] + assert old_output.name not in flow.indexes[-1][1], "the initial upload index exposed old generated text" + assert not old_output.exists() + assert any(path.read_bytes() == old_bytes for path in flow.video.parent.rglob("*") if path.is_file()), \ + "old generated bytes were destroyed instead of quarantined" + if outcome == "disabled": + assert not flow.queue.jobs_pending_queue + return + flow.controls.fail = outcome == "failure" + sync_job, sync_thread = flow.start() + assert flow.engine_started.wait(2) + if outcome == "cancel": + flow.queue.cancel_running_job(sync_job.job_id) + flow.release_engine.set() + sync_thread.join(3) + assert not sync_thread.is_alive() + if outcome == "success": + assert "Replacement upload" in old_output.read_text() + assert "Old generated output" not in old_output.read_text() + else: + assert not old_output.exists() + assert old_output.name not in flow.indexes[-1][1] + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +@pytest.mark.parametrize("running", [False, True], ids=["queued", "running"]) +def test_review_deleted_generated_destination_stays_deleted(upload_flow, monkeypatch, media_type, running): + from app.config import settings + from subtitles.tools import delete + + flow = upload_flow + monkeypatch.setattr(settings.subsync, "output_mode", "keep_all") + destination = flow.video.with_suffix(".en.ffsubsync.srt") + destination.write_text("Old generated output") + flow.submit(media_type) + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + if running: + sync_job, sync_thread = flow.start() + assert flow.engine_started.wait(2) + delete.delete_subtitles(media_type=media_type, language="en", forced=False, hi=False, + media_path=str(flow.video), subtitles_path=str(destination), + sonarr_series_id=10, sonarr_episode_id=20, radarr_id=30, arr_instance_id=7) + assert not destination.exists() + flow.release_engine.set() + if not running: + sync_job, sync_thread = flow.start() + sync_thread.join(3) + assert not sync_thread.is_alive() + assert not destination.exists(), "background sync recreated an explicitly deleted generated destination" + assert destination.name not in flow.indexes[-1][1] + + +@pytest.mark.parametrize("output_mode", ["overwrite", "keep_all"]) +@pytest.mark.parametrize("operation", ["edit", "edit-etag", "promote", "mods", "manual-sync", "google", "lingarr", "openrouter", "gemini", "manual-download", "auto-download", "postprocessing"]) +def test_review_editor_write_cannot_be_lost_at_sync_publication(upload_flow, monkeypatch, output_mode, operation): + import importlib + import os + import sys + from threading import current_thread + from flask import Flask + from app.config import settings + from subtitles.tools import subsync_engines + + existing = sys.modules.get("api.subtitles.content") + if existing is not None and not getattr(existing, "__file__", None): + monkeypatch.delitem(sys.modules, "api.subtitles.content") + with monkeypatch.context() as importing: + importing.setattr("app.check_update.check_releases", lambda *args, **kwargs: None) + importing.setattr("app.announcements.get_announcements_to_file", lambda *args, **kwargs: None) + content = importlib.import_module("api.subtitles.content") + flow = upload_flow + monkeypatch.setattr(settings.subsync, "output_mode", output_mode) + source = flow.video.with_suffix(".en.srt") + output = flow.video.with_suffix(".en.ffsubsync.srt") + promotion = flow.video.with_suffix(".en.alass.srt") + user_text = "1\n00:00:01,000 --> 00:00:02,000\nUser selected text\n" + metadata = {"mediaPath": str(flow.video), "mediaId": 30, "arrInstanceId": 7} + monkeypatch.setattr(content, "resolve_subtitle_path", lambda media_type, media_id, language, **kwargs: + (str(promotion if language == "en:sync-alass" else source), metadata)) + monkeypatch.setattr(content, "store_subtitles_movie", lambda *args, **kwargs: None) + monkeypatch.setattr(content, "event_stream", lambda **kwargs: None) + monkeypatch.setattr(content, "history_log_movie", lambda **kwargs: None) + monkeypatch.setattr(content, "language_from_alpha2", lambda value: "English") + reached_publication, release_publication, writer_started, writer_done = Event(), Event(), Event(), Event() + real_replace = os.replace + published_destination = output if output_mode == "keep_all" else source + + def replace(temporary, destination): + if (Path(destination) == published_destination and Path(temporary).name.startswith(".bazarr-sync-") + and current_thread().name != "competing-subtitle-writer"): + reached_publication.set() + assert release_publication.wait(5), "test did not release sync publication" + return real_replace(temporary, destination) + + monkeypatch.setattr(subsync_engines.os, "replace", replace) + flow.submit("movie") + upload_job, upload_thread = flow.start() + upload_thread.join(2) + assert upload_job.status == "completed" + promotion.write_text(user_text) + initial_etag = content.generate_etag(str(source)) + sync_job, sync_thread = flow.start() + assert flow.engine_started.wait(2) + flow.release_engine.set() + writer_results = [] + writer_errors = [] + + def write(): + try: + writer_started.set() + if operation in ("edit", "edit-etag"): + app = Flask(__name__) + with app.test_request_context(json={"content": user_text}, + headers={"If-Match": initial_etag} if operation == "edit-etag" else {}): + response = content._save_subtitle_content("movie", 30, "en", arr_instance_id=7) + writer_results.append(response.status_code if hasattr(response, "status_code") else response[1]) + elif operation == "promote": + _, status = content.promote_sync_subtitle("movie", 30, "en", "en:sync-alass", arr_instance_id=7) + writer_results.append(status) + elif operation == "mods": + from subtitles.tools import mods + monkeypatch.setattr(mods.Subtitle, "get_modified_content", lambda self, **kwargs: user_text.encode()) + mods.subtitles_apply_mods("en", str(source), [], str(flow.video), arr_instance_id=7) + writer_results.append(200) + elif operation == "manual-sync": + from subtitles.tools import subsyncer + + def external(self, engine, output_path, video_path): + output_path.write_text(user_text) + return {"offset_seconds": 0} + + monkeypatch.setattr(subsyncer.SubSyncer, "_run_external_engine", external) + result = subsyncer.SubSyncer().sync( + str(flow.video), str(source), "en", False, False, 60, False, False, + output_mode="overwrite", enabled_engines=["alass"], write_history=False) + writer_results.append(200 if result.success else 500) + elif operation in ("manual-download", "auto-download"): + from subtitles import manual, download, pool + from subliminal_patch.subtitle import Subtitle + from subzero.language import Language + + subtitle = Subtitle(Language("eng"), original_format=True) + subtitle.content = user_text.encode() + subtitle.provider_name = "fixture" + subtitle.matches = set() + video = Mock(original_path=str(flow.video)) + provider_pool = Mock(providers=["fixture"], discarded_providers=[]) + monkeypatch.setattr(pool, "_update_pool", lambda *args, **kwargs: False) + module = manual if operation == "manual-download" else download + monkeypatch.setattr(module, "get_video", lambda *args, **kwargs: video) + monkeypatch.setattr(module, "_get_pool", lambda *args, **kwargs: provider_pool) + monkeypatch.setattr(module, "get_target_folder", lambda *args, **kwargs: None) + monkeypatch.setattr(module, "process_subtitle", lambda **kwargs: "saved") + monkeypatch.setattr(module.subliminal, "region", Mock()) + if operation == "manual-download": + monkeypatch.setattr(manual.subtitle_cache, "get", lambda *args: subtitle) + monkeypatch.setattr(manual, "download_subtitles", lambda *args: None) + result = manual.manual_download_subtitle( + str(flow.video), [], False, False, "cached", "fixture", None, "Video", "movie", True, 1, + arr_instance_id=7) + assert result == "saved" + else: + monkeypatch.setattr(download, "_get_language_obj", lambda **kwargs: {Language("eng")}) + monkeypatch.setattr(download, "get_profiles_list", lambda **kwargs: {"originalFormat": True}) + monkeypatch.setattr(download, "download_best_subtitles", lambda **kwargs: {video: [subtitle]}) + monkeypatch.setattr(download, "clear_mismatch_for_video", lambda *args, **kwargs: None) + result = list(download.generate_subtitles( + str(flow.video), [("en", False, False)], [], None, "Video", "movie", 1, arr_instance_id=7)) + assert result == ["saved"] + writer_results.append(200) + elif operation == "postprocessing": + from subtitles import post_processing + + def popen(*args, **kwargs): + source.write_text(user_text) + return Mock(communicate=lambda: ("", "")) + + monkeypatch.setattr(post_processing.subprocess, "Popen", popen) + post_processing.postprocessing("fixture command", str(flow.video)) + writer_results.append(200) + else: + module = importlib.import_module(f"subtitles.tools.translate.services.{operation}_translator") + classes = {"google": "GoogleTranslatorService", "lingarr": "LingarrTranslatorService", + "openrouter": "OpenRouterTranslatorService", "gemini": "GeminiTranslatorService"} + service = getattr(module, classes[operation])( + source_srt_file=str(promotion), dest_srt_file=str(source), lang_obj=None, + to_lang="eng", from_lang="en", media_type="movie", video_path=str(flow.video), + orig_to_lang="en", forced=False, hi=False, sonarr_series_id=None, + sonarr_episode_id=None, radarr_id=30) + monkeypatch.setattr(module, "jobs_queue", Mock()) + monkeypatch.setattr(module, "history_log_movie", Mock()) + monkeypatch.setattr(module, "create_process_result", Mock()) + monkeypatch.setattr(module, "add_translator_info", lambda *args: None) + if operation == "google": + monkeypatch.setattr(service, "_translate_text", lambda *args: "User selected text") + elif operation == "lingarr": + monkeypatch.setattr(service, "_translate_content", lambda *args, **kwargs: + [{"position": 0, "line": "User selected text"}]) + elif operation == "openrouter": + monkeypatch.setattr(service, "_submit_and_poll", lambda *args, **kwargs: + [{"position": 0, "line": "User selected text"}]) + else: + monkeypatch.setattr(service, "_get_configured_api_keys", lambda: []) + monkeypatch.setattr(service, "_select_next_api_key", lambda: "fixture") + monkeypatch.setattr(service, "_check_saved_progress", lambda: None) + monkeypatch.setattr(module, "get_description", lambda *args, **kwargs: "") + monkeypatch.setattr(service, "_translate_with_gemini", lambda: + Path(service.output_file).write_text(user_text)) + service.translate(job_id=123) + writer_results.append(200) + + except BaseException as exc: + writer_errors.append(exc) + finally: + writer_done.set() + + writer = Thread(target=write, name="competing-subtitle-writer") + try: + assert reached_publication.wait(2) + writer.start() + assert writer_started.wait(2) + writer_done.wait(0.3) + release_publication.set() + sync_thread.join(3) + writer.join(3) + assert not writer.is_alive() and not sync_thread.is_alive() + assert not writer_errors + if operation == "edit-etag" and output_mode == "overwrite": + assert writer_results == [412] + assert "Synced" in source.read_text() + else: + assert writer_results == [204 if operation in ("edit", "edit-etag") else 200] + assert "User selected text" in source.read_text(), "sync publication overwrote the completed user write" + if output_mode == "keep_all": + assert not output.exists(), "an output for the superseded source remained available" + finally: + release_publication.set() + if writer.ident: + writer.join(3) + + +@pytest.mark.parametrize("failure", ["cancel-before-engine", "setup-error", "settings-error"]) +def test_review_terminal_job_releases_publication_lease(upload_flow, monkeypatch, failure): + from subtitles import sync + + flow = upload_flow + flow.submit("movie") + _, upload_thread = flow.start() + upload_thread.join(2) + job = flow.queue.jobs_pending_queue[0] + publication = job.kwargs["source_version"] + if failure == "settings-error": + monkeypatch.setattr(sync, "_resolve_subsync_overrides", Mock(side_effect=RuntimeError("controlled settings failure"))) + elif failure == "setup-error": + monkeypatch.setattr(sync, "SubSyncer", Mock(side_effect=RuntimeError("controlled setup failure"))) + else: + original = sync._report_progress + + def report(*args, **kwargs): + flow.queue.cancel_running_job(job.job_id) + return original(*args, **kwargs) + + monkeypatch.setattr(sync, "_report_progress", report) + _, thread = flow.start() + thread.join(3) + assert not thread.is_alive() + assert job.status in ("completed", "failed") + assert publication.state is None, "terminal Job kwargs retained the media coordinator" + + +def test_review_partial_publication_refreshes_consumers_after_cancellation(upload_flow, monkeypatch): + from app.config import settings + from app.jobs_queue import JobCancelled + from subtitles.tools import subsyncer + + flow = upload_flow + monkeypatch.setattr(settings.subsync, "output_mode", "keep_all") + monkeypatch.setattr(settings.subsync, "enabled_engines", ["ffsubsync", "alass"]) + def cancel(self, **kwargs): + flow.queue.cancel_running_job(self.job_id) + raise JobCancelled() + + monkeypatch.setattr(subsyncer.SubSyncer, "_run_external_engine", cancel) + flow.submit("movie") + _, thread = flow.start() + thread.join(2) + flow.release_engine.set() + job, thread = flow.start() + thread.join(3) + assert job.status == "completed" and job.cancelled + assert flow.video.with_suffix(".en.ffsubsync.srt").is_file() + assert sum("arr_client" in kwargs for _, kwargs in flow.notifications) == 2 + assert sum("arr_instance_id" in kwargs for _, kwargs in flow.notifications) == 1 + assert len(flow.history) == 1 + + +def test_review_failed_delete_preserves_existing_generated_output(upload_flow, monkeypatch): + import os + from subtitles.tools import delete + + flow = upload_flow + source = flow.video.with_suffix(".en.srt") + output = flow.video.with_suffix(".en.ffsubsync.srt") + source.write_text("Base") + output.write_text("Valid generated output") + remove = os.remove + + def reject(path): + if Path(path) == source: + raise PermissionError("controlled rejected delete") + return remove(path) + + monkeypatch.setattr(delete.os, "remove", reject) + assert flow.remove("movie") is False + assert source.read_text() == "Base" + assert output.read_text() == "Valid generated output" + + +@pytest.mark.parametrize("collision", [False, True], ids=["same-mapped-owner", "distinct-owners"]) +def test_review_shared_folder_quarantine_requires_unique_mapped_owner(upload_flow, schema_session, monkeypatch, collision): + from app import database as db_module + from app.database import TableMovies + from app.config import settings + from subtitles.tools.subsync_engines import quarantine_sync_outputs, SubtitlePublication, subtitle_source_version + + flow = upload_flow + folder = flow.video.parent / "shared" + folder.mkdir() + other = flow.video.parent / "other" / flow.video.name + other.parent.mkdir() + other.touch() + source = folder / "Video.en.srt" + output = folder / "Video.en.ffsubsync.srt" + source.write_text("Base") + output.write_text("Existing generated bytes") + for owner, path in [(7, flow.video), (8, other if collision else flow.video)]: + schema_session.add(TableMovies(id=owner, arr_instance_id=owner, radarrId=30, + title="Movie", path=str(path), tmdbId=str(owner), subtitles="[]")) + schema_session.commit() + monkeypatch.setattr(db_module, "database", schema_session) + monkeypatch.setattr(settings.general, "subfolder", "absolute") + monkeypatch.setattr(settings.general, "subfolder_custom", str(folder)) + quarantine_sync_outputs(str(flow.video), str(source)) + publication = SubtitlePublication(str(flow.video), str(source), subtitle_source_version(source)) + try: + if collision: + assert output.read_text() == "Existing generated bytes" + assert not publication.destination_unchanged(output), "ambiguous output remained eligible for overwrite" + from subtitles.indexer.utils import add_sync_engine_outputs + assert output.name not in add_sync_engine_outputs(str(folder), {}, video_path=str(flow.video)) + else: + assert not output.exists() + assert publication.destination_unchanged(output) + assert any(p.read_text() == "Existing generated bytes" for p in folder.glob("*.bak")) + finally: + publication.release() + + +def test_review_create_cannot_overwrite_a_later_upload(upload_flow, monkeypatch): + import importlib + from threading import current_thread + from flask import Flask + from app.config import settings + + with monkeypatch.context() as importing: + importing.setattr("app.check_update.check_releases", lambda *args, **kwargs: None) + importing.setattr("app.announcements.get_announcements_to_file", lambda *args, **kwargs: None) + content = importlib.import_module("api.subtitles.content") + flow = upload_flow + monkeypatch.setattr(settings.subsync, "use_subsync", False) + monkeypatch.setattr(content, "database", Mock(execute=Mock(return_value=Mock( + first=Mock(return_value=SimpleNamespace(path=str(flow.video), id=30)))))) + monkeypatch.setattr(content, "get_target_folder", lambda *args: None) + monkeypatch.setattr(content, "store_subtitles_movie", lambda *args, **kwargs: None) + monkeypatch.setattr(content, "event_stream", lambda **kwargs: None) + reached, release = Event(), Event() + mkstemp = content.tempfile.mkstemp + + def pause(*args, **kwargs): + if current_thread().name == "create-subtitle": + reached.set() + assert release.wait(5) + return mkstemp(*args, **kwargs) + + monkeypatch.setattr(content.tempfile, "mkstemp", pause) + results = [] + + def create(): + with Flask(__name__).test_request_context(json={"language": "en", "format": "srt", "content": "Created first"}): + results.append(content._create_subtitle("movie", 30, arr_instance_id=7)) + + creator = Thread(target=create, name="create-subtitle") + creator.start() + try: + assert reached.wait(2) + flow.submit("movie", content="Uploaded later") + job, uploader = flow.start() + uploader.join(0.3) + release.set() + creator.join(3) + uploader.join(3) + assert results[0][1] == 201 + assert job.status == "completed" + assert "Uploaded later" in flow.video.with_suffix(".en.srt").read_text() + finally: + release.set() + creator.join(3) + + +def test_review_combine_waits_for_an_existing_publication(upload_flow, monkeypatch): + from subtitles.tools.combine import main + from subtitles.tools.combine.rules import SourcePaths + from subtitles.tools.subsync_engines import subtitle_write_lock + + flow = upload_flow + first = flow.video.with_suffix(".en.srt") + second = flow.video.with_suffix(".hu.srt") + first.write_text("1\n00:00:01,000 --> 00:00:02,000\nFirst\n") + second.write_text("1\n00:00:01,000 --> 00:00:02,000\nSecond\n") + monkeypatch.setattr(main, "resolve_source_paths", lambda **kwargs: SourcePaths(str(first), [str(second)])) + monkeypatch.setattr(main, "_post_write", lambda *args: None) + output = flow.video.with_suffix(".en.combined-hu.srt") + results = [] + finished = Event() + + def combine(): + results.append(main.try_combine_for_video(str(flow.video), "movie", languages=["en", "hu"], format="srt")) + finished.set() + + with subtitle_write_lock(str(flow.video), str(flow.video.parent)): + worker = Thread(target=combine) + worker.start() + blocked = not finished.wait(0.3) + worker.join(3) + assert results[0].status == "built" + assert blocked, "combine published while another writer owned the publication lock" + assert output.is_file() + + +@pytest.mark.parametrize("media_type", ["series", "movie"]) +def test_review_external_index_waits_for_publication(upload_flow, monkeypatch, media_type): + from subtitles.indexer import movies, series + from subtitles.tools.subsync_engines import subtitle_write_lock + + flow = upload_flow + module = movies if media_type == "movie" else series + db = Mock() + db.execute.return_value.first.return_value = SimpleNamespace(arr_instance_id=7, subtitles="[]") + db.execute.return_value.all.return_value = [] + monkeypatch.setattr(module, "database", db) + monkeypatch.setattr("app.config.settings.general.use_embedded_subs", False) + monkeypatch.setattr(module, "get_language_set", lambda: set()) + monkeypatch.setattr(module, "event_stream", lambda **kwargs: None) + entered = Event() + monkeypatch.setattr(module, "search_external_subtitles", lambda *args, **kwargs: (entered.set() or {})) + errors = [] + + def index(): + try: + flow.real_indexers[media_type](str(flow.video), str(flow.video), arr_instance_id=7) + except Exception as exc: + errors.append(exc) + + with subtitle_write_lock(str(flow.video), str(flow.video.parent)): + worker = Thread(target=index) + worker.start() + entered_while_locked = entered.wait(0.3) + worker.join(3) + assert not worker.is_alive() and not errors + assert entered.is_set() + assert not entered_while_locked, "external scan ran inside another writer's publication transaction" + + +def test_review_duplicate_enqueue_keeps_the_existing_job_lease(upload_flow): + from subtitles import sync + + flow = upload_flow + flow.submit("movie") + _, thread = flow.start() + thread.join(2) + job = flow.queue.jobs_pending_queue[0] + publication = job.kwargs["source_version"] + assert sync.sync_subtitles(**job.kwargs) is False + assert list(flow.queue.jobs_pending_queue) == [job] + assert publication.state is not None, "duplicate rejection invalidated the original queued publication" + flow.release_engine.set() + _, thread = flow.start() + thread.join(3) + assert "Synced" in flow.video.with_suffix(".en.srt").read_text() + assert publication.state is None + + +def test_review_removed_pending_job_does_not_retain_the_coordinator(upload_flow): + import gc + import weakref + + flow = upload_flow + flow.submit("movie") + _, thread = flow.start() + thread.join(2) + job = flow.queue.jobs_pending_queue[0] + state = weakref.ref(job.kwargs["source_version"].state) + assert flow.queue.remove_job_from_pending_queue(job.job_id) + del job + gc.collect() + assert state() is None + + +@pytest.mark.parametrize("media_type", ["series", "movie"]) +def test_review_notification_failure_does_not_strand_upload_sync(upload_flow, monkeypatch, media_type): + from subtitles import upload + + flow = upload_flow + name = "send_notifications" if media_type == "series" else "send_notifications_movie" + monkeypatch.setattr(upload, name, Mock(side_effect=RuntimeError("controlled notification failure"))) + flow.submit(media_type) + job, thread = flow.start() + thread.join(2) + assert job.status == "completed" + assert len(flow.queue.jobs_pending_queue) == 1 + flow.release_engine.set() + _, thread = flow.start() + thread.join(3) + assert "Synced" in flow.video.with_suffix(".en.srt").read_text() + + +@pytest.fixture +def review_translator(upload_flow, monkeypatch, request): + import importlib + + flow = upload_flow + name, existing = request.param if isinstance(request.param, tuple) else (request.param, True) + module = importlib.import_module(f"subtitles.tools.translate.services.{name}_translator") + class_name = {"google": "GoogleTranslatorService", "lingarr": "LingarrTranslatorService", + "openrouter": "OpenRouterTranslatorService", "gemini": "GeminiTranslatorService"}[name] + source = flow.video.with_suffix(".hu.srt") + destination = flow.video.with_suffix(".en.srt") + source.write_text("1\n00:00:01,000 --> 00:00:02,000\nSource\n") + if existing: + destination.write_text("1\n00:00:01,000 --> 00:00:02,000\nOriginal destination\n") + service = getattr(module, class_name)( + source_srt_file=str(source), dest_srt_file=str(destination), lang_obj=None, + to_lang="eng", from_lang="en", media_type="movie", video_path=str(flow.video), + orig_to_lang="en", forced=False, hi=False, sonarr_series_id=None, sonarr_episode_id=None, radarr_id=30) + monkeypatch.setattr(module, "jobs_queue", flow.queue) + monkeypatch.setattr(module, "history_log_movie", Mock()) + monkeypatch.setattr(module, "create_process_result", Mock()) + monkeypatch.setattr(module, "add_translator_info", lambda *args: None) + started, release, computed = Event(), Event(), Event() + + def compute(*args, **kwargs): + started.set() + assert release.wait(5) + computed.set() + if name == "google": + return "Translated text" + if name == "gemini": + Path(service.output_file).write_text("1\n00:00:01,000 --> 00:00:02,000\nTranslated text\n") + return + return [{"position": 0, "line": "Translated text"}] + + transport = {"google": "_translate_text", "lingarr": "_translate_content", + "openrouter": "_submit_and_poll", "gemini": "_translate_with_gemini"}[name] + monkeypatch.setattr(service, transport, compute) + if name == "gemini": + monkeypatch.setattr(service, "_get_configured_api_keys", lambda: []) + monkeypatch.setattr(service, "_select_next_api_key", lambda: "fixture") + monkeypatch.setattr(service, "_check_saved_progress", lambda: None) + monkeypatch.setattr(module, "get_description", lambda *args, **kwargs: "") + job_id = flow.queue.feed_jobs_pending_queue("Translation", "fixture", "translate", is_progress=True) + job = flow.queue.jobs_pending_queue.popleft() + job.status = "running" + flow.queue.jobs_running_queue.append(job) + errors = [] + + def run(): + try: + service.translate(job_id=job_id) + except Exception as exc: + errors.append(exc) + + worker = Thread(target=run) + worker.start() + assert started.wait(2) + yield SimpleNamespace(flow=flow, service=service, destination=destination, release=release, + computed=computed, worker=worker, errors=errors, job_id=job_id) + release.set() + worker.join(5) + assert not worker.is_alive() + + +@pytest.mark.parametrize("review_translator", ["google", "lingarr", "openrouter", "gemini"], indirect=True) +@pytest.mark.parametrize("change", ["edit", "delete"]) +def test_review_translation_preserves_changes_made_during_remote_work(review_translator, monkeypatch, change): + from subtitles.tools import mods + from subtitles.tools.subsync_engines import SubtitleDestinationChanged + + run = review_translator + if change == "edit": + edited = b"1\n00:00:01,000 --> 00:00:02,000\nLater edit\n" + monkeypatch.setattr(mods.Subtitle, "get_modified_content", lambda *args, **kwargs: edited) + mods.subtitles_apply_mods("en", str(run.destination), [], str(run.flow.video), arr_instance_id=7) + else: + assert run.flow.remove("movie") + run.release.set() + run.worker.join(3) + assert not run.worker.is_alive() + assert any(isinstance(exc, SubtitleDestinationChanged) for exc in run.errors) + if change == "edit": + assert run.destination.read_bytes() == edited + else: + assert not run.destination.exists() + + +@pytest.mark.parametrize("review_translator", ["google", "lingarr", "openrouter", "gemini"], indirect=True) +def test_review_translation_cancellation_while_waiting_to_publish_keeps_old_file(review_translator): + from app.jobs_queue import JobCancelled + from subtitles.tools.subsync_engines import subtitle_write_lock + + run = review_translator + original = run.destination.read_bytes() + with subtitle_write_lock(str(run.flow.video), str(run.destination.parent)): + run.release.set() + assert run.computed.wait(2) + assert run.flow.queue.cancel_running_job(run.job_id) + run.worker.join(3) + assert not run.worker.is_alive() + assert any(isinstance(exc, JobCancelled) for exc in run.errors) + assert run.destination.read_bytes() == original + + +@pytest.mark.parametrize("review_translator", [(name, False) for name in ["google", "lingarr", "openrouter", "gemini"]], indirect=True) +def test_review_new_translation_preserves_process_umask(review_translator): + run = review_translator + control = run.destination.with_name("permission-control.txt") + control.write_text("control") + run.release.set() + run.worker.join(3) + assert not run.errors + assert run.destination.stat().st_mode & 0o777 == control.stat().st_mode & 0o777 + + +@pytest.mark.parametrize("failure", ["write-error", "empty-content"]) +def test_review_failed_upload_save_preserves_prior_subtitle_and_outputs(upload_flow, monkeypatch, failure): + import builtins + from subtitles import upload + + flow = upload_flow + source = flow.video.with_suffix(".en.srt") + output = flow.video.with_suffix(".en.ffsubsync.srt") + source.write_text("Prior base bytes") + output.write_text("Prior generated bytes") + if failure == "empty-content": + monkeypatch.setattr(upload.Subtitle, "get_modified_content", lambda *args, **kwargs: None) + else: + original = builtins.open + + def fail(file, mode="r", *args, **kwargs): + if "w" in mode and "b" in mode: + raise OSError("controlled subtitle write failure") + return original(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", fail) + flow.submit("movie") + _, thread = flow.start() + thread.join(3) + assert source.read_text() == "Prior base bytes" + assert output.read_text() == "Prior generated bytes" + assert not flow.queue.jobs_pending_queue + + +def test_review_failed_upgrade_keeps_previous_subtitle(upload_flow, monkeypatch): + from subtitles import download, pool + from subliminal_patch.subtitle import Subtitle + from subzero.language import Language + + flow = upload_flow + previous = flow.video.with_suffix(".en.ass") + output = flow.video.with_suffix(".en.ffsubsync.ass") + previous.write_text("Prior base bytes") + output.write_text("Prior generated bytes") + subtitle = Subtitle(Language("eng"), original_format=True) + subtitle.content = b"1\n00:00:01,000 --> 00:00:02,000\nReplacement\n" + subtitle.matches = set() + video = Mock(original_path=str(flow.video)) + monkeypatch.setattr(pool, "_update_pool", lambda *args, **kwargs: False) + monkeypatch.setattr(download, "get_video", lambda *args, **kwargs: video) + monkeypatch.setattr(download, "_get_pool", lambda *args, **kwargs: Mock(providers=["fixture"])) + monkeypatch.setattr(download, "_get_language_obj", lambda **kwargs: {Language("eng")}) + monkeypatch.setattr(download, "get_profiles_list", lambda **kwargs: {"originalFormat": True}) + monkeypatch.setattr(download, "download_best_subtitles", lambda **kwargs: {video: [subtitle]}) + monkeypatch.setattr(download, "get_target_folder", lambda *args: None) + monkeypatch.setattr(download, "save_subtitles", Mock(side_effect=OSError("controlled failed replacement"))) + list(download.generate_subtitles(str(flow.video), [("en", False, False)], [], None, "Video", "movie", 1, + is_upgrade=True, previous_subtitles_to_delete=str(previous), arr_instance_id=7)) + assert previous.read_text() == "Prior base bytes" + assert output.read_text() == "Prior generated bytes" + + +@pytest.fixture +def review_combine(upload_flow, monkeypatch): + from subtitles.tools.combine import main + from subtitles.tools.combine.rules import SourcePaths + + flow = upload_flow + primary = flow.video.with_suffix(".en.srt") + secondary = flow.video.with_suffix(".hu.srt") + primary.write_text("1\n00:00:01,000 --> 00:00:02,000\nFirst\n") + secondary.write_text("1\n00:00:01,000 --> 00:00:02,000\nSecond\n") + monkeypatch.setattr(main, "resolve_source_paths", lambda **kwargs: SourcePaths(str(primary), [str(secondary)])) + monkeypatch.setattr(main, "_post_write", lambda *args: None) + return SimpleNamespace(flow=flow, module=main, output=flow.video.with_suffix(".en.combined-hu.srt"), + run=lambda format="srt": main.try_combine_for_video( + str(flow.video), "movie", languages=["en", "hu"], format=format)) + + +def test_review_combine_cleanup_does_not_follow_sibling_symlinks(review_combine): + run = review_combine + foreign = run.flow.video.with_suffix(".fr.ass") + foreign.write_text("Foreign subtitle bytes") + run.output.with_suffix(".ass").symlink_to(foreign) + assert run.run().status == "built" + assert foreign.read_text() == "Foreign subtitle bytes" + assert run.output.with_suffix(".ass").is_symlink() + + +def test_review_combine_format_rebuilds_cannot_delete_each_other(review_combine, monkeypatch): + run = review_combine + first_cleanup, second_cleanup = Event(), Event() + original = run.module._remove_stale_combined_siblings + results = [] + + def cleanup(path, video): + mine, other = (first_cleanup, second_cleanup) if path.endswith(".srt") else (second_cleanup, first_cleanup) + mine.set() + other.wait(0.3) + original(path, video) + + monkeypatch.setattr(run.module, "_remove_stale_combined_siblings", cleanup) + workers = [Thread(target=lambda fmt=fmt: results.append(run.run(fmt))) for fmt in ("srt", "ass")] + for worker in workers: + worker.start() + for worker in workers: + worker.join(3) + assert not worker.is_alive() + assert all(result.status == "built" for result in results) + assert sum(run.output.with_suffix(ext).is_file() for ext in (".srt", ".ass")) == 1 + + +def test_review_combine_preserves_edit_during_composition(review_combine, monkeypatch): + from subtitles.tools import mods + + run = review_combine + run.output.write_text("1\n00:00:01,000 --> 00:00:02,000\nOriginal combined\n") + entered, release = Event(), Event() + original = run.module.compose + + def compose(**kwargs): + result = original(**kwargs) + entered.set() + assert release.wait(5) + return result + + monkeypatch.setattr(run.module, "compose", compose) + results = [] + worker = Thread(target=lambda: results.append(run.run())) + worker.start() + try: + assert entered.wait(2) + edited = b"1\n00:00:01,000 --> 00:00:02,000\nEdited combined\n" + monkeypatch.setattr(mods.Subtitle, "get_modified_content", lambda *args, **kwargs: edited) + mods.subtitles_apply_mods("en", str(run.output), [], str(run.flow.video), arr_instance_id=7) + release.set() + worker.join(3) + assert not worker.is_alive() + assert run.output.read_bytes() == edited + assert results[0].status != "built" + finally: + release.set() + worker.join(3) + + +def test_review_postprocessing_without_a_queued_job_invalidates_exact_outputs(upload_flow, monkeypatch): + from subtitles import post_processing + + flow = upload_flow + source = flow.video.with_suffix(".en.srt") + output = flow.video.with_suffix(".en.ffsubsync.srt") + source.write_text("Before processing") + output.write_text("Generated before processing") + + def popen(*args, **kwargs): + source.write_text("After processing") + return Mock(communicate=lambda: ("", "")) + + monkeypatch.setattr(post_processing.subprocess, "Popen", popen) + post_processing.postprocessing("fixture", str(flow.video), subtitle_path=str(source)) + assert source.read_text() == "After processing" + assert not output.exists() + assert any(path.read_text() == "Generated before processing" for path in flow.video.parent.glob("*.bak")) + + +def test_review_partial_multiformat_save_records_each_actual_write(upload_flow): + from functools import partial + from subliminal_patch.core import save_subtitles + from subliminal_patch.subtitle import Subtitle + from subzero.language import Language + from subtitles.tools.subsync_engines import write_subtitle_file, SubtitlePublication, subtitle_source_version + + flow = upload_flow + source = flow.video.with_suffix(".en.srt") + output = flow.video.with_suffix(".en.ffsubsync.srt") + source.write_text("Prior source") + output.write_text("Prior generated output") + publication = SubtitlePublication(str(flow.video), str(source), subtitle_source_version(source)) + subtitle = Subtitle(Language("eng"), original_format=True) + subtitle.content = b"1\n00:00:01,000 --> 00:00:02,000\nReplacement\n" + + def content(format, **kwargs): + if format == "ass": + raise ValueError("controlled second format failure") + return b"First format succeeded" + + subtitle.get_modified_content = content + written_paths = [] + try: + with pytest.raises(ValueError, match="second format"): + save_subtitles(str(flow.video), [subtitle], formats=("srt", "ass"), + write_subtitle=partial(write_subtitle_file, str(flow.video), written_paths=written_paths)) + assert written_paths == [str(source)] + assert source.read_text() == "First format succeeded" + assert not publication.source_unchanged() + assert not output.exists() + finally: + publication.release() + + +def test_review_index_failure_does_not_suppress_refresh_of_published_output(upload_flow): + flow = upload_flow + flow.submit("movie") + _, thread = flow.start() + thread.join(2) + job = flow.queue.jobs_pending_queue[0] + job.kwargs["callback"] = Mock(side_effect=RuntimeError("controlled final index failure")) + flow.release_engine.set() + _, thread = flow.start() + thread.join(3) + assert "Synced" in flow.video.with_suffix(".en.srt").read_text() + assert sum("arr_client" in kwargs for _, kwargs in flow.notifications) == 2 + assert job.kwargs["source_version"].state is None + + +def test_review_deleting_one_engine_destination_preserves_other_engine_publication(upload_flow, monkeypatch): + from app.config import settings + from subtitles.tools import subsyncer, delete + + flow = upload_flow + monkeypatch.setattr(settings.subsync, "enabled_engines", ["ffsubsync", "alass"]) + monkeypatch.setattr(settings.subsync, "output_mode", "keep_all") + monkeypatch.setattr(subsyncer.SubSyncer, "_run_external_engine", lambda self, engine, output_path, video_path: + (output_path.write_text("Other engine output") and {"offset_seconds": 0})) + flow.submit("movie") + _, thread = flow.start() + thread.join(2) + deleted = flow.video.with_suffix(".en.ffsubsync.srt") + assert delete.delete_subtitles("movie", "en", False, False, str(flow.video), str(deleted), + radarr_id=30, arr_instance_id=7) is False + _, thread = flow.start() + thread.join(3) + assert not flow.engine_calls + assert not deleted.exists() + assert flow.video.with_suffix(".en.alass.srt").read_text() == "Other engine output" + + +def test_review_quarantine_moves_only_exact_regular_variants(upload_flow): + from subtitles.tools.subsync_engines import quarantine_sync_outputs, SubtitlePublication, subtitle_source_version + + flow = upload_flow + source = flow.video.with_suffix(".en.hi.ass") + source.write_text("Source") + exact = flow.video.with_suffix(".en.hi.ffsubsync.ass") + exact.write_text("Owned generated bytes") + foreign = flow.video.with_suffix(".hu.srt") + foreign.write_text("Foreign bytes") + symlink = flow.video.with_suffix(".en.hi.alass.ass") + symlink.symlink_to(foreign) + untouched = [flow.video.with_suffix(suffix) for suffix in + (".en.ffsubsync.ass", ".en.hi.ffsubsync.srt", ".en.forced.ffsubsync.ass")] + for path in untouched: + path.write_text("Other variant bytes") + quarantine_sync_outputs(str(flow.video), str(source)) + assert not exact.exists() + assert symlink.is_symlink() and foreign.read_text() == "Foreign bytes" + assert all(path.read_text() == "Other variant bytes" for path in untouched) + assert any(path.read_text() == "Owned generated bytes" for path in flow.video.parent.glob("*.bak")) + publication = SubtitlePublication(str(flow.video), str(source), subtitle_source_version(source)) + try: + assert not publication.destination_unchanged(symlink) + finally: + publication.release() + + +def test_review_failed_quarantine_retains_bytes_and_destination_revision(upload_flow, monkeypatch): + from subtitles.tools import subsync_engines + + flow = upload_flow + source = flow.video.with_suffix(".en.srt") + source.write_text("Source") + output = flow.video.with_suffix(".en.ffsubsync.srt") + output.write_text("Prior generated bytes") + publication = subsync_engines.SubtitlePublication( + str(flow.video), str(source), subsync_engines.subtitle_source_version(source)) + monkeypatch.setattr(subsync_engines.os, "replace", Mock(side_effect=PermissionError("controlled move rejection"))) + try: + with pytest.raises(PermissionError, match="move rejection"): + subsync_engines.quarantine_sync_outputs(str(flow.video), str(source)) + assert output.read_text() == "Prior generated bytes" + assert publication.destination_unchanged(output) + assert not list(flow.video.parent.glob(".bazarr-sync-obsolete-*.bak")) + finally: + publication.release() + + +@pytest.fixture +def rejected_quarantine(upload_flow, monkeypatch): + from subtitles.tools import subsync_engines + + flow = upload_flow + source = flow.video.with_suffix(".en.srt") + source.write_text("Previous subtitle") + output = flow.video.with_suffix(".en.ffsubsync.srt") + output.write_text("Previous generated output") + other = flow.video.with_suffix(".hu.ffsubsync.srt") + other.write_text("Other language output") + publication = subsync_engines.SubtitlePublication( + str(flow.video), str(source), subsync_engines.subtitle_source_version(source)) + other_revision = publication.state.revision(other) + real_replace = subsync_engines.os.replace + rejected = [] + + def replace(src, dst, *args, **kwargs): + if Path(src) == output and Path(dst).name.startswith(".bazarr-sync-obsolete-"): + rejected.append(str(src)) + raise PermissionError("controlled quarantine failure after publication") + return real_replace(src, dst, *args, **kwargs) + + monkeypatch.setattr(subsync_engines.os, "replace", replace) + yield SimpleNamespace(flow=flow, source=source, output=output, other=other, publication=publication, + other_revision=other_revision, rejected=rejected) + publication.release() + + +def _assert_reported_quarantine_failure(probe, caplog): + assert probe.rejected == [str(probe.output)] + assert not probe.publication.source_unchanged() + assert probe.publication.state.revision(probe.source) > probe.publication.source_revision + assert probe.output.read_text() == "Previous generated output" + assert probe.publication.destination_unchanged(probe.output) + assert probe.other.read_text() == "Other language output" + assert probe.publication.state.revision(probe.other) == probe.other_revision + assert not list(probe.source.parent.glob(".bazarr-sync-obsolete-*.bak")) + assert any("cleanup failed" in record.message and "may remain available" in record.message + and str(probe.source) in record.message for record in caplog.records) + + +@pytest.mark.parametrize("media_type", ["movie", "series"]) +def test_review_published_upload_is_accounted_when_quarantine_fails(rejected_quarantine, caplog, media_type): + probe = rejected_quarantine + flow = probe.flow + flow.submit(media_type, content="Replacement uploaded text") + job, thread = flow.start() + thread.join(3) + assert not thread.is_alive() + assert "Replacement uploaded text" in probe.source.read_text() + assert flow.indexes, "durable upload publication was not indexed" + assert flow.indexes[0][0] == 7 + assert flow.indexes[0][1][probe.output.name] == "Previous generated output" + assert len(flow.history) == 1 + assert len(flow.notifications) == 2 + assert job.status == "completed" + assert len(flow.queue.jobs_pending_queue) == 1 + _assert_reported_quarantine_failure(probe, caplog) + + +@pytest.mark.parametrize("writer", ["manual", "automatic"]) +def test_review_published_provider_subtitle_is_accounted_when_quarantine_fails( + rejected_quarantine, monkeypatch, caplog, writer): + from subtitles import manual, download, pool + from subliminal_patch.subtitle import Subtitle + from subzero.language import Language + + probe = rejected_quarantine + flow = probe.flow + subtitle = Subtitle(Language("eng"), original_format=True) + subtitle.content = b"1\n00:00:01,000 --> 00:00:02,000\nReplacement provider text\n" + subtitle.provider_name = "fixture" + subtitle.matches = set() + video = Mock(original_path=str(flow.video)) + provider_pool = Mock(providers=["fixture"], discarded_providers=[]) + module = manual if writer == "manual" else download + monkeypatch.setattr(pool, "_update_pool", lambda *args, **kwargs: False) + monkeypatch.setattr(module, "get_video", lambda *args, **kwargs: video) + monkeypatch.setattr(module, "_get_pool", lambda *args, **kwargs: provider_pool) + monkeypatch.setattr(module, "get_target_folder", lambda *args, **kwargs: None) + monkeypatch.setattr(module.subliminal, "region", Mock()) + processed = [] + monkeypatch.setattr(module, "process_subtitle", lambda **kwargs: processed.append(kwargs) or "saved") + if writer == "manual": + monkeypatch.setattr(manual.subtitle_cache, "get", lambda *args: subtitle) + monkeypatch.setattr(manual, "download_subtitles", lambda *args: None) + manual.manual_download_subtitle(str(flow.video), [], False, False, "cached", "fixture", None, + "Video", "movie", True, 1, arr_instance_id=7) + else: + monkeypatch.setattr(download, "_get_language_obj", lambda **kwargs: {Language("eng")}) + monkeypatch.setattr(download, "get_profiles_list", lambda **kwargs: {"originalFormat": True}) + monkeypatch.setattr(download, "download_best_subtitles", lambda **kwargs: {video: [subtitle]}) + monkeypatch.setattr(download, "clear_mismatch_for_video", lambda *args, **kwargs: None) + list(download.generate_subtitles(str(flow.video), [("en", False, False)], [], None, + "Video", "movie", 1, arr_instance_id=7)) + assert "Replacement provider text" in probe.source.read_text() + assert len(processed) == 1, "durable provider publication was not passed to processing" + assert processed[0]["subtitle"].storage_path == str(probe.source) + assert processed[0]["path"] == str(flow.video) + assert processed[0]["media_type"] == "movie" + _assert_reported_quarantine_failure(probe, caplog) + + +@pytest.mark.parametrize("operation", ["delete", "manual-sync"]) +def test_review_completed_mutation_keeps_success_when_quarantine_fails(rejected_quarantine, caplog, operation): + from subtitles import sync + + probe = rejected_quarantine + flow = probe.flow + if operation == "delete": + assert flow.remove("movie") is True + assert not probe.source.exists() + assert flow.indexes + else: + flow.release_engine.set() + assert sync.sync_subtitles(str(flow.video), str(probe.source), "en", False, False, 100, + radarr_id=30, force_sync=True, track_job_progress=False, + arr_instance_id=7) is True + assert "Synced" in probe.source.read_text() + assert flow.sync_history + _assert_reported_quarantine_failure(probe, caplog)