From 452917347fad5912801272f7a1b3e27d72076ec4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 20:46:25 +0000 Subject: [PATCH] Add a data profile for generic large-data transfers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The copy engine was never camera-specific — it reads every byte once, checksums it, fans it out to N destinations and reads each copy back off disk — but the layer above it assumed camera originals: ffprobe metadata, contact-sheet thumbnails, the BRAW container check. This generalises the tool to any large one-way transfer without touching that engine. Add a Profile { media, data } selected with `--profile` (shorthand `--generic`). The data profile turns the media layer off: no ffprobe, no thumbnails, no BRAW check, so a dataset, disk image, render output or backup is copied, verified and documented with nothing depending on ffmpeg. The default stays `media`, so the camera-card workflow is unchanged. Profile is a first-class field on OffloadOptions (whose __post_init__ enforces the media knobs off for a data transfer, so a library caller that sets only the profile still gets a clean generic copy), on Job, and on saved presets (round-tripped through JSON, tolerant of a missing or garbage key). It is selectable in the desktop app's Simple mode and preset editor, where the media-only thumbnails control greys out for a data transfer. The CLI summary drops the "N video" line under the data profile, where it would only ever read "0 video". Docs reframe the tool as verified copy for large data transfers with camera-card offload built in, and the ROADMAP's "not a sync tool" entry now distinguishes the supported one-way transfer from the two-way sync that stays out of scope. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_017D2mLdX5ABymqURvDQEJsY --- CHANGELOG.md | 14 ++++ README.md | 58 ++++++++++++--- ROADMAP.md | 10 ++- pyproject.toml | 2 +- src/offloader/__init__.py | 3 +- src/offloader/cli.py | 24 +++++- src/offloader/engine.py | 16 ++++ src/offloader/gui/preset_editor.py | 19 ++++- src/offloader/gui/simple_mode.py | 18 ++++- src/offloader/models.py | 31 ++++++++ src/offloader/presets.py | 20 ++++- tests/test_profile.py | 116 +++++++++++++++++++++++++++++ 12 files changed, 308 insertions(+), 23 deletions(-) create mode 100644 tests/test_profile.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ef9d80f..0ee5eea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,20 @@ project uses [semantic versioning][semver]. ## [Unreleased] +### Added + +- **A `data` profile for generic large-data transfers.** The verified copy + engine was never camera-specific — it reads every byte once, checksums it, + fans it out to N destinations and reads it back — but the metadata layer + assumed camera originals. `--profile data` (or the shorthand `--generic`) + turns that layer off: no ffprobe, no thumbnails, no BRAW check, so a dataset, + disk image, render output or backup is copied, checksummed, verified and + documented (CSV, MHL, ASC MHL, PDF, HTML) with nothing depending on ffmpeg. + The default stays `media`, so the camera-card workflow is unchanged. The + profile is a first-class field on `OffloadOptions`, `Job` and saved presets, + and is selectable in the desktop app's Simple mode and preset editor. This is + a one-way verified transfer, not two-way sync — see `ROADMAP.md`. + ### Fixed Found by adding CI on Linux and macOS — the suite had only ever run on Windows. diff --git a/README.md b/README.md index 93317f3..291b30b 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,18 @@ [![Python](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/downloads/) [![Licence](https://img.shields.io/badge/licence-MIT-green)](LICENSE) -Verified media offload for camera cards, with job reports that match the layout -of [ShotPut Pro][spp]'s `JobReport.pdf`. - -Copy a card to one or more destinations, checksum every byte, and produce the -paperwork a post house expects: a PDF contact sheet with per-clip metadata, a -CSV manifest, MHL and ASC MHL manifests for re-verification downstream, and a -self-contained HTML page. +Verified copy for large data transfers. Read every byte once, checksum it, fan +it out to one or more destinations, and read it back to prove what landed — +then produce the paperwork: a CSV manifest, MHL and ASC MHL manifests for +re-verification downstream, and a self-contained HTML page. + +The flagship use is camera-card offload, with job reports that match the layout +of [ShotPut Pro][spp]'s `JobReport.pdf` — a PDF contact sheet with per-clip +metadata, ffprobe media details and Blackmagic RAW container checks. But that +media layer is a profile, not the engine: `--profile data` (shorthand +`--generic`) offloads any large one-way transfer — datasets, disk images, +render output, backups — with the same verified copy and manifests, and nothing +depending on ffmpeg. See [Generic data transfers](#generic-data-transfers). [spp]: https://www.imagineproducts.com/product/shotput-pro/windows @@ -86,6 +91,8 @@ offloader verify D:\video\080426\A001 | `--dest PATH` | destination root; repeat for multiple copies | | `--hash ALGO` | `xxh3-64` (default), `xxh3-128`, `xxh64`, `xxh64be`, `md5`, `sha1`, `sha256`, `c4`, `none` | | `--verify MODE` | `source-only` (default), `full`, `none` | +| `--profile P` | `media` (default: ffprobe, thumbnails, BRAW) or `data` (generic transfer, no media probing) | +| `--generic` | shorthand for `--profile data` | | `--report FMT[,FMT]` | `pdf` (default), `csv`, `mhl`, `ascmhl`, `html` | | `--report-dir PATH` | override the report location | | `--thumbs N` | frames per clip, 0 to disable (default 4) | @@ -143,6 +150,39 @@ the destination, at the cost of reading everything twice. - **HTML** — self-contained; thumbnails inlined as data URIs, light and dark themes, no external requests. +## Generic data transfers + +The copy engine has never been camera-specific: it streams the source once, +checksums it, writes N destinations in the same pass, evicts the page cache and +reads each copy back off the platter. Everything that made this a *camera* tool +— ffprobe metadata, contact-sheet thumbnails, the BRAW container check — sits in +a layer above it. + +`--profile data` (or `--generic`) switches that layer off: + +```sh +offloader offload \ + --source /mnt/instrument/run_1440 \ + --dest /archive/2026/run_1440 \ + --dest /nas/cold/run_1440 \ + --generic \ + --verify full \ + --hash sha256 \ + --report csv,ascmhl +``` + +Nothing is treated as a clip, ffmpeg is never invoked, and the run does not need +it installed. What you still get is the whole point of the tool: every byte +read once and fanned out, both copies verified off disk, a checksum manifest +beside each one, and `offloader verify` to re-check the archive months later for +bit rot. The PDF, CSV, MHL, ASC MHL and HTML reports all render a plain file +listing — the per-clip metadata block simply does not appear. + +This is a **one-way, write-once** transfer: the same model the tool has always +assumed, now stated for any large data rather than only camera originals. It is +deliberately not a sync tool — no two-way reconciliation, conflict resolution or +partial-file updates. See [`ROADMAP.md`](ROADMAP.md). + ## The desktop app ```sh @@ -276,13 +316,13 @@ what makes the report layer testable without moving bytes. ```sh pip install -e ".[dev]" -pytest # 400 tests, ~33s +pytest # 409 tests, ~33s pytest --fuzz # same suite, 3000 examples per property (~2 min) ruff check src tests pytest --cov=offloader --cov-report=term-missing ``` -400 tests at 83% line coverage. They cover formatting against the reference's +409 tests at 82% line coverage. They cover formatting against the reference's exact strings, checksum vectors and streaming equivalence, copy/verify behaviour including simulated destination corruption, pause/resume/cancel concurrency, retry discrimination, BRAW container parsing, ffprobe parsing, diff --git a/ROADMAP.md b/ROADMAP.md index 9483c01..c6e964b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -105,9 +105,13 @@ already does the stem-matching this needs. ## Not planned -- **A general-purpose file sync tool.** The design assumes camera originals: - written once, never modified, verified once and archived. Two-way sync, - conflict resolution and partial-file updates would compromise that. +- **A general-purpose file *sync* tool.** The `data` profile generalises the + engine to any large *one-way* transfer — written once, never modified, + verified once, archived — because that assumption is exactly what makes the + "Verified" verdict meaningful, and it holds for a dataset or a disk image as + well as a camera card. What stays out is everything that *breaks* it: + two-way reconciliation, conflict resolution and partial-file updates. Those + turn a copy you can prove into a merge you have to trust. - **Metadata fidelity beyond timestamps** — ACLs, alternate data streams, junctions. robocopy does these well and they do not apply to camera media. - **Replacing the copy loop with robocopy.** Measured and rejected; see diff --git a/pyproject.toml b/pyproject.toml index 19227b8..8db0d68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "offloader" version = "0.1.0" -description = "Verified media offload for camera cards, with ShotPut Pro-compatible reports, ASC MHL, and Blackmagic RAW support" +description = "Verified copy for large data transfers, with camera-card offload, ShotPut Pro-compatible reports, ASC MHL, and Blackmagic RAW support" readme = "README.md" requires-python = ">=3.10" license = { text = "MIT" } diff --git a/src/offloader/__init__.py b/src/offloader/__init__.py index c1c42bb..dcbebda 100644 --- a/src/offloader/__init__.py +++ b/src/offloader/__init__.py @@ -1,4 +1,5 @@ -"""Verified media offload with ShotPut Pro-compatible reporting.""" +"""Verified copy for large data transfers, with camera-card offload and +ShotPut Pro-compatible reporting built in.""" __version__ = "0.1.0" diff --git a/src/offloader/cli.py b/src/offloader/cli.py index fd3b9cd..6773f22 100644 --- a/src/offloader/cli.py +++ b/src/offloader/cli.py @@ -8,7 +8,7 @@ from pathlib import Path from . import PRODUCT_NAME, __version__, engine, hashers, longpath, probe, retry, thumbs -from .models import FileStatus, Job, VerificationMode +from .models import FileStatus, Job, Profile, VerificationMode from .reports import WRITERS from .util import format_elapsed, format_size @@ -122,9 +122,12 @@ def _summarize(job: Job, reports: list[Path]) -> None: failed = [f for f in job.files if f.status is FileStatus.FAILED] print() print(f" {job.name}: {job.final_status}") + # The video count is meaningful only when media was probed; a generic + # data transfer never looks inside a file, so reporting "0 video" would + # be noise rather than information. + video = f" ({job.video_files} video)" if job.profile.probes_media else "" print(f" {job.total_files} files, {format_size(job.total_bytes)}" - f" in {format_elapsed(job.elapsed_sec)}" - f" ({job.video_files} video)") + f" in {format_elapsed(job.elapsed_sec)}{video}") print(f" Verification: {job.verification_label}") for destination in job.destination_roots: print(f" -> {destination}") @@ -169,6 +172,14 @@ def _common_options(parser: argparse.ArgumentParser) -> None: parser.add_argument("--retry-wait", type=float, default=2.0, metavar="SECONDS", help="pause before the first retry, backing off after " "(default: %(default)s)") + parser.add_argument("--profile", choices=[p.value for p in Profile], + default=Profile.MEDIA.value, + help="'media' (default) offloads camera cards with " + "ffprobe metadata, thumbnails and the BRAW check; " + "'data' is a generic large-data transfer that " + "copies and verifies but skips all media probing") + parser.add_argument("--generic", action="store_true", + help="shorthand for --profile data") parser.add_argument("--no-probe", action="store_true", help="skip ffprobe metadata and thumbnails") parser.add_argument("--quiet", action="store_true", help="suppress progress") @@ -177,7 +188,8 @@ def _common_options(parser: argparse.ArgumentParser) -> None: def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="offloader", - description=f"{PRODUCT_NAME} — verified media offload with job reports.", + description=f"{PRODUCT_NAME} — verified copy for large data transfers, " + f"with camera-card offload and job reports built in.", ) parser.add_argument("--version", action="version", version=f"{PRODUCT_NAME} {__version__}") @@ -226,6 +238,7 @@ def build_parser() -> argparse.ArgumentParser: def _options_from(args: argparse.Namespace, destinations: list[Path]) -> engine.OffloadOptions: + profile = Profile.DATA if getattr(args, "generic", False) else Profile(args.profile) return engine.OffloadOptions( destinations=destinations, algorithm=args.hash, @@ -236,6 +249,7 @@ def _options_from(args: argparse.Namespace, destinations: list[Path]) -> engine. skip_existing=getattr(args, "skip_existing", False), job_name=args.name, extra_probe=not args.no_probe, + profile=profile, retry=retry.RetryPolicy(attempts=max(1, args.retries), delay=max(0.0, args.retry_wait)), ) @@ -356,6 +370,8 @@ def cmd_info(_args: argparse.Namespace) -> int: f" {prefix} prefix {note}") print(f" checksums: {', '.join(sorted(hashers.algorithm_keys()))}") print(f" reports: {', '.join(WRITERS)}") + print(f" profiles: {', '.join(p.value for p in Profile)} " + f"(--profile; 'data' skips media probing for generic transfers)") return 0 diff --git a/src/offloader/engine.py b/src/offloader/engine.py index 316b4bd..6660ceb 100644 --- a/src/offloader/engine.py +++ b/src/offloader/engine.py @@ -27,6 +27,7 @@ FileEntry, FileStatus, Job, + Profile, VerificationMode, ) @@ -176,10 +177,23 @@ class OffloadOptions: job_name: str | None = None thumbnail_dir: Path | None = None extra_probe: bool = True + #: The workflow this offload is. `Profile.DATA` is a generic large-data + #: transfer: no file is treated as media, so ffprobe, thumbnails and the + #: BRAW check are all switched off regardless of the media-only knobs above. + profile: Profile = Profile.MEDIA #: How hard to try again when a read fails for a transient-looking reason. #: Marginal cards and readers routinely succeed on a second attempt. retry: retry_mod.RetryPolicy = field(default_factory=retry_mod.RetryPolicy) + def __post_init__(self) -> None: + # The data profile is defined by the absence of media work, so enforce + # it here rather than trusting every caller to zero the media knobs. + # A library caller that sets only `profile=Profile.DATA` gets a clean + # generic transfer; the CLI and presets get the same guarantee. + if self.profile is Profile.DATA: + self.extra_probe = False + self.thumbnail_count = 0 + @dataclass class _Counters: @@ -361,6 +375,7 @@ def run(source_root: Path, options: OffloadOptions, source_root=source_root, destination_roots=dest_roots, verification=options.verification, + profile=options.profile, hash_label=algorithm.label, started=_dt.datetime.now(), os_version=host.os_version, @@ -640,6 +655,7 @@ def rescan(source_root: Path, destination_roots: Sequence[Path], source_root=source_root, destination_roots=[Path(d) for d in destination_roots] or [source_root], verification=options.verification, + profile=options.profile, hash_label=algorithm.label, started=_dt.datetime.now(), os_version=host.os_version, diff --git a/src/offloader/gui/preset_editor.py b/src/offloader/gui/preset_editor.py index 2e6ce42..68499fa 100644 --- a/src/offloader/gui/preset_editor.py +++ b/src/offloader/gui/preset_editor.py @@ -21,7 +21,7 @@ ) from ..hashers import ALGORITHMS -from ..models import VerificationMode +from ..models import Profile, VerificationMode from ..naming import TOKENS from ..presets import PRESET_COLORS, Preset from ..reports import WRITERS @@ -85,11 +85,21 @@ def __init__(self, preset: Preset | None = None, parent: QWidget | None = None) self._verification.setCurrentIndex( max(0, self._verification.findData(self._source.verification.value))) + self._profile = QComboBox() + self._profile.addItem("Media — camera card (ffprobe, thumbnails, BRAW)", + Profile.MEDIA.value) + self._profile.addItem("Data — any large transfer (copy and verify only)", + Profile.DATA.value) + self._profile.setCurrentIndex( + max(0, self._profile.findData(self._source.profile.value))) + self._profile.currentIndexChanged.connect(self._on_profile_changed) + self._thumbnails = QSpinBox() self._thumbnails.setRange(0, 8) self._thumbnails.setValue(self._source.thumbnail_count) self._thumbnails.setSuffix(" per clip") self._thumbnails.setSpecialValueText("Off") + self._thumbnails.setEnabled(self._source.profile is Profile.MEDIA) self._reports: dict[str, QCheckBox] = {} report_row = [] @@ -130,6 +140,7 @@ def __init__(self, preset: Preset | None = None, parent: QWidget | None = None) form.addRow("Colour", self._color) form.addRow("Destinations", self._destinations) form.addRow("", row(add, remove, None)) + form.addRow("Profile", self._profile) form.addRow("Checksum", self._algorithm) form.addRow("Verification", self._verification) form.addRow("Thumbnails", self._thumbnails) @@ -152,6 +163,11 @@ def __init__(self, preset: Preset | None = None, parent: QWidget | None = None) layout.addLayout(form) layout.addWidget(buttons) + def _on_profile_changed(self) -> None: + # A data transfer never decodes a file, so contact-sheet thumbnails do + # not apply — disable the control rather than let it imply otherwise. + self._thumbnails.setEnabled(self._profile.currentData() == Profile.MEDIA.value) + def _choose_logo(self) -> None: path, _ = QFileDialog.getOpenFileName( self, "Choose a logo", "", "Images (*.png *.jpg *.jpeg *.gif)") @@ -171,6 +187,7 @@ def _accept(self) -> None: destinations=self._destinations.paths(), algorithm=self._algorithm.currentData(), verification=VerificationMode(self._verification.currentData()), + profile=Profile(self._profile.currentData()), thumbnail_count=self._thumbnails.value(), reports=[key for key, box in self._reports.items() if box.isChecked()], naming_template=self._naming.text().strip() or "{card}", diff --git a/src/offloader/gui/simple_mode.py b/src/offloader/gui/simple_mode.py index 23381f6..9253dc8 100644 --- a/src/offloader/gui/simple_mode.py +++ b/src/offloader/gui/simple_mode.py @@ -20,7 +20,7 @@ ) from ..hashers import ALGORITHMS -from ..models import VerificationMode +from ..models import Profile, VerificationMode from ..presets import Preset from ..reports import WRITERS from .preset_editor import VERIFICATION_LABELS @@ -60,6 +60,13 @@ def __init__(self, parent: QWidget | None = None) -> None: self._verification.setCurrentIndex( max(0, self._verification.findData(VerificationMode.SOURCE_ONLY.value))) + self._profile = QComboBox() + self._profile.addItem("Media — camera card", Profile.MEDIA.value) + self._profile.addItem("Data — any large transfer", Profile.DATA.value) + self._profile.setCurrentIndex( + max(0, self._profile.findData(Profile.MEDIA.value))) + self._profile.currentIndexChanged.connect(self._on_profile_changed) + self._thumbnails = QSpinBox() self._thumbnails.setRange(0, 8) self._thumbnails.setValue(4) @@ -82,6 +89,7 @@ def __init__(self, parent: QWidget | None = None) -> None: form.setSpacing(10) form.setLabelAlignment(Qt.AlignRight | Qt.AlignVCenter) form.addRow("Job name", self._name) + form.addRow("Profile", self._profile) form.addRow("Checksum", self._algorithm) form.addRow("Verification", self._verification) form.addRow("Thumbnails", self._thumbnails) @@ -146,12 +154,20 @@ def _overlaps(source: Path, destination: Path) -> bool: return False return source == destination or source in destination.parents + def _on_profile_changed(self) -> None: + # Thumbnails are contact-sheet frames from a clip — meaningless for a + # generic data transfer, which never decodes a file. Grey the control + # so the disabled state explains itself. + is_media = self._profile.currentData() == Profile.MEDIA.value + self._thumbnails.setEnabled(is_media) + def build_preset(self) -> Preset: return Preset( name="Simple mode", destinations=self.destinations.paths(), algorithm=self._algorithm.currentData(), verification=VerificationMode(self._verification.currentData()), + profile=Profile(self._profile.currentData()), thumbnail_count=self._thumbnails.value(), reports=[key for key, box in self._reports.items() if box.isChecked()], preserve_structure=self._preserve.isChecked(), diff --git a/src/offloader/models.py b/src/offloader/models.py index f545d99..e07f3c4 100644 --- a/src/offloader/models.py +++ b/src/offloader/models.py @@ -13,6 +13,34 @@ from pathlib import Path +class Profile(str, Enum): + """What kind of source is being offloaded. + + The verified copy engine is identical for both — every byte is read once, + checksummed, fanned out and (optionally) read back. The profile only + decides how much *media-specific* work runs on top of that: + + - `MEDIA` (default): the camera-card workflow. ffprobe metadata, contact- + sheet thumbnails and the BRAW container check, with camera originals in + mind. + - `DATA`: any large one-way transfer — datasets, disk images, backups, + render output. Skips media probing entirely, so nothing depends on + ffmpeg and no file is treated as a clip. The copy, the checksums, the + verification and the MHL/CSV/PDF paperwork are exactly the same. + """ + + MEDIA = "media" + DATA = "data" + + @property + def label(self) -> str: + return {Profile.MEDIA: "Media", Profile.DATA: "Data"}[self] + + @property + def probes_media(self) -> bool: + return self is Profile.MEDIA + + class VerificationMode(str, Enum): """How much re-reading the engine does to prove the copy landed intact.""" @@ -181,6 +209,9 @@ class Job: source_root: Path destination_roots: list[Path] verification: VerificationMode = VerificationMode.SOURCE_ONLY + #: Which workflow produced this job. `DATA` means no media metadata was + #: collected, so reports show a plain file listing rather than clip rows. + profile: Profile = Profile.MEDIA hash_label: str = "XXHash3-64" started: _dt.datetime = field(default_factory=_dt.datetime.now) finished: _dt.datetime | None = None diff --git a/src/offloader/presets.py b/src/offloader/presets.py index 9e32a81..edaaf70 100644 --- a/src/offloader/presets.py +++ b/src/offloader/presets.py @@ -14,7 +14,7 @@ from .config import config_file, read_json, write_json from .engine import DEFAULT_EXCLUDES, OffloadOptions from .hashers import ALGORITHMS -from .models import VerificationMode +from .models import Profile, VerificationMode from .naming import DEFAULT_TEMPLATE from .retry import RetryPolicy @@ -40,6 +40,7 @@ class Preset: destinations: list[Path] = field(default_factory=list) algorithm: str = "xxh3-64" verification: VerificationMode = VerificationMode.SOURCE_ONLY + profile: Profile = Profile.MEDIA thumbnail_count: int = 4 reports: list[str] = field(default_factory=lambda: ["pdf"]) preserve_structure: bool = True @@ -81,7 +82,10 @@ def summary(self) -> str: VerificationMode.SOURCE_ONLY: "source-only verify", VerificationMode.FULL: "full verify", }[self.verification] - return f"{where} · {self.algorithm} · {verify} · {', '.join(self.reports) or 'no reports'}" + parts = [where, self.algorithm, verify, ", ".join(self.reports) or "no reports"] + if self.profile is Profile.DATA: + parts.insert(0, "data") + return " · ".join(parts) def mark_used(self) -> None: self.use_count += 1 @@ -99,8 +103,11 @@ def to_options(self, job_name: str | None = None) -> OffloadOptions: skip_existing=self.skip_existing, job_name=job_name, # Metadata is cheap next to the copy itself and useful even when - # thumbnails are switched off, so it is always collected. + # thumbnails are switched off, so it is always collected — unless + # this is a data-transfer preset, where OffloadOptions turns media + # probing off for the profile. extra_probe=True, + profile=self.profile, retry=RetryPolicy(attempts=max(1, self.retry_attempts), delay=max(0.0, self.retry_wait)), ) @@ -112,6 +119,7 @@ def to_dict(self) -> dict: "destinations": [str(p) for p in self.destinations], "algorithm": self.algorithm, "verification": self.verification.value, + "profile": self.profile.value, "thumbnail_count": self.thumbnail_count, "reports": list(self.reports), "preserve_structure": self.preserve_structure, @@ -161,6 +169,11 @@ def as_list(key): except (ValueError, TypeError): verification = VerificationMode.SOURCE_ONLY + try: + profile = Profile(value("profile", "media")) + except (ValueError, TypeError): + profile = Profile.MEDIA + algorithm = value("algorithm", "xxh3-64") if algorithm not in ALGORITHMS: algorithm = "xxh3-64" @@ -170,6 +183,7 @@ def as_list(key): destinations=[Path(p) for p in as_list("destinations")], algorithm=algorithm, verification=verification, + profile=profile, thumbnail_count=as_int("thumbnail_count", 4), # An explicitly empty list is a real choice — offload without # paperwork — so only a missing or null key falls back to the diff --git a/tests/test_profile.py b/tests/test_profile.py new file mode 100644 index 0000000..816ade8 --- /dev/null +++ b/tests/test_profile.py @@ -0,0 +1,116 @@ +"""The `data` profile: a generic large-data transfer. + +The verified copy engine is the same for both profiles; `data` only switches +off the media-specific work (ffprobe, thumbnails, the BRAW check). These tests +pin that contract at every layer it passes through — OffloadOptions, the engine, +the CLI and presets — so a future refactor cannot quietly start probing a data +transfer or stop probing a media one. +""" + +from __future__ import annotations + +from pathlib import Path + +from offloader import cli, engine +from offloader.models import FileStatus, Profile, VerificationMode +from offloader.presets import Preset + + +def _options(tmp_path: Path, **overrides) -> engine.OffloadOptions: + defaults = dict( + destinations=[tmp_path / "dest"], + algorithm="xxh3-64", + verification=VerificationMode.FULL, + ) + defaults.update(overrides) + return engine.OffloadOptions(**defaults) + + +def test_data_profile_forces_media_work_off(tmp_path: Path): + # A caller that sets only the profile still gets a clean generic transfer: + # the media knobs are zeroed regardless of what was passed alongside them. + options = _options(tmp_path, profile=Profile.DATA, + extra_probe=True, thumbnail_count=4) + assert options.extra_probe is False + assert options.thumbnail_count == 0 + + +def test_media_profile_leaves_media_work_alone(tmp_path: Path): + options = _options(tmp_path, profile=Profile.MEDIA, + extra_probe=True, thumbnail_count=4) + assert options.extra_probe is True + assert options.thumbnail_count == 4 + + +def test_data_profile_still_copies_and_verifies(source_tree: Path, tmp_path: Path): + job = engine.run(source_tree, _options(tmp_path, profile=Profile.DATA)) + + assert job.profile is Profile.DATA + assert job.total_files == 3 + assert job.final_status == "Verified" + assert all(f.status is FileStatus.VERIFIED for f in job.files) + + copied = tmp_path / "dest" / "Clips" / "A001_C001.mov" + assert copied.read_bytes() == (source_tree / "Clips" / "A001_C001.mov").read_bytes() + + +def test_data_profile_never_probes(source_tree: Path, tmp_path: Path, monkeypatch): + # Probing a data transfer is the exact thing the profile exists to prevent, + # so make it fail loudly if the engine ever reaches for ffprobe. + def boom(*_args, **_kwargs): + raise AssertionError("probe() must not run under the data profile") + + monkeypatch.setattr("offloader.engine.probe_mod.probe", boom) + job = engine.run(source_tree, _options(tmp_path, profile=Profile.DATA)) + + assert job.video_files == 0 + assert all(not f.media.is_video for f in job.files) + + +def test_cli_generic_flag_selects_the_data_profile(source_tree: Path, tmp_path: Path, + capsys): + dest = tmp_path / "dest" + code = cli.main([ + "offload", "--source", str(source_tree), "--dest", str(dest), + "--name", "DATASET", "--generic", "--quiet", + ]) + assert code == 0 + out = capsys.readouterr().out + assert "DATASET: Verified" in out + # The video count is media-only noise on a generic transfer. + assert "video)" not in out + assert (dest / "DATASET_Reports" / "JobReport.pdf").is_file() + + +def test_cli_profile_data_is_equivalent_to_generic(tmp_path): + parser = cli.build_parser() + args = parser.parse_args([ + "offload", "--source", str(tmp_path), "--dest", str(tmp_path / "d"), + "--profile", "data", + ]) + options = cli._options_from(args, [tmp_path / "d"]) + assert options.profile is Profile.DATA + + +def test_cli_defaults_to_the_media_profile(tmp_path): + parser = cli.build_parser() + args = parser.parse_args([ + "offload", "--source", str(tmp_path), "--dest", str(tmp_path / "d"), + ]) + options = cli._options_from(args, [tmp_path / "d"]) + assert options.profile is Profile.MEDIA + + +def test_preset_round_trips_the_profile(): + preset = Preset(name="Archive", profile=Profile.DATA) + restored = Preset.from_dict(preset.to_dict()) + assert restored.profile is Profile.DATA + assert restored.to_options().profile is Profile.DATA + + +def test_preset_defaults_to_media_and_tolerates_missing_key(): + assert Preset(name="x").profile is Profile.MEDIA + # A config written before profiles existed has no key at all. + assert Preset.from_dict({"name": "legacy"}).profile is Profile.MEDIA + # A garbage value must never brick a load. + assert Preset.from_dict({"name": "bad", "profile": "nonsense"}).profile is Profile.MEDIA