diff --git a/core/archives.py b/core/archives.py index cd10dbb4..8d3dcf8e 100644 --- a/core/archives.py +++ b/core/archives.py @@ -8,6 +8,7 @@ from __future__ import annotations +import contextlib import math import zipfile from collections.abc import Callable @@ -405,6 +406,7 @@ def _after_examine(member_label: str, declared_size: int) -> bool: elif archive_type == "7z": try: import py7zr + from py7zr.io import BytesIOFactory except ImportError: raise ArchiveUnsupportedError( "7z support requires py7zr: `uv sync --extra compressed` (uv) " @@ -417,23 +419,59 @@ def _after_examine(member_label: str, declared_size: int) -> bool: try: with py7zr.SevenZipFile(path, "r", password=pwd) as archive: # py7zr >= 0.22: use list(); older docs used files_list (removed in 1.x). + # Collect desired members first, then ONE extract() — solid archives + # re-decompress per extract call (#1250 / #1248). + desired: list[str] = [] for member in archive.list(): if member.is_directory: continue size = getattr(member, "uncompressed", 0) or 0 if _after_examine(member.filename, size): - return + # Stop collecting; still extract members already in desired + # (ZIP/TAR parity — do not discard the pre-budget queue). + break if size > max_inner_size: continue ext = Path(member.filename).suffix.lower() if ext not in allowed: continue + desired.append(member.filename) + + if not desired: + return + + # SevenZipFile is stateful after list(); reset before extract. + with contextlib.suppress(Exception): + archive.reset() + + try: + # Per-stream memory cap matches the per-member size filter above. + fac = BytesIOFactory(max_inner_size) + archive.extract(targets=desired, factory=fac) + except PasswordRequired: + raise + except (Exception, OSError) as e: + if on_member_read_failure is not None: + reason = classify_7z_member_read_failure( + e, password_provided=bool(pwd) + ) + # Batch extract failed as a whole — record one failure per target + # so scan_failures still surfaces the reason for each member. + for name in desired: + on_member_read_failure( + reason, + name, + f"{path.name}|{name}: {type(e).__name__}: {e}", + ) + return + + for name in desired: + bio = fac.products.get(name) + if bio is None: + continue try: - files = archive.read(targets=[member.filename]) - bio = files.get(member.filename) - if bio is not None: - data = bio.read() - yield (member.filename, data) + bio.seek(0) + data = bio.read() except (Exception, OSError) as e: if on_member_read_failure is not None: reason = classify_7z_member_read_failure( @@ -441,10 +479,11 @@ def _after_examine(member_label: str, declared_size: int) -> bool: ) on_member_read_failure( reason, - member.filename, - f"{path.name}|{member.filename}: {type(e).__name__}: {e}", + name, + f"{path.name}|{name}: {type(e).__name__}: {e}", ) continue + yield (name, data) except PasswordRequired as e: if on_member_read_failure is not None: on_member_read_failure( diff --git a/tests/test_archives_scan_failures.py b/tests/test_archives_scan_failures.py index c1c50d69..85a97385 100644 --- a/tests/test_archives_scan_failures.py +++ b/tests/test_archives_scan_failures.py @@ -316,3 +316,143 @@ def _bad_password_read(self, name, pwd=None): # type: ignore[no-untyped-def] wrong = [f for f in db.failures if f[1] == SCAN_FAILURE_REASON_WRONG_PASSWORD] assert len(wrong) >= 1 + + +# --- #1250 / #1248: real py7zr BytesIOFactory path (not ZipFile-.read lookalike) --- + + +def test_7z_iter_archive_members_roundtrip_yields_pii_content(tmp_path): + """Unencrypted .7z: inner text is materialized (was AttributeError before #1250).""" + pytest.importorskip("py7zr") + import py7zr + + from core.archives import iter_archive_members + + seven = tmp_path / "pii.7z" + payload = ( + b"Contact: demo.user@example.com\n" + b"CPF 123.456.789-00 synthetic sample for archive read proof.\n" + ) + with py7zr.SevenZipFile(seven, "w") as z: + z.writestr(payload, "inner/doc.txt") + + got = list( + iter_archive_members( + seven, + "7z", + max_inner_size=1_000_000, + allowed_extensions={".txt"}, + ) + ) + assert len(got) == 1 + name, data = got[0] + assert name == "inner/doc.txt" + assert b"123.456.789-00" in data + assert b"demo.user@example.com" in data + + +def test_7z_iter_archive_members_multi_and_filters(tmp_path): + """Multiple allowed members are read; disallowed ext / oversized skipped.""" + pytest.importorskip("py7zr") + import py7zr + + from core.archives import iter_archive_members + + seven = tmp_path / "multi.7z" + with py7zr.SevenZipFile(seven, "w") as z: + z.writestr(b"one", "a.txt") + z.writestr(b"two", "b.txt") + z.writestr(b"skip-bin", "c.bin") + z.writestr(b"X" * 5000, "huge.txt") + + got = { + name: data + for name, data in iter_archive_members( + seven, + "7z", + max_inner_size=100, + allowed_extensions={".txt"}, + ) + } + assert set(got) == {"a.txt", "b.txt"} + assert got["a.txt"] == b"one" + assert got["b.txt"] == b"two" + assert "c.bin" not in got + assert "huge.txt" not in got + + +def test_7z_encrypted_without_password_is_encrypted_no_password(tmp_path): + """Password-protected .7z without config password → encrypted_no_password, not archive_read_error.""" + pytest.importorskip("py7zr") + import py7zr + + from core.archives import ( + SCAN_FAILURE_REASON_ARCHIVE_READ_ERROR, + SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD, + iter_archive_members, + ) + + seven = tmp_path / "locked.7z" + with py7zr.SevenZipFile(seven, "w", password="correct-horse") as z: + z.writestr(b"secret payload", "vault.txt") + + failures: list[tuple[str, str, str]] = [] + + def _on_fail(reason: str, member: str, details: str) -> None: + failures.append((reason, member, details)) + + got = list( + iter_archive_members( + seven, + "7z", + max_inner_size=1_000_000, + allowed_extensions={".txt"}, + file_passwords={}, + on_member_read_failure=_on_fail, + ) + ) + assert got == [] + assert failures + assert all(r == SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD for r, _, _ in failures) + assert all(r != SCAN_FAILURE_REASON_ARCHIVE_READ_ERROR for r, _, _ in failures) + assert not any("AttributeError" in d for _, _, d in failures) + + +def test_7z_budget_max_members_yields_pre_budget_then_stops(tmp_path): + """#1253 Bugbot HIGH: budget stop must extract desired[], not return empty (ZIP parity).""" + pytest.importorskip("py7zr") + import py7zr + + from core.archives import ( + SCAN_FAILURE_REASON_ARCHIVE_BUDGET_EXCEEDED, + iter_archive_members, + ) + + seven = tmp_path / "budget.7z" + n_members = 6 + with py7zr.SevenZipFile(seven, "w") as z: + for i in range(n_members): + z.writestr(f"payload-{i}".encode(), f"m{i:02d}.txt") + + failures: list[tuple[str, str, str]] = [] + materialized: list[str] = [] + + def _on_fail(reason: str, member: str, details: str) -> None: + failures.append((reason, member, details)) + + for name, _data in iter_archive_members( + seven, + "7z", + max_inner_size=1_000_000, + allowed_extensions={".txt"}, + on_member_read_failure=_on_fail, + max_members=3, + max_total_uncompressed=1_000_000_000, + max_expansion_ratio=10_000.0, + ): + materialized.append(name) + + assert len(materialized) == 3 + assert len(materialized) < n_members + assert materialized # pre-budget members must not vanish + assert any(r == SCAN_FAILURE_REASON_ARCHIVE_BUDGET_EXCEEDED for r, _, _ in failures)