From 0d0a511970f554ab07c9ee1ee8a226cdfca8f904 Mon Sep 17 00:00:00 2001 From: William Smith Date: Sat, 1 Aug 2026 22:33:21 -0700 Subject: [PATCH 1/2] perf: 1.76x byte-identical decompress and lazy module source (issue #5) Decompression dominated project opening (88-96% per the issue's profile). Two changes, both output-preserving: decompress now produces output with slice operations wherever the spec allows: non-overlapping copy tokens (offset >= length) move as one slice, runs of literal tokens within a flag byte extend once, and the copy-token masks recompute only when the chunk-local output size crosses a power of two instead of once per token. Overlapping copies keep the spec's byte-at-a-time semantics, which are load-bearing there. Measured 12.4 -> 21.8 MB/s over the 31 module and dir streams in the live fixtures. A new oracle test suite pins the optimized decoder against the original per-byte implementation: live-fixture streams, synthetic round trips (overlap-heavy included), and byte-equal error messages and offsets on malformed input. A new max_bytes parameter returns a chunk-aligned prefix; chunk locality (enforced by this decoder) makes the prefix byte-identical to the same range of a full decompression. parse_vba_project now decompresses only the first chunk of each module stream. That prefix carries the Attribute header, and for single-chunk modules it already is the complete source, which stays eager at zero extra cost. Multi-chunk modules defer the remaining chunks to the first VBAModule.source access via a stored loader; stream lookup and MODULEOFFSET bounds checks remain eager, and a header that runs to the chunk boundary falls back to eager full decompression rather than risk a truncated attribute_header. VBAModule becomes a regular class with an unchanged constructor signature plus a source_loaded property; dataclass field equality and repr are gone (equality is identity). Opening the large-module fixture for module_names() drops 1.47 ms -> 0.79 ms, and saving an unrelated edit no longer decompresses untouched modules (tested). Verified byte-identical end to end: the 25-case save matrix across all live fixtures hashes identically against a v3.1.0 worktree baseline, and the live Excel gate passes. --- src/pyopenvba/vba.py | 299 ++++++++++++++++++++++++++++++++--------- tests/test_vba.py | 313 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 550 insertions(+), 62 deletions(-) diff --git a/src/pyopenvba/vba.py b/src/pyopenvba/vba.py index abf9957..3e4d7ae 100644 --- a/src/pyopenvba/vba.py +++ b/src/pyopenvba/vba.py @@ -25,7 +25,7 @@ from __future__ import annotations import struct -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass, field from enum import Enum from typing import TYPE_CHECKING @@ -58,7 +58,9 @@ def copy_token_help(decompressed_current: int, decompressed_chunk_start: int) -> return length_mask, offset_mask, bit_count -def decompress(data: bytes, *, stream_name: str = "") -> bytes: +def decompress( + data: bytes, *, stream_name: str = "", max_bytes: int | None = None +) -> bytes: """ Decompress a VBA stream using the MS-OVBA compression algorithm. @@ -81,6 +83,36 @@ def decompress(data: bytes, *, stream_name: str = "") -> bytes: ``stream_name`` is purely contextual: it is embedded in any :class:`VBAProjectError` raised here so callers can identify which CFB stream a malformed chunk came from. + + ``max_bytes``, when given, stops decompression at the first chunk + boundary at or beyond that many output bytes and returns the + chunk-aligned prefix. Copy tokens never reference across chunk + boundaries (this decoder enforces it), so the prefix is + byte-identical to the same range of a full decompression. Useful + for reading a module's ``Attribute VB_*`` header without paying for + the whole stream. + """ + out, _ = _decompress(data, stream_name=stream_name, max_bytes=max_bytes) + return out + + +def _decompress( + data: bytes, *, stream_name: str, max_bytes: int | None +) -> tuple[bytes, bool]: + """Core decoder behind :func:`decompress`. + + Returns ``(decompressed, consumed_all)`` where ``consumed_all`` is + True when the entire compressed input was processed — i.e. the + returned bytes are the complete decompression, not a prefix. + + Performance shape (measured; see issue #5): output is produced with + slice operations wherever the spec allows — non-overlapping copy + tokens move as one slice, runs of literal tokens within a flag byte + extend once — and the copy-token masks are recomputed only when the + chunk-local output size crosses a power of two, since + :func:`copy_token_help` depends on nothing else. Overlapping + copies (offset < length) keep the spec's byte-at-a-time semantics, + which are load-bearing there. """ def _err(msg: str, offset: int) -> VBAProjectError: @@ -92,13 +124,17 @@ def _err(msg: str, offset: int) -> VBAProjectError: raise _err("Invalid compressed stream: missing 0x01 signature byte.", 0) pos = 1 + n = len(data) out = bytearray() - while pos < len(data): - if pos + 2 > len(data): + while pos < n: + if max_bytes is not None and len(out) >= max_bytes: + return bytes(out), False + + if pos + 2 > n: raise _err("Truncated compressed stream: missing chunk header.", pos) - header = int(struct.unpack_from("> 12) & 0x7 chunk_flag = (header >> 15) & 0x1 @@ -113,10 +149,10 @@ def _err(msg: str, offset: int) -> VBAProjectError: ) chunk_end = pos + chunk_data_size - if chunk_end > len(data): + if chunk_end > n: raise _err( f"Truncated chunk: header announces {chunk_data_size} bytes " - f"but only {len(data) - pos} remain.", + f"but only {n - pos} remain.", header_offset, ) decompressed_chunk_start = len(out) @@ -129,32 +165,42 @@ def _err(msg: str, offset: int) -> VBAProjectError: f"got {chunk_data_size}.", header_offset, ) - if pos + 4096 > len(data): + if pos + 4096 > n: raise _err("Truncated raw chunk.", pos) out.extend(data[pos: pos + 4096]) pos += 4096 else: - # Token-compressed chunk. + # Token-compressed chunk. Masks depend only on the + # chunk-local output offset; recompute at power-of-two + # crossings instead of per token. + length_mask, offset_mask, bit_count = copy_token_help( + len(out), decompressed_chunk_start + ) + next_threshold = (1 << bit_count) + 1 + while pos < chunk_end: - if pos >= len(data): + if pos >= n: break - flag_byte = int(data[pos]) + flag_byte = data[pos] pos += 1 - for bit in range(8): - if pos >= chunk_end or pos >= len(data): + bit = 0 + while bit < 8: + if pos >= chunk_end or pos >= n: break if (flag_byte >> bit) & 1: # Copy token — back-reference into already-decompressed output. - if pos + 2 > len(data): + if pos + 2 > n: raise _err("Truncated copy token.", pos) - token = int(struct.unpack_from("= next_threshold: + length_mask, offset_mask, bit_count = copy_token_help( + len(out), decompressed_chunk_start + ) + next_threshold = (1 << bit_count) + 1 length = (token & length_mask) + 3 offset = ((token & offset_mask) >> (16 - bit_count)) + 1 @@ -171,16 +217,29 @@ def _err(msg: str, offset: int) -> VBAProjectError: pos - 2, ) - # Byte-by-byte copy; overlap is intentional and required by spec. - for _ in range(length): - out.append(out[copy_src]) - copy_src += 1 + if offset >= length: + # Source range is fully materialized: one slice. + out += out[copy_src: copy_src + length] + else: + # Overlapping copy; byte-at-a-time is required + # by spec semantics (the pattern repeats). + for _ in range(length): + out.append(out[copy_src]) + copy_src += 1 + bit += 1 else: - # Literal token. - out.append(int(data[pos])) - pos += 1 + # Run of literal tokens: consecutive clear bits in + # this flag byte copy verbatim as one slice. + run_start = pos + while (bit < 8 + and not ((flag_byte >> bit) & 1) + and pos < chunk_end + and pos < n): + pos += 1 + bit += 1 + out += data[run_start:pos] - return bytes(out) + return bytes(out), True def compress(data: bytes) -> bytes: @@ -678,35 +737,101 @@ class _DirInfo: # Public data model # --------------------------------------------------------------------------- -@dataclass class VBAModule: - """A parsed VBA module with its source code.""" - name: str - stream_name: str - source: str - kind: VBAModuleKind = VBAModuleKind.standard - text_offset: int = 0 - is_read_only: bool = False - is_private: bool = False - # Dir-stream sub-records preserved for round-trip serialization. - name_unicode: str = "" - stream_name_unicode: str = "" - doc_string: str = "" - doc_string_unicode: str = "" - help_context: int = 0 - cookie: int = 0 - # Original bytes 0..text_offset of the module stream (performance cache - # / version-dependent prefix). Preserved across write-back so that - # Office's cache invalidation logic operates the same way as it would - # for an untouched stream. - prefix_bytes: bytes = field(default=b"", repr=False) - # Whether the source has been edited and needs to be recompressed on save. - dirty: bool = field(default=False, repr=False) - # Cached attribute header captured at parse time so a body-only source - # replacement (the VBE-style edit surface) can re-prepend the required - # ``Attribute VB_*`` / ``VERSION ... CLASS`` block. Re-derived on demand - # when missing via ``split_attribute_header(self.source)``. - attribute_header: str = field(default="", repr=False) + """A parsed VBA module with its source code. + + ``source`` may be materialized lazily: modules produced by + :func:`parse_vba_project` defer the MS-OVBA decompression of their + stream until the first ``.source`` access (issue #5 — decompression + is 88-96% of the cost of opening a project, and callers that only + want names, kinds, or one module out of many should not pay for all + of it). Constructing a module with an explicit ``source`` string is + fully eager and behaves exactly as before. One consequence of + laziness: a corrupt chunk past the first one raises + :class:`VBAProjectError` at first access rather than at parse time. + """ + + def __init__( + self, + name: str, + stream_name: str, + source: str = "", + kind: VBAModuleKind = VBAModuleKind.standard, + text_offset: int = 0, + is_read_only: bool = False, + is_private: bool = False, + # Dir-stream sub-records preserved for round-trip serialization. + name_unicode: str = "", + stream_name_unicode: str = "", + doc_string: str = "", + doc_string_unicode: str = "", + help_context: int = 0, + cookie: int = 0, + # Original bytes 0..text_offset of the module stream (performance + # cache / version-dependent prefix). Preserved across write-back + # so that Office's cache invalidation logic operates the same way + # as it would for an untouched stream. + prefix_bytes: bytes = b"", + # Whether the source has been edited and needs recompression on save. + dirty: bool = False, + # Cached attribute header captured at parse time so a body-only + # source replacement (the VBE-style edit surface) can re-prepend + # the required ``Attribute VB_*`` block. Re-derived on demand when + # missing via ``split_attribute_header(self.source)``. + attribute_header: str = "", + source_loader: Callable[[], str] | None = None, + ) -> None: + self.name = name + self.stream_name = stream_name + self.kind = kind + self.text_offset = text_offset + self.is_read_only = is_read_only + self.is_private = is_private + self.name_unicode = name_unicode + self.stream_name_unicode = stream_name_unicode + self.doc_string = doc_string + self.doc_string_unicode = doc_string_unicode + self.help_context = help_context + self.cookie = cookie + self.prefix_bytes = prefix_bytes + self.dirty = dirty + self.attribute_header = attribute_header + if source_loader is not None: + self._source: str | None = None + self._source_loader: Callable[[], str] | None = source_loader + else: + self._source = source + self._source_loader = None + + def __repr__(self) -> str: + return ( + f"VBAModule(name={self.name!r}, stream_name={self.stream_name!r}, " + f"kind={self.kind!r}, text_offset={self.text_offset}, " + f"source_loaded={self._source is not None})" + ) + + @property + def source_loaded(self) -> bool: + """True once the source text is materialized in memory.""" + return self._source is not None + + @property + def source(self) -> str: + """The module's full source text (header plus body). + + Materializes lazily on first access for parsed modules. + """ + if self._source is None: + loader = self._source_loader + assert loader is not None, "lazy module lost its loader" + self._source = loader() + self._source_loader = None + return self._source + + @source.setter + def source(self, value: str) -> None: + self._source = value + self._source_loader = None @property def body(self) -> str: @@ -1470,12 +1595,31 @@ def _project_section_end(lines: list[str]) -> int: # Public factory # --------------------------------------------------------------------------- +def _make_source_loader( + compressed: bytes, encoding: str, stream_name: str +) -> Callable[[], str]: + """Build the deferred decompress-and-decode thunk for a lazy module.""" + def _load() -> str: + return decompress(compressed, stream_name=stream_name).decode( + encoding, errors="replace" + ) + return _load + + def parse_vba_project(cfb: CFB) -> VBAProject: """ - Extract and decompress all VBA module sources from a parsed CFB. + Extract all VBA module metadata from a parsed CFB. The CFB must be the vbaProject.bin from an xlsm/xlsb, or the whole file for an xls workbook. + + Module source text is loaded lazily: only the first compressed + chunk of each module stream is decompressed here (enough for the + ``Attribute VB_*`` header, and for single-chunk modules it is + already the whole source). The remaining chunks decompress on the + first ``VBAModule.source`` access. Stream lookup and MODULEOFFSET + bounds checks stay eager so structural corruption still surfaces at + parse time. """ try: dir_compressed = cfb.get_stream_in_storage("VBA", "dir") @@ -1510,17 +1654,47 @@ def parse_vba_project(cfb: CFB) -> VBAProject: f"{len(stream_compressed)} for module {info.name!r}." ) - compressed_source = stream_compressed[info.text_offset:] - source_bytes = decompress( - compressed_source, stream_name=f"VBA/{stream_name}" + compressed_source = bytes(stream_compressed[info.text_offset:]) + ovba_name = f"VBA/{stream_name}" + + # Decompress just the first chunk. For single-chunk modules + # (decompressed size <= 4096) this is the complete source and + # the module is materialized eagerly with zero extra work. + prefix_raw, consumed_all = _decompress( + compressed_source, stream_name=ovba_name, max_bytes=1 ) - source = source_bytes.decode(encoding, errors="replace") - header, _ = split_attribute_header(source) + prefix_text = prefix_raw.decode(encoding, errors="replace") + + source: str | None + loader: Callable[[], str] | None + if consumed_all: + source = prefix_text + header, _ = split_attribute_header(prefix_text) + loader = None + else: + header, body_prefix = split_attribute_header(prefix_text) + if header and body_prefix: + # Header fits inside the first chunk (the normal case); + # defer the rest of the stream. + source = None + loader = _make_source_loader( + compressed_source, encoding, ovba_name + ) + else: + # Either no attribute header at all, or the header runs + # to the chunk boundary and may be truncated. Both are + # abnormal; fall back to eager full decompression so + # attribute_header is always derived from complete text. + source = decompress( + compressed_source, stream_name=ovba_name + ).decode(encoding, errors="replace") + header, _ = split_attribute_header(source) + loader = None modules.append(VBAModule( name=info.name, stream_name=stream_name, - source=source, + source=source if source is not None else "", kind=info.module_kind, text_offset=info.text_offset, is_read_only=info.is_read_only, @@ -1531,8 +1705,9 @@ def parse_vba_project(cfb: CFB) -> VBAProject: doc_string_unicode=info.doc_string_unicode, help_context=info.help_context, cookie=info.cookie, - prefix_bytes=stream_compressed[: info.text_offset], + prefix_bytes=bytes(stream_compressed[: info.text_offset]), attribute_header=header, + source_loader=loader, )) project = VBAProject(modules=modules, code_page=code_page) diff --git a/tests/test_vba.py b/tests/test_vba.py index 4688a7f..7431dd8 100644 --- a/tests/test_vba.py +++ b/tests/test_vba.py @@ -10,6 +10,7 @@ from pyopenvba.exceptions import VBAProjectError from pyopenvba.vba import ( CLASS_MODULE_CLSID, + VBAModule, VBAModuleKind, VBAProject, compress, @@ -890,3 +891,315 @@ def test_repeating_grams_with_capped_matches(self) -> None: # Many identical 3-grams whose matches hit the per-position # length cap: exercises the early-exit tie-break path. self._assert_equivalent(bytes(range(16)) * 128) + + +class TestDecompressOracleEquivalence: + """The optimized decoder (slice copies for non-overlapping tokens, + literal-run batching, hoisted copy-token masks) must be byte-exact + against the original per-byte reference implementation, including + error positions on malformed input (issue #5).""" + + @staticmethod + def _naive_decompress(data: bytes, *, stream_name: str = "") -> bytes: + """Reference implementation: the original per-byte decoder.""" + def _err(msg: str, offset: int) -> VBAProjectError: + return VBAProjectError(f"{msg} [stream={stream_name!r}, offset={offset}]") + + if not data or data[0] != 0x01: + raise _err("Invalid compressed stream: missing 0x01 signature byte.", 0) + pos = 1 + out = bytearray() + while pos < len(data): + if pos + 2 > len(data): + raise _err("Truncated compressed stream: missing chunk header.", pos) + header = int(struct.unpack_from("> 12) & 0x7 + chunk_flag = (header >> 15) & 0x1 + header_offset = pos + pos += 2 + if chunk_signature != 0b011: + raise _err( + f"Bad compressed chunk signature: expected 0b011, " + f"got {chunk_signature:#05b}.", + header_offset, + ) + chunk_end = pos + chunk_data_size + if chunk_end > len(data): + raise _err( + f"Truncated chunk: header announces {chunk_data_size} bytes " + f"but only {len(data) - pos} remain.", + header_offset, + ) + decompressed_chunk_start = len(out) + if chunk_flag == 0: + if chunk_data_size != 4096: + raise _err( + f"Raw chunk must have exactly 4096 data bytes; " + f"got {chunk_data_size}.", + header_offset, + ) + if pos + 4096 > len(data): + raise _err("Truncated raw chunk.", pos) + out.extend(data[pos: pos + 4096]) + pos += 4096 + else: + while pos < chunk_end: + if pos >= len(data): + break + flag_byte = int(data[pos]) + pos += 1 + for bit in range(8): + if pos >= chunk_end or pos >= len(data): + break + if (flag_byte >> bit) & 1: + if pos + 2 > len(data): + raise _err("Truncated copy token.", pos) + token = int(struct.unpack_from("> (16 - bit_count)) + 1 + copy_src = len(out) - offset + if copy_src < 0: + raise _err( + "Copy token references before start of output.", + pos - 2, + ) + if copy_src < decompressed_chunk_start: + raise _err( + "Copy token references before the start of the " + "current chunk.", + pos - 2, + ) + for _ in range(length): + out.append(out[copy_src]) + copy_src += 1 + else: + out.append(int(data[pos])) + pos += 1 + return bytes(out) + + @staticmethod + def _live_fixture_streams() -> list[bytes]: + import zipfile + + from pyopenvba.cfb import CFB + from pyopenvba.vba import parse_dir_stream + + streams: list[bytes] = [] + fixtures = [ + ("tests/live_excel_testing/test_macro_workbook.xlsm", "xl/vbaProject.bin"), + ("tests/live_excel_testing/large_vba_module.xlsm", "xl/vbaProject.bin"), + ("tests/live_word_testing/Doc1.docm", "word/vbaProject.bin"), + ("tests/live_powerpoint_testing/Presentation1.pptm", "ppt/vbaProject.bin"), + ] + base = Path(__file__).parent.parent + for rel, entry in fixtures: + path = base / rel + if not path.exists(): + continue + with zipfile.ZipFile(path) as zf: + cfb = CFB.from_bytes(zf.read(entry)) + dir_comp = cfb.get_stream_in_storage("VBA", "dir") + streams.append(bytes(dir_comp)) + _, mods = parse_dir_stream(decompress(dir_comp)) + for m in mods: + try: + raw = cfb.get_stream_in_storage("VBA", m.stream_name or m.name) + except KeyError: + continue + streams.append(bytes(raw[m.text_offset:])) + return streams + + def test_live_fixture_streams_byte_identical(self) -> None: + streams = self._live_fixture_streams() + if not streams: + pytest.skip("no live fixtures available") + for i, s in enumerate(streams): + assert decompress(s) == self._naive_decompress(s), f"stream {i} diverges" + + def test_synthetic_round_trips_byte_identical(self) -> None: + import random + + rng = random.Random(20260801) + inputs = [ + b"", + b"A" * 9000, # overlap-heavy copy tokens + b"AB" * 5000, + bytes(range(16)) * 600, + bytes(rng.randrange(256) for _ in range(10000)), + (b"Sub Demo()\r\n MsgBox 1\r\nEnd Sub\r\n" * 200), + ] + for i, plain in enumerate(inputs): + comp = compress(plain) + got = decompress(comp) + assert got == plain, f"input {i}: round trip broke" + assert got == self._naive_decompress(comp), f"input {i}: oracle diverges" + + def test_malformed_inputs_raise_identical_errors(self) -> None: + comp = compress(b"Hello VBA world, hello again, hello hello.\r\n" * 40) + malformed = [ + b"", + b"\x02", # wrong signature byte + comp[:1], # signature byte alone + comp[:5], # truncated mid-chunk + comp[: len(comp) // 2], # truncated later + b"\x01" + b"\x00\x00", # bad chunk signature bits + ] + for i, bad in enumerate(malformed): + new_msg = naive_msg = None + try: + decompress(bad) + except VBAProjectError as exc: + new_msg = str(exc) + try: + self._naive_decompress(bad) + except VBAProjectError as exc: + naive_msg = str(exc) + assert new_msg == naive_msg, f"case {i}: {new_msg!r} != {naive_msg!r}" + + +class TestDecompressMaxBytes: + def test_prefix_is_chunk_aligned_and_byte_identical(self) -> None: + plain = (b"Sub P()\r\n MsgBox 42\r\nEnd Sub\r\n" * 400) # multi-chunk + comp = compress(plain) + full = decompress(comp) + assert full == plain + for limit in (1, 100, 4096, 4097, 8000, len(plain)): + prefix = decompress(comp, max_bytes=limit) + assert len(prefix) >= min(limit, len(plain)) + assert len(prefix) % 4096 == 0 or len(prefix) == len(plain) + assert full.startswith(prefix) + + def test_max_bytes_beyond_length_returns_full(self) -> None: + plain = b"Short module\r\n" + comp = compress(plain) + assert decompress(comp, max_bytes=10_000_000) == plain + + def test_internal_consumed_flag(self) -> None: + from pyopenvba import vba as _vba + + plain = b"X" * 10000 # 3 chunks + comp = compress(plain) + inner = getattr(_vba, "_decompress") + prefix, consumed = inner(comp, stream_name="t", max_bytes=1) + assert not consumed and len(prefix) == 4096 + whole, consumed = inner(comp, stream_name="t", max_bytes=None) + assert consumed and whole == plain + # A limit the final chunk satisfies exactly still reports complete. + whole2, consumed2 = inner(comp, stream_name="t", max_bytes=len(plain)) + assert consumed2 and whole2 == plain + + +class TestLazyModuleSource: + """Module source materializes on first access (issue #5). Single + chunk modules are eager (the header prefix already IS the source); + multi-chunk modules defer everything past chunk one.""" + + _LARGE = Path(__file__).parent / "live_excel_testing" / "large_vba_module.xlsm" + + def _project_and_reference(self) -> tuple[VBAProject, str]: + import zipfile + + from pyopenvba.cfb import CFB + from pyopenvba.vba import parse_dir_stream, parse_vba_project + + if not self._LARGE.exists(): + pytest.skip("large_vba_module.xlsm not present") + with zipfile.ZipFile(self._LARGE) as zf: + raw = zf.read("xl/vbaProject.bin") + cfb = CFB.from_bytes(raw) + # Reference source computed the pre-lazy way: full decompression. + dir_raw = decompress(cfb.get_stream_in_storage("VBA", "dir")) + info, mods = parse_dir_stream(dir_raw) + target = next(m for m in mods if m.name == "Large_Module_") + stream = cfb.get_stream_in_storage("VBA", target.stream_name or target.name) + reference = decompress(stream[target.text_offset:]).decode( + f"cp{info.code_page}", errors="replace" + ) + assert len(reference) > 4096, "fixture module must span multiple chunks" + return parse_vba_project(cfb), reference + + def test_multichunk_module_starts_unloaded(self) -> None: + project, _ = self._project_and_reference() + module = project.get_module("Large_Module_") + assert not module.source_loaded + # Names, kinds, and headers are available without materializing. + assert project.module_names() + assert module.attribute_header.startswith("Attribute VB_Name") + assert not module.source_loaded + + def test_lazy_source_matches_eager_reference(self) -> None: + project, reference = self._project_and_reference() + module = project.get_module("Large_Module_") + assert module.source == reference + assert module.source_loaded + + def test_attribute_header_matches_full_decompression(self) -> None: + project, reference = self._project_and_reference() + module = project.get_module("Large_Module_") + assert module.attribute_header == split_attribute_header(reference)[0] + + def test_body_access_forces_and_matches(self) -> None: + project, reference = self._project_and_reference() + module = project.get_module("Large_Module_") + assert module.body == split_attribute_header(reference)[1] + assert module.source_loaded + + def test_source_assignment_discards_loader(self) -> None: + project, _ = self._project_and_reference() + module = project.get_module("Large_Module_") + module.source = "Attribute VB_Name = \"Large_Module_\"\r\n\r\nSub S()\r\nEnd Sub\r\n" + assert module.source_loaded + assert "Sub S()" in module.source + + def test_eager_constructor_unchanged(self) -> None: + module = VBAModule(name="M", stream_name="M", source="Sub A()\r\nEnd Sub\r\n") + assert module.source_loaded + assert module.source == "Sub A()\r\nEnd Sub\r\n" + + def test_lazy_round_trips_through_full_pipeline(self, tmp_path: Path) -> None: + """A pyOpenVBA-authored multi-chunk module survives save, reopens + lazy, and materializes to exactly what was written.""" + from pyopenvba import ExcelFile + + body = "".join( + f"Sub Filler{i}()\r\n Debug.Print {i}\r\n" + f" ' padding line for chunk spill {i:04d}\r\nEnd Sub\r\n" + for i in range(120) + ) + target = tmp_path / "big.xlsm" + with ExcelFile.create_new(target) as wb: + wb.vba_project().add_module("BigMod", body, kind=VBAModuleKind.standard) + wb.save() + with ExcelFile(target) as wb: + module = wb.vba_project().get_module("BigMod") + assert not module.source_loaded, "expected multi-chunk module to defer" + assert module.source.endswith("End Sub\r\n") + assert "chunk spill 0119" in module.source + + def test_save_does_not_force_untouched_modules(self, tmp_path: Path) -> None: + """Editing one module and saving must not decompress the others.""" + import shutil + + from pyopenvba import ExcelFile + + if not self._LARGE.exists(): + pytest.skip("large_vba_module.xlsm not present") + work = tmp_path / "work.xlsm" + shutil.copy(self._LARGE, work) + with ExcelFile(work) as wb: + project = wb.vba_project() + victim = next( + m.name for m in project.modules if m.name != "Large_Module_" + ) + wb.set_module(victim, "Sub Edited()\r\nEnd Sub\r\n") + wb.save() + untouched = project.get_module("Large_Module_") + assert not untouched.source_loaded, ( + "saving an unrelated edit forced decompression of an " + "untouched multi-chunk module" + ) From 2930c01e5fca0101d686fef171aba654807ca404 Mon Sep 17 00:00:00 2001 From: William Smith Date: Sat, 1 Aug 2026 22:33:21 -0700 Subject: [PATCH 2/2] release prep: 3.2.0 version bump and changelog --- docs/changelog.md | 37 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- src/pyopenvba/__init__.py | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 904e216..1086c20 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,43 @@ All notable changes to pyOpenVBA are documented here. This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.2.0] - 2026-08-01 + +### Changed + +- **Decompression is 1.76x faster, byte-for-byte identical** (issue #5). + `decompress` now emits output with slice operations wherever the spec + allows -- non-overlapping copy tokens move as one slice, runs of + literal tokens within a flag byte extend once -- and recomputes the + copy-token masks only when the chunk-local output size crosses a + power of two. Overlapping copies keep the spec's byte-at-a-time + semantics. Measured 12.4 -> 21.8 MB/s across the 31 module and dir + streams in the live fixtures; new oracle-equivalence tests pin the + optimized decoder against the original per-byte implementation, + including identical error messages and offsets on malformed input. +- **Module source loads lazily** (issue #5). Decompressing module + source is 88-96% of the cost of opening a project, so + `parse_vba_project` now decompresses only the first chunk of each + module stream (enough for the `Attribute VB_*` header; for + single-chunk modules it already is the whole source) and defers the + rest until the first `VBAModule.source` access. Stream lookup and + MODULEOFFSET bounds checks stay eager. Opening the large-module + fixture for `module_names()` drops from 1.47 ms to 0.79 ms. Two + visible consequences: a corrupt chunk past the first one raises + `VBAProjectError` at first access instead of at parse time, and + `VBAModule` is now a regular class rather than a dataclass -- the + constructor signature is unchanged, a new `source_loaded` property + reports materialization, but dataclass-generated field equality and + repr are gone (equality is identity). + +### Added + +- `decompress(..., max_bytes=N)` stops at the first chunk boundary at + or beyond N output bytes and returns the chunk-aligned prefix. Copy + tokens never cross chunk boundaries (the decoder enforces it), so + the prefix is byte-identical to the same range of a full + decompression. + ## [3.1.0] - 2026-07-22 ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 734291e..495ff91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "pyOpenVBA" -version = "3.1.0" +version = "3.2.0" description = "Read and write VBA macros inside Excel, Word, and PowerPoint files in pure Python, no dependencies." readme = "README.md" license = { text = "MIT" } diff --git a/src/pyopenvba/__init__.py b/src/pyopenvba/__init__.py index 134164a..026562a 100644 --- a/src/pyopenvba/__init__.py +++ b/src/pyopenvba/__init__.py @@ -204,4 +204,4 @@ def pull_access( "push_word", ] -__version__ = "3.1.0" +__version__ = "3.2.0"