From 8aa4742337576e86a0bfbf3d1ce01dac367c41b4 Mon Sep 17 00:00:00 2001 From: themuffinator Date: Thu, 27 Aug 2026 02:01:54 +0100 Subject: [PATCH] Document v0.10 save compatibility repair --- .github/workflows/commit-validation.yml | 2 +- .github/workflows/push-verification.yml | 2 +- docs/dev/release-completion.md | 1 + docs/dev/releases/v0.12.0.md | 3 + docs/dev/savegame-compatibility-policy.md | 27 ++++++++- docs/dev/savegame-reliability.md | 46 +++++++++++++-- tools/tests/savegame_v3_contract.py | 72 +++++++++++++++++++++++ 7 files changed, 145 insertions(+), 8 deletions(-) diff --git a/.github/workflows/commit-validation.yml b/.github/workflows/commit-validation.yml index caf21e1d..da51a07f 100644 --- a/.github/workflows/commit-validation.yml +++ b/.github/workflows/commit-validation.yml @@ -14,7 +14,7 @@ permissions: env: # Keep the engine's shared File/FileSystem ABI locked to the companion # revision that implements GAME_API 45 and generated-animation cache v3. - OPENQ4_GAMELIBS_SHA: 79c9d7d1e26952ffd8e5352f7d51b64ee08b9d82 + OPENQ4_GAMELIBS_SHA: 09fa7e175efa5a5d819569ba577fd0e11c38a891 jobs: script-smoke: diff --git a/.github/workflows/push-verification.yml b/.github/workflows/push-verification.yml index 55b5bcc9..0325a26c 100644 --- a/.github/workflows/push-verification.yml +++ b/.github/workflows/push-verification.yml @@ -10,7 +10,7 @@ permissions: env: # Keep the engine's shared File/FileSystem ABI locked to the companion # revision that implements GAME_API 45 and generated-animation cache v3. - OPENQ4_GAMELIBS_SHA: 79c9d7d1e26952ffd8e5352f7d51b64ee08b9d82 + OPENQ4_GAMELIBS_SHA: 09fa7e175efa5a5d819569ba577fd0e11c38a891 concurrency: group: openq4-push-verification-${{ github.ref }} diff --git a/docs/dev/release-completion.md b/docs/dev/release-completion.md index 7be2dd6b..21d335fb 100644 --- a/docs/dev/release-completion.md +++ b/docs/dev/release-completion.md @@ -141,6 +141,7 @@ is `docs/dev/macos-moltenvk-decision.md`. - [x] Objective popups now preserve retail simple-window clipping behavior, so long stock objective titles such as "Regroup with Rhino Squad" render fully instead of losing their final characters. - [x] Failed or incompatible save restores no longer crash a second time while the game tears down a partially restored actor attachment. Cleanup now safely releases an attachment whose physics state was never installed, while preserving the normal visual and collision teardown path for fully restored entities. This fixes the save-load crash reported in GitHub issue #107. - [x] Savegame handling is stricter and more recoverable: new saves carry a generated engine/GameLibs source snapshot stamp plus payload sync markers, an end footer, an exact wire-ABI stamp, and whole-file CRC protection; the session save header/description and GameLibs payload writers detect short writes before committing staged files; and staged saves must validate their persistent-player dictionaries, payload version/ABI, CRC, and footer before replacing an existing slot. Generated description sidecars and preview TGA headers are validated before promotion, the load path commits persistent-player state only after the full save header is valid and preflights gameplay compatibility before tearing down the current map, the save/load menu filters malformed save headers and unsafe slot names, description sidecars are read through a bounded parser and must still map back to their save slot, outbound dict/winding/object data is bounded, AI movement path caches are initialized and serialized with bounded live entries, and source stamps are stable across Windows and Unix line endings. Build/source drift is diagnostic for schema-compatible version 3 saves; version 2 requires one of six exact Windows x64 snapshots, and unstamped legacy saves require the current build plus the Windows/MSVC x64 `raw1` ABI. Unsupported or corrupted saves now fail at preflight or the nearest later restore boundary with a clear ABI, checksum, marker, or footer diagnostic instead of surfacing later as invalid AI path lengths or other drift errors. +- [x] Released v0.10 Windows x64 saves now restore consistently across optimized and debug GameLib builds: the exact affected version-3 source tuple defaults its two absent player liquid-state fields, the reader recognizes the empty physics frame that MSVC identical-code folding caused optimized writers to omit, and current writers derive frame ownership from source declarations instead of linker-foldable function addresses. The reporter's exact issue #123 fixture passes its original physics and `ReadTrace` boundaries; a fresh stock `airdefense2` save/reload completed in active gameplay with a visually reviewed engine screenshot, while final HiDef-mod confirmation remains with the reporter because the supplied archive omitted that mod's GUI/assets. - [x] Savegame restore now also bounds SP/MP script interpreter state before rebuilding scripts from disk: call-stack depth, local-stack usage, function indices, statement pointers, stack bases, pop parameters, and multi-frame event names are validated so malformed saves fail at restore with a clear script error instead of indexing fixed interpreter arrays. - [x] Savegame restore now rejects malformed SP/MP scalar state before it can steer later arrays or script cleanup: script-program watermarks, actor animation channels, client entity numbers, crawl-effect joint indices, crawl directions, and simple animated-entity counters are range-checked during restore. - [x] Savegame sound and render restore is stricter: sound-world payloads now use checked primitive/string reads and writes with practical emitter/channel caps and sound-class validation, restored sound emitters retain their serialized indices instead of aliasing reusable empty emitters, trailing empty emitters are omitted from new saves, and SP/MP render entities now reject malformed saved bounds in release builds instead of relying only on debug asserts. diff --git a/docs/dev/releases/v0.12.0.md b/docs/dev/releases/v0.12.0.md index ee8e44ef..40cce90d 100644 --- a/docs/dev/releases/v0.12.0.md +++ b/docs/dev/releases/v0.12.0.md @@ -11,6 +11,7 @@ - **The complex `airdefense1` opening reaches gameplay faster and its CPU-limited scene runs more smoothly.** Cinematic fast-forward no longer spends time interpolating thousands of poses that cannot be displayed, and ordinary maps without baked light grids no longer build a bake-only probe layout while loading. The guarded learned-cache experiment remains off by default; these gains apply to the classic source path. - **Shadow-mapped props now cast complete, grounded shadows.** Open and thin stock models such as the buggy, crates, railings, and computer props around the `airdefense1` start no longer disappear from point-light shadow maps, while sealed meshes store their near surface instead of a detached far shell. Tighter balanced filtering and a world-space cap for huge point-light bias preserve contact corners; any caster that cannot enter the map is supplied by the matching stencil ownership or safely returns that ownership to full stencil. - **Stock maps no longer report or expose known startup gaps.** The engine first honours any loose or mod-provided versions of the three omitted brown-fluid effect images and the omitted large water-splash sample, then uses shape- and family-compatible media that did ship with Quake 4. Generated TrueType atlases also bind their uploaded images on first parse, wide loading backgrounds and generated image caches remain reliable from deeply nested save locations, and rigid bodies that the stock game deliberately clamps or forces to rest remain visible as developer diagnostics without being mislabeled as unresolved warnings. +- **Released v0.10 saves no longer depend on whether the game module was optimized.** The exact affected Windows x64 save layout now restores its omitted physics frame and two later player liquid-state fields safely, while new saves use one deterministic class-frame layout in debug and release builds. - **Polish and Russian are now built in.** Text rendering works by Unicode code point, the Polish tables use the correct Central European encoding path, and generated font data is checksum-validated so translated menus and HUD text remain reproducible across packages. - **The marine hovertank has its vehicle audio back.** Single-player once again creates and updates the engine and hover-pad loops used throughout the vehicle sequence, with safe cleanup during map or engine shutdown. - **Long macOS sessions keep their sound instead of dropping back to the menu.** Apple Silicon packages now include OpenAL Soft, avoiding the fixed source and buffer limits in Apple's legacy OpenAL framework that could stop stock levels after sustained audio use. @@ -27,6 +28,7 @@ - Mac users should replace the complete package to receive the bundled OpenAL Soft runtime; no separate OpenAL installation is needed. The package includes its corresponding licence notices and source offer. - Linux packages no longer contain duplicate root icons whose names differ only by letter case, so the same archive can be extracted safely on Windows and other case-insensitive filesystems. - Existing settings and compatible saves do not need to be reset. As a precaution, retain a copy of important saves before replacing an older installation. +- Players upgrading from released v0.10 Windows x64 builds can retry saves that previously stopped at a physics or `ReadTrace` restore error. A modded save still requires the same mod maps, GUIs, and other assets that created it; keep a backup before testing. - The exact older broad shadow-filter defaults are upgraded once to the new balanced contact-shadow values; customized shadow profiles are left untouched. `r_shadowMapCasterCulling 2` remains the recommended value but now means topology-aware automatic culling rather than far-shell storage. Use `0` for an always-two-sided comparison or `1` to force near-shell culling. - Level-load data under the active `fs_savepath` `generated/` directory is private and disposable. `com_levelLoadModernization` defaults to `0` and overrides older archived cache settings, so ordinary play uses the classic source path. Set it to `1` only for focused cache evaluation. Never delete the original retail or mod assets. - Retail Quake 4 assets are still required. Point openQ4 at a complete retail installation, including the unsuffixed base dialogue archive such as `zpak_english.pk4`. Numbered files such as `zpak_english_01.pk4` are patches and must not be renamed as a substitute; without the base archive, dialogue is silent and conversation-gated campaign scenes can fail to advance. @@ -78,6 +80,7 @@ - Restored the SDL OpenGL context before applying startup or live swap-interval changes, keeping `r_swapInterval` effective when native Wayland window setup temporarily detaches the context. - Initialized engine-side triangle-surface allocators for `dmap` and related geometry tools. - Restored clean, fixed-size manual save-game previews at widescreen resolutions and prevented renderer row padding from leaking into narrow captures. +- Restored the exact affected v0.10 Windows x64 version-3 save layout by defaulting its absent swim-speed and liquid-surface timer fields and accepting the empty physics frame omitted by optimized builds; current writers now use source-declared class ownership so linker folding cannot change save bytes. - Replaced plaintext-by-default remote console authentication with bounded `rcon2`, redacted private password CVars throughout console and persistence paths, restricted server-originated CVar dictionaries to their declared userinfo/network-sync authority with all-or-nothing decoding, constrained server package URLs to bounded HTTP/HTTPS with syntax-checked hosts, and retired executable download/launch behavior from the legacy updater. Standard packages keep direct PK4 transfer disabled; separately integrated curl-enabled builds add time-limited, no-redirect transfer containment. - Restored `si_pure` multiplayer enforcement with ordered asset-PK4 checks and a platform-independent Quake 4 1.4.2 protocol 2.41 compatibility token for already installed openQ4 game modules; pure negotiation cannot download, extract, or restart into executable code, and the token is not cryptographic module attestation. - Added fail-closed read-underflow tracking and range/type checks to audited queued-message, user-command, SP/MP snapshot, server-demo, player, projectile, and hit-scan decode paths. Audited leaf readers stage their fields until valid, and a malformed top-level snapshot tears down the session before another game or presentation frame. diff --git a/docs/dev/savegame-compatibility-policy.md b/docs/dev/savegame-compatibility-policy.md index 38610fe3..1512f3e0 100644 --- a/docs/dev/savegame-compatibility-policy.md +++ b/docs/dev/savegame-compatibility-policy.md @@ -9,6 +9,7 @@ load. Reliability mechanisms and implementation detail are documented in | Save payload | Current decision | | --- | --- | | Version 3, exact wire-ABI stamp and valid integrity/footer | Supported format path; build/source drift is diagnostic only | +| Version 3, released v0.10 build/source tuple on Windows/MSVC x64 `raw1` | Supported by an exact decoder for its two absent player liquid-state fields and optimized empty physics frame | | Version 3, different wire-ABI stamp | Rejected before map teardown | | Version 2, exact tuple in the allowlist below | Supported only on Windows/MSVC x64 little-endian `raw1` | | Version 2, any other tuple | Rejected before map teardown | @@ -44,6 +45,26 @@ Eligibility is not runtime certification. For example, a Linux arm64 v3 save may be format-eligible on another Linux arm64 `raw1` build, but it is supported for a release only when that platform has matching candidate runtime evidence. +One released v0.10 snapshot used version 3 before two player liquid-state fields +were added: + +```text +build 1 +source 19351be39d2d4077a74294c0442707ef9565fc7a2fa9af9b81e05fc9aca8b220 +404 files +windows-msvcabi-x64-le-raw1 +``` + +The SP and MP readers recognize only that complete tuple when deciding that +`idPhysics_Player::swimSpeed` and `idPlayer::nextLiquidSurfaceSoundTime` are +absent. Both fields receive safe defaults, and every unrelated v3 source snapshot +continues to use the current layout. A second v0.10 release-build defect came +from MSVC identical-code folding: the former pointer-address comparison could +omit the empty `idPhysics` class frame. Current readers use the surrounding sync +sequence to accept that known omission, while current writers derive class-frame +ownership from source declarations so optimization can no longer alter the wire +format. + ## Exact Version 2 Allowlist Version 2 did not carry its own wire-ABI field. The current reader assigns v2 only @@ -86,6 +107,8 @@ Backward compatibility means a newer runtime reading an older save. It is supported only through an explicit decoder or allowlist: - current v3 on the exact ABI path; +- the exact released v0.10 v3 snapshot through its bounded player-field decoder + and legacy empty-physics-frame recognition; - the six exact v2 snapshots above on Windows x64 `raw1`; and - the narrow same-build unstamped Windows x64 legacy path. @@ -164,7 +187,9 @@ Every versioned change must update, in one change set: 6. this policy, reliability documentation, and player-facing upgrade notes. The generated source hash is evidence and diagnosis for v3, not a substitute for -this versioning decision. +this versioning decision. The v0.10 decoder above is a narrowly reviewed repair +for a format that had already shipped without a required version bump; it is not +precedent for adding fields within v3. ## Failure and User-Message Policy diff --git a/docs/dev/savegame-reliability.md b/docs/dev/savegame-reliability.md index a8421afa..9c0309a5 100644 --- a/docs/dev/savegame-reliability.md +++ b/docs/dev/savegame-reliability.md @@ -62,6 +62,25 @@ compatibility gate: same-version payloads may load across build/source changes when the exact wire-ABI stamp still matches. This makes schema-version discipline mandatory. +### Released v0.10 Compatibility Repair + +The released Windows x64 v0.10 snapshot identified by build `1`, source SHA-256 +`19351be39d2d4077a74294c0442707ef9565fc7a2fa9af9b81e05fc9aca8b220`, 404 +source files, and `windows-msvcabi-x64-le-raw1` predates two fields that were +later added without a version bump. Its reader initializes `swimSpeed` and +`nextLiquidSurfaceSoundTime` to safe defaults instead of consuming bytes that +belong to the following fields. The complete tuple is required; other v3 source +snapshots are not guessed to have the older player layout. + +That release could also serialize a different class-frame sequence depending on +link optimization. The old dispatcher compared member-function addresses, and +MSVC identical-code folding made the empty `idPhysics` and `idClass` methods +appear identical in optimized builds. Current restore code recognizes the +missing empty physics frame from the next sync-marker sequence. Current writers +record whether each class declares its own `Save` and `Restore` methods, making +the emitted frame sequence deterministic across debug and optimized link +profiles. + ## Save Transaction and Crash Recovery The old slot is not replaced while a new payload is still being constructed. @@ -283,6 +302,22 @@ An affected old save may already contain indeterminate data. CRC cannot repair that data, and the restore path intentionally rejects it rather than inventing an AI path. +## Issue #123 + +The attached v0.10 save in issue #123 exposed both historical v3 differences +above. Reconstructing its source snapshot reproduced the embedded SHA-256 +exactly. With the targeted decoder, the original file passes the omitted physics +frame and both four-byte player-field boundaries, including the reported +`ReadTrace` offset, before reaching later state owned by the reporter's +unprovided HiDef mod GUI/assets. + +An independent current-source Windows x64 run created a fresh save on +`game/airdefense2`, reloaded it through `Game Map Init SaveGame`, captured an +engine-render-target screenshot in active gameplay, and exited normally. This +proves the deterministic current writer/reader round trip; completing the +third-party-mod fixture still requires reporter-side validation with the exact +mod installation. + ## Validation Evidence and Limits The focused regression set is: @@ -294,11 +329,12 @@ python tools/tests/savegame_corruption_contract.py ``` All three savegame contracts pass for the current tree. They cover the v3 -integrity/transaction model, total-file boundary arithmetic, pointer-width -safety, bounded and typed restore rules, raw-write inventory, and source parity; -static contracts are not runtime proof. Windows x64 client UI translation units -and both SP/MP GameLibs compile/link, while companion checks cover the ARM64 ABI -source contract and typed restored-object references. +integrity/transaction model, the exact v0.10 compatibility tuple and decoder, +source-declared class-frame ownership, total-file boundary arithmetic, +pointer-width safety, bounded and typed restore rules, raw-write inventory, and +source parity; static contracts are not runtime proof. Windows x64 client UI +translation units and both SP/MP GameLibs compile/link, while companion checks +cover the ARM64 ABI source contract and typed restored-object references. Current Windows x64 candidate runs on `game/airdefense1` record a fresh v3 save and `Game Map Init SaveGame`, rejection of a CRC-modified copy before map diff --git a/tools/tests/savegame_v3_contract.py b/tools/tests/savegame_v3_contract.py index 593eaf2e..d40dc86d 100644 --- a/tools/tests/savegame_v3_contract.py +++ b/tools/tests/savegame_v3_contract.py @@ -17,6 +17,12 @@ INTEGRITY_TRAILER_BYTES = 16 MENU_GUI_ASPECT = 640.0 / 480.0 MENU_PREVIEW_BOUNDS = (25.0, 78.0, 183.0, 137.0) +V3_PRE_PLAYER_LIQUID_FIELDS_SNAPSHOT = ( + 1, + "19351be39d2d4077a74294c0442707ef9565fc7a2fa9af9b81e05fc9aca8b220", + 404, + "windows-msvcabi-x64-le-raw1", +) def read(path: Path) -> str: @@ -274,8 +280,14 @@ def validate_source_contracts() -> None: engine_file_cpp = read(ROOT / "src/framework/File.cpp") sp = read(GAME_LIBS_ROOT / "src/game/gamesys/SaveGame.cpp") sp_h = read(GAME_LIBS_ROOT / "src/game/gamesys/SaveGame.h") + sp_class_h = read(GAME_LIBS_ROOT / "src/game/gamesys/Class.h") + sp_player = read(GAME_LIBS_ROOT / "src/game/Player.cpp") + sp_physics_player = read(GAME_LIBS_ROOT / "src/game/physics/Physics_Player.cpp") mp = read(GAME_LIBS_ROOT / "src/mpgame/gamesys/SaveGame.cpp") mp_h = read(GAME_LIBS_ROOT / "src/mpgame/gamesys/SaveGame.h") + mp_class_h = read(GAME_LIBS_ROOT / "src/mpgame/gamesys/Class.h") + mp_player = read(GAME_LIBS_ROOT / "src/mpgame/Player.cpp") + mp_physics_player = read(GAME_LIBS_ROOT / "src/mpgame/physics/Physics_Player.cpp") game_file_h = read(GAME_LIBS_ROOT / "src/framework/File.h") if constant(session, "SESSION_OPENQ4_SAVEGAME_COMPATIBILITY_VERSION") != 3: @@ -296,6 +308,66 @@ def validate_source_contracts() -> None: if any(wire_abi != "windows-msvcabi-x64-le-raw1" for _, _, _, wire_abi in engine_snapshots): raise AssertionError("Ambiguous unstamped v2 snapshots must stay restricted to their known wire ABI") + sp_pre_liquid_snapshots = snapshot_tuples( + sp, "OPENQ4_SAVEGAME_V3_PRE_PLAYER_LIQUID_FIELDS_SNAPSHOTS" + ) + mp_pre_liquid_snapshots = snapshot_tuples( + mp, "OPENQ4_SAVEGAME_V3_PRE_PLAYER_LIQUID_FIELDS_SNAPSHOTS" + ) + expected_pre_liquid_snapshots = [V3_PRE_PLAYER_LIQUID_FIELDS_SNAPSHOT] + if sp_pre_liquid_snapshots != expected_pre_liquid_snapshots or mp_pre_liquid_snapshots != expected_pre_liquid_snapshots: + raise AssertionError("SP/MP v0.10 player-liquid compatibility snapshots differ from the approved tuple") + + for source, header, class_header, player, physics_player, context in ( + (sp, sp_h, sp_class_h, sp_player, sp_physics_player, "SP GameLib"), + (mp, mp_h, mp_class_h, mp_player, mp_physics_player, "MP GameLib"), + ): + for token in ( + "SaveGame_IsV3PrePlayerLiquidFieldsSnapshot", + "bool idRestoreGame::HasOpenQ4PlayerLiquidSaveFields", + "HasNextSerializedEmptyClassFrame", + 'idStr::Icmp( cls->classname, "idPhysics" ) == 0', + "!cls->saveDeclaredHere", + "!cls->restoreDeclaredHere", + ): + require(source, token, f"{context} v0.10 compatibility decoder") + require(header, "HasOpenQ4PlayerLiquidSaveFields( void ) const", f"{context} compatibility accessor") + for token in ( + "struct idMemberFunctionOwner", + "struct idMemberFunctionDeclaredHere", + "decltype( &nameofclass::Save )", + "decltype( &nameofclass::Restore )", + "saveDeclaredHere", + "restoreDeclaredHere", + ): + require(class_header, token, f"{context} source-declared class-frame ownership") + for obsolete_comparison in ( + "cls->super->Save == cls->Save", + "cls->super->Restore == cls->Restore", + ): + if obsolete_comparison in source: + raise AssertionError(f"{context} still derives save frames from linker-foldable function addresses") + if re.search( + r"ReadInt\s*\(\s*previousWaterType\s*\)\s*;\s*" + r"nextLiquidSurfaceSoundTime\s*=\s*0\s*;\s*" + r"if\s*\(\s*savefile->HasOpenQ4PlayerLiquidSaveFields\s*\(\s*\)\s*\)\s*\{\s*" + r"savefile->ReadInt\s*\(\s*nextLiquidSurfaceSoundTime\s*\)\s*;\s*\}\s*" + r"savefile->ReadInt\s*\(\s*nextLiquidDamageTime\s*\)", + player, + re.DOTALL, + ) is None: + raise AssertionError(f"{context} does not restore the pre-v0.10 liquid-sound timer layout") + if re.search( + r"ReadFloat\s*\(\s*playerSpeed\s*\)\s*;\s*" + r"swimSpeed\s*=\s*0\.0f\s*;\s*" + r"if\s*\(\s*savefile->HasOpenQ4PlayerLiquidSaveFields\s*\(\s*\)\s*\)\s*\{\s*" + r"savefile->ReadFloat\s*\(\s*swimSpeed\s*\)\s*;\s*\}\s*" + r"savefile->ReadVec3\s*\(\s*viewForward\s*\)", + physics_player, + re.DOTALL, + ) is None: + raise AssertionError(f"{context} does not restore the pre-v0.10 swim-speed layout") + require(session, 'SESSION_LEGACY_SAVEGAME_WIRE_ABI = "windows-msvcabi-x64-le-raw1"', "engine unstamped legacy ABI restriction") require(session, "Session_GetSaveGameWireABI(), SESSION_LEGACY_SAVEGAME_WIRE_ABI",