From 42e765882593f7282a82452d122e4ceeeb9fb669 Mon Sep 17 00:00:00 2001 From: themuffinator Date: Wed, 19 Aug 2026 20:42:56 +0100 Subject: [PATCH 01/35] Implement idTech 5 modernization foundation Document the complete compatibility-safe roadmap and dated implementation state; land the P0 security, provenance, stock-evidence, PBR authoring/resource, packaging, renderer-readback, and multiplayer validation foundations. Companion game-code commit: 834ed9f28817e9d01355ea3aeecb0f144e1ce69b. --- .github/workflows/commit-validation.yml | 13 + .github/workflows/push-verification.yml | 13 + .vscode/launch.json | 147 + AGENTS.md | 1 + LICENSES/DOOM-3-ADDITIONAL-TERMS.txt | 19 + LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt | 58 + README.md | 5 +- content/baseoq4/pak0/default.cfg | 10 +- .../pak0/guis/menu/settings/controls.gui | 6 +- docs/dev/engine-capability-matrix.md | 80 + docs/dev/idtech5-modernization-roadmap.md | 393 +++ docs/dev/official-pk4-checksums.md | 10 +- .../plans/2026-05-19-pbr-material-support.md | 57 +- .../rbdoom3-bfg-parity-modernization-plan.md | 3 + docs/dev/release-completion.md | 19 +- docs/dev/releases/v0.12.0.md | 16 + docs/dev/renderer-validation-matrix.md | 4 + docs/dev/settings-menu-registry.json | 6 +- docs/dev/source-provenance-manifest.json | 45 + docs/dev/source-provenance.md | 49 + docs/dev/stock-asset-baseline.md | 159 ++ docs/user/server-security.md | 172 ++ docs/user/server-setup.md | 3 + meson.build | 6 +- src/framework/CVarSystem.cpp | 159 +- src/framework/CVarSystem.h | 12 + src/framework/CmdSystem.cpp | 33 +- src/framework/Common.cpp | 10 + src/framework/Console.cpp | 42 +- src/framework/EditField.cpp | 24 +- src/framework/EventLoop.cpp | 33 +- src/framework/FileSystem.cpp | 402 ++- src/framework/FileSystem.h | 18 +- src/framework/GameDirPolicy.h | 108 + src/framework/RemoteCVarPolicy.h | 35 + src/framework/Session.cpp | 63 +- src/framework/async/AsyncClient.cpp | 655 +++-- src/framework/async/AsyncClient.h | 35 +- src/framework/async/AsyncNetwork.cpp | 116 +- src/framework/async/AsyncNetwork.h | 7 +- src/framework/async/AsyncServer.cpp | 849 +++++- src/framework/async/AsyncServer.h | 55 + src/framework/async/MultiViewDemo.cpp | 4 +- src/framework/async/Rcon2Protocol.cpp | 69 + src/framework/async/Rcon2Protocol.h | 51 + src/framework/licensee.h | 1 + src/idlib/BitMsg.cpp | 287 +- src/idlib/BitMsg.h | 33 +- src/idlib/CmdArgs.cpp | 41 +- src/idlib/CmdArgs.h | 2 + src/idlib/CryptoHash.cpp | 320 +++ src/idlib/CryptoHash.h | 47 + src/idlib/PrivateCommand.h | 76 + src/idlib/Str.h | 13 + src/renderer/Image.h | 15 + src/renderer/ImageManager.cpp | 71 +- src/renderer/Image_load.cpp | 33 +- src/renderer/Material.cpp | 995 ++++++- src/renderer/Material.h | 65 + src/renderer/MaterialResourceTable.cpp | 649 ++++- src/renderer/MaterialResourceTable.h | 79 +- src/renderer/ModernGLDrawPlan.cpp | 5 + src/renderer/ModernGLExecutor.cpp | 17 +- src/renderer/ModernGLSubmitPlan.cpp | 8 +- src/renderer/RenderModuleAPI.h | 8 +- src/renderer/RenderSystem.cpp | 145 +- src/renderer/RenderSystem.h | 4 + src/renderer/RenderSystem_init.cpp | 20 + src/renderer/ScenePackets.cpp | 96 +- src/renderer/ScenePackets.h | 20 + src/renderer/tr_local.h | 5 + src/sys/URLPolicy.h | 292 ++ src/sys/linux/main.cpp | 30 +- src/sys/osx/macosx_misc.mm | 71 +- src/sys/posix/posix_main.cpp | 36 +- src/sys/posix/posix_syscon.cpp | 16 +- src/sys/win32/win_main.cpp | 18 +- src/sys/win32/win_syscon.cpp | 18 +- tools/build/stage_direct_run_game_module.py | 10 +- tools/build/stage_fast_install.py | 68 +- tools/build/windows_runtime.py | 112 + tools/debug/renderdoc_capture.ps1 | 1 + tools/debug/start_listen_server_client.ps1 | 2 + tools/tests/competitive_match_layer.py | 12 + tools/tests/filesystem_write_qpath_safety.py | 14 + tools/tests/key_bind_presentation.py | 115 +- tools/tests/macos_sdl3_backend_guard.py | 23 +- tools/tests/macos_static_policy.py | 19 +- tools/tests/mp_bot_navigation.py | 23 + tools/tests/native/CoreSafetyTest.cpp | 285 ++ tools/tests/network_security.py | 897 ++++++ tools/tests/openq4_pure_pack.py | 390 ++- tools/tests/openurl_security.py | 236 ++ tools/tests/p0_governance_evidence.py | 173 ++ tools/tests/packaging_safety.py | 147 +- tools/tests/release_tooling_safety.py | 3 +- tools/tests/renderer_gameplay_benchmark.py | 2 + tools/tests/renderer_mp_flat_items.py | 2 +- tools/tests/renderer_pbr_materials.py | 389 +++ tools/tests/renderer_screenshot_readback.py | 170 ++ tools/tests/renderer_validation_matrix.py | 39 +- tools/tests/settings_menu_coverage.py | 58 +- tools/tests/stock_asset_baseline.py | 1263 +++++++++ tools/tests/vscode_fast_build.py | 80 + tools/validation/audit_source_provenance.py | 329 +++ tools/validation/openq4_validate.py | 5 + tools/validation/stock_asset_baseline.py | 2493 +++++++++++++++++ 107 files changed, 14150 insertions(+), 820 deletions(-) create mode 100644 LICENSES/DOOM-3-ADDITIONAL-TERMS.txt create mode 100644 LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt create mode 100644 docs/dev/engine-capability-matrix.md create mode 100644 docs/dev/idtech5-modernization-roadmap.md create mode 100644 docs/dev/source-provenance-manifest.json create mode 100644 docs/dev/source-provenance.md create mode 100644 docs/dev/stock-asset-baseline.md create mode 100644 docs/user/server-security.md create mode 100644 src/framework/GameDirPolicy.h create mode 100644 src/framework/RemoteCVarPolicy.h create mode 100644 src/framework/async/Rcon2Protocol.cpp create mode 100644 src/framework/async/Rcon2Protocol.h create mode 100644 src/idlib/CryptoHash.cpp create mode 100644 src/idlib/CryptoHash.h create mode 100644 src/idlib/PrivateCommand.h create mode 100644 src/sys/URLPolicy.h create mode 100644 tools/tests/network_security.py create mode 100644 tools/tests/openurl_security.py create mode 100644 tools/tests/p0_governance_evidence.py create mode 100644 tools/tests/renderer_pbr_materials.py create mode 100644 tools/tests/stock_asset_baseline.py create mode 100644 tools/validation/audit_source_provenance.py create mode 100644 tools/validation/stock_asset_baseline.py diff --git a/.github/workflows/commit-validation.yml b/.github/workflows/commit-validation.yml index a6699de2..aa1d1e21 100644 --- a/.github/workflows/commit-validation.yml +++ b/.github/workflows/commit-validation.yml @@ -62,7 +62,9 @@ jobs: tools/build/write_pak_manifest.py \ src/sys/linux/pk4/id_utils.py \ tools/analysis/clang_tidy_input_safety.py \ + tools/validation/audit_source_provenance.py \ tools/validation/openq4_validate.py \ + tools/validation/stock_asset_baseline.py \ tools/tests/renderer_validation_matrix.py \ tools/tests/arena_campaign.py \ tools/tests/async_drop_client_contract.py \ @@ -157,7 +159,10 @@ jobs: tools/tests/native_glx_shutdown.py \ tools/tests/network_ipv4_support.py \ tools/tests/network_ipv6_support.py \ + tools/tests/network_security.py \ + tools/tests/openurl_security.py \ tools/tests/openq4_pure_pack.py \ + tools/tests/p0_governance_evidence.py \ tools/tests/packaging_safety.py \ tools/tests/preprocessor_macro_safety.py \ tools/tests/posix_memory_management.py \ @@ -167,6 +172,7 @@ jobs: tools/tests/release_tooling_safety.py \ tools/tests/renderer_cel_shading.py \ tools/tests/renderer_mp_flat_items.py \ + tools/tests/renderer_pbr_materials.py \ tools/tests/renderer_msaa_cvar_safety.py \ tools/tests/renderer_picmip_policy.py \ tools/tests/renderer_player_visibility.py \ @@ -186,6 +192,7 @@ jobs: tools/tests/steam_deck_support.py \ tools/tests/system_console_presentation.py \ tools/tests/startup_language_override.py \ + tools/tests/stock_asset_baseline.py \ tools/tests/ui_embedded_icons.py \ tools/tests/validation_hardening.py \ tools/tests/vk_shader_header_pin.py \ @@ -195,6 +202,7 @@ jobs: bash -n tools/validation/validate_macos_static.sh bash -n tools/validation/validate_push.sh bash -n tools/validation/validate_pr.sh + python tools/validation/audit_source_provenance.py --check python tools/tests/arena_campaign.py python tools/tests/async_drop_client_contract.py python tools/tests/campaign_split_state_transition.py @@ -284,7 +292,10 @@ jobs: python tools/tests/native_glx_shutdown.py python tools/tests/network_ipv4_support.py python tools/tests/network_ipv6_support.py + python tools/tests/network_security.py + python tools/tests/openurl_security.py python tools/tests/openq4_pure_pack.py + python tools/tests/p0_governance_evidence.py python tools/tests/packaging_safety.py python tools/tests/preprocessor_macro_safety.py python tools/tests/release_tooling_safety.py @@ -294,6 +305,7 @@ jobs: python tools/tests/posix_thread_shutdown.py python tools/tests/renderer_cel_shading.py python tools/tests/renderer_mp_flat_items.py + python tools/tests/renderer_pbr_materials.py python tools/tests/renderer_msaa_cvar_safety.py python tools/tests/renderer_picmip_policy.py python tools/tests/renderer_player_visibility.py @@ -313,6 +325,7 @@ jobs: python tools/tests/steam_deck_support.py python tools/tests/system_console_presentation.py python tools/tests/startup_language_override.py + python tools/tests/stock_asset_baseline.py python tools/tests/ui_embedded_icons.py python tools/tests/validation_hardening.py python tools/tests/vk_shader_header_pin.py diff --git a/.github/workflows/push-verification.yml b/.github/workflows/push-verification.yml index 5d60d2a7..bb969efc 100644 --- a/.github/workflows/push-verification.yml +++ b/.github/workflows/push-verification.yml @@ -62,7 +62,9 @@ jobs: tools/build/write_pak_manifest.py \ src/sys/linux/pk4/id_utils.py \ tools/analysis/clang_tidy_input_safety.py \ + tools/validation/audit_source_provenance.py \ tools/validation/openq4_validate.py \ + tools/validation/stock_asset_baseline.py \ tools/tests/renderer_validation_matrix.py \ tools/tests/arena_campaign.py \ tools/tests/async_drop_client_contract.py \ @@ -157,7 +159,10 @@ jobs: tools/tests/native_glx_shutdown.py \ tools/tests/network_ipv4_support.py \ tools/tests/network_ipv6_support.py \ + tools/tests/network_security.py \ + tools/tests/openurl_security.py \ tools/tests/openq4_pure_pack.py \ + tools/tests/p0_governance_evidence.py \ tools/tests/packaging_safety.py \ tools/tests/preprocessor_macro_safety.py \ tools/tests/posix_memory_management.py \ @@ -167,6 +172,7 @@ jobs: tools/tests/release_tooling_safety.py \ tools/tests/renderer_cel_shading.py \ tools/tests/renderer_mp_flat_items.py \ + tools/tests/renderer_pbr_materials.py \ tools/tests/renderer_msaa_cvar_safety.py \ tools/tests/renderer_picmip_policy.py \ tools/tests/renderer_player_visibility.py \ @@ -186,6 +192,7 @@ jobs: tools/tests/steam_deck_support.py \ tools/tests/system_console_presentation.py \ tools/tests/startup_language_override.py \ + tools/tests/stock_asset_baseline.py \ tools/tests/ui_embedded_icons.py \ tools/tests/validation_hardening.py \ tools/tests/vk_shader_header_pin.py \ @@ -195,6 +202,7 @@ jobs: bash -n tools/validation/validate_macos_static.sh bash -n tools/validation/validate_push.sh bash -n tools/validation/validate_pr.sh + python tools/validation/audit_source_provenance.py --check python tools/tests/arena_campaign.py python tools/tests/async_drop_client_contract.py python tools/tests/campaign_split_state_transition.py @@ -284,7 +292,10 @@ jobs: python tools/tests/native_glx_shutdown.py python tools/tests/network_ipv4_support.py python tools/tests/network_ipv6_support.py + python tools/tests/network_security.py + python tools/tests/openurl_security.py python tools/tests/openq4_pure_pack.py + python tools/tests/p0_governance_evidence.py python tools/tests/packaging_safety.py python tools/tests/preprocessor_macro_safety.py python tools/tests/release_tooling_safety.py @@ -294,6 +305,7 @@ jobs: python tools/tests/posix_thread_shutdown.py python tools/tests/renderer_cel_shading.py python tools/tests/renderer_mp_flat_items.py + python tools/tests/renderer_pbr_materials.py python tools/tests/renderer_msaa_cvar_safety.py python tools/tests/renderer_picmip_policy.py python tools/tests/renderer_player_visibility.py @@ -313,6 +325,7 @@ jobs: python tools/tests/steam_deck_support.py python tools/tests/system_console_presentation.py python tools/tests/startup_language_override.py + python tools/tests/stock_asset_baseline.py python tools/tests/ui_embedded_icons.py python tools/tests/validation_hardening.py python tools/tests/vk_shader_header_pin.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 751a77c1..45f3135d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -58,6 +58,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm1" ], @@ -121,6 +124,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm2" ], @@ -184,6 +190,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm3" ], @@ -247,6 +256,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm4" ], @@ -310,6 +322,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm5" ], @@ -373,6 +388,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm6" ], @@ -436,6 +454,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm7" ], @@ -499,6 +520,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm8" ], @@ -562,6 +586,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm9" ], @@ -625,6 +652,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm10" ], @@ -688,6 +718,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm11" ], @@ -751,6 +784,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp1" ], @@ -814,6 +850,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp2" ], @@ -877,6 +916,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp3" ], @@ -940,6 +982,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp4" ], @@ -1003,6 +1048,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp5" ], @@ -1066,6 +1114,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp6" ], @@ -1129,6 +1180,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp7" ], @@ -1192,6 +1246,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp8" ], @@ -1255,6 +1312,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp9" ], @@ -1318,6 +1378,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp10" ], @@ -1381,6 +1444,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp11" ], @@ -1444,6 +1510,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp12" ], @@ -1507,6 +1576,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp13" ], @@ -1570,6 +1642,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp14" ], @@ -1633,6 +1708,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4cmp15" ], @@ -1696,6 +1774,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xdm10" ], @@ -1759,6 +1840,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xdm11" ], @@ -1822,6 +1906,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xdm13" ], @@ -1885,6 +1972,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xdm14" ], @@ -1948,6 +2038,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xdm15" ], @@ -2011,6 +2104,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dm11v1" ], @@ -2074,6 +2170,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4tourney1" ], @@ -2137,6 +2236,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xtourney1" ], @@ -2200,6 +2302,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xtourney2" ], @@ -2263,6 +2368,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf1" ], @@ -2326,6 +2434,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf2" ], @@ -2389,6 +2500,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf3" ], @@ -2452,6 +2566,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf4" ], @@ -2515,6 +2632,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf5" ], @@ -2578,6 +2698,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf6" ], @@ -2641,6 +2764,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf7" ], @@ -2704,6 +2830,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4ctf8" ], @@ -2767,6 +2896,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4xctf6" ], @@ -2830,6 +2962,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dz1" ], @@ -2893,6 +3028,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dz2" ], @@ -2956,6 +3094,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dz3" ], @@ -3019,6 +3160,9 @@ "+set", "si_pure", "0", + "+set", + "ui_autoJoin", + "1", "+spawnServer", "mp/q4dz4" ], @@ -4966,6 +5110,9 @@ "si_gameType", "DM", "+set", + "ui_autoJoin", + "1", + "+set", "si_pure", "0" ], diff --git a/AGENTS.md b/AGENTS.md index 5d6bedbe..e2bb7e63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,6 +53,7 @@ This file describes project goals, rules, and upstream credits for anyone workin - Any existing custom `q4base/` content is treated as an expedient bootstrap, not a long-term solution. The goal is to remove this reliance by fixing engine compatibility issues rather than shipping replacement assets. - For investigations, reference the log file written by `logFileName` (VS Code launch uses `logs/openq4.log`), located under `fs_savepath\\` (e.g. `${workspaceFolder}\\.home\\baseoq4\\logs\\openq4.log`). - For runtime validation, use mode-specific launch tasks: use the SP launch task for single-player testing and the MP launch task for multiplayer testing. +- Keep `ui_autoJoin 1` enabled for multiplayer testing so clients enter gameplay automatically. Use an explicit `+set ui_autoJoin 0` only when the test specifically targets the join menu or initial spectator/join flow; do not rely on omission because the setting is archived. - Do not treat main-menu startup as sufficient validation; enter in-game/map gameplay relevant to the change before concluding tests. - For macOS testing/debugging, use the compliant Apple-hardware VM/host workflow in `docs/dev/macos-vm-testing-workflow.md` and `tools/macos/Invoke-openQ4MacOSWorkflow.ps1`. Keep macOS installer/restore-image inventory under `E:\ISO\macos\`. Do not use Windows VMware macOS unlocker/Hackintosh-style setup guides for openQ4 automation. - Use `.tmp/` directory in repository for any temporary files required for tasks. diff --git a/LICENSES/DOOM-3-ADDITIONAL-TERMS.txt b/LICENSES/DOOM-3-ADDITIONAL-TERMS.txt new file mode 100644 index 00000000..e0cb3835 --- /dev/null +++ b/LICENSES/DOOM-3-ADDITIONAL-TERMS.txt @@ -0,0 +1,19 @@ +ADDITIONAL TERMS APPLICABLE TO THE DOOM 3 GPL SOURCE CODE. + + The following additional terms (“Additional Terms”) supplement and modify the GNU General Public License, Version 3 (“GPL”) applicable to the Doom 3 GPL Source Code (“Doom 3 Source Code”). In addition to the terms and conditions of the GPL, the Doom 3 Source Code is subject to the further restrictions below. + +1. Replacement of Section 15. Section 15 of the GPL shall be deleted in its entirety and replaced with the following: + +“15. Disclaimer of Warranty. + +THE PROGRAM IS PROVIDED WITHOUT ANY WARRANTIES, WHETHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, TITLE AND MERCHANTABILITY. THE PROGRAM IS BEING DELIVERED OR MADE AVAILABLE “AS IS”, “WITH ALL FAULTS” AND WITHOUT WARRANTY OR REPRESENTATION. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.” + +2. Replacement of Section 16. Section 16 of the GPL shall be deleted in its entirety and replaced with the following: + +“16. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES SHALL ANY COPYRIGHT HOLDER OR ITS AFFILIATES, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, FOR ANY DAMAGES OR OTHER LIABILITY, INCLUDING ANY GENERAL, DIRECT, INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL OR PUNITIVE DAMAGES ARISING FROM, OUT OF OR IN CONNECTION WITH THE USE OR INABILITY TO USE THE PROGRAM OR OTHER DEALINGS WITH THE PROGRAM(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), WHETHER OR NOT ANY COPYRIGHT HOLDER OR SUCH OTHER PARTY RECEIVES NOTICE OF ANY SUCH DAMAGES AND WHETHER OR NOT SUCH DAMAGES COULD HAVE BEEN FORESEEN.” + +3. LEGAL NOTICES; NO TRADEMARK LICENSE; ORIGIN. You must reproduce faithfully all trademark, copyright and other proprietary and legal notices on any copies of the Program or any other required author attributions. This license does not grant you rights to use any copyright holder or any other party’s name, logo, or trademarks. Neither the name of the copyright holder or its affiliates, or any other party who modifies and/or conveys the Program may be used to endorse or promote products derived from this software without specific prior written permission. The origin of the Program must not be misrepresented; you must not claim that you wrote the original Program. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original Program. + +4. INDEMNIFICATION. IF YOU CONVEY A COVERED WORK AND AGREE WITH ANY RECIPIENT OF THAT COVERED WORK THAT YOU WILL ASSUME ANY LIABILITY FOR THAT COVERED WORK, YOU HEREBY AGREE TO INDEMNIFY, DEFEND AND HOLD HARMLESS THE OTHER LICENSORS AND AUTHORS OF THAT COVERED WORK FOR ANY DAMAEGS, DEMANDS, CLAIMS, LOSSES, CAUSES OF ACTION, LAWSUITS, JUDGMENTS EXPENSES (INCLUDING WITHOUT LIMITATION REASONABLE ATTORNEYS' FEES AND EXPENSES) OR ANY OTHER LIABLITY ARISING FROM, RELATED TO OR IN CONNECTION WITH YOUR ASSUMPTIONS OF LIABILITY. diff --git a/LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt b/LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt new file mode 100644 index 00000000..4dd74945 --- /dev/null +++ b/LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt @@ -0,0 +1,58 @@ +ADDITIONAL TERMS APPLICABLE TO THE Doom 3 BFG Edition GPL Source Code. + + The following additional terms ("Additional Terms") supplement and modify + the GNU General Public License, Version 3 ("GPL") applicable to the Doom 3 + BFG Edition GPL Source Code ("Doom 3 BFG Edition Source Code"). In addition + to the terms and conditions of the GPL, the Doom 3 BFG Edition Source Code is + subject to the further restrictions below. + +1. Replacement of Section 15. Section 15 of the GPL shall be deleted in its +entirety and replaced with the following: + +"15. Disclaimer of Warranty. + +THE PROGRAM IS PROVIDED WITHOUT ANY WARRANTIES, WHETHER EXPRESSED OR IMPLIED, +INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR +PURPOSE, NON-INFRINGEMENT, TITLE AND MERCHANTABILITY. THE PROGRAM IS BEING +DELIVERED OR MADE AVAILABLE "AS IS", "WITH ALL FAULTS" AND WITHOUT WARRANTY OR +REPRESENTATION. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST +OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION." + +2. Replacement of Section 16. Section 16 of the GPL shall be deleted in its +entirety and replaced with the following: + +"16. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES SHALL ANY COPYRIGHT HOLDER OR ITS AFFILIATES, OR ANY +OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE +LIABLE TO YOU, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, FOR ANY +DAMAGES OR OTHER LIABILITY, INCLUDING ANY GENERAL, DIRECT, INDIRECT, SPECIAL, +INCIDENTAL, CONSEQUENTIAL OR PUNITIVE DAMAGES ARISING FROM, OUT OF OR IN +CONNECTION WITH THE USE OR INABILITY TO USE THE PROGRAM OR OTHER DEALINGS WITH +THE PROGRAM(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED +INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE +PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), WHETHER OR NOT ANY COPYRIGHT +HOLDER OR SUCH OTHER PARTY RECEIVES NOTICE OF ANY SUCH DAMAGES AND WHETHER +OR NOT SUCH DAMAGES COULD HAVE BEEN FORESEEN." + +3. LEGAL NOTICES; NO TRADEMARK LICENSE; ORIGIN. You must reproduce faithfully +all trademark, copyright and other proprietary and legal notices on any copies +of the Program or any other required author attributions. This license does +not grant you rights to use any copyright holder or any other party’s name, +logo, or trademarks. Neither the name of the copyright holder or its +affiliates, or any other party who modifies and/or conveys the Program may be +used to endorse or promote products derived from this software without +specific prior written permission. The origin of the Program must not be +misrepresented; you must not claim that you wrote the original Program. + +Altered source versions must be plainly marked as such, and must not be +misrepresented as being the original Program. + +4. INDEMNIFICATION. IF YOU CONVEY A COVERED WORK AND AGREE WITH ANY RECIPIENT +OF THAT COVERED WORK THAT YOU WILL ASSUME ANY LIABILITY FOR THAT COVERED WORK, +YOU HEREBY AGREE TO INDEMNIFY, DEFEND AND HOLD HARMLESS THE OTHER LICENSORS +AND AUTHORS OF THAT COVERED WORK FOR ANY DAMAEGS, DEMANDS, CLAIMS, LOSSES, +CAUSES OF ACTION, LAWSUITS, JUDGMENTS EXPENSES (INCLUDING WITHOUT LIMITATION +REASONABLE ATTORNEYS' FEES AND EXPENSES) OR ANY OTHER LIABLITY ARISING FROM, +RELATED TO OR IN CONNECTION WITH YOUR ASSUMPTIONS OF LIABILITY. diff --git a/README.md b/README.md index a69b558d..fad4d0ef 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,8 @@ Packaged support currently focuses on Windows, Linux x64, Steam Deck/SteamOS, pr - It is **not** a drop-in runtime for the original proprietary Quake 4 DLL mods. - The project is still in **beta development**, so compatibility work is ongoing. +Developers and testers should use the [engine capability matrix](docs/dev/engine-capability-matrix.md) for authoritative implemented/experimental/missing status, the [idTech 5-level modernization roadmap](docs/dev/idtech5-modernization-roadmap.md) for the compatibility-safe implementation order, and the [stock-asset baseline](docs/dev/stock-asset-baseline.md) for reproducible PK4, SP/MP, save/load, demo, log, and engine-screenshot evidence. + If you run into problems, please use the [issue tracker](https://github.com/themuffinator/openQ4/issues) and include crash logs or setup details when possible. For experimental macOS crashes, use the [macOS support-data guide](docs/user/macos-support-data.md) before filing or updating an issue. --- @@ -170,6 +172,7 @@ Bug reports, compatibility reports, testing feedback, and code contributions are - **DarkMatter Productions** - project stewardship and website - **Justin Marshall** - Quake4Doom and early BSE reverse engineering reference work - **Robert Beckebans** - renderer modernization reference work, including RBDOOM-3-BFG inspiration +- **id Software's official Doom 3 and Doom 3 BFG source releases** - retained idTech 4 source lineage; see the [audited provenance inventory](docs/dev/source-provenance.md) - **id Software** and **Raven Software** - Quake 4 and the underlying technology - **akacross** (Discord user) - Thorough playtesting on Linux and Windows, a huge help moving the project forward! @@ -177,7 +180,7 @@ Bug reports, compatibility reports, testing feedback, and code contributions are ## License and disclaimer -openQ4 engine code is licensed under the [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0). See [LICENSE](LICENSE) for details. +openQ4 engine code is licensed under the [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0). See [LICENSE](LICENSE) for details. Files retaining Doom 3 or Doom 3 BFG Edition headers also retain their upstream notices and are accompanied by the corresponding published Additional Terms; the [source-provenance inventory](docs/dev/source-provenance.md) records their scope, pinned audit references, and intermediate lineage without offering a legal conclusion. The game-library code in [openQ4-game](https://github.com/themuffinator/openQ4-game) is derived from the Quake 4 SDK and remains subject to id Software's SDK EULA. Quake 4 assets remain the property of id Software and ZeniMax Media. diff --git a/content/baseoq4/pak0/default.cfg b/content/baseoq4/pak0/default.cfg index 315cdef2..dbc3ca2a 100644 --- a/content/baseoq4/pak0/default.cfg +++ b/content/baseoq4/pak0/default.cfg @@ -119,12 +119,12 @@ bind b "buymenu" // // FUNCTION KEYS // -bind F1 voteyes // Vote yes -bind F2 voteno // Vote no -bind F3 ready // Ready to play +bind F1 _impulse28 // Vote yes +bind F2 _impulse29 // Vote no +bind F3 _impulse17 // Toggle ready to play bind F5 "savegame quick" // Quick save -bind F6 toggleteam // Toggle team -bind F7 spectate // Spectate +bind F6 _impulse20 // Toggle team +bind F7 _impulse22 // Spectate bind F9 "loadgame quick" // Quick load bind F12 screenshot // Take screenshot diff --git a/content/baseoq4/pak0/guis/menu/settings/controls.gui b/content/baseoq4/pak0/guis/menu/settings/controls.gui index f67a6345..16f8d7c9 100644 --- a/content/baseoq4/pak0/guis/menu/settings/controls.gui +++ b/content/baseoq4/pak0/guis/menu/settings/controls.gui @@ -1052,7 +1052,7 @@ rect 452,215,149,18 visible 1 forecolor 1,0.745,0.137,0.8 - bind voteyes + bind _impulse28 textscale 0.24 font "fonts/lowpixel" @@ -1066,7 +1066,7 @@ rect 452,239,149,18 visible 1 forecolor 1,0.745,0.137,0.8 - bind voteno + bind _impulse29 textscale 0.24 font "fonts/lowpixel" @@ -1094,7 +1094,7 @@ rect 452,287,149,17 visible 1 forecolor 1,0.745,0.137,0.8 - bind ready + bind _impulse17 textscale 0.24 font "fonts/lowpixel" diff --git a/docs/dev/engine-capability-matrix.md b/docs/dev/engine-capability-matrix.md new file mode 100644 index 00000000..dd842edf --- /dev/null +++ b/docs/dev/engine-capability-matrix.md @@ -0,0 +1,80 @@ +# Engine Capability Matrix + +This is the authoritative current-state index for openQ4 engine capability claims. It answers whether a capability exists in the tree as audited on 2026-08-19; proposals describe intent, and release notes describe historical changes, but neither overrides this matrix. Domain documents linked below own their detailed acceptance evidence. + +Status means: + +- **Implemented** — a functional path is integrated and has the named evidence. The scope/qualification column limits the claim; it never implies every platform or asset is certified. +- **Experimental** — meaningful code exists, but it is opt-in, incomplete, missing parity evidence, or not yet the supported default. +- **Missing** — there is no production implementation. A plan, cvar stub, shader prototype, or unused data structure does not count. + +Any change that moves a row between these states must update this file and its evidence in the same change. When documentation disagrees, treat the narrower claim here as current. + +## P0 security and provenance + +| Capability | Status | Scope and qualification | Evidence | +|---|---|---|---| +| Network-driven executable updater | **Implemented** (retired safely) | The legacy version-check wire exchange remains compatible, but the client sends no GUID identity, ignores all server-provided text/URL/MIME/action fields, contains no updater download or execute path, and can only open the compile-time project releases URL without forcing exit. Pure-server package negotiation is a separate, bounded compatibility path described below. | [`AsyncClient.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncClient.cpp), [`licensee.h`](https://github.com/themuffinator/openQ4/blob/master/src/framework/licensee.h), [`openq4_pure_pack.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/openq4_pure_pack.py) | +| Server-supplied package transport | **Implemented** (validated redirect path; direct transfer build-conditional) | Pure-server redirects and PK4 download entries accept only bounded `http://` or `https://` URLs whose authority contains a syntactically valid DNS name, IPv4 literal, or bracketed IPv6 literal. Standard Meson packages do not enable libcurl, so syntax-validated redirect prompts remain available but in-process direct PK4 transfer reports unavailable. In a separately integrated curl-enabled build, the generic downloader revalidates the URL, restricts libcurl to HTTP(S), disables redirects, and applies connect, stall, and whole-transfer limits; package size, checksum, and destination-path validation remain mandatory. This validates syntax and protocol, not DNS ownership or server trust. | [`AsyncClient.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncClient.cpp), [`FileSystem.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/FileSystem.cpp), [`BuildDefines.h`](https://github.com/themuffinator/openQ4/blob/master/src/framework/BuildDefines.h), [`URLPolicy.h`](https://github.com/themuffinator/openQ4/blob/master/src/sys/URLPolicy.h), [`openurl_security.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/openurl_security.py) | +| Pure multiplayer game-module boundary | **Implemented** (contained protocol 2.41 compatibility path) | Pure mode enforces the ordered asset-PK4 list while the legacy game-code field carries the stock 1.4.2 `game300.pk4` checksum only as a platform-independent token for a module already resolved from trusted local openQ4 package/module roots. Pure negotiation never selects, downloads, extracts, or restarts into executable code. The token is not a hash of the loaded module, cryptographic module-equality proof, or anti-cheat guarantee. | [Official PK4 checksums](official-pk4-checksums.md), [server security](../user/server-security.md), [`FileSystem.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/FileSystem.cpp), [`openq4_pure_pack.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/openq4_pure_pack.py) | +| Malformed network and snapshot input handling | **Implemented** (hardened legacy boundary) | Bit-message reads record underflow instead of silently continuing, bounded queue and user-command decoders reject incomplete payloads, and SP/MP snapshot readers validate covered entity, player, spectator, weapon, instance, projectile-owner, PVS, hit-scan, and game-state fields before use. Audited leaf readers stage decoded fields until their payload is valid. A late malformed top-level snapshot tears down the affected session before another game or presentation frame; the legacy entity lifecycle is not claimed to provide whole-snapshot rollback. This is targeted hardening of the audited paths, not a claim that every legacy parser is formally verified. | [`BitMsg.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/idlib/BitMsg.cpp), [`AsyncNetwork.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncNetwork.cpp), [`network_security.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/network_security.py), [`openQ4-game`](https://github.com/themuffinator/openQ4-game) snapshot readers | +| Connection challenge entropy | **Implemented** | Client IDs, server-instance identity, and connection challenges use the OS CSPRNG and fail closed when it is unavailable. A connection challenge is endpoint/client-bound, expires after 30 seconds, and is consumed on successful admission; the legacy pure handshake may validate the same transaction across connect/pure/connect messages before admission. | [`AsyncClient.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncClient.cpp), [`AsyncServer.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncServer.cpp), [`network_security.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/network_security.py), [`CoreSafetyTest.cpp`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/native/CoreSafetyTest.cpp) | +| Authenticated remote console (`rcon2`) | **Implemented** | Challenge/proof exchange uses CSPRNG nonces, endpoint binding, PBKDF2-HMAC-SHA-256 password verification, request binding, constant-time proof comparison, one-use expiry, and secret zeroing. It does not transmit the password. | [`Rcon2Protocol.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/Rcon2Protocol.cpp), [`CryptoHash.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/idlib/CryptoHash.cpp), [`AsyncServer.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncServer.cpp), [`network_security.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/network_security.py) | +| Rcon abuse limits and secret redaction | **Implemented** | Per-source and global OOB/rcon limits plus cooldowns bound unauthenticated work. Private assignments are suppressed before dispatch from in-game, Win32, SDL, and TTY echo/history; old persistent history is purged; `$ cvar` expansion, startup dumps, config persistence, generic cvar output, and console completion previews redact; private journal payloads are omitted; and consumed command buffers are cleared. The password is case-sensitive/private. | [`AsyncServer.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncServer.cpp), [`CVarSystem.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/CVarSystem.cpp), [`Console.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/Console.cpp), [`PrivateCommand.h`](https://github.com/themuffinator/openQ4/blob/master/src/idlib/PrivateCommand.h), [`network_security.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/network_security.py) | +| Legacy plaintext rcon | **Implemented** (contained compatibility path) | Insecure plaintext rcon is disabled on both client and server by default. It is available only through explicit `net_clientUseLegacyRcon 1` / `net_serverAllowLegacyRcon 1` opt-in and should be limited to trusted legacy environments. | [`AsyncNetwork.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncNetwork.cpp), [`AsyncClient.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncClient.cpp), [`AsyncServer.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncServer.cpp), [`network_security.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/network_security.py) | +| Doom 3 / Doom 3 BFG provenance inventory | **Implemented** | Both retained header families, their distinct Additional Terms, official pinned snapshots, and the six BFG-headered OpenAL files found through an intermediate RBDOOM lineage are inventoried without asserting a legal conclusion. | [Source provenance](source-provenance.md), [`audit_source_provenance.py`](https://github.com/themuffinator/openQ4/blob/master/tools/validation/audit_source_provenance.py) | +| Reproducible retail-PK4 SP/MP compatibility evidence | **Implemented** | Offline retail/overlay/runtime identity, windowed SP save/restore, demo record/playback, pure MP listen/client, logs, and engine screenshots have one non-interactive harness. It binds the approved-manifest file and current Git provenance, rejects dry-run/failure-bearing reports, reconstructs each exact role launch, and inventories every packaged-overlay path that supersedes a retail virtual path instead of calling the run stock-only. MP roles explicitly use archived `ui_autoJoin 1` and prove an active, non-spectating player with a visible HUD; only a join-menu/initial-spectator test may explicitly set `ui_autoJoin 0`. A human visual/gameplay review and final-package/platform evidence remain required for promotion. | [Retail-PK4 compatibility baseline](stock-asset-baseline.md), [`stock_asset_baseline.py`](https://github.com/themuffinator/openQ4/blob/master/tools/validation/stock_asset_baseline.py) | + +## Quake 4 compatibility and engine foundation + +| Capability | Status | Scope and qualification | Evidence | +|---|---|---|---| +| Retail PK4 filesystem and idTech 4 asset/decl loaders | **Implemented** | Stock Quake 4 PK4 search, decls, materials, maps, MD5 models/animations, images, sounds, GUIs, cinematics, and scripts are the compatibility foundation. Full content behavior is still qualified scene-by-scene rather than claimed universally. | [`FileSystem.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/FileSystem.cpp), [`DeclManager.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/DeclManager.cpp), [renderer validation](renderer-validation-matrix.md) | +| Unified open-source SP and MP game modules | **Implemented** | `game_sp` and `game_mp` are built from the companion `openQ4-game` source and consumed under the unified `baseoq4/` runtime. Proprietary retail DLL compatibility is intentionally out of scope. | [`meson.build`](https://github.com/themuffinator/openQ4/blob/master/meson.build), [`meson_sources.py`](https://github.com/themuffinator/openQ4/blob/master/tools/build/meson_sources.py) | +| BSE effects runtime | **Implemented** | BSE is first-party in-tree client code and supports stock Quake 4 effect declarations; dedicated builds retain the disabled manager path. | [`BSE_Manager.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/bse/BSE_Manager.cpp), [BSE research/validation](quake4-bse-research-and-implementation-plan.md) | +| Save-game write/read compatibility | **Implemented** | Versioned openQ4 saves, integrity checking, recovery, legacy read support, entity-filter state, and save/load tests are present. Cross-version compatibility is governed explicitly rather than assumed. | [Save compatibility policy](savegame-compatibility-policy.md), [`savegame_v3_contract.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/savegame_v3_contract.py) | +| Render demos and multiview demos | **Implemented** | Render-demo record/playback and server multiview recording/playback exist with versioned compatibility checks. | [`Session.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/Session.cpp), [`MultiViewDemo.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/MultiViewDemo.cpp), [`demo_playback.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/demo_playback.py), [`multiview_demo.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/multiview_demo.py) | +| Fixed 60 Hz simulation with high-refresh presentation | **Implemented** | Authoritative simulation remains 60 Hz while presentation/interpolation and frame-pacing diagnostics support higher display rates. This does not change save/demo/network cadence. | [`Session.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/Session.cpp), [`renderer_gameplay_benchmark.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_gameplay_benchmark.py) | +| Dedicated server | **Implemented** | Headless dedicated builds avoid client renderer/BSE presentation and have stock-map smoke coverage. | [`dedicated.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/sys/linux/dedicated.cpp), [`linux_dedicated_stock_map_smoke.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/linux_dedicated_stock_map_smoke.py) | +| Background job system for general engine work | **Missing** | There is no portable idTech 5-style dependency/job-list substrate for renderer, animation, streaming, or archive work. Existing local worker uses do not provide that contract. | [`tr_local.h`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/tr_local.h), [idTech 5-level roadmap](idtech5-modernization-roadmap.md) | +| Pipelined async asset streaming and learned preload manifests | **Missing** | Level image loading and most PK4/decode/upload work remain synchronous; cache features do not yet form a cancellable read→decompress→decode→upload pipeline. | [`ImageManager.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ImageManager.cpp), [level-load cache](../user/level-load-cache.md), [idTech 5-level roadmap](idtech5-modernization-roadmap.md) | + +## Rendering + +| Capability | Status | Scope and qualification | Evidence | +|---|---|---|---| +| Classic ARB2 interaction renderer | **Implemented** | This is the supported/default visible-lighting path and compatibility rollback for stock assets. | [`draw_arb2.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_arb2.cpp), [renderer validation](renderer-validation-matrix.md) | +| Modern OpenGL scene packets/resources/render graph | **Experimental** | Scene packets, material/geometry tables, render-graph resources, upload rings, and a modern executor exist. They are opt-in infrastructure, not proof that a complete stock frame is modern-owned. | [`ScenePackets.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ScenePackets.cpp), [`MaterialResourceTable.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/MaterialResourceTable.cpp), [`GeometryResources.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/GeometryResources.cpp), [`RenderGraph.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/RenderGraph.cpp), [`ModernGLExecutor.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernGLExecutor.cpp) | +| Modern visible lighting ownership | **Experimental** | Interaction capability coverage exists, but exact math parity is not proven; ambient, fog, blend, stage-condition/color, and deform coverage still block production ownership. The current proven-domain count is zero, so ARB2 owns lit stock frames. | [Modern visible-lighting ownership](plans/2026-08-16-modern-visible-lighting-ownership.md), [renderer validation](renderer-validation-matrix.md) | +| GPU-driven GL submission / clustered Forward+ | **Experimental** | SSBO/compute/MDI, clustered-lighting, Hi-Z, and persistent/DSA paths exist at capable tiers, but remain opt-in and depend on the incomplete modern-visible path. | [`ModernClusteredLighting.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernClusteredLighting.cpp), [`ModernGLSubmitPlan.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernGLSubmitPlan.cpp), [`ModernGLExecutor.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernGLExecutor.cpp) | +| Vulkan renderer | **Experimental** | Broad stock material, interaction, decal, GUI, MD5R, and shadow support exists and can reach gameplay. It remains non-default while cubemap render targets, soft particles, debug tooling, SMP, custom programs, and other long-tail parity gaps remain. | [`vk_Backend.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Vulkan/vk_Backend.cpp), [`renderer_vulkan_world_interaction_compatibility.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_vulkan_world_interaction_compatibility.py), [`renderer_vulkan_shadow_compatibility.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_vulkan_shadow_compatibility.py) | +| Shadow maps | **Experimental** | Projected/point maps, CSM, cutout handling, caching, debug views, and stencil fallback exist, but `r_useShadowMap` remains opt-in/default-off pending complete promotion evidence. | [Shadow mapping](../user/shadow-mapping.md), [`Interaction.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Interaction.cpp), [`renderer_gameplay_benchmark.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_gameplay_benchmark.py) | +| Baked light grids | **Experimental** | Bake, packed atlas, visibility/distance moments, portal-aware sampling, streaming controls, and worker-assisted baking exist. They require generated per-map data and are not a stock-asset default. | [Light grids](../user/light-grids.md), [`RenderWorld_lightgrid.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/RenderWorld_lightgrid.cpp), [`draw_common.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_common.cpp) | +| SMAA post-process anti-aliasing | **Implemented** | Supported post-AA path for current renderers; it remains the compatibility/low-cost choice for future temporal work. | [`draw_common.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_common.cpp), [`material_smaa_edge.frag`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Vulkan/shaders/material_smaa_edge.frag) | +| Internal HDR scene/post chain and bloom | **Experimental** | Floating-point scene/post, exposure, tone mapping, bloom, and color controls exist, but modern-visible handoff and complete parity qualification constrain the path. This is not true HDR display output. | [`draw_common.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_common.cpp), [display settings](../user/display-settings.md) | +| True HDR display output (scRGB/HDR10) | **Missing** | Swapchain/window colorspace negotiation, paper-white UI composition, HDR screenshots, and platform qualification are not implemented. | [`VulkanDevice.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Vulkan/VulkanDevice.cpp), [display settings](../user/display-settings.md) | +| GPU skeletal skinning | **Missing** | Scene packets can classify a GPU-palette request, but modern draw planning rejects it and CPU deformation remains authoritative. | [`ScenePackets.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ScenePackets.cpp), [`ModernGLDrawPlan.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernGLDrawPlan.cpp) | +| Temporal AA / temporal upscaling | **Missing** | No production TAA/TAAU history, complete rigid/skinned/viewmodel/subview motion vectors, reactive masks, or disocclusion handling exists. | [`RenderSystem_init.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/RenderSystem_init.cpp), [PBR/material plan](plans/2026-05-19-pbr-material-support.md) | +| Automatic dynamic resolution | **Missing** | `r_screenFraction` and benchmark controls do not form a frame-time feedback controller. | [`RenderSystem_init.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/RenderSystem_init.cpp), [`draw_common.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_common.cpp) | +| Namespaced PBR materials and IBL/specular probes | **Missing** | A compatibility-safe Phase 0-3 foundation exists: opt-in namespaced material parsing, typed PBR image usage and lifecycle handling, classic ARB2 fallback generation, scene-packet metadata, resource-table semantics/metrics, and fail-closed exclusion from current visible-modern paths. It is infrastructure rather than a production PBR renderer: there are no PBR G-buffer shaders, direct PBR lighting, visible PBR ownership, IBL, or specular environment probes, and `gfxInfo` reports `modernLighting=0`. | [PBR material plan](plans/2026-05-19-pbr-material-support.md), [`Material.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Material.cpp), [`ScenePackets.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ScenePackets.cpp), [`MaterialResourceTable.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/MaterialResourceTable.cpp), [`renderer_pbr_materials.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_pbr_materials.py) | +| Clustered decals and reflection probes | **Missing** | Existing clustered lights and legacy projected decals are not a shared light/decal/probe cluster system. | [`ModernClusteredLighting.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernClusteredLighting.cpp), [`tr_deform.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/tr_deform.cpp) | +| Froxel volumetrics, SSR, and SSGI | **Missing** | No production froxel fog/lighting or screen-space reflection/GI pipeline exists. Classic fog/blend and SSAO-like post effects are separate capabilities. | [`draw_common.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_common.cpp), [renderer validation](renderer-validation-matrix.md) | +| Backend-neutral material/pass IR | **Missing** | GL and Vulkan still duplicate significant stage/program interpretation, and ModernGL-specific components are excluded from the Vulkan module build. | [`meson_sources.py`](https://github.com/themuffinator/openQ4/blob/master/tools/build/meson_sources.py), [`ModernGLExecutor.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernGLExecutor.cpp), [`vk_GuiExecutor.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Vulkan/vk_GuiExecutor.cpp) | + +## Platform, audio, and networking + +| Capability | Status | Scope and qualification | Evidence | +|---|---|---|---| +| SDL3 platform foundation | **Implemented** | Shared SDL3 window/display/input infrastructure is integrated with platform-specific bridges where required. Platform qualification is tracked separately below. | [`sdl3_backend.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/sys/sdl3/sdl3_backend.cpp), [SDL3 migration](sdl3-linux-macos-migration.md) | +| Windows x64 client/server | **Implemented** | Primary build/package target with staged client, dedicated server, renderer modules, and game modules. | [`meson_setup.ps1`](https://github.com/themuffinator/openQ4/blob/master/tools/build/meson_setup.ps1), [`platform-support.md`](platform-support.md) | +| Linux x64 client/server | **Implemented** | Native builds, Wayland/X11 paths, packaging, and physical-host stock SP/dedicated evidence exist. | [`platform-support.md`](platform-support.md), [`linux_wayland_stock_sp_smoke.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/linux_wayland_stock_sp_smoke.py) | +| macOS client/server | **Experimental** | Build/package/VM workflows and renderer corridors exist, but hardware qualification and backend/audio consistency remain narrower than Windows/Linux. | [macOS workflow](macos-vm-testing-workflow.md), [`macos_matrix_policy.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/macos_matrix_policy.py) | +| ARM64 | **Experimental** | Linux ARM64 cross/native release evidence exists; it is not yet a universal platform support claim. | [`linux_arm64_release_evidence.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/linux_arm64_release_evidence.py), [`platform-support.md`](platform-support.md) | +| OpenAL spatial audio, streaming, HRTF, and EFX | **Implemented** | The OpenAL backend includes streaming voices, device recovery, HRTF controls, EFX routing, and diagnostics. Provider/feature availability remains platform/device-dependent. | [`AL_SoundHardware.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/sound/OpenAL/AL_SoundHardware.cpp), [`AL_SoundVoice.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/sound/OpenAL/AL_SoundVoice.cpp), [`macos_openal_provider_policy.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/macos_openal_provider_policy.py) | +| IPv4/IPv6 transport and LAN discovery | **Implemented** | Dual-stack UDP, IPv6 literals/zones, DNS, fragmentation policy, and IPv6 multicast LAN discovery exist with self-tests. | [Multiplayer networking](../user/multiplayer-networking.md), [`network_ipv4_support.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/network_ipv4_support.py), [`network_ipv6_support.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/network_ipv6_support.py) | +| Prediction and lag compensation | **Implemented** | Client prediction and opt-in server rewind/lag compensation exist; gameplay defaults and tuning remain game-mode policy. | [Multiplayer networking](../user/multiplayer-networking.md), [`AsyncNetwork.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/AsyncNetwork.cpp) | +| Voice chat | **Missing** | A disabled game-side capture/playback skeleton exists, but reliable transport, codec, user controls, moderation, and production validation do not. | [Multiplayer networking](../user/multiplayer-networking.md) | + +## Promotion rule + +“Implemented” is not shorthand for “release-qualified everywhere.” Renderer default promotion still requires the full evidence token defined by the [renderer validation matrix](renderer-validation-matrix.md), including clean warnings, visual/gameplay/RenderDoc/performance/presentation/rollback passes with debug features off. Retail-asset compatibility changes additionally require a passing [retail-PK4 compatibility baseline](stock-asset-baseline.md) plus the manual review gates recorded with that bundle. diff --git a/docs/dev/idtech5-modernization-roadmap.md b/docs/dev/idtech5-modernization-roadmap.md new file mode 100644 index 00000000..895e9383 --- /dev/null +++ b/docs/dev/idtech5-modernization-roadmap.md @@ -0,0 +1,393 @@ +# idTech 5-Level Modernization Roadmap + +This document turns the project's modernization goal into an implementation +order that preserves the shipped Quake 4 asset and gameplay contracts. It is +based on the official Doom 3 BFG Edition source snapshot (`1caba197`) pinned in +the [source-provenance inventory](source-provenance.md), the current openQ4 +[capability matrix](engine-capability-matrix.md), and the stock-asset acceptance +harness. + +Doom 3 BFG is not the full idTech 5 or idTech 6 source tree. It is a late +idTech 4 branch containing several idTech 5-era architectural ideas: a parallel +job manager, jobbed renderer front end, GPU skinning, explicit GPU buffers, +binary/generated resources, preload manifests, GPU timing, automatic resolution +scaling, a refined front-end/back-end render command stream, and a newer +session/network stack. Those subsystems are useful references, but importing the +whole engine would replace Quake 4 contracts rather than modernize them. + +## Non-negotiable compatibility boundary + +Modernization is acceptable only when all of these remain true: + +- Retail Quake 4 PK4s remain the source of truth. Players are not required to + convert, unpack, or patch them. +- Classic material, decl, GUI, map, MD5/MD5R, BSE, sound, save, demo, and + protocol behavior remains available as the fallback and comparison path. +- Generated data is a disposable, versioned cache under `fs_savepath`, keyed to + its source identity (including the containing PK4 checksum where relevant). + Missing or invalid cache data falls back to the retail source. A cache never + becomes downloadable content or part of the pure-package authority set. +- New material features use namespaced, opt-in syntax. A stock material is not + silently reinterpreted as PBR content. +- Wire, save, and demo format changes are explicitly versioned. Protocol 2.41 + compatibility is not changed by an internal refactor. +- Dedicated builds do not acquire renderer, presentation, or client-only job + dependencies. +- Every promoted default passes the four-role stock baseline and a human review + of engine-written screenshots. MP validation explicitly uses + `+set ui_autoJoin 1`, `+set si_pure 1`, and + `+set net_serverAllowServerMod 0` unless the test is specifically for the join + menu or non-pure behavior. + +## What openQ4 already uses from the BFG lineage + +The most directly reusable BFG work is not hypothetical. openQ4 already carries +audited BFG-lineage code in these areas: + +- binary image loading/writing, image options, image programs, color-space + conversion, and DXT encode/decode; +- renderer image management and intrinsic-image support; +- the newer sound world/emitter/voice/sample architecture, with the OpenAL + implementation arriving through the documented RBDOOM lineage; +- small idlib utilities such as static strings, swapping, and sorting. + +The authoritative inventory currently records 37 BFG-headered files. Any +additional incorporation must retain the upstream notices, update that inventory +and the applicable Additional Terms coverage, and record any intermediate +lineage. This roadmap is technical guidance, not a legal conclusion. + +"Readily available" below means that the named implementation is present in the +audited official snapshot and can be studied or adapted without reverse +engineering. It does not mean drop-in: several paths assume Win32, PS3/SPU, +trusted pre-generated data, 32-bit offsets, or assert-only validation. New code +must use openQ4's portable interfaces and fail-closed input rules. + +## Current implementation state (2026-08-19) + +This table is the dated delivery snapshot for this roadmap. **Implemented** +means the compatibility-safe foundation is present and covered by the cited +project evidence; **Experimental** means substantial code exists but is not a +supported/default capability; **Partial** means only part of the required +contract exists; and **Planned** means the roadmap is still design guidance. +The [engine capability matrix](engine-capability-matrix.md) remains authoritative +when a status differs or a narrower qualification is needed. + +| Workstream | State | Current boundary and next work | +|---|---|---| +| Stock-compatibility and security foundation | **Implemented** | Protocol 2.41 preservation, pure-MP game-module containment, bounded malformed-input handling with immediate session teardown, challenge entropy, rcon2, private-CVar redaction/remote authority, HTTP(S)-only transfer policy, source-provenance auditing, archived MP auto-join test policy, and the four-role retail-PK4 evidence harness are present. Release promotion still requires a clean source pair, final-package capture, and retained human review. | +| Audited BFG-lineage image, sound, and idlib work | **Implemented** | The existing 37-file BFG inventory is tracked with source lineage and Additional Terms. Further imports must update the same manifest and notices. | +| PBR material authoring/resource foundation | **Implemented foundation; visible capability missing** | Namespaced parsing, typed color/data image usage, classic ARB2 fallbacks, scene-packet metadata, resource-table diagnostics, and fail-closed exclusion from unsupported modern-visible paths cover Phases 0-3 of the PBR plan. PBR shaders, direct lighting, visible ownership, IBL, and specular probes do not exist yet. | +| GPU measurement and dynamic resolution | **Partial** | OpenGL has a delayed four-frame non-blocking timer-query ring and the engine has manual render scaling. A backend-neutral full-frame result, Vulkan timestamp ring, automatic controller, discontinuity resets, and promotion evidence remain Milestones A/E. | +| General job system | **Planned** | Local workers do not provide a bounded dependency/job-list substrate. Milestone A begins with portable sleepable synchronization, cancellation, deterministic synchronous execution, and dedicated-server-safe ownership. | +| Generated caches, streaming, and learned preload manifests | **Partial** | Binary images and generated-animation patterns exist, but model/world/collision caches and a cancellable read -> decompress -> decode -> upload pipeline do not. Retail PK4 resolution remains authoritative. | +| Shared renderer contracts and GPU skinning | **Partial** | Scene packets, resource tables, upload infrastructure, and CPU skinning provide inputs, but there is no backend-neutral material/pass IR or supported joint-buffer/GPU deformation path. CPU deformation remains authoritative. | +| Modern classic-frame ownership | **Experimental** | Render-graph, modern OpenGL submission, clustered/MDI infrastructure, shadow maps, light grids, and Vulkan coverage exist, but no complete stock visible-lighting domain is promoted. ARB2 remains the supported/default owner. | +| Temporal presentation | **Planned** | Complete motion vectors, history ownership, TAA/TAAU, reactive/disocclusion handling, and dynamic-resolution integration are absent. SMAA remains the compatibility path. | +| Modern PBR lighting and idTech 6-like follow-ons | **Planned** | GGX/IBL, reflection probes, clustered decals/probes, froxel volumetrics, SSR/SSGI, GPU-driven visible ownership, and optional sparse residency all remain after the shared-contract and temporal gates. | + +The practical next target is **Milestone A**. It unlocks safe parallel loading, +cache generation, renderer-front-end work, and trustworthy GPU-budget feedback +without changing stock content interpretation. The PBR Phase 0-3 foundation is +intentionally not a reason to skip ahead to visible PBR lighting. + +## Best official Doom 3 BFG candidates + +The paths below are relative to official `DOOM-3-BFG/neo/` at the pinned +snapshot. + +| Candidate | Readily available BFG code | openQ4 use | Reuse level | Priority | +|---|---|---|---|---| +| Parallel job substrate | `idlib/ParallelJobList.*`, `idlib/Thread.*`, renderer consumers in `tr_frontend_addmodels.cpp` and `tr_frontend_addlights.cpp` | A bounded, dependency-aware worker pool for renderer front-end work, archive/decode jobs, animation work, and cache generation | Adapt architecture; replace platform primitives and spin waits with SDL3/portable C++, and retain deterministic single-thread fallback | **P1** | +| GPU skeletal skinning | `renderer/BufferObject.*`, `VertexCache.*`, `Model_md5.cpp`, `tr_frontend_addmodels.cpp`, `tr_backend_draw.cpp`, `RenderProgs*` | Joint-buffer uploads and optional four-weight GPU deformation for rendered MD5/MD5R draw surfaces and shadow-map casters while preserving CPU consumers | Port the algorithm into dedicated backend-neutral skin attributes and buffers; do not import BFG's GL backend or blindly reuse its vertex-color packing | **P1** | +| GPU timing and automatic resolution scaling | `renderer/ResolutionScale.*`, the timer query in `RenderSystem.cpp`, CPU profiling blocks in `RenderLog.*` | Feed a backend-neutral full-frame result from openQ4's existing non-blocking GL query ring and a new Vulkan timestamp path into a bounded controller | Reuse the controller logic, not BFG's single-query blocking readback | **P1** | +| Generated model, render-world, and collision caches | `renderer/Model.cpp`, `Model_md5.cpp`, `ModelManager.cpp`, `RenderWorld_load.cpp`, `cm/CollisionModel_files.cpp` | Cache parsed static/MD5 geometry, `.proc` world data, and collision data after first trusted-source load | Design a hardened openQ4 format; follow the generated-animation cache contract and include Quake 4 MD5R/source-PK4 identity | **P1** | +| Preload manifests | `framework/File_Manifest.*` and resource-type discovery in `FileSystem.cpp` | Record actual per-map image/model/animation/sample/collision use and replay it through a cancellable preload queue | Reuse the manifest concept, not BFG's retail manifest contents | **P1** | +| Resource containers | `framework/File_Resource.*` | Optional developer-generated, sequential cache containers for derived data | Reuse the access-order concept only; BFG uses 32-bit offsets and trusted tables, so prefer individual cache files or a new bounded 64-bit format and never require a BFG `.resources` package | **P2** | +| Parallel renderer front end | `renderer/tr_frontend_*`, `renderer/jobs/ShadowShared.*`, atomic frame allocation in `tr_frontend_main.cpp` | Parallel entity/light visibility, interaction preparation, and shadow-caster work after ownership is made immutable for a frame | Architectural port with substantial Quake 4/BSE and modern-renderer adaptation | **P1/P2** | +| Explicit transient/static GPU buffers | `renderer/BufferObject.*`, `VertexCache.*` | Complete the current upload manager with backend-neutral vertex/index/joint handles, frame rings, fences, budgets, and overflow diagnostics | Mine lifecycle and handle ideas; current openQ4 GL/Vulkan ownership must remain authoritative | **P1** | +| Render-matrix and culling utilities | `idlib/geometry/RenderMatrix.*` | Shared, tested MVP/frustum/depth-bounds math for scene packets, shadow planning, Hi-Z, and GL/Vulkan clip-space variants | Selectively adapt algorithms and tests; do not force BFG's matrix or vertex ABI onto Quake 4 data | **P2** | +| Render-program parameter model | `renderer/RenderProgs.*`, `RenderProgs_GLSL.cpp` | Common parameter names/layouts for optional skinning and shared passes | Selective reference only; build a backend-neutral material/pass IR rather than another GL-specific shader manager | **P2** | +| Stereo presentation | `renderer/RenderContext.h`, stereo portions of `GuiModel.cpp`, `RenderSystem.*`, and `OpenGL/gl_backend.cpp` | Optional side-by-side/top-bottom rendering and stereo-aware full-screen GUI depth | Adapt only after ordinary presentation is stable; use SDL/OpenXR-era platform interfaces instead of old WGL assumptions | **P3** | +| Lightweight compression | `sys/LightweightCompression.*` | Potential LZW/zero-run compression for new cache payloads where profiling proves a benefit | Reuse only behind a new bounded, fuzzed decoder and versioned container; do not insert it into protocol 2.41 | **P3** | + +### 1. Parallel jobs: the highest-leverage import + +BFG's `idParallelJobList` provides job lists, priorities, synchronization +points, list dependencies, bounded parallelism, timing, and a deterministic wait +boundary. Its real BFG renderer users are deliberately coarse: add visible +models, add lights, and build shadow work. That is a better starting point than +spawning ad-hoc threads throughout openQ4. + +The API should be adapted, not copied blindly: + +- back it with SDL3 threads/condition variables or a small portable C++ core; +- replace BFG's spinning `Wait()` behavior and fixed platform processing-unit + assumptions with blocking waits and explicit worker limits; +- make cancellation and shutdown explicit; +- make job payload ownership and lifetime visible in the type/API contract; +- provide a synchronous implementation used by dedicated builds, tests, and + deterministic debugging; +- bound queues and allocations; report saturation instead of silently growing; +- collect queue, execution, wait, and critical-path timings; +- prohibit renderer API calls from arbitrary workers unless the backend + explicitly owns that queue. + +First consumers should be work that already has a clean join point: learned +preload discovery, image decode/transcode, generated-cache writes, and then +renderer model/light preparation. PK4 archive mutation and game-state mutation +should not be the first consumers. + +### 2. GPU skinning: a concrete idTech 5-class capability + +BFG converts MD5 vertices to four normalized byte weights and four joint +indices, uploads joint matrices through an aligned joint buffer, and keeps a CPU +path for unsupported or special surfaces. That is a strong reference for +openQ4, where CPU deformation remains authoritative today. + +The packed layout is not itself a safe compatibility contract. BFG asserts that +a model has fewer than 256 joints, stores joint indices in `color`, stores +weights in `color2`, and sorts, truncates, then renormalizes vertices with more +than four influences. Its own source notes residual weights above 25 percent in +some assets. Quake 4's packed MD5R path can also carry diffuse vertex colors, so +openQ4 must not silently repurpose those channels. + +An openQ4 implementation needs additional compatibility work: + +- inventory joint counts and influence counts, then validate the top-four + reduction against stock Quake 4 MD5 and packed MD5R meshes; retain the CPU + path whenever a joint-index limit or residual-error threshold is exceeded; +- use dedicated skin-index/weight attributes, or prove that an existing packed + channel is semantically unused, so MD5R diffuse colors remain intact; +- preserve CPU deformation for collision, traces, deforms, software-only debug + tools, decals/overlays that need current positions, stencil shadow-volume + construction, and any shader/material path lacking the skinning contract; +- carry joint data through ambient surfaces, light interactions, shadow-map + casters, subviews, and view models, while explicitly routing incompatible + surfaces to CPU deformation; +- use one backend-neutral joint-buffer handle represented consistently in GL + and Vulkan; +- compare bounds, positions, normals/tangents, silhouettes, and screenshots + against the CPU path before enabling it by default. + +This should land as capability and parity infrastructure first. It should not be +coupled to PBR, TAA, or a renderer-default switch. + +### 3. Generated assets and learned preloading + +BFG assumes generated BFG resources exist. Stock Quake 4 installations do not, +so its packaged manifests and resource containers cannot be required. The safe +adaptation is the pattern openQ4 already uses for generated animation and binary +image caches: + +1. Load the original PK4 asset normally. +2. Record the resolved source path, containing PK4 checksum, parser/build + version, platform-independent format version, and relevant quality settings. +3. Write derived data under `fs_savepath/baseoq4/generated/` using an atomic + temporary-file replacement. +4. On the next run, validate every bound before allocation and every source key + before use. +5. Delete or ignore an invalid cache and fall back to the original asset. + +Useful next cache targets are parsed static/MD5/MD5R render models, `.proc` +render-world data, collision models, and a learned per-map preload manifest. +BFG's serializers are useful field inventories, but their timestamp checks and +trusted-data assumptions are not sufficient for a PK4-backed, fail-closed +runtime. A preload manifest should schedule work; it should not override VFS +resolution or become a second source of asset truth. + +### 4. Dynamic resolution from real GPU time + +BFG's resolution controller is compact and readily adaptable. It lowers +resolution quickly when GPU time exceeds a threshold and raises it more slowly +after several under-budget frames, avoiding constant oscillation. openQ4 already +has render scaling, renderer metrics, high-refresh presentation, and a +four-frame, non-blocking GL timer-query ring. What is missing is a +backend-neutral total-frame timing result, an equivalent Vulkan timestamp path, +the feedback controller, and complete promotion evidence. + +The production version should improve on the 2012 implementation: + +- extend the existing delayed GL query ring and add a Vulkan timestamp-query + ring; never wait on the current frame's result; +- target a user/display frame budget and account for VRR; +- quantize dimensions to backend-friendly alignments; +- expose minimum scale, response rate, and a conservative default-off rollout; +- reset history on map load, teleport, video restart, backend switch, and other + discontinuities; +- keep GUI/HUD composition at native output resolution; +- integrate with future TAAU, while remaining useful with SMAA/bilinear scaling. + +## BFG systems to study but not transplant + +| BFG subsystem | Why it is not a direct openQ4 import | Safer direction | +|---|---|---| +| `sys/PacketProcessor.*`, `Snapshot*`, lobby/session code | Different wire model, object snapshots, lobby assumptions, platform services, and game semantics; replacing it would break protocol 2.41 and existing Quake 4 networking | Keep the hardened Quake 4 path. If a new transport is justified, negotiate an explicit openQ4 protocol while retaining 2.41 as a separate path | +| Depth-fail stencil-shadow back end | The official BFG release expressly omits the code that enables Carmack's Reverse; the included shadow jobs and shared geometry helpers are not a complete replacement renderer | Keep openQ4's existing Quake 4-compatible stencil path authoritative. Study the BFG job boundaries independently of the omitted back-end operation | +| SWF UI runtime | Quake 4 ships idTech 4 GUI scripts, not BFG SWFs; replacing the UI runtime would strand stock menus and in-world GUIs | Modernize the existing GUI renderer/parser and add optional new UI surfaces without removing the stock path | +| XAudio2 backend | Windows-specific and redundant with openQ4's cross-platform OpenAL voice/HRTF/EFX work | Continue improving the current backend and SDL/platform device lifecycle | +| Doom 3 `d3xp` gameplay, aim, inventory, achievements, and save/session code | Different game rules, class layouts, scripts, maps, and save data | Port isolated engine-agnostic ideas only; implement Quake 4 gameplay changes canonically in `openQ4-game` | +| BFG render backend as a whole | Assumes BFG vertex formats, shaders, material behavior, generated resources, and GL/platform interfaces | Extract contracts and algorithms into the current backend-neutral GL/Vulkan architecture | +| BFG resource packages as shipped data | No corresponding generated packages exist in a retail Quake 4 installation | Generate disposable caches locally and retain source-PK4 fallback | +| Doom Classic integration and platform storefront code | Unrelated content and unavailable proprietary service pieces | Keep out of the runtime | + +## Capabilities beyond the official BFG drop + +Reaching an idTech 6-like standard requires work that the 2012 BFG source does +not provide. These should build on the BFG-derived foundations rather than be +treated as code-import tasks. + +The highest-leverage renderer step is to finish coherent ownership, not add one +more isolated experimental pass. openQ4 already has experimental scene packets, +a render graph, clustered/MDI submission, shadow maps, and Vulkan coverage, but +the capability matrix records no proven modern visible-lighting domain yet. +Classic ambient, interaction, fog, blend, stage-condition/color, deform, +subview, GUI, and fallback semantics must become explicit shared contracts before +temporal or PBR work multiplies the parity surface. + +### Backend and submission + +- A backend-neutral material/pass intermediate representation shared by GL and + Vulkan. +- Complete modern visible-lighting ownership before promoting GPU-driven + submission. +- Persistent resource descriptors, pipeline/shader caches, indirect draws, + Hi-Z culling, and a safe CPU rollback. +- Explicit resource state/lifetime tracking and asynchronous upload budgets. + +### Image quality + +- Complete motion vectors for rigid, skinned, particle, deform, subview, GUI, + and view-model surfaces. +- TAA/TAAU with reactive masks, disocclusion handling, camera-cut resets, and + SMAA fallback. +- Dynamic resolution driven by backend timestamps. +- Namespaced PBR material extensions, GGX lighting, IBL/specular probes, and + stock-material defaults that preserve the classic look. +- Froxel fog/volumetrics, SSR, and carefully bounded screen-space GI only after + depth/history infrastructure is reliable. +- True HDR output (scRGB/HDR10 negotiation, paper-white GUI composition, and HDR + screenshot policy), distinct from the existing internal HDR scene chain. + +### Streaming and CPU scalability + +- A staged read -> decompress -> decode -> upload pipeline with cancellation, + per-stage budgets, and map-generation ownership tokens. +- Learned preload manifests and priority changes driven by portal visibility, + not unconditional whole-level preloads. +- Optional virtual-texture/sparse-residency support for high-resolution community + content, after ordinary streaming is reliable. The official BFG drop does not + provide idTech 5's virtual-texturing implementation, and stock Quake 4 assets + must never depend on this path. +- GPU skinning plus jobbed animation/model preparation. +- Background shader/pipeline compilation with deterministic cache keys and an + always-available synchronous fallback. + +### Networking and operations + +- Keep protocol 2.41 for compatibility, but consider a separately negotiated + openQ4 transport for larger sequence spaces, stronger session authentication, + modern congestion/fragmentation behavior, and optional traffic protection. +- Continue bounded parser work, fuzzable decode APIs, rate limits, structured + diagnostics, and headless dedicated-server soak tests independently of any + future protocol. + +## Recommended implementation order + +| Milestone | Current state | Dependency that prevents promotion | +|---|---|---| +| A. Foundation and measurement | **Partial** | The GL timing ring exists; the portable job substrate, backend-neutral timing, Vulkan timestamps, and recorded budgets do not. | +| B. Loading and cache modernization | **Partial** | Existing binary-image/generated-animation patterns do not yet form learned manifests, bounded pipeline stages, or model/world/collision caches. | +| C. Shared renderer contracts and GPU animation | **Partial** | Packet/resource infrastructure exists, but shared GL/Vulkan pass semantics and GPU skinning parity are missing. | +| D. Modern classic-frame ownership | **Experimental** | Individual modern paths exist; no complete classic-visible domain has satisfied the cross-backend parity exit gate. | +| E. Temporal presentation | **Planned** | Depends on Milestones A, C, and D for timing, motion/resource contracts, and complete frame ownership. | +| F. Modern materials and advanced lighting | **Foundation only** | PBR authoring/resource Phases 0-3 exist, but visible PBR/IBL and advanced-lighting ownership must wait for Milestones C-E. | + +### Milestone A: foundation and measurement + +1. Land a portable bounded job manager with synchronous mode, dependency tests, + shutdown/cancellation tests, and timing counters. +2. Expose the existing delayed GL timer ring through a backend-neutral + total-frame timing result and add the equivalent Vulkan timestamp-query ring. +3. Establish per-map CPU/GPU budgets in the existing benchmark and stock + evidence tools. + +Exit gate: identical stock screenshots/game state with jobs on/off, clean +shutdown under repeated map changes, and trustworthy non-blocking timing. + +### Milestone B: loading and cache modernization + +1. Add a learned preload manifest keyed to the exact stock PK4 set and renderer + settings. +2. Move read/decompress/decode work into cancellable stages. +3. Add versioned binary render-model, render-world, and collision caches under + `fs_savepath`. + +Exit gate: cold and warm load measurements, bounded memory, cancellation during +map/restart/disconnect, corrupt-cache fallback, and zero required loose assets. + +### Milestone C: shared renderer contracts and GPU animation + +1. Define the minimum backend-neutral material/pass, clip-space, vertex-layout, + and buffer-handle contracts needed by both GL and Vulkan. +2. Add backend-neutral joint-buffer rings and dedicated four-weight vertex data. +3. Prove the rigid/MD5/MD5R CPU-vs-GPU parity corridor, including diffuse vertex + colors, residual weights, and joint-count fallbacks. +4. Extend coverage to interactions, decals, subviews, shadow-map paths, view + models, and explicit CPU stencil-volume fallback. + +Exit gate: stock SP/MP visual equivalence, CPU fallback parity, no collision or +hit-detection changes, and measured CPU-frame reduction in animation-heavy maps. + +### Milestone D: modern classic-frame ownership + +1. Express classic ambient, interaction, fog, blend, stage-condition/color, + deform, subview, GUI, and fallback behavior through the shared contracts. +2. Make GL and Vulkan consume the same semantic records, with backend-specific + execution and a per-domain classic rollback. +3. Promote scene-packet, resource-table, render-graph, and modern-visible domains + only when each domain's exact parity evidence is complete. + +Exit gate: at least one complete stock-frame domain is modern-owned on both +backends, no visible light or surface is silently dropped, and rollback produces +the classic result. + +### Milestone E: temporal presentation + +1. Promote the GPU-time controller to experimental dynamic resolution. +2. Add complete motion-vector ownership. +3. Implement TAA/TAAU with SMAA rollback and native-resolution UI. + +Exit gate: stable motion, camera cuts, particles, weapon view, portals/subviews, +menus, screenshots, save previews, and GL/Vulkan parity. + +### Milestone F: modern materials and advanced lighting + +1. Extend the shared material/pass IR with namespaced PBR/IBL semantics without + changing stock defaults. +2. Add reflection/specular-probe ownership and bounded clustered decal/probe + records. +3. Promote clustered/GPU-driven submission only after visible-lighting parity. +4. Stage froxel volumetrics and screen-space reflection/GI work behind separate + evidence gates rather than one all-or-nothing renderer switch. + +Exit gate: the modern renderer owns complete validated domains rather than +isolated passes, with the classic path remaining a one-setting rollback. + +## Promotion evidence for every milestone + +Each milestone should record: + +- source provenance and retained notices for incorporated code; +- focused unit/static tests and malformed-cache/input tests; +- Windows x64 builds plus Linux/macOS compile-policy coverage appropriate to the + changed subsystem; +- stock-only SP load/save/reload/demo evidence; +- pure MP listen-server and auto-joined client gameplay evidence; +- engine-render-target screenshots and human visual review; +- before/after CPU frame, GPU frame, load time, memory, and cache-size metrics; +- rollback results with the new feature disabled; +- confirmation that no new loose content is required. + +The [engine capability matrix](engine-capability-matrix.md) remains the current +truth. This roadmap describes sequence and acceptance gates; it does not mark a +capability implemented merely because BFG source exists or a prototype compiles. diff --git a/docs/dev/official-pk4-checksums.md b/docs/dev/official-pk4-checksums.md index fc1c1aef..a2fdef90 100644 --- a/docs/dev/official-pk4-checksums.md +++ b/docs/dev/official-pk4-checksums.md @@ -7,7 +7,15 @@ This table captures the PK4 checksums loaded from official installed game direct - Log source: `logs/openq4.log` startup lines (`Loaded pk4 ... with checksum ...`) under `fs_savepath\\` - Checksum format: engine PK4 checksum (`MD4` of zip-entry CRC list, as computed in `src/framework/FileSystem.cpp`) -openQ4 ignores the retail game-binary PK4 archives (`game000.pk4` through `game300.pk4`, plus `gamex*.pk4` variants) because it ships its own game modules. They are not required and are not verified. +openQ4 ignores the retail game-binary PK4 archives (`game000.pk4` through `game300.pk4`, plus `gamex*.pk4` variants) as loadable content because it ships its own game modules. They are not required, mounted, or content-verified. The one retained checksum value described below is a network compatibility token, not a requirement to install or trust the archive. + +## Pure multiplayer game-module token + +Quake 4 protocol 2.41 carries a game-code PK4 checksum after the ordered pure asset list. openQ4 preserves that field by sending the official Quake 4 1.4.2 `q4base/game300.pk4` engine checksum, `0x68fb90b1`, for the legacy Windows, Linux, and macOS OS IDs. The same value is used on every platform, so an openQ4 client and server do not need identical executable formats to complete the asset handshake. + +The value is only a protocol compatibility token. openQ4 does not open, extract, download, restart into, or execute `game300.pk4`. It accepts the token only when a game module has already been resolved from the trusted local openQ4 package/module roots; a missing local module or any other token fails the pure handshake as unavailable game code. The ordered retail/openQ4 asset PK4 list is still checked separately. + +`0x68fb90b1` is not a hash of the loaded openQ4 module, cryptographic attestation, or an anti-cheat guarantee. It preserves the stock 1.4.2 wire meaning while executable trust remains a local packaging decision. ## Required official baseline diff --git a/docs/dev/plans/2026-05-19-pbr-material-support.md b/docs/dev/plans/2026-05-19-pbr-material-support.md index e0db3135..8ebd8662 100644 --- a/docs/dev/plans/2026-05-19-pbr-material-support.md +++ b/docs/dev/plans/2026-05-19-pbr-material-support.md @@ -6,6 +6,12 @@ Add physically based material support to openQ4 without changing how shipped Qua The short version: stock materials stay stock; PBR materials get a modern material model; every unsupported case has a visible fallback reason and a legacy rendering path. +## Implementation Status + +As of 2026-08-19, the compatibility-safe Phase 0-3 foundation is present: opt-in controls and capability reporting, namespaced parser metadata, typed PBR image usage and lifecycle handling, classic fallback generation, scene-packet propagation, and material-resource-table records, metrics, dumps, and fail-closed ownership checks. These are authoring and renderer-infrastructure contracts, not user-visible PBR rendering. + +Phase 4-6 PBR G-buffer shaders, direct lighting, and visible ownership remain unimplemented, as do Phase 8 IBL and specular environment probes. The cross-engine capability therefore remains **Missing** in the [engine capability matrix](../engine-capability-matrix.md). A checked item below means current source and an automated source or engine self-test directly cover that item; runtime, visual, gameplay, and promotion acceptances remain unchecked until their own evidence exists. + ## Suitability Review This plan is suitable for openQ4 only if legacy fallback is treated as a first-class authoring contract, not as a nice-to-have conversion step. openQ4's shipped-content compatibility still comes from the classic material parser and the `SL_BUMP`/`SL_DIFFUSE`/`SL_SPECULAR` interaction model, so PBR metadata must never be the only runtime description for a material that can ship in `baseoq4/`. @@ -299,37 +305,38 @@ Add separate PBR controls instead of overloading enhanced materials: ### Phase 0: Baseline And Gates - [ ] Record stock-material baseline with PBR code absent or fully disabled. -- [ ] Add `r_pbrMaterials`, `r_pbrGeneratedLegacyFallback`, `r_pbrDebug`, and `r_pbrInferFromLegacyMaterials`. -- [ ] Add `gfxInfo` lines that show PBR parser support, PBR modern support, and fallback status. -- [ ] Add empty metrics counters with zero values on stock startup. +- [x] Add `r_pbrMaterials`, `r_pbrGeneratedLegacyFallback`, `r_pbrDebug`, and `r_pbrInferFromLegacyMaterials`. +- [x] Add `gfxInfo` lines that show PBR parser support, PBR modern support, and fallback status. +- [x] Add PBR metrics counters to the material-resource table. +- [ ] Confirm those counters remain zero on stock startup. - [ ] Acceptance: safe validation matrix passes, stock startup logs do not gain material warnings, and `r_pbrMaterials 0/1` changes nothing when no PBR materials are loaded. ### Phase 1: Parser Metadata Only -- [ ] Add `pbrMaterialInfo_t` to `idMaterial`. -- [ ] Parse `pbr { ... }` tokens into metadata without changing classic stages. -- [ ] Add image loading for PBR maps with correct texture usage classes. -- [ ] Update material lifecycle methods for PBR images. -- [ ] Add parser validation tests through material decl validation. -- [ ] Acceptance: PBR sample declarations parse, stock declarations compile identically, and no renderer path consumes PBR metadata yet. +- [x] Add `pbrMaterialInfo_t` to `idMaterial`. +- [x] Parse `pbr { ... }` tokens into metadata without changing classic stages. +- [x] Add image loading for PBR maps with correct texture usage classes. +- [x] Update material lifecycle methods for PBR images. +- [x] Add parser validation tests through material decl validation. +- [ ] Acceptance: PBR sample declarations parse, stock declarations compile identically, and no visible/shader path consumes PBR metadata yet. ### Phase 2: Legacy Fallback For PBR Authored Materials -- [ ] Detect whether a PBR material already has classic interaction stages. -- [ ] Add explicit `legacyBumpMap`/`legacyDiffuseMap`/`legacySpecularMap`/`legacyEmissiveMap` support. -- [ ] Add generated fallback stages for PBR-only materials as development fallback, not as release-quality fallback. -- [ ] Report approximate fallback warnings once per material. -- [ ] Track authored, explicit-generated, approximate, and missing fallback counts. +- [x] Detect whether a PBR material already has classic interaction stages. +- [x] Add explicit `legacyBumpMap`/`legacyDiffuseMap`/`legacySpecularMap`/`legacyEmissiveMap` support. +- [x] Add generated fallback stages for PBR-only materials as development fallback, not as release-quality fallback. +- [x] Report one aggregate approximate-fallback warning per material parse. +- [x] Track authored, explicit-generated, approximate, and missing fallback counts. - [ ] Add a self-test that creates a PBR-only material and verifies ARB2 sees bump/diffuse/specular interaction stages. - [ ] Acceptance: PBR materials render something sane under the default ARB2 path, explicit fallback maps generate the expected classic stages, approximate fallback is visible in metrics, and stock materials remain unchanged. ### Phase 3: Material Resource Table Integration -- [ ] Extend packet material records with PBR flags and first PBR texture handles. -- [ ] Extend `MaterialResourceTable` semantics, records, fallback reasons, and metrics. -- [ ] Dump PBR metadata in `rendererMaterialResourceTableDump`. -- [ ] Update draw/submit plan fallback checks to understand PBR-ready versus legacy-ready materials. -- [ ] Acceptance: modern side paths can identify PBR materials and explain why they are or are not renderable. +- [x] Extend packet material records with PBR flags and first PBR texture handles. +- [x] Extend `MaterialResourceTable` semantics, records, fallback reasons, and metrics. +- [x] Dump PBR metadata in `rendererMaterialResourceTableDump`. +- [x] Update draw/submit plan fallback checks to understand PBR-ready versus legacy-ready materials. +- [x] Acceptance: modern side paths can identify PBR materials and explain why they are or are not renderable. ### Phase 4: PBR G-buffer Side Path @@ -384,12 +391,12 @@ Add separate PBR controls instead of overloading enhanced materials: Add safe tests: -- `renderer-pbr-parser-selftest`: parser metadata, scalar registers, image usage classes, normal format enum, and error cases. -- `renderer-pbr-legacy-fallback-selftest`: generated classic stage fallback and explicit fallback maps. -- `renderer-pbr-material-table-selftest`: PBR semantics, texture binding counts, packed/separate maps, fallback reasons. -- `renderer-pbr-gbuffer-selftest`: G-buffer packing and debug overlays. -- `renderer-pbr-lighting-selftest`: deferred/forward PBR shader readiness and BRDF sanity. -- `renderer-pbr-visible-selftest`: guarded visible ownership on a synthetic packet frame. +- [x] Parser metadata, scalar registers, image usage classes, normal format enum, and error cases through `rendererPBRMaterialSelfTest`. +- [x] Generated classic-stage fallback and explicit fallback maps through `rendererPBRMaterialSelfTest`. +- [x] Packet propagation plus PBR semantics, texture binding counts, packed/separate maps, and fallback reasons through `rendererScenePacketSelfTest` and `rendererMaterialResourceTableSelfTest`. +- [ ] PBR G-buffer packing and debug overlays. +- [ ] Deferred/forward PBR shader readiness and BRDF sanity. +- [ ] Guarded visible PBR ownership on a synthetic packet frame. Add gameplay/manual coverage: diff --git a/docs/dev/proposals/rbdoom3-bfg-parity-modernization-plan.md b/docs/dev/proposals/rbdoom3-bfg-parity-modernization-plan.md index a19d4ec5..900eb58f 100644 --- a/docs/dev/proposals/rbdoom3-bfg-parity-modernization-plan.md +++ b/docs/dev/proposals/rbdoom3-bfg-parity-modernization-plan.md @@ -3,6 +3,9 @@ Date: 2026-02-09 Author: Codex (analysis + implementation plan) +> [!IMPORTANT] +> This is a historical proposal, not a current-state inventory. Several baseline statements below predate the modern GL, Vulkan, shadow-map, and light-grid work now in the tree. Use the [engine capability matrix](../engine-capability-matrix.md) for authoritative implemented/experimental/missing status, the [idTech 5-level modernization roadmap](../idtech5-modernization-roadmap.md) for the current compatibility-safe delivery order, and [source provenance](../source-provenance.md) before considering any upstream import. + ## 1. Goal Bring openQ4 to the same technical standard as modern RBDOOM-3-BFG, with special focus on lighting and shadowing, while preserving openQ4 rules: diff --git a/docs/dev/release-completion.md b/docs/dev/release-completion.md index 4ae75108..f2232c6b 100644 --- a/docs/dev/release-completion.md +++ b/docs/dev/release-completion.md @@ -36,6 +36,15 @@ is `docs/dev/macos-moltenvk-decision.md`. ## Ready For Changelog +- [x] Internet-server administration and package negotiation now fail closed at their remaining legacy edges. `rcon2` keeps remote-console passwords out of packets, applies bounded challenge and reply budgets, and redacts private settings from console output, history, journals, configuration serialization, command expansion, and completion previews; plaintext rcon is disabled unless both relevant sides explicitly opt into it. Server-originated userinfo and synchronized-CVar dictionaries decode transactionally and can update only the CVar class owned by that wire message, so they cannot borrow authority over private or unrelated settings. Server-provided redirects and PK4 entries accept only bounded HTTP or HTTPS URLs with a syntactically valid DNS, IPv4, or bracketed IPv6 host. Standard Meson packages keep in-process direct PK4 transfer disabled while preserving validated web-redirect prompts; a separately integrated curl-enabled build additionally rechecks the URL, refuses redirects, applies connect/stall/total-duration limits, and retains package path, size, and checksum validation. The obsolete network-driven executable updater no longer downloads or runs code and ignores server-controlled release text and links. +- [x] Pure multiplayer can remain enabled without giving a server control over executable code. `si_pure 1` is no longer silently disabled; the ordered asset-PK4 list is still enforced, while the protocol 2.41 game-code field uses the stock 1.4.2 `game300.pk4` checksum only as a platform-independent compatibility token for an already loaded module from trusted local openQ4 package/module roots. Missing or unexpected tokens, modules, asset lists, and platform IDs fail closed, code-bearing server mods require the explicit `net_serverAllowServerMod 1` opt-in, and neither pure negotiation nor package download can install or restart into game code. The token is not a cryptographic measurement of the loaded module, a module-equality proof, or an anti-cheat guarantee. +- [x] Truncated and malformed multiplayer state now fails at an explicit safety boundary. Bit-message underflow is tracked across normal and delta reads, queued messages and delta user commands are length-checked, and audited SP/MP leaf readers stage decoded fields until their payload is valid while validating referenced slots, types, ranges, and remaining data. Invalid covered traffic is dropped; a late malformed top-level snapshot tears down the affected session before another game or presentation frame. The legacy entity lifecycle is not a whole-snapshot transaction, and this targeted hardening does not claim formal verification of every parser. +- [x] Multiplayer validation now reaches gameplay without menu-input automation. Supported MP test profiles explicitly launch with the archived `ui_autoJoin 1` setting and the retail-PK4 compatibility baseline proves that the local client is active, not spectating, outside the session menu, and drawing its HUD on both sides of the screenshot. Only tests specifically covering the join menu or initial spectator/join flow may explicitly set `ui_autoJoin 0`. +- [x] Multiplayer function keys once again use Quake 4's canonical impulse actions: F1/F2 vote, F3 ready, F6 team, and F7 spectate bindings execute real key actions, while the controls menu edits the same vote/ready actions. Ready presses retain their throttle but reach the server through the reliable ready message, so the warmup prompt names a real key; its stock two-line non-tourney layout uses the inline keycap size so the instruction is not clipped out of the authored 40-pixel HUD window. Existing configs migrate only the exact historical openQ4 defaults (`voteyes`, `voteno`, `ready`, `toggleteam`, and `spectate`) on those five keys; custom and compound bindings are preserved, and the idempotent `ready` console command remains available. Casual no-time-limit matches also normalize a retained positive `si_overtime` value to sudden death with a zero timed period, avoiding a rejected competitive-rules import at stock deathmatch defaults. +- [x] Retail-asset compatibility reports now describe what actually ran: unchanged approved retail PK4 bytes beneath packaged openQ4 overlays. Verification rejects dry runs, stale or missing expected-asset bindings, mismatched source provenance, recorded failure arrays, and altered role launches; reports also count every packaged-overlay virtual path that supersedes retail content instead of presenting an overlay-colliding run as stock-only evidence. +- [x] Successful Windows full installs and VS Code fast stages now apply one shared, explicit cleanup manifest so extensionless POSIX client/dedicated binaries and obsolete renderer `.dll.mainbak` files cannot survive a platform switch. The old `baseoq4/skins` directory is removed only when empty and populated local content is preserved. Stage-root or directory links, junctions, and wrong entry types fail closed; a known stale leaf symlink is unlinked without following its target. +- [x] Manual save-game previews now render one coherent full-size frame before a bounded CPU center crop and resize produces the intended 320x240 thumbnail, eliminating repeated or nested screen-effect copies at widescreen resolutions. Renderer readback also keeps row-alignment padding out of captures whose widths are not multiples of four. +- [x] Liquids now behave like complete combat volumes: projectiles and hitscan shots pass through while producing reliable crossing splashes and sounds, submerged projectile smoke becomes bubbles, and hitscan wakes cover only the underwater ray segment. Quake 4-height probes and a gravity-aware launch arc make water jumps reliable from clear pool ledges; the wading basin is also normally jumpable. Clear water remains readable, liquid acoustics suppress hall-like wet tails and duplicate jump splashes, and drowning, slime, and lava report distinct localized obituaries with graphical icons. The well-lit `mp/liquid_lab` developer map provides recessed deep, shallow, and wading water plus vivid cellular slime and bright lava; its sky ceiling, shadow-casting beams, illuminated deep-pool floor, lava heat haze, steam, bubbling surfaces, submerged ambience, and full-arsenal respawns make every behavior immediately testable. - [x] The marine hovertank once again plays its engine and hover-pad loops throughout the vehicle sequence. Single-player now creates and updates the moving sound emitters just like the retail game and openQ4's multiplayer module; the later null-safe sound-world teardown keeps those emitters safe during map and engine shutdown. This addresses the missing vehicle audio reported in GitHub issue #114 without requiring different OpenAL, EAX, or speaker settings. - [x] The built-in `dmap` compiler and other engine-side geometry tools now initialize their own triangle-surface allocator copy before use. This prevents light-volume generation from dereferencing an uninitialized allocator after the renderer was split into a runtime module, fixing the Windows crash reported while compiling `game/airdefense1` in GitHub issue #108. - [x] Windows now resolves openQ4's per-user save root through the system Known Folder service, so redirected Local AppData and Saved Games locations are respected. Environment and working-directory fallbacks remain available for stripped-down sessions but reject malformed relative roots, while the displayed `fs_savepath` consistently uses native Windows separators. @@ -49,7 +58,7 @@ is `docs/dev/macos-moltenvk-decision.md`. - [x] macOS OpenAL Soft migration builds now fail clearly during configuration when the requested system package is not visible, instead of silently falling back to Apple's OpenAL framework and then failing to compile against `AL/...` headers. The migration lookup is pkg-config-only, and the provider guide documents the `PKG_CONFIG_PATH` needed for Homebrew's keg-only OpenAL Soft package. Release packages remain on Apple's framework. This resolves GitHub issue #94. - [x] Managed competitive multiplayer now has one cohesive, server-authoritative match system instead of scattered ready, vote, pause, team, demo and admin paths. An explicit competitive profile enables validated/frozen rules, readiness blockers, connection-scoped player/captain/coach/broadcaster/referee roles, durable rosters and substitutes, self-only coach/substitute withdrawal, Duel queues, team locks, tactical and technical pauses, typed proposals, recipient-filtered spectator follow/vitals, and BO1/BO3/BO5 map-pool/veto workflows; casual servers retain their existing behaviour. The localized Match Control surface exposes Status, Teams, Proposals, Rules, Series and Evidence from bounded recipient views with truthful denial reasons and confirmations. Series progress and its mutable report draft use one digest-protected schema-3 atomic checkpoint across maps without persisting names, addresses or client-slot authority; team-side mappings recover directly, while a trusted operator explicitly rebinds current Duel connections with `matchSeriesBind`. Managed matches also link series identity and an automatically owned server MVD into bounded schema-v2 per-map JSON evidence and an immutable schema-1 final series report promoted beneath `fs_savepath`, with recording/write failures isolated from the result. Managed activation cancels any inherited vote lifecycle, and legacy settings/kick/team/shuffle/restart/next-map adapters fail closed instead of bypassing typed authority. Local listen-server operators can grant a strictly observational broadcaster role through Match Control, and dedicated operators have the equivalent typed `matchBroadcaster` adapter. Supported placed, respawning major items are tracked authoritatively on the pause-safe match clock and projected only to explicit broadcaster/referee recipients; players, coaches and ordinary spectators receive no timers, and raw item tokens are never rendered. Captain invitations for neutral spectators remain fail-closed. The broadcaster role and recorded server MVD are supported, but no reachable engine Q4TV/repeater transport, delayed broadcast or live multi-POV path is shipped; the pure public-only repeater policy remains a fail-closed future integration boundary. - [x] Bordered window placement now accounts for the complete native frame on Windows, X11, and macOS instead of treating the drawable client rectangle as the whole window. Saved and restored positions track the outer-frame origin, usable-area constraints reserve the actual title-bar and resize-border extents, oversized client requests shrink enough for all chrome to remain reachable, display/DPI moves persist the updated frame position, and startup repeats placement after the window server finishes decorating the window. Native Wayland continues to delegate absolute positioning to the compositor by design. -- [x] Malformed legacy console, configuration, and numeric input now fails safely instead of risking an invalid memory access or overflowing an impulse command. Command argument collection is bounded by both slot and storage capacity, escaped argument reconstruction grows safely, lexer lookahead preserves the next token, and extreme string growth is checked before allocator rounding or legacy 32-bit narrowing. Relative file mutations reject traversal, rooted paths, platform-specific filename aliases, and other non-portable targets before touching the save directory. Server-offered package downloads are confined to new PK4 destinations beneath that directory, including Windows device-name protection, reject invalid or overflowing advertised sizes and response types, enforce accepted sizes against the received HTTP body, and release temporary files on every completion path; multiplayer negotiation now reports a deterministic empty game-package slot when no game-code package is requested. +- [x] Malformed legacy console, configuration, and numeric input now fails safely instead of risking an invalid memory access or overflowing an impulse command. Command argument collection is bounded by both slot and storage capacity, escaped argument reconstruction grows safely, lexer lookahead preserves the next token, and extreme string growth is checked before allocator rounding or legacy 32-bit narrowing. Relative file mutations reject traversal, rooted paths, platform-specific filename aliases, and other non-portable targets before touching the save directory. In separately integrated curl-enabled builds, server-offered package transfers are confined to new PK4 destinations beneath that directory, including Windows device-name protection, reject invalid or overflowing advertised sizes and response types, enforce accepted sizes against the received HTTP body, and release temporary files on every completion path; standard Meson packages leave that direct-transfer path disabled. Multiplayer negotiation reports a deterministic empty game-package slot when no game-code package is requested. - [x] Standalone game-module builds now refresh whenever either gameplay code or their shared engine support sources change, verify the complete staged snapshot by content, and compile the legacy inline-assembly SIMD implementations only for the 32-bit x86 target that supports them. This prevents stale modules and restores reliable x64 and modern-platform builds without dropping the existing x86 path. - [x] Windows x64 engine and standalone game-module builds are warning-clean again. Legacy native-width conversions now use checked boundaries or native-width accounting, intentional allocator alignment is explicit and layout-guarded, dead SDK remnants are removed, and the cleaned warning classes fail the build if they return. SDL configuration no longer reports a false missing-libdecor-version warning on platforms that do not use libdecor, and the Windows validation wrapper now forwards an explicitly selected companion repository correctly instead of treating its parameter name as a path. - [x] Bound controls are now easier to recognize at a glance: Settings key-bind rows show a bounded summary across keyboard, mouse, and controller inputs, while supported in-game prompts follow the device family the player used most recently. Keyboard bindings use clean labeled keycaps with square arrow symbols, upright mouse silhouettes fill the bound physical button without ambiguous numbers, and controller bindings use neutral positional graphics plus a localized Back/View label rather than assuming an Xbox, PlayStation, or Nintendo legend scheme. Inline icons now sit at nearly the full text-line height, important centered multiplayer prompts use a larger 150% treatment, and higher-sample rounded edges keep the procedural button art smooth at modern resolutions; the spectator HUD uses the same presentation for its follow-cycle and exit controls. Bind capture is safer too: Escape, Start, or Back cancels without erasing the action, while Backspace or Delete explicitly clears it. @@ -327,7 +336,7 @@ is `docs/dev/macos-moltenvk-decision.md`. - [x] Experimental macOS SDL3 renderer shutdown and error paths are safer: the default SDL3 backend now checks for a live, current OpenGL window/context before screen changes, swap, teardown, or `glFinish`, rejects empty OpenGL extension lookups before calling SDL, uses overflow-safe selected-display viewport math, clamps malformed mouse movement, wheel, controller, and rumble values before integer conversion or event queuing, normalizes unusual app-entry argument state, guards POSIX clipboard/console allocation copies plus null fatal-error formats, falls back cleanly when Cocoa display enumeration is partial, and has validation coverage to keep those detached-context and null-pointer crash guards in place. - [x] macOS fallback support is harder to crash during bring-up: the native Cocoa/OpenGL path now records a valid CGL context, validates pixel-format/context/window setup before use, releases a created context if the final make-current step fails, checks Cocoa screen and display-mode array bounds before `objectAtIndex:` calls, fails cleanly on fullscreen setup errors, unwinds partial display captures, treats missing or invalid-display VRAM telemetry as non-fatal, guards nil event helper entry points plus mouse capture, scroll-wheel overflow, and display/gamma-table lifetime paths, verifies the OpenGL context is current before swap/pause/resume/activation work, avoids stack allocation during extension lookup, returns clipboard text through the engine allocator, owns localized string fallbacks, displays fatal Cocoa alerts through a safe format wrapper, keeps obsolete Carbon/Xcode-era macOS sources out of the Meson build manifest, rejects raw C string builders from live macOS backend sources, and has validation coverage for the old undefined-return, transient-pointer, allocator-mismatch, allocation, overflow, stale-context, and null-pointer startup/shutdown hazards. - [x] Experimental macOS POSIX support paths now reject more bad runtime state before it can crash: thread and trigger-event helpers guard invalid indexes in release builds, pthread setup avoids uninitialized attribute use, requires error-checking mutex attributes before enabling critical sections, and tracks mutex/condition initialization before later lock/wait/signal calls, thread creation validates null function/tracking arguments and negative tracking counts before registering handles, terminal and desktop-console input cursors are clamped before pointer arithmetic, console scroll-wheel deltas clamp and saturate before mutating scroll state, console command and input events check allocation and oversized input before copying, null debug/print format strings are ignored or normalized, signal-number reporting avoids signed-overflow edge cases, and process handoff survives an unexpected null environment pointer from the macOS runtime. -- [x] macOS local handoff paths are tighter: URL opening now allows only normal web links plus existing local files under openQ4 runtime roots, process handoff requires an absolute executable file path instead of searching `PATH`, strips dynamic-loader injection variables such as `DYLD_*`/`LD_PRELOAD` before launching the child process, and native fatal-error dialogs use the modern `NSAlert` API. +- [x] macOS local handoff paths are tighter: URL opening allows only bounded HTTP or HTTPS web links with a syntactically valid host and rejects local-file URLs, process handoff requires an absolute executable file path instead of searching `PATH`, strips dynamic-loader injection variables such as `DYLD_*`/`LD_PRELOAD` before launching the child process, and native fatal-error dialogs use the modern `NSAlert` API. - [x] macOS static policy validation now guards the remaining platform-risk boundaries: validation rejects PATH-based process spawning, shell process helpers, broad AppKit URL opening, deprecated `NSRunAlertPanel`, Carbon leaking into the SDL3 path, unconditional Apple OpenAL linkage, and release workflows missing Developer ID/notarization/stapling while documenting the native fallback's legacy `NSOpenGL`/Carbon exceptions. - [x] Experimental macOS hardware signoff now has a structured Apple-host workflow action: `Invoke-openQ4MacOSWorkflow.ps1 -Action Signoff -MacOSGraphicsBridge both` builds the OpenGL and Metal bridge variants in separate Meson build directories, runs the real-asset smoke profile and macOS renderer matrix for each, installs the Desktop launcher, records the selected OpenAL provider, captures host/display/audio/USB/Bluetooth inventory, writes bridge-specific `macos-runtime-signoff.md` reports with the remaining manual input, audio, display-mode, and in-game renderer checks, and copies the matching result directories back to `.tmp/macos-vm/results/openq4-macos-results-.tar.gz` for review on the host. - [x] Collected experimental macOS hardware signoff archives now validate on the host immediately after copy-back: `tools/macos/validate_signoff_archive.py` rejects unsafe tar entries, requires both OpenGL and Metal bridge reports, checks workflow logs plus renderer smoke/matrix output, `-Action CollectResults -MacOSRunId ` can re-collect completed reports without rebuilding, and `-RequireCompletedSignoffChecklist` can fail final evidence if any manual hardware checklist item remains unchecked. @@ -357,7 +366,7 @@ is `docs/dev/macos-moltenvk-decision.md`. - [x] macOS guest signoff runs fail earlier on malformed workflow inputs: action, graphics bridge, OpenAL provider, build type, and platform backend tokens are validated before result directories or Meson setup arguments are created, keeping bad environment values from producing misleading signoff folders or logs. - [x] macOS host workflow cleanup is more reliable: the SSH runner now removes its per-run `/tmp/openq4-macos-` transfer directory from the Apple host in a `finally` block, while refusing to remove any path that does not match the expected temp-root pattern. - [x] macOS host/guest workflow paths and limits are more robust: documented `~/openq4-work` defaults now expand to the Apple user's home directory inside remote transfer, asset, build, and result-collection scripts; host parameters reject invalid ports, build types, empty build directories, and non-positive smoke limits; guest Meson setup refuses source/runtime/tool directories as build outputs; and signoff archives reject oversized non-text payloads or malformed action tokens. -- [x] macOS runtime handoff paths are more defensive: `Sys_OpenURL` now rejects overlong or control-character URLs before logging them, requires HTTP(S) URLs to include a host, keeps file URLs constrained to clean local runtime files, and `Sys_StartProcess` fails clearly if an executable bit cannot be applied before `execve`/`posix_spawn`; the macOS bootstrap also checks for `xcrun`, `plutil`, `lipo`, `otool`, and `codesign` before build/signoff work starts. +- [x] macOS runtime handoff paths are more defensive: `Sys_OpenURL` rejects overlong, control-character, non-HTTP(S), local-file, and malformed-host URLs before handoff, and `Sys_StartProcess` fails clearly if an executable bit cannot be applied before `execve`/`posix_spawn`; the macOS bootstrap also checks for `xcrun`, `plutil`, `lipo`, `otool`, and `codesign` before build/signoff work starts. - [x] Experimental macOS package metadata validation is stricter before artifacts are published: app bundle validation rejects empty icon/version/plist payloads and non-dictionary `Info.plist` roots, bundle creation fails immediately when no staged icon is available, localized `InfoPlist.strings` files are parsed for malformed or duplicate keys instead of substring-matched, and archive metadata members are capped so malformed packages cannot hide oversized app metadata. - [x] Sound-reactive world lights now match stock Quake 4 behavior more closely: hum-driven map lights such as Air Defense 1's corridor lighting use the authored sound-shader flicker envelope instead of dropping to black. - [x] Single-player enemy-hit feedback now matches retail Quake 4 again: the stock cursor GUI's per-component color aliases resolve correctly, so damaging enemies flashes the crosshair red without requiring custom HUD content or a new setting. @@ -399,7 +408,7 @@ is `docs/dev/macos-moltenvk-decision.md`. - [x] macOS app base-path discovery no longer depends only on Cocoa bundle metadata: when the executable path is inside `*.app/Contents/MacOS`, the engine derives the bundle's embedded resource/module roots directly, while complete older packages can still use their extracted adjacent root. - [x] macOS first-launch save/config handling is more robust: the default `~/Library/Application Support/openQ4` save root is normalized, created recursively with private permissions when missing, verified as writable/searchable before use, and falls back through the current working directory with clear diagnostics instead of silently selecting an unusable support-log path. - [x] Native macOS OpenGL context deactivation is safer during renderer skip/restart diagnostics: deactivation now tolerates missing contexts and only clears openQ4's own Cocoa context, avoiding accidental state disruption when Cocoa has no active game context. -- [x] macOS URL and process handoff diagnostics are safer for support logs: rejected URLs are no longer echoed before UTF-8 parsing, Foundation parsing, and the runtime-root/scheme allowlist pass, failed AppKit URL handoffs use a generic diagnostic, and malformed process command lines are rejected without copying the rejected command into logs. +- [x] macOS URL and process handoff diagnostics are safer for support logs: rejected URLs are no longer echoed before the shared HTTP(S)-only policy, UTF-8 parsing, and Foundation validation pass, failed AppKit URL handoffs use a generic diagnostic, and malformed process command lines are rejected without copying the rejected command into logs. - [x] Experimental macOS support is covered earlier in CI: pull-request and push validation now build/install both macOS ARM64 SDL3 OpenGL and Metal bridge variants, staged macOS payloads verify their icon and splash assets, and release packaging validates the generated `.app` executable and `Info.plist` before archiving. - [x] Experimental macOS packages now advertise the same macOS 11.0 minimum version that Meson builds target, avoiding an accidental macOS 12 metadata floor for users on supported Big Sur systems. - [x] macOS app launch diagnostics are clearer: the generated `.app` now embeds the packaged client binary directly, release CI checks that embedded executable against the adjacent package client plus key plist values, and executable-path discovery handles long `_NSGetExecutablePath` buffers without silently deriving paths from truncated strings. @@ -725,7 +734,7 @@ is `docs/dev/macos-moltenvk-decision.md`. - [x] Light-grid area transitions now blend across visible, unblocked portal neighbors near doorway/window boundaries, using `r_lightGridPortalBlend` to smooth indirect-light seams without changing baked asset formats. - [x] Light-grid atlas residency now keeps visible areas and unblocked portal neighbors hot while loose fallback atlases can still use an age-based window controlled by `r_lightGridResidencyFrames`, reducing reload churn and traversal hitches when moving through connected areas. - [x] Light-grid indirect rendering now uses a material-compiled representative diffuse stage instead of redrawing every diffuse stage, reducing extra geometry submissions for multi-diffuse materials while retaining a fallback when the preferred stage is conditionally disabled. -- [x] Startup asset validation now ignores retail Quake 4 game-binary PK4 archives (`game000.pk4` through `game300.pk4`, plus `gamex*.pk4` variants), so openQ4 only requires and verifies the stock media PK4s it actually uses. +- [x] Startup asset validation ignores retail Quake 4 game-binary PK4 archives (`game000.pk4` through `game300.pk4`, plus `gamex*.pk4` variants), so openQ4 only requires and verifies the stock media PK4s it actually uses. Pure multiplayer retains the official `game300.pk4` checksum solely as the protocol 2.41 compatibility token for trusted local openQ4 modules; the archive itself remains optional and is never loaded as game code. - [x] Retail decl loading parity improved: Quake 4 decl folders now honor retail recursion/unique-file behavior, no-cache decl lookups reach entityDef inheritance/media caching, legacy Doom 3 PDA/email/video/audio console commands are no longer exposed as supported decls, and playback/table/lip-sync/material-type decl parsing plus playback runtime sampling/record hooks now handle more shipped Raven asset data safely. - [x] Packed decl-manager parity advanced: the retail single-decl-file API is enabled again across engine/game headers, packed `.decls` sections can be read and written through controlled cvars/commands, stored decl indices are preserved during load, validation/allocation/tool hooks are available, packed stub decls expand lazily from their original source text, PDA/email/video/audio decl families are included in the framework packed section, and the startup/session/network asset-log wiring now selects and refreshes per-map packed decl files when the retail cvars are enabled. - [x] Packed decl asset-log wiring now follows the retail `assetlogs/` naming contract, including entity-filter suffixes, while the decl manager preserves the real `maps/...` path when selecting per-map `.decls` files. diff --git a/docs/dev/releases/v0.12.0.md b/docs/dev/releases/v0.12.0.md index 22eec21d..38ee2bb2 100644 --- a/docs/dev/releases/v0.12.0.md +++ b/docs/dev/releases/v0.12.0.md @@ -7,7 +7,10 @@ - **The marine hovertank has its vehicle audio back.** Single-player once again creates and updates the engine and hover-pad loops used throughout the vehicle sequence, with safe cleanup during map or engine shutdown. - **Arena Campaign scoring starts reliably.** Disabling ready-up now also clears the ready threshold, preventing deathmatch-backed Arena cards from remaining forever in a non-scoring warmup. - **Multiplayer setup is easier to control.** The server menu exposes guarded bot controls, and competitive free-for-all disclosure now accepts the active recipients intended by the match policy. +- **Multiplayer function keys work as shown.** F1/F2 voting, F3 ready-up, F6 team switching, and F7 spectating once again use Quake 4's real impulse actions. Ready presses travel over the reliable path, the two-line warmup instruction remains fully visible, and only the exact older openQ4 defaults are upgraded—custom bindings stay untouched. No-time-limit deathmatch also starts without a spurious competitive-rules rejection. - **Developer map compilation is more reliable.** `dmap` and other engine-side geometry tools initialize their triangle-surface allocator before generating light volumes, fixing the Windows crash seen while compiling `game/airdefense1`. +- **Liquids now look, sound, and behave like real volumes in combat.** Projectiles and hitscan shots cross the surface with a splash and sound, underwater travel produces bubble trails, clear water stays readable, and swimmers can reliably water-jump out over a clear ledge. Drowning, slime, and lava have distinct localized death-feed icons and messages. The multiplayer Liquid Volume Lab makes everything easy to inspect across deep, shallow, and wading water, vivid cellular slime, and bright heat-hazed, steaming lava, with restrained underwater ambience and boiling hazardous surfaces. +- **Internet multiplayer has safer administration, package negotiation, packet handling, and pure play.** The default remote console no longer sends its password over the network, private password settings stay out of console completion and persistence paths, and unauthenticated replies are bounded. A server can now update only the userinfo or synchronized-setting class owned by each message, and a truncated settings dictionary changes nothing. Server-provided package links are limited to bounded HTTP or HTTPS URLs with syntax-checked hosts; validated web redirects remain available, while standard packages keep in-process direct PK4 transfer disabled. Truncated covered messages and snapshots now fail safely and a malformed snapshot ends the affected session before another frame, the obsolete executable updater can no longer download or launch code, and pure servers keep asset checks enabled while game modules remain trusted local package components rather than server-supplied downloads. ## Upgrade Notes @@ -15,6 +18,8 @@ - Existing settings and compatible saves do not need to be reset. As a precaution, retain a copy of important saves before replacing an older installation. - Retail Quake 4 assets are still required. Point openQ4 at a complete retail installation, including the unsuffixed base dialogue archive such as `zpak_english.pk4`. - OpenGL remains the default and recommended renderer. Vulkan is experimental and opt-in with `r_renderApi vulkan`, applied after an engine restart. The modern OpenGL visible path also remains guarded and experimental. +- Remote administration uses authenticated `rcon2` by default. Legacy plaintext rcon remains available only through explicit client/server compatibility settings and should be limited to a trusted network; use a strong unique password and do not put it on the process command line. +- Servers that use `si_pure 1` now keep it enabled. The default `net_serverAllowServerMod 0` permits content-only mods to inherit the base module but rejects a mod-supplied game module; set it to `1` only when intentionally operating and separately distributing a trusted code-bearing mod. The protocol 2.41 game-code checksum is a compatibility token for an already installed local module, not a cryptographic check that client and server module bytes match and not an anti-cheat system. - Linux ARM64 downloads remain preview packages. The experimental macOS packages target Apple Silicon/arm64 and may be unsigned when Apple signing and notarization credentials are unavailable. Intel Mac packages are not published or supported, and Rosetta is not a supported substitute. OpenGL remains the recommended renderer; the Vulkan option runs through MoltenVK, a Vulkan-on-Metal translation layer. The `macos_graphics_bridge=metal` package is a Metal bridge around the OpenGL renderer, not a native Metal renderer, while `platform_backend=native` is comparison-only diagnostic infrastructure, not a release backend. Visual parity remains tracked by issue #98, and support promotion remains gated by `docs/dev/macos-signoff-evidence.md`. ## Change Log @@ -28,6 +33,17 @@ - Restored Arena Campaign countdown and scoring progression when ready-up is disabled. - Added multiplayer bot controls and validation to the server setup menu. - Allowed the intended active free-for-all recipients through competitive match disclosure policy. +- Restored the stock F1/F2/F3/F6/F7 impulse bindings, aligned the controls menu's vote/ready rows, routed the throttled ready toggle through the reliable operation, kept the two-line readiness prompt inside its authored HUD bounds, narrowly migrated only the exact historical openQ4 defaults, and normalized no-time-limit casual matches to sudden-death overtime with a zero period. - Corrected `s_muteUnfocused` transitions so returning to the game cannot leave audio muted for the rest of the level. - Initialized engine-side triangle-surface allocators for `dmap` and related geometry tools. +- Restored clean, fixed-size manual save-game previews at widescreen resolutions and prevented renderer row padding from leaking into narrow captures. +- Replaced plaintext-by-default remote console authentication with bounded `rcon2`, redacted private password CVars throughout console and persistence paths, restricted server-originated CVar dictionaries to their declared userinfo/network-sync authority with all-or-nothing decoding, constrained server package URLs to bounded HTTP/HTTPS with syntax-checked hosts, and retired executable download/launch behavior from the legacy updater. Standard packages keep direct PK4 transfer disabled; separately integrated curl-enabled builds add time-limited, no-redirect transfer containment. +- Restored `si_pure` multiplayer enforcement with ordered asset-PK4 checks and a platform-independent Quake 4 1.4.2 protocol 2.41 compatibility token for already installed openQ4 game modules; pure negotiation cannot download, extract, or restart into executable code, and the token is not cryptographic module attestation. +- Added fail-closed read-underflow tracking and range/type checks to audited queued-message, user-command, SP/MP snapshot, server-demo, player, projectile, and hit-scan decode paths. Audited leaf readers stage their fields until valid, and a malformed top-level snapshot tears down the session before another game or presentation frame. +- Made Windows staging self-cleaning for explicitly known cross-platform leftovers: extensionless POSIX engine binaries and obsolete renderer backup files are removed from successful full and fast stages, while the legacy empty `baseoq4/skins` directory is pruned only when it contains nothing. +- Added the well-lit `mp/liquid_lab` multiplayer showcase with recessed pools, a sky ceiling, shadow-casting beams, deep-pool floor lights, vivid animated slime, bright lava heat haze and steam, bubbling hazardous surfaces, submerged local ambience, and full-arsenal respawns. +- Let projectiles and hitscan weapons pass through liquid surfaces while retaining reliable entry splashes and sounds; submerged projectile trails and hitscan segments now render as liquid-specific bubbles. +- Restored practical water jumps with Quake 4-height ledge probes, gravity-aware launch arcs, and an escapable wading basin. +- Brightened clear-water visibility, removed hall-like underwater reverb tails, and debounced rapid surface sounds without suppressing their splash effects. +- Added localized drowning, slime, and lava obituaries with distinct graphical death-feed icons and normal burn feedback for slime. - Strengthened the experimental modern OpenGL lighting path with per-light image ownership and missing-binding fallbacks. diff --git a/docs/dev/renderer-validation-matrix.md b/docs/dev/renderer-validation-matrix.md index 26fcf65c..cf61761a 100644 --- a/docs/dev/renderer-validation-matrix.md +++ b/docs/dev/renderer-validation-matrix.md @@ -2,6 +2,8 @@ This matrix is the validation source of truth for the staged GL renderer work. It separates safe automated startup/self-test coverage from gameplay smoke coverage that must be run manually with the mode-specific SP/MP launch tasks. +For cross-engine feature status, use the [engine capability matrix](engine-capability-matrix.md). This document owns renderer acceptance evidence and promotion gates; it does not turn an experimental renderer capability into a supported/default one by itself. + ## Build And Stage Use the project wrapper: @@ -79,6 +81,8 @@ The visible-depth, G-buffer, clustered-light, deferred-resolve, forward+, modern The shader-library tier cases force `r_glTier gl33`, `gl41`, `gl43`, `gl45`, and `gl46`, run `rendererShaderLibrarySelfTest`, and require `gfxInfo` to report `Modern GL shader library: available` with program, kind, permutation, and sampler-reflection coverage. The runner marks these cases as assetless startup probes, because they only need renderer initialization and should not load game scripts just to validate internal shader variants. +The foundation self-test case also runs `rendererPBRMaterialSelfTest`, while `rendererScenePacketSelfTest` and `rendererMaterialResourceTableSelfTest` include the matching PBR packet/resource cases. Together these assetless contracts verify that opt-in `pbr {}` metadata leaves classic Quake 4 stages untouched, image usage and scalar registers survive parsing, explicit and approximate classic fallbacks are classified deterministically, packet records preserve PBR metadata, and packed, separate, scalar-only, unsupported-workflow, and missing-map resource records fail closed with observable reasons. They also require `pbrModernReady=0` and exclude PBR bindings from current classic-modern submission. They do not exercise PBR G-buffer shaders, direct lighting, visible ownership, IBL, or specular environment probes. + Gameplay benchmark acceptance should use wall-clock sampling for FPS claims. The `--sample-msec` option emits `waitMsec` into the generated cfg so the measurement window is a real duration rather than a frame count: ```powershell diff --git a/docs/dev/settings-menu-registry.json b/docs/dev/settings-menu-registry.json index 46a56b88..2cc95d42 100644 --- a/docs/dev/settings-menu-registry.json +++ b/docs/dev/settings-menu-registry.json @@ -914,7 +914,7 @@ "widget_type": "bind", "label_key": "#str_200129", "target_type": "bind", - "target": "voteyes", + "target": "_impulse28", "default": "F1", "default_keys": [ "F1" @@ -936,7 +936,7 @@ "widget_type": "bind", "label_key": "#str_200130", "target_type": "bind", - "target": "voteno", + "target": "_impulse29", "default": "F2", "default_keys": [ "F2" @@ -980,7 +980,7 @@ "widget_type": "bind", "label_key": "#str_201009", "target_type": "bind", - "target": "ready", + "target": "_impulse17", "default": "F3", "default_keys": [ "F3" diff --git a/docs/dev/source-provenance-manifest.json b/docs/dev/source-provenance-manifest.json new file mode 100644 index 00000000..e3d8a8fc --- /dev/null +++ b/docs/dev/source-provenance-manifest.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "auditedOn": "2026-08-19", + "scope": "Tracked source files under src/ whose retained id Software header identifies a covered source family", + "textHashNormalization": "UTF-8; CRLF and CR normalized to LF", + "families": { + "doom3": { + "displayName": "Doom 3 GPL Source Code", + "headerMarker": "Doom 3 GPL Source Code", + "expectedFileCount": 581, + "officialRepository": "https://github.com/id-Software/DOOM-3", + "auditedCommit": "a9c49da5afb18201d31e3f0a429a037e56ce2b9a", + "officialCopying": "https://github.com/id-Software/DOOM-3/blob/a9c49da5afb18201d31e3f0a429a037e56ce2b9a/COPYING.txt", + "officialCopyingSha256": "f83520e077c35722a56889cd54da8e46105e31278f8f97fa39318957e865c5c7", + "localAdditionalTerms": "LICENSES/DOOM-3-ADDITIONAL-TERMS.txt", + "localAdditionalTermsSha256": "0745111e721bad901aa67e16af761f35598e2d92f971a134ef732d3519b09a3e" + }, + "doom3_bfg": { + "displayName": "Doom 3 BFG Edition GPL Source Code", + "headerMarker": "Doom 3 BFG Edition GPL Source Code", + "expectedFileCount": 37, + "officialRepository": "https://github.com/id-Software/DOOM-3-BFG", + "auditedCommit": "1caba1979589971b5ed44e315d9ead30b278d8b4", + "officialCopying": "https://github.com/id-Software/DOOM-3-BFG/blob/1caba1979589971b5ed44e315d9ead30b278d8b4/COPYING.txt", + "officialCopyingSha256": "1689ee84a23d3985c7478b10080f45d10fef6f2945ddd2b67777d1990defe5c8", + "localAdditionalTerms": "LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt", + "localAdditionalTermsSha256": "de40806c9c1ab3cd2c180cc18cdc37c9b896f3e4ea4818739baad78a0abc45b2", + "officialPathRule": "src/ maps to neo/, except src/imagetools/ maps to neo/renderer/", + "intermediateSources": { + "rbdoom3_bfg": { + "repository": "https://github.com/RobertBeckebans/RBDOOM-3-BFG", + "auditedCommit": "ea29c006e84fedcc0e7c173c383a087ce0a8c0d5", + "pathOverrides": { + "src/sound/OpenAL/AL_SoundHardware.cpp": "neo/sound/OpenAL/AL_SoundHardware.cpp", + "src/sound/OpenAL/AL_SoundHardware.h": "neo/sound/OpenAL/AL_SoundHardware.h", + "src/sound/OpenAL/AL_SoundSample.cpp": "neo/sound/OpenAL/AL_SoundSample.cpp", + "src/sound/OpenAL/AL_SoundSample.h": "neo/sound/OpenAL/AL_SoundSample.h", + "src/sound/OpenAL/AL_SoundVoice.cpp": "neo/sound/OpenAL/AL_SoundVoice.cpp", + "src/sound/OpenAL/AL_SoundVoice.h": "neo/sound/OpenAL/AL_SoundVoice.h" + } + } + } + } + } +} diff --git a/docs/dev/source-provenance.md b/docs/dev/source-provenance.md new file mode 100644 index 00000000..50af9e72 --- /dev/null +++ b/docs/dev/source-provenance.md @@ -0,0 +1,49 @@ +# Source Provenance and Accompanying Terms + +This is the authoritative engineering inventory for retained id Software source headers under openQ4's `src/` tree. It records what the tree says about its origin and makes the accompanying notices available; it is not a legal opinion or a conclusion about licence compatibility. + +The repository-wide [GPLv3 text](../../LICENSE) remains the primary licence file. Two sets of tracked files also retain headers that expressly refer to a distinct set of Additional Terms: + +| Header family | Tracked files at the 2026-08-19 audit | Audited official source snapshot | Accompanying terms in this tree | +|---|---:|---|---| +| Doom 3 GPL Source Code | 581 | [id-Software/DOOM-3 `a9c49da`](https://github.com/id-Software/DOOM-3/tree/a9c49da5afb18201d31e3f0a429a037e56ce2b9a) | [`LICENSES/DOOM-3-ADDITIONAL-TERMS.txt`](https://github.com/themuffinator/openQ4/blob/master/LICENSES/DOOM-3-ADDITIONAL-TERMS.txt) | +| Doom 3 BFG Edition GPL Source Code | 37 | [id-Software/DOOM-3-BFG `1caba19`](https://github.com/id-Software/DOOM-3-BFG/tree/1caba1979589971b5ed44e315d9ead30b278d8b4) | [`LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt`](https://github.com/themuffinator/openQ4/blob/master/LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt) | + +The two Additional-Terms files preserve the wording published in the corresponding official `COPYING.txt`, including upstream spelling errors; only character encoding, line endings, and insignificant trailing whitespace are normalized for the repository. They are separate because their scope and published text identify different source releases. The machine-readable audit pins the official repositories, full commit object IDs, complete upstream `COPYING.txt` hashes, and canonical UTF-8 local notice hashes in [`source-provenance-manifest.json`](https://github.com/themuffinator/openQ4/blob/master/docs/dev/source-provenance-manifest.json). The local hash normalizes CRLF/CR to LF so Git's checkout policy cannot create a false mismatch; all other text changes fail. + +The official Doom 3 BFG repository describes its release as GPL source with omissions for Steam integration, Bink playback, and the depth-fail stencil-shadow implementation. Those omissions are a boundary on what can be copied from that release; they are not an invitation to reconstruct excluded code from non-source binaries. Quake 4 retail assets are not included in either source release and are not relicensed by openQ4. + +## Reproducible inventory + +Run the offline audit from the repository root: + +```text +python tools/validation/audit_source_provenance.py --check +python tools/validation/audit_source_provenance.py --family doom3_bfg +python tools/validation/audit_source_provenance.py --format json --output .tmp/source-provenance.json +python tools/validation/audit_source_provenance.py --check --doom3-source E:\_SOURCE\_CODE\DOOM-3-master --doom3-bfg-source E:\_SOURCE\_CODE\DOOM-3-BFG-master --rbdoom3-bfg-source E:\_SOURCE\_CODE\RBDOOM-3-BFG-master +``` + +The JSON report is the exact current file inventory. The check fails when a tracked header referring to Additional Terms is unknown, either accompanying notice is missing or differs from its canonical local SHA-256, or a family count changes without an explicit manifest review. When official source roots are supplied, the audit also checks the raw published bytes of each complete `COPYING.txt` against its separate official hash; the UTF-8/newline normalization applies only to the extracted local notice files. A header-family match establishes only that the local file retains that notice. It does not claim byte identity with the pinned official snapshot or reconstruct the complete chain of intermediate forks. + +The 37 BFG-marked files currently occupy these groups: + +| Classification | Files | Audited path source | +|---|---:|---| +| Path exists in official Doom 3 BFG snapshot `1caba19` | 31 | `neo/`; local `src/imagetools/` corresponds to official `neo/renderer/` | +| Path absent from the official snapshot; BFG-header lineage is present in RBDOOM-3-BFG snapshot `ea29c00` | 6 | `src/sound/OpenAL/AL_Sound{Hardware,Sample,Voice}.{cpp,h}` at `neo/sound/OpenAL/` | + +The audit prints every file with its classification. For the six OpenAL files, the pinned RBDOOM tree is an intermediate lineage reference, not proof of the exact commit from which openQ4 first received the file. The optional source-root arguments verify that every configured path actually exists in the identified reference tree. Most Doom 3-marked files came through the historical idTech 4/Quake 4 lineage and do not necessarily have a one-to-one path in the official Doom 3 snapshot, so the inventory intentionally does not invent such a mapping. + +## Import policy + +Before incorporating more external code: + +1. Use an official or otherwise auditable source revision and record its repository, immutable commit, path, and licence family. +2. Verify that every required licence and notice is present. Do not remove or rewrite an upstream header merely to make the tree look uniform. +3. Mark the openQ4 version as altered in the change and its documentation; never represent it as the original upstream program. +4. Add the new local path to the reproducible inventory by updating the expected count after reviewing the audit diff. +5. Add an elegant upstream credit in the relevant documentation or README. +6. Do not import excluded third-party code, game data, proprietary SDK binaries, or source recovered from a retail executable. + +The companion `openQ4-game` repository is a separate provenance boundary: its Quake4SDK-derived game-library source remains subject to the Quake 4 SDK EULA. This document inventories this engine repository only. diff --git a/docs/dev/stock-asset-baseline.md b/docs/dev/stock-asset-baseline.md new file mode 100644 index 00000000..7dacabe3 --- /dev/null +++ b/docs/dev/stock-asset-baseline.md @@ -0,0 +1,159 @@ +# Retail-PK4 Compatibility Baseline With Packaged openQ4 Overlays + +[`stock_asset_baseline.py`](https://github.com/themuffinator/openQ4/blob/master/tools/validation/stock_asset_baseline.py) is the authoritative P0 capture harness for Quake 4 retail-asset compatibility. It binds an evidence run to the exact retail PK4 bytes and approved-manifest file, selected client, packaged renderer modules, SP/MP game modules, other packaged shared libraries, openQ4 overlay packages, current openQ4 Git revision and dirty-state policy, logs, saves, demos, and engine-rendered screenshots. + +This is not an overlay-free or “stock-only” execution claim. `baseoq4` is the active game directory and its packaged openQ4 PK4s take normal virtual-filesystem precedence over the verified retail `q4base`/`q4mp` fallback. The report preserves both facts: retail archive bytes must match the separately approved manifest, while every packaged-overlay member that supersedes a retail virtual path is inventoried and counted. + +The harness is deliberately non-interactive: + +- every client is forced to bordered, non-desktop `r_fullscreen 0` / + `r_borderless 0` presentation at a fixed `r_windowWidth`/`r_windowHeight`; +- map actions are registered console commands in generated cfg files; +- images are produced by the engine `screenshot` command from the render target; +- it does not call an operating-system capture API; +- it does not control, inject, or capture mouse or keyboard input; +- every role uses an isolated save path below the chosen evidence directory. +- the staged package is mounted through the engine's working-directory + `fs_cdpath`, while writable `fs_devpath` output is confined to that role's + isolated evidence tree. + +## Evidence sequence + +| Role | Automated evidence | +|---|---| +| SP capture | Loads stock `game/storage1`, records a render demo, takes a full-size engine screenshot immediately before writing `StockBaselineSP`, loads that save, waits for an active restored draw, emits renderer/frame-pacing information, and takes a separate post-load engine screenshot. | +| SP demo playback | Reopens the recorded demo with `timeDemoQuit`; a completed timedemo line and clean exit prove that the recorded stream can be read, not merely that a file exists. | +| MP server | Starts a windowed `mp/q4dm1` listen server, accepts the loopback run, records a demo, emits diagnostics, and takes an engine screenshot. | +| MP client | Connects through IPv4 loopback with the existing archived `ui_autoJoin 1` userinfo setting, proves the local player is in-game and neither spectating nor requesting spectate, proves the session GUI is closed and the HUD is enabled, records a demo, emits diagnostics, and takes its own gameplay screenshot. | + +Each role must exit normally, reach its explicit completion marker, avoid the harness's blocking diagnostic denylist, and produce non-empty expected artifacts. That denylist is limited to fatal errors, line-start engine `ERROR` records, shader compile/program-link failures, Vulkan validation messages or VUIDs, and OpenGL errors. Other warnings are retained in the evidence but do not by themselves fail the baseline, so an automated pass is not a warning-free-log claim. The MP client also requires exact active-player proofs on both sides of the screenshot; those proofs include closed game/session menu state and an enabled HUD, so a joined player hidden behind the JOIN GAME screen cannot pass. TGA screenshots must fully decode, contain an exact uncompressed 24/32-bit payload, exactly match the renderer dimensions recorded by `gfxInfo`, and avoid blank or recursive scaled-strip corruption. The SP save preview is held to the same pixel-content checks and the retail 320x240 dimensions. It must also remain visually coherent with the full-size engine screenshot issued immediately before `saveGame`, while simulation state is unchanged. The later post-load screenshot remains separately required and proves that the restored game reaches a renderable gameplay frame; it is deliberately not the preview-comparison reference because the validation waits and renderer measurements allow the live scene, vehicle pose, and effects to advance. The harness centre-aspect-crops and bilinearly downsamples the same-state reference and preview to 80x60, then requires a luma correlation of at least 0.75. For the secondary colour check, it fits one shared, bounded affine exposure transform from preview luma to reference luma (gain 0.25–4.0 and bias -192–192), applies that same transform to all three colour channels, and requires the remaining mean absolute RGB error to be no greater than 48. A single shared transform tolerates capture-path exposure and contrast changes without concealing an arbitrary per-channel colour defect. Raw RGB error, fitted exposure parameters, and compensated error are all recorded for review. The combined correlation, colour, and recursive-strip gates deliberately permit modest post-process differences while rejecting structurally valid feedback/recursion captures. The report hashes every artifact with SHA-256. + +Both MP roles explicitly launch with `ui_autoJoin 1`. This follows the normal +game userinfo/server-policy path and avoids automating menu input. All openQ4 +MP validation should keep auto-join enabled unless the subject of the test is +the join menu or the initial spectator/join flow itself. Such a test must set +`ui_autoJoin 0` explicitly rather than relying on omission because the CVar is +archived. + +The MP server also runs with the normal `si_pure 1` policy and with +`net_serverAllowServerMod 0`. A baseline therefore has to complete the real +pure-PK4 negotiation; disabling pure mode or using the legacy server-mod escape +hatch cannot produce promotable compatibility evidence. + +Before reporting success, the harness re-hashes the retail assets, staged +runtime and overlays, and all captured artifacts. It also recomputes the SP +save-preview comparison against the same-state pre-save screenshot and requires +the recorded reference artifact, algorithm, thresholds, and metrics to match, +so updating an image's recorded hash cannot hide an incoherent preview. A run that mutates the staged +package or any other recorded input therefore fails in the same invocation. + +`--verify-report` fails closed unless the report represents a real capture +(`dryRun` is exactly `false`), every top-level and per-role failure array is +empty, the bound expected-assets file still exists at its recorded absolute +path and has its recorded SHA-256, and that manifest's retail inventory matches +the report. The verifier also requires the recorded openQ4 HEAD and dirty flag +to match the current checkout under the named provenance policy. A dirty flag +records only that uncommitted changes exist; it is not a fingerprint of those +changes, so promotion evidence should come from a clean checkout. + +Capture and dry-run output directories must be new or empty. The harness will not reuse earlier logs, screenshots, saves, or demos. Both process streams are captured and hashed even when empty; report verification requires the exact four roles, their lifecycle fields, completion markers, artifact kinds and paths, and unchanged engine log/stdout/stderr bytes. It reconstructs and compares the complete canonical argument list for every role, including `fs_game`, SP map/game type, MP listen map/game type/dedicated mode, loopback connection, `ui_autoJoin 1`, and pure/server-mod policy. The per-role engine logfile is authoritative for ordered lifecycle markers and MP screenshot-bracketing proofs. This avoids false duplicate-proof failures on POSIX builds that mirror identical engine diagnostics to stdout; stdout and stderr remain independently hashed and all three channels are scanned for the same blocking denylist described above, while general warnings remain review evidence rather than automatic failures. + +The timeout is applied independently to each SP role. Multiplayer uses one absolute timeout deadline beginning when the listen server is launched; the server and loopback client are monitored together and any roles still running at that deadline are terminated together. The client-start delay consumes part of that shared MP budget rather than granting either process a second sequential timeout. + +## Run it + +List the fixed cases and safety invariants without touching assets or launching: + +```text +python tools/validation/stock_asset_baseline.py --list +``` + +First create a PK4-only view. A normal Steam tree is not suitable because saves, +generated collision caches, extracted maps, or mod files can override the retail +archives even when every loose byte is hashed: + +```powershell +$source = "C:\Program Files (x86)\Steam\steamapps\common\Quake 4" +$assetRoot = ".tmp\stock-assets-pk4-only" +New-Item -ItemType Directory -Force "$assetRoot\q4base", "$assetRoot\q4mp" +Copy-Item "$source\q4base\*.pk4" "$assetRoot\q4base\" +Copy-Item "$source\q4mp\*.pk4" "$assetRoot\q4mp\" +``` + +Write the complete plan and the separately reviewed expected-assets manifest +without launching the game: + +```text +python tools/validation/stock_asset_baseline.py --dry-run --asset-root .tmp\stock-assets-pk4-only --output-dir .tmp\stock-baseline\approved +``` + +Capture the baseline. Every non-dry capture must bind to an approved manifest +from the same retail edition/language set; a missing binding, archive mismatch, +or loose q4base/q4mp file fails before openQ4 launches: + +```text +python tools/validation/stock_asset_baseline.py --asset-root .tmp\stock-assets-pk4-only --expected-assets .tmp\stock-baseline\approved\stock_pk4_manifest.json --output-dir .tmp\stock-baseline\candidate +``` + +By default the harness hashes and launches the canonical repository `.install` +package. If that directory is occupied by an unrelated running test, stage one +fresh, ordinary alternate package below `.tmp/stock-runtime/` with the canonical +fast-staging tool, then name that exact directory explicitly for capture and +verification: + +```text +python tools/build/stage_fast_install.py --build-dir builddir --install-dir .tmp/stock-runtime/current-build --temporary-runtime +python tools/validation/stock_asset_baseline.py --runtime-dir .tmp/stock-runtime/current-build --asset-root .tmp/stock-assets-pk4-only --expected-assets .tmp/stock-baseline/approved/stock_pk4_manifest.json --output-dir .tmp/stock-baseline/candidate +python tools/validation/stock_asset_baseline.py --runtime-dir .tmp/stock-runtime/current-build --asset-root .tmp/stock-assets-pk4-only --verify-report .tmp/stock-baseline/candidate/stock_asset_baseline_report.json +``` + +Temporary staging refuses an existing destination, links/junctions, or a path +outside `.tmp/stock-runtime/`. The report and runtime manifest record its +resolved root, and every collector, launch working directory, post-capture +rehash, and later verifier uses that same root. This proves the recorded current +build against verified retail assets plus its packaged openQ4 overlays; it does not replace the separate release-packaging +gate that restages and validates canonical `.install`. + +Re-hash an existing report's retail assets, bound expected-assets manifest, openQ4 runtime/overlays, current source provenance, exact launch contract, and recorded artifacts without launching. Keep the evidence directory and expected-manifest file at their recorded absolute locations and verify from the same openQ4 HEAD/dirty state: + +```text +python tools/validation/stock_asset_baseline.py --asset-root .tmp\stock-assets-pk4-only --verify-report .tmp\stock-baseline\candidate\stock_asset_baseline_report.json +``` + +The output directory contains: + +- `stock_asset_baseline_report.json`: full machine-readable plan, identities, results, artifact hashes, compatibility model, and retail/overlay path-collision inventory; +- `stock_asset_baseline_report.md`: concise reviewer report, collision counts by packaged overlay, and manual gates; +- `stock_pk4_manifest.json`: portable retail PK4 identity manifest; +- `openq4_runtime_manifest.json`: resolved runtime root plus the selected client, dedicated server, diagnostic symbols, all packaged renderer/game modules and shared libraries, overlay PK4s, and loose overlay files; +- `savepaths/`: isolated cfg files, engine logs, screenshots, demos, and saves; +- per-role stdout/stderr text. + +The retail manifest hashes every top-level `.pk4` under `q4base/` and `q4mp/`; any loose non-PK4 file is a hard failure. It establishes byte identity with the separately supplied approved manifest; by itself it does not establish ownership or authenticity. The runtime manifest separately hashes the selected client, dedicated server, diagnostic symbols, every packaged dynamic library under the recorded runtime root (including renderer modules, both game modules, and packaged dependencies such as OpenAL), every top-level `baseoq4/*.pk4`, and every loose file recursively visible under its `baseoq4/`. The verifier opens the retail and packaged-overlay PK4s as ZIP archives, compares their case-insensitive idTech virtual paths, and requires the recorded path-by-path collision inventory and count to match the current packages. + +The baseline fixes `r_renderApi gl`, requires the OpenGL renderer module on non-macOS packages, and records any additional packaged renderer module. Only the SP/MP modules, their top-level Windows symbol sidecars, and `mod.json` may be loose below the recorded runtime's `baseoq4/`; any other loose overlay file fails preflight before launch. Symbol sidecars are hashed but are not VFS game content. This prevents an unreported map, material, GUI, or script from weakening the declared retail-fallback-plus-packaged-overlay model. + +The harness fails closed when the supplied asset root has a non-empty `baseoq4/` directory. With `fs_game=baseoq4`, that second asset-root overlay would silently alter the explicitly inventoried precedence model. Use a clean retail fallback view containing only top-level retail PK4s in `q4base/` and `q4mp/`. The tool rejects every loose retail file, and by default rejects links at or inside those trees so recursive inventory cannot be bypassed. + +A boundary junction is accepted only with explicit opt-in and only when its +resolved directory has already been made PK4-only. Never junction the normal +Steam q4base/q4mp directories: loose generated or extracted files in those +trees take precedence over shipped archive members. + +```powershell +python tools/validation/stock_asset_baseline.py --allow-asset-dir-links --asset-root .tmp\stock-assets-linked-pk4-only --expected-assets .tmp\stock-baseline\approved\stock_pk4_manifest.json --output-dir .tmp\stock-baseline\candidate +``` + +The explicit flag permits links only at the `q4base`/`q4mp` boundary. Their resolved targets are recorded in the JSON, Markdown, and retail manifests; links nested inside either content tree and every loose file still fail closed. + +## Promotion gate + +An automated `pass` establishes the artifact and lifecycle contract only. Before using a bundle as the release baseline, a reviewer must still: + +1. inspect the engine screenshots for black frames, broken materials, missing effects, bad UI/subviews, and obvious shadow or post-process failures; +2. play representative SP and MP sequences to assess input, audio, scripting, collision, prediction, and subjective presentation; +3. capture from a clean source checkout, then retain the approved JSON report and PK4 manifest with the tested binary or immutable build identifier; +4. verify the final, freshly staged release package rather than treating a debug/development staging tree as release proof; +5. record platform/driver coverage separately rather than treating one host as universal qualification. + +The broader renderer visual/performance promotion rules remain in the [renderer validation matrix](renderer-validation-matrix.md). This P0 harness is the compatibility identity and lifecycle baseline, not a replacement for renderer-specific image references or target-hardware performance evidence. diff --git a/docs/user/server-security.md b/docs/user/server-security.md new file mode 100644 index 00000000..b2b448dd --- /dev/null +++ b/docs/user/server-security.md @@ -0,0 +1,172 @@ +# Server and Remote-Console Security + +openQ4 uses an authenticated remote-console protocol, `rcon2`, by default. +It replaces Quake 4's legacy packet, which sent the administration password +verbatim over UDP. Gameplay, server discovery, the connection handshake's wire +layout, and the retail Quake 4 asset format are unchanged. + +## Configure a strong password + +Set the same password on the server and on the administrator's client: + +```text +net_serverRemoteConsolePassword "a-long-random-password" +net_clientRemoteConsolePassword "a-long-random-password" +``` + +Then issue a command from the client in the usual form: + +```text +rcon status +``` + +`rcon2` requires at least 12 password bytes. Use a unique, randomly generated +password of 24 or more characters for an Internet-facing server. Do not reuse a +login, referee, database, or service password. + +Enter the password at a trusted local console, or put the server-side setting +in a configuration file readable only by the account that runs the server. +Do not commit that file, share it with a package, or use the password on the +process command line. In particular, never launch with +`+set net_serverRemoteConsolePassword ...` or +`+set net_clientRemoteConsolePassword ...`: launch arguments can be visible to +other local users, process-monitoring tools, crash reports, and service logs. + +Private CVars are redacted from broad console listings and direct console +queries, omitted from generic CVar serialization and dictionaries, blocked +from `$` CVar expansion, and suppressed from console echo, command history, +completion previews, startup-command reporting, and event journals. These +protections reduce accidental disclosure; they do not make an insecure +configuration file safe. + +Server-originated userinfo and synchronized-CVar messages have separate, +explicit authority. Each can update only registered CVars carrying the matching +network flag; unrelated and private settings are ignored. Dictionary decoding +is all-or-nothing, so a truncated update does not leave a partially changed +settings set behind. + +## Server-provided package links + +Pure-server redirects and PK4 download entries are accepted only as bounded +`http://` or `https://` URLs. Their authority must contain a syntactically valid +DNS name, IPv4 literal, or bracketed IPv6 literal; malformed labels, ports, +credentials, ambiguous authorities, control characters, and every other URL +scheme (including `file:` and SMB-style paths) are rejected before prompting or +queueing. The generic background downloader applies the same check again. + +Standard Meson packages do not enable libcurl. They can show a validated web +redirect offered by a server, but in-process direct PK4 transfer reports +unavailable; obtain any required package separately from a source you trust. +This is intentional and should not be diagnosed as a broken downloader. + +If a separately integrated build deliberately enables libcurl, package +transfers are limited to HTTP(S), do not follow redirects, time out if a +connection cannot be established within 15 seconds, stop after 30 seconds below +1 KiB/s, and have a one-hour absolute cap. Advertised and received sizes, +destination paths, and package checksums retain their separate validation. +These restrictions bound that optional legacy path and prevent +local-file/protocol substitution; they do not prove DNS ownership, make an +untrusted server trustworthy, or certify arbitrary package content. + +The platform URL opener follows the same HTTP(S)-only syntax policy on Windows, +Linux, and macOS. In particular, macOS no longer accepts local `file:` URLs. + +## What rcon2 protects + +The server issues short-lived, one-use random challenges. The proof binds the +client and server nonces, the exact UDP endpoint, a server binding value, and a +SHA-256 digest of the requested command. The server derives its proof verifier +with PBKDF2-HMAC-SHA-256 at 200,000 iterations, compares proofs without an +early-exit timing leak, consumes a challenge before accepting or rejecting the +proof, and rate-limits both individual sources and global unauthenticated reply +traffic. Connection challenges, client IDs, server IDs, download-request IDs, +and rcon2 nonces use the operating system's cryptographic random source; an +operation fails closed if that source is unavailable. + +`rcon2` authenticates the command sender to the server, but it is **not an encrypted transport**. +The command and server output still travel in cleartext +UDP packets. A captured rcon2 exchange also permits PBKDF2-throttled offline password guessing. +PBKDF2 makes each guess more expensive; it does not rescue a +short, reused, or predictable password. Anyone able to observe or alter the +network path can read commands and output and can disrupt the exchange. Use a +trusted network or an encrypted tunnel/VPN when command or output confidentiality +matters. + +## Pure multiplayer and local game modules + +`si_pure 1` is supported and remains enabled when selected; openQ4 no longer +silently turns it off at server startup. Pure mode compares the ordered PK4 +asset list used by the server and client. The protocol can identify missing +asset PK4s, but standard packages do not enable in-process direct transfer. Any +separately integrated curl-enabled package path remains asset-only and never +supplies executable game code. + +The legacy protocol also carries a game-code checksum. openQ4 sends the official +Quake 4 1.4.2 `game300.pk4` checksum, `0x68fb90b1`, only as a compatibility +token. It does not require, mount, download, extract, or execute that archive. +The engine accepts the token only when the required `game_sp` or `game_mp` +module was already loaded from trusted local openQ4 package/module roots. A missing +module, an unsupported platform ID, an empty pure asset list, or a different +token fails closed instead of weakening pure mode or starting a code download. + +By default, `net_serverAllowServerMod 0` also prevents a selected mod from +supplying its own game module. Content-only mods can continue to use the base +openQ4 module. An administrator who intentionally operates a code-bearing mod +must set `net_serverAllowServerMod 1` and distribute a complete, trusted package +to clients separately; the setting does not make module code downloadable. + +The compatibility token preserves the stock 1.4.2 packet field across Windows, +Linux, and macOS. It is not a cryptographic measurement of the loaded module and +does not turn pure mode into an anti-cheat system. It also does not guarantee +that an arbitrary gameplay mod remains compatible with stock clients. + +## Malformed network traffic + +The shared bit-message reader records attempts to read beyond the received +payload, including through delta messages. Covered queue and user-command +decoders check their encoded sizes, while audited SP and MP leaf readers stage +decoded fields until their payload is valid and check referenced entity/player +types and bounds, spectator and weapon values, tournament instances, projectile +owners, PVS and game state, and hit-scan fields. A late malformed top-level +snapshot tears down the affected session before another game or presentation +frame. The legacy entity lifecycle is not a whole-snapshot rollback +transaction. + +This is targeted hardening of the audited legacy paths. It is not a claim that +every parser is formally verified, and malformed-packet regression tests do not +replace normal Internet-server isolation, operating-system updates, or careful +review of future protocol changes. + +## Legacy compatibility mode + +Legacy plaintext rcon is disabled on both sides by default. It remains only as +an explicit compatibility escape hatch for an old client or administration +tool: + +```text +// Server: accept legacy packets. +net_serverAllowLegacyRcon 1 + +// openQ4 client: send a legacy packet. +net_clientUseLegacyRcon 1 +``` + +An openQ4 client talking to an openQ4 server needs both settings to use the +legacy path. A third-party legacy tool needs the server setting; an openQ4 +client talking to an old server needs the client setting. Legacy mode sends the +password and command in plaintext and should be enabled only temporarily on a +trusted network. Return both variables to `0` when compatibility testing ends. + +Stock Quake 4 clients can still join and play on an openQ4 server because the +gameplay and connection packet layouts remain compatible. The intentional +administration compatibility change is that old plaintext-rcon tools receive no +response until the server operator explicitly enables the legacy path. + +## Standards and provenance + +The engine implementation is original GPL code based on the public algorithm +specifications, not on the Quake 4 SDK game-library implementation: + +- [FIPS 180-4: Secure Hash Standard (SHA-256)](https://csrc.nist.gov/pubs/fips/180-4/upd1/final) +- [RFC 2104: HMAC](https://www.rfc-editor.org/rfc/rfc2104) +- [RFC 8018: PBKDF2](https://www.rfc-editor.org/rfc/rfc8018) diff --git a/docs/user/server-setup.md b/docs/user/server-setup.md index a5a76edf..9c65e7a5 100644 --- a/docs/user/server-setup.md +++ b/docs/user/server-setup.md @@ -33,6 +33,9 @@ Example startup flow: openQ4-ded_x64 +set net_ip 0.0.0.0 +set net_port 28004 +set si_name "My openQ4 Server" +set si_map mp/q4dm1 +set si_gameType DM +spawnServer ``` +For remote administration, follow the [server and remote-console security +guide](server-security.md). Never put an rcon password in launch arguments. + ## IPv4 Binding and Ports For predictable hosting, set the interface and UDP port before starting the server: diff --git a/meson.build b/meson.build index 00c0a519..8cf22dca 100644 --- a/meson.build +++ b/meson.build @@ -207,7 +207,11 @@ endif if get_option('build_native_tests') openq4_core_safety_test = executable( 'openq4-core-safety-test', - files('tools/tests/native/CoreSafetyTest.cpp'), + files( + 'tools/tests/native/CoreSafetyTest.cpp', + 'src/idlib/CryptoHash.cpp', + 'src/framework/async/Rcon2Protocol.cpp', + ), include_directories: root_include_dir, install: false, ) diff --git a/src/framework/CVarSystem.cpp b/src/framework/CVarSystem.cpp index c2483b10..8da3f488 100644 --- a/src/framework/CVarSystem.cpp +++ b/src/framework/CVarSystem.cpp @@ -29,6 +29,9 @@ If you have questions concerning this license or the applicable additional terms +#include "../idlib/PrivateCommand.h" +#include "RemoteCVarPolicy.h" + idCVar * idCVar::staticVars = NULL; /* @@ -90,12 +93,31 @@ static void ArgCompletion_CvarName( const idCmdArgs &args, void(*callback)( cons s_cvarNameArgCompletionCommand = NULL; } +static void CVar_AssignString( idStr &target, const char *newValue, bool privateValue ) { + if ( newValue == NULL ) { + newValue = ""; + } + if ( !privateValue ) { + target = newValue; + return; + } + + // newValue can point into target (or another long-lived CVar string), so + // preserve it before clearing the complete current allocation. + idStr replacement( newValue ); + target.SecureClear(); + target = replacement; + replacement.SecureClear(); +} + /* ============ idInternalCVar::idInternalCVar ============ */ idInternalCVar::idInternalCVar( void ) { + flags = 0; + valueStrings = NULL; } /* @@ -106,9 +128,9 @@ idInternalCVar::idInternalCVar idInternalCVar::idInternalCVar( const char *newName, const char *newValue, int newFlags ) { nameString = newName; name = nameString.c_str(); - valueString = newValue; + CVar_AssignString( valueString, newValue, ( newFlags & CVAR_PRIVATE ) != 0 ); value = valueString.c_str(); - resetString = newValue; + CVar_AssignString( resetString, newValue, ( newFlags & CVAR_PRIVATE ) != 0 ); descriptionString = ""; description = descriptionString.c_str(); flags = ( newFlags & ~CVAR_STATIC ) | CVAR_MODIFIED; @@ -129,9 +151,9 @@ idInternalCVar::idInternalCVar idInternalCVar::idInternalCVar( const idCVar *cvar ) { nameString = cvar->GetName(); name = nameString.c_str(); - valueString = cvar->GetString(); + CVar_AssignString( valueString, cvar->GetString(), ( cvar->GetFlags() & CVAR_PRIVATE ) != 0 ); value = valueString.c_str(); - resetString = cvar->GetString(); + CVar_AssignString( resetString, cvar->GetString(), ( cvar->GetFlags() & CVAR_PRIVATE ) != 0 ); descriptionString = cvar->GetDescription(); description = descriptionString.c_str(); flags = cvar->GetFlags() | CVAR_MODIFIED; @@ -150,6 +172,10 @@ idInternalCVar::~idInternalCVar ============ */ idInternalCVar::~idInternalCVar( void ) { + if ( flags & CVAR_PRIVATE ) { + valueString.SecureClear(); + resetString.SecureClear(); + } Mem_Free( valueStrings ); valueStrings = NULL; } @@ -208,6 +234,7 @@ idInternalCVar::Update ============ */ void idInternalCVar::Update( const idCVar *cvar ) { + const bool privateValue = ( ( flags | cvar->GetFlags() ) & CVAR_PRIVATE ) != 0; // if this is a statically declared variable if ( cvar->GetFlags() & CVAR_STATIC ) { @@ -228,7 +255,7 @@ void idInternalCVar::Update( const idCVar *cvar ) { } // the code is now specifying a variable that the user already set a value for, take the new value as the reset value - resetString = cvar->GetString(); + CVar_AssignString( resetString, cvar->GetString(), privateValue ); descriptionString = cvar->GetDescription(); description = descriptionString.c_str(); valueMin = cvar->GetMinValue(); @@ -246,9 +273,13 @@ void idInternalCVar::Update( const idCVar *cvar ) { // only allow one non-empty reset string without a warning if ( resetString.Length() == 0 ) { - resetString = cvar->GetString(); + CVar_AssignString( resetString, cvar->GetString(), privateValue ); } else if ( cvar->GetString()[0] && resetString.Cmp( cvar->GetString() ) != 0 ) { - common->Warning( "cvar \"%s\" given initial values: \"%s\" and \"%s\"\n", nameString.c_str(), resetString.c_str(), cvar->GetString() ); + if ( privateValue ) { + common->Warning( "private cvar \"%s\" was declared with conflicting initial values\n", nameString.c_str() ); + } else { + common->Warning( "cvar \"%s\" given initial values: \"%s\" and \"%s\"\n", nameString.c_str(), resetString.c_str(), cvar->GetString() ); + } } } @@ -382,11 +413,13 @@ void idInternalCVar::Set( const char *newValue, bool force, bool fromServer ) { } } - if ( valueString.Icmp( newValue ) == 0 ) { + const bool unchanged = ( flags & CVAR_CASE_SENSITIVE ) ? + valueString.Cmp( newValue ) == 0 : valueString.Icmp( newValue ) == 0; + if ( unchanged ) { return; } - valueString = newValue; + CVar_AssignString( valueString, newValue, ( flags & CVAR_PRIVATE ) != 0 ); value = valueString.c_str(); UpdateValue(); @@ -400,7 +433,7 @@ idInternalCVar::Reset ============ */ void idInternalCVar::Reset( void ) { - valueString = resetString; + CVar_AssignString( valueString, resetString.c_str(), ( flags & CVAR_PRIVATE ) != 0 ); value = valueString.c_str(); UpdateValue(); } @@ -498,6 +531,8 @@ class idCVarSystemLocal : public idCVarSystem { virtual const idDict * MoveCVarsToDict( int flags ) const; virtual void SetCVarsFromDict( const idDict &dict ); + virtual bool SetCVarsFromDictByFlags( const idDict &dict, int requiredFlag ); + virtual bool CommandContainsPrivateCVar( const char *commandText ) const; void RegisterInternal( idCVar *cvar ); idInternalCVar * FindInternal( const char *name ) const; @@ -768,6 +803,50 @@ float idCVarSystemLocal::GetCVarFloat( const char *name ) const { return 0.0f; } +/* +============ +idCVarSystemLocal::CommandContainsPrivateCVar + +Scan the full, possibly semicolon-separated input instead of trusting argv(0). +The case-insensitive private name must be bounded by non-cvar characters. Also +inspect the expanded tokens: otherwise `set $target secret`, where target names +a private CVar, reaches the private assignment after raw echo/history/journaling. +============ +*/ +bool idCVarSystemLocal::CommandContainsPrivateCVar( const char *commandText ) const { + if ( commandText == NULL || commandText[0] == '\0' ) { + return false; + } + for ( int index = 0; index < cvars.Num(); ++index ) { + const idInternalCVar *cvar = cvars[ index ]; + if ( !( cvar->GetFlags() & CVAR_PRIVATE ) ) { + continue; + } + if ( idPrivateCommand::ContainsBoundedCaseInsensitiveToken( + commandText, cvar->GetName() ) ) { + return true; + } + } + + idCmdArgs expandedArgs; + expandedArgs.TokenizeString( commandText, false ); + bool containsPrivateCVar = false; + for ( int argIndex = 0; argIndex < expandedArgs.Argc() && !containsPrivateCVar; ++argIndex ) { + for ( int cvarIndex = 0; cvarIndex < cvars.Num(); ++cvarIndex ) { + const idInternalCVar *cvar = cvars[ cvarIndex ]; + if ( ( cvar->GetFlags() & CVAR_PRIVATE ) && + idPrivateCommand::ContainsBoundedCaseInsensitiveToken( + expandedArgs.Argv( argIndex ), cvar->GetName() ) ) { + containsPrivateCVar = true; + break; + } + } + } + // Token expansion may have copied the assignment value into this temporary. + expandedArgs.ClearSensitive(); + return containsPrivateCVar; +} + /* ============ idCVarSystemLocal::Command @@ -790,14 +869,19 @@ bool idCVarSystemLocal::Command( const idCmdArgs &args ) { if ( args.Argc() == 1 ) { // print the variable + const char *value = ( internal->GetFlags() & CVAR_PRIVATE ) ? "" : internal->valueString.c_str(); + const char *defaultValue = ( internal->GetFlags() & CVAR_PRIVATE ) ? "" : internal->resetString.c_str(); common->Printf( "\"%s\" is:\"%s\"" S_COLOR_WHITE " default:\"%s\"\n", - internal->nameString.c_str(), internal->valueString.c_str(), internal->resetString.c_str() ); + internal->nameString.c_str(), value, defaultValue ); if ( idStr::Length( internal->GetDescription() ) > 0 ) { common->Printf( S_COLOR_WHITE "%s\n", internal->GetDescription() ); } } else { // set the value internal->Set( args.Args(), false, false ); + if ( internal->GetFlags() & CVAR_PRIVATE ) { + idCmdArgs::ClearArgsScratch(); + } } return true; } @@ -905,7 +989,7 @@ with the "flags" flag set to true. void idCVarSystemLocal::WriteFlaggedVariables( int flags, const char *setCmd, idFile *f ) const { for( int i = 0; i < cvars.Num(); i++ ) { idInternalCVar *cvar = cvars[i]; - if ( cvar->GetFlags() & flags ) { + if ( ( cvar->GetFlags() & flags ) && !( cvar->GetFlags() & CVAR_PRIVATE ) ) { f->Printf( "%s %s \"%s\"\n", setCmd, cvar->GetName(), cvar->GetString() ); } } @@ -920,7 +1004,7 @@ const idDict* idCVarSystemLocal::MoveCVarsToDict( int flags ) const { moveCVarsToDict.Clear(); for( int i = 0; i < cvars.Num(); i++ ) { idCVar *cvar = cvars[i]; - if ( cvar->GetFlags() & flags ) { + if ( ( cvar->GetFlags() & flags ) && !( cvar->GetFlags() & CVAR_PRIVATE ) ) { moveCVarsToDict.Set( cvar->GetName(), cvar->GetString() ); } } @@ -933,17 +1017,53 @@ idCVarSystemLocal::SetCVarsFromDict ============ */ void idCVarSystemLocal::SetCVarsFromDict( const idDict &dict ) { + // Retain the historical API for local/legacy callers, but never let it + // force-set PRIVATE variables or anything outside the three legacy network + // dictionary classes. New protocol consumers must use + // SetCVarsFromDictByFlags so one wire opcode cannot borrow another class's + // authority. idInternalCVar *internal; for( int i = 0; i < dict.GetNumKeyVals(); i++ ) { const idKeyValue *kv = dict.GetKeyVal( i ); internal = FindInternal( kv->GetKey() ); - if ( internal ) { + if ( internal && + ( internal->GetFlags() & ( CVAR_USERINFO | CVAR_SERVERINFO | CVAR_NETWORKSYNC ) ) != 0 && + !( internal->GetFlags() & CVAR_PRIVATE ) ) { internal->InternalServerSetString( kv->GetValue() ); } } } +/* +============ +idCVarSystemLocal::SetCVarsFromDictByFlags + +Apply a dictionary under one explicit network authority. Requiring a single +known flag prevents a userinfo packet from setting network-sync/server-info +state (or vice versa), while the private check is defense in depth for any +future CVar that is accidentally declared with both flag classes. +============ +*/ +bool idCVarSystemLocal::SetCVarsFromDictByFlags( const idDict &dict, int requiredFlag ) { + const int allowedRemoteFlags = CVAR_USERINFO | CVAR_SERVERINFO | CVAR_NETWORKSYNC; + if ( !idRemoteCVarPolicy::IsSingleAllowedAuthority( requiredFlag, allowedRemoteFlags ) ) { + common->Warning( "SetCVarsFromDictByFlags: invalid remote CVar authority 0x%x", requiredFlag ); + return false; + } + + for ( int i = 0; i < dict.GetNumKeyVals(); ++i ) { + const idKeyValue *kv = dict.GetKeyVal( i ); + idInternalCVar *internal = FindInternal( kv->GetKey() ); + if ( internal == NULL || !idRemoteCVarPolicy::CanApply( internal->GetFlags(), + requiredFlag, allowedRemoteFlags, CVAR_PRIVATE ) ) { + continue; + } + internal->InternalServerSetString( kv->GetValue() ); + } + return true; +} + /* ============ idCVarSystemLocal::Toggle_f @@ -969,6 +1089,10 @@ void idCVarSystemLocal::Toggle_f( const idCmdArgs &args ) { common->Warning( "Toggle_f: cvar \"%s\" not found", args.Argv( 1 ) ); return; } + if ( cvar->GetFlags() & CVAR_PRIVATE ) { + common->Printf( "toggle is unavailable for private CVar %s\n", cvar->GetName() ); + return; + } if ( argc > 3 ) { // cycle through multiple values @@ -1014,6 +1138,10 @@ void idCVarSystemLocal::Set_f( const idCmdArgs &args ) { str = args.Args( 2, args.Argc() - 1 ); localCVarSystem.SetCVarString( args.Argv(1), str ); + idInternalCVar *cvar = localCVarSystem.FindInternal( args.Argv( 1 ) ); + if ( cvar != NULL && ( cvar->GetFlags() & CVAR_PRIVATE ) ) { + idCmdArgs::ClearArgsScratch(); + } } /* @@ -1171,7 +1299,8 @@ void idCVarSystemLocal::ListByFlags( const idCmdArgs &args, cvarFlags_t flags ) case SHOW_VALUE: { for ( i = 0; i < cvarList.Num(); i++ ) { cvar = cvarList[i]; - common->Printf( FORMAT_STRING S_COLOR_WHITE "\"%s\"\n", cvar->nameString.c_str(), cvar->valueString.c_str() ); + const char *value = ( cvar->GetFlags() & CVAR_PRIVATE ) ? "" : cvar->valueString.c_str(); + common->Printf( FORMAT_STRING S_COLOR_WHITE "\"%s\"\n", cvar->nameString.c_str(), value ); } break; } diff --git a/src/framework/CVarSystem.h b/src/framework/CVarSystem.h index 3abb4314..317e99cd 100644 --- a/src/framework/CVarSystem.h +++ b/src/framework/CVarSystem.h @@ -81,6 +81,7 @@ typedef enum { CVAR_SPECIAL_CONCAT = BIT(22), // special concatination of the incoming string to the cvar system, will remove space between ^ and the code that is produced by tokenzier CVAR_STRIPTRAILING = BIT(23), // always strip trailing / on that cvar CVAR_REPEATERINFO = BIT(24), // sent from repeaters, available to menu + CVAR_PRIVATE = BIT(25), // secret value: redact from console output and omit from generic serialization } cvarFlags_t; @@ -253,6 +254,17 @@ class idCVarSystem { // Moves CVars to and from dictionaries. virtual const idDict * MoveCVarsToDict( int flags ) const = 0; virtual void SetCVarsFromDict( const idDict &dict ) = 0; + // Applies only CVars carrying exactly the authority represented by one of + // CVAR_USERINFO, CVAR_SERVERINFO, or CVAR_NETWORKSYNC. Network decoders + // must use this entry point instead of granting a received dictionary the + // authority to force-set arbitrary registered CVars. + virtual bool SetCVarsFromDictByFlags( const idDict &dict, int requiredFlag ) = 0; + + // ABI rule: append new virtual methods here. Inserting one above the + // legacy SetCVarsFromDict tail silently changes every later vtable slot for + // an otherwise version-compatible game module. + // Console front ends use this before echoing or retaining typed input. + virtual bool CommandContainsPrivateCVar( const char *commandText ) const = 0; }; extern idCVarSystem * cvarSystem; diff --git a/src/framework/CmdSystem.cpp b/src/framework/CmdSystem.cpp index 64d42d73..480e6d90 100644 --- a/src/framework/CmdSystem.cpp +++ b/src/framework/CmdSystem.cpp @@ -264,6 +264,11 @@ void idCmdSystemLocal::Vstr_f( const idCmdArgs &args ) { return; } + idCVar *cvar = cvarSystem->Find( args.Argv( 1 ) ); + if ( cvar != NULL && ( cvar->GetFlags() & CVAR_PRIVATE ) ) { + common->Printf( "vstr is unavailable for private CVar %s\n", cvar->GetName() ); + return; + } v = cvarSystem->GetCVarString( args.Argv( 1 ) ); cmdSystemLocal.BufferCommandText( CMD_EXEC_APPEND, va( "%s\n", v ) ); @@ -744,9 +749,25 @@ void idCmdSystemLocal::ExecuteCommandBuffer( void ) { text[i] = 0; + bool privateCommand = cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( text ); if ( !idStr::Cmp( text, "_execTokenized" ) ) { - args = tokenizedCmds[ 0 ]; - tokenizedCmds.RemoveIndex( 0 ); + if ( tokenizedCmds.Num() == 0 ) { + common->Warning( "ignored unmatched internal tokenized-command marker" ); + // args is reused across loop iterations. An unmatched marker must + // not execute the preceding command a second time. + args.ClearSensitive(); + } else { + args = tokenizedCmds[ 0 ]; + for ( int argIndex = 0; argIndex < args.Argc() && !privateCommand; ++argIndex ) { + privateCommand = cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( args.Argv( argIndex ) ); + } + if ( privateCommand ) { + tokenizedCmds[ 0 ].ClearSensitive(); + } + tokenizedCmds.RemoveIndex( 0 ); + } } else { args.TokenizeString( text, false ); } @@ -755,6 +776,7 @@ void idCmdSystemLocal::ExecuteCommandBuffer( void ) { // this is necessary because commands (exec) can insert data at the // beginning of the text buffer + const int previousTextLength = textLength; if ( i == textLength ) { textLength = 0; } else { @@ -762,9 +784,16 @@ void idCmdSystemLocal::ExecuteCommandBuffer( void ) { textLength -= i; memmove( text, text+i, textLength ); } + // Do not leave consumed command arguments (which can include private + // CVar assignments) in the unused tail of this long-lived buffer. + memset( textBuf + textLength, 0, previousTextLength - textLength ); // execute the command line that we have already tokenized ExecuteTokenizedString( args ); + if ( privateCommand ) { + args.ClearSensitive(); + idCmdArgs::ClearArgsScratch(); + } } } diff --git a/src/framework/Common.cpp b/src/framework/Common.cpp index 29206d28..ef79c73a 100644 --- a/src/framework/Common.cpp +++ b/src/framework/Common.cpp @@ -1576,6 +1576,10 @@ idCommonLocal::ClearCommandLine ================== */ void idCommonLocal::ClearCommandLine( void ) { + for ( int i = 0; i < com_numConsoleLines; ++i ) { + com_consoleLines[ i ].ClearSensitive(); + } + idCmdArgs::ClearArgsScratch(); com_numConsoleLines = 0; } @@ -6455,6 +6459,12 @@ void idCommonLocal::Init( int argc, const char **argv, const char *cmdline ) { if ( !com_consoleLines[ i ].Argc() ) { continue; } + const char *lineText = com_consoleLines[ i ].Args( 0, com_consoleLines[ i ].Argc() - 1 ); + if ( cvarSystem->CommandContainsPrivateCVar( lineText ) ) { + idCmdArgs::ClearArgsScratch(); + Printf( " %d: \n", i ); + continue; + } Printf( " %d:", i ); for ( int j = 0; j < com_consoleLines[ i ].Argc(); ++j ) { Printf( " %s", com_consoleLines[ i ].Argv( j ) ); diff --git a/src/framework/Console.cpp b/src/framework/Console.cpp index 2d63be75..c7afa59a 100644 --- a/src/framework/Console.cpp +++ b/src/framework/Console.cpp @@ -1569,6 +1569,7 @@ void idConsoleLocal::LoadCommandHistory( void ) { return; } + bool removedPrivateCommand = false; int lineStart = 0; for ( int i = 0; i <= fileLength; ++i ) { const bool atEnd = ( i == fileLength ); @@ -1588,6 +1589,14 @@ void idConsoleLocal::LoadCommandHistory( void ) { if ( lineLength > 0 ) { line.Append( fileBuffer + lineStart, lineLength ); } + if ( cvarSystem != NULL && cvarSystem->CommandContainsPrivateCVar( line.c_str() ) ) { + // Older builds persisted the complete command line, including values + // assigned to secret CVars. Purge those entries instead of bringing a + // credential back into live history (or writing it out again). + removedPrivateCommand = true; + lineStart = i + 1; + continue; + } historyEditLines[nextHistoryLine % COMMAND_HISTORY].SetBuffer( line.c_str() ); nextHistoryLine++; @@ -1595,7 +1604,15 @@ void idConsoleLocal::LoadCommandHistory( void ) { } historyLine = nextHistoryLine; + if ( removedPrivateCommand && fileLength > 0 ) { + // The old history buffer can contain the credential that was just + // purged from disk. Scrub it before returning the allocation. + memset( const_cast( fileBuffer ), 0, fileLength ); + } fileSystem->FreeFile( ( void * )fileBuffer ); + if ( removedPrivateCommand ) { + SaveCommandHistory(); + } } /* @@ -3150,7 +3167,12 @@ bool idConsoleLocal::GetCompletionCvarInfo( const char *match, char *value, int *modified = cvar->IsModified(); } if ( value != NULL && valueSize > 0 ) { - const char *cvarValue = cvar->GetString(); + // Completion candidates are populated from the full CVar-name list, so a + // partial prefix can reach this path before the edit-field private-command + // guard sees the complete name. Never let the popup turn completion into a + // private-value query. + const char *cvarValue = ( cvar->GetFlags() & CVAR_PRIVATE ) ? + "" : cvar->GetString(); idStr::Copynz( value, ( cvarValue != NULL && cvarValue[0] != '\0' ) ? cvarValue : "\"\"", valueSize ); } return true; @@ -4113,14 +4135,22 @@ void idConsoleLocal::KeyDownEvent( int key ) { return; } - common->Printf( "]%s\n", consoleField.GetBuffer() ); + const bool privateCommand = cvarSystem != NULL && + cvarSystem->CommandContainsPrivateCVar( consoleField.GetBuffer() ); + if ( privateCommand ) { + common->Printf( "]\n" ); + } else { + common->Printf( "]%s\n", consoleField.GetBuffer() ); + } cmdSystem->BufferCommandText( CMD_EXEC_APPEND, consoleField.GetBuffer() ); cmdSystem->BufferCommandText( CMD_EXEC_APPEND, "\n" ); - historyEditLines[nextHistoryLine % COMMAND_HISTORY] = consoleField; - nextHistoryLine++; - historyLine = nextHistoryLine; - SaveCommandHistory(); + if ( !privateCommand ) { + historyEditLines[nextHistoryLine % COMMAND_HISTORY] = consoleField; + nextHistoryLine++; + historyLine = nextHistoryLine; + SaveCommandHistory(); + } consoleField.Clear(); consoleField.SetWidthInChars( lineWidth ); diff --git a/src/framework/EditField.cpp b/src/framework/EditField.cpp index 952106f3..bc20e752 100644 --- a/src/framework/EditField.cpp +++ b/src/framework/EditField.cpp @@ -195,7 +195,10 @@ PrintCvarMatches static void PrintCvarMatches( const char *s ) { if ( idStr::Icmpn( s, globalAutoComplete.currentMatch, idLib::SizeToInt( strlen( globalAutoComplete.currentMatch ), "PrintCvarMatches" ) ) == 0 ) { - common->Printf( " %s" S_COLOR_WHITE " = \"%s\"\n", s, cvarSystem->GetCVarString( s ) ); + idCVar *cvar = cvarSystem->Find( s ); + const char *value = ( cvar != NULL && ( cvar->GetFlags() & CVAR_PRIVATE ) ) ? + "" : cvarSystem->GetCVarString( s ); + common->Printf( " %s" S_COLOR_WHITE " = \"%s\"\n", s, value ); } } @@ -223,11 +226,13 @@ idEditField::Clear =============== */ void idEditField::Clear( void ) { - buffer[0] = 0; + // Edit fields can hold console-entered credentials. Clear the backing + // storage, not only its first byte, so discarded private commands do not + // remain recoverable in stale field memory. + memset( buffer, 0, sizeof( buffer ) ); cursor = 0; scroll = 0; - autoComplete.length = 0; - autoComplete.valid = false; + memset( &autoComplete, 0, sizeof( autoComplete ) ); } /* @@ -365,6 +370,12 @@ idEditField::AutoComplete void idEditField::AutoComplete( void ) { char completionArgString[MAX_EDIT_LINE]; idCmdArgs args; + if ( cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( buffer ) ) { + idCmdArgs::ClearArgsScratch(); + common->Printf( "autocomplete is unavailable for private CVar commands\n" ); + return; + } if ( !autoComplete.valid ) { const bool explicitCommandPrefix = ( buffer[0] == '/' || buffer[0] == '\\' ); @@ -501,6 +512,10 @@ static int QueryCompletionInternal( const char *cmd, bool *appendSpace, editFiel if ( cmd == NULL || cmd[0] == '\0' ) { return 0; } + if ( cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( cmd ) ) { + return 0; + } NormalizeCompletionCommandString( cmd, normalizedCmd, sizeof( normalizedCmd ) ); if ( normalizedCmd[0] == '\0' ) { @@ -787,6 +802,7 @@ void idEditField::Paste( void ) { CharEvent( cbd[i] ); } + memset( cbd, 0, pasteLen ); Mem_Free( cbd ); } diff --git a/src/framework/EventLoop.cpp b/src/framework/EventLoop.cpp index 90190748..13fb20f1 100644 --- a/src/framework/EventLoop.cpp +++ b/src/framework/EventLoop.cpp @@ -34,6 +34,18 @@ idCVar idEventLoop::com_journal( "com_journal", "0", CVAR_INIT|CVAR_SYSTEM, "1 = idEventLoop eventLoopLocal; idEventLoop *eventLoop = &eventLoopLocal; +static bool EventLoop_IsPrivateConsoleEvent( const sysEvent_t &event ) { + if ( event.evType != SE_CONSOLE || event.evPtr == NULL || event.evPtrLength <= 0 || + cvarSystem == NULL || !cvarSystem->IsInitialized() ) { + return false; + } + if ( memchr( event.evPtr, '\0', static_cast( event.evPtrLength ) ) == NULL ) { + // A malformed console event is not safe to persist as text. + return true; + } + return cvarSystem->CommandContainsPrivateCVar( static_cast( event.evPtr ) ); +} + /* ================= @@ -81,13 +93,20 @@ sysEvent_t idEventLoop::GetRealEvent( void ) { // write the journal value out if needed if ( com_journal.GetInteger() == 1 ) { - r = com_journalFile->Write( &ev, sizeof(ev) ); + static const char PRIVATE_EVENT_TEXT[] = ""; + sysEvent_t journalEvent = ev; + const void *journalData = ev.evPtr; + if ( EventLoop_IsPrivateConsoleEvent( ev ) ) { + journalEvent.evPtrLength = sizeof( PRIVATE_EVENT_TEXT ); + journalData = PRIVATE_EVENT_TEXT; + } + r = com_journalFile->Write( &journalEvent, sizeof(journalEvent) ); if ( r != sizeof(ev) ) { common->FatalError( "Error writing to journal file" ); } - if ( ev.evPtrLength ) { - r = com_journalFile->Write( ev.evPtr, ev.evPtrLength ); - if ( r != ev.evPtrLength ) { + if ( journalEvent.evPtrLength ) { + r = com_journalFile->Write( journalData, journalEvent.evPtrLength ); + if ( r != journalEvent.evPtrLength ) { common->FatalError( "Error writing to journal file" ); } } @@ -117,6 +136,9 @@ void idEventLoop::PushEvent( sysEvent_t *event ) { } if ( ev->evPtr ) { + if ( EventLoop_IsPrivateConsoleEvent( *ev ) ) { + memset( ev->evPtr, 0, ev->evPtrLength ); + } Mem_Free( ev->evPtr ); } com_pushedEventsTail++; @@ -166,6 +188,9 @@ void idEventLoop::ProcessEvent( sysEvent_t ev ) { // free any block data if ( ev.evPtr ) { + if ( EventLoop_IsPrivateConsoleEvent( ev ) ) { + memset( ev.evPtr, 0, ev.evPtrLength ); + } Mem_Free( ev.evPtr ); } } diff --git a/src/framework/FileSystem.cpp b/src/framework/FileSystem.cpp index 20426e40..ba839a45 100644 --- a/src/framework/FileSystem.cpp +++ b/src/framework/FileSystem.cpp @@ -30,7 +30,9 @@ If you have questions concerning this license or the applicable additional terms #include "Unzip.h" +#include "GameDirPolicy.h" #include "openq4_paks_generated.h" +#include "../sys/URLPolicy.h" #include #include @@ -57,6 +59,11 @@ If you have questions concerning this license or the applicable additional terms #if ID_ENABLE_CURL #include "../curl/include/curl/curl.h" + #if defined( LIBCURL_VERSION_NUM ) && LIBCURL_VERSION_NUM >= 0x071304 && defined( CURL_VERSION_ASYNCHDNS ) + #define OPENQ4_CURL_CAPABLE_BUILD 1 + #else + #define OPENQ4_CURL_CAPABLE_BUILD 0 + #endif #endif int Com_GetNumStartupCommandLines( void ); @@ -1344,11 +1351,22 @@ typedef struct searchpath_s { // + .jpg and .tga #define MAX_CACHED_DIRS 6 -// how many OSes to handle game paks for ( we don't have to know them precisely ) -#define MAX_GAME_OS 6 #define BINARY_CONFIG "binary.conf" #define ADDON_CONFIG "addon.conf" +// Protocol 2.41 identifies compatible game code with the official 1.4.2 +// q4base/game300.pk4 checksum. openQ4 never loads executable code from that +// archive; the unchanged value is a platform-independent wire token for a +// module that FindDLL resolved from the trusted package/module roots. +static const int OPENQ4_Q4_142_GAME300_PAK_CHECKSUM = 0x68fb90b1; + +typedef enum { + GAME_MODULE_ORIGIN_NONE = 0, + GAME_MODULE_ORIGIN_ACTIVE_MOD, + GAME_MODULE_ORIGIN_BASE_GAME, + GAME_MODULE_ORIGIN_PACKAGE_ROOT +} gameModuleOrigin_t; + class idDEntry : public idStrList { public: idDEntry() {} @@ -1513,6 +1531,7 @@ class idFileSystemLocal : public idFileSystem { int restartGamePakChecksum; int gameDLLChecksum; // the checksum of the last loaded game DLL int gamePakChecksum; // the checksum of the pak holding the loaded game DLL + gameModuleOrigin_t gameModuleOrigin; // trusted root and active-mod provenance bool isFileLoadingAllowed; idStr currentAssetLog; idStr currentAssetLogUnfiltered; @@ -1616,12 +1635,29 @@ idFileSystemLocal::idFileSystemLocal( void ) { d3xp = 0; loadedFileFromDir = false; restartGamePakChecksum = 0; + gameDLLChecksum = 0; + gamePakChecksum = 0; + gameModuleOrigin = GAME_MODULE_ORIGIN_NONE; + memset( gamePakForOS, 0, sizeof( gamePakForOS ) ); isFileLoadingAllowed = false; currentAssetLog.Clear(); currentAssetLogUnfiltered.Clear(); assetLog.Clear(); backgroundDownloads = NULL; - memset( &defaultBackgroundDownload, 0, sizeof( defaultBackgroundDownload ) ); + defaultBackgroundDownload.next = NULL; + defaultBackgroundDownload.opcode = DLTYPE_FILE; + defaultBackgroundDownload.f = NULL; + defaultBackgroundDownload.file.position = 0; + defaultBackgroundDownload.file.length = 0; + defaultBackgroundDownload.file.buffer = NULL; + defaultBackgroundDownload.url.url.Clear(); + defaultBackgroundDownload.url.dlerror[ 0 ] = '\0'; + defaultBackgroundDownload.url.expectedSize = 0; + defaultBackgroundDownload.url.dltotal = 0; + defaultBackgroundDownload.url.dlnow = 0; + defaultBackgroundDownload.url.dlstatus = 0; + defaultBackgroundDownload.url.status = DL_WAIT; + defaultBackgroundDownload.completed = true; memset( &backgroundThread, 0, sizeof( backgroundThread ) ); addonPaks = NULL; } @@ -4254,6 +4290,12 @@ bool idFileSystemLocal::GetModInfo( const char *modDir, idModInfo &modInfo, idSt } return false; } + if ( !idGameDirPolicy::IsPortableSegment( modDir ) ) { + if ( reason != NULL ) { + *reason = "mod directory must be one portable directory segment"; + } + return false; + } const char *search[ 3 ]; search[ 0 ] = fs_cdpath.GetString(); @@ -5560,64 +5602,23 @@ idFileSystemLocal::UpdateGamePakChecksums ===================== */ bool idFileSystemLocal::UpdateGamePakChecksums( void ) { - searchpath_t *search; - fileInPack_t *pakFile; - int confHash; - idFile *confFile; - char *buf; - idLexer *lexConf; - idToken token; - int id; - - confHash = HashFileName( BINARY_CONFIG ); - memset( gamePakForOS, 0, sizeof( gamePakForOS ) ); - for ( search = searchPaths; search; search = search->next ) { - if ( !search->pack ) { - continue; - } - search->pack->binary = BINARY_NO; - for ( pakFile = search->pack->hashTable[confHash]; pakFile; pakFile = pakFile->next ) { - if ( !FilenameCompare( pakFile->name, BINARY_CONFIG ) ) { - search->pack->binary = BINARY_YES; - confFile = ReadFileFromZip( search->pack, pakFile, BINARY_CONFIG ); - if ( confFile == NULL ) { - search->pack->binary = BINARY_NO; - break; - } - buf = new char[ confFile->Length() + 1 ]; - confFile->Read( (void *)buf, confFile->Length() ); - buf[ confFile->Length() ] = '\0'; - lexConf = new idLexer( buf, confFile->Length(), confFile->GetFullPath() ); - while ( lexConf->ReadToken( &token ) ) { - if ( token.IsNumeric() ) { - id = atoi( token ); - if ( id < MAX_GAME_OS && !gamePakForOS[ id ] ) { - if ( fs_debug.GetBool() ) { - common->Printf( "Adding game pak checksum for OS %d: %s 0x%x\n", id, confFile->GetFullPath(), search->pack->checksum ); - } - gamePakForOS[ id ] = search->pack->checksum; - } - } - } - CloseFile( confFile ); - delete lexConf; - delete[] buf; - } - } + if ( gameDLLChecksum == 0 || gamePakChecksum != OPENQ4_Q4_142_GAME300_PAK_CHECKSUM || + gameModuleOrigin == GAME_MODULE_ORIGIN_NONE ) { + common->Warning( "No trusted openQ4 game module is loaded for pure-server startup" ); + return false; } - - // some sanity checks on the game code references - // make sure that at least the local OS got a pure reference - if ( !gamePakForOS[ BUILD_OS_ID ] ) { - common->Warning( "No game code pak reference found for the local OS" ); + if ( gameModuleOrigin == GAME_MODULE_ORIGIN_ACTIVE_MOD && + !cvarSystem->GetCVarBool( "net_serverAllowServerMod" ) ) { + common->Warning( "The active mod supplies game code (net_serverAllowServerMod is off)" ); return false; } - if ( !cvarSystem->GetCVarBool( "net_serverAllowServerMod" ) && - gamePakChecksum != gamePakForOS[ BUILD_OS_ID ] ) { - common->Warning( "The current game code doesn't match pak files (net_serverAllowServerMod is off)" ); - return false; + // Protocol 2.41 defines Windows, Linux and macOS as OS IDs 0..2. The + // compatibility token is deliberately platform-independent; executable + // bytes remain outside the downloadable/pure-PK4 path. + for ( int os = 0; os <= 2; ++os ) { + gamePakForOS[ os ] = OPENQ4_Q4_142_GAME300_PAK_CHECKSUM; } return true; @@ -5676,6 +5677,16 @@ int idFileSystemLocal::ValidateDownloadPakForChecksum( int checksum, char path[ } // check the binary // a pure server sets the binary flag when starting the game + if ( pak->binary == BINARY_UNKNOWN ) { + const int confHash = HashFileName( BINARY_CONFIG ); + pak->binary = BINARY_NO; + for ( fileInPack_t *pakFile = pak->hashTable[ confHash ]; pakFile; pakFile = pakFile->next ) { + if ( !FilenameCompare( pakFile->name, BINARY_CONFIG ) ) { + pak->binary = BINARY_YES; + break; + } + } + } assert( pak->binary != BINARY_UNKNOWN ); pakBinary = ( pak->binary == BINARY_YES ) ? true : false; if ( isBinary != pakBinary ) { @@ -5709,6 +5720,7 @@ idFileSystemLocal::ClearPureChecksums void idFileSystemLocal::ClearPureChecksums( void ) { common->DPrintf( "Cleared pure server lock\n" ); serverPaks.Clear(); + memset( gamePakForOS, 0, sizeof( gamePakForOS ) ); } /* @@ -5722,8 +5734,10 @@ can be: some pak files currently referenced are not referenced by the server wrong order - if the pak order doesn't match, means some stuff could have been loaded from somewhere else server referenced files are prepended to the list if possible ( that doesn't break pureness ) -DLL: - the checksum of the pak containing the DLL is maintained seperately, the server can send different replies by OS +Game code: + protocol 2.41 carries the official q4base/game300.pk4 checksum as a + compatibility token, but openQ4 resolves executable modules only from its + trusted package/module roots and never from a server-selected PK4 ===================== */ fsPureReply_t idFileSystemLocal::SetPureServerChecksums( const int pureChecksums[ MAX_PURE_PAKS ], int _gamePakChecksum, int missingChecksums[ MAX_PURE_PAKS ], int *missingGamePakChecksum ) { @@ -5731,18 +5745,24 @@ fsPureReply_t idFileSystemLocal::SetPureServerChecksums( const int pureChecksums int i, j, imissing; bool success = true; bool canPrepend = true; - char dllName[MAX_OSPATH]; - int dllHash; - fileInPack_t * pakFile; - - sys->DLL_GetFileName( "game", dllName, MAX_OSPATH ); - dllHash = HashFileName( dllName ); imissing = 0; missingChecksums[ 0 ] = 0; assert( missingGamePakChecksum ); *missingGamePakChecksum = 0; + // Validate the legacy game-code field before touching the server-selected + // asset list. It is only a wire-compatibility token in openQ4; it must never + // select, download, extract, or restart into executable code. + if ( _gamePakChecksum != OPENQ4_Q4_142_GAME300_PAK_CHECKSUM || + gamePakChecksum != OPENQ4_Q4_142_GAME300_PAK_CHECKSUM || + gameDLLChecksum == 0 || gameModuleOrigin == GAME_MODULE_ORIGIN_NONE ) { + if ( fs_debug.GetBool() ) { + common->Printf( "pure server supplied unsupported game-code token 0x%x\n", _gamePakChecksum ); + } + return PURE_NODLL; + } + if ( pureChecksums[ 0 ] == 0 ) { ClearPureChecksums(); return PURE_OK; @@ -5815,48 +5835,6 @@ fsPureReply_t idFileSystemLocal::SetPureServerChecksums( const int pureChecksums j++; } - // DLL checksuming - if ( !_gamePakChecksum ) { - // server doesn't have knowledge of code we can use ( OS issue ) - return PURE_NODLL; - } - assert( gameDLLChecksum ); -#if ID_FAKE_PURE - gamePakChecksum = _gamePakChecksum; -#endif - if ( _gamePakChecksum != gamePakChecksum ) { - // current DLL is wrong, search for a pak with the approriate checksum - // ( search all paks, the pure list is not relevant here ) - pack = GetPackForChecksum( _gamePakChecksum ); - if ( !pack ) { - if ( fs_debug.GetBool() ) { - common->Printf( "missing the game code pak ( 0x%x )\n", _gamePakChecksum ); - } - // if there are other paks missing they have also been marked above - *missingGamePakChecksum = _gamePakChecksum; - return PURE_MISSING; - } - // if assets paks are missing, don't try any of the DLL restart / NODLL - if ( imissing ) { - return PURE_MISSING; - } - // we have a matching pak - if ( fs_debug.GetBool() ) { - common->Printf( "server's game code pak candidate is '%s' ( 0x%x )\n", pack->pakFilename.c_str(), pack->checksum ); - } - // make sure there is a valid DLL for us - if ( pack->hashTable[ dllHash ] ) { - for ( pakFile = pack->hashTable[ dllHash ]; pakFile; pakFile = pakFile->next ) { - if ( !FilenameCompare( pakFile->name, dllName ) ) { - gamePakChecksum = _gamePakChecksum; // this will be used to extract the DLL in pure mode FindDLL - return PURE_RESTART; - } - } - } - common->Warning( "media is misconfigured. server claims pak '%s' ( 0x%x ) has media for us, but '%s' is not found\n", pack->pakFilename.c_str(), pack->checksum, dllName ); - return PURE_NODLL; - } - // we reply to missing after DLL check so it can be part of the list if ( imissing ) { return PURE_MISSING; @@ -5885,10 +5863,12 @@ void idFileSystemLocal::GetPureServerChecksums( int checksums[ MAX_PURE_PAKS ], } checksums[ i ] = 0; if ( _gamePakChecksum ) { - if ( OS >= 0 ) { + if ( OS >= 0 && OS < MAX_GAME_OS ) { *_gamePakChecksum = gamePakForOS[ OS ]; - } else { + } else if ( OS == -1 ) { *_gamePakChecksum = gamePakChecksum; + } else { + *_gamePakChecksum = 0; } } } @@ -6032,6 +6012,10 @@ void idFileSystemLocal::Restart( void ) { // see if we are going to allow add-ons SetRestrictions(); + // Shutdown( true ) stops the worker, so filesystem restarts must restore it + // before another background file read or pure-pack download is queued. + StartBackgroundDownloadThread(); + // if we can't find default.cfg, assume that the paths are // busted and error out now, rather than getting an unreadable // graphics screen when the font fails to load @@ -6075,6 +6059,8 @@ void idFileSystemLocal::Shutdown( bool reloading ) { loadedFileFromDir = false; gameDLLChecksum = 0; gamePakChecksum = 0; + gameModuleOrigin = GAME_MODULE_ORIGIN_NONE; + memset( gamePakForOS, 0, sizeof( gamePakForOS ) ); ClearDirCache(); @@ -6831,6 +6817,55 @@ int idFileSystemLocal::CurlProgressFunction( void *clientp, double dltotal, doub } #if ID_ENABLE_CURL +static bool fsCurlGlobalInitialized = false; +static bool fsCurlHTTPReady = false; + +static void FS_ShutdownCurl() { + fsCurlHTTPReady = false; + if ( fsCurlGlobalInitialized ) { + curl_global_cleanup(); + fsCurlGlobalInitialized = false; + } +} + +static bool FS_InitializeCurl() { +#if !OPENQ4_CURL_CAPABLE_BUILD + common->Warning( "HTTP downloads disabled: libcurl 7.19.4 or newer with asynchronous DNS is required" ); + return false; +#else + if ( fsCurlHTTPReady ) { + return true; + } + + const CURLcode initResult = curl_global_init( CURL_GLOBAL_DEFAULT ); + if ( initResult != CURLE_OK ) { + common->Warning( "HTTP downloads disabled: curl_global_init failed (%s)", curl_easy_strerror( initResult ) ); + return false; + } + fsCurlGlobalInitialized = true; + + const curl_version_info_data *versionInfo = curl_version_info( CURLVERSION_NOW ); + bool hasHTTP = false; + if ( versionInfo != NULL && versionInfo->protocols != NULL ) { + for ( const char *const *protocol = versionInfo->protocols; *protocol != NULL; ++protocol ) { + if ( idStr::Icmp( *protocol, "http" ) == 0 ) { + hasHTTP = true; + break; + } + } + } + if ( versionInfo == NULL || versionInfo->version_num < 0x071304 || + ( versionInfo->features & CURL_VERSION_ASYNCHDNS ) == 0 || !hasHTTP ) { + common->Warning( "HTTP downloads disabled: libcurl lacks the required version, HTTP, or asynchronous-DNS capability" ); + FS_ShutdownCurl(); + return false; + } + + fsCurlHTTPReady = true; + return true; +#endif +} + class idScopedCurlEasySession { public: explicit idScopedCurlEasySession( CURL *session ) : session( session ) {} @@ -6843,6 +6878,52 @@ class idScopedCurlEasySession { idScopedCurlEasySession &operator=( const idScopedCurlEasySession & ) = delete; CURL *session; }; + +static CURLcode FS_ConfigureBoundedHTTPTransfer( CURL *session ) { +#if !OPENQ4_CURL_CAPABLE_BUILD + return CURLE_FAILED_INIT; +#else + CURLcode result; + + // Keep package transfer and shutdown behavior bounded even when a remote + // endpoint accepts a connection and then stops making progress. Protocol + // masks are repeated here so future callers cannot bypass URLPolicy by + // reaching libcurl directly. + result = curl_easy_setopt( session, CURLOPT_PROTOCOLS, static_cast( CURLPROTO_HTTP | CURLPROTO_HTTPS ) ); + if ( result != CURLE_OK ) { + return result; + } + result = curl_easy_setopt( session, CURLOPT_REDIR_PROTOCOLS, static_cast( CURLPROTO_HTTP | CURLPROTO_HTTPS ) ); + if ( result != CURLE_OK ) { + return result; + } + result = curl_easy_setopt( session, CURLOPT_FOLLOWLOCATION, 0L ); + if ( result != CURLE_OK ) { + return result; + } + result = curl_easy_setopt( session, CURLOPT_MAXREDIRS, 0L ); + if ( result != CURLE_OK ) { + return result; + } + result = curl_easy_setopt( session, CURLOPT_CONNECTTIMEOUT, 15L ); + if ( result != CURLE_OK ) { + return result; + } + result = curl_easy_setopt( session, CURLOPT_LOW_SPEED_LIMIT, 1024L ); + if ( result != CURLE_OK ) { + return result; + } + result = curl_easy_setopt( session, CURLOPT_LOW_SPEED_TIME, 30L ); + if ( result != CURLE_OK ) { + return result; + } + result = curl_easy_setopt( session, CURLOPT_TIMEOUT, 3600L ); + if ( result != CURLE_OK ) { + return result; + } + return curl_easy_setopt( session, CURLOPT_NOSIGNAL, 1L ); +#endif +} #endif /* @@ -6875,8 +6956,27 @@ dword BackgroundDownloadThread( void *parms ) { // use the low level read function, because fread may allocate memory fread(bgl->file.buffer, bgl->file.length, 1, static_cast(bgl->f)->GetFilePtr()); bgl->completed = true; - } else { + } else if ( bgl->opcode == DLTYPE_URL ) { #if ID_ENABLE_CURL + dlStatus_t expectedStatus = DL_WAIT; + if ( !bgl->url.status.compare_exchange_strong( expectedStatus, DL_INPROGRESS ) ) { + bgl->url.dlstatus = -1; + bgl->url.status = DL_FAILED; + idStr::Copynz( bgl->url.dlerror, + expectedStatus == DL_ABORTING ? "download cancelled" : "invalid download state", + MAX_STRING_CHARS ); + bgl->completed = true; + continue; + } + if ( !fsCurlHTTPReady ) { + bgl->url.dlstatus = CURLE_FAILED_INIT; + bgl->url.status = DL_FAILED; + idStr::Copynz( bgl->url.dlerror, + "HTTP download support is unavailable with this libcurl runtime", + MAX_STRING_CHARS ); + bgl->completed = true; + continue; + } // DLTYPE_URL // use a local buffer for curl error since the size define is local char error_buf[ CURL_ERROR_SIZE ]; @@ -6907,7 +7007,14 @@ dword BackgroundDownloadThread( void *parms ) { bgl->completed = true; continue; } - ret = curl_easy_setopt( session, CURLOPT_FAILONERROR, 1 ); + ret = FS_ConfigureBoundedHTTPTransfer( session ); + if ( ret ) { + bgl->url.dlstatus = ret; + bgl->url.status = DL_FAILED; + bgl->completed = true; + continue; + } + ret = curl_easy_setopt( session, CURLOPT_FAILONERROR, 1L ); if ( ret ) { bgl->url.dlstatus = ret; bgl->url.status = DL_FAILED; @@ -6928,7 +7035,7 @@ dword BackgroundDownloadThread( void *parms ) { bgl->completed = true; continue; } - ret = curl_easy_setopt( session, CURLOPT_NOPROGRESS, 0 ); + ret = curl_easy_setopt( session, CURLOPT_NOPROGRESS, 0L ); if ( ret ) { bgl->url.dlstatus = ret; bgl->url.status = DL_FAILED; @@ -6951,7 +7058,6 @@ dword BackgroundDownloadThread( void *parms ) { } bgl->url.dlnow = 0; bgl->url.dltotal = 0; - bgl->url.status = DL_INPROGRESS; ret = curl_easy_perform( session ); if ( ret ) { const char *errorMessage = bgl->url.dlerror[ 0 ] != '\0' ? bgl->url.dlerror : @@ -6975,12 +7081,26 @@ dword BackgroundDownloadThread( void *parms ) { bgl->completed = true; continue; } - bgl->url.status = DL_DONE; + expectedStatus = DL_INPROGRESS; + if ( !bgl->url.status.compare_exchange_strong( expectedStatus, DL_DONE ) ) { + bgl->url.dlstatus = -1; + bgl->url.status = DL_FAILED; + idStr::Copynz( bgl->url.dlerror, + expectedStatus == DL_ABORTING ? "download cancelled" : "invalid download completion state", + MAX_STRING_CHARS ); + } bgl->completed = true; #else bgl->url.status = DL_FAILED; + bgl->url.dlstatus = -1; + idStr::Copynz( bgl->url.dlerror, "HTTP download support is unavailable in this build", MAX_STRING_CHARS ); bgl->completed = true; #endif + } else { + bgl->url.status = DL_FAILED; + bgl->url.dlstatus = -1; + idStr::Copynz( bgl->url.dlerror, "unsupported background download operation", MAX_STRING_CHARS ); + bgl->completed = true; } } return 0; @@ -6993,9 +7113,15 @@ idFileSystemLocal::StartBackgroundReadThread */ void idFileSystemLocal::StartBackgroundDownloadThread() { if ( !backgroundThread.threadHandle ) { +#if ID_ENABLE_CURL + FS_InitializeCurl(); +#endif Sys_CreateThread( (xthread_t)BackgroundDownloadThread, NULL, THREAD_NORMAL, backgroundThread, "backgroundDownload", g_threads, &g_thread_count ); if ( !backgroundThread.threadHandle ) { common->Warning( "idFileSystemLocal::StartBackgroundDownloadThread: failed" ); +#if ID_ENABLE_CURL + FS_ShutdownCurl(); +#endif } } else { common->Printf( "background thread already running\n" ); @@ -7027,6 +7153,10 @@ void idFileSystemLocal::StopBackgroundDownloadThread() { bgl->completed = true; bgl = next; } + +#if ID_ENABLE_CURL + FS_ShutdownCurl(); +#endif } /* @@ -7035,6 +7165,28 @@ idFileSystemLocal::BackgroundDownload ================= */ void idFileSystemLocal::BackgroundDownload( backgroundDownload_t *bgl ) { + if ( bgl == NULL ) { + return; + } + if ( bgl->opcode != DLTYPE_FILE && bgl->opcode != DLTYPE_URL ) { + bgl->url.status = DL_FAILED; + bgl->url.dlstatus = -1; + idStr::Copynz( bgl->url.dlerror, "unsupported background download operation", sizeof( bgl->url.dlerror ) ); + bgl->completed = true; + return; + } + if ( bgl->opcode == DLTYPE_URL && !idURLPolicy::IsAllowedHTTPURL( bgl->url.url.c_str() ) ) { + // Keep the generic downloader fail-closed too. Today the async pure-pack + // client is the sole URL producer, but a future caller must not silently + // re-enable file:, SMB, or another libcurl-supported protocol. + bgl->url.status = DL_FAILED; + bgl->url.dlstatus = -1; + idStr::Copynz( bgl->url.dlerror, + "download URL must be bounded HTTP or HTTPS with a host", + sizeof( bgl->url.dlerror ) ); + bgl->completed = true; + return; + } if ( bgl->opcode == DLTYPE_FILE ) { if ( dynamic_cast(bgl->f) ) { // add the bgl to the background download list @@ -7169,6 +7321,7 @@ void idFileSystemLocal::FindDLL( const char *name, char _dllPath[ MAX_OSPATH ], idFile *dllFile = NULL; char dllName[MAX_OSPATH]; idStr dllPath; + gameModuleOrigin_t resolvedOrigin = GAME_MODULE_ORIGIN_NONE; sys->DLL_GetFileName( name, dllName, MAX_OSPATH ); @@ -7207,6 +7360,16 @@ void idFileSystemLocal::FindDLL( const char *name, char _dllPath[ MAX_OSPATH ], if ( !moduleGameDir[0] ) { moduleGameDir = OPENQ4_GAMEDIR; } + if ( !idGameDirPolicy::IsPortableSegment( moduleGameDir ) ) { + common->Warning( "Refusing game module lookup for unsafe fs_game '%s'", moduleGameDir ); + if ( updateChecksum ) { + gameDLLChecksum = 0; + gamePakChecksum = 0; + gameModuleOrigin = GAME_MODULE_ORIGIN_NONE; + } + _dllPath[ 0 ] = '\0'; + return; + } idStr trustedModuleRoots; idStr attemptedModulePaths; for ( int i = 0; i < numModuleSearchRoots; ++i ) { @@ -7218,12 +7381,19 @@ void idFileSystemLocal::FindDLL( const char *name, char _dllPath[ MAX_OSPATH ], for ( int i = 0; !dllFile && i < numModuleSearchRoots; i++ ) { FS_AppendGameModuleSearchPath( attemptedModulePaths, moduleSearchRoots[i], moduleGameDir, dllName ); dllFile = FS_OpenGameModuleFromExeDir( this, moduleSearchRoots[i], moduleGameDir, dllName, dllPath ); + if ( dllFile ) { + resolvedOrigin = idStr::Icmp( moduleGameDir, OPENQ4_GAMEDIR ) == 0 ? + GAME_MODULE_ORIGIN_BASE_GAME : GAME_MODULE_ORIGIN_ACTIVE_MOD; + } } if ( !dllFile && idStr::Icmp( moduleGameDir, OPENQ4_GAMEDIR ) != 0 ) { for ( int i = 0; !dllFile && i < numModuleSearchRoots; i++ ) { FS_AppendGameModuleSearchPath( attemptedModulePaths, moduleSearchRoots[i], OPENQ4_GAMEDIR, dllName ); dllFile = FS_OpenGameModuleFromExeDir( this, moduleSearchRoots[i], OPENQ4_GAMEDIR, dllName, dllPath ); + if ( dllFile ) { + resolvedOrigin = GAME_MODULE_ORIGIN_BASE_GAME; + } } if ( dllFile ) { common->DPrintf( "Game DLL '%s' not found in mod directory '%s'; falling back to '%s'.\n", dllName, moduleGameDir, OPENQ4_GAMEDIR ); @@ -7235,6 +7405,9 @@ void idFileSystemLocal::FindDLL( const char *name, char _dllPath[ MAX_OSPATH ], dllPath = moduleSearchRoots[i]; dllPath.AppendPath( dllName ); dllFile = OpenExplicitFileRead( dllPath ); + if ( dllFile ) { + resolvedOrigin = GAME_MODULE_ORIGIN_PACKAGE_ROOT; + } } if ( updateChecksum ) { @@ -7243,7 +7416,8 @@ void idFileSystemLocal::FindDLL( const char *name, char _dllPath[ MAX_OSPATH ], } else { gameDLLChecksum = 0; } - gamePakChecksum = 0; + gameModuleOrigin = resolvedOrigin; + gamePakChecksum = dllFile ? OPENQ4_Q4_142_GAME300_PAK_CHECKSUM : 0; } if ( dllFile ) { dllPath = dllFile->GetFullPath( ); diff --git a/src/framework/FileSystem.h b/src/framework/FileSystem.h index d86222c7..675f4a40 100644 --- a/src/framework/FileSystem.h +++ b/src/framework/FileSystem.h @@ -29,6 +29,8 @@ If you have questions concerning this license or the applicable additional terms #ifndef __FILESYSTEM_H__ #define __FILESYSTEM_H__ +#include + /* =============================================================================== @@ -55,6 +57,7 @@ If you have questions concerning this license or the applicable additional terms static const ID_TIME_T FILE_NOT_FOUND_TIMESTAMP = static_cast( -1 ); static const int MAX_PURE_PAKS = 128; static const int MAX_OSPATH = 256; +static const int MAX_GAME_OS = 6; // modes for OpenFileByMode. used as bit mask internally typedef enum { @@ -83,11 +86,6 @@ typedef enum { DL_FAILED } dlStatus_t; -typedef enum { - FILE_EXEC, - FILE_OPEN -} dlMime_t; - typedef enum { FIND_NO, FIND_YES, @@ -98,10 +96,10 @@ typedef struct urlDownload_s { idStr url; char dlerror[ MAX_STRING_CHARS ]; int expectedSize; // immutable transfer limit; zero permits an unrestricted download - int dltotal; - int dlnow; - int dlstatus; - dlStatus_t status; + std::atomic dltotal; + std::atomic dlnow; + std::atomic dlstatus; + std::atomic status; } urlDownload_t; typedef struct fileDownload_s { @@ -116,7 +114,7 @@ typedef struct backgroundDownload_s { idFile * f; fileDownload_t file; urlDownload_t url; - volatile bool completed; + std::atomic completed; } backgroundDownload_t; // file list for directory listings diff --git a/src/framework/GameDirPolicy.h b/src/framework/GameDirPolicy.h new file mode 100644 index 00000000..10f3cc90 --- /dev/null +++ b/src/framework/GameDirPolicy.h @@ -0,0 +1,108 @@ +/* +=========================================================================== + +openQ4 GPL Source Code +Copyright (C) 2026 the openQ4 contributors. + +This file is part of the openQ4 Source Code. See docs/legal for details. + +=========================================================================== +*/ + +#ifndef __GAMEDIRPOLICY_H__ +#define __GAMEDIRPOLICY_H__ + +// fs_game values cross network, package and host-filesystem boundaries. Keep +// them to one filename segment that has the same meaning on every supported +// host; in particular, never let a separator or Windows alias reach a path +// join performed under a trusted package root. +namespace idGameDirPolicy { + +static const int MAX_SEGMENT_BYTES = 255; + +inline unsigned char LowerASCII( unsigned char value ) { + return value >= 'A' && value <= 'Z' ? static_cast( value + ( 'a' - 'A' ) ) : value; +} + +inline bool HasASCIIStem( const char *segment, int stemLength, const char *expected, int expectedLength ) { + if ( stemLength != expectedLength ) { + return false; + } + for ( int index = 0; index < expectedLength; ++index ) { + if ( LowerASCII( static_cast( segment[ index ] ) ) != + static_cast( expected[ index ] ) ) { + return false; + } + } + return true; +} + +inline bool HasASCIIPrefix( const char *segment, const char *expected, int expectedLength ) { + for ( int index = 0; index < expectedLength; ++index ) { + if ( LowerASCII( static_cast( segment[ index ] ) ) != + static_cast( expected[ index ] ) ) { + return false; + } + } + return true; +} + +inline bool IsWindowsDeviceName( const char *segment, int segmentLength ) { + int stemLength = 0; + while ( stemLength < segmentLength && segment[ stemLength ] != '.' ) { + stemLength++; + } + + if ( HasASCIIStem( segment, stemLength, "con", 3 ) || + HasASCIIStem( segment, stemLength, "prn", 3 ) || + HasASCIIStem( segment, stemLength, "aux", 3 ) || + HasASCIIStem( segment, stemLength, "nul", 3 ) ) { + return true; + } + + if ( stemLength < 4 ) { + return false; + } + const bool portPrefix = HasASCIIPrefix( segment, "com", 3 ) || HasASCIIPrefix( segment, "lpt", 3 ); + if ( !portPrefix ) { + return false; + } + + const unsigned char digit = static_cast( segment[ 3 ] ); + if ( stemLength == 4 ) { + return ( digit >= '1' && digit <= '9' ) || digit == 0xB9 || digit == 0xB2 || digit == 0xB3; + } + + return stemLength == 5 && digit == 0xC2 && + ( static_cast( segment[ 4 ] ) == 0xB9 || + static_cast( segment[ 4 ] ) == 0xB2 || + static_cast( segment[ 4 ] ) == 0xB3 ); +} + +inline bool IsPortableSegment( const char *segment ) { + if ( segment == nullptr || segment[ 0 ] == '\0' ) { + return false; + } + + int segmentLength = 0; + for ( const unsigned char *scan = reinterpret_cast( segment ); *scan != '\0'; ++scan ) { + if ( ++segmentLength > MAX_SEGMENT_BYTES || *scan < 32 || *scan == '/' || *scan == '\\' || + *scan == ':' || *scan == '<' || *scan == '>' || *scan == '"' || *scan == '|' || + *scan == '?' || *scan == '*' ) { + return false; + } + } + + if ( ( segmentLength == 1 && segment[ 0 ] == '.' ) || + ( segmentLength == 2 && segment[ 0 ] == '.' && segment[ 1 ] == '.' ) || + segment[ 0 ] == ' ' || segment[ segmentLength - 1 ] == '.' || + segment[ segmentLength - 1 ] == ' ' ) { + return false; + } + + return !IsWindowsDeviceName( segment, segmentLength ); +} + +} // namespace idGameDirPolicy + +#endif /* !__GAMEDIRPOLICY_H__ */ diff --git a/src/framework/RemoteCVarPolicy.h b/src/framework/RemoteCVarPolicy.h new file mode 100644 index 00000000..cbc133dd --- /dev/null +++ b/src/framework/RemoteCVarPolicy.h @@ -0,0 +1,35 @@ +/* +=========================================================================== + +openQ4 GPL Source Code +Copyright (C) 2026 the openQ4 contributors. + +This file is part of the openQ4 Source Code. See docs/legal for details. + +=========================================================================== +*/ + +#ifndef __REMOTECVARPOLICY_H__ +#define __REMOTECVARPOLICY_H__ + +// This small, dependency-free policy is shared with the native safety test. +// The caller supplies the engine's concrete CVar flag mask, keeping this +// header independent of the large CVarSystem interface and game-module ABI. +namespace idRemoteCVarPolicy { + +inline bool IsSingleAllowedAuthority( int requiredFlag, int allowedRemoteFlags ) { + return requiredFlag != 0 && + ( requiredFlag & allowedRemoteFlags ) == requiredFlag && + ( requiredFlag & ( requiredFlag - 1 ) ) == 0; +} + +inline bool CanApply( int variableFlags, int requiredFlag, int allowedRemoteFlags, + int forbiddenFlags ) { + return IsSingleAllowedAuthority( requiredFlag, allowedRemoteFlags ) && + ( variableFlags & requiredFlag ) != 0 && + ( variableFlags & forbiddenFlags ) == 0; +} + +} // namespace idRemoteCVarPolicy + +#endif /* !__REMOTECVARPOLICY_H__ */ diff --git a/src/framework/Session.cpp b/src/framework/Session.cpp index dcf69f29..960283b9 100644 --- a/src/framework/Session.cpp +++ b/src/framework/Session.cpp @@ -1423,6 +1423,25 @@ class sessionSaveOperationGuard_t { int previousLastCheckPoint; bool complete; }; + +class sessionRenderCropGuard_t { +public: + sessionRenderCropGuard_t( int width, int height ) : active( false ) { + if ( renderSystem != NULL && renderSystem->IsOpenGLRunning() && width > 0 && height > 0 ) { + renderSystem->CropRenderSize( width, height, false, true ); + active = true; + } + } + + ~sessionRenderCropGuard_t() { + if ( active && renderSystem != NULL ) { + renderSystem->UnCrop(); + } + } + +private: + bool active; +}; #endif void idSessionLocal::ResetFramePacingStats( void ) { @@ -2409,6 +2428,22 @@ static bool Session_PrepareExpandedLoadingBackground( const idStr &backgroundPat fileSystem->RemoveFileChecked( stagingPath.c_str(), "fs_savepath" ); common->Warning( "Could not publish expanded loading background '%s'; using the source levelshot", generatedPath.c_str() ); + } else if ( !Session_FileExistsInSearchPaths( generatedPath.c_str() ) ) { + // A pure server deliberately excludes loose image files even when this + // client just authored them under fs_savepath. Publication alone is not + // permission to consume the file: require the active VFS policy to expose + // it before handing its name to the loading GUI/material system. Keeping + // published=false leaves the caller's stock levelshot selected. + common->DPrintf( "Expanded loading background '%s' is unavailable through the active VFS; using the source levelshot\n", + generatedPath.c_str() ); + published = false; + } else { + const idMaterial *generatedMaterial = declManager->FindMaterial( generatedPath.c_str() ); + if ( generatedMaterial == NULL || generatedMaterial->TestMaterialFlag( MF_DEFAULTED ) ) { + common->DPrintf( "Expanded loading background '%s' has no usable material; using the source levelshot\n", + generatedPath.c_str() ); + published = false; + } } R_StaticFree( centerPic ); @@ -3519,6 +3554,24 @@ static void Session_openQ4AssertMapState_f( const idCmdArgs &args ) { actualEntityFilter.c_str() ); } } + +/* +================== +Session_openQ4AssertMPGameplayView_f +================== +*/ +static void Session_openQ4AssertMPGameplayView_f( const idCmdArgs &args ) { + if ( !sessLocal.IsMapSpawned() || !sessLocal.IsMultiplayer() || + sessLocal.GetActiveGUI() != NULL ) { + common->Error( "openq4_assertMPGameplayView failed: map=%d multiplayer=%d gui=%d", + sessLocal.IsMapSpawned() ? 1 : 0, + sessLocal.IsMultiplayer() ? 1 : 0, + sessLocal.GetActiveGUI() != NULL ? 1 : 0 ); + return; + } + + common->Printf( "OPENQ4_STOCK_BASELINE_MP_CLIENT_VIEW gui=0\n" ); +} #endif /* @@ -6133,14 +6186,17 @@ bool idSessionLocal::SaveGame( const char *saveName, saveType_t saveType ) { // Write screenshot if ( saveType != ST_AUTO ) { - renderSystem->CropRenderSize( 320, 240, false ); if ( rw ) { rw->PushMarkedDefs(); } + // The current crop can be smaller than the drawable in legacy + // r_screenFraction mode. Push a physical-size crop so all screen-space + // feedback targets see one coherent frame, then restore the prior crop. + sessionRenderCropGuard_t previewCrop( renderSystem->GetScreenWidth(), renderSystem->GetScreenHeight() ); common->SetRenderableGameFrame( true ); game->Draw( 0 ); - renderSystem->CaptureRenderToFile( tempPreviewFile, true ); - renderSystem->UnCrop(); + // The renderer reduces the coherent physical frame after readback. + renderSystem->CaptureRenderToFile( tempPreviewFile, true, 320, 240 ); } mapName = mapSpawnData.serverInfo.GetString( "si_map" ); @@ -7719,6 +7775,7 @@ void idSessionLocal::Init() { #ifndef ID_DEDICATED cmdSystem->AddCommand( "openq4_startSingleplayer", Session_openQ4StartSingleplayer_f, CMD_FL_SYSTEM, "internal helper to start singleplayer after game-module switches" ); cmdSystem->AddCommand( "openq4_assertMapState", Session_openQ4AssertMapState_f, CMD_FL_SYSTEM|CMD_FL_CHEAT, "asserts the active map and entity filter for validation harnesses" ); + cmdSystem->AddCommand( "openq4_assertMPGameplayView", Session_openQ4AssertMPGameplayView_f, CMD_FL_SYSTEM|CMD_FL_CHEAT, "asserts that multiplayer rendering is not covered by an active session GUI" ); cmdSystem->AddCommand( "openq4_resumeBakeLightGrids", Session_openQ4ResumeBakeLightGrids_f, CMD_FL_SYSTEM|CMD_FL_CHEAT, "internal helper to continue light-grid baking after game-module switches" ); cmdSystem->AddCommand( "iamtheduke", Session_IAmTheDuke_f, CMD_FL_SYSTEM|CMD_FL_CHEAT, "toggles the SP-only iamtheduke cheat text overlay" ); cmdSystem->AddCommand( "bakeLightGrids", Session_BakeLightGrids_f, CMD_FL_SYSTEM|CMD_FL_CHEAT, "bakes openQ4-compatible lightgrid metadata and irradiance atlases for the current map or a batch of maps" ); diff --git a/src/framework/async/AsyncClient.cpp b/src/framework/async/AsyncClient.cpp index 6007a096..1aad71ff 100644 --- a/src/framework/async/AsyncClient.cpp +++ b/src/framework/async/AsyncClient.cpp @@ -30,10 +30,54 @@ If you have questions concerning this license or the applicable additional terms #include "AsyncNetwork.h" +#include "Rcon2Protocol.h" #include "../ArenaCampaign.h" #include "../Session_local.h" #include "../../sys/NetworkEndpoint.h" +#include "../../sys/URLPolicy.h" + +#include + +static const int ASYNC_CLIENT_MAX_NETWORK_TIME = idMath::INT_MAX - 65536; +static const int ASYNC_CLIENT_MAX_PREDICTION_MSEC = 60000; + +static ID_INLINE int AsyncClient_MaxNetworkGameFrame() { + const std::int64_t msecNumerator = common->GetUserCmdMsecNumerator(); + const std::int64_t msecDenominator = common->GetUserCmdMsecDenominator(); + const std::int64_t legacyTickMsec = common->GetUserCmdMSec(); + if ( msecNumerator <= 0 || msecDenominator <= 0 || legacyTickMsec <= 0 ) { + return 0; + } + + // Both exact-tic and legacy integer-tic conversions are used below. Leave + // one frame of headroom for GetUserCmdTime( frame + 1 ) as well. + const std::int64_t exactLimit = + ( static_cast( ASYNC_CLIENT_MAX_NETWORK_TIME ) * msecDenominator ) / msecNumerator; + const std::int64_t legacyLimit = + static_cast( ASYNC_CLIENT_MAX_NETWORK_TIME ) / legacyTickMsec; + const std::int64_t maximumFrame = ( exactLimit < legacyLimit ? exactLimit : legacyLimit ) - 1; + return maximumFrame > 0 ? static_cast( maximumFrame ) : 0; +} + +static ID_INLINE bool AsyncClient_ValidNetworkTiming( int gameFrame, int gameTime ) { + return gameFrame >= 0 && gameFrame <= AsyncClient_MaxNetworkGameFrame() && + gameTime >= 0 && gameTime <= ASYNC_CLIENT_MAX_NETWORK_TIME; +} + +static void AsyncClient_StopAfterMalformedSnapshot() { + // ClientReadSnapshot may already have published timing, entities, or PVS state + // before a later field proves that the packet is malformed. Disconnecting the + // channel alone leaves that partially updated map available to the remainder of + // the outer frame, so tear down the loaded session immediately. + arenaCampaign.AbortMatch(); + session->Stop(); +} + +static ID_INLINE int AsyncClient_MaxPredictionMsec() { + return idMath::ClampInt( 0, ASYNC_CLIENT_MAX_PREDICTION_MSEC, + idAsyncNetwork::clientMaxPrediction.GetInteger() ); +} static ID_INLINE int AsyncClient_NextGameFrameMsec( int gameFrame ) { return common->GetUserCmdDeltaMsec( gameFrame + 1 ); @@ -42,11 +86,35 @@ static ID_INLINE int AsyncClient_NextGameFrameMsec( int gameFrame ) { static ID_INLINE int AsyncClient_ConfiguredPredictionMsec( int gameFrame ) { // Preserve the legacy default as "one exact tic" instead of a literal 16 ms. if ( idAsyncNetwork::clientPrediction.GetInteger() == 16 ) { - return AsyncClient_NextGameFrameMsec( gameFrame ); + return Min( AsyncClient_NextGameFrameMsec( gameFrame ), AsyncClient_MaxPredictionMsec() ); } - return Max( 0, idAsyncNetwork::clientPrediction.GetInteger() ); + return idMath::ClampInt( 0, AsyncClient_MaxPredictionMsec(), + idAsyncNetwork::clientPrediction.GetInteger() ); +} + +static ID_INLINE bool AsyncClient_SameEndpoint( const netadr_t &left, const netadr_t &right ) { + return left.port == right.port && Sys_CompareNetAdrBase( left, right ); } +static ID_INLINE std::uint32_t AsyncClient_Elapsed( int now, int then ) { + return static_cast( now ) - static_cast( then ); +} + +static bool AsyncClient_SecureConnectionId( int &identifier ) { + std::uint32_t randomValue = 0; + if ( !Sys_GetSecureRandomBytes( &randomValue, sizeof( randomValue ) ) ) { + return false; + } + identifier = static_cast( randomValue & CONNECTIONLESS_MESSAGE_ID_MASK ); + if ( identifier == CONNECTIONLESS_MESSAGE_ID_MASK ) { + identifier = 0; + } + return true; +} + +static const int RCON2_CLIENT_TIMEOUT_MSEC = 10000; +static const int RCON2_CLIENT_RESEND_MSEC = 1000; + static bool AsyncClient_IsWindowsDevicePathSegment( const char *segment, int segmentLength ) { int stemLength = 0; while ( stemLength < segmentLength && segment[ stemLength ] != '.' ) { @@ -179,6 +247,9 @@ void idAsyncClient::Clear( void ) { backgroundDownload.completed = true; backgroundDownload.url.expectedSize = 0; lastRconTime = 0; + memset( &lastRconAddress, 0, sizeof( lastRconAddress ) ); + idCrypto::SecureZero( &rcon2Request, sizeof( rcon2Request ) ); + rcon2Request.state = RCON_REPLY_NONE; showUpdateMessage = false; lastFrameDelta = 0; @@ -197,9 +268,6 @@ idAsyncClient::Shutdown void idAsyncClient::Shutdown( void ) { guiNetMenu = NULL; updateMSG.Clear(); - updateURL.Clear(); - updateFile.Clear(); - updateFallback.Clear(); backgroundDownload.url.url.Clear(); dlList.Clear(); } @@ -229,6 +297,7 @@ idAsyncClient::ClosePort ================== */ void idAsyncClient::ClosePort( void ) { + ClearRemoteConsoleRequest(); clientPort.Close(); } @@ -302,8 +371,11 @@ void idAsyncClient::ConnectToServer( const netadr_t adr ) { // clear the client state Clear(); - // get a pseudo random client id, but don't use the id which is reserved for connectionless packets - clientId = Sys_Milliseconds() & CONNECTIONLESS_MESSAGE_ID_MASK; + // Keep the legacy 15-bit wire field but remove the predictable clock seed. + if ( !AsyncClient_SecureConnectionId( clientId ) ) { + common->Warning( "OS secure random unavailable; connection attempt cancelled" ); + return; + } // calculate a checksum on some of the essential data used clientDataChecksum = declManager->GetChecksum(); @@ -578,12 +650,14 @@ idAsyncClient::RemoteConsole */ void idAsyncClient::RemoteConsole( const char *command ) { netadr_t adr; - idBitMsg msg; - byte msgBuf[MAX_MESSAGE_SIZE]; if ( !InitPort() ) { return; } + if ( command == NULL || command[0] == '\0' || strlen( command ) >= MAX_STRING_CHARS ) { + common->Printf( "usage: rcon (maximum %d bytes)\n", MAX_STRING_CHARS - 1 ); + return; + } if ( active ) { adr = serverAddress; @@ -599,16 +673,205 @@ void idAsyncClient::RemoteConsole( const char *command ) { adr.port = PORT_SERVER; } + ClearRemoteConsoleRequest(); lastRconAddress = adr; lastRconTime = realTime; + const char *password = idAsyncNetwork::clientRemoteConsolePassword.GetString(); + if ( password[0] == '\0' ) { + common->Printf( "Set net_clientRemoteConsolePassword before using rcon.\n" ); + ClearRemoteConsoleRequest(); + return; + } + + if ( idAsyncNetwork::clientUseLegacyRcon.GetBool() ) { + byte msgBuf[MAX_MESSAGE_SIZE]; + idBitMsg msg; + msg.Init( msgBuf, sizeof( msgBuf ) ); + msg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); + msg.WriteString( "rcon" ); + msg.WriteString( password ); + msg.WriteString( command ); + rcon2Request.state = RCON_REPLY_LEGACY; + rcon2Request.address = adr; + rcon2Request.startTime = realTime; + common->Warning( "sending legacy rcon password as plaintext because net_clientUseLegacyRcon is enabled" ); + clientPort.SendPacket( adr, msg.GetData(), msg.GetSize() ); + idCrypto::SecureZero( msgBuf, sizeof( msgBuf ) ); + return; + } + + if ( strlen( password ) < idRcon2::MIN_PASSWORD_BYTES ) { + common->Printf( "rcon2 requires a password of at least %u bytes.\n", + static_cast( idRcon2::MIN_PASSWORD_BYTES ) ); + ClearRemoteConsoleRequest(); + return; + } + + rcon2Request.state = RCON_REPLY_CHALLENGE; + rcon2Request.address = adr; + rcon2Request.startTime = realTime; + rcon2Request.lastSendTime = realTime; + idStr::Copynz( rcon2Request.command, command, sizeof( rcon2Request.command ) ); + if ( !Sys_GetSecureRandomBytes( rcon2Request.clientNonce, sizeof( rcon2Request.clientNonce ) ) ) { + common->Warning( "OS secure random unavailable; rcon2 request cancelled" ); + ClearRemoteConsoleRequest(); + return; + } + idRcon2::HashRequest( rcon2Request.command, rcon2Request.requestDigest ); + SendRemoteConsole2Challenge(); +} + +/* +================== +idAsyncClient::ClearRemoteConsoleRequest +================== +*/ +void idAsyncClient::ClearRemoteConsoleRequest( void ) { + idCrypto::SecureZero( &rcon2Request, sizeof( rcon2Request ) ); + rcon2Request.state = RCON_REPLY_NONE; + memset( &lastRconAddress, 0, sizeof( lastRconAddress ) ); + lastRconTime = 0; +} + +void idAsyncClient::SendRemoteConsole2Challenge( void ) { + if ( rcon2Request.state != RCON_REPLY_CHALLENGE ) { + return; + } + byte msgBuf[128]; + idBitMsg msg; + msg.Init( msgBuf, sizeof( msgBuf ) ); + msg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); + msg.WriteString( "rcon2Challenge" ); + msg.WriteByte( idRcon2::PROTOCOL_VERSION ); + msg.WriteData( rcon2Request.clientNonce, sizeof( rcon2Request.clientNonce ) ); + msg.WriteData( rcon2Request.requestDigest, sizeof( rcon2Request.requestDigest ) ); + clientPort.SendPacket( rcon2Request.address, msg.GetData(), msg.GetSize() ); + rcon2Request.lastSendTime = realTime; + lastRconTime = realTime; + idCrypto::SecureZero( msgBuf, sizeof( msgBuf ) ); +} + +void idAsyncClient::SendRemoteConsole2Proof( void ) { + if ( rcon2Request.state != RCON_REPLY_OUTPUT ) { + return; + } + byte msgBuf[MAX_MESSAGE_SIZE]; + idBitMsg msg; msg.Init( msgBuf, sizeof( msgBuf ) ); msg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); - msg.WriteString( "rcon" ); - msg.WriteString( idAsyncNetwork::clientRemoteConsolePassword.GetString() ); - msg.WriteString( command ); + msg.WriteString( "rcon2" ); + msg.WriteByte( idRcon2::PROTOCOL_VERSION ); + msg.WriteData( rcon2Request.clientNonce, sizeof( rcon2Request.clientNonce ) ); + msg.WriteData( rcon2Request.serverNonce, sizeof( rcon2Request.serverNonce ) ); + msg.WriteString( rcon2Request.command ); + msg.WriteData( rcon2Request.proof, sizeof( rcon2Request.proof ) ); + clientPort.SendPacket( rcon2Request.address, msg.GetData(), msg.GetSize() ); + rcon2Request.lastSendTime = realTime; + lastRconTime = realTime; + idCrypto::SecureZero( msgBuf, sizeof( msgBuf ) ); +} - clientPort.SendPacket( adr, msg.GetData(), msg.GetSize() ); +void idAsyncClient::UpdateRemoteConsoleRequest( void ) { + if ( rcon2Request.state == RCON_REPLY_NONE ) { + return; + } + if ( AsyncClient_Elapsed( realTime, rcon2Request.startTime ) > RCON2_CLIENT_TIMEOUT_MSEC ) { + common->Printf( "remote console request timed out\n" ); + ClearRemoteConsoleRequest(); + return; + } + if ( rcon2Request.state == RCON_REPLY_LEGACY ) { + return; + } + if ( AsyncClient_Elapsed( realTime, rcon2Request.lastSendTime ) < RCON2_CLIENT_RESEND_MSEC ) { + return; + } + if ( rcon2Request.state == RCON_REPLY_CHALLENGE ) { + SendRemoteConsole2Challenge(); + } else if ( rcon2Request.state == RCON_REPLY_OUTPUT ) { + SendRemoteConsole2Proof(); + } +} + +/* +================== +idAsyncClient::ProcessRemoteConsole2ChallengeResponse +================== +*/ +void idAsyncClient::ProcessRemoteConsole2ChallengeResponse( const netadr_t from, const idBitMsg &msg ) { + const int responseBytes = 1 + idRcon2::NONCE_BYTES + idRcon2::NONCE_BYTES + + idRcon2::SALT_BYTES + 4 + idRcon2::ENDPOINT_BINDING_BYTES + idRcon2::REQUEST_DIGEST_BYTES; + if ( rcon2Request.state != RCON_REPLY_CHALLENGE || + !AsyncClient_SameEndpoint( from, rcon2Request.address ) || + msg.GetRemainingData() != responseBytes || + msg.ReadByte() != idRcon2::PROTOCOL_VERSION ) { + return; + } + byte clientNonce[idRcon2::NONCE_BYTES]; + byte serverNonce[idRcon2::NONCE_BYTES]; + byte salt[idRcon2::SALT_BYTES]; + byte endpointBinding[idRcon2::ENDPOINT_BINDING_BYTES]; + byte requestDigest[idRcon2::REQUEST_DIGEST_BYTES]; + byte verifier[idRcon2::VERIFIER_BYTES]; + msg.ReadData( clientNonce, sizeof( clientNonce ) ); + msg.ReadData( serverNonce, sizeof( serverNonce ) ); + msg.ReadData( salt, sizeof( salt ) ); + const int iterations = msg.ReadLong(); + msg.ReadData( endpointBinding, sizeof( endpointBinding ) ); + msg.ReadData( requestDigest, sizeof( requestDigest ) ); + + const bool responseMatches = iterations == static_cast( idRcon2::PBKDF2_ITERATIONS ) && + idCrypto::ConstantTimeEquals( clientNonce, rcon2Request.clientNonce, sizeof( clientNonce ) ) && + idCrypto::ConstantTimeEquals( requestDigest, rcon2Request.requestDigest, sizeof( requestDigest ) ); + if ( !responseMatches || !idRcon2::DeriveVerifier( + idAsyncNetwork::clientRemoteConsolePassword.GetString(), salt, verifier ) ) { + common->Warning( "invalid rcon2 challenge response" ); + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( serverNonce, sizeof( serverNonce ) ); + idCrypto::SecureZero( salt, sizeof( salt ) ); + idCrypto::SecureZero( endpointBinding, sizeof( endpointBinding ) ); + idCrypto::SecureZero( requestDigest, sizeof( requestDigest ) ); + idCrypto::SecureZero( verifier, sizeof( verifier ) ); + ClearRemoteConsoleRequest(); + return; + } + + memcpy( rcon2Request.serverNonce, serverNonce, sizeof( rcon2Request.serverNonce ) ); + idRcon2::ComputeProof( verifier, rcon2Request.clientNonce, serverNonce, + endpointBinding, rcon2Request.requestDigest, rcon2Request.proof ); + rcon2Request.state = RCON_REPLY_OUTPUT; + rcon2Request.lastSendTime = realTime; + SendRemoteConsole2Proof(); + + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( serverNonce, sizeof( serverNonce ) ); + idCrypto::SecureZero( salt, sizeof( salt ) ); + idCrypto::SecureZero( endpointBinding, sizeof( endpointBinding ) ); + idCrypto::SecureZero( requestDigest, sizeof( requestDigest ) ); + idCrypto::SecureZero( verifier, sizeof( verifier ) ); +} + +void idAsyncClient::ProcessRemoteConsole2Complete( const netadr_t from, const idBitMsg &msg ) { + const int completeBytes = 1 + idRcon2::NONCE_BYTES + idRcon2::NONCE_BYTES; + if ( rcon2Request.state != RCON_REPLY_OUTPUT || + !AsyncClient_SameEndpoint( from, rcon2Request.address ) || + msg.GetRemainingData() != completeBytes || + msg.ReadByte() != idRcon2::PROTOCOL_VERSION ) { + return; + } + byte clientNonce[idRcon2::NONCE_BYTES]; + byte serverNonce[idRcon2::NONCE_BYTES]; + msg.ReadData( clientNonce, sizeof( clientNonce ) ); + msg.ReadData( serverNonce, sizeof( serverNonce ) ); + const bool matches = idCrypto::ConstantTimeEquals( clientNonce, + rcon2Request.clientNonce, sizeof( clientNonce ) ) && + idCrypto::ConstantTimeEquals( serverNonce, rcon2Request.serverNonce, sizeof( serverNonce ) ); + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( serverNonce, sizeof( serverNonce ) ); + if ( matches ) { + ClearRemoteConsoleRequest(); + } } /* @@ -835,6 +1098,10 @@ void idAsyncClient::SendUsercmdsToServer( void ) { if ( idAsyncNetwork::verbose.GetInteger() == 2 ) { common->Printf( "sending usercmd to server: gameInitId = %d, gameFrame = %d, gameTime = %d\n", gameInitId, gameFrame, gameTime ); } + if ( gameFrame < 0 || gameFrame > AsyncClient_MaxNetworkGameFrame() ) { + common->Warning( "cannot send a user command for invalid game frame %d", gameFrame ); + return; + } // generate user command for this client index = gameFrame & ( MAX_USERCMD_BACKUP - 1 ); @@ -850,7 +1117,9 @@ void idAsyncClient::SendUsercmdsToServer( void ) { msg.WriteByte( CLIENT_UNRELIABLE_MESSAGE_USERCMD ); msg.WriteShort( clientPrediction ); - numUsercmds = idMath::ClampInt( 0, 10, idAsyncNetwork::clientUsercmdBackup.GetInteger() ) + 1; + const int requestedUsercmds = idMath::ClampInt( 0, MAX_USERCMD_PACKET_COMMANDS - 1, + idAsyncNetwork::clientUsercmdBackup.GetInteger() ) + 1; + numUsercmds = gameFrame >= requestedUsercmds - 1 ? requestedUsercmds : gameFrame + 1; // write the user commands msg.WriteLong( gameFrame ); @@ -898,6 +1167,11 @@ void idAsyncClient::ProcessUnreliableServerMessage( const idBitMsg &msg ) { usercmd_t *last; bool pureWait; + if ( msg.GetRemainingReadBits() < 32 + 8 ) { + common->Warning( "server sent a truncated unreliable message; disconnecting safely" ); + DisconnectFromServer(); + return; + } serverGameInitId = msg.ReadLong(); id = msg.ReadByte(); @@ -912,13 +1186,28 @@ void idAsyncClient::ProcessUnreliableServerMessage( const idBitMsg &msg ) { if ( idAsyncNetwork::verbose.GetInteger() == 2 ) { common->Printf( "received ping message from server\n" ); } + if ( msg.GetRemainingReadBits() < 32 ) { + common->Warning( "server sent a truncated ping; disconnecting safely" ); + DisconnectFromServer(); + return; + } SendPingResponseToServer( msg.ReadLong() ); break; } case SERVER_UNRELIABLE_MESSAGE_GAMEINIT: { + if ( msg.GetRemainingReadBits() < 32 + 32 ) { + common->Warning( "server sent a truncated game-init message; disconnecting safely" ); + DisconnectFromServer(); + return; + } serverGameFrame = msg.ReadLong(); serverGameTime = msg.ReadLong(); msg.ReadDeltaDict( serverSI, NULL ); + if ( msg.IsReadOverflowed() || !AsyncClient_ValidNetworkTiming( serverGameFrame, serverGameTime ) ) { + common->Warning( "server sent invalid game-init timing; disconnecting safely" ); + DisconnectFromServer(); + return; + } pureWait = serverSI.GetBool( "si_pure" ); InitGame( serverGameInitId, serverGameFrame, serverGameTime, serverSI ); @@ -956,11 +1245,25 @@ void idAsyncClient::ProcessUnreliableServerMessage( const idBitMsg &msg ) { break; } - snapshotSequence = msg.ReadLong(); - snapshotGameFrame = msg.ReadLong(); - snapshotGameTime = msg.ReadLong(); + if ( msg.GetRemainingReadBits() < 32 + 32 + 32 + 8 + 16 ) { + common->Warning( "server sent a truncated snapshot header; disconnecting safely" ); + DisconnectFromServer(); + return; + } + const int receivedSnapshotSequence = msg.ReadLong(); + const int receivedSnapshotGameFrame = msg.ReadLong(); + const int receivedSnapshotGameTime = msg.ReadLong(); numDuplicatedUsercmds = msg.ReadByte(); aheadOfServer = msg.ReadShort(); + if ( msg.IsReadOverflowed() || + !AsyncClient_ValidNetworkTiming( receivedSnapshotGameFrame, receivedSnapshotGameTime ) ) { + common->Warning( "server sent invalid snapshot timing; disconnecting safely" ); + DisconnectFromServer(); + return; + } + snapshotSequence = receivedSnapshotSequence; + snapshotGameFrame = receivedSnapshotGameFrame; + snapshotGameTime = receivedSnapshotGameTime; // read the game snapshot if ( !game->ClientReadSnapshot( @@ -968,20 +1271,38 @@ void idAsyncClient::ProcessUnreliableServerMessage( const idBitMsg &msg ) { numDuplicatedUsercmds, aheadOfServer, msg ) ) { common->Warning( "server sent malformed snapshot %d; disconnecting safely", snapshotSequence ); - DisconnectFromServer(); + AsyncClient_StopAfterMalformedSnapshot(); return; } // read user commands of other clients from the snapshot - for ( last = NULL, i = msg.ReadByte(); i < MAX_ASYNC_CLIENTS; i = msg.ReadByte() ) { - numUsercmds = msg.ReadByte(); - if ( numUsercmds > MAX_USERCMD_RELAY ) { - common->Error( "snapshot %d contains too many user commands for client %d", snapshotSequence, i ); + last = NULL; + while ( true ) { + i = msg.ReadByte(); + if ( msg.IsReadOverflowed() || i > MAX_ASYNC_CLIENTS ) { + common->Warning( "snapshot %d has an invalid user-command terminator; disconnecting safely", + snapshotSequence ); + AsyncClient_StopAfterMalformedSnapshot(); + return; + } + if ( i == MAX_ASYNC_CLIENTS ) { break; } + numUsercmds = msg.ReadByte(); + if ( msg.IsReadOverflowed() || numUsercmds < 1 || numUsercmds > MAX_USERCMD_RELAY ) { + common->Warning( "snapshot %d contains an invalid user-command count for client %d; disconnecting safely", + snapshotSequence, i ); + AsyncClient_StopAfterMalformedSnapshot(); + return; + } for ( j = 0; j < numUsercmds; j++ ) { index = ( snapshotGameFrame + j ) & ( MAX_USERCMD_BACKUP - 1 ); - idAsyncNetwork::ReadUserCmdDelta( msg, userCmds[index][i], last ); + if ( !idAsyncNetwork::ReadUserCmdDelta( msg, userCmds[index][i], last ) ) { + common->Warning( "snapshot %d contains a truncated user command for client %d; disconnecting safely", + snapshotSequence, i ); + AsyncClient_StopAfterMalformedSnapshot(); + return; + } userCmds[index][i].gameFrame = snapshotGameFrame + j; userCmds[index][i].duplicateCount = 0; last = &userCmds[index][i]; @@ -1005,19 +1326,24 @@ void idAsyncClient::ProcessUnreliableServerMessage( const idBitMsg &msg ) { } // if the snapshot is newer than the clients current game time - if ( gameTime < snapshotGameTime || gameTime > snapshotGameTime + idAsyncNetwork::clientMaxPrediction.GetInteger() ) { + const int maximumPredictionMsec = AsyncClient_MaxPredictionMsec(); + if ( gameTime < snapshotGameTime || gameTime > snapshotGameTime + maximumPredictionMsec ) { gameFrame = snapshotGameFrame; gameTime = snapshotGameTime; - gameTimeResidual = idMath::ClampInt( -idAsyncNetwork::clientMaxPrediction.GetInteger(), idAsyncNetwork::clientMaxPrediction.GetInteger(), gameTimeResidual ); - clientPredictTime = idMath::ClampInt( -idAsyncNetwork::clientMaxPrediction.GetInteger(), idAsyncNetwork::clientMaxPrediction.GetInteger(), clientPredictTime ); + gameTimeResidual = idMath::ClampInt( -maximumPredictionMsec, maximumPredictionMsec, gameTimeResidual ); + clientPredictTime = idMath::ClampInt( -maximumPredictionMsec, maximumPredictionMsec, clientPredictTime ); } // adjust the client prediction time based on the snapshot time const int configuredPredictionMsec = AsyncClient_ConfiguredPredictionMsec( gameFrame ); clientPrediction -= ( 1 - ( INTSIGNBITSET( aheadOfServer - configuredPredictionMsec ) << 1 ) ); - clientPrediction = idMath::ClampInt( configuredPredictionMsec, idAsyncNetwork::clientMaxPrediction.GetInteger(), clientPrediction ); + clientPrediction = idMath::ClampInt( configuredPredictionMsec, maximumPredictionMsec, clientPrediction ); delta = gameTime - ( snapshotGameTime + clientPrediction ); - clientPredictTime -= ( delta / PREDICTION_FAST_ADJUST ) + ( 1 - ( INTSIGNBITSET( delta ) << 1 ) ); + const std::int64_t adjustedPredictTime = static_cast( clientPredictTime ) - + ( delta / PREDICTION_FAST_ADJUST ) - ( 1 - ( INTSIGNBITSET( delta ) << 1 ) ); + clientPredictTime = adjustedPredictTime < -maximumPredictionMsec ? -maximumPredictionMsec : + ( adjustedPredictTime > maximumPredictionMsec ? maximumPredictionMsec : + static_cast( adjustedPredictTime ) ); lastSnapshotTime = clientTime; @@ -1126,6 +1452,11 @@ void idAsyncClient::ProcessReliableServerMessages( void ) { while ( channel.GetReliableMessage( msg ) ) { id = msg.ReadByte(); + if ( msg.IsReadOverflowed() ) { + common->Warning( "server sent an empty reliable message; disconnecting safely" ); + DisconnectFromServer(); + return; + } switch( id ) { case SERVER_RELIABLE_MESSAGE_CLIENTINFO: { int clientNum; @@ -1133,13 +1464,19 @@ void idAsyncClient::ProcessReliableServerMessages( void ) { // openQ4: wire value, so it indexes nothing until it is known to be a // client slot. userInfo only has MAX_ASYNC_CLIENTS entries. - if ( clientNum < 0 || clientNum >= MAX_ASYNC_CLIENTS ) { + if ( msg.IsReadOverflowed() || clientNum < 0 || clientNum >= MAX_ASYNC_CLIENTS ) { common->Warning( "SERVER_RELIABLE_MESSAGE_CLIENTINFO: bad client number %d, ignored", clientNum ); - break; + DisconnectFromServer(); + return; } idDict &info = sessLocal.mapSpawnData.userInfo[ clientNum ]; bool haveBase = ( msg.ReadBits( 1 ) != 0 ); + if ( msg.IsReadOverflowed() ) { + common->Warning( "SERVER_RELIABLE_MESSAGE_CLIENTINFO: truncated base flag" ); + DisconnectFromServer(); + return; + } #if ID_CLIENTINFO_TAGS int checksum = info.Checksum(); @@ -1157,11 +1494,16 @@ void idAsyncClient::ProcessReliableServerMessages( void ) { } else { msg.ReadDeltaDict( info, NULL ); } + if ( msg.IsReadOverflowed() ) { + common->Warning( "SERVER_RELIABLE_MESSAGE_CLIENTINFO: malformed userinfo dictionary" ); + DisconnectFromServer(); + return; + } // server forces us to a different userinfo if ( clientNum == idAsyncClient::clientNum ) { common->DPrintf( "local user info modified by server\n" ); - cvarSystem->SetCVarsFromDict( info ); + cvarSystem->SetCVarsFromDictByFlags( info, CVAR_USERINFO ); cvarSystem->ClearModifiedFlags( CVAR_USERINFO ); // don't emit back } game->SetUserInfo( clientNum, info, true ); @@ -1170,7 +1512,12 @@ void idAsyncClient::ProcessReliableServerMessages( void ) { case SERVER_RELIABLE_MESSAGE_SYNCEDCVARS: { idDict &info = sessLocal.mapSpawnData.syncedCVars; msg.ReadDeltaDict( info, &info ); - cvarSystem->SetCVarsFromDict( info ); + if ( msg.IsReadOverflowed() ) { + common->Warning( "SERVER_RELIABLE_MESSAGE_SYNCEDCVARS: malformed CVar dictionary" ); + DisconnectFromServer(); + return; + } + cvarSystem->SetCVarsFromDictByFlags( info, CVAR_NETWORKSYNC ); if ( !idAsyncNetwork::AreCheatsEnabled() ) { cvarSystem->ResetFlaggedVariables( CVAR_CHEAT ); } @@ -1299,7 +1646,7 @@ void idAsyncClient::ProcessChallengeResponseMessage( const netadr_t from, const return; } - common->Printf( "received challenge response 0x%x from %s\n", serverChallenge, Sys_NetAdrToString( from ) ); + common->DPrintf( "received connection challenge response from %s\n", Sys_NetAdrToString( from ) ); // start sending connect packets instead of challenge request packets clientState = CS_CONNECTING; @@ -1333,35 +1680,42 @@ void idAsyncClient::ProcessConnectResponseMessage( const netadr_t from, const id return; } - common->Printf( "received connect response from %s\n", Sys_NetAdrToString( from ) ); - + if ( msg.GetRemainingReadBits() < 32 + 32 + 32 + 32 ) { + common->Warning( "server sent a truncated connect response - aborting the connection" ); + cmdSystem->BufferCommandText( CMD_EXEC_APPEND, "disconnect\n" ); + return; + } const int serverClientNum = msg.ReadLong(); + serverGameInitId = msg.ReadLong(); + serverGameFrame = msg.ReadLong(); + serverGameTime = msg.ReadLong(); + msg.ReadDeltaDict( serverSI, NULL ); // openQ4: this wire value ends up indexing userInfo and userCmds for the whole // session, so refuse the connection outright rather than accept a bad slot. - if ( serverClientNum < 0 || serverClientNum >= MAX_ASYNC_CLIENTS ) { - common->Warning( "connect response assigned client number %d, out of range - aborting the connection", serverClientNum ); + if ( msg.IsReadOverflowed() || serverClientNum < 0 || serverClientNum >= MAX_ASYNC_CLIENTS || + !AsyncClient_ValidNetworkTiming( serverGameFrame, serverGameTime ) ) { + common->Warning( "server sent a malformed connect response - aborting the connection" ); cmdSystem->BufferCommandText( CMD_EXEC_APPEND, "disconnect\n" ); return; } + common->Printf( "received connect response from %s\n", Sys_NetAdrToString( from ) ); channel.Init( from, clientId ); clientNum = serverClientNum; clientState = CS_CONNECTED; lastPacketTime = -9999; - serverGameInitId = msg.ReadLong(); - serverGameFrame = msg.ReadLong(); - serverGameTime = msg.ReadLong(); - msg.ReadDeltaDict( serverSI, NULL ); - InitGame( serverGameInitId, serverGameFrame, serverGameTime, serverSI ); // load map session->SetGUI( NULL, NULL ); sessLocal.ExecuteMapChange(); - clientPredictTime = clientPrediction = idMath::ClampInt( 0, idAsyncNetwork::clientMaxPrediction.GetInteger(), clientTime - lastConnectTime ); + const std::uint32_t connectElapsed = AsyncClient_Elapsed( clientTime, lastConnectTime ); + const std::uint32_t maximumPrediction = static_cast( AsyncClient_MaxPredictionMsec() ); + clientPredictTime = clientPrediction = static_cast( + connectElapsed < maximumPrediction ? connectElapsed : maximumPrediction ); } /* @@ -1374,7 +1728,7 @@ void idAsyncClient::ProcessDisconnectMessage( const netadr_t from, const idBitMs common->Printf( "Disconnect packet while not connected.\n" ); return; } - if ( !Sys_CompareNetAdrBase( from, serverAddress ) ) { + if ( !AsyncClient_SameEndpoint( from, serverAddress ) ) { common->Printf( "Disconnect packet from unknown server.\n" ); return; } @@ -1585,22 +1939,27 @@ idAsyncClient::ProcessVersionMessage ================== */ void idAsyncClient::ProcessVersionMessage( const netadr_t from, const idBitMsg &msg ) { - char string[ MAX_STRING_CHARS ]; + char ignoredNetworkField[ MAX_STRING_CHARS ]; + (void)from; if ( updateState != UPDATE_SENT ) { common->Printf( "ProcessVersionMessage: version reply, != UPDATE_SENT\n" ); return; } + // Consume the complete legacy payload so old master servers remain wire + // compatible. The message, direct-download flag, URL, MIME action, and + // fallback URL are intentionally ignored: an unauthenticated datagram must + // never supply instructions or choose what the client downloads, opens, or + // executes. + msg.ReadString( ignoredNetworkField, sizeof( ignoredNetworkField ) ); + (void)msg.ReadByte(); + msg.ReadString( ignoredNetworkField, sizeof( ignoredNetworkField ) ); + (void)msg.ReadByte(); + msg.ReadString( ignoredNetworkField, sizeof( ignoredNetworkField ) ); + + updateMSG = common->GetLanguageDict()->GetString( "#str_104330" ); common->Printf( "A new version is available\n" ); - msg.ReadString( string, MAX_STRING_CHARS ); - updateMSG = string; - updateDirectDownload = ( msg.ReadByte() != 0 ); - msg.ReadString( string, MAX_STRING_CHARS ); - updateURL = string; - updateMime = (dlMime_t)msg.ReadByte(); - msg.ReadString( string, MAX_STRING_CHARS ); - updateFallback = string; updateState = UPDATE_READY; } @@ -1679,7 +2038,10 @@ bool idAsyncClient::ValidatePureServerChecksums( const netadr_t from, const idBi common->DPrintf( "game code pak: 0x%x\n", missingGamePakChecksum ); } // store the requested downloads - GetDownloadRequest( missingChecksums, numMissingChecksums, missingGamePakChecksum ); + if ( GetDownloadRequest( missingChecksums, numMissingChecksums, missingGamePakChecksum ) == -1 ) { + common->Warning( "OS secure random unavailable; download request cancelled" ); + return false; + } // build the download request message // NOTE: in a specific function? dlmsg.Init( msgBuf, sizeof( msgBuf ) ); @@ -1789,9 +2151,51 @@ void idAsyncClient::ConnectionlessMessage( const netadr_t from, const idBitMsg & } } - // ignore if not from the current/last server - if ( !Sys_CompareNetAdrBase( from, serverAddress ) && ( lastRconTime + 10000 < realTime || !Sys_CompareNetAdrBase( from, lastRconAddress ) ) ) { - common->DPrintf( "got message '%s' from bad source: %s\n", string, Sys_NetAdrToString( from ) ); + const bool fromCurrentServer = active && AsyncClient_SameEndpoint( from, serverAddress ); + const bool fromPendingRcon = rcon2Request.state != RCON_REPLY_NONE && + AsyncClient_Elapsed( realTime, rcon2Request.startTime ) <= RCON2_CLIENT_TIMEOUT_MSEC && + AsyncClient_SameEndpoint( from, rcon2Request.address ); + const bool fromPendingRconOutput = fromPendingRcon && + ( rcon2Request.state == RCON_REPLY_OUTPUT || rcon2Request.state == RCON_REPLY_LEGACY ); + + // Remote-console opcodes have their own exact-endpoint capability. A + // pending rcon request must not authorize this endpoint to inject game + // control messages, and the game server must not manufacture rcon replies + // when no matching request is pending. + if ( idStr::Icmp( string, "rcon2ChallengeResponse" ) == 0 ) { + if ( !fromPendingRcon ) { + common->DPrintf( "got rcon2 challenge response from bad source: %s\n", Sys_NetAdrToString( from ) ); + return; + } + ProcessRemoteConsole2ChallengeResponse( from, msg ); + return; + } + if ( idStr::Icmp( string, "rcon2Complete" ) == 0 ) { + if ( !fromPendingRcon ) { + common->DPrintf( "got rcon2 completion from bad source: %s\n", Sys_NetAdrToString( from ) ); + return; + } + ProcessRemoteConsole2Complete( from, msg ); + return; + } + if ( idStr::Icmp( string, "print" ) == 0 ) { + // A challenge request has not authorized an output window yet. Secure + // rcon2 opens it only after the proof is sent; legacy mode keeps its + // explicitly insecure bounded window for compatibility. Ordinary game + // prints remain tied to the exact current-server endpoint. + if ( !fromCurrentServer && !fromPendingRconOutput ) { + common->DPrintf( "got print from bad source: %s\n", Sys_NetAdrToString( from ) ); + return; + } + ProcessPrintMessage( from, msg ); + return; + } + + // Everything below is game-session control and therefore requires the + // exact current server endpoint. Merely owning an rcon reply window is not + // sufficient authority. + if ( !fromCurrentServer ) { + common->DPrintf( "got game control message '%s' from bad source: %s\n", string, Sys_NetAdrToString( from ) ); return; } @@ -1814,12 +2218,6 @@ void idAsyncClient::ConnectionlessMessage( const netadr_t from, const idBitMsg & return; } - // print request from server - if ( idStr::Icmp( string, "print" ) == 0 ) { - ProcessPrintMessage( from, msg ); - return; - } - // server pure list if ( idStr::Icmp( string, "pureServer" ) == 0 ) { ProcessPureMessage( from, msg ); @@ -1828,6 +2226,7 @@ void idAsyncClient::ConnectionlessMessage( const netadr_t from, const idBitMsg & if ( idStr::Icmp( string, "downloadInfo" ) == 0 ) { ProcessDownloadInfoMessage( from, msg ); + return; } common->DPrintf( "ignored message from %s: %s\n", Sys_NetAdrToString( from ), string ); @@ -1894,7 +2293,7 @@ void idAsyncClient::SetupConnection( void ) { msg.WriteLong( clientId ); clientPort.SendPacket( serverAddress, msg.GetData(), msg.GetSize() ); } else if ( clientState == CS_CONNECTING ) { - common->Printf( "sending connect to %s with challenge 0x%x\n", Sys_NetAdrToString( serverAddress ), serverChallenge ); + common->DPrintf( "sending authenticated connect request to %s\n", Sys_NetAdrToString( serverAddress ) ); msg.Init( msgBuf, sizeof( msgBuf ) ); msg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); msg.WriteString( "connect" ); @@ -2001,6 +2400,7 @@ void idAsyncClient::RunFrame( bool allowBlocking ) { if ( !clientPort.GetPort() ) { return; } + UpdateRemoteConsoleRequest(); // handle ongoing pk4 downloads and patch downloads HandleDownloads(); @@ -2160,7 +2560,9 @@ void idAsyncClient::SendVersionCheck( bool fromMenu ) { msg.WriteLong( ASYNC_PROTOCOL_VERSION ); msg.WriteShort( BUILD_OS_ID ); msg.WriteString( cvarSystem->GetCVarString( "si_version" ) ); - msg.WriteString( cvarSystem->GetCVarString( "com_guid" ) ); + // Retain the legacy field position without leaking the persistent client + // identifier to the unauthenticated UDP update service. + msg.WriteString( "" ); clientPort.SendPacket( idAsyncNetwork::GetMasterAddress(), msg.GetData(), msg.GetSize() ); common->DPrintf( "sent a version check request\n" ); @@ -2170,27 +2572,6 @@ void idAsyncClient::SendVersionCheck( bool fromMenu ) { showUpdateMessage = fromMenu; } -/* -================== -idAsyncClient::SendVersionDLUpdate - -sending those packets is not strictly necessary. just a way to tell the update server -about what is going on. allows the update server to have a more precise view of the overall -network load for the updates -================== -*/ -void idAsyncClient::SendVersionDLUpdate( int state ) { - idBitMsg msg; - byte msgBuf[MAX_MESSAGE_SIZE]; - - msg.Init( msgBuf, sizeof( msgBuf ) ); - msg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); - msg.WriteString( "versionDL" ); - msg.WriteLong( ASYNC_PROTOCOL_VERSION ); - msg.WriteShort( state ); - clientPort.SendPacket( idAsyncNetwork::GetMasterAddress(), msg.GetData(), msg.GetSize() ); -} - /* ================== idAsyncClient::HandleDownloads @@ -2209,76 +2590,14 @@ void idAsyncClient::HandleDownloads( void ) { } else if ( backgroundDownload.completed ) { // only enter these if the download slot is free if ( updateState == UPDATE_READY ) { - // - if ( session->MessageBox( MSG_YESNO, updateMSG, common->GetLanguageDict()->GetString ( "#str_04330" ), true, "yes" )[0] ) { - if ( !updateDirectDownload ) { - sys->OpenURL( updateURL, true ); - updateState = UPDATE_DONE; - } else { - - // we're just creating the file at toplevel inside fs_savepath - updateURL.ExtractFileName( updateFile ); - idFile_Permanent *f = static_cast< idFile_Permanent *>( fileSystem->OpenFileWrite( updateFile ) ); - if ( f == NULL ) { - common->Warning( "could not create update download destination '%s'", updateFile.c_str() ); - updateState = UPDATE_DONE; - SendVersionDLUpdate( 2 ); - session->MessageBox( MSG_OK, common->GetLanguageDict()->GetString ( "#str_04335" ), common->GetLanguageDict()->GetString ( "#str_04336" ), true ); - if ( updateFallback.Length() ) { - sys->OpenURL( updateFallback.c_str(), true ); - } else { - common->Printf( "no fallback URL\n" ); - } - return; - } - dltotal = 0; - dlnow = 0; - - backgroundDownload.completed = false; - backgroundDownload.opcode = DLTYPE_URL; - backgroundDownload.f = f; - backgroundDownload.url.status = DL_WAIT; - backgroundDownload.url.expectedSize = 0; - backgroundDownload.url.dlnow = 0; - backgroundDownload.url.dltotal = 0; - backgroundDownload.url.url = updateURL; - fileSystem->BackgroundDownload( &backgroundDownload ); - - updateState = UPDATE_DLING; - SendVersionDLUpdate( 0 ); - session->DownloadProgressBox( &backgroundDownload, va( "Downloading %s\n", updateFile.c_str() ) ); - updateState = UPDATE_DONE; - if ( backgroundDownload.url.status == DL_DONE ) { - SendVersionDLUpdate( 1 ); - idStr fullPath = f->GetFullPath(); - AsyncClient_CloseBackgroundDownloadFile( backgroundDownload ); - if ( session->MessageBox( MSG_YESNO, common->GetLanguageDict()->GetString ( "#str_04331" ), common->GetLanguageDict()->GetString ( "#str_04332" ), true, "yes" )[0] ) { - if ( updateMime == FILE_EXEC ) { - sys->StartProcess( fullPath, true ); - } else { - sys->OpenURL( va( "file://%s", fullPath.c_str() ), true ); - } - } else { - session->MessageBox( MSG_OK, va( common->GetLanguageDict()->GetString ( "#str_04333" ), fullPath.c_str() ), common->GetLanguageDict()->GetString ( "#str_04334" ), true ); - } - } else { - if ( backgroundDownload.url.dlerror[ 0 ] ) { - common->Warning( "update download failed. curl error: %s", backgroundDownload.url.dlerror ); - } - SendVersionDLUpdate( 2 ); - idStr name = f->GetName(); - AsyncClient_CloseBackgroundDownloadFile( backgroundDownload ); - fileSystem->RemoveFile( name ); - session->MessageBox( MSG_OK, common->GetLanguageDict()->GetString ( "#str_04335" ), common->GetLanguageDict()->GetString ( "#str_04336" ), true ); - if ( updateFallback.Length() ) { - sys->OpenURL( updateFallback.c_str(), true ); - } else { - common->Printf( "no fallback URL\n" ); - } - } - } - } else { - updateState = UPDATE_DONE; + const bool openReleasePage = session->MessageBox( MSG_YESNO, updateMSG, + common->GetLanguageDict()->GetString( "#str_04330" ), true, "yes" )[ 0 ] != '\0'; + updateState = UPDATE_DONE; + showUpdateMessage = false; + if ( openReleasePage ) { + // This compile-time HTTPS destination is the only action a legacy + // version reply may trigger. The network-provided URLs are inert. + sys->OpenURL( PROJECT_RELEASES_URL, false ); } } else if ( dlList.Num() ) { @@ -2478,6 +2797,10 @@ void idAsyncClient::ProcessDownloadInfoMessage( const netadr_t from, const idBit if ( infoType == SERVER_DL_REDIRECT ) { msg.ReadString( buf, MAX_STRING_CHARS ); cmdSystem->BufferCommandText( CMD_EXEC_NOW, "disconnect" ); + if ( !idURLPolicy::IsAllowedHTTPURL( buf ) ) { + common->Warning( "server supplied an invalid download information URL; request ignored" ); + return; + } // "You are missing required pak files to connect to this server.\nThe server gave a web page though:\n%s\nDo you want to go there now?" // "Missing required files" if ( session->MessageBox( MSG_YESNO, va( common->GetLanguageDict()->GetString( "#str_07217" ), buf ), @@ -2517,6 +2840,11 @@ void idAsyncClient::ProcessDownloadInfoMessage( const netadr_t from, const idBit } entry.filename = buf; msg.ReadString( buf, MAX_STRING_CHARS ); + if ( !idURLPolicy::IsAllowedHTTPURL( buf ) ) { + common->Warning( "server supplied a package URL outside the bounded HTTP/HTTPS policy; download list ignored" ); + dlList.Clear(); + return; + } entry.url = buf; entry.size = msg.ReadLong(); if ( entry.size <= 0 || totalDlSize > idMath::INT_MAX - entry.size ) { @@ -2611,14 +2939,21 @@ int idAsyncClient::GetDownloadRequest( const int checksums[ MAX_PURE_PAKS ], int common->Warning( "download request checksum count %d exceeds storage capacity %d", count, storedChecksumCount ); } if ( count != storedChecksumCount || memcmp( dlChecksums + 1, checksums, sizeof( int ) * storedChecksumCount ) || gamePakChecksum != dlChecksums[ 0 ] ) { - idRandom newreq; - memset( dlChecksums, 0, sizeof( dlChecksums ) ); dlChecksums[ 0 ] = gamePakChecksum; memcpy( dlChecksums + 1, checksums, sizeof( int ) * storedChecksumCount ); - newreq.SetSeed( Sys_Milliseconds() ); - dlRequest = newreq.RandomInt(); + std::uint32_t secureRequest = 0; + if ( !Sys_GetSecureRandomBytes( &secureRequest, sizeof( secureRequest ) ) ) { + memset( dlChecksums, 0, sizeof( dlChecksums ) ); + dlRequest = -1; + dlCount = -1; + return -1; + } + if ( secureRequest == static_cast( -1 ) ) { + secureRequest = 0; + } + dlRequest = static_cast( secureRequest ); dlCount = storedChecksumCount + 1; return dlRequest; } diff --git a/src/framework/async/AsyncClient.h b/src/framework/async/AsyncClient.h index 51df14c6..2f603e64 100644 --- a/src/framework/async/AsyncClient.h +++ b/src/framework/async/AsyncClient.h @@ -51,6 +51,25 @@ typedef enum { AUTHKEY_GUID } authKeyMsg_t; +typedef enum { + RCON_REPLY_NONE, + RCON_REPLY_LEGACY, + RCON_REPLY_CHALLENGE, + RCON_REPLY_OUTPUT +} rconReplyState_t; + +typedef struct rcon2ClientRequest_s { + rconReplyState_t state; + netadr_t address; + int startTime; + int lastSendTime; + byte clientNonce[16]; + byte serverNonce[16]; + byte requestDigest[32]; + byte proof[32]; + char command[MAX_STRING_CHARS]; +} rcon2ClientRequest_t; + typedef enum { AUTHKEY_BAD_INVALID, AUTHKEY_BAD_BANNED, @@ -62,7 +81,6 @@ typedef enum { UPDATE_NONE, UPDATE_SENT, UPDATE_READY, - UPDATE_DLING, UPDATE_DONE } clientUpdateState_t; @@ -136,6 +154,7 @@ class idAsyncClient { netadr_t lastRconAddress; // last rcon address we emitted to int lastRconTime; // when last rcon emitted + rcon2ClientRequest_t rcon2Request; idMsgChannel channel; // message channel to server int lastConnectTime; // last time a connect message was sent @@ -159,16 +178,9 @@ class idAsyncClient { clientUpdateState_t updateState; int updateSentTime; idStr updateMSG; - idStr updateURL; - bool updateDirectDownload; - idStr updateFile; - dlMime_t updateMime; - idStr updateFallback; bool showUpdateMessage; backgroundDownload_t backgroundDownload; - int dltotal; - int dlnow; int lastFrameDelta; @@ -194,6 +206,8 @@ class idAsyncClient { void ProcessDisconnectMessage( const netadr_t from, const idBitMsg &msg ); void ProcessInfoResponseMessage( const netadr_t from, const idBitMsg &msg ); void ProcessPrintMessage( const netadr_t from, const idBitMsg &msg ); + void ProcessRemoteConsole2ChallengeResponse( const netadr_t from, const idBitMsg &msg ); + void ProcessRemoteConsole2Complete( const netadr_t from, const idBitMsg &msg ); void ProcessServersListMessage( const netadr_t from, const idBitMsg &msg ); void ProcessServersListExtMessage( const netadr_t from, const idBitMsg &msg ); void ProcessAuthKeyMessage( const netadr_t from, const idBitMsg &msg ); @@ -206,7 +220,6 @@ class idAsyncClient { void ProcessReliableMessagePure( const idBitMsg &msg ); static const char* HandleGuiCommand( const char *cmd ); const char* HandleGuiCommandInternal( const char *cmd ); - void SendVersionDLUpdate( int state ); void HandleDownloads( void ); void Idle( void ); int UpdateTime( int clamp ); @@ -214,6 +227,10 @@ class idAsyncClient { bool CheckTimeout( void ); void ProcessDownloadInfoMessage( const netadr_t from, const idBitMsg &msg ); int GetDownloadRequest( const int checksums[ MAX_PURE_PAKS ], int count, int gamePakChecksum ); + void ClearRemoteConsoleRequest( void ); + void SendRemoteConsole2Challenge( void ); + void SendRemoteConsole2Proof( void ); + void UpdateRemoteConsoleRequest( void ); }; #endif /* !__ASYNCCLIENT_H__ */ diff --git a/src/framework/async/AsyncNetwork.cpp b/src/framework/async/AsyncNetwork.cpp index a96c7f0f..d5d33237 100644 --- a/src/framework/async/AsyncNetwork.cpp +++ b/src/framework/async/AsyncNetwork.cpp @@ -53,12 +53,14 @@ idCVar idAsyncNetwork::serverZombieTimeout( "net_serverZombieTimeout", "5", C idCVar idAsyncNetwork::serverClientTimeout( "net_serverClientTimeout", "40", CVAR_SYSTEM | CVAR_INTEGER | CVAR_NOCHEAT, "client time out in seconds" ); idCVar idAsyncNetwork::clientServerTimeout( "net_clientServerTimeout", "40", CVAR_SYSTEM | CVAR_INTEGER | CVAR_NOCHEAT, "server time out in seconds" ); idCVar idAsyncNetwork::serverDrawClient( "net_serverDrawClient", "-1", CVAR_SYSTEM | CVAR_INTEGER, "number of client for which to draw view on server" ); -idCVar idAsyncNetwork::serverRemoteConsolePassword( "net_serverRemoteConsolePassword", "", CVAR_SYSTEM | CVAR_NOCHEAT, "remote console password" ); +idCVar idAsyncNetwork::serverRemoteConsolePassword( "net_serverRemoteConsolePassword", "", CVAR_SYSTEM | CVAR_NOCHEAT | CVAR_PRIVATE | CVAR_CASE_SENSITIVE, "private remote console password (rcon2 requires at least 12 bytes)" ); +idCVar idAsyncNetwork::serverAllowLegacyRcon( "net_serverAllowLegacyRcon", "0", CVAR_SYSTEM | CVAR_BOOL | CVAR_NOCHEAT, "allow the insecure legacy plaintext rcon protocol" ); idCVar idAsyncNetwork::clientPrediction( "net_clientPrediction", "16", CVAR_SYSTEM | CVAR_INTEGER | CVAR_NOCHEAT, "additional client side prediction in milliseconds (legacy value 16 follows one exact base tic)" ); idCVar idAsyncNetwork::clientMaxPrediction( "net_clientMaxPrediction", "1000", CVAR_SYSTEM | CVAR_INTEGER | CVAR_NOCHEAT, "maximum number of milliseconds a client can predict ahead of server." ); idCVar idAsyncNetwork::clientUsercmdBackup( "net_clientUsercmdBackup", "5", CVAR_SYSTEM | CVAR_INTEGER | CVAR_NOCHEAT, "number of usercmds to resend" ); idCVar idAsyncNetwork::clientRemoteConsoleAddress( "net_clientRemoteConsoleAddress", "localhost", CVAR_SYSTEM | CVAR_NOCHEAT, "remote console address" ); -idCVar idAsyncNetwork::clientRemoteConsolePassword( "net_clientRemoteConsolePassword", "", CVAR_SYSTEM | CVAR_NOCHEAT, "remote console password" ); +idCVar idAsyncNetwork::clientRemoteConsolePassword( "net_clientRemoteConsolePassword", "", CVAR_SYSTEM | CVAR_NOCHEAT | CVAR_PRIVATE | CVAR_CASE_SENSITIVE, "private remote console password" ); +idCVar idAsyncNetwork::clientUseLegacyRcon( "net_clientUseLegacyRcon", "0", CVAR_SYSTEM | CVAR_BOOL | CVAR_NOCHEAT, "send the insecure legacy plaintext rcon protocol" ); idCVar idAsyncNetwork::master0( "net_master0", IDNET_HOST ":" IDNET_MASTER_PORT, CVAR_SYSTEM | CVAR_ROM, "idnet master server address" ); idCVar idAsyncNetwork::master1( "net_master1", "", CVAR_SYSTEM | CVAR_ARCHIVE, "1st master server address" ); idCVar idAsyncNetwork::master2( "net_master2", "", CVAR_SYSTEM | CVAR_ARCHIVE, "2nd master server address" ); @@ -164,7 +166,7 @@ void idAsyncNetwork::Init( void ) { cmdSystem->AddCommand( "serverInfo", GetServerInfo_f, CMD_FL_SYSTEM, "shows server info" ); cmdSystem->AddCommand( "LANScan", GetLANServers_f, CMD_FL_SYSTEM, "scans LAN for servers" ); cmdSystem->AddCommand( "listServers", ListServers_f, CMD_FL_SYSTEM, "lists scanned servers" ); - cmdSystem->AddCommand( "rcon", RemoteConsole_f, CMD_FL_SYSTEM, "sends remote console command to server" ); + cmdSystem->AddCommand( "rcon", RemoteConsole_f, CMD_FL_SYSTEM, "sends an authenticated remote console command to the server" ); cmdSystem->AddCommand( "heartbeat", Heartbeat_f, CMD_FL_SYSTEM, "send a heartbeat to the the master servers" ); cmdSystem->AddCommand( "kick", Kick_f, CMD_FL_SYSTEM, "kick a client by connection number" ); cmdSystem->AddCommand( "checkNewVersion", CheckNewVersion_f, CMD_FL_SYSTEM, "check if a new version of the game is available" ); @@ -293,33 +295,93 @@ void idAsyncNetwork::WriteUserCmdDelta( idBitMsg &msg, const usercmd_t &cmd, con idAsyncNetwork::ReadUserCmdDelta ================== */ -void idAsyncNetwork::ReadUserCmdDelta( const idBitMsg &msg, usercmd_t &cmd, const usercmd_t *base ) { - memset( &cmd, 0, sizeof( cmd ) ); +static bool AsyncNetwork_ProbeDeltaField( idBitMsg &probe, const int valueBits ) { + if ( probe.GetRemainingReadBits() < 1 ) { + return false; + } + const bool changed = probe.ReadBits( 1 ) != 0; + if ( changed ) { + if ( probe.GetRemainingReadBits() < valueBits ) { + return false; + } + probe.ReadBits( valueBits ); + } + return true; +} - if ( base ) { - cmd.gameTime = msg.ReadDeltaLongCounter( base->gameTime ); - cmd.buttons = msg.ReadDeltaShort( base->buttons ); - cmd.mx = msg.ReadDeltaShort( base->mx ); - cmd.my = msg.ReadDeltaShort( base->my ); - cmd.forwardmove = msg.ReadDeltaChar( base->forwardmove ); - cmd.rightmove = msg.ReadDeltaChar( base->rightmove ); - cmd.upmove = msg.ReadDeltaChar( base->upmove ); - cmd.angles[0] = msg.ReadDeltaShort( base->angles[0] ); - cmd.angles[1] = msg.ReadDeltaShort( base->angles[1] ); - cmd.angles[2] = msg.ReadDeltaShort( base->angles[2] ); - return; +static bool AsyncNetwork_ProbeDeltaLongCounter( idBitMsg &probe ) { + if ( probe.GetRemainingReadBits() < 5 ) { + return false; + } + const int valueBits = probe.ReadBits( 5 ); + if ( valueBits < 0 || valueBits > 31 ) { + return false; + } + if ( probe.GetRemainingReadBits() < valueBits ) { + return false; } + if ( valueBits > 0 ) { + probe.ReadBits( valueBits ); + } + return true; +} + +static bool AsyncNetwork_CanReadUserCmdDelta( const idBitMsg &msg, const bool hasBase ) { + idBitMsg probe = msg; + if ( !hasBase ) { + // long + three shorts + three chars + three angle shorts + return probe.GetRemainingReadBits() >= 152; + } + + return AsyncNetwork_ProbeDeltaLongCounter( probe ) && + AsyncNetwork_ProbeDeltaField( probe, 16 ) && + AsyncNetwork_ProbeDeltaField( probe, 16 ) && + AsyncNetwork_ProbeDeltaField( probe, 16 ) && + AsyncNetwork_ProbeDeltaField( probe, 8 ) && + AsyncNetwork_ProbeDeltaField( probe, 8 ) && + AsyncNetwork_ProbeDeltaField( probe, 8 ) && + AsyncNetwork_ProbeDeltaField( probe, 16 ) && + AsyncNetwork_ProbeDeltaField( probe, 16 ) && + AsyncNetwork_ProbeDeltaField( probe, 16 ); +} + +bool idAsyncNetwork::ReadUserCmdDelta( const idBitMsg &msg, usercmd_t &cmd, const usercmd_t *base ) { + if ( !AsyncNetwork_CanReadUserCmdDelta( msg, base != NULL ) ) { + return false; + } + + usercmd_t decoded; + memset( &decoded, 0, sizeof( decoded ) ); - cmd.gameTime = msg.ReadLong(); - cmd.buttons = msg.ReadShort(); - cmd.mx = msg.ReadShort(); - cmd.my = msg.ReadShort(); - cmd.forwardmove = msg.ReadChar(); - cmd.rightmove = msg.ReadChar(); - cmd.upmove = msg.ReadChar(); - cmd.angles[0] = msg.ReadShort(); - cmd.angles[1] = msg.ReadShort(); - cmd.angles[2] = msg.ReadShort(); + if ( base ) { + decoded.gameTime = msg.ReadDeltaLongCounter( base->gameTime ); + decoded.buttons = msg.ReadDeltaShort( base->buttons ); + decoded.mx = msg.ReadDeltaShort( base->mx ); + decoded.my = msg.ReadDeltaShort( base->my ); + decoded.forwardmove = msg.ReadDeltaChar( base->forwardmove ); + decoded.rightmove = msg.ReadDeltaChar( base->rightmove ); + decoded.upmove = msg.ReadDeltaChar( base->upmove ); + decoded.angles[0] = msg.ReadDeltaShort( base->angles[0] ); + decoded.angles[1] = msg.ReadDeltaShort( base->angles[1] ); + decoded.angles[2] = msg.ReadDeltaShort( base->angles[2] ); + } else { + decoded.gameTime = msg.ReadLong(); + decoded.buttons = msg.ReadShort(); + decoded.mx = msg.ReadShort(); + decoded.my = msg.ReadShort(); + decoded.forwardmove = msg.ReadChar(); + decoded.rightmove = msg.ReadChar(); + decoded.upmove = msg.ReadChar(); + decoded.angles[0] = msg.ReadShort(); + decoded.angles[1] = msg.ReadShort(); + decoded.angles[2] = msg.ReadShort(); + } + + if ( msg.IsReadOverflowed() ) { + return false; + } + cmd = decoded; + return true; } /* diff --git a/src/framework/async/AsyncNetwork.h b/src/framework/async/AsyncNetwork.h index 6e2d95f4..f2309f62 100644 --- a/src/framework/async/AsyncNetwork.h +++ b/src/framework/async/AsyncNetwork.h @@ -44,6 +44,9 @@ const int ASYNC_PROTOCOL_VERSION = ( ASYNC_PROTOCOL_MAJOR << 16 ) + ASYNC_PROTOC const int MAX_ASYNC_CLIENTS = 32; const int MAX_USERCMD_BACKUP = 256; +// The wire protocol sends the current command plus at most ten backups. +// Keep packet work bounded independently of the larger circular history. +const int MAX_USERCMD_PACKET_COMMANDS = 11; const int MAX_USERCMD_DUPLICATION = 25; const int MAX_USERCMD_RELAY = 10; @@ -151,7 +154,7 @@ class idAsyncNetwork { static void RunFrame( void ); static void WriteUserCmdDelta( idBitMsg &msg, const usercmd_t &cmd, const usercmd_t *base ); - static void ReadUserCmdDelta( const idBitMsg &msg, usercmd_t &cmd, const usercmd_t *base ); + static bool ReadUserCmdDelta( const idBitMsg &msg, usercmd_t &cmd, const usercmd_t *base ); static bool DuplicateUsercmd( const usercmd_t &previousUserCmd, usercmd_t ¤tUserCmd, int frame, int time ); static bool UsercmdInputChanged( const usercmd_t &previousUserCmd, const usercmd_t ¤tUserCmd ); @@ -185,11 +188,13 @@ class idAsyncNetwork { static idCVar clientServerTimeout; // time out in seconds for server static idCVar serverDrawClient; // the server draws the view of this client static idCVar serverRemoteConsolePassword; // remote console password + static idCVar serverAllowLegacyRcon; // explicit plaintext compatibility opt-in static idCVar clientPrediction; // how many additional milliseconds the clients runs ahead static idCVar clientMaxPrediction; // max milliseconds into the future a client can run prediction static idCVar clientUsercmdBackup; // how many usercmds the client sends from previous frames static idCVar clientRemoteConsoleAddress; // remote console address static idCVar clientRemoteConsolePassword; // remote console password + static idCVar clientUseLegacyRcon; // explicit plaintext compatibility opt-in static idCVar master0; // idnet master server static idCVar master1; // 1st master server static idCVar master2; // 2nd master server diff --git a/src/framework/async/AsyncServer.cpp b/src/framework/async/AsyncServer.cpp index 9b582138..cec08d28 100644 --- a/src/framework/async/AsyncServer.cpp +++ b/src/framework/async/AsyncServer.cpp @@ -30,9 +30,12 @@ If you have questions concerning this license or the applicable additional terms #include "AsyncNetwork.h" +#include "Rcon2Protocol.h" #include "../Session_local.h" +#include + static ID_INLINE int AsyncServer_NextGameFrameMsec( int gameFrame ) { return common->GetUserCmdDeltaMsec( gameFrame + 1 ); } @@ -44,6 +47,67 @@ const int NOINPUT_IDLE_TIME = 30000; const int HEARTBEAT_MSEC = 5*60*1000; +const int CONNECTION_CHALLENGE_TIMEOUT_MSEC = 30000; +const int RCON2_CHALLENGE_TIMEOUT_MSEC = 10000; +const int RCON2_RESEND_MIN_MSEC = 500; +const int RCON2_RATE_WINDOW_MSEC = 10000; +const int RCON2_RATE_MAX_CHALLENGES = 5; +const int RCON2_FAILURE_WINDOW_MSEC = 60000; +const int RCON2_FAILURE_MAX_ATTEMPTS = 5; +const int RCON2_FAILURE_BLOCK_MSEC = 30000; +const int OOB_RATE_WINDOW_MSEC = 1000; +const int OOB_INFO_MAX_PER_SOURCE = 16; +const int OOB_CHALLENGE_MAX_PER_SOURCE = 4; +const int OOB_INFO_MAX_GLOBAL = 512; +const int OOB_CHALLENGE_MAX_GLOBAL = 256; + +static ID_INLINE bool AsyncServer_SameEndpoint( const netadr_t &left, const netadr_t &right ) { + return left.port == right.port && Sys_CompareNetAdrBase( left, right ); +} + +static ID_INLINE std::uint32_t AsyncServer_Elapsed( int now, int then ) { + return static_cast( now ) - static_cast( then ); +} + +static ID_INLINE bool AsyncServer_TimeBefore( int now, int future ) { + return static_cast( static_cast( now ) - + static_cast( future ) ) < 0; +} + +static bool AsyncServer_SecureConnectionId( int &identifier ) { + std::uint32_t randomValue = 0; + if ( !Sys_GetSecureRandomBytes( &randomValue, sizeof( randomValue ) ) ) { + return false; + } + identifier = static_cast( randomValue & CONNECTIONLESS_MESSAGE_ID_MASK ); + if ( identifier == CONNECTIONLESS_MESSAGE_ID_MASK ) { + identifier = 0; + } + return true; +} + +static void AsyncServer_ClearChallenge( challenge_t &challenge ) { + challenge.valid = false; + memset( &challenge.address, 0, sizeof( challenge.address ) ); + challenge.clientId = 0; + challenge.challenge = 0; + challenge.time = 0; + challenge.pingTime = 0; + challenge.connected = false; + challenge.authState = CDK_WAIT; + challenge.authReply = AUTH_NONE; + challenge.authReplyMsg = AUTH_REPLY_WAITING; + challenge.authReplyPrint.Clear(); + challenge.guid[ 0 ] = '\0'; + challenge.OS = 0; +} + +static void AsyncServer_ClearChallenges( challenge_t challenges[ MAX_CHALLENGES ] ) { + for ( int index = 0; index < MAX_CHALLENGES; ++index ) { + AsyncServer_ClearChallenge( challenges[ index ] ); + } +} + // openQ4: below this much room in the message channel there is no point building // a snapshot - the channel would refuse it and the game would have already // consumed the client's unreliable message queue writing it. @@ -88,7 +152,10 @@ idAsyncServer::idAsyncServer( void ) { gameFrame = 0; gameTime = 0; gameTimeResidual = 0; - memset( challenges, 0, sizeof( challenges ) ); + AsyncServer_ClearChallenges( challenges ); + memset( rcon2Challenges, 0, sizeof( rcon2Challenges ) ); + memset( rconRateLimits, 0, sizeof( rconRateLimits ) ); + memset( oobRateLimits, 0, sizeof( oobRateLimits ) ); memset( userCmds, 0, sizeof( userCmds ) ); for ( i = 0; i < MAX_ASYNC_CLIENTS; i++ ) { ClearClient( i ); @@ -97,6 +164,13 @@ idAsyncServer::idAsyncServer( void ) { nextHeartbeatTime = 0; nextAsyncStatsTime = 0; noRconOutput = true; + rcon2VerifierInitialized = false; + rcon2VerifierValid = false; + memset( rcon2Salt, 0, sizeof( rcon2Salt ) ); + memset( rcon2Verifier, 0, sizeof( rcon2Verifier ) ); + oobWindowStart = 0; + oobInfoResponses = 0; + oobChallengeResponses = 0; lastAuthTime = 0; memset( stats_outrate, 0, sizeof( stats_outrate ) ); @@ -155,6 +229,7 @@ void idAsyncServer::ClosePort( void ) { for ( i = 0; i < MAX_CHALLENGES; i++ ) { challenges[ i ].authReplyPrint.Clear(); } + ClearRconSecurityState( true ); } /* @@ -187,7 +262,13 @@ void idAsyncServer::Spawn( void ) { cvarSystem->ResetFlaggedVariables( CVAR_CHEAT ); } - memset( challenges, 0, sizeof( challenges ) ); + AsyncServer_ClearChallenges( challenges ); + memset( rcon2Challenges, 0, sizeof( rcon2Challenges ) ); + memset( rconRateLimits, 0, sizeof( rconRateLimits ) ); + memset( oobRateLimits, 0, sizeof( oobRateLimits ) ); + oobWindowStart = serverTime; + oobInfoResponses = 0; + oobChallengeResponses = 0; memset( userCmds, 0, sizeof( userCmds ) ); for ( i = 0; i < MAX_ASYNC_CLIENTS; i++ ) { ClearClient( i ); @@ -199,10 +280,16 @@ void idAsyncServer::Spawn( void ) { serverDataChecksum = declManager->GetChecksum(); common->DPrintf( "Server decl checksum: 0x%08x\n", static_cast( serverDataChecksum ) ); - // get a pseudo random server id, but don't use the id which is reserved for connectionless packets - serverId = Sys_Milliseconds() & CONNECTIONLESS_MESSAGE_ID_MASK; + // A server id is only a short wire correlation value, not an authentication + // secret, but making it unpredictable closes an unnecessary spoofing aid. + if ( !AsyncServer_SecureConnectionId( serverId ) ) { + common->Warning( "OS secure random unavailable; refusing to spawn network server" ); + serverPort.Close(); + return; + } active = true; + RefreshRcon2Verifier(); nextHeartbeatTime = 0; nextAsyncStatsTime = 0; @@ -245,6 +332,7 @@ void idAsyncServer::Kill( void ) { fileSystem->ClearPureChecksums(); active = false; + ClearRconSecurityState( true ); // shutdown any current game session->Stop(); @@ -327,14 +415,6 @@ void idAsyncServer::ExecuteMapChange( void ) { serverTime = 0; - // openQ4 dev/staging runs from directory overrides (fs_cdpath). Keep - // multiplayer server startup non-pure to avoid pure-lockdown failures. - if ( sessLocal.mapSpawnData.serverInfo.GetInt( "si_pure" ) ) { - sessLocal.mapSpawnData.serverInfo.SetInt( "si_pure", 0 ); - cvarSystem->SetCVarBool( "si_pure", false ); - common->Printf( "openQ4: forcing si_pure 0 for local server startup\n" ); - } - // initialize game id and time gameInitId ^= Sys_Milliseconds(); // NOTE: make sure the gameInitId is always a positive number because negative numbers have special meaning gameFrame = 0; @@ -418,7 +498,9 @@ void idAsyncServer::ExecuteMapChange( void ) { for ( i = 0; i < MAX_ASYNC_CLIENTS; i++ ) { if ( clients[ i ].clientState == SCS_PUREWAIT ) { if ( !SendReliablePureToClient( i ) ) { - clients[ i ].clientState = SCS_CONNECTED; + // Never promote a client when the server cannot prove the + // prerequisites for its own pure policy. + DropClient( i, "#str_04337" ); } } } @@ -1016,7 +1098,7 @@ void idAsyncServer::SendUserInfoBroadcast( int userInfoNum, const idDict &info, if ( userInfoNum == localClientNum ) { common->DPrintf( "local user info modified by server\n" ); - cvarSystem->SetCVarsFromDict( *gameInfo ); + cvarSystem->SetCVarsFromDictByFlags( *gameInfo, CVAR_USERINFO ); cvarSystem->ClearModifiedFlags( CVAR_USERINFO ); // don't emit back } @@ -1292,8 +1374,12 @@ bool idAsyncServer::SendSnapshotToClient( int clientNum ) { return false; } - // how far is the client ahead of the server minus the packet delay - client.clientAheadTime = client.gameTime - ( gameTime + gameTimeResidual ); + // How far the client is ahead of the server minus the packet delay. The + // client-reported time is untrusted, so keep every intermediate widened. + const std::int64_t clientAheadTime = static_cast( client.gameTime ) - + static_cast( gameTime ) - static_cast( gameTimeResidual ); + client.clientAheadTime = clientAheadTime < idMath::INT_MIN ? idMath::INT_MIN : + ( clientAheadTime > idMath::INT_MAX ? idMath::INT_MAX : static_cast( clientAheadTime ) ); // write the snapshot msg.Init( msgBuf, sizeof( msgBuf ) ); @@ -1371,6 +1457,10 @@ void idAsyncServer::ProcessUnreliableClientMessage( int clientNum, const idBitMs if ( client.clientState == SCS_ZOMBIE ) { return; } + if ( msg.GetRemainingReadBits() < 64 ) { + DropClient( clientNum, "#str_07138" ); + return; + } acknowledgeSequence = msg.ReadLong(); clientGameInitId = msg.ReadLong(); @@ -1395,7 +1485,8 @@ void idAsyncServer::ProcessUnreliableClientMessage( int clientNum, const idBitMs if ( sessLocal.mapSpawnData.serverInfo.GetBool( "si_pure" ) ) { client.clientState = SCS_PUREWAIT; if ( !SendReliablePureToClient( clientNum ) ) { - client.clientState = SCS_CONNECTED; + DropClient( clientNum, "#str_04337" ); + return; } } } else if ( idAsyncNetwork::verbose.GetInteger() ) { @@ -1403,6 +1494,10 @@ void idAsyncServer::ProcessUnreliableClientMessage( int clientNum, const idBitMs } return; } + if ( msg.GetRemainingReadBits() < 32 + 8 ) { + DropClient( clientNum, "#str_07138" ); + return; + } client.acknowledgeSnapshotSequence = msg.ReadLong(); @@ -1446,19 +1541,44 @@ void idAsyncServer::ProcessUnreliableClientMessage( int clientNum, const idBitMs break; } case CLIENT_UNRELIABLE_MESSAGE_PINGRESPONSE: { - client.clientPing = realTime - msg.ReadLong(); + if ( msg.GetRemainingReadBits() < 32 ) { + DropClient( clientNum, "#str_07138" ); + return; + } + const int echoedPingTime = msg.ReadLong(); + if ( echoedPingTime != client.lastPingTime ) { + break; + } + client.clientPing = static_cast( Min( AsyncServer_Elapsed( realTime, echoedPingTime ), 32767u ) ); break; } case CLIENT_UNRELIABLE_MESSAGE_USERCMD: { + if ( msg.GetRemainingReadBits() < 16 + 32 + 8 ) { + DropClient( clientNum, "#str_07138" ); + return; + } client.clientPrediction = msg.ReadShort(); // read user commands clientGameFrame = msg.ReadLong(); numUsercmds = msg.ReadByte(); - for ( last = NULL, i = clientGameFrame - numUsercmds + 1; i <= clientGameFrame; i++ ) { + if ( numUsercmds < 1 || numUsercmds > MAX_USERCMD_PACKET_COMMANDS || + clientGameFrame < numUsercmds - 1 || + static_cast( clientGameFrame ) > static_cast( gameFrame ) + MAX_USERCMD_BACKUP ) { + DropClient( clientNum, "#str_07138" ); + return; + } + + const int firstClientGameFrame = clientGameFrame - numUsercmds + 1; + last = NULL; + for ( int commandIndex = 0; commandIndex < numUsercmds; commandIndex++ ) { + i = firstClientGameFrame + commandIndex; index = i & ( MAX_USERCMD_BACKUP - 1 ); - idAsyncNetwork::ReadUserCmdDelta( msg, userCmds[index][clientNum], last ); + if ( !idAsyncNetwork::ReadUserCmdDelta( msg, userCmds[index][clientNum], last ) ) { + DropClient( clientNum, "#str_07138" ); + return; + } userCmds[index][clientNum].gameFrame = i; userCmds[index][clientNum].duplicateCount = 0; if ( idAsyncNetwork::UsercmdInputChanged( userCmds[( i - 1 ) & ( MAX_USERCMD_BACKUP - 1 )][clientNum], userCmds[index][clientNum] ) ) { @@ -1500,10 +1620,18 @@ void idAsyncServer::ProcessReliableClientMessages( int clientNum ) { while ( client.channel.GetReliableMessage( msg ) ) { id = msg.ReadByte(); + if ( msg.IsReadOverflowed() ) { + DropClient( clientNum, "#str_07138" ); + return; + } switch( id ) { case CLIENT_RELIABLE_MESSAGE_CLIENTINFO: { idDict info; msg.ReadDeltaDict( info, &sessLocal.mapSpawnData.userInfo[clientNum] ); + if ( msg.IsReadOverflowed() ) { + DropClient( clientNum, "#str_07138" ); + return; + } SendUserInfoBroadcast( clientNum, info ); break; } @@ -1569,7 +1697,8 @@ void idAsyncServer::ProcessAuthMessage( const idBitMsg &msg ) { // no message parsing below for ( i = 0; i < MAX_CHALLENGES; i++ ) { - if ( !challenges[i].connected && challenges[ i ].clientId == clientId ) { + if ( challenges[ i ].valid && !challenges[i].connected && + challenges[ i ].clientId == clientId ) { // return if something is wrong // break if we have found a valid auth if ( !strlen( challenges[ i ].guid ) ) { @@ -1593,7 +1722,7 @@ void idAsyncServer::ProcessAuthMessage( const idBitMsg &msg ) { } if ( challenges[ i ].authState != CDK_WAIT ) { - common->DWarning( "auth: challenge 0x%x %s authState %d != CDK_WAIT", challenges[ i ].challenge, Sys_NetAdrToString( challenges[ i ].address ), challenges[ i ].authState ); + common->DWarning( "auth: challenge for %s has authState %d instead of CDK_WAIT", Sys_NetAdrToString( challenges[ i ].address ), challenges[ i ].authState ); return; } @@ -1617,39 +1746,307 @@ void idAsyncServer::ProcessAuthMessage( const idBitMsg &msg ) { } } +/* +================== +idAsyncServer::AllowConnectionlessResponse + +Bound the two unauthenticated replies that can amplify a spoofed UDP request. +LAN browser traffic gets enough per-source headroom to probe every legacy +server port in one pass, while the global budget also limits distributed +reflection traffic. +================== +*/ +bool idAsyncServer::AllowConnectionlessResponse( const netadr_t from, bool infoResponse ) { + if ( AsyncServer_Elapsed( serverTime, oobWindowStart ) >= OOB_RATE_WINDOW_MSEC ) { + oobWindowStart = serverTime; + oobInfoResponses = 0; + oobChallengeResponses = 0; + } + + int slot = -1; + int oldest = 0; + std::uint32_t oldestAge = 0; + for ( int index = 0; index < MAX_OOB_RATE_LIMITS; ++index ) { + if ( oobRateLimits[ index ].active && + Sys_CompareNetAdrBase( from, oobRateLimits[ index ].address ) ) { + slot = index; + break; + } + if ( !oobRateLimits[ index ].active ) { + oldest = index; + oldestAge = static_cast( -1 ); + } else { + const std::uint32_t age = AsyncServer_Elapsed( serverTime, + oobRateLimits[ index ].windowStart ); + if ( oldestAge != static_cast( -1 ) && age > oldestAge ) { + oldest = index; + oldestAge = age; + } + } + } + if ( slot < 0 ) { + slot = oldest; + memset( &oobRateLimits[ slot ], 0, sizeof( oobRateLimits[ slot ] ) ); + oobRateLimits[ slot ].active = true; + oobRateLimits[ slot ].address = from; + oobRateLimits[ slot ].windowStart = serverTime; + } + + oobRateLimit_t &source = oobRateLimits[ slot ]; + if ( AsyncServer_Elapsed( serverTime, source.windowStart ) >= OOB_RATE_WINDOW_MSEC ) { + source.windowStart = serverTime; + source.infoResponses = 0; + source.challengeResponses = 0; + } + + if ( infoResponse ) { + const int sourceLimit = Sys_IsLANAddress( from ) ? OOB_INFO_MAX_PER_SOURCE * 2 : OOB_INFO_MAX_PER_SOURCE; + if ( source.infoResponses >= sourceLimit || oobInfoResponses >= OOB_INFO_MAX_GLOBAL ) { + return false; + } + source.infoResponses++; + oobInfoResponses++; + } else { + if ( source.challengeResponses >= OOB_CHALLENGE_MAX_PER_SOURCE || + oobChallengeResponses >= OOB_CHALLENGE_MAX_GLOBAL ) { + return false; + } + source.challengeResponses++; + oobChallengeResponses++; + } + return true; +} + +/* +================== +idAsyncServer::ClearRconSecurityState +================== +*/ +void idAsyncServer::ClearRconSecurityState( bool clearRateLimits ) { + idCrypto::SecureZero( rcon2Challenges, sizeof( rcon2Challenges ) ); + idCrypto::SecureZero( rcon2Salt, sizeof( rcon2Salt ) ); + idCrypto::SecureZero( rcon2Verifier, sizeof( rcon2Verifier ) ); + rcon2VerifierInitialized = false; + rcon2VerifierValid = false; + if ( clearRateLimits ) { + idCrypto::SecureZero( rconRateLimits, sizeof( rconRateLimits ) ); + } +} + +/* +================== +idAsyncServer::RefreshRcon2Verifier + +The expensive password KDF runs once per configured password, never once per +untrusted request. Changing the password invalidates every outstanding proof. +================== +*/ +bool idAsyncServer::RefreshRcon2Verifier( void ) { + if ( rcon2VerifierInitialized && !idAsyncNetwork::serverRemoteConsolePassword.IsModified() ) { + return rcon2VerifierValid; + } + + idCrypto::SecureZero( rcon2Challenges, sizeof( rcon2Challenges ) ); + idCrypto::SecureZero( rcon2Salt, sizeof( rcon2Salt ) ); + idCrypto::SecureZero( rcon2Verifier, sizeof( rcon2Verifier ) ); + rcon2VerifierInitialized = true; + rcon2VerifierValid = false; + idAsyncNetwork::serverRemoteConsolePassword.ClearModified(); + + const char *password = idAsyncNetwork::serverRemoteConsolePassword.GetString(); + const size_t passwordBytes = strlen( password ); + if ( passwordBytes == 0 ) { + return false; + } + if ( passwordBytes < idRcon2::MIN_PASSWORD_BYTES ) { + common->Warning( "rcon2 is disabled: net_serverRemoteConsolePassword must contain at least %u bytes", + static_cast( idRcon2::MIN_PASSWORD_BYTES ) ); + return false; + } + if ( !Sys_GetSecureRandomBytes( rcon2Salt, sizeof( rcon2Salt ) ) ) { + common->Warning( "rcon2 is disabled: OS secure random is unavailable" ); + return false; + } + if ( !idRcon2::DeriveVerifier( password, rcon2Salt, rcon2Verifier ) ) { + idCrypto::SecureZero( rcon2Salt, sizeof( rcon2Salt ) ); + common->Warning( "rcon2 verifier derivation failed" ); + return false; + } + rcon2VerifierValid = true; + return true; +} + +/* +================== +idAsyncServer::AllowRconChallenge +================== +*/ +bool idAsyncServer::AllowRconChallenge( const netadr_t from ) { + int slot = -1; + int oldest = 0; + std::uint32_t oldestAge = 0; + for ( int index = 0; index < MAX_RCON_RATE_LIMITS; ++index ) { + if ( rconRateLimits[ index ].active && + Sys_CompareNetAdrBase( from, rconRateLimits[ index ].address ) ) { + slot = index; + break; + } + if ( !rconRateLimits[ index ].active ) { + oldest = index; + oldestAge = static_cast( -1 ); + } else { + const std::uint32_t age = AsyncServer_Elapsed( serverTime, + rconRateLimits[ index ].challengeWindowStart ); + if ( oldestAge != static_cast( -1 ) && age > oldestAge ) { + oldest = index; + oldestAge = age; + } + } + } + if ( slot < 0 ) { + slot = oldest; + memset( &rconRateLimits[ slot ], 0, sizeof( rconRateLimits[ slot ] ) ); + rconRateLimits[ slot ].active = true; + rconRateLimits[ slot ].address = from; + rconRateLimits[ slot ].challengeWindowStart = serverTime; + rconRateLimits[ slot ].failureWindowStart = serverTime; + } + + rconRateLimit_t &limit = rconRateLimits[ slot ]; + if ( limit.blockedUntil != 0 ) { + if ( AsyncServer_TimeBefore( serverTime, limit.blockedUntil ) ) { + return false; + } + limit.blockedUntil = 0; + } + if ( limit.challengeCount > 0 && + AsyncServer_Elapsed( serverTime, limit.lastChallengeTime ) < RCON2_RESEND_MIN_MSEC ) { + return false; + } + if ( AsyncServer_Elapsed( serverTime, limit.challengeWindowStart ) >= RCON2_RATE_WINDOW_MSEC ) { + limit.challengeWindowStart = serverTime; + limit.challengeCount = 0; + } + if ( limit.challengeCount >= RCON2_RATE_MAX_CHALLENGES ) { + return false; + } + limit.lastChallengeTime = serverTime; + limit.challengeCount++; + return true; +} + +bool idAsyncServer::AllowRconProofAttempt( const netadr_t from ) { + for ( int index = 0; index < MAX_RCON_RATE_LIMITS; ++index ) { + if ( rconRateLimits[ index ].active && + Sys_CompareNetAdrBase( from, rconRateLimits[ index ].address ) ) { + rconRateLimit_t &limit = rconRateLimits[ index ]; + if ( limit.blockedUntil == 0 ) { + return true; + } + if ( AsyncServer_TimeBefore( serverTime, limit.blockedUntil ) ) { + return false; + } + limit.blockedUntil = 0; + return true; + } + } + return true; +} + +void idAsyncServer::RecordRconFailure( const netadr_t from ) { + // Ensure a rate slot exists without allowing the failure to bypass a full + // table. A proof can only reach this point after a challenge used this path. + for ( int index = 0; index < MAX_RCON_RATE_LIMITS; ++index ) { + rconRateLimit_t &limit = rconRateLimits[ index ]; + if ( !limit.active || !Sys_CompareNetAdrBase( from, limit.address ) ) { + continue; + } + if ( AsyncServer_Elapsed( serverTime, limit.failureWindowStart ) >= RCON2_FAILURE_WINDOW_MSEC ) { + limit.failureWindowStart = serverTime; + limit.failureCount = 0; + } + limit.failureCount++; + if ( limit.failureCount >= RCON2_FAILURE_MAX_ATTEMPTS ) { + limit.blockedUntil = static_cast( static_cast( serverTime ) + + static_cast( RCON2_FAILURE_BLOCK_MSEC ) ); + for ( int challengeIndex = 0; challengeIndex < MAX_RCON2_CHALLENGES; ++challengeIndex ) { + if ( rcon2Challenges[ challengeIndex ].active && + Sys_CompareNetAdrBase( from, rcon2Challenges[ challengeIndex ].address ) ) { + idCrypto::SecureZero( &rcon2Challenges[ challengeIndex ], + sizeof( rcon2Challenges[ challengeIndex ] ) ); + } + } + } + return; + } +} + /* ================== idAsyncServer::ProcessChallengeMessage ================== */ void idAsyncServer::ProcessChallengeMessage( const netadr_t from, const idBitMsg &msg ) { - int i, clientId, oldest, oldestTime; + int i, clientId, oldest; idBitMsg outMsg; byte msgBuf[MAX_MESSAGE_SIZE]; + if ( msg.GetRemainingData() != 4 ) { + return; + } clientId = msg.ReadLong(); oldest = 0; - oldestTime = 0x7fffffff; + bool foundFree = false; + std::uint32_t oldestAge = 0; // see if we already have a challenge for this ip for ( i = 0; i < MAX_CHALLENGES; i++ ) { - if ( !challenges[i].connected && Sys_CompareNetAdrBase( from, challenges[i].address ) && clientId == challenges[i].clientId ) { + if ( challenges[ i ].valid && + AsyncServer_Elapsed( serverTime, challenges[ i ].time ) > CONNECTION_CHALLENGE_TIMEOUT_MSEC ) { + AsyncServer_ClearChallenge( challenges[ i ] ); + } + if ( challenges[i].valid && !challenges[i].connected && + AsyncServer_SameEndpoint( from, challenges[i].address ) && + clientId == challenges[i].clientId ) { break; } - if ( challenges[i].time < oldestTime ) { - oldestTime = challenges[i].time; - oldest = i; + if ( !challenges[ i ].valid ) { + if ( !foundFree ) { + oldest = i; + foundFree = true; + } + } else if ( !foundFree ) { + const std::uint32_t age = AsyncServer_Elapsed( serverTime, challenges[ i ].time ); + if ( age > oldestAge ) { + oldestAge = age; + oldest = i; + } } } + if ( !AllowConnectionlessResponse( from, false ) ) { + return; + } + if ( i >= MAX_CHALLENGES ) { // this is the first time this client has asked for a challenge i = oldest; + std::uint32_t secureChallenge = 0; + if ( !Sys_GetSecureRandomBytes( &secureChallenge, sizeof( secureChallenge ) ) ) { + common->Warning( "OS secure random unavailable; connection challenge not issued" ); + return; + } + if ( secureChallenge == 0 ) { + secureChallenge = 1; + } + AsyncServer_ClearChallenge( challenges[ i ] ); + challenges[i].valid = true; challenges[i].address = from; challenges[i].clientId = clientId; - challenges[i].challenge = ( (rand() << 16) ^ rand() ) ^ serverTime; + challenges[i].challenge = static_cast( secureChallenge ); challenges[i].time = serverTime; + challenges[i].pingTime = serverTime; challenges[i].connected = false; challenges[i].authState = CDK_WAIT; challenges[i].authReply = AUTH_NONE; @@ -1657,9 +2054,7 @@ void idAsyncServer::ProcessChallengeMessage( const netadr_t from, const idBitMsg challenges[i].authReplyPrint = ""; challenges[i].guid[0] = '\0'; } - challenges[i].pingTime = serverTime; - - common->Printf( "sending challenge 0x%x to %s\n", challenges[i].challenge, Sys_NetAdrToString( from ) ); + common->DPrintf( "sending connection challenge to %s\n", Sys_NetAdrToString( from ) ); outMsg.Init( msgBuf, sizeof( msgBuf ) ); outMsg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); @@ -1696,9 +2091,9 @@ bool idAsyncServer::SendPureServerMessage( const netadr_t to, int OS ) { int i; fileSystem->GetPureServerChecksums( serverChecksums, OS, &gamePakChecksum ); - if ( !serverChecksums[ 0 ] ) { + if ( !serverChecksums[ 0 ] || !gamePakChecksum ) { // happens if you run fully expanded assets with si_pure 1 - common->Warning( "pure server has no pak files referenced" ); + common->Warning( "pure server has no referenced pak files or compatible game module" ); return false; } common->DPrintf( "client %s: sending pure pak list\n", Sys_NetAdrToString( to ) ); @@ -1734,9 +2129,9 @@ bool idAsyncServer::SendReliablePureToClient( int clientNum ) { int gamePakChecksum; fileSystem->GetPureServerChecksums( serverChecksums, clients[ clientNum ].OS, &gamePakChecksum ); - if ( !serverChecksums[ 0 ] ) { + if ( !serverChecksums[ 0 ] || !gamePakChecksum ) { // happens if you run fully expanded assets with si_pure 1 - common->Warning( "pure server has no pak files referenced" ); + common->Warning( "pure server has no referenced pak files or compatible game module" ); return false; } @@ -1774,7 +2169,7 @@ int idAsyncServer::ValidateChallenge( const netadr_t from, int challenge, int cl } if ( Sys_CompareNetAdrBase( from, client.channel.GetRemoteAddress() ) && ( clientId == client.clientId || from.port == client.channel.GetRemoteAddress().port ) ) { - if ( serverTime - client.lastConnectTime < MIN_RECONNECT_TIME ) { + if ( AsyncServer_Elapsed( serverTime, client.lastConnectTime ) < MIN_RECONNECT_TIME ) { common->Printf( "%s: reconnect rejected : too soon\n", Sys_NetAdrToString( from ) ); return -1; } @@ -1783,14 +2178,24 @@ int idAsyncServer::ValidateChallenge( const netadr_t from, int challenge, int cl } for ( i = 0; i < MAX_CHALLENGES; i++ ) { - if ( Sys_CompareNetAdrBase( from, challenges[i].address ) && from.port == challenges[i].address.port ) { + if ( challenges[ i ].valid && + AsyncServer_Elapsed( serverTime, challenges[ i ].time ) > CONNECTION_CHALLENGE_TIMEOUT_MSEC ) { + AsyncServer_ClearChallenge( challenges[ i ] ); + continue; + } + if ( challenges[ i ].valid && !challenges[ i ].connected && + AsyncServer_Elapsed( serverTime, challenges[ i ].time ) <= CONNECTION_CHALLENGE_TIMEOUT_MSEC && + AsyncServer_SameEndpoint( from, challenges[i].address ) && + clientId == challenges[ i ].clientId ) { if ( challenge == challenges[i].challenge ) { break; } } } if ( i == MAX_CHALLENGES ) { - PrintOOB( from, SERVER_PRINT_BADCHALLENGE, "#str_04840" ); + if ( AllowConnectionlessResponse( from, false ) ) { + PrintOOB( from, SERVER_PRINT_BADCHALLENGE, "#str_04840" ); + } return -1; } return i; @@ -1809,21 +2214,45 @@ void idAsyncServer::ProcessConnectMessage( const netadr_t from, const idBitMsg & char password[ 17 ]; int i, ichallenge, islot, OS, numClients; + // Parse the complete fixed header before deciding whether to reply. All + // response paths below require the endpoint-bound challenge first, which + // prevents malformed or spoofed connect packets from becoming reflectors. + const int fixedHeaderBytes = 4 + 2 + 4 + 4 + 2 + 4; + if ( msg.GetRemainingData() < fixedHeaderBytes ) { + return; + } protocol = msg.ReadLong(); OS = msg.ReadShort(); + clientDataChecksum = msg.ReadLong(); + challenge = msg.ReadLong(); + clientId = msg.ReadShort(); + clientRate = msg.ReadLong(); + if ( OS < 0 || OS >= MAX_GAME_OS ) { + common->DPrintf( "connect from %s rejected: invalid OS %d\n", Sys_NetAdrToString( from ), OS ); + return; + } + if ( sessLocal.mapSpawnData.serverInfo.GetInt( "si_pure" ) ) { + const int osMask = fileSystem->GetOSMask(); + if ( osMask < 0 || ( static_cast( osMask ) & ( 1u << OS ) ) == 0 ) { + common->DPrintf( "connect from %s rejected: unsupported pure OS %d\n", Sys_NetAdrToString( from ), OS ); + return; + } + } + + if ( ( ichallenge = ValidateChallenge( from, challenge, clientId ) ) == -1 ) { + return; + } + if ( !AllowConnectionlessResponse( from, false ) ) { + return; + } - // check the protocol version + // check the protocol version only after the request proves possession of + // the challenge issued to this exact endpoint. if ( protocol != ASYNC_PROTOCOL_VERSION ) { - // that's a msg back to a client, we don't know about it's localization, so send english PrintOOB( from, SERVER_PRINT_BADPROTOCOL, va( "server uses protocol %d.%d\n", ASYNC_PROTOCOL_MAJOR, ASYNC_PROTOCOL_MINOR ) ); return; } - clientDataChecksum = msg.ReadLong(); - challenge = msg.ReadLong(); - clientId = msg.ReadShort(); - clientRate = msg.ReadLong(); - // check the client data - only for non pure servers if ( !sessLocal.mapSpawnData.serverInfo.GetInt( "si_pure" ) && clientDataChecksum != serverDataChecksum ) { common->DPrintf( "Decl checksum mismatch from %s: client=0x%08x server=0x%08x (non-pure)\n", @@ -1833,16 +2262,21 @@ void idAsyncServer::ProcessConnectMessage( const netadr_t from, const idBitMsg & return; } - if ( ( ichallenge = ValidateChallenge( from, challenge, clientId ) ) == -1 ) { + msg.ReadString( guid, sizeof( guid ) ); + if ( msg.IsReadOverflowed() ) { + common->DPrintf( "connect from %s rejected: truncated GUID\n", Sys_NetAdrToString( from ) ); return; } challenges[ ichallenge ].OS = OS; - msg.ReadString( guid, sizeof( guid ) ); - switch ( challenges[ ichallenge ].authState ) { case CDK_PUREWAIT: - SendPureServerMessage( from, OS ); + if ( !SendPureServerMessage( from, OS ) ) { + common->DPrintf( "client %s: pure challenge resend failed; rejecting connection\n", + Sys_NetAdrToString( from ) ); + PrintOOB( from, SERVER_PRINT_MISC, "#str_04337" ); + AsyncServer_ClearChallenge( challenges[ ichallenge ] ); + } return; case CDK_ONLYLAN: common->DPrintf( "%s: not a lan client\n", Sys_NetAdrToString( from ) ); @@ -1864,8 +2298,14 @@ void idAsyncServer::ProcessConnectMessage( const netadr_t from, const idBitMsg & // if authState == CDK_PUREOK, the check was already performed once before entering pure checks // but meanwhile, the max players may have been reached msg.ReadString( password, sizeof( password ) ); + if ( msg.IsReadOverflowed() ) { + idCrypto::SecureZero( password, sizeof( password ) ); + common->DPrintf( "connect from %s rejected: truncated password\n", Sys_NetAdrToString( from ) ); + return; + } char reason[MAX_STRING_CHARS]; allowReply_t reply = game->ServerAllowClient(clientId, numClients, Sys_NetAdrToString( from ), guid, password, password, reason ); + idCrypto::SecureZero( password, sizeof( password ) ); if ( reply != ALLOW_YES ) { common->DPrintf( "game denied connection for %s\n", Sys_NetAdrToString( from ) ); @@ -1883,10 +2323,15 @@ void idAsyncServer::ProcessConnectMessage( const netadr_t from, const idBitMsg & // enter pure checks if necessary if ( sessLocal.mapSpawnData.serverInfo.GetInt( "si_pure" ) && challenges[ ichallenge ].authState != CDK_PUREOK ) { - if ( SendPureServerMessage( from, OS ) ) { - challenges[ ichallenge ].authState = CDK_PUREWAIT; + if ( !SendPureServerMessage( from, OS ) ) { + common->DPrintf( "client %s: pure challenge could not be issued; rejecting connection\n", + Sys_NetAdrToString( from ) ); + PrintOOB( from, SERVER_PRINT_MISC, "#str_04337" ); + AsyncServer_ClearChallenge( challenges[ ichallenge ] ); return; } + challenges[ ichallenge ].authState = CDK_PUREWAIT; + return; } // push back decl checksum here when running pure. just an additional safe check @@ -1898,7 +2343,9 @@ void idAsyncServer::ProcessConnectMessage( const netadr_t from, const idBitMsg & return; } - ping = serverTime - challenges[ ichallenge ].pingTime; + const std::uint32_t pingElapsed = AsyncServer_Elapsed( serverTime, challenges[ ichallenge ].pingTime ); + ping = pingElapsed > static_cast( idMath::INT_MAX ) ? idMath::INT_MAX : + static_cast( pingElapsed ); common->Printf( "challenge from %s connecting with %d ping\n", Sys_NetAdrToString( from ), ping ); challenges[ ichallenge ].connected = true; @@ -1964,7 +2411,7 @@ void idAsyncServer::ProcessConnectMessage( const netadr_t from, const idBitMsg & clients[clientNum].snapshotSequence = 1; // clear the challenge struct so a reconnect from this client IP starts clean - memset( &challenges[ ichallenge ], 0, sizeof( challenge_t ) ); + AsyncServer_ClearChallenge( challenges[ ichallenge ] ); } /* @@ -2042,7 +2489,9 @@ void idAsyncServer::ProcessPureMessage( const netadr_t from, const idBitMsg &msg } if ( !VerifyChecksumMessage( iclient, &from, msg, reply, challenges[ iclient ].OS ) ) { - PrintOOB( from, SERVER_PRINT_MISC, reply ); + if ( AllowConnectionlessResponse( from, false ) ) { + PrintOOB( from, SERVER_PRINT_MISC, reply ); + } return; } @@ -2072,7 +2521,7 @@ void idAsyncServer::ProcessReliablePure( int clientNum, const idBitMsg &msg ) { common->DPrintf( "client %d: got reliable pure while != SCS_PUREWAIT, sending a reload\n", clientNum ); outMsg.Init( msgBuf, sizeof( msgBuf ) ); outMsg.WriteByte( SERVER_RELIABLE_MESSAGE_RELOAD ); - SendReliableMessage( clientNum, msg ); + SendReliableMessage( clientNum, outMsg ); // go back to SCS_CONNECTED to sleep on the client until it goes away for a reconnect clients[ clientNum ].clientState = SCS_CONNECTED; return; @@ -2107,41 +2556,264 @@ void RConRedirect( const char *string ) { /* ================== -idAsyncServer::ProcessRemoteConsoleMessage +idAsyncServer::ExecuteRemoteConsoleCommand ================== */ -void idAsyncServer::ProcessRemoteConsoleMessage( const netadr_t from, const idBitMsg &msg ) { - idBitMsg outMsg; +void idAsyncServer::ExecuteRemoteConsoleCommand( const netadr_t from, const char *command, bool authenticated ) { byte msgBuf[952]; - char string[MAX_STRING_CHARS]; + common->Printf( "%s remote console command accepted from %s\n", + authenticated ? "authenticated" : "legacy plaintext", Sys_NetAdrToString( from ) ); - if ( idAsyncNetwork::serverRemoteConsolePassword.GetString()[0] == '\0' ) { - PrintOOB( from, SERVER_PRINT_MISC, "#str_04846" ); - return; + rconAddress = from; + noRconOutput = true; + common->BeginRedirect( (char *)msgBuf, sizeof( msgBuf ), RConRedirect ); + cmdSystem->BufferCommandText( CMD_EXEC_NOW, command ); + common->EndRedirect(); + + if ( noRconOutput ) { + PrintOOB( rconAddress, SERVER_PRINT_RCON, "#str_04848" ); } +} - msg.ReadString( string, sizeof( string ) ); +/* +================== +idAsyncServer::SendRemoteConsole2Complete +================== +*/ +void idAsyncServer::SendRemoteConsole2Complete( const netadr_t to, + const byte clientNonce[16], const byte serverNonce[16] ) { + byte msgBuf[128]; + idBitMsg outMsg; + outMsg.Init( msgBuf, sizeof( msgBuf ) ); + outMsg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); + outMsg.WriteString( "rcon2Complete" ); + outMsg.WriteByte( idRcon2::PROTOCOL_VERSION ); + outMsg.WriteData( clientNonce, idRcon2::NONCE_BYTES ); + outMsg.WriteData( serverNonce, idRcon2::NONCE_BYTES ); + serverPort.SendPacket( to, outMsg.GetData(), outMsg.GetSize() ); +} - if ( idStr::Icmp( string, idAsyncNetwork::serverRemoteConsolePassword.GetString() ) != 0 ) { - PrintOOB( from, SERVER_PRINT_MISC, "#str_04847" ); +/* +================== +idAsyncServer::ProcessRemoteConsoleMessage + +Quake 4's original packet sends the password verbatim. It remains available +only as an explicit two-sided compatibility escape hatch and is still bounded +and constant-time checked. +================== +*/ +void idAsyncServer::ProcessRemoteConsoleMessage( const netadr_t from, const idBitMsg &msg ) { + char suppliedPassword[MAX_STRING_CHARS] = {}; + char command[MAX_STRING_CHARS] = {}; + if ( !idAsyncNetwork::serverAllowLegacyRcon.GetBool() || + idAsyncNetwork::serverRemoteConsolePassword.GetString()[0] == '\0' ) { + return; + } + if ( !AllowRconChallenge( from ) || !AllowConnectionlessResponse( from, false ) ) { + return; + } + msg.ReadString( suppliedPassword, sizeof( suppliedPassword ) ); + msg.ReadString( command, sizeof( command ) ); + if ( msg.GetRemainingData() != 0 || command[0] == '\0' ) { + idCrypto::SecureZero( suppliedPassword, sizeof( suppliedPassword ) ); + idCrypto::SecureZero( command, sizeof( command ) ); return; } - msg.ReadString( string, sizeof( string ) ); - - common->Printf( "rcon from %s: %s\n", Sys_NetAdrToString( from ), string ); + const char *configuredPassword = idAsyncNetwork::serverRemoteConsolePassword.GetString(); + const size_t suppliedBytes = strlen( suppliedPassword ); + const size_t configuredBytes = strlen( configuredPassword ); + const bool passwordMatches = suppliedBytes == configuredBytes && + idCrypto::ConstantTimeEquals( suppliedPassword, configuredPassword, configuredBytes ); + if ( !passwordMatches ) { + RecordRconFailure( from ); + PrintOOB( from, SERVER_PRINT_MISC, "#str_04847" ); + idCrypto::SecureZero( suppliedPassword, sizeof( suppliedPassword ) ); + idCrypto::SecureZero( command, sizeof( command ) ); + return; + } - rconAddress = from; - noRconOutput = true; - common->BeginRedirect( (char *)msgBuf, sizeof( msgBuf ), RConRedirect ); + idCrypto::SecureZero( suppliedPassword, sizeof( suppliedPassword ) ); + ExecuteRemoteConsoleCommand( from, command, false ); + idCrypto::SecureZero( command, sizeof( command ) ); +} - cmdSystem->BufferCommandText( CMD_EXEC_NOW, string ); +/* +================== +idAsyncServer::ProcessRemoteConsole2ChallengeMessage +================== +*/ +void idAsyncServer::ProcessRemoteConsole2ChallengeMessage( const netadr_t from, const idBitMsg &msg ) { + const int requestBytes = 1 + idRcon2::NONCE_BYTES + idRcon2::REQUEST_DIGEST_BYTES; + if ( !active || msg.GetRemainingData() != requestBytes || !RefreshRcon2Verifier() ) { + return; + } + if ( msg.ReadByte() != idRcon2::PROTOCOL_VERSION ) { + return; + } + byte clientNonce[idRcon2::NONCE_BYTES]; + byte requestDigest[idRcon2::REQUEST_DIGEST_BYTES]; + msg.ReadData( clientNonce, sizeof( clientNonce ) ); + msg.ReadData( requestDigest, sizeof( requestDigest ) ); + + int slot = -1; + int oldest = 0; + std::uint32_t oldestAge = 0; + for ( int index = 0; index < MAX_RCON2_CHALLENGES; ++index ) { + rcon2Challenge_t &candidate = rcon2Challenges[ index ]; + if ( candidate.active && + AsyncServer_Elapsed( serverTime, candidate.createdTime ) > RCON2_CHALLENGE_TIMEOUT_MSEC ) { + idCrypto::SecureZero( &candidate, sizeof( candidate ) ); + } + if ( candidate.active && AsyncServer_SameEndpoint( from, candidate.address ) && + idCrypto::ConstantTimeEquals( clientNonce, candidate.clientNonce, sizeof( clientNonce ) ) && + idCrypto::ConstantTimeEquals( requestDigest, candidate.requestDigest, sizeof( requestDigest ) ) ) { + slot = index; + break; + } + if ( !candidate.active ) { + oldest = index; + oldestAge = static_cast( -1 ); + } else { + const std::uint32_t age = AsyncServer_Elapsed( serverTime, candidate.createdTime ); + if ( oldestAge != static_cast( -1 ) && age > oldestAge ) { + oldest = index; + oldestAge = age; + } + } + } + if ( !AllowRconChallenge( from ) || !AllowConnectionlessResponse( from, false ) ) { + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( requestDigest, sizeof( requestDigest ) ); + return; + } + if ( slot < 0 ) { + slot = oldest; + rcon2Challenge_t &issued = rcon2Challenges[ slot ]; + idCrypto::SecureZero( &issued, sizeof( issued ) ); + byte randomValues[idRcon2::NONCE_BYTES + idRcon2::ENDPOINT_BINDING_BYTES]; + if ( !Sys_GetSecureRandomBytes( randomValues, sizeof( randomValues ) ) ) { + common->Warning( "OS secure random unavailable; rcon2 challenge not issued" ); + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( requestDigest, sizeof( requestDigest ) ); + return; + } + issued.active = true; + issued.address = from; + issued.createdTime = serverTime; + memcpy( issued.clientNonce, clientNonce, sizeof( issued.clientNonce ) ); + memcpy( issued.serverNonce, randomValues, sizeof( issued.serverNonce ) ); + memcpy( issued.endpointBinding, randomValues + sizeof( issued.serverNonce ), + sizeof( issued.endpointBinding ) ); + memcpy( issued.requestDigest, requestDigest, sizeof( issued.requestDigest ) ); + idCrypto::SecureZero( randomValues, sizeof( randomValues ) ); + } + + rcon2Challenge_t &issued = rcon2Challenges[ slot ]; + issued.lastResponseTime = serverTime; + byte msgBuf[192]; + idBitMsg outMsg; + outMsg.Init( msgBuf, sizeof( msgBuf ) ); + outMsg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); + outMsg.WriteString( "rcon2ChallengeResponse" ); + outMsg.WriteByte( idRcon2::PROTOCOL_VERSION ); + outMsg.WriteData( issued.clientNonce, sizeof( issued.clientNonce ) ); + outMsg.WriteData( issued.serverNonce, sizeof( issued.serverNonce ) ); + outMsg.WriteData( rcon2Salt, sizeof( rcon2Salt ) ); + outMsg.WriteLong( static_cast( idRcon2::PBKDF2_ITERATIONS ) ); + outMsg.WriteData( issued.endpointBinding, sizeof( issued.endpointBinding ) ); + outMsg.WriteData( issued.requestDigest, sizeof( issued.requestDigest ) ); + serverPort.SendPacket( from, outMsg.GetData(), outMsg.GetSize() ); + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( requestDigest, sizeof( requestDigest ) ); +} - common->EndRedirect(); +/* +================== +idAsyncServer::ProcessRemoteConsole2Message +================== +*/ +void idAsyncServer::ProcessRemoteConsole2Message( const netadr_t from, const idBitMsg &msg ) { + const int fixedPrefixBytes = 1 + idRcon2::NONCE_BYTES + idRcon2::NONCE_BYTES; + if ( !active || msg.GetRemainingData() < fixedPrefixBytes + 1 + idRcon2::PROOF_BYTES || + !RefreshRcon2Verifier() || !AllowRconProofAttempt( from ) ) { + return; + } + if ( msg.ReadByte() != idRcon2::PROTOCOL_VERSION ) { + return; + } + byte clientNonce[idRcon2::NONCE_BYTES]; + byte serverNonce[idRcon2::NONCE_BYTES]; + msg.ReadData( clientNonce, sizeof( clientNonce ) ); + msg.ReadData( serverNonce, sizeof( serverNonce ) ); + + int slot = -1; + for ( int index = 0; index < MAX_RCON2_CHALLENGES; ++index ) { + const rcon2Challenge_t &candidate = rcon2Challenges[ index ]; + if ( candidate.active && + AsyncServer_Elapsed( serverTime, candidate.createdTime ) <= RCON2_CHALLENGE_TIMEOUT_MSEC && + AsyncServer_SameEndpoint( from, candidate.address ) && + idCrypto::ConstantTimeEquals( clientNonce, candidate.clientNonce, sizeof( clientNonce ) ) && + idCrypto::ConstantTimeEquals( serverNonce, candidate.serverNonce, sizeof( serverNonce ) ) ) { + slot = index; + break; + } + } + if ( slot < 0 ) { + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( serverNonce, sizeof( serverNonce ) ); + return; + } - if ( noRconOutput ) { - PrintOOB( rconAddress, SERVER_PRINT_RCON, "#str_04848" ); + rcon2Challenge_t issued = rcon2Challenges[ slot ]; + // Consume before parsing or checking the proof: malformed, failed, and + // successful attempts are all one-shot and cannot be replayed. + idCrypto::SecureZero( &rcon2Challenges[ slot ], sizeof( rcon2Challenges[ slot ] ) ); + char command[MAX_STRING_CHARS] = {}; + msg.ReadString( command, sizeof( command ) ); + if ( command[0] == '\0' || msg.GetRemainingData() != idRcon2::PROOF_BYTES ) { + RecordRconFailure( from ); + idCrypto::SecureZero( command, sizeof( command ) ); + idCrypto::SecureZero( &issued, sizeof( issued ) ); + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( serverNonce, sizeof( serverNonce ) ); + return; } + byte suppliedProof[idRcon2::PROOF_BYTES]; + byte expectedProof[idRcon2::PROOF_BYTES]; + byte requestDigest[idRcon2::REQUEST_DIGEST_BYTES]; + msg.ReadData( suppliedProof, sizeof( suppliedProof ) ); + idRcon2::HashRequest( command, requestDigest ); + idRcon2::ComputeProof( rcon2Verifier, issued.clientNonce, issued.serverNonce, + issued.endpointBinding, issued.requestDigest, expectedProof ); + const bool requestMatches = idCrypto::ConstantTimeEquals( requestDigest, + issued.requestDigest, sizeof( requestDigest ) ); + const bool proofMatches = idCrypto::ConstantTimeEquals( suppliedProof, + expectedProof, sizeof( suppliedProof ) ); + + if ( !requestMatches || !proofMatches ) { + RecordRconFailure( from ); + PrintOOB( from, SERVER_PRINT_MISC, "#str_04847" ); + } else { + for ( int index = 0; index < MAX_RCON_RATE_LIMITS; ++index ) { + if ( rconRateLimits[ index ].active && + Sys_CompareNetAdrBase( from, rconRateLimits[ index ].address ) ) { + rconRateLimits[ index ].failureCount = 0; + rconRateLimits[ index ].failureWindowStart = serverTime; + break; + } + } + ExecuteRemoteConsoleCommand( from, command, true ); + } + SendRemoteConsole2Complete( from, issued.clientNonce, issued.serverNonce ); + + idCrypto::SecureZero( command, sizeof( command ) ); + idCrypto::SecureZero( suppliedProof, sizeof( suppliedProof ) ); + idCrypto::SecureZero( expectedProof, sizeof( expectedProof ) ); + idCrypto::SecureZero( requestDigest, sizeof( requestDigest ) ); + idCrypto::SecureZero( &issued, sizeof( issued ) ); + idCrypto::SecureZero( clientNonce, sizeof( clientNonce ) ); + idCrypto::SecureZero( serverNonce, sizeof( serverNonce ) ); } /* @@ -2157,6 +2829,9 @@ void idAsyncServer::ProcessGetInfoMessage( const netadr_t from, const idBitMsg & if ( !IsActive() ) { return; } + if ( msg.GetRemainingData() != 4 || !AllowConnectionlessResponse( from, true ) ) { + return; + } common->DPrintf( "Sending info response to %s\n", Sys_NetAdrToString( from ) ); @@ -2235,9 +2910,19 @@ bool idAsyncServer::ConnectionlessMessage( const netadr_t from, const idBitMsg & ProcessRemoteConsoleMessage( from, msg ); return true; } + if ( idStr::Icmp( string, "rcon2Challenge" ) == 0 ) { + ProcessRemoteConsole2ChallengeMessage( from, msg ); + return false; + } + if ( idStr::Icmp( string, "rcon2" ) == 0 ) { + ProcessRemoteConsole2Message( from, msg ); + return true; + } if ( !active ) { - PrintOOB( from, SERVER_PRINT_MISC, "#str_04849" ); + if ( AllowConnectionlessResponse( from, false ) ) { + PrintOOB( from, SERVER_PRINT_MISC, "#str_04849" ); + } return false; } @@ -2336,7 +3021,12 @@ bool idAsyncServer::ProcessMessage( const netadr_t from, idBitMsg &msg ) { } // if we received a sequenced packet from an address we don't recognize, - // send an out of band disconnect packet to it + // send an out of band disconnect packet to it. This is still a pre-auth + // response, so share the global/per-source control budget used by the + // connectionless admission paths. + if ( !AllowConnectionlessResponse( from, false ) ) { + return false; + } outMsg.Init( msgBuf, sizeof( msgBuf ) ); outMsg.WriteShort( CONNECTIONLESS_MESSAGE_ID ); outMsg.WriteString( "disconnect" ); @@ -2826,6 +3516,9 @@ void idAsyncServer::ProcessDownloadRequestMessage( const netadr_t from, const id common->DPrintf( "client %s: got download request message, not in CDK_PUREWAIT\n", Sys_NetAdrToString( from ) ); return; } + if ( !AllowConnectionlessResponse( from, false ) ) { + return; + } // the first token of the pak names list passed to the game will be empty if no game pak is requested dlGamePak = msg.ReadLong(); diff --git a/src/framework/async/AsyncServer.h b/src/framework/async/AsyncServer.h index 79d75bad..64deeffb 100644 --- a/src/framework/async/AsyncServer.h +++ b/src/framework/async/AsyncServer.h @@ -77,6 +77,7 @@ typedef enum { } authReplyMsg_t; typedef struct challenge_s { + bool valid; // entry contains an issued, unexpired challenge netadr_t address; // client address int clientId; // client identification int challenge; // challenge code @@ -91,6 +92,40 @@ typedef struct challenge_s { int OS; } challenge_t; +const int MAX_RCON2_CHALLENGES = 64; +const int MAX_RCON_RATE_LIMITS = 64; +const int MAX_OOB_RATE_LIMITS = 128; + +typedef struct rcon2Challenge_s { + bool active; + netadr_t address; + int createdTime; + int lastResponseTime; + byte clientNonce[ 16 ]; + byte serverNonce[ 16 ]; + byte endpointBinding[ 16 ]; + byte requestDigest[ 32 ]; +} rcon2Challenge_t; + +typedef struct rconRateLimit_s { + bool active; + netadr_t address; + int challengeWindowStart; + int lastChallengeTime; + int challengeCount; + int failureWindowStart; + int failureCount; + int blockedUntil; +} rconRateLimit_t; + +typedef struct oobRateLimit_s { + bool active; + netadr_t address; + int windowStart; + int infoResponses; + int challengeResponses; +} oobRateLimit_t; + typedef enum { SCS_FREE, // can be reused for a new connection SCS_ZOMBIE, // client has been disconnected, but don't reuse connection for a couple seconds @@ -196,6 +231,9 @@ class idAsyncServer { int localClientNum; // local client on listen server challenge_t challenges[MAX_CHALLENGES]; // to prevent invalid IPs from connecting + rcon2Challenge_t rcon2Challenges[MAX_RCON2_CHALLENGES]; + rconRateLimit_t rconRateLimits[MAX_RCON_RATE_LIMITS]; + oobRateLimit_t oobRateLimits[MAX_OOB_RATE_LIMITS]; serverClient_t clients[MAX_ASYNC_CLIENTS]; // clients usercmd_t userCmds[MAX_USERCMD_BACKUP][MAX_ASYNC_CLIENTS]; @@ -212,6 +250,13 @@ class idAsyncServer { bool serverReloadingEngine; // flip-flop to not loop over when net_serverReloadEngine is on bool noRconOutput; // for default rcon response when command is silent + bool rcon2VerifierInitialized; + bool rcon2VerifierValid; + byte rcon2Salt[16]; + byte rcon2Verifier[32]; + int oobWindowStart; + int oobInfoResponses; + int oobChallengeResponses; int lastAuthTime; // global for auth server timeout @@ -250,6 +295,8 @@ class idAsyncServer { void ProcessChallengeMessage( const netadr_t from, const idBitMsg &msg ); void ProcessConnectMessage( const netadr_t from, const idBitMsg &msg ); void ProcessRemoteConsoleMessage( const netadr_t from, const idBitMsg &msg ); + void ProcessRemoteConsole2ChallengeMessage( const netadr_t from, const idBitMsg &msg ); + void ProcessRemoteConsole2Message( const netadr_t from, const idBitMsg &msg ); void ProcessGetInfoMessage( const netadr_t from, const idBitMsg &msg ); bool ConnectionlessMessage( const netadr_t from, const idBitMsg &msg ); bool ProcessMessage( const netadr_t from, idBitMsg &msg ); @@ -257,6 +304,14 @@ class idAsyncServer { bool SendPureServerMessage( const netadr_t to, int OS ); // returns false if no pure paks on the list void ProcessPureMessage( const netadr_t from, const idBitMsg &msg ); int ValidateChallenge( const netadr_t from, int challenge, int clientId ); // returns -1 if validate failed + bool AllowConnectionlessResponse( const netadr_t from, bool infoResponse ); + bool RefreshRcon2Verifier( void ); + bool AllowRconChallenge( const netadr_t from ); + bool AllowRconProofAttempt( const netadr_t from ); + void RecordRconFailure( const netadr_t from ); + void ClearRconSecurityState( bool clearRateLimits ); + void ExecuteRemoteConsoleCommand( const netadr_t from, const char *command, bool authenticated ); + void SendRemoteConsole2Complete( const netadr_t to, const byte clientNonce[16], const byte serverNonce[16] ); bool SendReliablePureToClient( int clientNum ); void ProcessReliablePure( int clientNum, const idBitMsg &msg ); bool VerifyChecksumMessage( int clientNum, const netadr_t *from, const idBitMsg &msg, idStr &reply, int OS ); // if from is NULL, clientNum is used for error messages diff --git a/src/framework/async/MultiViewDemo.cpp b/src/framework/async/MultiViewDemo.cpp index da227c1f..ad696f68 100644 --- a/src/framework/async/MultiViewDemo.cpp +++ b/src/framework/async/MultiViewDemo.cpp @@ -2003,7 +2003,7 @@ bool idMultiViewDemo::StartPlayback( const idCmdArgs &args ) { return false; } - cvarSystem->SetCVarsFromDict( sessLocal.mapSpawnData.syncedCVars ); + cvarSystem->SetCVarsFromDictByFlags( sessLocal.mapSpawnData.syncedCVars, CVAR_NETWORKSYNC ); sessLocal.ExecuteMapChange(); game->SetDemoState( DEMO_PLAYING, true, false ); for ( int i = 0; i < MAX_ASYNC_CLIENTS; i++ ) { @@ -2072,7 +2072,7 @@ bool idMultiViewDemo::ResetPlaybackStream() { return false; } - cvarSystem->SetCVarsFromDict( sessLocal.mapSpawnData.syncedCVars ); + cvarSystem->SetCVarsFromDictByFlags( sessLocal.mapSpawnData.syncedCVars, CVAR_NETWORKSYNC ); sessLocal.ExecuteMapChange(); game->SetDemoState( DEMO_PLAYING, true, false ); for ( int i = 0; i < MAX_ASYNC_CLIENTS; i++ ) { diff --git a/src/framework/async/Rcon2Protocol.cpp b/src/framework/async/Rcon2Protocol.cpp new file mode 100644 index 00000000..57c54f50 --- /dev/null +++ b/src/framework/async/Rcon2Protocol.cpp @@ -0,0 +1,69 @@ +/* +=========================================================================== + +openQ4 authenticated remote-console protocol +Copyright (C) 2026 DarkMatter Productions + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +=========================================================================== +*/ + +#include "Rcon2Protocol.h" + +#include + +namespace idRcon2 { + +void HashRequest( const char *command, + std::uint8_t digest[ REQUEST_DIGEST_BYTES ] ) { + if ( command == nullptr ) { + idCrypto::SHA256( nullptr, 0, digest ); + return; + } + idCrypto::SHA256( command, std::strlen( command ), digest ); +} + +bool DeriveVerifier( const char *password, + const std::uint8_t salt[ SALT_BYTES ], + std::uint8_t verifier[ VERIFIER_BYTES ] ) { + if ( password == nullptr || salt == nullptr || verifier == nullptr ) { + return false; + } + const std::size_t passwordBytes = std::strlen( password ); + if ( passwordBytes < MIN_PASSWORD_BYTES ) { + return false; + } + return idCrypto::PBKDF2HMACSHA256( password, passwordBytes, + salt, SALT_BYTES, PBKDF2_ITERATIONS, verifier, VERIFIER_BYTES ); +} + +void ComputeProof( const std::uint8_t verifier[ VERIFIER_BYTES ], + const std::uint8_t clientNonce[ NONCE_BYTES ], + const std::uint8_t serverNonce[ NONCE_BYTES ], + const std::uint8_t endpointBinding[ ENDPOINT_BINDING_BYTES ], + const std::uint8_t requestDigest[ REQUEST_DIGEST_BYTES ], + std::uint8_t proof[ PROOF_BYTES ] ) { + static constexpr char PROOF_DOMAIN[] = "openQ4-rcon2-proof-v1"; + static constexpr std::size_t PROOF_DOMAIN_BYTES = sizeof( PROOF_DOMAIN ) - 1; + std::uint8_t message[ PROOF_DOMAIN_BYTES + 1 + NONCE_BYTES + NONCE_BYTES + + ENDPOINT_BINDING_BYTES + REQUEST_DIGEST_BYTES ]; + std::size_t offset = 0; + std::memcpy( message + offset, PROOF_DOMAIN, PROOF_DOMAIN_BYTES ); + offset += PROOF_DOMAIN_BYTES; + message[ offset++ ] = PROTOCOL_VERSION; + std::memcpy( message + offset, clientNonce, NONCE_BYTES ); + offset += NONCE_BYTES; + std::memcpy( message + offset, serverNonce, NONCE_BYTES ); + offset += NONCE_BYTES; + std::memcpy( message + offset, endpointBinding, ENDPOINT_BINDING_BYTES ); + offset += ENDPOINT_BINDING_BYTES; + std::memcpy( message + offset, requestDigest, REQUEST_DIGEST_BYTES ); + idCrypto::HMACSHA256( verifier, VERIFIER_BYTES, message, sizeof( message ), proof ); + idCrypto::SecureZero( message, sizeof( message ) ); +} + +} // namespace idRcon2 diff --git a/src/framework/async/Rcon2Protocol.h b/src/framework/async/Rcon2Protocol.h new file mode 100644 index 00000000..651986a0 --- /dev/null +++ b/src/framework/async/Rcon2Protocol.h @@ -0,0 +1,51 @@ +/* +=========================================================================== + +openQ4 authenticated remote-console protocol +Copyright (C) 2026 DarkMatter Productions + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +=========================================================================== +*/ + +#ifndef __RCON2PROTOCOL_H__ +#define __RCON2PROTOCOL_H__ + +#include "../../idlib/CryptoHash.h" + +#include +#include + +namespace idRcon2 { + +static constexpr std::uint8_t PROTOCOL_VERSION = 1; +static constexpr std::size_t NONCE_BYTES = 16; +static constexpr std::size_t SALT_BYTES = 16; +static constexpr std::size_t ENDPOINT_BINDING_BYTES = 16; +static constexpr std::size_t REQUEST_DIGEST_BYTES = idCrypto::SHA256_DIGEST_BYTES; +static constexpr std::size_t PROOF_BYTES = idCrypto::SHA256_DIGEST_BYTES; +static constexpr std::size_t VERIFIER_BYTES = idCrypto::SHA256_DIGEST_BYTES; +static constexpr std::uint32_t PBKDF2_ITERATIONS = 200000; +static constexpr std::size_t MIN_PASSWORD_BYTES = 12; + +void HashRequest( const char *command, + std::uint8_t digest[ REQUEST_DIGEST_BYTES ] ); + +bool DeriveVerifier( const char *password, + const std::uint8_t salt[ SALT_BYTES ], + std::uint8_t verifier[ VERIFIER_BYTES ] ); + +void ComputeProof( const std::uint8_t verifier[ VERIFIER_BYTES ], + const std::uint8_t clientNonce[ NONCE_BYTES ], + const std::uint8_t serverNonce[ NONCE_BYTES ], + const std::uint8_t endpointBinding[ ENDPOINT_BINDING_BYTES ], + const std::uint8_t requestDigest[ REQUEST_DIGEST_BYTES ], + std::uint8_t proof[ PROOF_BYTES ] ); + +} // namespace idRcon2 + +#endif /* !__RCON2PROTOCOL_H__ */ diff --git a/src/framework/licensee.h b/src/framework/licensee.h index 1acd0d0e..3a751a65 100644 --- a/src/framework/licensee.h +++ b/src/framework/licensee.h @@ -19,6 +19,7 @@ #define PROJECT_VERSION_DATE OPENQ4_VERSION_DATE #define PROJECT_WEBSITE "www.darkmatter-quake.com" #define PROJECT_REPO "https://github.com/themuffinator/openQ4" +#define PROJECT_RELEASES_URL PROJECT_REPO "/releases" #define GAME_NAME PROJECT_NAME // appears on errors and secondary windows #define GAME_ICON "q4icon.bmp" diff --git a/src/idlib/BitMsg.cpp b/src/idlib/BitMsg.cpp index a308303a..de3558df 100644 --- a/src/idlib/BitMsg.cpp +++ b/src/idlib/BitMsg.cpp @@ -81,7 +81,7 @@ idBitMsg::WriteBits */ void idBitMsg::WriteBits( int value, int numBits ) { int put; - int fraction; + uint32_t fraction; if ( !writeData ) { idLib::common->Error( "idBitMsg::WriteBits: cannot write to message" ); @@ -95,13 +95,12 @@ void idBitMsg::WriteBits( int value, int numBits ) { // check for value overflows if ( numBits != 32 ) { if ( numBits > 0 ) { - if ( value > ( 1 << numBits ) - 1 ) { - idLib::common->Warning( "idBitMsg::WriteBits: value overflow %d %d", value, numBits ); - } else if ( value < 0 ) { + const uint32_t maximumValue = ( 1u << numBits ) - 1u; + if ( value < 0 || static_cast( value ) > maximumValue ) { idLib::common->Warning( "idBitMsg::WriteBits: value overflow %d %d", value, numBits ); } } else { - int r = 1 << ( - 1 - numBits ); + const int64_t r = static_cast( 1 ) << ( -1 - numBits ); if ( value > r - 1 ) { idLib::common->Warning( "idBitMsg::WriteBits: value overflow %d %d", value, numBits ); } else if ( value < -r ) { @@ -113,6 +112,7 @@ void idBitMsg::WriteBits( int value, int numBits ) { if ( numBits < 0 ) { numBits = -numBits; } + uint32_t unsignedValue = static_cast( value ); // check for msg overflow if ( CheckOverflow( numBits ) ) { @@ -129,10 +129,10 @@ void idBitMsg::WriteBits( int value, int numBits ) { if ( put > numBits ) { put = numBits; } - fraction = value & ( ( 1 << put ) - 1 ); - writeData[curSize - 1] |= fraction << writeBit; + fraction = unsignedValue & ( ( 1u << put ) - 1u ); + writeData[curSize - 1] |= static_cast( fraction << writeBit ); numBits -= put; - value >>= put; + unsignedValue >>= put; writeBit = ( writeBit + put ) & 7; } } @@ -204,18 +204,26 @@ idBitMsg::WriteDeltaByteCounter ================ */ void idBitMsg::WriteDeltaByteCounter( int oldValue, int newValue ) { - int i, x; + int i; + const uint32_t x = static_cast( oldValue ^ newValue ); - x = oldValue ^ newValue; - for ( i = 7; i > 0; i-- ) { - if ( x & ( 1 << i ) ) { + if ( x & ( 1u << 7 ) ) { + idLib::common->Warning( "idBitMsg::WriteDeltaByteCounter: unencodable high-bit change" ); + WriteBits( 0, 3 ); + return; + } + for ( i = 6; i >= 0; i-- ) { + if ( x & ( 1u << i ) ) { i++; break; } } + if ( i < 0 ) { + i = 0; + } WriteBits( i, 3 ); if ( i ) { - WriteBits( ( ( 1 << i ) - 1 ) & newValue, i ); + WriteBits( static_cast( ( ( 1u << i ) - 1u ) & static_cast( newValue ) ), i ); } } @@ -225,18 +233,26 @@ idBitMsg::WriteDeltaShortCounter ================ */ void idBitMsg::WriteDeltaShortCounter( int oldValue, int newValue ) { - int i, x; + int i; + const uint32_t x = static_cast( oldValue ^ newValue ); - x = oldValue ^ newValue; - for ( i = 15; i > 0; i-- ) { - if ( x & ( 1 << i ) ) { + if ( x & ( 1u << 15 ) ) { + idLib::common->Warning( "idBitMsg::WriteDeltaShortCounter: unencodable high-bit change" ); + WriteBits( 0, 4 ); + return; + } + for ( i = 14; i >= 0; i-- ) { + if ( x & ( 1u << i ) ) { i++; break; } } + if ( i < 0 ) { + i = 0; + } WriteBits( i, 4 ); if ( i ) { - WriteBits( ( ( 1 << i ) - 1 ) & newValue, i ); + WriteBits( static_cast( ( ( 1u << i ) - 1u ) & static_cast( newValue ) ), i ); } } @@ -246,18 +262,26 @@ idBitMsg::WriteDeltaLongCounter ================ */ void idBitMsg::WriteDeltaLongCounter( int oldValue, int newValue ) { - int i, x; + int i; + const uint32_t x = static_cast( oldValue ^ newValue ); - x = oldValue ^ newValue; - for ( i = 31; i > 0; i-- ) { - if ( x & ( 1 << i ) ) { + if ( x & ( 1u << 31 ) ) { + idLib::common->Warning( "idBitMsg::WriteDeltaLongCounter: unencodable high-bit change" ); + WriteBits( 0, 5 ); + return; + } + for ( i = 30; i >= 0; i-- ) { + if ( x & ( 1u << i ) ) { i++; break; } } + if ( i < 0 ) { + i = 0; + } WriteBits( i, 5 ); if ( i ) { - WriteBits( ( ( 1 << i ) - 1 ) & newValue, i ); + WriteBits( static_cast( ( ( 1u << i ) - 1u ) & static_cast( newValue ) ), i ); } } @@ -321,10 +345,10 @@ idBitMsg::ReadBits ================ */ int idBitMsg::ReadBits( int numBits ) const { - int value; + uint32_t value; int valueBits; int get; - int fraction; + uint32_t fraction; bool sgn; if ( !readData ) { @@ -347,7 +371,8 @@ int idBitMsg::ReadBits( int numBits ) const { } // check for overflow - if ( numBits > GetRemainingReadBits() ) { + if ( IsReadOverflowed() || numBits > GetRemainingReadBits() ) { + MarkReadOverflowed(); return -1; } @@ -361,7 +386,7 @@ int idBitMsg::ReadBits( int numBits ) const { } fraction = readData[readCount - 1]; fraction >>= readBit; - fraction &= ( 1 << get ) - 1; + fraction &= ( 1u << get ) - 1u; value |= fraction << valueBits; valueBits += get; @@ -369,12 +394,14 @@ int idBitMsg::ReadBits( int numBits ) const { } if ( sgn ) { - if ( value & ( 1 << ( numBits - 1 ) ) ) { - value |= -1 ^ ( ( 1 << numBits ) - 1 ); + if ( value & ( 1u << ( numBits - 1 ) ) ) { + value |= ~( ( 1u << numBits ) - 1u ); } } - return value; + int result; + memcpy( &result, &value, sizeof( result ) ); + return result; } /* @@ -419,13 +446,20 @@ int idBitMsg::ReadData( void *data, int length ) const { int cnt; ReadByteAlign(); + if ( IsReadOverflowed() || length < 0 ) { + MarkReadOverflowed(); + return 0; + } cnt = readCount; - if ( readCount + length > curSize ) { + if ( length > curSize - readCount ) { + const int remaining = curSize - readCount; if ( data ) { - memcpy( data, readData + readCount, GetRemainingData() ); + memcpy( data, readData + readCount, remaining ); + memset( static_cast( data ) + remaining, 0, length - remaining ); } - readCount = curSize; + MarkReadOverflowed(); + return remaining; } else { if ( data ) { memcpy( data, readData + readCount, length ); @@ -473,11 +507,16 @@ int idBitMsg::ReadDeltaByteCounter( int oldValue ) const { int i, newValue; i = ReadBits( 3 ); - if ( !i ) { + if ( IsReadOverflowed() || i <= 0 ) { return oldValue; } newValue = ReadBits( i ); - return ( oldValue & ~( ( 1 << i ) - 1 ) | newValue ); + if ( IsReadOverflowed() ) { + return oldValue; + } + const unsigned int mask = ( 1u << i ) - 1u; + return static_cast( ( static_cast( oldValue ) & ~mask ) | + ( static_cast( newValue ) & mask ) ); } /* @@ -489,11 +528,16 @@ int idBitMsg::ReadDeltaShortCounter( int oldValue ) const { int i, newValue; i = ReadBits( 4 ); - if ( !i ) { + if ( IsReadOverflowed() || i <= 0 ) { return oldValue; } newValue = ReadBits( i ); - return ( oldValue & ~( ( 1 << i ) - 1 ) | newValue ); + if ( IsReadOverflowed() ) { + return oldValue; + } + const unsigned int mask = ( 1u << i ) - 1u; + return static_cast( ( static_cast( oldValue ) & ~mask ) | + ( static_cast( newValue ) & mask ) ); } /* @@ -505,11 +549,16 @@ int idBitMsg::ReadDeltaLongCounter( int oldValue ) const { int i, newValue; i = ReadBits( 5 ); - if ( !i ) { + if ( IsReadOverflowed() || i <= 0 || i > 31 ) { return oldValue; } newValue = ReadBits( i ); - return ( oldValue & ~( ( 1 << i ) - 1 ) | newValue ); + if ( IsReadOverflowed() ) { + return oldValue; + } + const unsigned int mask = ( 1u << i ) - 1u; + return static_cast( ( static_cast( oldValue ) & ~mask ) | + ( static_cast( newValue ) & mask ) ); } /* @@ -521,24 +570,43 @@ bool idBitMsg::ReadDeltaDict( idDict &dict, const idDict *base ) const { char key[MAX_STRING_CHARS]; char value[MAX_STRING_CHARS]; bool changed = false; + idDict decoded; if ( base != NULL ) { - dict = *base; + decoded = *base; } else { - dict.Clear(); + decoded.Clear(); } - while( ReadString( key, sizeof( key ) ) != 0 ) { + while ( true ) { + ReadString( key, sizeof( key ) ); + if ( IsReadOverflowed() ) { + return false; + } + if ( key[ 0 ] == '\0' ) { + break; + } ReadString( value, sizeof( value ) ); - dict.Set( key, value ); + if ( IsReadOverflowed() ) { + return false; + } + decoded.Set( key, value ); changed = true; } - while( ReadString( key, sizeof( key ) ) != 0 ) { - dict.Delete( key ); + while ( true ) { + ReadString( key, sizeof( key ) ); + if ( IsReadOverflowed() ) { + return false; + } + if ( key[ 0 ] == '\0' ) { + break; + } + decoded.Delete( key ); changed = true; } + dict = decoded; return changed; } @@ -617,7 +685,7 @@ void idBitMsgDelta::WriteBits( int value, int numBits ) { changed = true; } else { int baseValue = base->ReadBits( numBits ); - if ( baseValue == value ) { + if ( !base->IsReadOverflowed() && baseValue == value ) { writeDelta->WriteBits( 0, 1 ); } else { writeDelta->WriteBits( 1, 1 ); @@ -647,7 +715,7 @@ void idBitMsgDelta::WriteDelta( int oldValue, int newValue, int numBits ) { changed = true; } else { int baseValue = base->ReadBits( numBits ); - if ( baseValue == newValue ) { + if ( !base->IsReadOverflowed() && baseValue == newValue ) { writeDelta->WriteBits( 0, 1 ); } else { writeDelta->WriteBits( 1, 1 ); @@ -676,8 +744,14 @@ int idBitMsgDelta::ReadBits( int numBits ) const { changed = true; } else { int baseValue = base->ReadBits( numBits ); - if ( !readDelta || readDelta->ReadBits( 1 ) == 0 ) { + const bool baseOverflowed = base->IsReadOverflowed(); + if ( !readDelta ) { + value = baseValue; + } else if ( readDelta->ReadBits( 1 ) == 0 ) { value = baseValue; + if ( baseOverflowed ) { + readDelta->MarkReadOverflowed(); + } } else { value = readDelta->ReadBits( numBits ); changed = true; @@ -707,8 +781,14 @@ int idBitMsgDelta::ReadDelta( int oldValue, int numBits ) const { changed = true; } else { int baseValue = base->ReadBits( numBits ); - if ( !readDelta || readDelta->ReadBits( 1 ) == 0 ) { + const bool baseOverflowed = base->IsReadOverflowed(); + if ( !readDelta ) { value = baseValue; + } else if ( readDelta->ReadBits( 1 ) == 0 ) { + value = baseValue; + if ( baseOverflowed ) { + readDelta->MarkReadOverflowed(); + } } else if ( readDelta->ReadBits( 1 ) == 0 ) { value = oldValue; changed = true; @@ -740,7 +820,7 @@ void idBitMsgDelta::WriteString( const char *s, int maxLength ) { } else { char baseString[MAX_DATA_BUFFER]; base->ReadString( baseString, sizeof( baseString ) ); - if ( idStr::Cmp( s, baseString ) == 0 ) { + if ( !base->IsReadOverflowed() && idStr::Cmp( s, baseString ) == 0 ) { writeDelta->WriteBits( 0, 1 ); } else { writeDelta->WriteBits( 1, 1 ); @@ -767,7 +847,7 @@ void idBitMsgDelta::WriteData( const void *data, int length ) { byte baseData[MAX_DATA_BUFFER]; assert( length < sizeof( baseData ) ); base->ReadData( baseData, length ); - if ( memcmp( data, baseData, length ) == 0 ) { + if ( !base->IsReadOverflowed() && memcmp( data, baseData, length ) == 0 ) { writeDelta->WriteBits( 0, 1 ); } else { writeDelta->WriteBits( 1, 1 ); @@ -812,7 +892,7 @@ void idBitMsgDelta::WriteDeltaByteCounter( int oldValue, int newValue ) { changed = true; } else { int baseValue = base->ReadBits( 8 ); - if ( baseValue == newValue ) { + if ( !base->IsReadOverflowed() && baseValue == newValue ) { writeDelta->WriteBits( 0, 1 ); } else { writeDelta->WriteBits( 1, 1 ); @@ -837,7 +917,7 @@ void idBitMsgDelta::WriteDeltaShortCounter( int oldValue, int newValue ) { changed = true; } else { int baseValue = base->ReadBits( 16 ); - if ( baseValue == newValue ) { + if ( !base->IsReadOverflowed() && baseValue == newValue ) { writeDelta->WriteBits( 0, 1 ); } else { writeDelta->WriteBits( 1, 1 ); @@ -862,7 +942,7 @@ void idBitMsgDelta::WriteDeltaLongCounter( int oldValue, int newValue ) { changed = true; } else { int baseValue = base->ReadBits( 32 ); - if ( baseValue == newValue ) { + if ( !base->IsReadOverflowed() && baseValue == newValue ) { writeDelta->WriteBits( 0, 1 ); } else { writeDelta->WriteBits( 1, 1 ); @@ -884,8 +964,14 @@ void idBitMsgDelta::ReadString( char *buffer, int bufferSize ) const { } else { char baseString[MAX_DATA_BUFFER]; base->ReadString( baseString, sizeof( baseString ) ); - if ( !readDelta || readDelta->ReadBits( 1 ) == 0 ) { + const bool baseOverflowed = base->IsReadOverflowed(); + if ( !readDelta ) { + idStr::Copynz( buffer, baseString, bufferSize ); + } else if ( readDelta->ReadBits( 1 ) == 0 ) { idStr::Copynz( buffer, baseString, bufferSize ); + if ( baseOverflowed ) { + readDelta->MarkReadOverflowed(); + } } else { readDelta->ReadString( buffer, bufferSize ); changed = true; @@ -910,8 +996,14 @@ void idBitMsgDelta::ReadData( void *data, int length ) const { char baseData[MAX_DATA_BUFFER]; assert( length < sizeof( baseData ) ); base->ReadData( baseData, length ); - if ( !readDelta || readDelta->ReadBits( 1 ) == 0 ) { + const bool baseOverflowed = base->IsReadOverflowed(); + if ( !readDelta ) { + memcpy( data, baseData, length ); + } else if ( readDelta->ReadBits( 1 ) == 0 ) { memcpy( data, baseData, length ); + if ( baseOverflowed ) { + readDelta->MarkReadOverflowed(); + } } else { readDelta->ReadData( data, length ); changed = true; @@ -960,8 +1052,14 @@ int idBitMsgDelta::ReadDeltaByteCounter( int oldValue ) const { changed = true; } else { int baseValue = base->ReadBits( 8 ); - if ( !readDelta || readDelta->ReadBits( 1 ) == 0 ) { + const bool baseOverflowed = base->IsReadOverflowed(); + if ( !readDelta ) { + value = baseValue; + } else if ( readDelta->ReadBits( 1 ) == 0 ) { value = baseValue; + if ( baseOverflowed ) { + readDelta->MarkReadOverflowed(); + } } else { value = readDelta->ReadDeltaByteCounter( oldValue ); changed = true; @@ -987,8 +1085,14 @@ int idBitMsgDelta::ReadDeltaShortCounter( int oldValue ) const { changed = true; } else { int baseValue = base->ReadBits( 16 ); - if ( !readDelta || readDelta->ReadBits( 1 ) == 0 ) { + const bool baseOverflowed = base->IsReadOverflowed(); + if ( !readDelta ) { + value = baseValue; + } else if ( readDelta->ReadBits( 1 ) == 0 ) { value = baseValue; + if ( baseOverflowed ) { + readDelta->MarkReadOverflowed(); + } } else { value = readDelta->ReadDeltaShortCounter( oldValue ); changed = true; @@ -1014,8 +1118,14 @@ int idBitMsgDelta::ReadDeltaLongCounter( int oldValue ) const { changed = true; } else { int baseValue = base->ReadBits( 32 ); - if ( !readDelta || readDelta->ReadBits( 1 ) == 0 ) { + const bool baseOverflowed = base->IsReadOverflowed(); + if ( !readDelta ) { + value = baseValue; + } else if ( readDelta->ReadBits( 1 ) == 0 ) { value = baseValue; + if ( baseOverflowed ) { + readDelta->MarkReadOverflowed(); + } } else { value = readDelta->ReadDeltaLongCounter( oldValue ); changed = true; @@ -1106,15 +1216,34 @@ bool idMsgQueue::Get( byte *data, int dataSize, int &size, bool sequencing ) { size = 0; return false; } - int sequence; + const int headerSize = sequencing ? 6 : 2; + const int totalSize = GetTotalSize(); + if ( totalSize < headerSize ) { + common->Warning( "idMsgQueue::Get: truncated queue record header" ); + size = 0; + first = last; + startIndex = endIndex; + return false; + } size = ReadUShort(); - if ( data && size > dataSize ) { - common->Error( "idMsgQueue::Get buffer size of %d < get size of %d", dataSize, size ); + if ( size <= 0 || size > totalSize - headerSize || ( data && size > dataSize ) ) { + common->Warning( "idMsgQueue::Get: invalid record size %d (available %d, destination %d)", + size, totalSize - headerSize, data ? dataSize : 0 ); + size = 0; + first = last; + startIndex = endIndex; + return false; } if ( sequencing ) { - sequence = ReadLong(); - assert( sequence == first ); + const int sequence = ReadLong(); + if ( sequence != first ) { + common->Warning( "idMsgQueue::Get: invalid record sequence %d (expected %d)", sequence, first ); + size = 0; + first = last; + startIndex = endIndex; + return false; + } } ReadData( data, size ); first++; @@ -1321,13 +1450,16 @@ idMsgQueue::ReadFrom void idMsgQueue::ReadFrom( const idBitMsg &msg ) { Init( 0 ); const int encodedSize = msg.ReadUShort(); + const int available = msg.GetRemainingData(); + const int remaining = available > 0 ? available : 0; if ( encodedSize < 0 || encodedSize >= MAX_MSG_QUEUE_SIZE || - encodedSize > msg.GetRemainingData() ) { + encodedSize > remaining ) { common->Warning( "idMsgQueue::ReadFrom: invalid encoded queue size %d", encodedSize ); - const int discard = idMath::ClampInt( 0, msg.GetRemainingData(), encodedSize ); + const int discard = idMath::ClampInt( 0, remaining, encodedSize ); if ( discard > 0 ) { msg.ReadData( NULL, discard ); } + msg.MarkReadOverflowed(); endIndex = 0; return; } @@ -1335,6 +1467,29 @@ void idMsgQueue::ReadFrom( const idBitMsg &msg ) { if ( msg.ReadData( buffer, endIndex ) != endIndex ) { common->Warning( "idMsgQueue::ReadFrom: truncated encoded queue" ); endIndex = 0; + return; + } + + // ReadFrom is used for serialized unreliable-message queues, whose nested + // records are a little-endian ushort length followed by that payload. + // Validate every nested boundary now so Get cannot later walk uninitialized + // ring storage when a snapshot supplied a forged record size. + for ( int offset = 0; offset < endIndex; ) { + if ( endIndex - offset < 2 ) { + common->Warning( "idMsgQueue::ReadFrom: truncated nested record header" ); + msg.MarkReadOverflowed(); + Init( 0 ); + return; + } + const int recordSize = buffer[ offset ] | ( buffer[ offset + 1 ] << 8 ); + offset += 2; + if ( recordSize <= 0 || recordSize > endIndex - offset ) { + common->Warning( "idMsgQueue::ReadFrom: invalid nested record size %d", recordSize ); + msg.MarkReadOverflowed(); + Init( 0 ); + return; + } + offset += recordSize; } } diff --git a/src/idlib/BitMsg.h b/src/idlib/BitMsg.h index 05167570..5cb69dc0 100644 --- a/src/idlib/BitMsg.h +++ b/src/idlib/BitMsg.h @@ -27,6 +27,8 @@ class idBitMsg { int GetMaxSize( void ) const; // get the maximum message size void SetAllowOverflow( bool set ); // generate error if not set and message is overflowed bool IsOverflowed( void ) const; // returns true if the message was overflowed + bool IsReadOverflowed( void ) const; // returns true after a read requested data beyond the message + void MarkReadOverflowed( void ) const; // fail a higher-level semantic decode without changing object layout int GetSize( void ) const; // size of the message in bytes void SetSize( int size ); // set the message size @@ -166,6 +168,15 @@ ID_INLINE bool idBitMsg::IsOverflowed( void ) const { return overflowed; } +ID_INLINE bool idBitMsg::IsReadOverflowed( void ) const { + return readBit == 8; +} + +ID_INLINE void idBitMsg::MarkReadOverflowed( void ) const { + readCount = curSize; + readBit = 8; +} + ID_INLINE int idBitMsg::GetSize( void ) const { return curSize; } @@ -339,7 +350,9 @@ ID_INLINE void idBitMsg::BeginReading( void ) const { } ID_INLINE void idBitMsg::ReadByteAlign( void ) const { - readBit = 0; + if ( !IsReadOverflowed() ) { + readBit = 0; + } } ID_INLINE int idBitMsg::ReadChar( void ) const { @@ -430,6 +443,8 @@ class idBitMsgDelta { void InitWriting( const idBitMsg *base, idBitMsg *newBase, idBitMsg *delta ); void InitReading( const idBitMsg *base, idBitMsg *newBase, const idBitMsg *delta ); bool HasChanged( void ) const; + bool IsReadOverflowed( void ) const; + void MarkReadOverflowed( void ) const; void WriteBits( int value, int numBits ); void WriteChar( int c ); @@ -523,6 +538,22 @@ ID_INLINE idBitMsgDelta::idBitMsgDelta() { changed = false; } +ID_INLINE bool idBitMsgDelta::IsReadOverflowed( void ) const { + // A conditional delta tail may legitimately extend beyond its trusted old + // base. The readers below promote that condition to wire overflow only if + // the delta asks to reuse a base value that is not available. + return readDelta != NULL ? readDelta->IsReadOverflowed() : + ( base != NULL && base->IsReadOverflowed() ); +} + +ID_INLINE void idBitMsgDelta::MarkReadOverflowed( void ) const { + if ( readDelta != NULL ) { + readDelta->MarkReadOverflowed(); + } else if ( base != NULL ) { + base->MarkReadOverflowed(); + } +} + ID_INLINE void idBitMsgDelta::InitWriting( const idBitMsg *base, idBitMsg *newBase, idBitMsg *delta ) { this->base = base; this->newBase = newBase; diff --git a/src/idlib/CmdArgs.cpp b/src/idlib/CmdArgs.cpp index aeb1b2f6..cd56c627 100644 --- a/src/idlib/CmdArgs.cpp +++ b/src/idlib/CmdArgs.cpp @@ -22,8 +22,9 @@ void idCmdArgs::operator=( const idCmdArgs &args ) { idCmdArgs::Args ============ */ +static idStr cmd_args; + const char *idCmdArgs::Args( int start, int end, bool escapeArgs ) const { - static idStr cmd_args; int i; if ( start < 0 ) { @@ -34,7 +35,9 @@ const char *idCmdArgs::Args( int start, int end, bool escapeArgs ) const { } else if ( end >= argc ) { end = argc - 1; } - cmd_args.Clear(); + // The previous result may have contained a private CVar value. Scrub the + // complete allocation before reusing this process-wide scratch string. + cmd_args.SecureClear(); if ( escapeArgs ) { cmd_args += "\""; } @@ -121,7 +124,12 @@ void idCmdArgs::TokenizeString( const char *text, bool keepAsStrings ) { return; } if ( idLib::cvarSystem ) { - token = idLib::cvarSystem->GetCVarString( token.c_str() ); + idCVar *expandedCVar = idLib::cvarSystem->Find( token.c_str() ); + if ( expandedCVar != NULL && ( expandedCVar->GetFlags() & CVAR_PRIVATE ) ) { + token = ""; + } else { + token = idLib::cvarSystem->GetCVarString( token.c_str() ); + } } else { token = ""; } @@ -181,6 +189,33 @@ void idCmdArgs::AppendArg( const char *text ) { argc++; } +/* +============ +idCmdArgs::ClearSensitive + +Clear command tokens that may have held a private CVar value. This is kept +separate from the hot-path Clear() used for ordinary argument objects. +============ +*/ +void idCmdArgs::ClearSensitive( void ) { + volatile byte *cursor = reinterpret_cast( this ); + for ( size_t remaining = sizeof( *this ); remaining > 0; --remaining ) { + *cursor++ = 0; + } +} + +/* +============ +idCmdArgs::ClearArgsScratch + +Args() assembles its result in a shared idStr. Private command values must not +survive there after the command object itself has been wiped. +============ +*/ +void idCmdArgs::ClearArgsScratch( void ) { + cmd_args.SecureClear(); +} + /* ============ idCmdArgs::GetArgs diff --git a/src/idlib/CmdArgs.h b/src/idlib/CmdArgs.h index 017af078..1b16868f 100644 --- a/src/idlib/CmdArgs.h +++ b/src/idlib/CmdArgs.h @@ -32,6 +32,8 @@ class idCmdArgs { void AppendArg( const char *text ); void Clear( void ) { argc = 0; } + void ClearSensitive( void ); + static void ClearArgsScratch( void ); const char ** GetArgs( int *argc ); private: diff --git a/src/idlib/CryptoHash.cpp b/src/idlib/CryptoHash.cpp new file mode 100644 index 00000000..e65ec8f3 --- /dev/null +++ b/src/idlib/CryptoHash.cpp @@ -0,0 +1,320 @@ +/* +=========================================================================== + +openQ4 cryptographic hash primitives +Copyright (C) 2026 DarkMatter Productions + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +This is an original implementation of the published FIPS 180-4 SHA-256, +RFC 2104 HMAC, and RFC 8018 PBKDF2 specifications. No source code from the +Quake 4 SDK game module or another cryptographic implementation is used here. + +=========================================================================== +*/ + +#include "CryptoHash.h" + +#include + +namespace idCrypto { +namespace { + +struct sha256Context_t { + std::uint32_t state[ 8 ]; + std::uint64_t totalBytes; + std::uint8_t buffer[ SHA256_BLOCK_BYTES ]; + std::size_t bufferedBytes; +}; + +static constexpr std::uint32_t SHA256_ROUND_CONSTANTS[ 64 ] = { + 0x428a2f98u, 0x71374491u, 0xb5c0fbcfu, 0xe9b5dba5u, + 0x3956c25bu, 0x59f111f1u, 0x923f82a4u, 0xab1c5ed5u, + 0xd807aa98u, 0x12835b01u, 0x243185beu, 0x550c7dc3u, + 0x72be5d74u, 0x80deb1feu, 0x9bdc06a7u, 0xc19bf174u, + 0xe49b69c1u, 0xefbe4786u, 0x0fc19dc6u, 0x240ca1ccu, + 0x2de92c6fu, 0x4a7484aau, 0x5cb0a9dcu, 0x76f988dau, + 0x983e5152u, 0xa831c66du, 0xb00327c8u, 0xbf597fc7u, + 0xc6e00bf3u, 0xd5a79147u, 0x06ca6351u, 0x14292967u, + 0x27b70a85u, 0x2e1b2138u, 0x4d2c6dfcu, 0x53380d13u, + 0x650a7354u, 0x766a0abbu, 0x81c2c92eu, 0x92722c85u, + 0xa2bfe8a1u, 0xa81a664bu, 0xc24b8b70u, 0xc76c51a3u, + 0xd192e819u, 0xd6990624u, 0xf40e3585u, 0x106aa070u, + 0x19a4c116u, 0x1e376c08u, 0x2748774cu, 0x34b0bcb5u, + 0x391c0cb3u, 0x4ed8aa4au, 0x5b9cca4fu, 0x682e6ff3u, + 0x748f82eeu, 0x78a5636fu, 0x84c87814u, 0x8cc70208u, + 0x90befffau, 0xa4506cebu, 0xbef9a3f7u, 0xc67178f2u +}; + +static inline std::uint32_t RotateRight( std::uint32_t value, unsigned int count ) { + return ( value >> count ) | ( value << ( 32u - count ) ); +} + +static inline std::uint32_t ReadBigEndian32( const std::uint8_t *bytes ) { + return ( static_cast( bytes[ 0 ] ) << 24 ) | + ( static_cast( bytes[ 1 ] ) << 16 ) | + ( static_cast( bytes[ 2 ] ) << 8 ) | + static_cast( bytes[ 3 ] ); +} + +static inline void WriteBigEndian32( std::uint8_t *bytes, std::uint32_t value ) { + bytes[ 0 ] = static_cast( value >> 24 ); + bytes[ 1 ] = static_cast( value >> 16 ); + bytes[ 2 ] = static_cast( value >> 8 ); + bytes[ 3 ] = static_cast( value ); +} + +static void SHA256Transform( sha256Context_t &context, + const std::uint8_t block[ SHA256_BLOCK_BYTES ] ) { + std::uint32_t schedule[ 64 ]; + for ( int index = 0; index < 16; ++index ) { + schedule[ index ] = ReadBigEndian32( block + index * 4 ); + } + for ( int index = 16; index < 64; ++index ) { + const std::uint32_t sigma0 = RotateRight( schedule[ index - 15 ], 7 ) ^ + RotateRight( schedule[ index - 15 ], 18 ) ^ ( schedule[ index - 15 ] >> 3 ); + const std::uint32_t sigma1 = RotateRight( schedule[ index - 2 ], 17 ) ^ + RotateRight( schedule[ index - 2 ], 19 ) ^ ( schedule[ index - 2 ] >> 10 ); + schedule[ index ] = schedule[ index - 16 ] + sigma0 + + schedule[ index - 7 ] + sigma1; + } + + std::uint32_t a = context.state[ 0 ]; + std::uint32_t b = context.state[ 1 ]; + std::uint32_t c = context.state[ 2 ]; + std::uint32_t d = context.state[ 3 ]; + std::uint32_t e = context.state[ 4 ]; + std::uint32_t f = context.state[ 5 ]; + std::uint32_t g = context.state[ 6 ]; + std::uint32_t h = context.state[ 7 ]; + + for ( int index = 0; index < 64; ++index ) { + const std::uint32_t sum1 = RotateRight( e, 6 ) ^ RotateRight( e, 11 ) ^ RotateRight( e, 25 ); + const std::uint32_t choose = ( e & f ) ^ ( ( ~e ) & g ); + const std::uint32_t temp1 = h + sum1 + choose + + SHA256_ROUND_CONSTANTS[ index ] + schedule[ index ]; + const std::uint32_t sum0 = RotateRight( a, 2 ) ^ RotateRight( a, 13 ) ^ RotateRight( a, 22 ); + const std::uint32_t majority = ( a & b ) ^ ( a & c ) ^ ( b & c ); + const std::uint32_t temp2 = sum0 + majority; + + h = g; + g = f; + f = e; + e = d + temp1; + d = c; + c = b; + b = a; + a = temp1 + temp2; + } + + context.state[ 0 ] += a; + context.state[ 1 ] += b; + context.state[ 2 ] += c; + context.state[ 3 ] += d; + context.state[ 4 ] += e; + context.state[ 5 ] += f; + context.state[ 6 ] += g; + context.state[ 7 ] += h; + idCrypto::SecureZero( schedule, sizeof( schedule ) ); +} + +static void SHA256Init( sha256Context_t &context ) { + context.state[ 0 ] = 0x6a09e667u; + context.state[ 1 ] = 0xbb67ae85u; + context.state[ 2 ] = 0x3c6ef372u; + context.state[ 3 ] = 0xa54ff53au; + context.state[ 4 ] = 0x510e527fu; + context.state[ 5 ] = 0x9b05688cu; + context.state[ 6 ] = 0x1f83d9abu; + context.state[ 7 ] = 0x5be0cd19u; + context.totalBytes = 0; + context.bufferedBytes = 0; + std::memset( context.buffer, 0, sizeof( context.buffer ) ); +} + +static void SHA256Update( sha256Context_t &context, const void *data, std::size_t dataBytes ) { + const std::uint8_t *cursor = static_cast( data ); + context.totalBytes += static_cast( dataBytes ); + + if ( context.bufferedBytes != 0 ) { + const std::size_t wanted = SHA256_BLOCK_BYTES - context.bufferedBytes; + const std::size_t copied = dataBytes < wanted ? dataBytes : wanted; + if ( copied != 0 ) { + std::memcpy( context.buffer + context.bufferedBytes, cursor, copied ); + context.bufferedBytes += copied; + cursor += copied; + dataBytes -= copied; + } + if ( context.bufferedBytes == SHA256_BLOCK_BYTES ) { + SHA256Transform( context, context.buffer ); + context.bufferedBytes = 0; + } + } + + while ( dataBytes >= SHA256_BLOCK_BYTES ) { + SHA256Transform( context, cursor ); + cursor += SHA256_BLOCK_BYTES; + dataBytes -= SHA256_BLOCK_BYTES; + } + if ( dataBytes != 0 ) { + std::memcpy( context.buffer, cursor, dataBytes ); + context.bufferedBytes = dataBytes; + } +} + +static void SHA256Final( sha256Context_t &context, + std::uint8_t digest[ SHA256_DIGEST_BYTES ] ) { + const std::uint64_t totalBits = context.totalBytes * 8u; + context.buffer[ context.bufferedBytes++ ] = 0x80u; + if ( context.bufferedBytes > 56 ) { + std::memset( context.buffer + context.bufferedBytes, 0, + SHA256_BLOCK_BYTES - context.bufferedBytes ); + SHA256Transform( context, context.buffer ); + context.bufferedBytes = 0; + } + std::memset( context.buffer + context.bufferedBytes, 0, 56 - context.bufferedBytes ); + for ( int index = 0; index < 8; ++index ) { + context.buffer[ 56 + index ] = static_cast( totalBits >> ( 56 - index * 8 ) ); + } + SHA256Transform( context, context.buffer ); + for ( int index = 0; index < 8; ++index ) { + WriteBigEndian32( digest + index * 4, context.state[ index ] ); + } + SecureZero( &context, sizeof( context ) ); +} + +struct hmacSHA256Prepared_t { + sha256Context_t inner; + sha256Context_t outer; +}; + +static void HMACPrepare( const void *key, std::size_t keyBytes, + hmacSHA256Prepared_t &prepared ) { + std::uint8_t normalizedKey[ SHA256_BLOCK_BYTES ] = {}; + if ( keyBytes > SHA256_BLOCK_BYTES ) { + SHA256( key, keyBytes, normalizedKey ); + } else if ( keyBytes != 0 ) { + std::memcpy( normalizedKey, key, keyBytes ); + } + + std::uint8_t innerPad[ SHA256_BLOCK_BYTES ]; + std::uint8_t outerPad[ SHA256_BLOCK_BYTES ]; + for ( std::size_t index = 0; index < SHA256_BLOCK_BYTES; ++index ) { + innerPad[ index ] = normalizedKey[ index ] ^ 0x36u; + outerPad[ index ] = normalizedKey[ index ] ^ 0x5cu; + } + SHA256Init( prepared.inner ); + SHA256Update( prepared.inner, innerPad, sizeof( innerPad ) ); + SHA256Init( prepared.outer ); + SHA256Update( prepared.outer, outerPad, sizeof( outerPad ) ); + SecureZero( normalizedKey, sizeof( normalizedKey ) ); + SecureZero( innerPad, sizeof( innerPad ) ); + SecureZero( outerPad, sizeof( outerPad ) ); +} + +static void HMACFinish( const hmacSHA256Prepared_t &prepared, + const void *data, std::size_t dataBytes, + std::uint8_t digest[ SHA256_DIGEST_BYTES ] ) { + sha256Context_t inner = prepared.inner; + sha256Context_t outer = prepared.outer; + std::uint8_t innerDigest[ SHA256_DIGEST_BYTES ]; + SHA256Update( inner, data, dataBytes ); + SHA256Final( inner, innerDigest ); + SHA256Update( outer, innerDigest, sizeof( innerDigest ) ); + SHA256Final( outer, digest ); + SecureZero( innerDigest, sizeof( innerDigest ) ); +} + +} // namespace + +void SHA256( const void *data, std::size_t dataBytes, + std::uint8_t digest[ SHA256_DIGEST_BYTES ] ) { + sha256Context_t context; + SHA256Init( context ); + if ( dataBytes != 0 ) { + SHA256Update( context, data, dataBytes ); + } + SHA256Final( context, digest ); +} + +void HMACSHA256( const void *key, std::size_t keyBytes, + const void *data, std::size_t dataBytes, + std::uint8_t digest[ SHA256_DIGEST_BYTES ] ) { + hmacSHA256Prepared_t prepared; + HMACPrepare( key, keyBytes, prepared ); + HMACFinish( prepared, data, dataBytes, digest ); + SecureZero( &prepared, sizeof( prepared ) ); +} + +bool PBKDF2HMACSHA256( const void *password, std::size_t passwordBytes, + const void *salt, std::size_t saltBytes, std::uint32_t iterations, + void *output, std::size_t outputBytes ) { + if ( ( passwordBytes != 0 && password == nullptr ) || + ( saltBytes != 0 && salt == nullptr ) || output == nullptr || + outputBytes == 0 || outputBytes > SHA256_DIGEST_BYTES || iterations == 0 ) { + return false; + } + + hmacSHA256Prepared_t prepared; + HMACPrepare( password, passwordBytes, prepared ); + sha256Context_t firstInner = prepared.inner; + const std::uint8_t blockIndex[ 4 ] = { 0, 0, 0, 1 }; + std::uint8_t iteration[ SHA256_DIGEST_BYTES ]; + std::uint8_t aggregate[ SHA256_DIGEST_BYTES ]; + + if ( saltBytes != 0 ) { + SHA256Update( firstInner, salt, saltBytes ); + } + SHA256Update( firstInner, blockIndex, sizeof( blockIndex ) ); + std::uint8_t firstDigest[ SHA256_DIGEST_BYTES ]; + SHA256Final( firstInner, firstDigest ); + sha256Context_t firstOuter = prepared.outer; + SHA256Update( firstOuter, firstDigest, sizeof( firstDigest ) ); + SHA256Final( firstOuter, iteration ); + std::memcpy( aggregate, iteration, sizeof( aggregate ) ); + SecureZero( firstDigest, sizeof( firstDigest ) ); + + for ( std::uint32_t round = 1; round < iterations; ++round ) { + std::uint8_t next[ SHA256_DIGEST_BYTES ]; + HMACFinish( prepared, iteration, sizeof( iteration ), next ); + for ( std::size_t index = 0; index < sizeof( aggregate ); ++index ) { + aggregate[ index ] ^= next[ index ]; + } + std::memcpy( iteration, next, sizeof( iteration ) ); + SecureZero( next, sizeof( next ) ); + } + + std::memcpy( output, aggregate, outputBytes ); + SecureZero( iteration, sizeof( iteration ) ); + SecureZero( aggregate, sizeof( aggregate ) ); + SecureZero( &prepared, sizeof( prepared ) ); + return true; +} + +bool ConstantTimeEquals( const void *left, const void *right, std::size_t bytes ) { + if ( bytes == 0 ) { + return true; + } + if ( left == nullptr || right == nullptr ) { + return false; + } + const std::uint8_t *leftBytes = static_cast( left ); + const std::uint8_t *rightBytes = static_cast( right ); + volatile std::uint8_t difference = 0; + for ( std::size_t index = 0; index < bytes; ++index ) { + difference |= leftBytes[ index ] ^ rightBytes[ index ]; + } + return difference == 0; +} + +void SecureZero( void *memory, std::size_t bytes ) { + volatile std::uint8_t *cursor = static_cast( memory ); + while ( bytes-- != 0 ) { + *cursor++ = 0; + } +} + +} // namespace idCrypto diff --git a/src/idlib/CryptoHash.h b/src/idlib/CryptoHash.h new file mode 100644 index 00000000..7031e718 --- /dev/null +++ b/src/idlib/CryptoHash.h @@ -0,0 +1,47 @@ +/* +=========================================================================== + +openQ4 cryptographic hash primitives +Copyright (C) 2026 DarkMatter Productions + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +=========================================================================== +*/ + +#ifndef __CRYPTOHASH_H__ +#define __CRYPTOHASH_H__ + +#include +#include + +namespace idCrypto { + +static constexpr std::size_t SHA256_DIGEST_BYTES = 32; +static constexpr std::size_t SHA256_BLOCK_BYTES = 64; + +// These routines implement the algorithms specified by FIPS 180-4, RFC 2104, +// and RFC 8018. They deliberately have no dependency on engine globals so the +// exact primitives can be exercised by the native safety test target. +void SHA256( const void *data, std::size_t dataBytes, + std::uint8_t digest[ SHA256_DIGEST_BYTES ] ); + +void HMACSHA256( const void *key, std::size_t keyBytes, + const void *data, std::size_t dataBytes, + std::uint8_t digest[ SHA256_DIGEST_BYTES ] ); + +// openQ4 only needs one SHA-256-sized PBKDF2 block. Keeping the interface +// bounded prevents accidental unbounded work or counter-wrap mistakes. +bool PBKDF2HMACSHA256( const void *password, std::size_t passwordBytes, + const void *salt, std::size_t saltBytes, std::uint32_t iterations, + void *output, std::size_t outputBytes ); + +bool ConstantTimeEquals( const void *left, const void *right, std::size_t bytes ); +void SecureZero( void *memory, std::size_t bytes ); + +} // namespace idCrypto + +#endif /* !__CRYPTOHASH_H__ */ diff --git a/src/idlib/PrivateCommand.h b/src/idlib/PrivateCommand.h new file mode 100644 index 00000000..8893fe8b --- /dev/null +++ b/src/idlib/PrivateCommand.h @@ -0,0 +1,76 @@ +/* +=========================================================================== + +openQ4 private command matching helpers +Copyright (C) 2026 DarkMatter Productions + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +=========================================================================== +*/ + +#ifndef __PRIVATECOMMAND_H__ +#define __PRIVATECOMMAND_H__ + +#include +#include + +namespace idPrivateCommand { + +static inline bool IsNameCharacter( const unsigned char value ) { + return ( value >= 'a' && value <= 'z' ) || + ( value >= 'A' && value <= 'Z' ) || + ( value >= '0' && value <= '9' ) || value == '_' || value == '.'; +} + +static inline unsigned char FoldASCII( const unsigned char value ) { + return value >= 'A' && value <= 'Z' ? + static_cast( value + ( 'a' - 'A' ) ) : value; +} + +// Matches a complete CVar-name token anywhere in a command line. Lengths are +// established before the scan so a command ending in a prefix of name never +// causes the comparison or right-boundary check to read beyond its NUL byte. +static inline bool ContainsBoundedCaseInsensitiveToken( const char *commandText, + const char *name ) { + if ( commandText == NULL || name == NULL || commandText[ 0 ] == '\0' || name[ 0 ] == '\0' ) { + return false; + } + + const std::size_t commandBytes = std::strlen( commandText ); + const std::size_t nameBytes = std::strlen( name ); + if ( nameBytes > commandBytes ) { + return false; + } + + for ( std::size_t offset = 0; offset <= commandBytes - nameBytes; ++offset ) { + bool equal = true; + for ( std::size_t index = 0; index < nameBytes; ++index ) { + if ( FoldASCII( static_cast( commandText[ offset + index ] ) ) != + FoldASCII( static_cast( name[ index ] ) ) ) { + equal = false; + break; + } + } + if ( !equal ) { + continue; + } + + const bool leftBounded = offset == 0 || + !IsNameCharacter( static_cast( commandText[ offset - 1 ] ) ); + const std::size_t rightOffset = offset + nameBytes; + const bool rightBounded = rightOffset == commandBytes || + !IsNameCharacter( static_cast( commandText[ rightOffset ] ) ); + if ( leftBounded && rightBounded ) { + return true; + } + } + return false; +} + +} // namespace idPrivateCommand + +#endif /* !__PRIVATECOMMAND_H__ */ diff --git a/src/idlib/Str.h b/src/idlib/Str.h index f8f91cb0..7d255159 100644 --- a/src/idlib/Str.h +++ b/src/idlib/Str.h @@ -198,6 +198,7 @@ class idStr { void Empty( void ); bool IsEmpty( void ) const; void Clear( void ); + void SecureClear( void ); void Append( const char a ); void Append( const idStr &text ); void Append( const char *text ); @@ -831,6 +832,18 @@ ID_INLINE void idStr::Clear( void ) { Init(); } +ID_INLINE void idStr::SecureClear( void ) { + if ( data != NULL ) { + const int bytes = data != baseBuffer ? alloced : static_cast( sizeof( baseBuffer ) ); + volatile char *cursor = data; + for ( int remaining = bytes; remaining > 0; --remaining ) { + *cursor++ = 0; + } + } + FreeData(); + Init(); +} + ID_INLINE void idStr::Append( const char a ) { EnsureAlloced( idStrAllocationDetail::SaturatingAdd( static_cast( len ), 2 ) ); data[ len ] = a; diff --git a/src/renderer/Image.h b/src/renderer/Image.h index 4f9ac200..5d423988 100644 --- a/src/renderer/Image.h +++ b/src/renderer/Image.h @@ -60,6 +60,11 @@ typedef enum { // Preserve retail Quake 4's explicit high-quality / uncompressed material // bucket without renumbering the existing generated-image cache keys. TD_HIGH_QUALITY, + // Appended openQ4 PBR usages keep all historical binary-image cache keys + // stable. Colour inputs receive gamma-correct mip generation; material + // data (ORM/metallic/roughness/AO) always remains linear. + TD_PBR_COLOR, + TD_MATERIAL_DATA, } textureUsage_t; typedef enum { @@ -173,7 +178,9 @@ class idImage { int GetUploadHeight() const { return opts.height; } textureFilter_t GetFilter() const { return filter; } textureRepeat_t GetRepeat() const { return repeat; } + textureUsage_t GetUsage() const { return usage; } bool IsDefaulted() const { return defaulted; } + bool IsScratchImage() const { return scratchImage; } void SetReferencedOutsideLevelLoad() { referencedOutsideLevelLoad = true; } void SetReferencedInsideLevelLoad() { levelLoadReferenced = true; } @@ -248,6 +255,7 @@ class idImage { bool referencedOutsideLevelLoad; bool levelLoadReferenced; // for determining if it needs to be purged bool defaulted; // true if the default image was generated because a file couldn't be loaded + bool scratchImage; // storage is owned/mutated by a runtime render target or upload path ID_TIME_T sourceFileTime; // the most recent of all images used in creation, for reloadImages command ID_TIME_T binaryFileTime; // the time stamp of the binary file idStr loadedSourceName; // source or automatic DDS replacement used by the last successful load @@ -285,6 +293,7 @@ ID_INLINE idImage::idImage(const char* name) : imgName(name) { referencedOutsideLevelLoad = false; levelLoadReferenced = false; defaulted = false; + scratchImage = false; sourceFileTime = FILE_NOT_FOUND_TIMESTAMP; binaryFileTime = FILE_NOT_FOUND_TIMESTAMP; loadedSourceName.Clear(); @@ -292,6 +301,12 @@ ID_INLINE idImage::idImage(const char* name) : imgName(name) { useCount = 0; } +// Mutable renderer-owned images must never be treated as static PBR material +// resources. The name form catches targets before lazy allocation; the image +// form also catches arbitrary names registered through ScratchImage(). +bool R_IsMutableRenderImageName( const char *name ); +bool R_IsMutableRenderImage( const idImage *image ); + // data is RGBA bool R_WriteTGA(const char* filename, const byte* data, int width, int height, bool flipVertical = false, const char* basePath = "fs_savepath"); diff --git a/src/renderer/ImageManager.cpp b/src/renderer/ImageManager.cpp index 5dfdf5ac..41aa4391 100644 --- a/src/renderer/ImageManager.cpp +++ b/src/renderer/ImageManager.cpp @@ -32,6 +32,32 @@ If you have questions concerning this license or the applicable additional terms #include "tr_local.h" +bool R_IsMutableRenderImageName( const char *name ) { + if ( name == NULL || name[0] == '\0' ) { + return false; + } + static const char *prefixes[] = { + "_cinematic", "_scratch", "_accum", + "_reflectionRender", "_refractionRender", + "_currentRender", "_originalCurrentRender", "_currentDepth", + "_forwardRenderResolved", "_postProcessAlbedo", + "_scenePreserveDepth", "_hdrScene", "_ssao", "_cel", + "_motionVector", "_bloom", "_hdrLum", "_underwaterDepth", + "_shadowMap", "_pointShadowMap", + "_translucentShadowMap", "_pointTranslucentShadowMap" + }; + for ( int i = 0; i < static_cast( sizeof( prefixes ) / sizeof( prefixes[0] ) ); ++i ) { + if ( idStr::Icmpn( name, prefixes[i], static_cast( strlen( prefixes[i] ) ) ) == 0 ) { + return true; + } + } + return !idStr::Icmp( name, "BlurTexture1" ) || !idStr::Icmp( name, "DepthTexture" ); +} + +bool R_IsMutableRenderImage( const idImage *image ) { + return image != NULL && ( image->IsScratchImage() || R_IsMutableRenderImageName( image->GetName() ) ); +} + // do this with a pointer, in case we want to make the actual manager // a private virtual subclass idImageManager imageManager; @@ -289,6 +315,22 @@ static bool R_IsQ4LightImageNamespace( const char *name ) { || idStr::Icmpn( name, "gfx/lights/", 11 ) == 0 ); } +static textureUsage_t R_ImageUsageForName( const char *name, textureUsage_t requestedUsage ) { + // Namespace remaps are a legacy convenience for generic callers only. + // Explicit PBR and high-quality usage classes are cache/storage contracts + // and must survive regardless of the source path chosen by an author. + if ( requestedUsage != TD_DEFAULT ) { + return requestedUsage; + } + if ( idStr::Icmpn( name, "fonts", 5 ) == 0 || idStr::Icmpn( name, "newfonts", 8 ) == 0 ) { + return TD_FONT; + } + if ( R_IsQ4LightImageNamespace( name ) ) { + return TD_LIGHT; + } + return requestedUsage; +} + static bool R_IsQ4PresentationImageNamespace( const char *name ) { return name != NULL && ( idStr::Icmpn( name, "gfx/guis/", 9 ) == 0 @@ -594,14 +636,7 @@ idImage *idImageManager::GetImageWithParameters( const char *_name, textureFilte declManager->MediaPrint( "DEFAULTED\n" ); return globalImages->defaultImage; } - if ( usage == TD_DEFAULT ) { - if ( idStr::Icmpn( _name, "fonts", 5 ) == 0 || idStr::Icmpn( _name, "newfonts", 8 ) == 0 ) { - usage = TD_FONT; - } - if ( R_IsQ4LightImageNamespace( _name ) ) { - usage = TD_LIGHT; - } - } + usage = R_ImageUsageForName( _name, usage ); // strip any .tga file extensions from anywhere in the _name, including image program parameters idStr name = _name; name.Replace( ".tga", "" ); @@ -648,12 +683,7 @@ idImage *idImageManager::ImageFromFile( const char *_name, textureFilter_t filte declManager->MediaPrint( "DEFAULTED\n" ); return globalImages->defaultImage; } - if ( idStr::Icmpn( _name, "fonts", 5 ) == 0 || idStr::Icmpn( _name, "newfonts", 8 ) == 0 ) { - usage = TD_FONT; - } - if ( R_IsQ4LightImageNamespace( _name ) ) { - usage = TD_LIGHT; - } + usage = R_ImageUsageForName( _name, usage ); // strip any .tga file extensions from anywhere in the _name, including image program parameters idStr name = _name; @@ -743,16 +773,7 @@ idImage *idImageManager::ImageHandleDeferred( const char *_name, textureFilter_t declManager->MediaPrint( "DEFAULTED\n" ); return globalImages->defaultImage; } - if ( usage == TD_DEFAULT ) { - // Keep the legacy convenience remap for generic callers, but preserve any - // explicit material-requested usage class such as TD_HIGH_QUALITY. - if ( idStr::Icmpn( _name, "fonts", 5 ) == 0 || idStr::Icmpn( _name, "newfonts", 8 ) == 0 ) { - usage = TD_FONT; - } - if ( R_IsQ4LightImageNamespace( _name ) ) { - usage = TD_LIGHT; - } - } + usage = R_ImageUsageForName( _name, usage ); idStr name = _name; name.Replace( ".tga", "" ); @@ -817,6 +838,7 @@ idImage * idImageManager::ScratchImage( const char *_name, idImageOpts *imgOpts, for ( int i = imageHash.First( hash ); i != -1; i = imageHash.Next( i ) ) { idImage * image = images[i]; if ( name.Icmp( image->GetName() ) == 0 ) { + image->scratchImage = true; image->usage = usage; image->levelLoadReferenced = true; image->referencedOutsideLevelLoad = true; @@ -839,6 +861,7 @@ idImage * idImageManager::ScratchImage( const char *_name, idImageOpts *imgOpts, // idImage* newImage = AllocImage( name ); if ( newImage != NULL ) { + newImage->scratchImage = true; newImage->usage = usage; newImage->levelLoadReferenced = true; newImage->referencedOutsideLevelLoad = true; diff --git a/src/renderer/Image_load.cpp b/src/renderer/Image_load.cpp index 13182506..d16c3f1d 100644 --- a/src/renderer/Image_load.cpp +++ b/src/renderer/Image_load.cpp @@ -58,6 +58,19 @@ static unsigned int R_GetImageDownsizeSignature( const char *name, textureUsage_ static void R_DownsizeLoadedImageData( const char *name, textureUsage_t usage, bool allowDownSize, byte *&pic, int &width, int &height ); static void R_DownsizeLoadedCubeImageData( const char *name, textureUsage_t usage, bool allowDownSize, byte *pics[6], int &size ); +static void R_LoadImageProgramForDeclaredUsage( const char *name, byte **pic, int *width, int *height, + ID_TIME_T *timestamp, textureUsage_t &usage ) { + // Classic image programs infer TD_BUMP for normal-producing operations. + // PBR semantics are explicit authoring contracts and form part of the image + // cache key, so decoding must not mutate them after lookup/name generation. + const textureUsage_t declaredUsage = usage; + textureUsage_t inferredUsage = usage; + R_LoadImageProgram( name, pic, width, height, timestamp, &inferredUsage ); + if ( declaredUsage != TD_PBR_COLOR && declaredUsage != TD_MATERIAL_DATA ) { + usage = inferredUsage; + } +} + /* ======================== idImage::DeriveOpts @@ -117,6 +130,16 @@ ID_INLINE void idImage::DeriveOpts() { opts.colorFormat = CFM_DEFAULT; opts.format = FMT_RGBA8; break; + case TD_PBR_COLOR: + opts.gammaMips = true; + opts.colorFormat = CFM_DEFAULT; + opts.format = FMT_RGBA8; + break; + case TD_MATERIAL_DATA: + opts.gammaMips = false; + opts.colorFormat = CFM_DEFAULT; + opts.format = FMT_RGBA8; + break; default: opts.gammaMips = false; opts.format = FMT_RGBA8; @@ -453,7 +476,7 @@ void idImage::ActuallyLoadImage( bool fromBackEnd ) { R_ResolvePreferredDDSImageSource( GetName(), preferredDDSName, &preferredDDSFileTime, true, &preferredDDSPrecompressed ); if ( preferredDDSImage && !fileSystem->InProductionMode() ) { ID_TIME_T originalSourceTime = FILE_NOT_FOUND_TIMESTAMP; - R_LoadImageProgram( GetName(), NULL, NULL, NULL, &originalSourceTime, &usage ); + R_LoadImageProgramForDeclaredUsage( GetName(), NULL, NULL, NULL, &originalSourceTime, usage ); if ( R_IsPreferredDDSStale( preferredDDSName, preferredDDSFileTime, originalSourceTime ) ) { if ( cvarSystem->GetCVarBool( "image_showPrecompressedTextures" ) ) { common->Printf( "Ignoring stale DDS replacement %s for %s\n", preferredDDSName.c_str(), GetName() ); @@ -541,7 +564,7 @@ void idImage::ActuallyLoadImage( bool fromBackEnd ) { } else if ( preferredDDSImage ) { sourceFileTime = preferredDDSFileTime; } else { - R_LoadImageProgram( GetName(), NULL, NULL, NULL, &sourceFileTime, &usage ); + R_LoadImageProgramForDeclaredUsage( GetName(), NULL, NULL, NULL, &sourceFileTime, usage ); } sourceFileTimeKnown = true; } @@ -653,12 +676,12 @@ void idImage::ActuallyLoadImage( bool fromBackEnd ) { } // load the full specification, and perform any image program calculations - R_LoadImageProgram( fallbackLoadSourceName, &pic, &width, &height, &sourceFileTime, &usage ); + R_LoadImageProgramForDeclaredUsage( fallbackLoadSourceName, &pic, &width, &height, &sourceFileTime, usage ); if ( pic == NULL && preferredDDSImage && !preferredDDSPrecompressed ) { common->Warning( "Couldn't decode preferred DDS replacement %s for %s; falling back to original source", loadSourceName, GetName() ); selectedSourceName = GetName(); sourceFileTime = FILE_NOT_FOUND_TIMESTAMP; - R_LoadImageProgram( GetName(), &pic, &width, &height, &sourceFileTime, &usage ); + R_LoadImageProgramForDeclaredUsage( GetName(), &pic, &width, &height, &sourceFileTime, usage ); } sourceFileTimeKnown = true; @@ -857,7 +880,7 @@ been; anything else falls back to a general resample. */ static bool R_ImageUsageUsesGammaMips( textureUsage_t usage ) { // mirrors the gammaMips choices DeriveOpts makes for each usage - return usage == TD_FONT || usage == TD_LIGHT; + return usage == TD_FONT || usage == TD_LIGHT || usage == TD_PBR_COLOR; } static int R_CountExactHalvings( int width, int height, int scaledWidth, int scaledHeight ) { diff --git a/src/renderer/Material.cpp b/src/renderer/Material.cpp index 1e409367..721332a9 100644 --- a/src/renderer/Material.cpp +++ b/src/renderer/Material.cpp @@ -310,6 +310,17 @@ void idMaterial::CommonInit() { globalUseCount = 0; portalImage = nullptr; // jmarshall end + memset( &pbrInfo, 0, sizeof( pbrInfo ) ); + pbrInfo.workflow = PBR_WORKFLOW_NONE; + pbrInfo.normalFormat = PBR_NORMAL_UNSPECIFIED; + pbrInfo.metallicRegister = -1; + pbrInfo.roughnessRegister = -1; + pbrInfo.aoRegister = -1; + pbrInfo.normalScaleRegister = -1; + pbrInfo.emissiveColorRegisters[0] = -1; + pbrInfo.emissiveColorRegisters[1] = -1; + pbrInfo.emissiveColorRegisters[2] = -1; + pbrInfo.autoLegacyFallback = true; decalInfo.stayTime = 10000; decalInfo.maxAngle = 0.1f; @@ -424,6 +435,19 @@ void idMaterial::FreeData() { materialTypeArrayName.Clear(); MTAWidth = 0; MTAHeight = 0; + // Purged declarations remain queryable before their next parse. Do not + // leave stale PBR image pointers or register indices visible in that state. + memset( &pbrInfo, 0, sizeof( pbrInfo ) ); + pbrInfo.workflow = PBR_WORKFLOW_NONE; + pbrInfo.normalFormat = PBR_NORMAL_UNSPECIFIED; + pbrInfo.metallicRegister = -1; + pbrInfo.roughnessRegister = -1; + pbrInfo.aoRegister = -1; + pbrInfo.normalScaleRegister = -1; + pbrInfo.emissiveColorRegisters[0] = -1; + pbrInfo.emissiveColorRegisters[1] = -1; + pbrInfo.emissiveColorRegisters[2] = -1; + pbrInfo.autoLegacyFallback = true; } /* @@ -450,6 +474,8 @@ idImage *idMaterial::GetEditorImage( void ) const { if ( !editorImage ) { editorImage = stages[0].texture.image; } + } else if ( pbrInfo.enabled && pbrInfo.albedo.image != NULL ) { + editorImage = pbrInfo.albedo.image; } else { editorImage = globalImages->defaultImage; } @@ -2483,6 +2509,516 @@ void idMaterial::ParseStage( idLexer &src, const textureRepeat_t trpDefault ) { } } +static bool R_IsUnsupportedPBRImageProgramToken( const idToken &token ) { + return R_IsMutableRenderImageName( token.c_str() ) + || !token.Icmp( "videoMap" ) || !token.Icmp( "soundMap" ) + || !token.Icmp( "mirrorRenderMap" ) || !token.Icmp( "remoteRenderMap" ) + || !token.Icmp( "reflectionRenderMap" ) || !token.Icmp( "refractionRenderMap" ) + || !token.Icmp( "xrayRenderMap" ) + || !token.Icmp( "cameraCubeMap" ) || !token.Icmp( "cubeMap" ) + || !token.Icmp( "program" ) || !token.Icmp( "vertexProgram" ) + || !token.Icmp( "fragmentProgram" ) || !token.Icmp( "fp20Program" ) + || !token.Icmp( "glslProgram" ) || !token.Icmp( "vertexParm" ) + || !token.Icmp( "fragmentParm" ) || !token.Icmp( "fragmentMap" ) + || !token.Icmp( "shaderParm" ) || !token.Icmp( "shaderTexture" ) + || !token.Icmp( "customLighting" ) + // Stage-only state is not an image name. Reject it here as well as + // when nested inside an otherwise valid image program. Keep image + // program operators such as add() and scale() available. + || !token.Icmp( "blend" ) || !token.Icmp( "map" ) + || !token.Icmp( "screen" ) || !token.Icmp( "screen2" ) || !token.Icmp( "glassWarp" ) + || !token.Icmp( "texGen" ) || !token.Icmp( "if" ) + || !token.Icmp( "alphaTest" ) || !token.Icmp( "alphaFunc" ) + || !token.Icmp( "scroll" ) || !token.Icmp( "translate" ) + || !token.Icmp( "centerScale" ) || !token.Icmp( "shear" ) || !token.Icmp( "rotate" ) + || !token.Icmp( "vertexColor" ) || !token.Icmp( "inverseVertexColor" ) + || !token.Icmp( "color" ) || !token.Icmp( "colored" ) + || !token.Icmp( "red" ) || !token.Icmp( "green" ) || !token.Icmp( "blue" ) + || !token.Icmp( "alpha" ) || !token.Icmp( "rgb" ) || !token.Icmp( "rgba" ) + || !token.Icmp( "maskRed" ) || !token.Icmp( "maskGreen" ) + || !token.Icmp( "maskBlue" ) || !token.Icmp( "maskAlpha" ) + || !token.Icmp( "maskColor" ) || !token.Icmp( "maskDepth" ) + || !token.Icmp( "privatePolygonOffset" ) || !token.Icmp( "polygonOffset" ) + || !token.Icmp( "ignoreAlphaTest" ); +} + +/* +================ +idMaterial::ParsePBRImage + +Parses one static image-program reference from a PBR metadata line. Dynamic +render/video maps and arbitrary shader state deliberately remain classic-stage +features until a modern pass owns their complete lifetime and fallback rules. +================ +*/ +bool idMaterial::ParsePBRImage( idLexer &src, pbrMaterialTexture_t &target, const int usage, const textureRepeat_t trpDefault ) { + if ( target.present ) { + src.Warning( "duplicate PBR image semantic in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + + textureFilter_t filter = TF_DEFAULT; + textureRepeat_t repeat = trpDefault; + bool allowPicmip = true; + bool noMips = false; + bool highQuality = false; + bool forceHighQuality = false; + unsigned int imageFlags = 0; + idToken token; + bool haveImageToken = false; + + while ( src.ReadTokenOnLine( &token ) ) { + if ( !token.Icmp( "nearest" ) ) { + filter = TF_NEAREST; + continue; + } + if ( !token.Icmp( "linear" ) ) { + filter = TF_LINEAR; + continue; + } + if ( !token.Icmp( "clamp" ) ) { + repeat = TR_CLAMP; + continue; + } + if ( !token.Icmp( "noclamp" ) ) { + repeat = TR_REPEAT; + continue; + } + if ( !token.Icmp( "zeroclamp" ) ) { + repeat = TR_CLAMP_TO_ZERO; + continue; + } + if ( !token.Icmp( "alphazeroclamp" ) ) { + repeat = TR_CLAMP_TO_ZERO_ALPHA; + continue; + } + if ( !token.Icmp( "mirroredrepeat" ) ) { + repeat = TR_MIRRORED_REPEAT; + continue; + } + if ( !token.Icmp( "nopicmip" ) ) { + allowPicmip = false; + continue; + } + if ( !token.Icmp( "nomips" ) ) { + noMips = true; + imageFlags = R_ApplyMaterialNoMipFlags( imageFlags ); + continue; + } + if ( !token.Icmp( "forceHighQuality" ) ) { + highQuality = true; + forceHighQuality = true; + continue; + } + if ( !token.Icmp( "uncompressed" ) || !token.Icmp( "highquality" ) ) { + // TD_PBR_COLOR and TD_MATERIAL_DATA are already lossless RGBA8 + // identities. Retain the authoring hint for any generated classic + // fallback without discarding the PBR colour-vs-data mip semantics. + highQuality = true; + continue; + } + + if ( R_IsUnsupportedPBRImageProgramToken( token ) ) { + src.Warning( "dynamic or cube image token '%s' is not supported in the PBR block for '%s'", token.c_str(), GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + + src.UnreadToken( &token ); + haveImageToken = true; + break; + } + + if ( !haveImageToken ) { + src.Warning( "PBR image semantic expects an image program in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + + const char *parsedName = R_ParsePastImageProgram( src ); + if ( parsedName == NULL || parsedName[0] == '\0' ) { + src.Warning( "PBR image semantic has an empty image program in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + + // R_ParsePastImageProgram accepts arbitrary leaf names, so validating only + // the first authored token would let a dynamic/render/program token hide + // inside add(), scale(), or another nested image operation. Re-lex the + // canonical expression and reject forbidden tokens at every nesting depth. + idLexer imageProgram; + imageProgram.LoadMemory( parsedName, idLib::SizeToInt( strlen( parsedName ), "ParsePBRImage validation" ), "pbrImageProgram" ); + imageProgram.SetFlags( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_NOSTRINGESCAPECHARS | LEXFL_ALLOWPATHNAMES ); + idToken imageProgramToken; + while ( imageProgram.ReadToken( &imageProgramToken ) ) { + if ( R_IsUnsupportedPBRImageProgramToken( imageProgramToken ) ) { + src.Warning( "dynamic or cube image token '%s' is not supported in the PBR block for '%s'", imageProgramToken.c_str(), GetName() ); + imageProgram.FreeSource(); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + } + imageProgram.FreeSource(); + + idStr imageName = parsedName; + target.image = R_LoadMaterialImage( imageName.c_str(), filter, repeat, + static_cast( usage ), CF_2D, allowPicmip, imageFlags ); + if ( target.image == NULL ) { + target.image = globalImages->defaultImage; + } + if ( R_IsMutableRenderImage( target.image ) ) { + src.Warning( "mutable render image '%s' is not supported in the PBR block for '%s'", imageName.c_str(), GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + target.filter = static_cast( filter ); + target.repeat = static_cast( repeat ); + target.allowPicmip = allowPicmip; + target.noMips = noMips; + target.highQuality = highQuality; + target.forceHighQuality = forceHighQuality; + target.present = true; + return true; +} + +/* +================ +idMaterial::ParsePBRBlock +================ +*/ +bool idMaterial::ParsePBRBlock( idLexer &src, const textureRepeat_t trpDefault ) { + if ( pbrInfo.enabled ) { + src.Warning( "multiple PBR blocks in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + if ( !src.ExpectTokenString( "{" ) ) { + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + + pbrInfo.enabled = true; + // Allocate PBR-only defaults only after an authored PBR block opts in. Doing + // this in ParseMaterial's common prologue would shift expression-register + // indices for every retail material even though no PBR metadata is present. + pbrInfo.metallicRegister = GetExpressionConstant( 0.0f ); + pbrInfo.roughnessRegister = GetExpressionConstant( 0.5f ); + pbrInfo.aoRegister = GetExpressionConstant( 1.0f ); + pbrInfo.normalScaleRegister = GetExpressionConstant( 1.0f ); + pbrInfo.emissiveColorRegisters[0] = GetExpressionConstant( 0.0f ); + pbrInfo.emissiveColorRegisters[1] = GetExpressionConstant( 0.0f ); + pbrInfo.emissiveColorRegisters[2] = GetExpressionConstant( 0.0f ); + idToken token; + while ( src.ReadToken( &token ) ) { + if ( token == "}" ) { + break; + } + + if ( !token.Icmp( "workflow" ) ) { + idToken value; + if ( !src.ReadTokenOnLine( &value ) ) { + src.Warning( "PBR workflow expects a value in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + if ( !value.Icmp( "metallicRoughness" ) ) { + pbrInfo.workflow = PBR_WORKFLOW_METALLIC_ROUGHNESS; + } else if ( !value.Icmp( "specularGlossiness" ) ) { + pbrInfo.workflow = PBR_WORKFLOW_SPECULAR_GLOSSINESS; + } else { + src.Warning( "unknown PBR workflow '%s' in material '%s'", value.c_str(), GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + continue; + } + + if ( !token.Icmp( "normalFormat" ) ) { + idToken value; + if ( !src.ReadTokenOnLine( &value ) ) { + src.Warning( "PBR normalFormat expects a value in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + if ( !value.Icmp( "quake4AGB" ) ) { + pbrInfo.normalFormat = PBR_NORMAL_QUAKE4_AGB; + } else if ( !value.Icmp( "tangentRG" ) ) { + pbrInfo.normalFormat = PBR_NORMAL_TANGENT_RG; + } else if ( !value.Icmp( "tangentXYZ" ) ) { + pbrInfo.normalFormat = PBR_NORMAL_TANGENT_XYZ; + } else { + src.Warning( "unknown PBR normalFormat '%s' in material '%s'", value.c_str(), GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + continue; + } + + if ( !token.Icmp( "albedoMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.albedo, TD_PBR_COLOR, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "normalMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.normal, TD_BUMP, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "ormMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.orm, TD_MATERIAL_DATA, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "metallicMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.metallic, TD_MATERIAL_DATA, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "roughnessMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.roughness, TD_MATERIAL_DATA, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "aoMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.ao, TD_MATERIAL_DATA, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "emissiveMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.emissive, TD_PBR_COLOR, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "legacyBumpMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.legacyBump, TD_BUMP, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "legacyDiffuseMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.legacyDiffuse, TD_DIFFUSE, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "legacySpecularMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.legacySpecular, TD_SPECULAR, trpDefault ) ) return false; + continue; + } + if ( !token.Icmp( "legacyEmissiveMap" ) ) { + if ( !ParsePBRImage( src, pbrInfo.legacyEmissive, TD_PBR_COLOR, trpDefault ) ) return false; + continue; + } + + if ( !token.Icmp( "metallic" ) ) { + pbrInfo.metallicRegister = ParseExpression( src ); + continue; + } + if ( !token.Icmp( "roughness" ) ) { + pbrInfo.roughnessRegister = ParseExpression( src ); + continue; + } + if ( !token.Icmp( "ao" ) ) { + pbrInfo.aoRegister = ParseExpression( src ); + continue; + } + if ( !token.Icmp( "normalScale" ) ) { + pbrInfo.normalScaleRegister = ParseExpression( src ); + continue; + } + if ( !token.Icmp( "emissiveColor" ) ) { + for ( int i = 0; i < 3; ++i ) { + pbrInfo.emissiveColorRegisters[i] = ParseExpression( src ); + if ( i < 2 ) { + idToken separator; + if ( src.ReadToken( &separator ) && separator != "," ) { + src.UnreadToken( &separator ); + } + } + } + continue; + } + if ( !token.Icmp( "autoLegacyFallback" ) ) { + idToken value; + if ( !src.ReadTokenOnLine( &value ) || ( value != "0" && value != "1" ) ) { + src.Warning( "PBR autoLegacyFallback expects 0 or 1 in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + pbrInfo.autoLegacyFallback = value == "1"; + continue; + } + + src.Warning( "unknown PBR parameter '%s' in material '%s'", token.c_str(), GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + + if ( token != "}" ) { + src.Warning( "unterminated PBR block in material '%s'", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + if ( pbrInfo.workflow == PBR_WORKFLOW_NONE ) { + src.Warning( "PBR material '%s' has no workflow", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + if ( !pbrInfo.albedo.present ) { + src.Warning( "PBR material '%s' has no albedoMap", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + if ( pbrInfo.normal.present && pbrInfo.normalFormat == PBR_NORMAL_UNSPECIFIED ) { + src.Warning( "PBR normalMap in '%s' requires normalFormat", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + if ( pbrInfo.orm.present && ( pbrInfo.metallic.present || pbrInfo.roughness.present || pbrInfo.ao.present ) ) { + src.Warning( "PBR material '%s' cannot combine ormMap with separate metallic/roughness/AO maps", GetName() ); + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + + pbrInfo.hasExplicitLegacyFallback = pbrInfo.legacyBump.present || + pbrInfo.legacyDiffuse.present || pbrInfo.legacySpecular.present || + pbrInfo.legacyEmissive.present; + return true; +} + +/* +================ +idMaterial::AddPBRLegacyFallbackStages +================ +*/ +void idMaterial::AddPBRLegacyFallbackStages( const textureRepeat_t trpDefault ) { + if ( !pbrInfo.enabled ) { + return; + } + + bool hasBump = false; + bool hasDiffuse = false; + bool hasSpecular = false; + bool hasAmbient = false; + for ( int i = 0; i < numStages; ++i ) { + hasBump |= pd->parseStages[i].lighting == SL_BUMP; + hasDiffuse |= pd->parseStages[i].lighting == SL_DIFFUSE; + hasSpecular |= pd->parseStages[i].lighting == SL_SPECULAR; + hasAmbient |= pd->parseStages[i].lighting == SL_AMBIENT; + } + pbrInfo.hasAuthoredClassicFallback = hasBump && hasDiffuse; + + auto addStage = [&]( const char *blendName, const pbrMaterialTexture_t &texture ) -> bool { + if ( texture.image == NULL || numStages >= MAX_SHADER_STAGES ) { + SetMaterialFlag( MF_DEFAULTED ); + return false; + } + idStr buffer; + buffer = "blend "; + buffer.Append( blendName ); + buffer.Append( "\n" ); + switch ( static_cast( texture.filter ) ) { + case TF_NEAREST: buffer.Append( "nearest\n" ); break; + case TF_LINEAR: buffer.Append( "linear\n" ); break; + default: break; + } + const textureRepeat_t repeat = static_cast( texture.repeat ); + if ( repeat != trpDefault ) { + switch ( repeat ) { + case TR_REPEAT: buffer.Append( "noclamp\n" ); break; + case TR_MIRRORED_REPEAT: buffer.Append( "mirroredrepeat\n" ); break; + case TR_CLAMP: buffer.Append( "clamp\n" ); break; + case TR_CLAMP_TO_ZERO: buffer.Append( "zeroclamp\n" ); break; + case TR_CLAMP_TO_ZERO_ALPHA: buffer.Append( "alphazeroclamp\n" ); break; + default: break; + } + } + if ( !texture.allowPicmip ) buffer.Append( "nopicmip\n" ); + if ( texture.noMips ) buffer.Append( "nomips\n" ); + if ( texture.forceHighQuality ) { + buffer.Append( "forceHighQuality\n" ); + } else if ( texture.highQuality ) { + buffer.Append( "highquality\n" ); + } + buffer.Append( "map " ); + buffer.Append( texture.image->GetName() ); + buffer.Append( "\n}\n" ); + idLexer generated; + generated.LoadMemory( buffer.c_str(), buffer.Length(), "pbrLegacyFallback" ); + generated.SetFlags( LEXFL_NOFATALERRORS | LEXFL_NOSTRINGCONCAT | LEXFL_NOSTRINGESCAPECHARS | LEXFL_ALLOWPATHNAMES ); + ParseStage( generated, trpDefault ); + generated.FreeSource(); + return !TestMaterialFlag( MF_DEFAULTED ); + }; + auto defaultTexture = [&]( idImage *image ) -> pbrMaterialTexture_t { + pbrMaterialTexture_t texture; + memset( &texture, 0, sizeof( texture ) ); + texture.image = image; + texture.present = image != NULL; + texture.filter = static_cast( TF_DEFAULT ); + texture.repeat = static_cast( trpDefault ); + texture.allowPicmip = true; + return texture; + }; + + if ( !hasBump && pbrInfo.legacyBump.present ) { + if ( !addStage( "bumpmap", pbrInfo.legacyBump ) ) return; + hasBump = true; + pbrInfo.usesGeneratedLegacyFallback = true; + } + if ( !hasDiffuse && pbrInfo.legacyDiffuse.present ) { + if ( !addStage( "diffusemap", pbrInfo.legacyDiffuse ) ) return; + hasDiffuse = true; + pbrInfo.usesGeneratedLegacyFallback = true; + } + if ( !hasSpecular && pbrInfo.legacySpecular.present ) { + if ( !addStage( "specularmap", pbrInfo.legacySpecular ) ) return; + hasSpecular = true; + pbrInfo.usesGeneratedLegacyFallback = true; + } + if ( !hasAmbient && pbrInfo.legacyEmissive.present ) { + if ( !addStage( "add", pbrInfo.legacyEmissive ) ) return; + hasAmbient = true; + pbrInfo.usesGeneratedLegacyFallback = true; + } + + // An authored bump+diffuse interaction is already a complete classic + // fallback. Classic specular is optional, and a PBR emissive map must not + // silently inject an ambient stage into that authored path. + const bool hasUsableClassicInteraction = hasBump && hasDiffuse; + const bool allowApproximate = !hasUsableClassicInteraction + && pbrInfo.autoLegacyFallback + && r_pbrGeneratedLegacyFallback.GetBool(); + if ( allowApproximate ) { + if ( !hasBump ) { + // The classic Quake 4 interaction decoder understands only the A/G + // normal convention. Tangent RG/XYZ sources stay available to the + // future PBR shader, but their classic fallback must remain neutral. + const bool classicNormalCompatible = pbrInfo.normal.present + && pbrInfo.normalFormat == PBR_NORMAL_QUAKE4_AGB; + const pbrMaterialTexture_t texture = classicNormalCompatible + ? pbrInfo.normal + : defaultTexture( globalImages->flatNormalMap ); + if ( !addStage( "bumpmap", texture ) ) return; + hasBump = true; + pbrInfo.usesGeneratedLegacyFallback = true; + pbrInfo.usesApproximateLegacyFallback = true; + } + if ( !hasDiffuse ) { + const pbrMaterialTexture_t texture = pbrInfo.albedo.present ? pbrInfo.albedo : defaultTexture( globalImages->whiteImage ); + if ( !addStage( "diffusemap", texture ) ) return; + hasDiffuse = true; + pbrInfo.usesGeneratedLegacyFallback = true; + pbrInfo.usesApproximateLegacyFallback = true; + } + if ( !hasSpecular ) { + if ( !addStage( "specularmap", defaultTexture( globalImages->blackImage ) ) ) return; + hasSpecular = true; + pbrInfo.usesGeneratedLegacyFallback = true; + pbrInfo.usesApproximateLegacyFallback = true; + } + if ( !hasAmbient && pbrInfo.emissive.present ) { + if ( !addStage( "add", pbrInfo.emissive ) ) return; + pbrInfo.usesGeneratedLegacyFallback = true; + pbrInfo.usesApproximateLegacyFallback = true; + } + } + + if ( pbrInfo.usesApproximateLegacyFallback ) { + common->Warning( "PBR material '%s' uses an approximate generated classic fallback", GetName() ); + } +} + /* =============== idMaterial::ParseDeform @@ -2689,7 +3225,6 @@ void idMaterial::ParseMaterial( idLexer &src ) { for ( i = 0 ; i < numRegisters ; i++ ) { pd->registerIsTemporary[i] = true; // they aren't constants that can be folded } - numStages = 0; textureRepeat_t trpDefault = TR_REPEAT; // allow a global setting for repeat @@ -2977,6 +3512,12 @@ void idMaterial::ParseMaterial( idLexer &src ) { } continue; } + else if ( !token.Icmp( "pbr" ) || !token.Icmp( "physicallyBased" ) ) { + if ( !ParsePBRBlock( src, trpDefault ) ) { + return; + } + continue; + } // diffusemap for stage shortcut else if ( !token.Icmp( "diffusemap" ) ) { str = R_ParsePastImageProgram( src ); @@ -3050,9 +3591,34 @@ void idMaterial::ParseMaterial( idLexer &src ) { } } + // PBR metadata never mutates an authored classic stage. For a PBR-only + // declaration, explicitly requested or development-only generated stages + // are added before the ordinary classic interaction completion/sort pass. + AddPBRLegacyFallbackStages( trpDefault ); + if ( TestMaterialFlag( MF_DEFAULTED ) ) { + return; + } + // add _flat or _white stages if needed AddImplicitStages(); + // A PBR declaration is still valid metadata when it deliberately omits a + // classic fallback, but the current renderer cannot draw it until a native + // PBR lighting path exists. Preserve that state explicitly instead of + // letting a zero-stage declaration disappear from authoring diagnostics. + if ( pbrInfo.enabled ) { + bool hasBump = false; + bool hasDiffuse = false; + for ( int i = 0; i < numStages; ++i ) { + hasBump |= pd->parseStages[i].lighting == SL_BUMP; + hasDiffuse |= pd->parseStages[i].lighting == SL_DIFFUSE; + } + pbrInfo.legacyFallbackMissing = !( hasBump && hasDiffuse ); + if ( pbrInfo.legacyFallbackMissing ) { + common->Warning( "PBR material '%s' has no usable classic bump+diffuse fallback", GetName() ); + } + } + // order the diffuse / bump / specular stages properly SortInteractionStages(); @@ -3401,6 +3967,18 @@ void idMaterial::Print() const { common->Printf( "%i = %i %s %i\n", op->c, op->a, opNames[ op->opType ], op->b ); } } + if ( pbrInfo.enabled ) { + common->Printf( + "PBR: workflow=%d normalFormat=%d albedo=%s normal=%s orm=%s generatedFallback=%d approximateFallback=%d missingFallback=%d\n", + static_cast( pbrInfo.workflow ), + static_cast( pbrInfo.normalFormat ), + pbrInfo.albedo.image != NULL ? pbrInfo.albedo.image->GetName() : "", + pbrInfo.normal.image != NULL ? pbrInfo.normal.image->GetName() : "", + pbrInfo.orm.image != NULL ? pbrInfo.orm.image->GetName() : "", + pbrInfo.usesGeneratedLegacyFallback ? 1 : 0, + pbrInfo.usesApproximateLegacyFallback ? 1 : 0, + pbrInfo.legacyFallbackMissing ? 1 : 0 ); + } } /* @@ -3431,6 +4009,18 @@ void idMaterial::AddReference() { if ( portalImage ) { portalImage->AddReference(); } + + pbrMaterialTexture_t *pbrTextures[] = { + &pbrInfo.albedo, &pbrInfo.normal, &pbrInfo.orm, &pbrInfo.metallic, + &pbrInfo.roughness, &pbrInfo.ao, &pbrInfo.emissive, + &pbrInfo.legacyBump, &pbrInfo.legacyDiffuse, + &pbrInfo.legacySpecular, &pbrInfo.legacyEmissive + }; + for ( unsigned int i = 0; i < sizeof( pbrTextures ) / sizeof( pbrTextures[0] ); ++i ) { + if ( pbrTextures[i]->present && pbrTextures[i]->image != NULL ) { + pbrTextures[i]->image->AddReference(); + } + } } /* @@ -3466,6 +4056,18 @@ void idMaterial::ResolveUse() { if ( portalImage != NULL ) { portalImage->AddUseCount( useCount ); } + + pbrMaterialTexture_t *pbrTextures[] = { + &pbrInfo.albedo, &pbrInfo.normal, &pbrInfo.orm, &pbrInfo.metallic, + &pbrInfo.roughness, &pbrInfo.ao, &pbrInfo.emissive, + &pbrInfo.legacyBump, &pbrInfo.legacyDiffuse, + &pbrInfo.legacySpecular, &pbrInfo.legacyEmissive + }; + for ( unsigned int i = 0; i < sizeof( pbrTextures ) / sizeof( pbrTextures[0] ); ++i ) { + if ( pbrTextures[i]->present && pbrTextures[i]->image != NULL ) { + pbrTextures[i]->image->AddUseCount( useCount ); + } + } } /* @@ -3847,6 +4449,9 @@ idMaterial::ImageName */ const char *idMaterial::ImageName( void ) const { if ( numStages == 0 ) { + if ( pbrInfo.enabled && pbrInfo.albedo.image != NULL ) { + return pbrInfo.albedo.image->GetName(); + } return "_scratch"; } idImage *image = stages[0].texture.image; @@ -4083,6 +4688,382 @@ bool R_MaterialCustomGLSLReceiverHelperSelfTest( void ) { && !MaterialStagesHaveActiveStockLightingInteractions( inactiveStockStages, 3, shaderRegisters ); } +/* +=================== +R_PBRMaterialParserSelfTest + +Runtime parser test using only intrinsic images. It proves metadata parsing, +classic-stage non-interference, explicit fallback generation, and the two +important authoring failures without requiring repository content. +=================== +*/ +bool R_PBRMaterialParserSelfTest( void ) { + static const char dualAuthored[] = + "material _pbr_selftest_dual {\n" + " bumpmap _flat\n" + " diffusemap _white\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap heightmap( _white, 1 )\n" + " normalMap _flat\n" + " normalFormat tangentRG\n" + " ormMap smoothnormals( _flat )\n" + " emissiveMap _white\n" + " metallic 0.25\n" + " roughness 0.6\n" + " ao 0.9\n" + " normalScale 0.75\n" + " emissiveColor 0.1 0.2 0.3\n" + " }\n" + "}\n"; + + idDecl *dualDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( dualDecl == NULL ) { + return false; + } + idMaterial *dual = static_cast( dualDecl ); + bool ok = dual->Parse( dualAuthored, idLib::SizeToInt( sizeof( dualAuthored ) - 1, "R_PBRMaterialParserSelfTest dual" ) ); + if ( !ok ) { + common->Printf( "RendererPBRMaterial parser self-test: dual declaration did not parse\n" ); + } + if ( ok ) { + const pbrMaterialInfo_t &info = dual->GetPBRInfo(); + const int registerCount = dual->GetNumRegisters(); + float evaluatedRegisters[MAX_EXPRESSION_REGISTERS]; + float entityParms[MAX_ENTITY_SHADER_PARMS]; + viewDef_t evaluationView; + memset( evaluatedRegisters, 0, sizeof( evaluatedRegisters ) ); + memset( entityParms, 0, sizeof( entityParms ) ); + memset( &evaluationView, 0, sizeof( evaluationView ) ); + dual->EvaluateRegisters( evaluatedRegisters, entityParms, &evaluationView ); + auto registerMatches = [&]( int index, float expected ) -> bool { + return index >= 0 && index < registerCount + && idMath::Fabs( evaluatedRegisters[index] - expected ) < 0.0001f; + }; + const idImage *sameAlbedo = info.albedo.image != NULL ? globalImages->GetImageWithParameters( + info.albedo.image->GetName(), + static_cast( info.albedo.filter ), + static_cast( info.albedo.repeat ), + TD_PBR_COLOR, CF_2D, info.albedo.allowPicmip, 0 ) : NULL; + const idImage *sameORM = info.orm.image != NULL ? globalImages->GetImageWithParameters( + info.orm.image->GetName(), + static_cast( info.orm.filter ), + static_cast( info.orm.repeat ), + TD_MATERIAL_DATA, CF_2D, info.orm.allowPicmip, 0 ) : NULL; + ok = dual->HasPBR() + && info.workflow == PBR_WORKFLOW_METALLIC_ROUGHNESS + && info.normalFormat == PBR_NORMAL_TANGENT_RG + && info.albedo.present && info.normal.present && info.orm.present + && info.albedo.image->GetUsage() == TD_PBR_COLOR + && info.orm.image->GetUsage() == TD_MATERIAL_DATA + && sameAlbedo == info.albedo.image + && sameORM == info.orm.image + && registerMatches( info.metallicRegister, 0.25f ) + && registerMatches( info.roughnessRegister, 0.6f ) + && registerMatches( info.aoRegister, 0.9f ) + && registerMatches( info.normalScaleRegister, 0.75f ) + && registerMatches( info.emissiveColorRegisters[0], 0.1f ) + && registerMatches( info.emissiveColorRegisters[1], 0.2f ) + && registerMatches( info.emissiveColorRegisters[2], 0.3f ) + && info.hasAuthoredClassicFallback + && !info.usesGeneratedLegacyFallback + && dual->GetNumStages() == 2; + if ( !ok ) { + common->Printf( + "RendererPBRMaterial parser self-test: dual contract failed has=%d workflow=%d normal=%d maps=%d/%d/%d usage=%d/%d cache=%d/%d stages=%d authored=%d generated=%d registers=%d/%d/%d/%d/%d/%d/%d\n", + dual->HasPBR() ? 1 : 0, + static_cast( info.workflow ), + static_cast( info.normalFormat ), + info.albedo.present ? 1 : 0, + info.normal.present ? 1 : 0, + info.orm.present ? 1 : 0, + info.albedo.image != NULL ? static_cast( info.albedo.image->GetUsage() ) : -1, + info.orm.image != NULL ? static_cast( info.orm.image->GetUsage() ) : -1, + sameAlbedo == info.albedo.image ? 1 : 0, + sameORM == info.orm.image ? 1 : 0, + dual->GetNumStages(), + info.hasAuthoredClassicFallback ? 1 : 0, + info.usesGeneratedLegacyFallback ? 1 : 0, + registerMatches( info.metallicRegister, 0.25f ) ? 1 : 0, + registerMatches( info.roughnessRegister, 0.6f ) ? 1 : 0, + registerMatches( info.aoRegister, 0.9f ) ? 1 : 0, + registerMatches( info.normalScaleRegister, 0.75f ) ? 1 : 0, + registerMatches( info.emissiveColorRegisters[0], 0.1f ) ? 1 : 0, + registerMatches( info.emissiveColorRegisters[1], 0.2f ) ? 1 : 0, + registerMatches( info.emissiveColorRegisters[2], 0.3f ) ? 1 : 0 ); + } + } + DeclManager_FreeAllocatedDecl( dualDecl ); + if ( !ok ) { + return false; + } + + static const char explicitFallback[] = + "material _pbr_selftest_explicit {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " legacyBumpMap _flat\n" + " legacyDiffuseMap nearest clamp nopicmip nomips forceHighQuality makeIntensity( _white )\n" + " autoLegacyFallback 0\n" + " }\n" + "}\n"; + idDecl *explicitDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( explicitDecl == NULL ) { + return false; + } + idMaterial *explicitMaterial = static_cast( explicitDecl ); + const bool oldMakingBuild = com_makingBuild.GetBool(); + com_makingBuild.SetBool( true ); + ok = explicitMaterial->Parse( explicitFallback, idLib::SizeToInt( sizeof( explicitFallback ) - 1, "R_PBRMaterialParserSelfTest explicit" ) ); + com_makingBuild.SetBool( oldMakingBuild ); + if ( ok ) { + const pbrMaterialInfo_t &info = explicitMaterial->GetPBRInfo(); + const shaderStage_t *diffuseStage = NULL; + for ( int i = 0; i < explicitMaterial->GetNumStages(); ++i ) { + const shaderStage_t *stage = explicitMaterial->GetStage( i ); + if ( stage != NULL && stage->lighting == SL_DIFFUSE ) { + diffuseStage = stage; + break; + } + } + const idImage *expectedDiffuse = globalImages->GetImageWithParameters( + info.legacyDiffuse.image != NULL ? info.legacyDiffuse.image->GetName() : "", + TF_NEAREST, + TR_CLAMP, + TD_HIGH_QUALITY, + CF_2D, + false, + IMAGEFLAG_NOMIPS ); + ok = info.hasExplicitLegacyFallback + && info.usesGeneratedLegacyFallback + && !info.usesApproximateLegacyFallback + && info.legacyDiffuse.filter == static_cast( TF_NEAREST ) + && info.legacyDiffuse.repeat == static_cast( TR_CLAMP ) + && !info.legacyDiffuse.allowPicmip + && info.legacyDiffuse.noMips + && info.legacyDiffuse.highQuality + && info.legacyDiffuse.forceHighQuality + && explicitMaterial->GetNumStages() == 2 + && diffuseStage != NULL + && diffuseStage->texture.image == expectedDiffuse; + } + DeclManager_FreeAllocatedDecl( explicitDecl ); + if ( !ok ) { + return false; + } + + static const char missingFallback[] = + "material _pbr_selftest_missing_fallback {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " autoLegacyFallback 0\n" + " }\n" + "}\n"; + idDecl *missingDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( missingDecl == NULL ) { + return false; + } + idMaterial *missingMaterial = static_cast( missingDecl ); + ok = missingMaterial->Parse( missingFallback, idLib::SizeToInt( sizeof( missingFallback ) - 1, "R_PBRMaterialParserSelfTest missing fallback" ) ); + if ( ok ) { + const pbrMaterialInfo_t &info = missingMaterial->GetPBRInfo(); + ok = missingMaterial->HasPBR() + && info.legacyFallbackMissing + && !info.usesGeneratedLegacyFallback + && !info.usesApproximateLegacyFallback + && missingMaterial->GetNumStages() == 0; + } + DeclManager_FreeAllocatedDecl( missingDecl ); + if ( !ok ) { + return false; + } + + auto validatesGeneratedNormalFallback = []( const char *declaration, const char *label, bool expectReuse ) -> bool { + idDecl *decl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( decl == NULL ) { + return false; + } + idMaterial *material = static_cast( decl ); + bool valid = material->Parse( declaration, idLib::SizeToInt( strlen( declaration ), label ) ); + if ( valid ) { + const pbrMaterialInfo_t &info = material->GetPBRInfo(); + const shaderStage_t *bumpStage = NULL; + for ( int i = 0; i < material->GetNumStages(); ++i ) { + const shaderStage_t *stage = material->GetStage( i ); + if ( stage != NULL && stage->lighting == SL_BUMP ) { + bumpStage = stage; + break; + } + } + valid = info.usesApproximateLegacyFallback + && !info.legacyFallbackMissing + && info.normal.image != NULL + && bumpStage != NULL + && ( expectReuse + ? bumpStage->texture.image == info.normal.image + : bumpStage->texture.image == globalImages->flatNormalMap + && info.normal.image != globalImages->flatNormalMap ); + } + DeclManager_FreeAllocatedDecl( decl ); + return valid; + }; + static const char tangentNormalFallback[] = + "material _pbr_selftest_tangent_normal_fallback {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " normalMap makeIntensity( _white )\n" + " normalFormat tangentRG\n" + " }\n" + "}\n"; + static const char quake4NormalFallback[] = + "material _pbr_selftest_quake4_normal_fallback {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " normalMap makeIntensity( _white )\n" + " normalFormat quake4AGB\n" + " }\n" + "}\n"; + const bool oldGeneratedFallback = r_pbrGeneratedLegacyFallback.GetBool(); + r_pbrGeneratedLegacyFallback.SetBool( true ); + const bool normalFallbacksValid = validatesGeneratedNormalFallback( + tangentNormalFallback, "R_PBRMaterialParserSelfTest tangent fallback", false ) + && validatesGeneratedNormalFallback( + quake4NormalFallback, "R_PBRMaterialParserSelfTest quake4 fallback", true ); + r_pbrGeneratedLegacyFallback.SetBool( oldGeneratedFallback ); + if ( !normalFallbacksValid ) { + return false; + } + + static const char missingNormalFormat[] = + "material _pbr_selftest_bad_normal {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " normalMap _flat\n" + " }\n" + "}\n"; + idDecl *badNormalDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( badNormalDecl == NULL ) { + return false; + } + idMaterial *badNormal = static_cast( badNormalDecl ); + const bool badNormalAccepted = badNormal->Parse( missingNormalFormat, idLib::SizeToInt( sizeof( missingNormalFormat ) - 1, "R_PBRMaterialParserSelfTest normal" ) ); + DeclManager_FreeAllocatedDecl( badNormalDecl ); + if ( badNormalAccepted ) { + return false; + } + + static const char conflictingMaps[] = + "material _pbr_selftest_bad_maps {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " ormMap _white\n" + " roughnessMap _white\n" + " }\n" + "}\n"; + idDecl *badMapsDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( badMapsDecl == NULL ) { + return false; + } + idMaterial *badMaps = static_cast( badMapsDecl ); + const bool badMapsAccepted = badMaps->Parse( conflictingMaps, idLib::SizeToInt( sizeof( conflictingMaps ) - 1, "R_PBRMaterialParserSelfTest maps" ) ); + DeclManager_FreeAllocatedDecl( badMapsDecl ); + if ( badMapsAccepted ) { + return false; + } + + static const char fractionalFallback[] = + "material _pbr_selftest_bad_fallback_value {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " autoLegacyFallback 0.5\n" + " }\n" + "}\n"; + idDecl *fractionalDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( fractionalDecl == NULL ) { + return false; + } + idMaterial *fractionalMaterial = static_cast( fractionalDecl ); + const bool fractionalAccepted = fractionalMaterial->Parse( fractionalFallback, idLib::SizeToInt( sizeof( fractionalFallback ) - 1, "R_PBRMaterialParserSelfTest fallback value" ) ); + DeclManager_FreeAllocatedDecl( fractionalDecl ); + if ( fractionalAccepted ) { + return false; + } + + auto rejectsPBRDeclaration = []( const char *declaration, const char *label ) -> bool { + idDecl *decl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( decl == NULL ) { + return false; + } + idMaterial *material = static_cast( decl ); + const bool accepted = material->Parse( declaration, idLib::SizeToInt( strlen( declaration ), label ) ); + DeclManager_FreeAllocatedDecl( decl ); + return !accepted; + }; + static const char dynamicImageToken[] = + "material _pbr_selftest_dynamic_image {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap reflectionRenderMap\n" + " }\n" + "}\n"; + static const char shaderImageToken[] = + "material _pbr_selftest_shader_image {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap glslProgram\n" + " }\n" + "}\n"; + static const char nestedDynamicImageToken[] = + "material _pbr_selftest_nested_dynamic_image {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap add( _white, reflectionRenderMap )\n" + " }\n" + "}\n"; + static const char nestedShaderImageToken[] = + "material _pbr_selftest_nested_shader_image {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap add( _white, glslProgram )\n" + " }\n" + "}\n"; + static const char nestedSceneCaptureToken[] = + "material _pbr_selftest_nested_scene_capture {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap add( _white, _currentRender )\n" + " }\n" + "}\n"; + static const char nestedMutableRenderTargetToken[] = + "material _pbr_selftest_nested_mutable_target {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap add( _white, _reflectionRender )\n" + " }\n" + "}\n"; + static const char nestedStageStateToken[] = + "material _pbr_selftest_nested_stage_state {\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap add( _white, alphaTest )\n" + " }\n" + "}\n"; + return rejectsPBRDeclaration( dynamicImageToken, "R_PBRMaterialParserSelfTest dynamic image" ) + && rejectsPBRDeclaration( shaderImageToken, "R_PBRMaterialParserSelfTest shader image" ) + && rejectsPBRDeclaration( nestedDynamicImageToken, "R_PBRMaterialParserSelfTest nested dynamic image" ) + && rejectsPBRDeclaration( nestedShaderImageToken, "R_PBRMaterialParserSelfTest nested shader image" ) + && rejectsPBRDeclaration( nestedSceneCaptureToken, "R_PBRMaterialParserSelfTest nested scene capture" ) + && rejectsPBRDeclaration( nestedMutableRenderTargetToken, "R_PBRMaterialParserSelfTest nested mutable target" ) + && rejectsPBRDeclaration( nestedStageStateToken, "R_PBRMaterialParserSelfTest nested stage state" ); +} + /* =================== idMaterial::ReloadImages @@ -4110,4 +5091,16 @@ void idMaterial::ReloadImages( bool force ) const if ( portalImage ) { portalImage->Reload( force ); } + + const pbrMaterialTexture_t *pbrTextures[] = { + &pbrInfo.albedo, &pbrInfo.normal, &pbrInfo.orm, &pbrInfo.metallic, + &pbrInfo.roughness, &pbrInfo.ao, &pbrInfo.emissive, + &pbrInfo.legacyBump, &pbrInfo.legacyDiffuse, + &pbrInfo.legacySpecular, &pbrInfo.legacyEmissive + }; + for ( unsigned int i = 0; i < sizeof( pbrTextures ) / sizeof( pbrTextures[0] ); ++i ) { + if ( pbrTextures[i]->present && pbrTextures[i]->image != NULL ) { + pbrTextures[i]->image->Reload( force ); + } + } } diff --git a/src/renderer/Material.h b/src/renderer/Material.h index d5326d83..ae808e3a 100644 --- a/src/renderer/Material.h +++ b/src/renderer/Material.h @@ -41,6 +41,61 @@ class idImage; class idCinematic; class idUserInterface; +// openQ4-only, opt-in physically based material metadata. This is kept +// separate from the classic stage list so retail Quake 4 materials retain +// their exact bump/diffuse/specular parsing and rendering contract. +typedef enum { + PBR_WORKFLOW_NONE = 0, + PBR_WORKFLOW_METALLIC_ROUGHNESS, + PBR_WORKFLOW_SPECULAR_GLOSSINESS +} pbrWorkflow_t; + +typedef enum { + PBR_NORMAL_UNSPECIFIED = -1, + PBR_NORMAL_QUAKE4_AGB = 0, + PBR_NORMAL_TANGENT_RG, + PBR_NORMAL_TANGENT_XYZ +} pbrNormalFormat_t; + +typedef struct { + idImage * image; + bool present; + int filter; // textureFilter_t, retained for classic fallback generation + int repeat; // textureRepeat_t, retained for classic fallback generation + bool allowPicmip; + bool noMips; + bool highQuality; + bool forceHighQuality; +} pbrMaterialTexture_t; + +typedef struct { + bool enabled; + pbrWorkflow_t workflow; + pbrNormalFormat_t normalFormat; + pbrMaterialTexture_t albedo; + pbrMaterialTexture_t normal; + pbrMaterialTexture_t orm; + pbrMaterialTexture_t metallic; + pbrMaterialTexture_t roughness; + pbrMaterialTexture_t ao; + pbrMaterialTexture_t emissive; + pbrMaterialTexture_t legacyBump; + pbrMaterialTexture_t legacyDiffuse; + pbrMaterialTexture_t legacySpecular; + pbrMaterialTexture_t legacyEmissive; + int metallicRegister; + int roughnessRegister; + int aoRegister; + int normalScaleRegister; + int emissiveColorRegisters[3]; + bool autoLegacyFallback; + bool hasAuthoredClassicFallback; + bool hasExplicitLegacyFallback; + bool usesGeneratedLegacyFallback; + bool usesApproximateLegacyFallback; + bool legacyFallbackMissing; +} pbrMaterialInfo_t; + // moved from image.h for default parm typedef enum { TF_LINEAR, @@ -505,6 +560,11 @@ class idMaterial : public idDecl { // get a specific stage const shaderStage_t* GetStage(const int index) const { assert(index >= 0 && index < numStages); return &stages[index]; } + // PBR metadata is consumed only by explicitly enabled modern renderer paths. + // Classic callers continue to use the unchanged stage list. + bool HasPBR(void) const { return pbrInfo.enabled; } + const pbrMaterialInfo_t& GetPBRInfo(void) const { return pbrInfo; } + // Retarget a parsed stage without rewriting authored declaration text. // Runtime-generated images (for example the scalable console font atlas) // must not change the declaration checksum used by multiplayer handshakes. @@ -801,6 +861,9 @@ class idMaterial : public idDecl { void ParseShaderParm(idLexer& src, newShaderStage_t* newStage); void ParseShaderTexture(idLexer& src, newShaderStage_t* newStage); void ParseStage(idLexer& src, const textureRepeat_t trpDefault = TR_REPEAT); + bool ParsePBRBlock(idLexer& src, const textureRepeat_t trpDefault); + bool ParsePBRImage(idLexer& src, pbrMaterialTexture_t& target, const int usage, const textureRepeat_t trpDefault); + void AddPBRLegacyFallbackStages(const textureRepeat_t trpDefault); void ParseDeform(idLexer& src); void ParseDecalInfo(idLexer& src); bool CheckSurfaceParm(idToken* token); @@ -906,11 +969,13 @@ class idMaterial : public idDecl { bool suppressInSubview; bool portalSky; + pbrMaterialInfo_t pbrInfo; int refCount; }; // Parser-free regression hook for custom GLSL receiver compatibility helpers. bool R_MaterialCustomGLSLReceiverHelperSelfTest( void ); +bool R_PBRMaterialParserSelfTest( void ); typedef idList idMatList; diff --git a/src/renderer/MaterialResourceTable.cpp b/src/renderer/MaterialResourceTable.cpp index 89719b32..f3e70021 100644 --- a/src/renderer/MaterialResourceTable.cpp +++ b/src/renderer/MaterialResourceTable.cpp @@ -82,12 +82,55 @@ const char *MaterialResourceTextureSemantic_Name( materialResourceTextureSemanti return "gui"; case MATERIAL_RESOURCE_TEXTURE_POST_PROCESS: return "post"; + case MATERIAL_RESOURCE_TEXTURE_ALBEDO: + return "pbrAlbedo"; + case MATERIAL_RESOURCE_TEXTURE_NORMAL: + return "pbrNormal"; + case MATERIAL_RESOURCE_TEXTURE_ORM: + return "pbrORM"; + case MATERIAL_RESOURCE_TEXTURE_METALLIC: + return "pbrMetallic"; + case MATERIAL_RESOURCE_TEXTURE_ROUGHNESS: + return "pbrRoughness"; + case MATERIAL_RESOURCE_TEXTURE_AO: + return "pbrAO"; + case MATERIAL_RESOURCE_TEXTURE_EMISSIVE_PBR: + return "pbrEmissive"; case MATERIAL_RESOURCE_TEXTURE_NONE: default: return "none"; } } +const char *MaterialResourcePBRFallbackReason_Name( materialResourcePBRFallbackReason_t reason ) { + switch ( reason ) { + case MATERIAL_RESOURCE_PBR_FALLBACK_NONE: + return "none"; + case MATERIAL_RESOURCE_PBR_FALLBACK_DISABLED: + return "disabled"; + case MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_WORKFLOW: + return "unsupportedWorkflow"; + case MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_MATERIAL_CLASS: + return "unsupportedMaterialClass"; + case MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_ALBEDO: + return "missingAlbedo"; + case MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_NORMAL_FORMAT: + return "missingNormalFormat"; + case MATERIAL_RESOURCE_PBR_FALLBACK_CONFLICTING_LAYOUT: + return "conflictingLayout"; + case MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_IMAGE: + return "missingImage"; + case MATERIAL_RESOURCE_PBR_FALLBACK_CLASSIC_FEATURE: + return "classicFeature"; + case MATERIAL_RESOURCE_PBR_FALLBACK_TOO_MANY_TEXTURES: + return "tooManyTextures"; + case MATERIAL_RESOURCE_PBR_FALLBACK_SHADER_PATH_UNAVAILABLE: + return "shaderPathUnavailable"; + default: + return "unknown"; + } +} + unsigned int MaterialResourceTextureSemantic_Bit( materialResourceTextureSemantic_t semantic ) { if ( semantic <= MATERIAL_RESOURCE_TEXTURE_NONE || semantic >= MATERIAL_RESOURCE_TEXTURE_COUNT ) { return 0; @@ -191,6 +234,12 @@ static void R_MaterialResourceTable_AddFallback( materialResourceTableRecord_t & } } +static void R_MaterialResourceTable_AddPBRFallback( materialResourceTableRecord_t &record, materialResourcePBRFallbackReason_t reason ) { + if ( record.pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_NONE ) { + record.pbrFallbackReason = reason; + } +} + static bool R_MaterialResourceTable_ImageIsPostProcess( const idImage *image ) { if ( image == NULL ) { return false; @@ -206,21 +255,7 @@ static bool R_MaterialResourceTable_ImageIsPostProcess( const idImage *image ) { } static bool R_MaterialResourceTable_ImageIsSceneCapture( const idImage *image ) { - if ( image == NULL ) { - return false; - } - if ( globalImages != NULL - && ( image == globalImages->currentRenderImage - || image == globalImages->originalCurrentRenderImage - || image == globalImages->currentDepthImage ) ) { - return true; - } - const char *name = image->GetName(); - if ( name == NULL ) { - return false; - } - return idStr::Icmpn( name, "_currentRender", 14 ) == 0 - || idStr::Icmpn( name, "_currentDepth", 13 ) == 0; + return R_IsMutableRenderImage( image ); } static bool R_MaterialResourceTable_TexgenIsScreenSpace( texgen_t texgen ) { @@ -372,6 +407,16 @@ static int R_MaterialResourceTable_SemanticSlot( materialResourceTextureSemantic return 4; case MATERIAL_RESOURCE_TEXTURE_POST_PROCESS: return 5; + case MATERIAL_RESOURCE_TEXTURE_ALBEDO: + case MATERIAL_RESOURCE_TEXTURE_NORMAL: + case MATERIAL_RESOURCE_TEXTURE_ORM: + case MATERIAL_RESOURCE_TEXTURE_METALLIC: + case MATERIAL_RESOURCE_TEXTURE_ROUGHNESS: + case MATERIAL_RESOURCE_TEXTURE_AO: + case MATERIAL_RESOURCE_TEXTURE_EMISSIVE_PBR: + // Phase 3 records PBR handles but does not allocate visible shader units. + // The existing classic and shadow bindings therefore remain untouched. + return -1; default: return -1; } @@ -552,6 +597,27 @@ static void R_MaterialResourceTable_UpdateRecordSemanticFlags( materialResourceT case MATERIAL_RESOURCE_TEXTURE_POST_PROCESS: record.hasPostProcess |= present; break; + case MATERIAL_RESOURCE_TEXTURE_ALBEDO: + record.hasPBRAlbedo |= present; + break; + case MATERIAL_RESOURCE_TEXTURE_NORMAL: + record.hasPBRNormal |= present; + break; + case MATERIAL_RESOURCE_TEXTURE_ORM: + record.hasPBRORM |= present; + break; + case MATERIAL_RESOURCE_TEXTURE_METALLIC: + record.hasPBRMetallic |= present; + break; + case MATERIAL_RESOURCE_TEXTURE_ROUGHNESS: + record.hasPBRRoughness |= present; + break; + case MATERIAL_RESOURCE_TEXTURE_AO: + record.hasPBRAO |= present; + break; + case MATERIAL_RESOURCE_TEXTURE_EMISSIVE_PBR: + record.hasPBREmissive |= present; + break; default: break; } @@ -630,6 +696,10 @@ static bool R_MaterialResourceTable_HasSemanticBinding( const materialResourceTa return R_MaterialResourceTable_FindTextureBindingIndex( record, semantic ) >= 0; } +static bool R_MaterialResourceTable_IsPBRSemantic( materialResourceTextureSemantic_t semantic ) { + return semantic >= MATERIAL_RESOURCE_TEXTURE_ALBEDO && semantic < MATERIAL_RESOURCE_TEXTURE_COUNT; +} + static void R_MaterialResourceTable_AddTextureBinding( materialResourceTableRecord_t &record, materialResourceTextureSemantic_t semantic, @@ -637,8 +707,12 @@ static void R_MaterialResourceTable_AddTextureBinding( const shaderStage_t *stage, int stageIndex ) { if ( semantic <= MATERIAL_RESOURCE_TEXTURE_NONE || semantic >= MATERIAL_RESOURCE_TEXTURE_COUNT || image == NULL ) { - record.hasMissingImage = true; - R_MaterialResourceTable_AddFallback( record, MATERIAL_RESOURCE_FALLBACK_MISSING_IMAGE, MATERIAL_RESOURCE_FALLBACK_FLAG_MISSING_IMAGE ); + if ( R_MaterialResourceTable_IsPBRSemantic( semantic ) ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_IMAGE ); + } else { + record.hasMissingImage = true; + R_MaterialResourceTable_AddFallback( record, MATERIAL_RESOURCE_FALLBACK_MISSING_IMAGE, MATERIAL_RESOURCE_FALLBACK_FLAG_MISSING_IMAGE ); + } rg_materialResourceTable.stats.missingImages++; return; } @@ -647,7 +721,11 @@ static void R_MaterialResourceTable_AddTextureBinding( return; } if ( record.textureBindingCount >= MATERIAL_RESOURCE_TABLE_MAX_TEXTURE_BINDINGS ) { - R_MaterialResourceTable_AddFallback( record, MATERIAL_RESOURCE_FALLBACK_TOO_MANY_TEXTURES, MATERIAL_RESOURCE_FALLBACK_FLAG_TOO_MANY_TEXTURES ); + if ( R_MaterialResourceTable_IsPBRSemantic( semantic ) ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_TOO_MANY_TEXTURES ); + } else { + R_MaterialResourceTable_AddFallback( record, MATERIAL_RESOURCE_FALLBACK_TOO_MANY_TEXTURES, MATERIAL_RESOURCE_FALLBACK_FLAG_TOO_MANY_TEXTURES ); + } rg_materialResourceTable.stats.unsupportedFeatures++; return; } @@ -662,7 +740,11 @@ static void R_MaterialResourceTable_AddTextureBinding( binding.repeat = image->GetRepeat(); binding.classicUnit = R_MaterialResourceTable_SemanticSlot( semantic ); if ( binding.classicUnit >= rg_materialResourceTable.maxClassicTextureUnits ) { - R_MaterialResourceTable_AddFallback( record, MATERIAL_RESOURCE_FALLBACK_TOO_MANY_TEXTURES, MATERIAL_RESOURCE_FALLBACK_FLAG_TOO_MANY_TEXTURES ); + if ( R_MaterialResourceTable_IsPBRSemantic( semantic ) ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_TOO_MANY_TEXTURES ); + } else { + R_MaterialResourceTable_AddFallback( record, MATERIAL_RESOURCE_FALLBACK_TOO_MANY_TEXTURES, MATERIAL_RESOURCE_FALLBACK_FLAG_TOO_MANY_TEXTURES ); + } binding.classicUnit = -1; } binding.stageIndex = stageIndex; @@ -719,7 +801,9 @@ static void R_MaterialResourceTable_AddTextureBinding( if ( semantic == MATERIAL_RESOURCE_TEXTURE_DIFFUSE || semantic == MATERIAL_RESOURCE_TEXTURE_EMISSIVE || semantic == MATERIAL_RESOURCE_TEXTURE_GUI - || semantic == MATERIAL_RESOURCE_TEXTURE_POST_PROCESS ) { + || semantic == MATERIAL_RESOURCE_TEXTURE_POST_PROCESS + || semantic == MATERIAL_RESOURCE_TEXTURE_ALBEDO + || semantic == MATERIAL_RESOURCE_TEXTURE_EMISSIVE_PBR ) { record.renderableColorTextureMask |= semanticBit; } } @@ -748,6 +832,16 @@ static void R_MaterialResourceTable_BuildTextureArrayTable( void ) { R_MaterialResourceTable_SeedTextureArrayFallbacks(); for ( int recordIndex = 0; recordIndex < rg_materialResourceTable.stats.records; ++recordIndex ) { materialResourceTableRecord_t &record = rg_materialResourceTable.records[recordIndex]; + // PBR records stay on the legacy/classic owner until a dedicated PBR + // shader path exists. Do not let their bindings consume the shared + // classic-modern texture table and perturb unrelated classic records. + if ( record.hasPBR ) { + for ( int bindingIndex = 0; bindingIndex < record.textureBindingCount; ++bindingIndex ) { + record.textures[bindingIndex].textureArrayCandidate = false; + record.textures[bindingIndex].textureArrayLayer = -1; + } + continue; + } for ( int bindingIndex = 0; bindingIndex < record.textureBindingCount; ++bindingIndex ) { materialResourceTextureBinding_t &binding = record.textures[bindingIndex]; if ( !binding.loaded || binding.textureHandle == 0 ) { @@ -1023,6 +1117,169 @@ static void R_MaterialResourceTable_AddSourceImages( materialResourceTableRecord } } +static void R_MaterialResourceTable_CountPBR( const materialResourceTableRecord_t &record ) { + if ( !record.hasPBR ) { + rg_materialResourceTable.stats.classicRecords++; + return; + } + materialResourceTableStats_t &stats = rg_materialResourceTable.stats; + stats.pbrRecords++; + stats.pbrResourceReadyRecords += record.pbrResourceReady ? 1 : 0; + stats.pbrModernReadyRecords += record.pbrModernReady ? 1 : 0; + stats.pbrPackedMapRecords += record.pbrPackedMaterialData ? 1 : 0; + stats.pbrSeparateMapRecords += record.pbrSeparateMaterialData ? 1 : 0; + stats.pbrAuthoredClassicFallbackRecords += record.pbrHasAuthoredClassicFallback ? 1 : 0; + stats.pbrExplicitGeneratedFallbackRecords += record.pbrHasExplicitLegacyFallback && record.pbrUsesGeneratedLegacyFallback ? 1 : 0; + stats.pbrGeneratedFallbackRecords += record.pbrUsesGeneratedLegacyFallback ? 1 : 0; + stats.pbrApproximateFallbackRecords += record.pbrUsesApproximateLegacyFallback ? 1 : 0; + stats.pbrMissingLegacyFallbackRecords += record.pbrLegacyFallbackMissing ? 1 : 0; + stats.pbrMissingAlbedoMapRecords += record.hasPBRAlbedo ? 0 : 1; + stats.pbrMissingNormalMapRecords += record.hasPBRNormal ? 0 : 1; + stats.pbrMissingORMMapRecords += record.hasPBRORM ? 0 : 1; + if ( record.pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_NONE ) { + return; + } + stats.pbrFallbackRecords++; + switch ( record.pbrFallbackReason ) { + case MATERIAL_RESOURCE_PBR_FALLBACK_DISABLED: + stats.pbrFallbackDisabled++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_WORKFLOW: + stats.pbrFallbackUnsupportedWorkflow++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_MATERIAL_CLASS: + stats.pbrFallbackUnsupportedMaterialClass++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_ALBEDO: + stats.pbrFallbackMissingAlbedo++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_NORMAL_FORMAT: + stats.pbrFallbackMissingNormalFormat++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_CONFLICTING_LAYOUT: + stats.pbrFallbackConflictingLayout++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_IMAGE: + stats.pbrFallbackMissingImage++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_CLASSIC_FEATURE: + stats.pbrFallbackClassicFeature++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_TOO_MANY_TEXTURES: + stats.pbrFallbackTooManyTextures++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_SHADER_PATH_UNAVAILABLE: + stats.pbrFallbackShaderPathUnavailable++; + break; + case MATERIAL_RESOURCE_PBR_FALLBACK_NONE: + default: + break; + } +} + +static void R_MaterialResourceTable_AddPBRSourceImages( materialResourceTableRecord_t &record, const materialResourceRecord_t &sourceRecord ) { + if ( !sourceRecord.hasPBR ) { + return; + } + record.hasPBR = true; + record.pbrWorkflow = sourceRecord.pbrWorkflow; + record.pbrNormalFormat = sourceRecord.pbrNormalFormat; + record.pbrHasAuthoredClassicFallback = sourceRecord.pbrHasAuthoredClassicFallback; + record.pbrHasExplicitLegacyFallback = sourceRecord.pbrHasExplicitLegacyFallback; + record.pbrUsesGeneratedLegacyFallback = sourceRecord.pbrUsesGeneratedLegacyFallback; + record.pbrUsesApproximateLegacyFallback = sourceRecord.pbrUsesApproximateLegacyFallback; + record.pbrLegacyFallbackMissing = sourceRecord.pbrLegacyFallbackMissing; + record.pbrMetallicRegister = sourceRecord.pbrMetallicRegister; + record.pbrRoughnessRegister = sourceRecord.pbrRoughnessRegister; + record.pbrAORegister = sourceRecord.pbrAORegister; + record.pbrNormalScaleRegister = sourceRecord.pbrNormalScaleRegister; + memcpy( record.pbrEmissiveColorRegisters, sourceRecord.pbrEmissiveColorRegisters, sizeof( record.pbrEmissiveColorRegisters ) ); + + if ( sourceRecord.pbrAlbedoImage != NULL ) { + R_MaterialResourceTable_AddTextureBinding( record, MATERIAL_RESOURCE_TEXTURE_ALBEDO, sourceRecord.pbrAlbedoImage, NULL, -1 ); + } + if ( sourceRecord.pbrNormalImage != NULL ) { + R_MaterialResourceTable_AddTextureBinding( record, MATERIAL_RESOURCE_TEXTURE_NORMAL, sourceRecord.pbrNormalImage, NULL, -1 ); + } + if ( sourceRecord.pbrORMImage != NULL ) { + R_MaterialResourceTable_AddTextureBinding( record, MATERIAL_RESOURCE_TEXTURE_ORM, sourceRecord.pbrORMImage, NULL, -1 ); + } + if ( sourceRecord.pbrMetallicImage != NULL ) { + R_MaterialResourceTable_AddTextureBinding( record, MATERIAL_RESOURCE_TEXTURE_METALLIC, sourceRecord.pbrMetallicImage, NULL, -1 ); + } + if ( sourceRecord.pbrRoughnessImage != NULL ) { + R_MaterialResourceTable_AddTextureBinding( record, MATERIAL_RESOURCE_TEXTURE_ROUGHNESS, sourceRecord.pbrRoughnessImage, NULL, -1 ); + } + if ( sourceRecord.pbrAOImage != NULL ) { + R_MaterialResourceTable_AddTextureBinding( record, MATERIAL_RESOURCE_TEXTURE_AO, sourceRecord.pbrAOImage, NULL, -1 ); + } + if ( sourceRecord.pbrEmissiveImage != NULL ) { + R_MaterialResourceTable_AddTextureBinding( record, MATERIAL_RESOURCE_TEXTURE_EMISSIVE_PBR, sourceRecord.pbrEmissiveImage, NULL, -1 ); + } + record.pbrPackedMaterialData = sourceRecord.pbrORMImage != NULL; + record.pbrSeparateMaterialData = sourceRecord.pbrMetallicImage != NULL + || sourceRecord.pbrRoughnessImage != NULL + || sourceRecord.pbrAOImage != NULL; +} + +static bool R_MaterialResourceTable_PBRBindingReady( const materialResourceTableRecord_t &record, materialResourceTextureSemantic_t semantic, bool required ) { + const int index = R_MaterialResourceTable_FindTextureBindingIndex( record, semantic ); + if ( index < 0 ) { + return !required; + } + const materialResourceTextureBinding_t &binding = record.textures[index]; + return binding.image != NULL + && binding.loaded + && !binding.defaulted + && !R_IsMutableRenderImage( binding.image ); +} + +static void R_MaterialResourceTable_FinalizePBRContract( materialResourceTableRecord_t &record ) { + if ( !record.hasPBR ) { + return; + } + + if ( !record.hasPBRAlbedo ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_ALBEDO ); + } + if ( record.hasPBRNormal && record.pbrNormalFormat == PBR_NORMAL_UNSPECIFIED ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_NORMAL_FORMAT ); + } + if ( record.pbrPackedMaterialData && record.pbrSeparateMaterialData ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_CONFLICTING_LAYOUT ); + } + if ( record.pbrWorkflow != PBR_WORKFLOW_METALLIC_ROUGHNESS ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_WORKFLOW ); + } + if ( record.materialClass != RENDER_MATERIAL_OPAQUE && record.materialClass != RENDER_MATERIAL_PERFORATED ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_MATERIAL_CLASS ); + } + if ( !R_MaterialResourceTable_PBRBindingReady( record, MATERIAL_RESOURCE_TEXTURE_ALBEDO, true ) + || !R_MaterialResourceTable_PBRBindingReady( record, MATERIAL_RESOURCE_TEXTURE_NORMAL, record.hasPBRNormal ) + || !R_MaterialResourceTable_PBRBindingReady( record, MATERIAL_RESOURCE_TEXTURE_ORM, record.hasPBRORM ) + || !R_MaterialResourceTable_PBRBindingReady( record, MATERIAL_RESOURCE_TEXTURE_METALLIC, record.hasPBRMetallic ) + || !R_MaterialResourceTable_PBRBindingReady( record, MATERIAL_RESOURCE_TEXTURE_ROUGHNESS, record.hasPBRRoughness ) + || !R_MaterialResourceTable_PBRBindingReady( record, MATERIAL_RESOURCE_TEXTURE_AO, record.hasPBRAO ) + || !R_MaterialResourceTable_PBRBindingReady( record, MATERIAL_RESOURCE_TEXTURE_EMISSIVE_PBR, record.hasPBREmissive ) ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_IMAGE ); + } + if ( record.fallbackReason != MATERIAL_RESOURCE_FALLBACK_NONE ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_CLASSIC_FEATURE ); + } + + record.pbrResourceReady = record.pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_NONE; + if ( record.pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_NONE && !r_pbrMaterials.GetBool() ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_DISABLED ); + } + // Phase 3 publishes complete resource ownership but deliberately does not + // claim a visible PBR shader path. Phase 4 replaces this reason only after + // G-buffer and forward/deferred program readiness are proven. + if ( record.pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_NONE ) { + R_MaterialResourceTable_AddPBRFallback( record, MATERIAL_RESOURCE_PBR_FALLBACK_SHADER_PATH_UNAVAILABLE ); + } + record.pbrModernReady = false; +} + static void R_MaterialResourceTable_FinalizeShadowContract( materialResourceTableRecord_t &record ) { record.shadowFallbackFlags |= record.fallbackFlags; record.shadowCasterSupported = record.castsShadow && record.fallbackReason == MATERIAL_RESOURCE_FALLBACK_NONE; @@ -1054,7 +1311,10 @@ static bool R_MaterialResourceTable_AddRecordFromSource( const materialResourceR } record.tableIndex = rg_materialResourceTable.stats.records; record.sourceMaterialRecordIndex = sourceIndex; - record.materialId = sourceRecord.material != NULL ? sourceRecord.material->Index() : -1; + // The scene-packet record owns the stable decl index. Synthetic validation + // decls allocated through declManager intentionally have no idDeclBase and + // therefore must never be asked for Index() directly. + record.materialId = sourceRecord.material != NULL ? sourceRecord.resourceTableIndex : -1; record.material = sourceRecord.material; if ( !R_MaterialResourceTable_CopyDebugString( record.materialName, sizeof( record.materialName ), sourceRecord.material != NULL ? sourceRecord.material->GetName() : "" ) ) { R_MaterialResourceTable_RecordDebugStringTruncation( "material record name" ); @@ -1097,6 +1357,7 @@ static bool R_MaterialResourceTable_AddRecordFromSource( const materialResourceR } else { R_MaterialResourceTable_AddSourceImages( record, sourceRecord ); } + R_MaterialResourceTable_AddPBRSourceImages( record, sourceRecord ); if ( R_MaterialResourceTable_RecordNeedsSurfaceImage( record ) && !record.hasDiffuse && !record.hasEmissive @@ -1108,6 +1369,7 @@ static bool R_MaterialResourceTable_AddRecordFromSource( const materialResourceR } R_MaterialResourceTable_ValidateStageColorContract( record ); R_MaterialResourceTable_ValidateAmbientOverlayContract( record ); + R_MaterialResourceTable_FinalizePBRContract( record ); if ( !scanMaterialStages ) { record.shadowCasterSupported = record.castsShadow && record.fallbackReason == MATERIAL_RESOURCE_FALLBACK_NONE; record.shadowFallbackFlags = record.fallbackFlags; @@ -1116,6 +1378,7 @@ static bool R_MaterialResourceTable_AddRecordFromSource( const materialResourceR R_MaterialResourceTable_FinalizeRegisterRange( record ); R_MaterialResourceTable_CountClass( record ); R_MaterialResourceTable_CountFallbacks( record ); + R_MaterialResourceTable_CountPBR( record ); R_MaterialResourceTable_HashInsert( record.material, rg_materialResourceTable.stats.records ); rg_materialResourceTable.stats.records++; @@ -1237,11 +1500,12 @@ int R_MaterialResourceTable_TextureArrayTableIndexForHandle( unsigned int textur void R_MaterialResourceTable_PrintGfxInfo( void ) { const materialResourceTableStats_t &stats = R_MaterialResourceTable_Stats(); common->Printf( - "Material resource table: initialized=%d available=%d prepared=%d records=%d source=%d draws=%d textures=%d classic=%d arrays=%d table=%d/%d desc=%d overflow=%d views=%d bindless=%d/%d fallback=%d missing=%d unsupported=%d custom=%d/%d dynamic=%d current=%d texgen=%d(screen=%d sky=%d) condition=%d stageColor=%d matrix=%d vertexColor=%d offset=%d debugTrunc=%d source='%s' status='%s'\n", + "Material resource table: initialized=%d available=%d prepared=%d records=%d classicRecords=%d source=%d draws=%d textures=%d classicTextures=%d arrays=%d table=%d/%d desc=%d overflow=%d views=%d bindless=%d/%d fallback=%d missing=%d unsupported=%d custom=%d/%d dynamic=%d current=%d texgen=%d(screen=%d sky=%d) condition=%d stageColor=%d matrix=%d vertexColor=%d offset=%d debugTrunc=%d source='%s' status='%s'\n", stats.initialized ? 1 : 0, stats.available ? 1 : 0, stats.prepared ? 1 : 0, stats.records, + stats.classicRecords, stats.sourceMaterialRecords, stats.drawPacketReferences, stats.textureBindings, @@ -1272,15 +1536,49 @@ void R_MaterialResourceTable_PrintGfxInfo( void ) { stats.debugStringTruncations, stats.debugStringTruncationSource, stats.lastFailure ); + common->Printf( + "PBR material resources: records=%d resourceReady=%d modernReady=%d packed=%d separate=%d fallback=%d disabled=%d workflow=%d class=%d albedo=%d normalFormat=%d layout=%d image=%d classic=%d units=%d shader=%d authored=%d explicitGenerated=%d generated=%d approximate=%d missingFallback=%d mapsMissing=%d/%d/%d\n", + stats.pbrRecords, + stats.pbrResourceReadyRecords, + stats.pbrModernReadyRecords, + stats.pbrPackedMapRecords, + stats.pbrSeparateMapRecords, + stats.pbrFallbackRecords, + stats.pbrFallbackDisabled, + stats.pbrFallbackUnsupportedWorkflow, + stats.pbrFallbackUnsupportedMaterialClass, + stats.pbrFallbackMissingAlbedo, + stats.pbrFallbackMissingNormalFormat, + stats.pbrFallbackConflictingLayout, + stats.pbrFallbackMissingImage, + stats.pbrFallbackClassicFeature, + stats.pbrFallbackTooManyTextures, + stats.pbrFallbackShaderPathUnavailable, + stats.pbrAuthoredClassicFallbackRecords, + stats.pbrExplicitGeneratedFallbackRecords, + stats.pbrGeneratedFallbackRecords, + stats.pbrApproximateFallbackRecords, + stats.pbrMissingLegacyFallbackRecords, + stats.pbrMissingAlbedoMapRecords, + stats.pbrMissingNormalMapRecords, + stats.pbrMissingORMMapRecords ); +} + +bool R_MaterialResourceTable_ClassicModernPathEligible( const materialResourceTableRecord_t &record ) { + // The current modern-visible programs implement the classic Quake 4 stage + // contract. PBR metadata must remain on the compatible legacy owner until a + // dedicated PBR pipeline explicitly consumes pbrModernReady and its bindings. + return !record.hasPBR; } void R_MaterialResourceTable_DumpLatest( void ) { const materialResourceTableStats_t &stats = R_MaterialResourceTable_Stats(); common->Printf( - "MaterialResourceTable dump: prepared=%d available=%d records=%d source=%d draws=%d textures=%d table=%d/%d desc=%d overflow=%d fallback=%d missing=%d defaulted=%d unsupported=%d debugTrunc=%d source='%s' status='%s'\n", + "MaterialResourceTable dump: prepared=%d available=%d records=%d classicRecords=%d source=%d draws=%d textures=%d table=%d/%d desc=%d overflow=%d fallback=%d missing=%d defaulted=%d unsupported=%d debugTrunc=%d source='%s' status='%s'\n", stats.prepared ? 1 : 0, stats.available ? 1 : 0, stats.records, + stats.classicRecords, stats.sourceMaterialRecords, stats.drawPacketReferences, stats.textureBindings, @@ -1295,6 +1593,19 @@ void R_MaterialResourceTable_DumpLatest( void ) { stats.debugStringTruncations, stats.debugStringTruncationSource, stats.lastFailure ); + common->Printf( + "PBR summary: records=%d resourceReady=%d modernReady=%d packed=%d separate=%d fallback=%d approximate=%d missingFallback=%d mapsMissing=%d/%d/%d\n", + stats.pbrRecords, + stats.pbrResourceReadyRecords, + stats.pbrModernReadyRecords, + stats.pbrPackedMapRecords, + stats.pbrSeparateMapRecords, + stats.pbrFallbackRecords, + stats.pbrApproximateFallbackRecords, + stats.pbrMissingLegacyFallbackRecords, + stats.pbrMissingAlbedoMapRecords, + stats.pbrMissingNormalMapRecords, + stats.pbrMissingORMMapRecords ); for ( int i = 0; i < stats.records; ++i ) { const materialResourceTableRecord_t &record = rg_materialResourceTable.records[i]; common->Printf( @@ -1336,6 +1647,36 @@ void R_MaterialResourceTable_DumpLatest( void ) { record.twoSided ? 1 : 0, record.shouldCreateBackSides ? 1 : 0, record.shadowFallbackFlags ); + if ( record.hasPBR ) { + common->Printf( + " pbr workflow=%d normalFormat=%d resourceReady=%d modernReady=%d fallback=%s packed=%d separate=%d maps=a%d n%d orm%d m%d r%d ao%d e%d authored=%d explicit=%d generated=%d approximate=%d missingFallback=%d regs=%d/%d/%d/%d emit=%d,%d,%d\n", + record.pbrWorkflow, + record.pbrNormalFormat, + record.pbrResourceReady ? 1 : 0, + record.pbrModernReady ? 1 : 0, + MaterialResourcePBRFallbackReason_Name( record.pbrFallbackReason ), + record.pbrPackedMaterialData ? 1 : 0, + record.pbrSeparateMaterialData ? 1 : 0, + record.hasPBRAlbedo ? 1 : 0, + record.hasPBRNormal ? 1 : 0, + record.hasPBRORM ? 1 : 0, + record.hasPBRMetallic ? 1 : 0, + record.hasPBRRoughness ? 1 : 0, + record.hasPBRAO ? 1 : 0, + record.hasPBREmissive ? 1 : 0, + record.pbrHasAuthoredClassicFallback ? 1 : 0, + record.pbrHasExplicitLegacyFallback ? 1 : 0, + record.pbrUsesGeneratedLegacyFallback ? 1 : 0, + record.pbrUsesApproximateLegacyFallback ? 1 : 0, + record.pbrLegacyFallbackMissing ? 1 : 0, + record.pbrMetallicRegister, + record.pbrRoughnessRegister, + record.pbrAORegister, + record.pbrNormalScaleRegister, + record.pbrEmissiveColorRegisters[0], + record.pbrEmissiveColorRegisters[1], + record.pbrEmissiveColorRegisters[2] ); + } for ( int bindingIndex = 0; bindingIndex < record.textureBindingCount; ++bindingIndex ) { const materialResourceTextureBinding_t &binding = record.textures[bindingIndex]; common->Printf( @@ -1368,6 +1709,249 @@ void R_MaterialResourceTable_DumpLatest( void ) { } } +static bool R_MaterialResourceTable_RunPBRContractSelfTest( void ) { + if ( globalImages == NULL ) { + common->Printf( "RendererMaterialResourceTable PBR self-test skipped: images unavailable\n" ); + return true; + } + static const char declaration[] = + "material _pbr_resource_table_selftest {\n" + " bumpmap _flat\n" + " diffusemap _white\n" + " specularmap _black\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " normalMap _flat\n" + " normalFormat tangentRG\n" + " ormMap _white\n" + " metallic 0.2\n" + " roughness 0.7\n" + " ao 1.0\n" + " }\n" + "}\n"; + + idDecl *materialDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( materialDecl == NULL ) { + common->Printf( "RendererMaterialResourceTable self-test failed: PBR declaration allocation\n" ); + return false; + } + idMaterial *material = static_cast( materialDecl ); + if ( !material->Parse( declaration, idLib::SizeToInt( sizeof( declaration ) - 1, "PBR resource-table self-test" ) ) ) { + common->Printf( "RendererMaterialResourceTable self-test failed: PBR declaration parse\n" ); + DeclManager_FreeAllocatedDecl( materialDecl ); + return false; + } + const int savedMaxClassicTextureUnits = rg_materialResourceTable.maxClassicTextureUnits; + // Vulkan can invoke this CPU-only contract before the classic material table + // is initialized. Scope the same minimum unit budget used by this synthetic + // bump/diffuse/specular material, then restore the live backend state. + rg_materialResourceTable.maxClassicTextureUnits = Max( savedMaxClassicTextureUnits, 3 ); + const pbrMaterialInfo_t &pbr = material->GetPBRInfo(); + materialResourceRecord_t source; + memset( &source, 0, sizeof( source ) ); + source.material = material; + source.diffuseImage = globalImages->whiteImage; + source.normalImage = globalImages->flatNormalMap; + source.specularImage = globalImages->blackImage; + source.hasPBR = true; + source.pbrWorkflow = static_cast( pbr.workflow ); + source.pbrNormalFormat = static_cast( pbr.normalFormat ); + source.pbrAlbedoImage = pbr.albedo.image; + source.pbrNormalImage = pbr.normal.image; + source.pbrORMImage = pbr.orm.image; + source.pbrHasAuthoredClassicFallback = pbr.hasAuthoredClassicFallback; + source.pbrHasExplicitLegacyFallback = pbr.hasExplicitLegacyFallback; + source.pbrUsesGeneratedLegacyFallback = pbr.usesGeneratedLegacyFallback; + source.pbrUsesApproximateLegacyFallback = pbr.usesApproximateLegacyFallback; + source.pbrLegacyFallbackMissing = pbr.legacyFallbackMissing; + source.pbrMetallicRegister = pbr.metallicRegister; + source.pbrRoughnessRegister = pbr.roughnessRegister; + source.pbrAORegister = pbr.aoRegister; + source.pbrNormalScaleRegister = pbr.normalScaleRegister; + memcpy( source.pbrEmissiveColorRegisters, pbr.emissiveColorRegisters, sizeof( source.pbrEmissiveColorRegisters ) ); + source.permutation.materialClass = RENDER_MATERIAL_OPAQUE; + source.permutation.alphaMode = MC_OPAQUE; + source.resourceTableIndex = 200; + + R_MaterialResourceTable_ResetFrameStats(); + rg_materialResourceTable.stats.prepared = true; + const bool added = R_MaterialResourceTable_AddRecordFromSource( source, 200, true ); + + materialResourceRecord_t separateSource = source; + separateSource.pbrORMImage = NULL; + separateSource.pbrMetallicImage = globalImages->whiteImage; + separateSource.pbrRoughnessImage = globalImages->whiteImage; + separateSource.pbrAOImage = globalImages->whiteImage; + separateSource.resourceTableIndex = 201; + const bool separateAdded = R_MaterialResourceTable_AddRecordFromSource( separateSource, 201, true ); + + materialResourceRecord_t scalarSource = source; + scalarSource.pbrNormalImage = NULL; + scalarSource.pbrNormalFormat = PBR_NORMAL_UNSPECIFIED; + scalarSource.pbrORMImage = NULL; + scalarSource.resourceTableIndex = 202; + const bool scalarAdded = R_MaterialResourceTable_AddRecordFromSource( scalarSource, 202, true ); + + materialResourceRecord_t explicitSource = source; + explicitSource.pbrHasAuthoredClassicFallback = false; + explicitSource.pbrHasExplicitLegacyFallback = true; + explicitSource.pbrUsesGeneratedLegacyFallback = true; + explicitSource.pbrUsesApproximateLegacyFallback = false; + explicitSource.resourceTableIndex = 203; + const bool explicitAdded = R_MaterialResourceTable_AddRecordFromSource( explicitSource, 203, true ); + + materialResourceRecord_t unsupportedSource = source; + unsupportedSource.pbrWorkflow = PBR_WORKFLOW_SPECULAR_GLOSSINESS; + unsupportedSource.resourceTableIndex = 204; + const bool unsupportedAdded = R_MaterialResourceTable_AddRecordFromSource( unsupportedSource, 204, true ); + + materialResourceRecord_t missingFallbackSource = source; + missingFallbackSource.pbrHasAuthoredClassicFallback = false; + missingFallbackSource.pbrLegacyFallbackMissing = true; + missingFallbackSource.resourceTableIndex = 205; + const bool missingFallbackAdded = R_MaterialResourceTable_AddRecordFromSource( missingFallbackSource, 205, true ); + + materialResourceRecord_t missingAlbedoSource = source; + missingAlbedoSource.pbrAlbedoImage = NULL; + missingAlbedoSource.resourceTableIndex = 206; + const bool missingAlbedoAdded = R_MaterialResourceTable_AddRecordFromSource( missingAlbedoSource, 206, true ); + + idImageOpts mutableImageOpts = globalImages->whiteImage->GetOpts(); + idImage *mutableImage = globalImages->ScratchImage( + "_pbr_resource_table_mutable_selftest", + &mutableImageOpts, + TF_DEFAULT, + TR_REPEAT, + TD_PBR_COLOR ); + materialResourceRecord_t mutableImageSource = source; + mutableImageSource.pbrAlbedoImage = mutableImage; + mutableImageSource.resourceTableIndex = 207; + const bool mutableImageAdded = mutableImage != NULL + && R_MaterialResourceTable_AddRecordFromSource( mutableImageSource, 207, true ); + + // Authored legacy-map metadata is not an explicit-generated fallback when + // the complete classic interaction already made stage generation redundant. + materialResourceRecord_t redundantExplicitSource = source; + redundantExplicitSource.pbrHasExplicitLegacyFallback = true; + redundantExplicitSource.pbrUsesGeneratedLegacyFallback = false; + redundantExplicitSource.resourceTableIndex = 208; + const bool redundantExplicitAdded = R_MaterialResourceTable_AddRecordFromSource( redundantExplicitSource, 208, true ); + + R_MaterialResourceTable_BuildTextureArrayTable(); + const materialResourceTableRecord_t *record = R_MaterialResourceTable_RecordForIndex( 0 ); + const materialResourceTableRecord_t *separateRecord = R_MaterialResourceTable_RecordForIndex( 1 ); + const materialResourceTableRecord_t *scalarRecord = R_MaterialResourceTable_RecordForIndex( 2 ); + const materialResourceTableRecord_t *explicitRecord = R_MaterialResourceTable_RecordForIndex( 3 ); + const materialResourceTableRecord_t *unsupportedRecord = R_MaterialResourceTable_RecordForIndex( 4 ); + const materialResourceTableRecord_t *missingFallbackRecord = R_MaterialResourceTable_RecordForIndex( 5 ); + const materialResourceTableRecord_t *missingAlbedoRecord = R_MaterialResourceTable_RecordForIndex( 6 ); + const materialResourceTableRecord_t *mutableImageRecord = R_MaterialResourceTable_RecordForIndex( 7 ); + const materialResourceTableRecord_t *redundantExplicitRecord = R_MaterialResourceTable_RecordForIndex( 8 ); + const materialResourceTableStats_t &stats = R_MaterialResourceTable_Stats(); + const materialResourcePBRFallbackReason_t expectedReason = r_pbrMaterials.GetBool() + ? MATERIAL_RESOURCE_PBR_FALLBACK_SHADER_PATH_UNAVAILABLE + : MATERIAL_RESOURCE_PBR_FALLBACK_DISABLED; + bool pbrBindingsExcludedFromClassicTable = stats.records == 9; + for ( int recordIndex = 0; recordIndex < stats.records; ++recordIndex ) { + const materialResourceTableRecord_t *testRecord = R_MaterialResourceTable_RecordForIndex( recordIndex ); + pbrBindingsExcludedFromClassicTable &= testRecord != NULL && testRecord->hasPBR; + if ( testRecord != NULL ) { + for ( int i = 0; i < testRecord->textureBindingCount; ++i ) { + pbrBindingsExcludedFromClassicTable &= !testRecord->textures[i].textureArrayCandidate + && testRecord->textures[i].textureArrayLayer == -1; + } + } + } + const bool ok = added && separateAdded && scalarAdded && explicitAdded + && unsupportedAdded && missingFallbackAdded && missingAlbedoAdded && mutableImageAdded && redundantExplicitAdded + && record != NULL + && record->hasPBR + && record->pbrResourceReady + && !record->pbrModernReady + && !R_MaterialResourceTable_ClassicModernPathEligible( *record ) + && record->pbrFallbackReason == expectedReason + && record->pbrPackedMaterialData + && !record->pbrSeparateMaterialData + && record->pbrHasAuthoredClassicFallback + && R_MaterialResourceTable_TextureBindingForSemantic( *record, MATERIAL_RESOURCE_TEXTURE_ALBEDO ) != NULL + && R_MaterialResourceTable_TextureBindingForSemantic( *record, MATERIAL_RESOURCE_TEXTURE_NORMAL ) != NULL + && R_MaterialResourceTable_TextureBindingForSemantic( *record, MATERIAL_RESOURCE_TEXTURE_ORM ) != NULL + && separateRecord != NULL && separateRecord->pbrResourceReady + && !separateRecord->pbrPackedMaterialData && separateRecord->pbrSeparateMaterialData + && scalarRecord != NULL && scalarRecord->pbrResourceReady + && !scalarRecord->pbrPackedMaterialData && !scalarRecord->pbrSeparateMaterialData + && !scalarRecord->hasPBRNormal && !scalarRecord->hasPBRORM + && explicitRecord != NULL && explicitRecord->pbrHasExplicitLegacyFallback + && explicitRecord->pbrUsesGeneratedLegacyFallback + && !explicitRecord->pbrUsesApproximateLegacyFallback + && unsupportedRecord != NULL + && unsupportedRecord->pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_WORKFLOW + && missingFallbackRecord != NULL && missingFallbackRecord->pbrLegacyFallbackMissing + && missingAlbedoRecord != NULL && !missingAlbedoRecord->hasPBRAlbedo + && missingAlbedoRecord->pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_ALBEDO + && mutableImageRecord != NULL && mutableImageRecord->hasPBRAlbedo + && !mutableImageRecord->pbrResourceReady + && mutableImageRecord->pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_IMAGE + && redundantExplicitRecord != NULL && redundantExplicitRecord->pbrHasExplicitLegacyFallback + && !redundantExplicitRecord->pbrUsesGeneratedLegacyFallback + && stats.pbrRecords == 9 + && stats.classicRecords == 0 + && stats.pbrResourceReadyRecords == 6 + && stats.pbrModernReadyRecords == 0 + && stats.pbrPackedMapRecords == 7 + && stats.pbrSeparateMapRecords == 1 + && stats.pbrAuthoredClassicFallbackRecords == 7 + && stats.pbrExplicitGeneratedFallbackRecords == 1 + && stats.pbrGeneratedFallbackRecords == 1 + && stats.pbrApproximateFallbackRecords == 0 + && stats.pbrMissingLegacyFallbackRecords == 1 + && stats.pbrMissingAlbedoMapRecords == 1 + && stats.pbrMissingNormalMapRecords == 1 + && stats.pbrMissingORMMapRecords == 2 + && stats.pbrFallbackMissingImage == 1 + && stats.pbrFallbackRecords == 9 + && pbrBindingsExcludedFromClassicTable + && stats.textureArrayTableDescriptors == 0; + if ( !ok ) { + common->Printf( + "RendererMaterialResourceTable self-test failed: PBR contract added=%d/%d/%d/%d/%d/%d/%d/%d/%d record=%d resource=%d modern=%d fallback=%s records=%d packed=%d separate=%d missingFallback=%d mapsMissing=%d/%d/%d\n", + added ? 1 : 0, + separateAdded ? 1 : 0, + scalarAdded ? 1 : 0, + explicitAdded ? 1 : 0, + unsupportedAdded ? 1 : 0, + missingFallbackAdded ? 1 : 0, + missingAlbedoAdded ? 1 : 0, + mutableImageAdded ? 1 : 0, + redundantExplicitAdded ? 1 : 0, + record != NULL ? 1 : 0, + record != NULL && record->pbrResourceReady ? 1 : 0, + record != NULL && record->pbrModernReady ? 1 : 0, + record != NULL ? MaterialResourcePBRFallbackReason_Name( record->pbrFallbackReason ) : "missing", + stats.pbrRecords, + stats.pbrPackedMapRecords, + stats.pbrSeparateMapRecords, + stats.pbrMissingLegacyFallbackRecords, + stats.pbrMissingAlbedoMapRecords, + stats.pbrMissingNormalMapRecords, + stats.pbrMissingORMMapRecords ); + } + // The table owns no declaration lifetime. Remove every retained synthetic + // pointer and invalidate the hash before releasing the temporary material, + // including failure exits that callers may inspect afterward. + for ( int i = 0; i < stats.records; ++i ) { + rg_materialResourceTable.records[i].material = NULL; + } + DeclManager_FreeAllocatedDecl( materialDecl ); + R_MaterialResourceTable_ResetFrameStats(); + rg_materialResourceTable.maxClassicTextureUnits = savedMaxClassicTextureUnits; + if ( ok ) { + common->Printf( "RendererMaterialResourceTable PBR contract self-test passed\n" ); + } + return ok; +} + static bool R_MaterialResourceTable_RunSyntheticRecordSelfTest( void ) { R_MaterialResourceTable_ResetFrameStats(); rg_materialResourceTable.stats.prepared = true; @@ -1409,6 +1993,7 @@ static bool R_MaterialResourceTable_RunSyntheticRecordSelfTest( void ) { const materialResourceTableStats_t &stats = R_MaterialResourceTable_Stats(); if ( stats.records != 6 + || stats.classicRecords != 6 || stats.opaqueRecords != 2 || stats.perforatedRecords != 1 || stats.translucentRecords != 1 @@ -1417,8 +2002,9 @@ static bool R_MaterialResourceTable_RunSyntheticRecordSelfTest( void ) { || stats.alphaTestRecords != 1 || stats.fallbackMissingImage <= 0 ) { common->Printf( - "RendererMaterialResourceTable self-test failed: synthetic counts records=%d opaque=%d perforated=%d translucent=%d gui=%d post=%d alpha=%d missingFallback=%d\n", + "RendererMaterialResourceTable self-test failed: synthetic counts records=%d classic=%d opaque=%d perforated=%d translucent=%d gui=%d post=%d alpha=%d missingFallback=%d\n", stats.records, + stats.classicRecords, stats.opaqueRecords, stats.perforatedRecords, stats.translucentRecords, @@ -1459,8 +2045,15 @@ static bool R_MaterialResourceTable_RunSyntheticRecordSelfTest( void ) { } bool RendererMaterialResourceTable_RunSelfTest( void ) { - if ( !rg_materialResourceTable.stats.initialized || !rg_materialResourceTable.stats.available ) { - common->Printf( "RendererMaterialResourceTable self-test skipped: material resource table unavailable\n" ); + if ( !R_MaterialResourceTable_RunPBRContractSelfTest() ) { + return false; + } + if ( !rg_materialResourceTable.stats.initialized ) { + common->Printf( "RendererMaterialResourceTable full self-test skipped: material resource table uninitialized\n" ); + return true; + } + if ( !rg_materialResourceTable.stats.available ) { + common->Printf( "RendererMaterialResourceTable full self-test skipped: scene packets unavailable\n" ); return true; } @@ -1511,13 +2104,15 @@ bool RendererMaterialResourceTable_RunSelfTest( void ) { const int expectedRecords = tr.defaultMaterial != NULL ? 1 : 0; if ( stats.sourceMaterialRecords != materialRecordCount || stats.records != expectedRecords + || stats.classicRecords != expectedRecords || stats.drawPacketReferences != ( tr.defaultMaterial != NULL ? drawPacketCount : 0 ) || stats.overflow ) { common->Printf( - "RendererMaterialResourceTable self-test failed: packet build mismatch source=%d/%d records=%d expected=%d drawRefs=%d overflow=%d\n", + "RendererMaterialResourceTable self-test failed: packet build mismatch source=%d/%d records=%d classic=%d expected=%d drawRefs=%d overflow=%d\n", stats.sourceMaterialRecords, materialRecordCount, stats.records, + stats.classicRecords, expectedRecords, stats.drawPacketReferences, stats.overflow ? 1 : 0 ); diff --git a/src/renderer/MaterialResourceTable.h b/src/renderer/MaterialResourceTable.h index e1d2d28e..a5e9b4b2 100644 --- a/src/renderer/MaterialResourceTable.h +++ b/src/renderer/MaterialResourceTable.h @@ -21,7 +21,11 @@ */ const int MATERIAL_RESOURCE_TABLE_MAX_RECORDS = SCENE_PACKET_MAX_MATERIAL_RECORDS; -const int MATERIAL_RESOURCE_TABLE_MAX_TEXTURE_BINDINGS = 8; +// GL 3.3 guarantees at least 16 fragment texture units. Keep enough record +// slots for a dual-authored material's classic fallback plus the initial PBR +// semantics. The descriptor-array capacity remains 12 so units 12..15 stay +// reserved for the existing shadow contract. +const int MATERIAL_RESOURCE_TABLE_MAX_TEXTURE_BINDINGS = 16; const int MATERIAL_RESOURCE_TABLE_TEXTURE_ARRAY_CAPACITY = 12; enum materialResourceBlendMode_t { @@ -42,9 +46,30 @@ enum materialResourceTextureSemantic_t { MATERIAL_RESOURCE_TEXTURE_EMISSIVE, MATERIAL_RESOURCE_TEXTURE_GUI, MATERIAL_RESOURCE_TEXTURE_POST_PROCESS, + MATERIAL_RESOURCE_TEXTURE_ALBEDO, + MATERIAL_RESOURCE_TEXTURE_NORMAL, + MATERIAL_RESOURCE_TEXTURE_ORM, + MATERIAL_RESOURCE_TEXTURE_METALLIC, + MATERIAL_RESOURCE_TEXTURE_ROUGHNESS, + MATERIAL_RESOURCE_TEXTURE_AO, + MATERIAL_RESOURCE_TEXTURE_EMISSIVE_PBR, MATERIAL_RESOURCE_TEXTURE_COUNT }; +enum materialResourcePBRFallbackReason_t { + MATERIAL_RESOURCE_PBR_FALLBACK_NONE = 0, + MATERIAL_RESOURCE_PBR_FALLBACK_DISABLED, + MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_WORKFLOW, + MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_MATERIAL_CLASS, + MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_ALBEDO, + MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_NORMAL_FORMAT, + MATERIAL_RESOURCE_PBR_FALLBACK_CONFLICTING_LAYOUT, + MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_IMAGE, + MATERIAL_RESOURCE_PBR_FALLBACK_CLASSIC_FEATURE, + MATERIAL_RESOURCE_PBR_FALLBACK_TOO_MANY_TEXTURES, + MATERIAL_RESOURCE_PBR_FALLBACK_SHADER_PATH_UNAVAILABLE +}; + enum materialResourceSortGroup_t { MATERIAL_RESOURCE_SORT_UNKNOWN = 0, MATERIAL_RESOURCE_SORT_SUBVIEW, @@ -173,6 +198,31 @@ typedef struct materialResourceTableRecord_s { bool hasSpecular; bool hasEmissive; bool hasPostProcess; + bool hasPBR; + bool pbrResourceReady; + bool pbrModernReady; + int pbrWorkflow; + int pbrNormalFormat; + materialResourcePBRFallbackReason_t pbrFallbackReason; + bool hasPBRAlbedo; + bool hasPBRNormal; + bool hasPBRORM; + bool hasPBRMetallic; + bool hasPBRRoughness; + bool hasPBRAO; + bool hasPBREmissive; + bool pbrPackedMaterialData; + bool pbrSeparateMaterialData; + bool pbrHasAuthoredClassicFallback; + bool pbrHasExplicitLegacyFallback; + bool pbrUsesGeneratedLegacyFallback; + bool pbrUsesApproximateLegacyFallback; + bool pbrLegacyFallbackMissing; + int pbrMetallicRegister; + int pbrRoughnessRegister; + int pbrAORegister; + int pbrNormalScaleRegister; + int pbrEmissiveColorRegisters[3]; bool hasConditionRegisters; bool hasTextureMatrix; bool hasVertexColor; @@ -215,6 +265,7 @@ typedef struct materialResourceTableStats_s { int sourceMaterialRecords; int drawPacketReferences; int records; + int classicRecords; int opaqueRecords; int perforatedRecords; int translucentRecords; @@ -254,6 +305,30 @@ typedef struct materialResourceTableStats_s { int fallbackVertexColor; int fallbackPolygonOffset; int fallbackTooManyTextures; + int pbrRecords; + int pbrResourceReadyRecords; + int pbrModernReadyRecords; + int pbrPackedMapRecords; + int pbrSeparateMapRecords; + int pbrAuthoredClassicFallbackRecords; + int pbrExplicitGeneratedFallbackRecords; + int pbrGeneratedFallbackRecords; + int pbrApproximateFallbackRecords; + int pbrMissingLegacyFallbackRecords; + int pbrMissingAlbedoMapRecords; + int pbrMissingNormalMapRecords; + int pbrMissingORMMapRecords; + int pbrFallbackRecords; + int pbrFallbackDisabled; + int pbrFallbackUnsupportedWorkflow; + int pbrFallbackUnsupportedMaterialClass; + int pbrFallbackMissingAlbedo; + int pbrFallbackMissingNormalFormat; + int pbrFallbackConflictingLayout; + int pbrFallbackMissingImage; + int pbrFallbackClassicFeature; + int pbrFallbackTooManyTextures; + int pbrFallbackShaderPathUnavailable; int debugStringTruncations; char debugStringTruncationSource[64]; char lastFailure[96]; @@ -265,6 +340,7 @@ void R_MaterialResourceTable_PrepareFrame( const idScenePacketFrame &packetFrame const materialResourceTableStats_t &R_MaterialResourceTable_Stats( void ); const materialResourceTableRecord_t *R_MaterialResourceTable_RecordForIndex( int tableIndex ); const materialResourceTableRecord_t *R_MaterialResourceTable_FindRecordForMaterial( const idMaterial *material ); +bool R_MaterialResourceTable_ClassicModernPathEligible( const materialResourceTableRecord_t &record ); const unsigned int *R_MaterialResourceTable_TextureArrayTable( int &count ); int R_MaterialResourceTable_TextureArrayTableIndexForHandle( unsigned int textureHandle ); const char *MaterialResourceBlendMode_Name( materialResourceBlendMode_t blendMode ); @@ -272,6 +348,7 @@ const char *MaterialResourceTextureSemantic_Name( materialResourceTextureSemanti unsigned int MaterialResourceTextureSemantic_Bit( materialResourceTextureSemantic_t semantic ); const materialResourceTextureBinding_t *R_MaterialResourceTable_TextureBindingForSemantic( const materialResourceTableRecord_t &record, materialResourceTextureSemantic_t semantic ); const char *MaterialResourceFallbackReason_Name( materialResourceFallbackReason_t reason ); +const char *MaterialResourcePBRFallbackReason_Name( materialResourcePBRFallbackReason_t reason ); void R_MaterialResourceTable_PrintGfxInfo( void ); void R_MaterialResourceTable_DumpLatest( void ); bool RendererMaterialResourceTable_RunSelfTest( void ); diff --git a/src/renderer/ModernGLDrawPlan.cpp b/src/renderer/ModernGLDrawPlan.cpp index 1a6e862d..de2740d7 100644 --- a/src/renderer/ModernGLDrawPlan.cpp +++ b/src/renderer/ModernGLDrawPlan.cpp @@ -401,6 +401,11 @@ bool idModernGLDrawPlan::Build( const idScenePacketFrame &packetFrame, const idR stats.missingMaterialTableDraws++; continue; } + if ( !R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord ) ) { + stats.fallbackDraws++; + stats.materialFallbackDraws++; + continue; + } const bool forwardPlusCandidate = R_ModernGLDrawPlan_ShouldUseForwardPlus( context, draw, *materialRecord, pipeline, shaderKind ); if ( materialRecord->fallbackReason != MATERIAL_RESOURCE_FALLBACK_NONE && ( !forwardPlusCandidate || !R_ModernGLDrawPlan_MaterialFallbackAllowedForForwardPlus( draw, *materialRecord ) ) ) { diff --git a/src/renderer/ModernGLExecutor.cpp b/src/renderer/ModernGLExecutor.cpp index ddb2ca2d..ea7ea190 100644 --- a/src/renderer/ModernGLExecutor.cpp +++ b/src/renderer/ModernGLExecutor.cpp @@ -1899,6 +1899,11 @@ static void R_ModernGLExecutor_RecordPacketFallbackBlockers( const idScenePacket R_ModernGLExecutor_SetOwnershipBlocker( stats, "draw", viewIndex, draw.passCategory, i, "material", "missing-material-record" ); continue; } + if ( !R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord ) ) { + stats.modernVisibleMaterialFallbackDraws++; + R_ModernGLExecutor_SetOwnershipBlocker( stats, "material", viewIndex, draw.passCategory, materialRecord->materialId, materialRecord->materialName, "pbr-dedicated-pipeline-unavailable" ); + continue; + } if ( R_ModernGLExecutor_DrawPacketNeedsLegacySceneGui( draw, *materialRecord ) && !legacyFeedbackSurface ) { stats.modernVisibleMaterialFallbackDraws++; R_ModernGLExecutor_SetOwnershipBlocker( stats, "material", viewIndex, draw.passCategory, materialRecord->materialId, materialRecord->materialName, "legacy-scene-gui" ); @@ -4464,7 +4469,9 @@ static void R_ModernGLExecutor_CountVisibleDepthFallback( const modernGLSubmitCo static bool R_ModernGLExecutor_VisibleDepthMaterialSupported( const modernGLSubmitCommand_t &command, modernGLExecutorStats_t &stats ) { const materialResourceTableRecord_t *materialRecord = R_ModernGLExecutor_MaterialRecordForCommand( command ); - if ( materialRecord == NULL || materialRecord->fallbackReason != MATERIAL_RESOURCE_FALLBACK_NONE ) { + if ( materialRecord == NULL + || !R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord ) + || materialRecord->fallbackReason != MATERIAL_RESOURCE_FALLBACK_NONE ) { stats.visibleDepthMaterialFallbackDraws++; return false; } @@ -5030,7 +5037,9 @@ static bool R_ModernGLExecutor_MaterialContractPromotable( const materialResourc static bool R_ModernGLExecutor_GBufferMaterialSupported( const modernGLSubmitCommand_t &command, modernGLExecutorStats_t &stats ) { const materialResourceTableRecord_t *materialRecord = R_ModernGLExecutor_MaterialRecordForCommand( command ); - if ( materialRecord == NULL || materialRecord->fallbackReason != MATERIAL_RESOURCE_FALLBACK_NONE ) { + if ( materialRecord == NULL + || !R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord ) + || materialRecord->fallbackReason != MATERIAL_RESOURCE_FALLBACK_NONE ) { stats.opaqueGBufferMaterialFallbackDraws++; return false; } @@ -5687,6 +5696,10 @@ static bool R_ModernGLExecutor_ForwardPlusMaterialSupported( const modernGLSubmi stats.forwardPlusMaterialFallbackDraws++; return false; } + if ( !R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord ) ) { + stats.forwardPlusMaterialFallbackDraws++; + return false; + } const bool decalMaterial = command.forwardPlusDecal; if ( materialRecord->fallbackReason != MATERIAL_RESOURCE_FALLBACK_NONE && ( !decalMaterial || !R_ModernGLExecutor_DecalFallbackAllowedForForwardPlus( *materialRecord, command.drawPlanEntry != NULL ? command.drawPlanEntry->drawPacket : NULL ) ) ) { diff --git a/src/renderer/ModernGLSubmitPlan.cpp b/src/renderer/ModernGLSubmitPlan.cpp index 5c1a3f33..4d4a814d 100644 --- a/src/renderer/ModernGLSubmitPlan.cpp +++ b/src/renderer/ModernGLSubmitPlan.cpp @@ -734,11 +734,16 @@ bool idModernGLSubmitPlan::AddCommand( const modernGLDrawPlanEntry_t &entry ) { stats.fallbackDraws++; return false; } + const materialResourceTableRecord_t *materialRecord = R_MaterialResourceTable_RecordForIndex( entry.materialTableIndex ); + if ( materialRecord != NULL && !R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord ) ) { + stats.fallbackDraws++; + return false; + } modernGLSubmitCommand_t &command = commands[numCommands]; memset( &command, 0, sizeof( command ) ); command.drawPlanEntry = &entry; - command.materialRecord = R_MaterialResourceTable_RecordForIndex( entry.materialTableIndex ); + command.materialRecord = materialRecord; command.viewDef = draw->viewDef; command.passCategory = entry.passCategory; command.pipeline = entry.pipeline; @@ -774,7 +779,6 @@ bool idModernGLSubmitPlan::AddCommand( const modernGLDrawPlanEntry_t &entry ) { command.instanceRecordIndex = entry.instanceRecordIndex; command.originalSubmitOrder = numCommands; command.sortBucket = -1; - const materialResourceTableRecord_t *materialRecord = command.materialRecord; command.blendMode = materialRecord != NULL ? materialRecord->blendMode : MATERIAL_RESOURCE_BLEND_OPAQUE; command.cullType = materialRecord != NULL ? materialRecord->cullType : CT_FRONT_SIDED; command.materialStableId = entry.materialStableId; diff --git a/src/renderer/RenderModuleAPI.h b/src/renderer/RenderModuleAPI.h index 7eeeb7ab..7b1f9d8c 100644 --- a/src/renderer/RenderModuleAPI.h +++ b/src/renderer/RenderModuleAPI.h @@ -46,9 +46,11 @@ // surface kind so the engine creates the window for the right API, and // the window services gain SDL-mediated Vulkan instance-extension and // surface creation (the module never links SDL) -// Version 7 keeps stale renderer modules from consuming the extended -// renderEntity_t presentation contract (flat diffuse colour and sweep flags). -#define RENDER_API_VERSION 7 +// 7 - Extended renderEntity_t presentation contract (flat diffuse colour +// and sweep flags) +// 8 - Append-only idRenderSystem slot for coherent full-frame capture with +// CPU center-crop/resampling to caller-selected output dimensions +#define RENDER_API_VERSION 8 #define RENDER_API_ENTRY_POINT "GetRenderAPI" class idSys; diff --git a/src/renderer/RenderSystem.cpp b/src/renderer/RenderSystem.cpp index a99e4779..50910929 100644 --- a/src/renderer/RenderSystem.cpp +++ b/src/renderer/RenderSystem.cpp @@ -1755,11 +1755,111 @@ CaptureRenderToFile ============== */ void idRenderSystemLocal::CaptureRenderToFile( const char *fileName, bool fixAlpha ) { + CaptureRenderToFile( fileName, fixAlpha, 0, 0 ); +} + +static const int MAX_CAPTURE_DIMENSION = 16384; +static const int64 MAX_CAPTURE_PIXELS = 33554432; + +/* +============== +R_ResampleCaptureToAspectRGBA + +Preserves the bottom-up row order returned by glReadPixels. R_WriteTGA's +flipVertical argument then describes those rows exactly as it does for an +unscaled capture. +============== +*/ +static byte *R_ResampleCaptureToAspectRGBA( const byte *source, int sourceWidth, int sourceHeight, + int outputWidth, int outputHeight ) { + int cropX = 0; + int cropY = 0; + int cropWidth = sourceWidth; + int cropHeight = sourceHeight; + + const int64 sourceAspectProduct = (int64)sourceWidth * outputHeight; + const int64 outputAspectProduct = (int64)sourceHeight * outputWidth; + if ( sourceAspectProduct > outputAspectProduct ) { + cropWidth = (int)( (int64)sourceHeight * outputWidth / outputHeight ); + cropWidth = Max( cropWidth, 1 ); + cropX = ( sourceWidth - cropWidth ) / 2; + } else if ( sourceAspectProduct < outputAspectProduct ) { + cropHeight = (int)( (int64)sourceWidth * outputHeight / outputWidth ); + cropHeight = Max( cropHeight, 1 ); + cropY = ( sourceHeight - cropHeight ) / 2; + } + + const size_t outputPixels = (size_t)outputWidth * (size_t)outputHeight; + byte *output = (byte *)R_StaticAlloc( outputPixels * 4 ); + const int cropMaxX = cropX + cropWidth - 1; + const int cropMaxY = cropY + cropHeight - 1; + + for ( int y = 0; y < outputHeight; y++ ) { + const double sourceY = cropY + ( ( y + 0.5 ) * cropHeight / outputHeight ) - 0.5; + int y0 = (int)floor( sourceY ); + double yFraction = sourceY - y0; + if ( y0 < cropY ) { + y0 = cropY; + yFraction = 0.0; + } else if ( y0 >= cropMaxY ) { + y0 = cropMaxY; + yFraction = 0.0; + } + const int y1 = Min( y0 + 1, cropMaxY ); + byte *outputRow = output + (size_t)y * (size_t)outputWidth * 4; + + for ( int x = 0; x < outputWidth; x++ ) { + const double sourceX = cropX + ( ( x + 0.5 ) * cropWidth / outputWidth ) - 0.5; + int x0 = (int)floor( sourceX ); + double xFraction = sourceX - x0; + if ( x0 < cropX ) { + x0 = cropX; + xFraction = 0.0; + } else if ( x0 >= cropMaxX ) { + x0 = cropMaxX; + xFraction = 0.0; + } + const int x1 = Min( x0 + 1, cropMaxX ); + + const byte *topLeft = source + ( (size_t)y0 * sourceWidth + x0 ) * 4; + const byte *topRight = source + ( (size_t)y0 * sourceWidth + x1 ) * 4; + const byte *bottomLeft = source + ( (size_t)y1 * sourceWidth + x0 ) * 4; + const byte *bottomRight = source + ( (size_t)y1 * sourceWidth + x1 ) * 4; + byte *destination = outputRow + x * 4; + + for ( int channel = 0; channel < 4; channel++ ) { + const double top = topLeft[channel] + ( topRight[channel] - topLeft[channel] ) * xFraction; + const double bottom = bottomLeft[channel] + ( bottomRight[channel] - bottomLeft[channel] ) * xFraction; + destination[channel] = (byte)( top + ( bottom - top ) * yFraction + 0.5 ); + } + } + } + + return output; +} + +void idRenderSystemLocal::CaptureRenderToFile( const char *fileName, bool fixAlpha, + int outputWidth, int outputHeight ) { if ( !glConfig.isInitialized ) { return; } renderCrop_t *rc = &renderCrops[currentRenderCrop]; + const bool resample = outputWidth != 0 || outputHeight != 0; + const int64 sourcePixelCount = (int64)rc->width * rc->height; + if ( rc->width < 1 || rc->height < 1 || + rc->width > MAX_CAPTURE_DIMENSION || rc->height > MAX_CAPTURE_DIMENSION || + sourcePixelCount > MAX_CAPTURE_PIXELS ) { + common->Warning( "CaptureRenderToFile: invalid source dimensions %d x %d", rc->width, rc->height ); + return; + } + const int64 outputPixelCount = (int64)outputWidth * outputHeight; + if ( resample && ( outputWidth < 1 || outputHeight < 1 || + outputWidth > MAX_CAPTURE_DIMENSION || outputHeight > MAX_CAPTURE_DIMENSION || + outputPixelCount > MAX_CAPTURE_PIXELS ) ) { + common->Warning( "CaptureRenderToFile: invalid output dimensions %d x %d", outputWidth, outputHeight ); + return; + } guiModel->EmitFullScreen(); guiModel->Clear(); @@ -1769,26 +1869,45 @@ void idRenderSystemLocal::CaptureRenderToFile( const char *fileName, bool fixAlp glReadBuffer( GL_BACK ); - // include extra space for OpenGL padding to word boundaries - int c = ( rc->width + 3 ) * rc->height; - byte *data = (byte *)R_StaticAlloc( c * 3 ); + // GL_RGB readback rows use the default four-byte pack alignment. Convert + // each row independently so the padding never becomes pixel data. + const int sourceStride = ( rc->width * 3 + 3 ) & ~3; + const size_t sourceBytes = (size_t)sourceStride * (size_t)rc->height; + byte *data = (byte *)R_StaticAlloc( sourceBytes ); + memset( data, 0, sourceBytes ); glReadPixels( rc->x, rc->y, rc->width, rc->height, GL_RGB, GL_UNSIGNED_BYTE, data ); tr.takingScreenshot = wasTakingScreenshot; - byte *data2 = (byte *)R_StaticAlloc( c * 4 ); + byte *data2 = (byte *)R_StaticAlloc( (size_t)sourcePixelCount * 4 ); - for ( int i = 0 ; i < c ; i++ ) { - data2[ i * 4 ] = data[ i * 3 ]; - data2[ i * 4 + 1 ] = data[ i * 3 + 1 ]; - data2[ i * 4 + 2 ] = data[ i * 3 + 2 ]; - data2[ i * 4 + 3 ] = 0xff; + for ( int y = 0; y < rc->height; y++ ) { + const byte *sourceRow = data + (size_t)y * sourceStride; + byte *destinationRow = data2 + (size_t)y * rc->width * 4; + for ( int x = 0; x < rc->width; x++ ) { + destinationRow[ x * 4 ] = sourceRow[ x * 3 ]; + destinationRow[ x * 4 + 1 ] = sourceRow[ x * 3 + 1 ]; + destinationRow[ x * 4 + 2 ] = sourceRow[ x * 3 + 2 ]; + destinationRow[ x * 4 + 3 ] = 0xff; + } } - - R_WriteTGA( fileName, data2, rc->width, rc->height, true ); - R_StaticFree( data ); - R_StaticFree( data2 ); + data = NULL; + + byte *outputData = data2; + int capturedWidth = rc->width; + int capturedHeight = rc->height; + if ( resample ) { + outputData = R_ResampleCaptureToAspectRGBA( data2, rc->width, rc->height, + outputWidth, outputHeight ); + R_StaticFree( data2 ); + data2 = NULL; + capturedWidth = outputWidth; + capturedHeight = outputHeight; + } + + R_WriteTGA( fileName, outputData, capturedWidth, capturedHeight, true ); + R_StaticFree( outputData ); } diff --git a/src/renderer/RenderSystem.h b/src/renderer/RenderSystem.h index adcaa182..2e19b0e3 100644 --- a/src/renderer/RenderSystem.h +++ b/src/renderer/RenderSystem.h @@ -636,6 +636,10 @@ class idRenderSystem { // fogDistance is how far light travels through this liquid before it is fully absorbed, in // world units - the knob that separates clear water from lava you cannot see a foot into. virtual bool SetUnderwaterView( float amount, const idVec3 &tint, float fogDistance ) = 0; + + // Capture the coherent current crop, then center-crop and resample it on the CPU. + // Kept at the end of the interface so existing render-system vtable slots remain stable. + virtual void CaptureRenderToFile( const char *fileName, bool fixAlpha, int outputWidth, int outputHeight ) = 0; }; extern idRenderSystem * renderSystem; diff --git a/src/renderer/RenderSystem_init.cpp b/src/renderer/RenderSystem_init.cpp index c8f1b8e7..0ec167c6 100644 --- a/src/renderer/RenderSystem_init.cpp +++ b/src/renderer/RenderSystem_init.cpp @@ -219,6 +219,10 @@ idCVar r_useLightPortalFlow( "r_useLightPortalFlow", "1", CVAR_RENDERER | CVAR_B idCVar r_multiSamples( "r_multiSamples", "0", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_INTEGER, "MSAA sample count: 0 = off, 2/4/8/16 = supported quality steps", 0, 16, idCmdSystem::ArgCompletion_String ); idCVar r_postAA( "r_postAA", "0", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_INTEGER, "post AA mode: 0 = off, 1 = SMAA 1x medium, 2 = SMAA 1x high, 3 = SMAA 1x ultra, 4 = SMAA 1x colour-edge prototype", 0, 4, idCmdSystem::ArgCompletion_Integer<0,4> ); idCVar r_postAAStatePoisonTest( "r_postAAStatePoisonTest", "0", CVAR_RENDERER | CVAR_BOOL, "intentionally dirty GL texture/client state before SMAA post-AA draws for validation" ); +idCVar r_pbrMaterials( "r_pbrMaterials", "0", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_BOOL, "allow explicitly PBR-authored materials on supported modern renderer paths" ); +idCVar r_pbrGeneratedLegacyFallback( "r_pbrGeneratedLegacyFallback", "1", CVAR_RENDERER | CVAR_BOOL, "allow development-only classic fallback generation for PBR-only materials" ); +idCVar r_pbrDebug( "r_pbrDebug", "0", CVAR_RENDERER | CVAR_INTEGER, "PBR debug view: 0=off, 1=albedo, 2=normal, 3=metallic, 4=roughness, 5=AO, 6=emissive, 7=fallback", 0, 7, idCmdSystem::ArgCompletion_Integer<0,7> ); +idCVar r_pbrInferFromLegacyMaterials( "r_pbrInferFromLegacyMaterials", "0", CVAR_RENDERER | CVAR_BOOL, "research-only legacy material reinterpretation; never used for stock rendering by default" ); idCVar r_bloom( "r_bloom", "0", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_BOOL, "enable bloom post-process" ); idCVar r_bloomThreshold( "r_bloomThreshold", "0.45", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_FLOAT, "bloom bright-pass threshold in scene-referred units", 0.0f, 16.0f ); idCVar r_bloomSoftKnee( "r_bloomSoftKnee", "0.15", CVAR_RENDERER | CVAR_ARCHIVE | CVAR_FLOAT, "relative bloom soft-threshold knee", 0.0f, 1.0f ); @@ -807,6 +811,15 @@ static void R_RendererMaterialResourceTableSelfTest_f( const idCmdArgs &args ) { } } +static void R_RendererPBRMaterialSelfTest_f( const idCmdArgs &args ) { + (void)args; + if ( !R_PBRMaterialParserSelfTest() ) { + common->Warning( "Renderer PBR material parser self-test failed" ); + return; + } + common->Printf( "RendererPBRMaterial self-test passed\n" ); +} + static void R_RendererMaterialResourceTableDump_f( const idCmdArgs &args ) { (void)args; R_MaterialResourceTable_DumpLatest(); @@ -3688,6 +3701,12 @@ void GfxInfo_f( const idCmdArgs &args ) { RENDER_GRAPH_MAX_RESOURCE_ACCESSES ); R_RenderGraphResources_PrintGfxInfo(); R_MaterialResourceTable_PrintGfxInfo(); + common->Printf( + "PBR materials: parser=1 modernLighting=0 enabled=%d generatedLegacyFallback=%d inferLegacy=%d debug=%d\n", + r_pbrMaterials.GetBool() ? 1 : 0, + r_pbrGeneratedLegacyFallback.GetBool() ? 1 : 0, + r_pbrInferFromLegacyMaterials.GetBool() ? 1 : 0, + r_pbrDebug.GetInteger() ); R_ModernGLExecutor_PrintGfxInfo(); R_ModernClusteredLighting_PrintGfxInfo(); R_ModernShadowPlanner_PrintGfxInfo(); @@ -4079,6 +4098,7 @@ void R_InitCommands( void ) { cmdSystem->AddCommand( "rendererRenderGraphResourceSelfTest", R_RendererRenderGraphResourceSelfTest_f, CMD_FL_RENDERER, "run renderer graph resource owner self tests" ); cmdSystem->AddCommand( "rendererRenderGraphResourceDump", R_RendererRenderGraphResourceDump_f, CMD_FL_RENDERER, "dump the latest renderer graph resource handles" ); cmdSystem->AddCommand( "rendererMaterialResourceTableSelfTest", R_RendererMaterialResourceTableSelfTest_f, CMD_FL_RENDERER, "run renderer material resource-table self tests" ); + cmdSystem->AddCommand( "rendererPBRMaterialSelfTest", R_RendererPBRMaterialSelfTest_f, CMD_FL_RENDERER, "run PBR material parser and classic fallback self tests" ); cmdSystem->AddCommand( "rendererMaterialResourceTableDump", R_RendererMaterialResourceTableDump_f, CMD_FL_RENDERER, "dump the latest renderer material resource table" ); cmdSystem->AddCommand( "rendererGeometryResourceSelfTest", R_RendererGeometryResourceSelfTest_f, CMD_FL_RENDERER, "run renderer geometry and instance packet self tests" ); cmdSystem->AddCommand( "rendererGLStateCacheSelfTest", R_RendererGLStateCacheSelfTest_f, CMD_FL_RENDERER, "run renderer GL state-cache self tests" ); diff --git a/src/renderer/ScenePackets.cpp b/src/renderer/ScenePackets.cpp index a9fae7f0..d4fc5037 100644 --- a/src/renderer/ScenePackets.cpp +++ b/src/renderer/ScenePackets.cpp @@ -501,6 +501,29 @@ int idScenePacketFrame::FindOrAddMaterialRecord( const drawSurf_t *drawSurf ) { } record.normalImage = R_ScenePackets_FirstStageImage( material, SL_BUMP ); record.specularImage = R_ScenePackets_FirstStageImage( material, SL_SPECULAR ); + if ( material->HasPBR() ) { + const pbrMaterialInfo_t &pbr = material->GetPBRInfo(); + record.hasPBR = true; + record.pbrWorkflow = static_cast( pbr.workflow ); + record.pbrNormalFormat = static_cast( pbr.normalFormat ); + record.pbrAlbedoImage = pbr.albedo.present ? pbr.albedo.image : NULL; + record.pbrNormalImage = pbr.normal.present ? pbr.normal.image : NULL; + record.pbrORMImage = pbr.orm.present ? pbr.orm.image : NULL; + record.pbrMetallicImage = pbr.metallic.present ? pbr.metallic.image : NULL; + record.pbrRoughnessImage = pbr.roughness.present ? pbr.roughness.image : NULL; + record.pbrAOImage = pbr.ao.present ? pbr.ao.image : NULL; + record.pbrEmissiveImage = pbr.emissive.present ? pbr.emissive.image : NULL; + record.pbrHasAuthoredClassicFallback = pbr.hasAuthoredClassicFallback; + record.pbrHasExplicitLegacyFallback = pbr.hasExplicitLegacyFallback; + record.pbrUsesGeneratedLegacyFallback = pbr.usesGeneratedLegacyFallback; + record.pbrUsesApproximateLegacyFallback = pbr.usesApproximateLegacyFallback; + record.pbrLegacyFallbackMissing = pbr.legacyFallbackMissing; + record.pbrMetallicRegister = pbr.metallicRegister; + record.pbrRoughnessRegister = pbr.roughnessRegister; + record.pbrAORegister = pbr.aoRegister; + record.pbrNormalScaleRegister = pbr.normalScaleRegister; + memcpy( record.pbrEmissiveColorRegisters, pbr.emissiveColorRegisters, sizeof( record.pbrEmissiveColorRegisters ) ); + } record.resourceTableIndex = material->Index(); record.permutation.materialClass = R_ScenePackets_MaterialClassForDrawSurf( drawSurf ); record.permutation.lightingMode = @@ -1818,13 +1841,15 @@ void R_ScenePackets_LogIfVerbose( const idScenePacketFrame &packetFrame ) { for ( int i = 0; i < materialRecordLogCount; ++i ) { const materialResourceRecord_t &record = packetFrame.MaterialRecord( i ); common->Printf( - "scenePackets material[%d]=%s class=%s diffuse=%s normal=%s specular=%s table=%d\n", + "scenePackets material[%d]=%s class=%s diffuse=%s normal=%s specular=%s pbr=%d workflow=%d table=%d\n", i, record.material ? record.material->GetName() : "", RendererMaterialClass_Name( static_cast( record.permutation.materialClass ) ), record.diffuseImage ? record.diffuseImage->GetName() : "", record.normalImage ? record.normalImage->GetName() : "", record.specularImage ? record.specularImage->GetName() : "", + record.hasPBR ? 1 : 0, + record.pbrWorkflow, record.resourceTableIndex ); } const int geometryRecordLogCount = Min( packetFrame.NumGeometryRecords(), 8 ); @@ -2045,6 +2070,73 @@ bool RendererScenePacket_RunSelfTest( void ) { return false; } - common->Printf( "RendererScenePacket self-test passed (backend, frontend, post commands)\n" ); + // Exercise the actual material-record capture path with opt-in PBR metadata. + // The ambient stage makes this a drawable synthetic surface while the absent + // bump+diffuse interaction deliberately proves the missing-fallback bit. + static const char pbrPacketDeclaration[] = + "material _pbr_scene_packet_selftest {\n" + " {\n" + " map _white\n" + " }\n" + " pbr {\n" + " workflow metallicRoughness\n" + " albedoMap _white\n" + " metallic 0.35\n" + " roughness 0.65\n" + " ao 0.85\n" + " autoLegacyFallback 0\n" + " }\n" + "}\n"; + idDecl *pbrPacketDecl = declManager->AllocateDecl( DECL_MATERIAL ); + if ( pbrPacketDecl == NULL ) { + common->Printf( "RendererScenePacket self-test failed: PBR declaration allocation\n" ); + return false; + } + idMaterial *pbrPacketMaterial = static_cast( pbrPacketDecl ); + bool pbrPacketValid = pbrPacketMaterial->Parse( + pbrPacketDeclaration, + idLib::SizeToInt( sizeof( pbrPacketDeclaration ) - 1, "RendererScenePacket PBR self-test" ) ); + if ( pbrPacketValid ) { + drawSurf_t pbrDrawSurf; + memset( &pbrDrawSurf, 0, sizeof( pbrDrawSurf ) ); + pbrDrawSurf.material = pbrPacketMaterial; + pbrDrawSurf.sort = pbrPacketMaterial->GetSort(); + { + idScenePacketFrame pbrPacketFrame; + pbrPacketValid = pbrPacketFrame.AddDrawPacket( &pbrDrawSurf, RENDER_PASS_AMBIENT, 0 ); + const drawPacket_t *pbrDrawPacket = pbrPacketValid && pbrPacketFrame.NumDrawPackets() == 1 + ? &pbrPacketFrame.DrawPacket( 0 ) + : NULL; + const materialResourceRecord_t *pbrRecord = pbrDrawPacket != NULL && pbrDrawPacket->materialRecordIndex >= 0 + ? &pbrPacketFrame.MaterialRecord( pbrDrawPacket->materialRecordIndex ) + : NULL; + const pbrMaterialInfo_t &pbr = pbrPacketMaterial->GetPBRInfo(); + const int registerCount = pbrPacketMaterial->GetNumRegisters(); + auto validRegister = [&]( int index ) -> bool { return index >= 0 && index < registerCount; }; + pbrPacketValid = pbrRecord != NULL + && pbrDrawPacket->materialRecord == pbrRecord + && pbrRecord->material == pbrPacketMaterial + && pbrRecord->hasPBR + && pbrRecord->pbrWorkflow == static_cast( pbr.workflow ) + && pbrRecord->pbrAlbedoImage == pbr.albedo.image + && pbrRecord->pbrLegacyFallbackMissing + && pbrRecord->pbrLegacyFallbackMissing == pbr.legacyFallbackMissing + && pbrRecord->pbrMetallicRegister == pbr.metallicRegister + && pbrRecord->pbrRoughnessRegister == pbr.roughnessRegister + && pbrRecord->pbrAORegister == pbr.aoRegister + && pbrRecord->pbrNormalScaleRegister == pbr.normalScaleRegister + && validRegister( pbrRecord->pbrMetallicRegister ) + && validRegister( pbrRecord->pbrRoughnessRegister ) + && validRegister( pbrRecord->pbrAORegister ) + && validRegister( pbrRecord->pbrNormalScaleRegister ); + } + } + DeclManager_FreeAllocatedDecl( pbrPacketDecl ); + if ( !pbrPacketValid ) { + common->Printf( "RendererScenePacket self-test failed: PBR packet metadata propagation\n" ); + return false; + } + + common->Printf( "RendererScenePacket self-test passed (backend, frontend, post commands, PBR metadata)\n" ); return true; } diff --git a/src/renderer/ScenePackets.h b/src/renderer/ScenePackets.h index 67fb8ba6..240a4da2 100644 --- a/src/renderer/ScenePackets.h +++ b/src/renderer/ScenePackets.h @@ -97,6 +97,26 @@ typedef struct materialResourceRecord_s { const idImage *diffuseImage; const idImage *normalImage; const idImage *specularImage; + const idImage *pbrAlbedoImage; + const idImage *pbrNormalImage; + const idImage *pbrORMImage; + const idImage *pbrMetallicImage; + const idImage *pbrRoughnessImage; + const idImage *pbrAOImage; + const idImage *pbrEmissiveImage; + bool hasPBR; + int pbrWorkflow; + int pbrNormalFormat; + bool pbrHasAuthoredClassicFallback; + bool pbrHasExplicitLegacyFallback; + bool pbrUsesGeneratedLegacyFallback; + bool pbrUsesApproximateLegacyFallback; + bool pbrLegacyFallbackMissing; + int pbrMetallicRegister; + int pbrRoughnessRegister; + int pbrAORegister; + int pbrNormalScaleRegister; + int pbrEmissiveColorRegisters[3]; int resourceTableIndex; rendererPermutationKey_t permutation; } materialResourceRecord_t; diff --git a/src/renderer/tr_local.h b/src/renderer/tr_local.h index d77f1642..d179a369 100644 --- a/src/renderer/tr_local.h +++ b/src/renderer/tr_local.h @@ -794,6 +794,7 @@ class idRenderSystemLocal : public idRenderSystem { virtual void CropRenderSize( int width, int height, bool makePowerOfTwo = false, bool forceDimensions = false ); virtual void CaptureRenderToImage( const char *imageName ); virtual void CaptureRenderToFile( const char *fileName, bool fixAlpha ); + virtual void CaptureRenderToFile( const char *fileName, bool fixAlpha, int outputWidth, int outputHeight ); virtual void SetPortalSkyCaptureViewCallback( renderPortalSkyCaptureViewCallback_t callback ); virtual void UnCrop(); virtual void GetCardCaps( bool &oldCard, bool &nv10or20 ); @@ -980,6 +981,10 @@ extern idCVar r_windowHeight; // windowed mode height extern idCVar r_multiSamples; // number of antialiasing samples extern idCVar r_postAA; // post AA mode: 0 = off, 1/2/3 = SMAA medium/high/ultra, 4 = colour-edge prototype extern idCVar r_postAAStatePoisonTest; // intentionally dirty GL texture/client state before SMAA draws +extern idCVar r_pbrMaterials; // allow explicitly PBR-authored materials on modern paths +extern idCVar r_pbrGeneratedLegacyFallback; // allow development-only generated classic fallbacks +extern idCVar r_pbrDebug; // PBR attachment/fallback debug view +extern idCVar r_pbrInferFromLegacyMaterials; // research-only classic-material inference extern idCVar r_bloom; // enable bloom post-process extern idCVar r_bloomThreshold; // bloom bright-pass threshold extern idCVar r_bloomSoftKnee; // relative bloom soft threshold knee diff --git a/src/sys/URLPolicy.h b/src/sys/URLPolicy.h new file mode 100644 index 00000000..306e86ed --- /dev/null +++ b/src/sys/URLPolicy.h @@ -0,0 +1,292 @@ +/* +=========================================================================== + +openQ4 external URL validation helpers. +Copyright (C) 2026 DarkMatter Productions + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +=========================================================================== +*/ + +#ifndef __URL_POLICY_H__ +#define __URL_POLICY_H__ + +#include + +namespace idURLPolicy { + +// These helpers intentionally have no engine dependencies so every platform +// launcher and the native safety tests use the same fail-closed policy. + +// Includes the terminating NUL. Accepted URLs therefore contain at most 4095 +// bytes, matching the former macOS bound without allowing an unbounded scan. +const size_t MAX_URL_BYTES = 4096; + +inline char ASCIILower( const char value ) { + return ( value >= 'A' && value <= 'Z' ) ? static_cast( value + ( 'a' - 'A' ) ) : value; +} + +inline bool ASCIIStartsWith( const char *text, const char *prefix ) { + if ( text == NULL || prefix == NULL ) { + return false; + } + while ( *prefix != '\0' ) { + if ( *text == '\0' || ASCIILower( *text ) != ASCIILower( *prefix ) ) { + return false; + } + text++; + prefix++; + } + return true; +} + +inline bool ParsePort( const char *begin, const char *end ) { + if ( begin == NULL || end == NULL || begin == end || end < begin ) { + return false; + } + + unsigned int port = 0; + for ( const char *cursor = begin; cursor != end; cursor++ ) { + if ( *cursor < '0' || *cursor > '9' ) { + return false; + } + const unsigned int digit = static_cast( *cursor - '0' ); + if ( port > 6553 || ( port == 6553 && digit > 5 ) ) { + return false; + } + port = port * 10 + digit; + } + return true; +} + +inline bool IsASCIIAlphaNumeric( const char value ) { + return ( value >= 'a' && value <= 'z' ) || ( value >= 'A' && value <= 'Z' ) || + ( value >= '0' && value <= '9' ); +} + +inline bool IsASCIIHexDigit( const char value ) { + return ( value >= '0' && value <= '9' ) || ( value >= 'a' && value <= 'f' ) || + ( value >= 'A' && value <= 'F' ); +} + +inline bool IsIPv4Literal( const char *begin, const char *end ) { + int components = 0; + const char *cursor = begin; + while ( cursor != end ) { + if ( components == 4 ) { + return false; + } + unsigned int value = 0; + int digits = 0; + while ( cursor != end && *cursor != '.' ) { + if ( *cursor < '0' || *cursor > '9' || digits == 3 ) { + return false; + } + value = value * 10 + static_cast( *cursor - '0' ); + digits++; + cursor++; + } + if ( digits == 0 || value > 255 ) { + return false; + } + components++; + if ( cursor != end ) { + cursor++; + if ( cursor == end ) { + return false; + } + } + } + return components == 4; +} + +inline bool IsIPv6Literal( const char *begin, const char *end ) { + if ( begin == end ) { + return false; + } + + int groups = 0; + bool compressed = false; + const char *cursor = begin; + if ( *cursor == ':' ) { + if ( cursor + 1 == end || cursor[ 1 ] != ':' ) { + return false; + } + compressed = true; + cursor += 2; + if ( cursor == end ) { + return true; + } + } + + while ( cursor != end ) { + const char *groupBegin = cursor; + bool dotted = false; + while ( cursor != end && *cursor != ':' ) { + dotted = dotted || *cursor == '.'; + cursor++; + } + if ( dotted ) { + if ( cursor != end || !IsIPv4Literal( groupBegin, cursor ) ) { + return false; + } + groups += 2; + break; + } + const size_t groupLength = static_cast( cursor - groupBegin ); + if ( groupLength == 0 || groupLength > 4 ) { + return false; + } + for ( const char *digit = groupBegin; digit != cursor; digit++ ) { + if ( !IsASCIIHexDigit( *digit ) ) { + return false; + } + } + if ( ++groups > 8 ) { + return false; + } + if ( cursor == end ) { + break; + } + cursor++; + if ( cursor == end ) { + return false; + } + if ( *cursor == ':' ) { + if ( compressed ) { + return false; + } + compressed = true; + cursor++; + if ( cursor == end ) { + break; + } + } + } + return compressed ? groups < 8 : groups == 8; +} + +inline bool IsDNSOrIPv4Host( const char *begin, const char *end ) { + const size_t hostLength = static_cast( end - begin ); + if ( hostLength == 0 || hostLength > 253 ) { + return false; + } + + bool numericAndDotsOnly = true; + bool sawDot = false; + const char *labelBegin = begin; + for ( const char *cursor = begin; ; cursor++ ) { + if ( cursor == end || *cursor == '.' ) { + const size_t labelLength = static_cast( cursor - labelBegin ); + if ( labelLength == 0 || labelLength > 63 || + !IsASCIIAlphaNumeric( *labelBegin ) || !IsASCIIAlphaNumeric( cursor[ -1 ] ) ) { + return false; + } + if ( cursor == end ) { + break; + } + sawDot = true; + labelBegin = cursor + 1; + continue; + } + if ( !IsASCIIAlphaNumeric( *cursor ) && *cursor != '-' ) { + return false; + } + if ( *cursor < '0' || *cursor > '9' ) { + numericAndDotsOnly = false; + } + } + return !numericAndDotsOnly || ( sawDot && IsIPv4Literal( begin, end ) ); +} + +inline bool AuthorityHasHost( const char *begin, const char *end ) { + if ( begin == NULL || end == NULL || begin == end || end < begin ) { + return false; + } + + // User information and percent-encoded authority components create URL + // display/parser ambiguity. Neither is needed by an engine-owned web link. + for ( const char *cursor = begin; cursor != end; cursor++ ) { + if ( *cursor == '@' || *cursor == '%' ) { + return false; + } + } + + if ( *begin == '[' ) { + const char *closeBracket = begin + 1; + while ( closeBracket != end && *closeBracket != ']' ) { + closeBracket++; + } + if ( closeBracket == end || !IsIPv6Literal( begin + 1, closeBracket ) ) { + return false; + } + if ( closeBracket + 1 == end ) { + return true; + } + return closeBracket[ 1 ] == ':' && ParsePort( closeBracket + 2, end ); + } + + const char *portSeparator = NULL; + for ( const char *cursor = begin; cursor != end; cursor++ ) { + if ( *cursor == '[' || *cursor == ']' ) { + return false; + } + if ( *cursor == ':' ) { + if ( portSeparator != NULL ) { + // IPv6 literals must use brackets so the authority is unambiguous. + return false; + } + portSeparator = cursor; + } + } + + const char *hostEnd = portSeparator != NULL ? portSeparator : end; + if ( hostEnd == begin ) { + return false; + } + if ( !IsDNSOrIPv4Host( begin, hostEnd ) ) { + return false; + } + return portSeparator == NULL || ParsePort( portSeparator + 1, end ); +} + +inline bool IsAllowedHTTPURL( const char *url ) { + if ( url == NULL ) { + return false; + } + + size_t length = 0; + for ( ; length < MAX_URL_BYTES && url[ length ] != '\0'; length++ ) { + const unsigned char value = static_cast( url[ length ] ); + if ( value <= 32 || value == 127 || value == '"' || value == '<' || value == '>' || + value == '\\' || value == '^' || value == '`' || value == '{' || value == '|' || value == '}' ) { + return false; + } + } + if ( length == 0 || length == MAX_URL_BYTES ) { + return false; + } + + const char *authority = NULL; + if ( ASCIIStartsWith( url, "https://" ) ) { + authority = url + 8; + } else if ( ASCIIStartsWith( url, "http://" ) ) { + authority = url + 7; + } else { + return false; + } + + const char *authorityEnd = authority; + while ( *authorityEnd != '\0' && *authorityEnd != '/' && *authorityEnd != '?' && *authorityEnd != '#' ) { + authorityEnd++; + } + return AuthorityHasHost( authority, authorityEnd ); +} + +} // namespace idURLPolicy + +#endif /* !__URL_POLICY_H__ */ diff --git a/src/sys/linux/main.cpp b/src/sys/linux/main.cpp index 4a4484bc..0b04abd2 100644 --- a/src/sys/linux/main.cpp +++ b/src/sys/linux/main.cpp @@ -28,6 +28,7 @@ If you have questions concerning this license or the applicable additional terms #include "../../idlib/precompiled.h" #include "../posix/posix_public.h" #include "../sys_local.h" +#include "../URLPolicy.h" #include #include @@ -227,24 +228,6 @@ static bool Sys_QueueOrStartProcessArgs( char *const argv[], bool quit ) { return Sys_ExecProcessArgs( argv, true ); } -static bool Sys_IsSafeURL( const char *url ) { - if ( url == NULL || url[0] == '\0' || Sys_StringHasControlCharacters( url ) ) { - return false; - } - if ( !isalpha( static_cast( url[0] ) ) ) { - return false; - } - for ( const char *scan = url + 1; *scan != '\0'; ++scan ) { - if ( *scan == ':' ) { - return true; - } - if ( !( isalnum( static_cast( *scan ) ) || *scan == '+' || *scan == '-' || *scan == '.' ) ) { - return false; - } - } - return false; -} - static bool Sys_FindExecutableOnPath( const char *name, idStr &resolvedPath ) { resolvedPath.Clear(); if ( name == NULL || name[0] == '\0' ) { @@ -900,17 +883,18 @@ void idSysLocal::OpenURL( const char *url, bool quit ) { static bool quit_spamguard = false; - if ( quit_spamguard ) { - common->DPrintf( "Sys_OpenURL: already in a doexit sequence, ignoring %s\n", url ? url : "" ); + if ( !idURLPolicy::IsAllowedHTTPURL( url ) ) { + common->Printf( "OpenURL rejected: expected a bounded HTTP or HTTPS URL with a host\n" ); return; } - common->Printf( "Open URL: %s\n", url ); - if ( !Sys_IsSafeURL( url ) ) { - common->Printf( "OpenURL '%s' rejected: expected a URL with a safe scheme\n", url ? url : "" ); + if ( quit_spamguard ) { + common->DPrintf( "Sys_OpenURL: already in a doexit sequence, ignoring request\n" ); return; } + common->Printf( "Open URL: %s\n", url ); + // opening an URL on *nix can mean a lot of things .. // prefer a user-provided script, then fall back to freedesktop helpers. diff --git a/src/sys/osx/macosx_misc.mm b/src/sys/osx/macosx_misc.mm index 1b8d425b..1a9fda54 100644 --- a/src/sys/osx/macosx_misc.mm +++ b/src/sys/osx/macosx_misc.mm @@ -40,6 +40,7 @@ #import #import #include "../sys_local.h" +#include "../URLPolicy.h" #if defined(USE_SDL3) #include @@ -49,7 +50,6 @@ static const int MAX_OSX_PROCESS_ARGS = 32; static const int MAX_OSX_PROCESS_COMMAND = 4096; -static const int MAX_OSX_URL_LENGTH = 4096; #if defined(USE_SDL3) static int OSX_WindowBorderExtent( CGFloat extent ) { @@ -206,61 +206,6 @@ static bool OSX_ParseProcessCommandLine( const char *command, char *buffer, size return argc > 0 && argv[0][0] != '\0'; } -static bool OSX_URLHasSafeSchemeSyntax( const char *url ) { - if ( url == NULL || url[0] == '\0' || OSX_StringHasControlCharacters( url ) ) { - return false; - } - if ( strlen( url ) >= MAX_OSX_URL_LENGTH ) { - return false; - } - if ( !isalpha( static_cast( url[0] ) ) ) { - return false; - } - for ( const char *scan = url + 1; *scan != '\0'; ++scan ) { - if ( *scan == ':' ) { - return true; - } - if ( !( isalnum( static_cast( *scan ) ) || *scan == '+' || *scan == '-' || *scan == '.' ) ) { - return false; - } - } - return false; -} - -static bool OSX_ResolvedPathIsUnderDirectory( const char *path, const char *directory ) { - if ( path == NULL || path[0] == '\0' || directory == NULL || directory[0] == '\0' ) { - return false; - } - - char resolvedPath[PATH_MAX]; - char resolvedDirectory[PATH_MAX]; - if ( realpath( path, resolvedPath ) == NULL || realpath( directory, resolvedDirectory ) == NULL ) { - return false; - } - - const size_t directoryLength = strlen( resolvedDirectory ); - if ( directoryLength == 0 || idStr::Cmpn( resolvedPath, resolvedDirectory, static_cast( directoryLength ) ) != 0 ) { - return false; - } - return resolvedPath[directoryLength] == '\0' || resolvedPath[directoryLength] == '/'; -} - -static bool OSX_FileURLIsLocalRuntimeFile( NSURL *url ) { - if ( url == nil || ![url isFileURL] ) { - return false; - } - - NSString *pathString = [url path]; - const char *path = pathString != nil ? [pathString fileSystemRepresentation] : NULL; - if ( path == NULL || OSX_StringHasControlCharacters( path ) || !OSX_IsAbsolutePath( path ) || !OSX_IsRegularFile( path ) ) { - return false; - } - - const char *savePath = cvarSystem != NULL ? cvarSystem->GetCVarString( "fs_savepath" ) : NULL; - const char *basePath = cvarSystem != NULL ? cvarSystem->GetCVarString( "fs_basepath" ) : NULL; - return OSX_ResolvedPathIsUnderDirectory( path, savePath ) || OSX_ResolvedPathIsUnderDirectory( path, basePath ); -} - static bool OSX_IsAllowedURL( NSURL *url ) { if ( url == nil ) { return false; @@ -274,9 +219,6 @@ static bool OSX_IsAllowedURL( NSURL *url ) { NSString *host = [url host]; return host != nil && [host length] > 0; } - if ( [scheme caseInsensitiveCompare:@"file"] == NSOrderedSame ) { - return OSX_FileURLIsLocalRuntimeFile( url ); - } return false; } @@ -378,15 +320,16 @@ static bool OSX_StartProcessArgs( char *const argv[], bool dofork ) { void idSysLocal::OpenURL( const char *url, bool doexit ) { static bool quit_spamguard = false; - if ( quit_spamguard ) { - common->DPrintf( "Sys_OpenURL: already in a doexit sequence, ignoring request\n" ); + if ( !idURLPolicy::IsAllowedHTTPURL( url ) ) { + common->Printf( "OpenURL rejected: expected a bounded HTTP or HTTPS URL with a host\n" ); return; } - if ( !OSX_URLHasSafeSchemeSyntax( url ) ) { - common->Printf( "OpenURL rejected: expected a bounded URL with a safe scheme\n" ); + if ( quit_spamguard ) { + common->DPrintf( "Sys_OpenURL: already in a doexit sequence, ignoring request\n" ); return; } + NSString *urlString = [ NSString stringWithUTF8String: url ]; if ( urlString == nil ) { common->Printf( "OpenURL rejected: URL is not valid UTF-8\n" ); @@ -399,7 +342,7 @@ static bool OSX_StartProcessArgs( char *const argv[], bool dofork ) { return; } if ( !OSX_IsAllowedURL( nsURL ) ) { - common->Printf( "OpenURL rejected: scheme is not allowed\n" ); + common->Printf( "OpenURL rejected: Foundation did not preserve the required HTTP(S) host\n" ); return; } diff --git a/src/sys/posix/posix_main.cpp b/src/sys/posix/posix_main.cpp index f8fa5adc..7abd4046 100644 --- a/src/sys/posix/posix_main.cpp +++ b/src/sys/posix/posix_main.cpp @@ -1214,20 +1214,26 @@ char *Posix_ConsoleInput( void ) { const char *inputBuffer = tty_InputState( inputLength, inputCursor ); idStr::Copynz( input_ret, inputBuffer, sizeof( input_ret ) ); } - assert( hidden ); - tty_Show(); - write( STDOUT_FILENO, &key, 1 ); - input_field.Clear(); - if ( history_count < COMMAND_HISTORY ) { - history[ history_count ] = input_ret; - history_count++; - } else { - history[ history_start ] = input_ret; - history_start++; - history_start %= COMMAND_HISTORY; + { + const bool privateCommand = cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( input_ret ); + assert( hidden ); + tty_Show(); + write( STDOUT_FILENO, &key, 1 ); + input_field.Clear(); + if ( !privateCommand ) { + if ( history_count < COMMAND_HISTORY ) { + history[ history_count ] = input_ret; + history_count++; + } else { + history[ history_start ] = input_ret; + history_start++; + history_start %= COMMAND_HISTORY; + } + } + history_current = 0; + return input_ret; } - history_current = 0; - return input_ret; case '\t': input_field.AutoComplete(); break; @@ -1488,6 +1494,10 @@ void Sys_GenerateEvents( void ) { return; } idStr::Copynz( b, s, len ); + if ( cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( s ) ) { + memset( s, 0, len ); + } Posix_QueEvent( SE_CONSOLE, 0, 0, len, b ); } } diff --git a/src/sys/posix/posix_syscon.cpp b/src/sys/posix/posix_syscon.cpp index 3f5275cf..918a439e 100644 --- a/src/sys/posix/posix_syscon.cpp +++ b/src/sys/posix/posix_syscon.cpp @@ -825,12 +825,20 @@ static void Posix_ConsoleSubmitInput( void ) { return; } - Sys_Printf( "]%s\n", command ); + const bool privateCommand = cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( command ); + if ( privateCommand ) { + Sys_Printf( "]\n" ); + } else { + Sys_Printf( "]%s\n", command ); + } Posix_ConsoleQueueCommand( command ); - s_consoleWindow.history[ s_consoleWindow.nextHistoryLine % POSIX_CONSOLE_HISTORY ] = s_consoleWindow.inputField; - s_consoleWindow.nextHistoryLine++; - s_consoleWindow.historyLine = s_consoleWindow.nextHistoryLine; + if ( !privateCommand ) { + s_consoleWindow.history[ s_consoleWindow.nextHistoryLine % POSIX_CONSOLE_HISTORY ] = s_consoleWindow.inputField; + s_consoleWindow.nextHistoryLine++; + s_consoleWindow.historyLine = s_consoleWindow.nextHistoryLine; + } s_consoleWindow.inputField.Clear(); s_consoleWindow.scrollLines = 0; } diff --git a/src/sys/win32/win_main.cpp b/src/sys/win32/win_main.cpp index dcd2e270..4baf9526 100644 --- a/src/sys/win32/win_main.cpp +++ b/src/sys/win32/win_main.cpp @@ -45,6 +45,7 @@ If you have questions concerning this license or the applicable additional terms #endif #include "../sys_local.h" +#include "../URLPolicy.h" #include "win_local.h" #include "win_crash.h" #include "rc/CreateResourceIDs.h" @@ -1600,10 +1601,15 @@ void Sys_GenerateEvents(void) { if (s) { char* b; int len; + const bool privateCommand = cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( s ); len = idLib::SizeToInt( strlen( s ) + 1, "Sys_GenerateEvents console command" ); b = (char*)Mem_Alloc(len); strcpy(b, s); + if ( privateCommand ) { + memset( s, 0, len ); + } Sys_QueEvent(0, SE_CONSOLE, 0, 0, len, b); } @@ -2198,15 +2204,21 @@ void idSysLocal::OpenURL(const char* url, bool doexit) { static bool doexit_spamguard = false; HWND wnd; + if ( !idURLPolicy::IsAllowedHTTPURL( url ) ) { + common->Printf( "OpenURL rejected: expected a bounded HTTP or HTTPS URL with a host\n" ); + return; + } + if (doexit_spamguard) { - common->DPrintf("OpenURL: already in an exit sequence, ignoring %s\n", url); + common->DPrintf("OpenURL: already in an exit sequence, ignoring request\n"); return; } common->Printf("Open URL: %s\n", url); - if (!ShellExecute(NULL, "open", url, NULL, NULL, SW_RESTORE)) { - common->Error("Could not open url: '%s' ", url); + const HINSTANCE result = ShellExecute(NULL, "open", url, NULL, NULL, SW_RESTORE); + if (reinterpret_cast(result) <= 32) { + common->Printf("OpenURL failed after validation\n"); return; } diff --git a/src/sys/win32/win_syscon.cpp b/src/sys/win32/win_syscon.cpp index 93558832..2d7f04e6 100644 --- a/src/sys/win32/win_syscon.cpp +++ b/src/sys/win32/win_syscon.cpp @@ -827,6 +827,8 @@ LRESULT CALLBACK InputLineWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lP // enter the line if (key == K_ENTER || key == K_KP_ENTER) { const char *inputText = s_wcd.consoleField.GetBuffer(); + const bool privateCommand = cvarSystem != NULL && cvarSystem->IsInitialized() && + cvarSystem->CommandContainsPrivateCVar( inputText ); size_t used = strlen( s_wcd.consoleText ); if ( used < sizeof( s_wcd.consoleText ) - 1 ) { const size_t available = sizeof( s_wcd.consoleText ) - used - 1; @@ -841,12 +843,18 @@ LRESULT CALLBACK InputLineWndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lP } SetWindowText(s_wcd.hwndInputLine, ""); - Sys_Printf("]%s\n", s_wcd.consoleField.GetBuffer()); + if ( privateCommand ) { + Sys_Printf("]\n"); + } else { + Sys_Printf("]%s\n", s_wcd.consoleField.GetBuffer()); + } // copy line to history buffer - s_wcd.historyEditLines[s_wcd.nextHistoryLine % COMMAND_HISTORY] = s_wcd.consoleField; - s_wcd.nextHistoryLine++; - s_wcd.historyLine = s_wcd.nextHistoryLine; + if ( !privateCommand ) { + s_wcd.historyEditLines[s_wcd.nextHistoryLine % COMMAND_HISTORY] = s_wcd.consoleField; + s_wcd.nextHistoryLine++; + s_wcd.historyLine = s_wcd.nextHistoryLine; + } s_wcd.consoleField.Clear(); @@ -1147,7 +1155,7 @@ char* Sys_ConsoleInput(void) { } idStr::Copynz( s_wcd.returnedText, s_wcd.consoleText, sizeof( s_wcd.returnedText ) ); - s_wcd.consoleText[0] = 0; + memset( s_wcd.consoleText, 0, sizeof( s_wcd.consoleText ) ); return s_wcd.returnedText; } diff --git a/tools/build/stage_direct_run_game_module.py b/tools/build/stage_direct_run_game_module.py index d6a38a78..412a5efb 100644 --- a/tools/build/stage_direct_run_game_module.py +++ b/tools/build/stage_direct_run_game_module.py @@ -12,8 +12,14 @@ def atomic_copy_if_changed(source: Path, destination: Path) -> bool: - if destination.is_file() and filecmp.cmp(source, destination, shallow=False): - return False + try: + if destination.is_file() and filecmp.cmp(source, destination, shallow=False): + return False + except FileNotFoundError: + # A Meson regeneration can remove an old direct-run output between the + # existence probe and filecmp opening it. Treat that as a changed file; + # the atomic replacement below is already the recovery path. + pass destination.parent.mkdir(parents=True, exist_ok=True) temporary_path: Path | None = None diff --git a/tools/build/stage_fast_install.py b/tools/build/stage_fast_install.py index 421a8b5a..2dc1d6b7 100644 --- a/tools/build/stage_fast_install.py +++ b/tools/build/stage_fast_install.py @@ -4,10 +4,12 @@ from __future__ import annotations import argparse +import stat import sys from pathlib import Path from openq4_pak import copy_file_if_changed, is_relative_to +from windows_runtime import cleanup_windows_stage_target, is_windows_host ROOT_RUNTIME_PATTERNS = ( @@ -15,6 +17,10 @@ "openQ4-client_*.pdb", "openQ4-ded_*.exe", "openQ4-ded_*.pdb", + "renderer-gl_*.dll", + "renderer-gl_*.pdb", + "renderer-vk_*.dll", + "renderer-vk_*.pdb", "OpenAL32.dll", ) GAME_RUNTIME_PATTERNS = ( @@ -46,6 +52,14 @@ def parse_args(argv: list[str]) -> argparse.Namespace: default=str(Path(__file__).resolve().parents[2]), help="openQ4 source root used to validate the fast staging target.", ) + parser.add_argument( + "--temporary-runtime", + action="store_true", + help=( + "Allow one fresh alternate runtime below /.tmp/stock-runtime/. " + "This is for isolated compatibility captures and never replaces canonical .install staging." + ), + ) return parser.parse_args(argv[1:]) @@ -53,22 +67,55 @@ def copy_if_changed(source: Path, destination: Path) -> bool: return copy_file_if_changed(source, destination) -def validate_stage_roots(source_root: Path, build_dir: Path, install_dir: Path) -> None: +def is_link_or_junction(path: Path) -> bool: + is_junction = getattr(path, "is_junction", None) + if path.is_symlink() or bool(is_junction and is_junction()): + return True + try: + attributes = getattr(path.lstat(), "st_file_attributes", 0) + return bool(attributes & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + except OSError: + return False + + +def validate_stage_roots( + source_root: Path, + build_dir: Path, + install_dir: Path, + temporary_runtime: bool = False, +) -> None: for label, path in ( ("source root", source_root), ("build directory", build_dir), ("install directory", install_dir), ): - if path.is_symlink(): - raise RuntimeError(f"fast staging {label} must not be a symlink: {path}") + if is_link_or_junction(path): + raise RuntimeError(f"fast staging {label} must not be a link or junction: {path}") source_root = source_root.resolve() build_dir = build_dir.resolve() install_dir = install_dir.resolve() expected_install_dir = source_root / ".install" + temporary_parent = source_root / ".tmp" / "stock-runtime" if install_dir != expected_install_dir: - raise RuntimeError(f"fast staging install directory must be {expected_install_dir}: {install_dir}") + if not temporary_runtime: + raise RuntimeError(f"fast staging install directory must be {expected_install_dir}: {install_dir}") + if install_dir == temporary_parent or not is_relative_to(install_dir, temporary_parent): + raise RuntimeError( + f"temporary runtime directory must stay below {temporary_parent}: {install_dir}" + ) + if install_dir.exists(): + raise RuntimeError(f"temporary runtime directory must be new: {install_dir}") + current = source_root + for part in install_dir.relative_to(source_root).parts[:-1]: + current /= part + if current.exists() and is_link_or_junction(current): + raise RuntimeError( + f"temporary runtime parent must not be a link or junction: {current}" + ) + elif temporary_runtime: + raise RuntimeError("--temporary-runtime is only valid for an alternate runtime directory") if not is_relative_to(build_dir, source_root): raise RuntimeError(f"fast staging build directory must stay under {source_root}: {build_dir}") if build_dir == install_dir or is_relative_to(install_dir, build_dir) or is_relative_to(build_dir, install_dir): @@ -108,7 +155,9 @@ def main(argv: list[str]) -> int: install_dir = Path(args.install_dir) try: - validate_stage_roots(source_root, build_dir, install_dir) + validate_stage_roots( + source_root, build_dir, install_dir, args.temporary_runtime + ) source_root = source_root.resolve() build_dir = build_dir.resolve() install_dir = install_dir.resolve() @@ -117,6 +166,11 @@ def main(argv: list[str]) -> int: install_dir.mkdir(parents=True, exist_ok=True) install_game_dir.mkdir(parents=True, exist_ok=True) + stage_cleanup = ( + cleanup_windows_stage_target(install_dir) + if is_windows_host() + else {"removed_stale_files": [], "removed_empty_directories": []} + ) removed = remove_matches(install_dir, NON_RUNTIME_PATTERNS) removed += remove_matches(install_game_dir, NON_RUNTIME_PATTERNS + ("*.so", "*.dylib")) copied = copy_matches(build_dir, install_dir, ROOT_RUNTIME_PATTERNS) @@ -127,7 +181,9 @@ def main(argv: list[str]) -> int: print( f"fast-staged .install: copied={len(copied)} " - f"removed_non_runtime={len(removed)}" + f"removed_non_runtime={len(removed)} " + f"removed_stale={len(stage_cleanup['removed_stale_files'])} " + f"removed_empty_dirs={len(stage_cleanup['removed_empty_directories'])}" ) for path in copied[:20]: print(f" copied {path}") diff --git a/tools/build/windows_runtime.py b/tools/build/windows_runtime.py index 0f6f9724..094c22c8 100644 --- a/tools/build/windows_runtime.py +++ b/tools/build/windows_runtime.py @@ -5,6 +5,7 @@ import os import shutil +import stat import struct from pathlib import Path @@ -21,6 +22,30 @@ WINDOWS_ROOT_RUNTIME_PATTERNS = ( "OpenAL32.dll", ) +# Meson installs overlay their destination and deliberately do not remove files +# from an earlier build. Keep the Windows cleanup policy as an explicit, +# narrow manifest: these names can only be foreign POSIX engine binaries or +# obsolete renderer-validation backups in a Windows runtime tree. +WINDOWS_STALE_STAGE_FILE_MANIFEST = ( + "openQ4-client_x86", + "openQ4-client_x64", + "openQ4-client_arm64", + "openQ4-ded_x86", + "openQ4-ded_x64", + "openQ4-ded_arm64", + "renderer-gl_x86.dll.mainbak", + "renderer-gl_x64.dll.mainbak", + "renderer-gl_arm64.dll.mainbak", + "renderer-vk_x86.dll.mainbak", + "renderer-vk_x64.dll.mainbak", + "renderer-vk_arm64.dll.mainbak", +) +# These directories were emitted by older content staging. They are removed +# only when empty; populated directories may contain intentional local content +# and are therefore left untouched. +WINDOWS_EMPTY_STAGE_DIRECTORY_MANIFEST = ( + "baseoq4/skins", +) RUNTIME_BINARY_PATTERNS = ( f"{PRODUCT_NAME}-client_*.exe", f"{PRODUCT_NAME}-ded_*.exe", @@ -256,6 +281,85 @@ def clear_staged_runtime_files(root_dir: Path) -> None: path.unlink() +def _manifest_path(root_dir: Path, relative_name: str) -> Path: + relative_path = Path(relative_name) + if relative_path.is_absolute() or not relative_path.parts or ".." in relative_path.parts: + raise RuntimeError(f"invalid Windows stage cleanup manifest path: {relative_name!r}") + parent = root_dir + for part in relative_path.parts[:-1]: + parent /= part + if _is_link_or_junction(parent): + raise RuntimeError(f"Windows stage cleanup parent is a link or junction: {parent}") + if parent.exists() and not parent.is_dir(): + raise RuntimeError(f"Windows stage cleanup parent is not a directory: {parent}") + candidate = root_dir.joinpath(*relative_path.parts) + if not _is_relative_to(candidate, root_dir): + raise RuntimeError(f"Windows stage cleanup path escapes its runtime target: {relative_name!r}") + return candidate + + +def _is_link_or_junction(path: Path) -> bool: + is_junction = getattr(path, "is_junction", None) + if path.is_symlink() or bool(is_junction and is_junction()): + return True + try: + attributes = getattr(path.lstat(), "st_file_attributes", 0) + return bool(attributes & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + except OSError: + return False + + +def cleanup_windows_stage_target(root_dir: Path) -> dict[str, list[str]]: + """Remove only explicitly known stale entries from a Windows stage root.""" + + root_dir = validate_runtime_directory(root_dir, "Windows runtime target", must_exist=False) + removed_files: list[str] = [] + removed_directories: list[str] = [] + + if not root_dir.exists(): + return { + "removed_stale_files": removed_files, + "removed_empty_directories": removed_directories, + } + + for relative_name in WINDOWS_STALE_STAGE_FILE_MANIFEST: + candidate = _manifest_path(root_dir, relative_name) + if _is_link_or_junction(candidate): + # Unlinking a known stale leaf is safe and does not follow its + # target. Directory links are rejected rather than recursively + # traversed or removed. + if candidate.is_dir(): + raise RuntimeError(f"stale Windows runtime file is a directory link or junction: {candidate}") + candidate.unlink() + removed_files.append(relative_name) + continue + if not candidate.exists(): + continue + if not candidate.is_file(): + raise RuntimeError(f"stale Windows runtime file path is not a regular file: {candidate}") + candidate.unlink() + removed_files.append(relative_name) + + for relative_name in WINDOWS_EMPTY_STAGE_DIRECTORY_MANIFEST: + candidate = _manifest_path(root_dir, relative_name) + if _is_link_or_junction(candidate): + raise RuntimeError(f"empty Windows runtime directory is a link or junction: {candidate}") + if not candidate.exists(): + continue + if not candidate.is_dir(): + raise RuntimeError(f"empty Windows runtime directory path is not a directory: {candidate}") + try: + next(candidate.iterdir()) + except StopIteration: + candidate.rmdir() + removed_directories.append(relative_name) + + return { + "removed_stale_files": removed_files, + "removed_empty_directories": removed_directories, + } + + def _is_relative_to(path: Path, root: Path) -> bool: try: @@ -294,6 +398,8 @@ def _copy_runtime_tree(source_dir: Path, destination_dir: Path, ignore_patterns: def validate_runtime_directory(path: Path, label: str, *, must_exist: bool) -> Path: if path.is_symlink(): raise RuntimeError(f"{label} must not be a symlink: {path}") + if _is_link_or_junction(path): + raise RuntimeError(f"{label} must not be a junction or reparse point: {path}") if path.exists() and not path.is_dir(): raise RuntimeError(f"{label} exists but is not a directory: {path}") if must_exist and not path.is_dir(): @@ -390,6 +496,10 @@ def stage_runtime_payloads( ] staged_build_game = stage_build_game_directory(source_root, build_root) + cleanup_results: dict[str, dict[str, list[str]]] = {} + for target in targets: + cleanup_results[str(target)] = cleanup_windows_stage_target(target) + binaries = collect_runtime_binaries(build_root) if not binaries: return { @@ -397,6 +507,7 @@ def stage_runtime_payloads( "runtime_flavor": RuntimeFlavor.NONE, "targets": [str(target) for target in targets], "copied_files": [], + "stage_cleanup": cleanup_results, "staged_build_game": staged_build_game, "validated_binaries": [], } @@ -438,6 +549,7 @@ def stage_runtime_payloads( "runtime_flavor": flavor, "targets": [str(target) for target in targets], "copied_files": sorted(set(copied_files)), + "stage_cleanup": cleanup_results, "staged_build_game": staged_build_game, "validated_binaries": [str(path) for path in binaries], } diff --git a/tools/debug/renderdoc_capture.ps1 b/tools/debug/renderdoc_capture.ps1 index 1cec6846..cd67ff39 100644 --- a/tools/debug/renderdoc_capture.ps1 +++ b/tools/debug/renderdoc_capture.ps1 @@ -95,6 +95,7 @@ if ($Mode -eq "SP") { } else { $gameArgs += @( "+set", "net_serverDedicated", "0", + "+set", "ui_autoJoin", "1", "+seta", "si_pure", "0", "+set", "net_serverAllowServerMod", "1", "+set", "sv_cheats", "1", diff --git a/tools/debug/start_listen_server_client.ps1 b/tools/debug/start_listen_server_client.ps1 index 17935870..96d2ee7f 100644 --- a/tools/debug/start_listen_server_client.ps1 +++ b/tools/debug/start_listen_server_client.ps1 @@ -135,6 +135,7 @@ $serverArgs = New-openQ4CommonArgs ` $serverArgs += @( "+set", "net_serverDedicated", "0", "+set", "net_port", $Port.ToString(), + "+set", "ui_autoJoin", "1", "+seta", "si_pure", "0", "+set", "net_serverAllowServerMod", "1", "+set", "sv_cheats", "1", @@ -154,6 +155,7 @@ $clientArgs = New-openQ4CommonArgs ` -ShowFramePacing $ShowFramePacing $clientArgs += @( + "+set", "ui_autoJoin", "1", "+set", "ui_name", $ClientName, "+connect", ("127.0.0.1:{0}" -f $Port) ) diff --git a/tools/tests/competitive_match_layer.py b/tools/tests/competitive_match_layer.py index 8ff6b605..43bb4edd 100644 --- a/tools/tests/competitive_match_layer.py +++ b/tools/tests/competitive_match_layer.py @@ -231,6 +231,18 @@ def main() -> None: require(multiplayer, "gameLocal.gameType == GAME_TDM || roundMode", "round rows") require(multiplayer, 'common->GetLocalizedString( "#str_41404" )', "round limit") + # A casual no-time-limit server may retain the stock positive si_overtime + # default. Timed overtime is valid only when regulation itself is timed; + # otherwise the typed rule import must normalize to sudden death with no + # timed period instead of rejecting the complete rules snapshot. + for token in ( + 'const int timeLimitMinutes = gameLocal.serverInfo.GetInt( "si_timeLimit" );', + "const bool useTimedOvertime = timeLimitMinutes > 0 && overtimeSeconds > 0;", + "useTimedOvertime ? MP_OVERTIME_TIMED_PERIODS : MP_OVERTIME_SUDDEN_DEATH", + "useTimedOvertime ? overtimeSeconds : 0", + ): + require(multiplayer, token, "casual overtime normalization") + # The managed-match HUD and scoreboard are projections of exactly the same # pre-localized state set. One parent gate makes their casual path inert. validate_match_context_surface( diff --git a/tools/tests/filesystem_write_qpath_safety.py b/tools/tests/filesystem_write_qpath_safety.py index 91d7c601..03143df6 100644 --- a/tools/tests/filesystem_write_qpath_safety.py +++ b/tools/tests/filesystem_write_qpath_safety.py @@ -242,11 +242,25 @@ def validate_generated_loadscreen_publication() -> None: require(prepare, "R_WriteTGA( stagingPath.c_str()", "generated loadscreen staged write") require(prepare, "fileSystem->PromoteFile( stagingPath.c_str(), generatedPath.c_str()", "generated loadscreen atomic publication") + require(prepare, "else if ( !Session_FileExistsInSearchPaths( generatedPath.c_str() ) )", + "generated loadscreen active-VFS visibility gate") + require(prepare, "declManager->FindMaterial( generatedPath.c_str() )", + "generated loadscreen material preflight") + require(prepare, "generatedMaterial->TestMaterialFlag( MF_DEFAULTED )", + "generated loadscreen default-material rejection") + require(prepare, "using the source levelshot", + "generated loadscreen unavailable-path source fallback") require(prepare, "fileSystem->RemoveFileChecked( stagingPath.c_str()", "generated loadscreen failed-stage cleanup") require(prepare, "return published;", "generated loadscreen failure falls back to source") require_order(prepare, "R_WriteTGA( stagingPath.c_str()", "fileSystem->PromoteFile(", "generated loadscreen write-before-publish order") + require_order(prepare, "fileSystem->PromoteFile(", + "Session_FileExistsInSearchPaths( generatedPath.c_str() )", + "generated loadscreen publish-before-VFS-validation order") + require_order(prepare, "Session_FileExistsInSearchPaths( generatedPath.c_str() )", + "declManager->FindMaterial( generatedPath.c_str() )", + "generated loadscreen VFS-before-material-validation order") require_order(prepare, "Sys_GetSecureRandomBytes( stagingNonce, sizeof( stagingNonce ) )", 'const idStr stagingPath = va( "%s.%016llx%016llx.%u.partial"', "generated loadscreen secure-token-before-path order") diff --git a/tools/tests/key_bind_presentation.py b/tools/tests/key_bind_presentation.py index 04cc4b19..b3158e23 100644 --- a/tools/tests/key_bind_presentation.py +++ b/tools/tests/key_bind_presentation.py @@ -645,6 +645,21 @@ def validate_bind_menu_and_spectator_consumers() -> None: mphud = read(ROOT / "content" / "baseoq4" / "pak0" / "guis" / "mphud.gui") require(mphud, '"gui::spectatetext1"', "multiplayer spectator HUD binding prompt") + ready_prompt_window = body_of(mphud, "windowDef Spectate1", "mphud.gui Spectate1") + if re.search(r"\brect\s+0\s*,\s*141\s*,\s*640\s*,\s*40\b", ready_prompt_window) is None: + raise AssertionError("The stock non-tourney readiness prompt must retain its 640x40 HUD window") + require(ready_prompt_window, "textscale\t0.31", "stock readiness prompt text scale") + + code_strings = read( + ROOT / "content" / "baseoq4" / "pak0" / "strings" / "english_code.lang" + ) + for localized_string in ("#str_107710", "#str_107711"): + if re.search( + rf'^\s*"{localized_string}"\s*"[^"\r\n]*\\nPress %s to [^"\r\n]+"\s*$', + code_strings, + re.MULTILINE, + ) is None: + raise AssertionError(f"{localized_string} must retain its explicit two-line readiness layout") checked_tree = False for tree in ("mpgame", "game"): @@ -662,7 +677,23 @@ def validate_bind_menu_and_spectator_consumers() -> None: reject(update_hud, 'KeysFromBinding( "_attack" )', f"{tree} non-emphasized spectator attack binding") all_ready = body_of(multiplayer_source, "bool idMultiplayerGame::AllPlayersReady(", str(multiplayer)) - require(all_ready, 'KeysFromBindingForPrompt( "_impulse17" )', f"{tree} emphasized readiness prompt") + for localized_string in ("#str_110017", "#str_110018"): + require( + all_ready, + f'GetLocalizedString( "{localized_string}" ), common->KeysFromBindingForPrompt( "_impulse17" )', + f"{tree} emphasized single-line tourney readiness prompt", + ) + for localized_string in ("#str_107710", "#str_107711"): + require( + all_ready, + f'GetLocalizedString( "{localized_string}" ), common->KeysFromBinding( "_impulse17" )', + f"{tree} inline two-line readiness prompt", + ) + require( + all_ready, + "stock non-tourney HUD fits two normal-height lines", + f"{tree} stock readiness prompt height rationale", + ) start_vote = body_of(multiplayer_source, "void idMultiplayerGame::ClientStartPackedVote(", str(multiplayer)) require(start_vote, 'KeysFromBindingForPrompt("_impulse28")', f"{tree} emphasized vote-yes prompt") require(start_vote, 'KeysFromBindingForPrompt("_impulse29")', f"{tree} emphasized vote-no prompt") @@ -671,6 +702,87 @@ def validate_bind_menu_and_spectator_consumers() -> None: raise AssertionError(f"No companion game-library source trees found below {GAME_LIBS_ROOT}") +def validate_ready_binding_contract() -> None: + default_cfg = read(ROOT / "content" / "baseoq4" / "pak0" / "default.cfg") + controls = read( + ROOT + / "content" + / "baseoq4" + / "pak0" + / "guis" + / "menu" + / "settings" + / "controls.gui" + ) + if re.search(r"(?m)^\s*bind\s+F3\s+_impulse17(?:\s+//.*)?$", default_cfg) is None: + raise AssertionError("The shipped F3 ready default must use stock _impulse17") + for key, impulse in (("F1", "_impulse28"), ("F2", "_impulse29"), ("F6", "_impulse20"), ("F7", "_impulse22")): + if re.search(rf"(?m)^\s*bind\s+{key}\s+{impulse}(?:\s+//.*)?$", default_cfg) is None: + raise AssertionError(f"The shipped {key} default must use stock {impulse}") + if re.search( + r"bindDef\s+set_ctrls_other_ready_key\s*\{.*?\bbind\s+_impulse17\b", + controls, + re.DOTALL, + ) is None: + raise AssertionError("The controls-menu ready row must edit stock _impulse17") + for widget, impulse in (("voteyes", "_impulse28"), ("voteno", "_impulse29")): + if re.search( + rf"bindDef\s+set_ctrls_other_{widget}_key\s*\{{.*?\bbind\s+{impulse}\b", + controls, + re.DOTALL, + ) is None: + raise AssertionError(f"The controls-menu {widget} row must edit stock {impulse}") + + multiplayer_path = GAME_LIBS_ROOT / "src" / "mpgame" / "MultiplayerGame.cpp" + multiplayer = read(multiplayer_path) + toggle_ready = body_of( + multiplayer, + "void idMultiplayerGame::ToggleReady(", + str(multiplayer_path), + ) + require(toggle_ready, "MPSendReady( !ready );", "reliable impulse-17 ready toggle") + reject(toggle_ready, "SetCVarString", "userinfo-only impulse-17 ready toggle") + + send_ready = body_of(multiplayer, "static void MPSendReady(", str(multiplayer_path)) + require( + send_ready, + "GAME_RELIABLE_MESSAGE_READY", + "casual client ready reliable message", + ) + require( + send_ready, + "ServerSetPlayerReady", + "listen-server authoritative ready path", + ) + ready_command = body_of( + multiplayer, + "void idMultiplayerGame::Ready_f(", + str(multiplayer_path), + ) + require(ready_command, "MPSendReady( true );", "idempotent ready command") + + reset = body_of(multiplayer, "void idMultiplayerGame::Reset(", str(multiplayer_path)) + require( + reset, + 'common->BindingFromKey( "F3" )', + "exact legacy F3 ready migration lookup", + ) + require( + reset, + '"bind F3 _impulse17\\n"', + "exact legacy F3 ready migration update", + ) + for key, legacy, impulse in ( + ("F1", "voteyes", "_impulse28"), + ("F2", "voteno", "_impulse29"), + ("F6", "toggleteam", "_impulse20"), + ("F7", "spectate", "_impulse22"), + ): + require(reset, f'common->BindingFromKey( "{key}" )', f"exact legacy {key} migration lookup") + require(reset, f'"{legacy}"', f"exact legacy {key} binding match") + require(reset, f'"bind {key} {impulse}\\n"', f"exact legacy {key} migration update") + + def main() -> int: try: validate_binding_formatter() @@ -680,6 +792,7 @@ def main() -> int: validate_bind_widget_fit_and_capture() validate_localized_controls_hint() validate_bind_menu_and_spectator_consumers() + validate_ready_binding_contract() except AssertionError as error: print(f"key_bind_presentation: FAILED - {error}") return 1 diff --git a/tools/tests/macos_sdl3_backend_guard.py b/tools/tests/macos_sdl3_backend_guard.py index e4919d50..d1132b94 100644 --- a/tools/tests/macos_sdl3_backend_guard.py +++ b/tools/tests/macos_sdl3_backend_guard.py @@ -85,7 +85,6 @@ def validate_macos_process_handoff_guards() -> None: filtered_environment = function_body(source, "static char **OSX_CreateFilteredProcessEnvironment() {") do_start_process = function_body(source, "void Sys_DoStartProcess( const char *exeName, bool dofork ) {") allowed_url = function_body(source, "static bool OSX_IsAllowedURL( NSURL *url ) {") - file_url = function_body(source, "static bool OSX_FileURLIsLocalRuntimeFile( NSURL *url ) {") open_url = function_body(source, "void idSysLocal::OpenURL( const char *url, bool doexit ) {") require(source, "static bool OSX_IsAbsolutePath( const char *path )", "macOS process handoff absolute-path helper") @@ -113,14 +112,14 @@ def validate_macos_process_handoff_guards() -> None: require(do_start_process, 'common->Printf( "Sys_DoStartProcess: invalid command line\\n" );', "macOS invalid process command diagnostic") reject(do_start_process, 'invalid command line \'%s\'', "macOS invalid process command diagnostic") - require(source, "MAX_OSX_URL_LENGTH", "macOS URL length guard") - require(open_url, "OSX_URLHasSafeSchemeSyntax( url )", "macOS URL syntax guard") - require(open_url, "OpenURL rejected: expected a bounded URL with a safe scheme", "macOS unsafe URL diagnostic") + require(source, '#include "../URLPolicy.h"', "shared macOS URL policy") + require(open_url, "idURLPolicy::IsAllowedHTTPURL( url )", "macOS HTTP(S)-only URL guard") + require(open_url, "OpenURL rejected: expected a bounded HTTP or HTTPS URL with a host", "macOS unsafe URL diagnostic") require(open_url, "OpenURL rejected: URL is not valid UTF-8", "macOS unsafe URL diagnostic") require(open_url, "OpenURL rejected: Foundation could not parse URL", "macOS unsafe URL diagnostic") - require(open_url, "OpenURL rejected: scheme is not allowed", "macOS unsafe URL diagnostic") + require(open_url, "OpenURL rejected: Foundation did not preserve the required HTTP(S) host", "macOS unsafe URL diagnostic") require(open_url, "OpenURL failed after validation", "macOS URL handoff diagnostic") - require_before(open_url, "OSX_URLHasSafeSchemeSyntax( url )", 'NSString *urlString = [ NSString stringWithUTF8String: url ];', "macOS URL syntax before UTF-8 bridge") + require_before(open_url, "idURLPolicy::IsAllowedHTTPURL( url )", 'NSString *urlString = [ NSString stringWithUTF8String: url ];', "macOS URL policy before UTF-8 bridge") require_before(open_url, "OSX_IsAllowedURL( nsURL )", 'common->Printf( "Open URL: %s\\n", url );', "macOS URL logging after allowlist guard") require_before(open_url, 'common->Printf( "Open URL: %s\\n", url );', "openURL: nsURL", "macOS approved URL logging before AppKit handoff") require(open_url, "OSX_IsAllowedURL( nsURL )", "macOS URL allowlist guard") @@ -129,15 +128,9 @@ def validate_macos_process_handoff_guards() -> None: require(allowed_url, '[scheme caseInsensitiveCompare:@"https"]', "macOS URL allowlist") require(allowed_url, '[scheme caseInsensitiveCompare:@"http"]', "macOS URL allowlist") require(allowed_url, "return host != nil && [host length] > 0;", "macOS HTTP URL host guard") - require(allowed_url, '[scheme caseInsensitiveCompare:@"file"]', "macOS URL allowlist") - require(file_url, "[url isFileURL]", "macOS file URL local-file guard") - require(file_url, "OSX_StringHasControlCharacters( path )", "macOS file URL local-file guard") - require(file_url, "OSX_IsAbsolutePath( path )", "macOS file URL local-file guard") - require(file_url, "OSX_IsRegularFile( path )", "macOS file URL local-file guard") - require(file_url, 'GetCVarString( "fs_savepath" )', "macOS file URL runtime-root guard") - require(file_url, 'GetCVarString( "fs_basepath" )', "macOS file URL runtime-root guard") - require(file_url, "OSX_ResolvedPathIsUnderDirectory( path, savePath )", "macOS file URL runtime-root guard") - reject(source, "static bool OSX_IsSafeURL", "macOS URL broad scheme guard") + reject(allowed_url, '[scheme caseInsensitiveCompare:@"file"]', "macOS file URL policy") + reject(source, "OSX_FileURLIsLocalRuntimeFile", "macOS file URL launcher") + reject(source, "OSX_URLHasSafeSchemeSyntax", "macOS broad URL syntax-only policy") def validate_sdl3_context_teardown_guards() -> None: diff --git a/tools/tests/macos_static_policy.py b/tools/tests/macos_static_policy.py index acc50b1a..cb890444 100644 --- a/tools/tests/macos_static_policy.py +++ b/tools/tests/macos_static_policy.py @@ -139,15 +139,15 @@ def validate_url_open_policy() -> None: misc = read("src/sys/osx/macosx_misc.mm") open_url = function_body(misc, "void idSysLocal::OpenURL( const char *url, bool doexit ) {") allowed_url = function_body(misc, "static bool OSX_IsAllowedURL( NSURL *url ) {") - file_url = function_body(misc, "static bool OSX_FileURLIsLocalRuntimeFile( NSURL *url ) {") - require(misc, "MAX_OSX_URL_LENGTH", "macOS URL length guard") - require(open_url, "OpenURL rejected: expected a bounded URL with a safe scheme", "macOS unsafe URL diagnostic") + require(misc, '#include "../URLPolicy.h"', "shared macOS URL policy") + require(open_url, "idURLPolicy::IsAllowedHTTPURL( url )", "macOS HTTP(S)-only URL guard") + require(open_url, "OpenURL rejected: expected a bounded HTTP or HTTPS URL with a host", "macOS unsafe URL diagnostic") require(open_url, "OpenURL rejected: URL is not valid UTF-8", "macOS unsafe URL diagnostic") require(open_url, "OpenURL rejected: Foundation could not parse URL", "macOS unsafe URL diagnostic") - require(open_url, "OpenURL rejected: scheme is not allowed", "macOS unsafe URL diagnostic") + require(open_url, "OpenURL rejected: Foundation did not preserve the required HTTP(S) host", "macOS unsafe URL diagnostic") require(open_url, "OpenURL failed after validation", "macOS URL handoff diagnostic") - require_before(open_url, "OSX_URLHasSafeSchemeSyntax( url )", 'NSString *urlString = [ NSString stringWithUTF8String: url ];', "macOS URL syntax before UTF-8 bridge") + require_before(open_url, "idURLPolicy::IsAllowedHTTPURL( url )", 'NSString *urlString = [ NSString stringWithUTF8String: url ];', "macOS URL policy before UTF-8 bridge") require_before(open_url, "OSX_IsAllowedURL( nsURL )", 'common->Printf( "Open URL: %s\\n", url );', "macOS URL logging after allowlist guard") require_before(open_url, 'common->Printf( "Open URL: %s\\n", url );', "openURL: nsURL", "macOS approved URL logging before AppKit handoff") require_before(open_url, "OSX_IsAllowedURL( nsURL )", "openURL: nsURL", "macOS URL allowlist before AppKit handoff") @@ -156,12 +156,9 @@ def validate_url_open_policy() -> None: require(allowed_url, '[scheme caseInsensitiveCompare:@"https"]', "macOS URL scheme allowlist") require(allowed_url, '[scheme caseInsensitiveCompare:@"http"]', "macOS URL scheme allowlist") require(allowed_url, "return host != nil && [host length] > 0;", "macOS HTTP URL host guard") - require(allowed_url, '[scheme caseInsensitiveCompare:@"file"]', "macOS URL scheme allowlist") - require(file_url, "OSX_StringHasControlCharacters( path )", "macOS file URL path control-character guard") - require(file_url, 'GetCVarString( "fs_savepath" )', "macOS file URL runtime-root policy") - require(file_url, 'GetCVarString( "fs_basepath" )', "macOS file URL runtime-root policy") - require(file_url, "OSX_ResolvedPathIsUnderDirectory( path, savePath )", "macOS file URL runtime-root policy") - reject(misc, "static bool OSX_IsSafeURL", "macOS broad URL syntax-only policy") + reject(allowed_url, '[scheme caseInsensitiveCompare:@"file"]', "macOS file URL policy") + reject(misc, "OSX_FileURLIsLocalRuntimeFile", "macOS file URL launcher") + reject(misc, "OSX_URLHasSafeSchemeSyntax", "macOS broad URL syntax-only policy") def validate_deprecated_api_boundaries() -> None: diff --git a/tools/tests/mp_bot_navigation.py b/tools/tests/mp_bot_navigation.py index f703c480..c3ec8160 100644 --- a/tools/tests/mp_bot_navigation.py +++ b/tools/tests/mp_bot_navigation.py @@ -56,6 +56,17 @@ def require_order(haystack: str, first: str, second: str, context: str) -> None: raise AssertionError(f"{first!r} must appear before {second!r} in {context}") +def cvar_signature(source: str, name: str, context: str) -> tuple[str, set[str]]: + match = re.search( + rf'idCVar\s+\w+\(\s*"{re.escape(name)}"\s*,\s*"([^"]*)"\s*,\s*([^,]+),', + source, + re.IGNORECASE, + ) + if match is None: + raise AssertionError(f"Missing CVar declaration for {name!r} in {context}") + return match.group(1), {flag.strip() for flag in match.group(2).split("|")} + + def validate_engine() -> None: source = read(ROOT / "src" / "framework" / "async" / "AsyncServer.cpp") @@ -497,6 +508,7 @@ def validate_cvars() -> None: declared = read(mp / "gamesys" / "SysCvar.cpp") exported = read(mp / "gamesys" / "SysCvar.h") + sp_bots = read(GAME_LIBS_ROOT / "src" / "game" / "bots" / "Bot.cpp") for name in ( "bot_enable", @@ -510,6 +522,17 @@ def validate_cvars() -> None: require(declared, f'idCVar {name}(', "mpgame SysCvar.cpp") require(exported, f"extern idCVar {name};", "mpgame SysCvar.h") + # These legacy SP bot diagnostics share global names with the richer MP + # diagnostics. Keep their declaration signatures aligned so loading SP + # before MP cannot leave the CVar system with conflicting types or policy. + for name in ("bot_debug", "bot_debugnav"): + sp_signature = cvar_signature(sp_bots, name, "game bots/Bot.cpp") + mp_signature = cvar_signature(declared, name, "mpgame SysCvar.cpp") + if sp_signature != mp_signature: + raise AssertionError(f"SP/MP bot CVar signatures differ for {name!r}") + if "CVAR_INTEGER" not in sp_signature[1] or "CVAR_BOOL" in sp_signature[1]: + raise AssertionError(f"{name} must retain the shared integer diagnostic type") + commands = read(mp / "gamesys" / "SysCmds.cpp") for command in ("addbot", "removebot", "kickbots", "botlist", "navmesh"): require(commands, f'"{command}"', "mpgame console commands") diff --git a/tools/tests/native/CoreSafetyTest.cpp b/tools/tests/native/CoreSafetyTest.cpp index 68ce63d0..260eb95e 100644 --- a/tools/tests/native/CoreSafetyTest.cpp +++ b/tools/tests/native/CoreSafetyTest.cpp @@ -1,9 +1,17 @@ #include "src/idlib/NumericString.h" #include "src/idlib/StrAllocation.h" +#include "src/idlib/CryptoHash.h" +#include "src/idlib/PrivateCommand.h" +#include "src/framework/GameDirPolicy.h" +#include "src/framework/RemoteCVarPolicy.h" +#include "src/framework/async/Rcon2Protocol.h" #include "src/sys/NetworkEndpoint.h" +#include "src/sys/URLPolicy.h" #include +#include #include +#include #include static int failures = 0; @@ -42,6 +50,165 @@ static void ExpectAllocation( const bool condition, const char *label ) { } } +static void ExpectPrivateToken( const char *command, const char *name, + const bool expected, const char *label ) { + const bool actual = idPrivateCommand::ContainsBoundedCaseInsensitiveToken( command, name ); + if ( actual != expected ) { + std::fprintf( stderr, "Private command-token match failed for %s: expected %d, got %d\n", + label, expected, actual ); + failures++; + } +} + +static int HexNibble( const char value ) { + if ( value >= '0' && value <= '9' ) { + return value - '0'; + } + if ( value >= 'a' && value <= 'f' ) { + return value - 'a' + 10; + } + if ( value >= 'A' && value <= 'F' ) { + return value - 'A' + 10; + } + return -1; +} + +static void ExpectCryptoBytes( const std::uint8_t *actual, const std::size_t bytes, + const char *expectedHex, const char *label ) { + if ( std::strlen( expectedHex ) != bytes * 2 ) { + std::fprintf( stderr, "Invalid expected crypto vector for %s\n", label ); + failures++; + return; + } + for ( std::size_t index = 0; index < bytes; ++index ) { + const int high = HexNibble( expectedHex[ index * 2 ] ); + const int low = HexNibble( expectedHex[ index * 2 + 1 ] ); + if ( high < 0 || low < 0 || actual[ index ] != static_cast( ( high << 4 ) | low ) ) { + std::fprintf( stderr, "Crypto vector failed for %s at byte %zu\n", label, index ); + failures++; + return; + } + } +} + +static void ExerciseCryptoVectors() { + std::uint8_t digest[idCrypto::SHA256_DIGEST_BYTES]; + idCrypto::SHA256( nullptr, 0, digest ); + ExpectCryptoBytes( digest, sizeof( digest ), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "SHA-256 empty" ); + idCrypto::SHA256( "abc", 3, digest ); + ExpectCryptoBytes( digest, sizeof( digest ), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "SHA-256 abc" ); + + std::uint8_t hmacKey[20]; + std::memset( hmacKey, 0x0b, sizeof( hmacKey ) ); + idCrypto::HMACSHA256( hmacKey, sizeof( hmacKey ), "Hi There", 8, digest ); + ExpectCryptoBytes( digest, sizeof( digest ), + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", "RFC 4231 HMAC-SHA-256" ); + + const char password[] = "password"; + const char salt[] = "salt"; + if ( !idCrypto::PBKDF2HMACSHA256( password, 8, salt, 4, 1, digest, sizeof( digest ) ) ) { + std::fprintf( stderr, "PBKDF2 iteration-1 vector rejected\n" ); + failures++; + } else { + ExpectCryptoBytes( digest, sizeof( digest ), + "120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b", "PBKDF2-HMAC-SHA-256 c=1" ); + } + if ( !idCrypto::PBKDF2HMACSHA256( password, 8, salt, 4, 2, digest, sizeof( digest ) ) ) { + std::fprintf( stderr, "PBKDF2 iteration-2 vector rejected\n" ); + failures++; + } else { + ExpectCryptoBytes( digest, sizeof( digest ), + "ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43", "PBKDF2-HMAC-SHA-256 c=2" ); + } + if ( !idCrypto::PBKDF2HMACSHA256( password, 8, salt, 4, 4096, digest, sizeof( digest ) ) ) { + std::fprintf( stderr, "PBKDF2 iteration-4096 vector rejected\n" ); + failures++; + } else { + ExpectCryptoBytes( digest, sizeof( digest ), + "c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a", "PBKDF2-HMAC-SHA-256 c=4096" ); + } + if ( idCrypto::PBKDF2HMACSHA256( password, 8, salt, 4, 0, digest, sizeof( digest ) ) ) { + std::fprintf( stderr, "PBKDF2 accepted zero iterations\n" ); + failures++; + } + + std::uint8_t verifier[idRcon2::VERIFIER_BYTES]; + std::uint8_t clientNonce[idRcon2::NONCE_BYTES]; + std::uint8_t serverNonce[idRcon2::NONCE_BYTES]; + std::uint8_t endpointBinding[idRcon2::ENDPOINT_BINDING_BYTES]; + std::uint8_t requestDigest[idRcon2::REQUEST_DIGEST_BYTES]; + std::uint8_t proof[idRcon2::PROOF_BYTES]; + for ( std::size_t index = 0; index < sizeof( verifier ); ++index ) { + verifier[index] = static_cast( index ); + } + for ( std::size_t index = 0; index < sizeof( clientNonce ); ++index ) { + clientNonce[index] = static_cast( index ); + serverNonce[index] = static_cast( index + 16 ); + endpointBinding[index] = static_cast( index + 32 ); + } + for ( std::size_t index = 0; index < sizeof( requestDigest ); ++index ) { + requestDigest[index] = static_cast( index + 48 ); + } + idRcon2::ComputeProof( verifier, clientNonce, serverNonce, endpointBinding, requestDigest, proof ); + ExpectCryptoBytes( proof, sizeof( proof ), + "b5a5953f5327179b21f82fb5fe6a158896c82df1126feb48b0c8ce96a2f65c20", "rcon2 proof domain" ); + idRcon2::HashRequest( "status", requestDigest ); + ExpectCryptoBytes( requestDigest, sizeof( requestDigest ), + "073c1634c496cdb649d1afe0a312bbb4b7e1741b271542e4a436c3b8824b1761", "rcon2 request digest" ); + + std::uint8_t different[idCrypto::SHA256_DIGEST_BYTES]; + std::memcpy( different, digest, sizeof( different ) ); + different[31] ^= 1; + if ( !idCrypto::ConstantTimeEquals( digest, digest, sizeof( digest ) ) || + idCrypto::ConstantTimeEquals( digest, different, sizeof( digest ) ) ) { + std::fprintf( stderr, "Constant-time equality result mismatch\n" ); + failures++; + } + idCrypto::SecureZero( different, sizeof( different ) ); + for ( std::uint8_t value : different ) { + if ( value != 0 ) { + std::fprintf( stderr, "SecureZero left data behind\n" ); + failures++; + break; + } + } +} + +static void ExpectOpenURLPolicy( const char *url, const bool expected, const char *label ) { + const bool actual = idURLPolicy::IsAllowedHTTPURL( url ); + if ( actual != expected ) { + std::fprintf( stderr, "Open URL policy failed for %s: expected %d, got %d\n", label, expected, actual ); + failures++; + } +} + +static void ExpectRemoteCVarPolicy( const int variableFlags, const int requiredFlag, + const bool expected, const char *label ) { + const int userInfo = 1 << 0; + const int serverInfo = 1 << 1; + const int networkSync = 1 << 2; + const int privateFlag = 1 << 3; + const int allowed = userInfo | serverInfo | networkSync; + const bool actual = idRemoteCVarPolicy::CanApply( + variableFlags, requiredFlag, allowed, privateFlag ); + if ( actual != expected ) { + std::fprintf( stderr, "Remote CVar policy failed for %s: expected %d, got %d\n", + label, expected, actual ); + failures++; + } +} + +static void ExpectGameDirSegment( const char *segment, const bool expected, const char *label ) { + const bool actual = idGameDirPolicy::IsPortableSegment( segment ); + if ( actual != expected ) { + std::fprintf( stderr, "Game-directory policy failed for %s: expected %d, got %d\n", + label, expected, actual ); + failures++; + } +} + static void ExpectEndpoint( const char *text, const bool expected, const char *expectedHost, const bool expectedHasPort, const unsigned short expectedPort, const char *label ) { char host[256] = "untouched"; @@ -164,6 +331,36 @@ static void ExpectBindPlan( const char *ipv4Text, const char *ipv6Text, const bo } int main() { + ExerciseCryptoVectors(); + const int remoteUserInfo = 1 << 0; + const int remoteServerInfo = 1 << 1; + const int remoteNetworkSync = 1 << 2; + const int remotePrivate = 1 << 3; + const int localInit = 1 << 4; + ExpectRemoteCVarPolicy( remoteUserInfo, remoteUserInfo, true, "matching userinfo authority" ); + ExpectRemoteCVarPolicy( remoteServerInfo, remoteServerInfo, true, "matching serverinfo authority" ); + ExpectRemoteCVarPolicy( remoteNetworkSync, remoteNetworkSync, true, "matching networksync authority" ); + ExpectRemoteCVarPolicy( remoteUserInfo, remoteNetworkSync, false, "cross-class authority" ); + ExpectRemoteCVarPolicy( localInit, remoteUserInfo, false, "local-only CVar" ); + ExpectRemoteCVarPolicy( remoteUserInfo | remotePrivate, remoteUserInfo, false, "private CVar" ); + ExpectRemoteCVarPolicy( remoteUserInfo, 0, false, "empty authority" ); + ExpectRemoteCVarPolicy( remoteUserInfo | remoteNetworkSync, + remoteUserInfo | remoteNetworkSync, false, "combined authority" ); + ExpectRemoteCVarPolicy( localInit, localInit, false, "unknown authority" ); + ExpectPrivateToken( "set net_serverRemoteConsolePassword secret", + "net_serverRemoteConsolePassword", true, "ordinary assignment" ); + ExpectPrivateToken( "status; SET NET_SERVERREMOTECONSOLEPASSWORD secret", + "net_serverRemoteConsolePassword", true, "case-insensitive semicolon assignment" ); + ExpectPrivateToken( "net_serverRemoteConsolePasswor", + "net_serverRemoteConsolePassword", false, "command ends in private-name prefix" ); + ExpectPrivateToken( "net_serverRemoteConsolePasswordSuffix", + "net_serverRemoteConsolePassword", false, "private name is a longer token prefix" ); + ExpectPrivateToken( "xnet_serverRemoteConsolePassword", + "net_serverRemoteConsolePassword", false, "private name lacks left boundary" ); + ExpectPrivateToken( "echo net_serverRemoteConsolePassword", + "net_serverRemoteConsolePassword", true, "private name at command end" ); + ExpectPrivateToken( nullptr, "net_private", false, "null command" ); + ExpectPrivateToken( "set net_private x", "", false, "empty private name" ); const char *validDecimals[] = { "0", "-0", "123", "-123", "1.0", ".5", "-.5", "5." }; @@ -190,6 +387,39 @@ int main() { ExpectBounded( "12x", 127, false, 0, "non-digit" ); ExpectBounded( "0", -1, false, 0, "negative maximum" ); + ExpectGameDirSegment( "baseoq4", true, "ordinary game directory" ); + ExpectGameDirSegment( "my-mod_2", true, "portable punctuation" ); + ExpectGameDirSegment( ".hidden-mod", true, "non-dot hidden directory" ); + ExpectGameDirSegment( nullptr, false, "null game directory" ); + ExpectGameDirSegment( "", false, "empty game directory" ); + ExpectGameDirSegment( ".", false, "dot game directory" ); + ExpectGameDirSegment( "..", false, "parent game directory" ); + ExpectGameDirSegment( "../escape", false, "parent path escape" ); + ExpectGameDirSegment( "mod/child", false, "forward-slash path" ); + ExpectGameDirSegment( "mod\\child", false, "backslash path" ); + ExpectGameDirSegment( "C:mod", false, "volume-relative path" ); + ExpectGameDirSegment( " leading", false, "leading space" ); + ExpectGameDirSegment( "trailing. ", false, "normalized trailing characters" ); + ExpectGameDirSegment( "bad*mod", false, "reserved punctuation" ); + ExpectGameDirSegment( "CON", false, "Windows device directory" ); + ExpectGameDirSegment( "com1.mod", false, "numbered Windows device directory" ); + ExpectGameDirSegment( "lpt\xC2\xB3", false, "UTF-8 superscript Windows device directory" ); + { + char maximumSegment[idGameDirPolicy::MAX_SEGMENT_BYTES + 1]; + for ( int index = 0; index < idGameDirPolicy::MAX_SEGMENT_BYTES; ++index ) { + maximumSegment[index] = 'm'; + } + maximumSegment[idGameDirPolicy::MAX_SEGMENT_BYTES] = '\0'; + ExpectGameDirSegment( maximumSegment, true, "maximum game-directory segment" ); + + char oversizedSegment[idGameDirPolicy::MAX_SEGMENT_BYTES + 2]; + for ( int index = 0; index <= idGameDirPolicy::MAX_SEGMENT_BYTES; ++index ) { + oversizedSegment[index] = 'm'; + } + oversizedSegment[idGameDirPolicy::MAX_SEGMENT_BYTES + 1] = '\0'; + ExpectGameDirSegment( oversizedSegment, false, "oversized game-directory segment" ); + } + using namespace idStrAllocationDetail; const size_t sizeMaximum = ( std::numeric_limits::max )(); ExpectAllocation( SaturatingAdd( 17, 25 ) == 42, "ordinary addition" ); @@ -210,6 +440,61 @@ int main() { ExpectAllocation( !TryRoundUpToInt( sizeMaximum, 32, roundedAmount ), "SIZE_MAX round-up" ); ExpectAllocation( !TryRoundUpToInt( 1, 0, roundedAmount ), "zero granularity" ); + ExpectOpenURLPolicy( "http://example.com", true, "ordinary HTTP URL" ); + ExpectOpenURLPolicy( "HTTPS://example.com:443/releases?q=openq4#download", true, "ordinary HTTPS URL" ); + ExpectOpenURLPolicy( "https://localhost", true, "localhost HTTPS URL" ); + ExpectOpenURLPolicy( "http://127.0.0.1:8080/path", true, "IPv4 HTTP URL" ); + ExpectOpenURLPolicy( "https://[2001:db8::1]:65535/path", true, "IPv6 HTTPS URL" ); + ExpectOpenURLPolicy( "https://example.com/path@name?redirect=%2Fsafe", true, "reserved path characters" ); + ExpectOpenURLPolicy( nullptr, false, "null URL" ); + ExpectOpenURLPolicy( "", false, "empty URL" ); + ExpectOpenURLPolicy( "ftp://example.com/file", false, "FTP scheme" ); + ExpectOpenURLPolicy( "file:///tmp/update", false, "file scheme" ); + ExpectOpenURLPolicy( "javascript:alert(1)", false, "script scheme" ); + ExpectOpenURLPolicy( "https:example.com", false, "HTTPS URL without authority delimiter" ); + ExpectOpenURLPolicy( "https:///missing-host", false, "empty authority" ); + ExpectOpenURLPolicy( "https://:443/path", false, "empty host with port" ); + ExpectOpenURLPolicy( "https://user@example.com/path", false, "userinfo authority" ); + ExpectOpenURLPolicy( "https://example.com%40attacker.invalid/path", false, "encoded authority ambiguity" ); + ExpectOpenURLPolicy( "https://example.com:65536/path", false, "out-of-range URL port" ); + ExpectOpenURLPolicy( "https://example.com:/path", false, "empty URL port" ); + ExpectOpenURLPolicy( "https://example.com:443:444/path", false, "ambiguous URL port" ); + ExpectOpenURLPolicy( "https://2001:db8::1/path", false, "unbracketed IPv6 URL" ); + ExpectOpenURLPolicy( "https://[::1]suffix/path", false, "text after bracketed URL host" ); + ExpectOpenURLPolicy( "https://!/path", false, "invalid hostname character" ); + ExpectOpenURLPolicy( "https://./path", false, "empty DNS labels" ); + ExpectOpenURLPolicy( "https://-example.com/path", false, "leading hostname hyphen" ); + ExpectOpenURLPolicy( "https://example-.com/path", false, "trailing hostname hyphen" ); + ExpectOpenURLPolicy( "https://999.1.2.3/path", false, "invalid IPv4 octet" ); + ExpectOpenURLPolicy( "https://127.1/path", false, "ambiguous abbreviated IPv4" ); + ExpectOpenURLPolicy( "https://[not-an-ip]/path", false, "invalid bracketed IP literal" ); + ExpectOpenURLPolicy( "https://[::::]/path", false, "invalid IPv6 compression" ); + ExpectOpenURLPolicy( "https://example.com/line\nbreak", false, "URL newline" ); + ExpectOpenURLPolicy( "https://example.com/space here", false, "URL space" ); + ExpectOpenURLPolicy( "https://example.com\\attacker.invalid", false, "URL backslash" ); + ExpectOpenURLPolicy( "https://example.com/\x7f", false, "URL delete control" ); + + { + char maximumLengthURL[idURLPolicy::MAX_URL_BYTES]; + const char prefix[] = "https://example.com/"; + for ( size_t index = 0; index < sizeof( prefix ) - 1; index++ ) { + maximumLengthURL[index] = prefix[index]; + } + for ( size_t index = sizeof( prefix ) - 1; index < idURLPolicy::MAX_URL_BYTES - 1; index++ ) { + maximumLengthURL[index] = 'a'; + } + maximumLengthURL[idURLPolicy::MAX_URL_BYTES - 1] = '\0'; + ExpectOpenURLPolicy( maximumLengthURL, true, "maximum bounded URL" ); + + char oversizedURL[idURLPolicy::MAX_URL_BYTES + 1]; + for ( size_t index = 0; index < idURLPolicy::MAX_URL_BYTES; index++ ) { + oversizedURL[index] = maximumLengthURL[index]; + } + oversizedURL[idURLPolicy::MAX_URL_BYTES - 1] = 'a'; + oversizedURL[idURLPolicy::MAX_URL_BYTES] = '\0'; + ExpectOpenURLPolicy( oversizedURL, false, "oversized URL" ); + } + ExpectEndpoint( "127.0.0.1", true, "127.0.0.1", false, 0, "numeric IPv4" ); ExpectEndpoint( "127.0.0.1:65535", true, "127.0.0.1", true, 65535, "maximum IPv4 port" ); ExpectEndpoint( "4.example.com:27960", true, "4.example.com", true, 27960, "digit-leading hostname" ); diff --git a/tools/tests/network_security.py b/tools/tests/network_security.py new file mode 100644 index 00000000..60a09a61 --- /dev/null +++ b/tools/tests/network_security.py @@ -0,0 +1,897 @@ +#!/usr/bin/env python3 +"""Focused source contracts for connection, rcon2, and private-CVar security.""" + +from __future__ import annotations + +import os +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +GAME_LIBS_ROOT = Path( + os.environ.get("OPENQ4_GAMELIBS_REPO", ROOT.parent / "openQ4-game") +).resolve() + + +def read(relative_path: str) -> str: + path = ROOT / relative_path + if not path.is_file(): + raise AssertionError(f"Required file not found: {path}") + return path.read_text(encoding="utf-8", errors="strict") + + +def read_game(relative_path: str) -> str: + path = GAME_LIBS_ROOT / relative_path + if not path.is_file(): + raise AssertionError(f"Required openQ4-game file not found: {path}") + return path.read_text(encoding="utf-8", errors="strict") + + +def require(text: str, token: str, context: str) -> None: + if token not in text: + raise AssertionError(f"Missing {token!r} in {context}") + + +def reject(text: str, token: str, context: str) -> None: + if token in text: + raise AssertionError(f"Unexpected {token!r} in {context}") + + +def require_before(text: str, first: str, second: str, context: str) -> None: + require(text, first, context) + require(text, second, context) + if text.index(first) >= text.index(second): + raise AssertionError(f"Expected {first!r} before {second!r} in {context}") + + +def function_body(source: str, signature: str, context: str) -> str: + start = source.find(signature) + if start < 0: + raise AssertionError(f"Missing {signature!r} in {context}") + opening = source.find("{", start + len(signature)) + if opening < 0: + raise AssertionError(f"Missing body for {signature!r} in {context}") + depth = 0 + for index in range(opening, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start : index + 1] + raise AssertionError(f"Unbalanced body for {signature!r} in {context}") + + +def validate_crypto_and_native_vectors() -> None: + crypto = read("src/idlib/CryptoHash.cpp") + protocol = read("src/framework/async/Rcon2Protocol.cpp") + protocol_header = read("src/framework/async/Rcon2Protocol.h") + native = read("tools/tests/native/CoreSafetyTest.cpp") + meson = read("meson.build") + + for token in ( + "original implementation of the published FIPS 180-4 SHA-256", + "void SHA256(", + "void HMACSHA256(", + "bool PBKDF2HMACSHA256(", + "bool ConstantTimeEquals(", + "void SecureZero(", + ): + require(crypto, token, "engine-owned cryptographic primitives") + require(protocol, 'PROOF_DOMAIN[] = "openQ4-rcon2-proof-v1"', "rcon2 domain separation") + require(protocol_header, "PBKDF2_ITERATIONS = 200000", "fixed rcon2 KDF work factor") + require(protocol_header, "MIN_PASSWORD_BYTES = 12", "minimum rcon2 password length") + + for token in ( + "RFC 4231 HMAC-SHA-256", + "PBKDF2-HMAC-SHA-256 c=1", + "PBKDF2-HMAC-SHA-256 c=2", + "PBKDF2-HMAC-SHA-256 c=4096", + "rcon2 proof domain", + "rcon2 request digest", + "SecureZero left data behind", + '"command ends in private-name prefix"', + '"private name is a longer token prefix"', + '"private name at command end"', + '"parent path escape"', + '"forward-slash path"', + '"backslash path"', + '"Windows device directory"', + '"oversized game-directory segment"', + ): + require(native, token, "native crypto/private-token coverage") + for token in ( + "src/idlib/CryptoHash.cpp", + "src/framework/async/Rcon2Protocol.cpp", + ): + require(meson, token, "native safety target sources") + + +def validate_secure_random_and_connection_challenges() -> None: + client = read("src/framework/async/AsyncClient.cpp") + server = read("src/framework/async/AsyncServer.cpp") + + for source, label in ((client, "async client"), (server, "async server")): + reject(source, "rand()", label) + reject(source, "idRandom random", label) + reject(source, "Sys_Milliseconds() & CONNECTIONLESS_MESSAGE_ID_MASK", label) + + client_id = function_body(client, "static bool AsyncClient_SecureConnectionId(", "client ID generator") + server_id = function_body(server, "static bool AsyncServer_SecureConnectionId(", "server ID generator") + issue = function_body(server, "void idAsyncServer::ProcessChallengeMessage(", "connection challenge issuance") + validate = function_body(server, "int idAsyncServer::ValidateChallenge(", "connection challenge validation") + download = function_body(client, "int idAsyncClient::GetDownloadRequest(", "download request ID") + + for body, label in ( + (client_id, "client ID generator"), + (server_id, "server ID generator"), + (issue, "connection challenge issuance"), + (download, "download request ID"), + ): + require(body, "Sys_GetSecureRandomBytes", label) + require(issue, "msg.GetRemainingData() != 4", "bounded challenge payload") + require(issue, "challenges[i].valid = true", "explicit challenge validity") + require(issue, "challenges[i].address = from", "challenge endpoint binding") + require(issue, "challenges[i].clientId = clientId", "challenge client-ID binding") + require(issue, "AsyncServer_Elapsed( serverTime, challenges[ i ].time )", "wrap-safe challenge age") + reject(issue, "challenges[i].time < oldestTime", "raw signed challenge eviction") + + for token in ( + "challenges[ i ].valid", + "!challenges[ i ].connected", + "AsyncServer_Elapsed( serverTime, challenges[ i ].time ) <= CONNECTION_CHALLENGE_TIMEOUT_MSEC", + "AsyncServer_SameEndpoint( from, challenges[i].address )", + "clientId == challenges[ i ].clientId", + "challenge == challenges[i].challenge", + ): + require(validate, token, "bound, expiring challenge validation") + require(validate, "AllowConnectionlessResponse( from, false )", "bounded bad-challenge reply") + require(download, "dlRequest = -1", "download request fail-closed path") + require(server, "AsyncServer_ClearChallenge( challenges[ ichallenge ] );", "one-use successful connection challenge") + + +def validate_oob_budgets() -> None: + server = read("src/framework/async/AsyncServer.cpp") + limiter = function_body(server, "bool idAsyncServer::AllowConnectionlessResponse(", "OOB response limiter") + info = function_body(server, "void idAsyncServer::ProcessGetInfoMessage(", "getInfo handler") + challenge = function_body(server, "void idAsyncServer::ProcessChallengeMessage(", "challenge handler") + rcon_challenge = function_body( + server, + "void idAsyncServer::ProcessRemoteConsole2ChallengeMessage(", + "rcon2 challenge handler", + ) + connect = function_body(server, "void idAsyncServer::ProcessConnectMessage(", "connect handler") + pure = function_body(server, "void idAsyncServer::ProcessPureMessage(", "pure handler") + download = function_body(server, "void idAsyncServer::ProcessDownloadRequestMessage(", "download handler") + connectionless = function_body(server, "bool idAsyncServer::ConnectionlessMessage(", "server OOB dispatcher") + process_message = function_body(server, "bool idAsyncServer::ProcessMessage(", "server packet dispatcher") + + for token in ( + "OOB_INFO_MAX_PER_SOURCE", + "OOB_CHALLENGE_MAX_PER_SOURCE", + "OOB_INFO_MAX_GLOBAL", + "OOB_CHALLENGE_MAX_GLOBAL", + "Sys_IsLANAddress( from )", + "oobInfoResponses >= OOB_INFO_MAX_GLOBAL", + "oobChallengeResponses >= OOB_CHALLENGE_MAX_GLOBAL", + ): + require(limiter, token, "per-source/global OOB response budget") + require(info, "msg.GetRemainingData() != 4", "bounded getInfo request") + require(info, "AllowConnectionlessResponse( from, true )", "bounded infoResponse") + require_before(challenge, "AllowConnectionlessResponse( from, false )", "serverPort.SendPacket", "challenge response budget") + require_before(rcon_challenge, "AllowConnectionlessResponse( from, false )", "serverPort.SendPacket", "rcon2 response budget") + require_before(connect, "ValidateChallenge( from, challenge, clientId )", "protocol != ASYNC_PROTOCOL_VERSION", "challenge before protocol reply") + require_before(connect, "OS < 0 || OS >= MAX_GAME_OS", "ValidateChallenge( from, challenge, clientId )", "client OS bounds before challenge state") + require_before(connect, "AllowConnectionlessResponse( from, false )", "protocol != ASYNC_PROTOCOL_VERSION", "connect reply budget") + require_before(connect, "AllowConnectionlessResponse( from, false )", "clientDataChecksum != serverDataChecksum", "checksum reply budget") + require(pure, "AllowConnectionlessResponse( from, false )", "pure failure reply budget") + require(download, "AllowConnectionlessResponse( from, false )", "download response budget") + inactive = connectionless[connectionless.index("if ( !active )") :] + require_before(inactive, "AllowConnectionlessResponse( from, false )", "PrintOOB", "inactive-server reply budget") + unknown_client = process_message[process_message.index("// if we received a sequenced packet") :] + require_before( + unknown_client, + "AllowConnectionlessResponse( from, false )", + "serverPort.SendPacket", + "unknown sequenced-client disconnect budget", + ) + + +def validate_usercmd_packet_bounds() -> None: + header = read("src/framework/async/AsyncNetwork.h") + network = read("src/framework/async/AsyncNetwork.cpp") + client = read("src/framework/async/AsyncClient.cpp") + server = read("src/framework/async/AsyncServer.cpp") + decoder = function_body( + network, + "bool idAsyncNetwork::ReadUserCmdDelta(", + "bounded user-command decoder", + ) + handler = function_body( + server, + "void idAsyncServer::ProcessUnreliableClientMessage(", + "server unreliable-client handler", + ) + sender = function_body( + client, + "void idAsyncClient::SendUsercmdsToServer(", + "client user-command sender", + ) + connect_response = function_body( + client, + "void idAsyncClient::ProcessConnectResponseMessage(", + "client connect-response handler", + ) + connect_request = function_body( + server, + "void idAsyncServer::ProcessConnectMessage(", + "server connect-request handler", + ) + snapshot_sender = function_body( + server, + "bool idAsyncServer::SendSnapshotToClient(", + "server snapshot sender", + ) + client_handler = function_body( + client, + "void idAsyncClient::ProcessUnreliableServerMessage(", + "client unreliable-server handler", + ) + game_init = client_handler[client_handler.index("case SERVER_UNRELIABLE_MESSAGE_GAMEINIT") :] + snapshot = client_handler[client_handler.index("case SERVER_UNRELIABLE_MESSAGE_SNAPSHOT") :] + + require(header, "MAX_USERCMD_PACKET_COMMANDS = 11", "wire user-command work bound") + require(header, "static bool\t\t\t\tReadUserCmdDelta", "fallible user-command decoder API") + require(client, "MAX_USERCMD_PACKET_COMMANDS - 1", "client packet-count bound") + require(sender, "gameFrame >= requestedUsercmds - 1 ? requestedUsercmds : gameFrame + 1", "startup user-command history clamp") + require(sender, "gameFrame > AsyncClient_MaxNetworkGameFrame()", "client frame-domain guard") + reject(sender, "Min( requestedUsercmds, gameFrame + 1 )", "overflow-prone startup history clamp") + require(network, "AsyncNetwork_CanReadUserCmdDelta", "user-command bit preflight") + require(network, "probe.GetRemainingReadBits() >= 152", "full-command bit bound") + require(decoder, "if ( !AsyncNetwork_CanReadUserCmdDelta", "decode before mutation") + require(decoder, "usercmd_t decoded;", "transactional user-command decode") + require(decoder, "cmd = decoded;", "commit only complete user-command") + require(handler, "msg.GetRemainingReadBits() < 64", "fixed unreliable-message prefix bound") + require(handler, "msg.GetRemainingReadBits() < 32 + 8", "snapshot/id header bound") + ping_handler = handler[handler.index("case CLIENT_UNRELIABLE_MESSAGE_PINGRESPONSE") :] + require_before(ping_handler, "msg.GetRemainingReadBits() < 32", "msg.ReadLong()", "ping-response payload bound") + require(handler, "msg.GetRemainingReadBits() < 16 + 32 + 8", "fixed user-command header bound") + require(handler, "numUsercmds > MAX_USERCMD_PACKET_COMMANDS", "server packet-count bound") + require(handler, "clientGameFrame < numUsercmds - 1", "negative user-command history rejection") + require( + handler, + "static_cast( clientGameFrame ) > static_cast( gameFrame ) + MAX_USERCMD_BACKUP", + "implausible future client-frame rejection", + ) + require(handler, "commandIndex < numUsercmds", "counted user-command loop") + require(handler, "if ( !idAsyncNetwork::ReadUserCmdDelta", "truncated client packet rejection") + require(handler, "echoedPingTime != client.lastPingTime", "ping response challenge binding") + require(handler, "AsyncServer_Elapsed( realTime, echoedPingTime )", "overflow-safe ping calculation") + reject(handler, "i <= clientGameFrame", "overflow-prone frame-sentinel loop") + require(client, "if ( !idAsyncNetwork::ReadUserCmdDelta", "truncated server snapshot rejection") + require(client, "i > MAX_ASYNC_CLIENTS", "snapshot relay terminator bound") + require(client, "i == MAX_ASYNC_CLIENTS", "snapshot relay exact terminator") + require(client, "numUsercmds < 1 || numUsercmds > MAX_USERCMD_RELAY", "snapshot relay count bound") + require(client, "server sent a truncated snapshot header", "snapshot fixed-header bound") + require(client, "server sent invalid snapshot timing", "snapshot timing-domain validation") + require(client, "server sent invalid game-init timing", "game-init timing-domain validation") + + teardown = function_body( + client, + "static void AsyncClient_StopAfterMalformedSnapshot(", + "malformed snapshot session teardown", + ) + require(teardown, "arenaCampaign.AbortMatch();", "malformed snapshot Arena rollback") + require(teardown, "session->Stop();", "malformed snapshot map teardown") + post_game_decode = snapshot[snapshot.index("if ( !game->ClientReadSnapshot(") :] + if post_game_decode.count("AsyncClient_StopAfterMalformedSnapshot();") != 4: + raise AssertionError( + "all post-game snapshot failures must tear down the session before the outer frame continues" + ) + reject(post_game_decode, "DisconnectFromServer();", "post-game snapshot channel-only teardown") + + require(client, "AsyncClient_MaxPredictionMsec", "bounded prediction arithmetic") + require(client, "const std::int64_t adjustedPredictTime", "widened prediction adjustment") + require(client, "adjustedPredictTime > maximumPredictionMsec", "prediction adjustment clamp") + require(client, "static ID_INLINE bool AsyncClient_ValidNetworkTiming", "tick-derived network timing helper") + require(client, "common->GetUserCmdMsecNumerator()", "exact user-command tick frame bound") + require(client, "common->GetUserCmdMSec()", "legacy user-command tick frame bound") + require(connect_response, "msg.GetRemainingReadBits() < 32 + 32 + 32 + 32", "connect-response fixed-header bound") + require(connect_response, "AsyncClient_ValidNetworkTiming( serverGameFrame, serverGameTime )", "connect-response timing bound") + require_before(connect_response, "msg.IsReadOverflowed()", "channel.Init", "connect response validation before state mutation") + require(connect_response, "AsyncClient_Elapsed( clientTime, lastConnectTime )", "overflow-safe initial prediction interval") + require(connect_response, "AsyncClient_MaxPredictionMsec()", "bounded initial prediction clamp") + reject(connect_response, "clientTime - lastConnectTime", "overflow-prone initial prediction interval") + require_before(game_init, "!AsyncClient_ValidNetworkTiming", "InitGame", "game-init timing validation before state mutation") + require_before(snapshot, "!AsyncClient_ValidNetworkTiming", "snapshotGameFrame = receivedSnapshotGameFrame", "snapshot timing validation before state mutation") + require(connect_request, "connect from %s rejected: truncated GUID", "truncated connect GUID rejection") + require(connect_request, "connect from %s rejected: truncated password", "truncated connect password rejection") + require_before(connect_request, "truncated password", "game->ServerAllowClient", "password validation before game callback") + require(connect_request, "AsyncServer_Elapsed( serverTime, challenges[ ichallenge ].pingTime )", "overflow-safe connect ping") + require(snapshot_sender, "const std::int64_t clientAheadTime", "widened client-ahead arithmetic") + require(snapshot_sender, "clientAheadTime < idMath::INT_MIN", "client-ahead lower clamp") + require(snapshot_sender, "clientAheadTime > idMath::INT_MAX", "client-ahead upper clamp") + + +def validate_snapshot_decode_bounds() -> None: + engine_bitmsg = read("src/idlib/BitMsg.cpp") + engine_bitmsg_header = read("src/idlib/BitMsg.h") + game_bitmsg = read_game("src/idlib/BitMsg.cpp") + game_bitmsg_header = read_game("src/idlib/BitMsg.h") + sp_network = read_game("src/game/Game_network.cpp") + mp_network = read_game("src/mpgame/Game_network.cpp") + sp_player = read_game("src/game/Player.cpp") + mp_player = read_game("src/mpgame/Player.cpp") + sp_multiplayer = read_game("src/game/MultiplayerGame.cpp") + mp_multiplayer = read_game("src/mpgame/MultiplayerGame.cpp") + sp_projectile = read_game("src/game/Projectile.cpp") + mp_projectile = read_game("src/mpgame/Projectile.cpp") + sp_particle = read_game("src/game/physics/Physics_Particle.cpp") + mp_particle = read_game("src/mpgame/physics/Physics_Particle.cpp") + sp_player_physics = read_game("src/game/physics/Physics_Player.cpp") + mp_player_physics = read_game("src/mpgame/physics/Physics_Player.cpp") + sp_entity = read_game("src/game/Entity.cpp") + mp_entity = read_game("src/mpgame/Entity.cpp") + sp_weapon = read_game("src/game/Weapon.cpp") + mp_weapon = read_game("src/mpgame/Weapon.cpp") + sp_af = read_game("src/game/physics/Physics_AF.cpp") + mp_af = read_game("src/mpgame/physics/Physics_AF.cpp") + + for header, label in ( + (engine_bitmsg_header, "engine bit-message API"), + (game_bitmsg_header, "game bit-message API"), + ): + require(header, "IsReadOverflowed", label) + require(header, "MarkReadOverflowed", label) + require(header, "return readBit == 8;", label) + require(header, "if ( !IsReadOverflowed() )", label) + + delta_overflow = function_body( + header, + "ID_INLINE bool idBitMsgDelta::IsReadOverflowed(", + f"{label} delta overflow policy", + ) + require( + delta_overflow, + "return readDelta != NULL ? readDelta->IsReadOverflowed()", + f"{label} conditional-tail base exhaustion policy", + ) + reject( + delta_overflow, + "( base != NULL && base->IsReadOverflowed() ) ||", + f"{label} false aggregate base-overflow rejection", + ) + + for source, label in ( + (engine_bitmsg, "engine bit-message implementation"), + (game_bitmsg, "game bit-message implementation"), + ): + read_bits = function_body(source, "int idBitMsg::ReadBits(", label) + read_data = function_body(source, "int idBitMsg::ReadData(", label) + long_counter = function_body(source, "int idBitMsg::ReadDeltaLongCounter(", label) + require_before(read_bits, "numBits > GetRemainingReadBits()", "return -1;", label) + require(read_bits, "MarkReadOverflowed();", label) + require(read_bits, "uint32_t\tvalue;", label) + require(read_bits, "1u << ( numBits - 1 )", label) + require(read_data, "length < 0", label) + require(read_data, "memset( static_cast( data ) + remaining, 0, length - remaining );", label) + require(read_data, "MarkReadOverflowed();", label) + require(long_counter, "i > 31", label) + require(long_counter, "1u << i", label) + reject(long_counter, "1 << i", label) + + delta_write_bits = function_body(source, "void idBitMsgDelta::WriteBits(", label) + delta_read_bits = function_body(source, "int idBitMsgDelta::ReadBits(", label) + delta_read_delta = function_body(source, "int idBitMsgDelta::ReadDelta(", label) + require( + delta_write_bits, + "!base->IsReadOverflowed() && baseValue == value", + f"{label} exhausted base forces explicit replacement", + ) + for reader, reader_label in ( + (delta_read_bits, "plain delta field"), + (delta_read_delta, "old-value delta field"), + ): + require(reader, "const bool baseOverflowed = base->IsReadOverflowed();", f"{label} {reader_label}") + require_before( + reader, + "readDelta->ReadBits( 1 ) == 0", + "readDelta->MarkReadOverflowed();", + f"{label} {reader_label} unavailable-base reuse rejection", + ) + + queue = function_body(game_bitmsg, "void idMsgQueue::ReadFrom(", "game unreliable-message queue") + require(queue, "encodedSize >= MAX_MSG_QUEUE_SIZE", "game unreliable-message queue bound") + require(queue, "encodedSize > remaining", "game unreliable-message queue payload bound") + require(queue, "invalid nested record size", "game unreliable-message nested record bound") + require(queue, "msg.MarkReadOverflowed();", "game unreliable-message semantic failure propagation") + require_before(queue, "encodedSize >= MAX_MSG_QUEUE_SIZE", "msg.ReadData( buffer", "queue bound before copy") + + for source, label, client_bound in ( + (sp_network, "SP snapshot", "clientNum < 0 || clientNum >= MAX_CLIENTS"), + (mp_network, "MP snapshot", "clientNum < 0 || clientNum > MAX_CLIENTS"), + ): + snapshot = function_body(source, "bool idGameLocal::ClientReadSnapshot(", label) + require_before(snapshot, client_bound, "entities[ clientNum ]", f"{label} client bound") + require(snapshot, "truncated unreliable-message queue", label) + require(snapshot, "deltaMsg.IsReadOverflowed()", label) + require(snapshot, "truncated entity state", label) + require(snapshot, "truncated PVS state", label) + require(snapshot, "truncated baseline state", label) + + sp_snapshot = function_body(sp_network, "bool idGameLocal::ClientReadSnapshot(", "SP snapshot") + require_before(sp_snapshot, "targetPlayer < 0 || targetPlayer >= MAX_CLIENTS", "entities[ targetPlayer ]", "SP target-player bound") + require(sp_snapshot, "non-player entity %d has player state", "SP player-state type check") + require(sp_player, "msg.ReadBits( -idMath::BitsForInteger( MAX_WEAPONS ) )", "SP ideal-weapon signed wire decode") + + mp_snapshot = function_body(mp_network, "bool idGameLocal::ClientReadSnapshot(", "MP snapshot") + require_before(mp_snapshot, "clientEntity->IsType( idPlayer::GetClassType() )", "static_cast( clientEntity )", "MP local-player type check before cast") + repeater = function_body(mp_network, "void idGameLocal::ClientReadRepeaterSnapshot(", "MP repeater snapshot") + require(repeater, "if ( !ClientReadSnapshot(", "MP repeater malformed-snapshot propagation") + require(repeater, "common->Error( \"ClientReadRepeaterSnapshot: malformed snapshot %d\"", "MP repeater session abort") + require(repeater, "\n\t\treturn;", "MP repeater decl-validation return") + reject(repeater, "(void)ClientReadSnapshot", "MP repeater ignored decode result") + + for source, label in ((sp_player, "SP player snapshot"), (mp_player, "MP player snapshot")): + player = function_body(source, "void idPlayer::ReadFromSnapshot(", label) + require_before(player, "decodedSpectator < 0 || decodedSpectator >= MAX_CLIENTS", "gameLocal.entities[ spectator ]", label) + require(player, "newIdealWeapon >= MAX_WEAPONS", label) + require(player, "msg.MarkReadOverflowed();", label) + require(player, "static_cast( snapshotSequence )", f"{label} wrap-safe sequence delta") + require(player, "physicsObj.DecodeSnapshotState", label) + require(player, "DecodeBindSnapshotInfo", label) + require(player, "rvWeapon::DecodeSnapshotAmmo", label) + require(player, "if ( msg.IsReadOverflowed() ) {\n\t\treturn;\n\t}\n\n\tlastSnapshotSequence = snapshotSequence;", label) + require_before(player, "if ( msg.IsReadOverflowed() )", "lastSnapshotSequence = snapshotSequence;", label) + require_before(player, "if ( msg.IsReadOverflowed() )", "physicsObj.ApplySnapshotState", label) + require_before(player, "if ( msg.IsReadOverflowed() )", "ApplyBindSnapshotInfo", label) + require_before(player, "if ( msg.IsReadOverflowed() )", "weapon->ApplySnapshotAmmo", label) + reject(player, "physicsObj.ReadFromSnapshot( msg )", label) + reject(player, "ReadBindFromSnapshot( msg )", label) + reject(player, "health = msg.ReadShort()", label) + + for source, label in ( + (sp_multiplayer, "SP multiplayer state"), + (mp_multiplayer, "MP multiplayer state"), + ): + multiplayer = function_body(source, "void idMultiplayerGame::ReadFromSnapshot(", label) + require(multiplayer, "ingame[ MAX_CLIENTS / 8 ] = { 0 }", label) + require(multiplayer, "msg.IsReadOverflowed()", label) + require(multiplayer, "!ent->IsType( idPlayer::GetClassType() )", label) + require(multiplayer, "newInstance >= MAX_INSTANCES", label) + require(multiplayer, "mpPlayerState_t decodedPlayerState[ MAX_CLIENTS ]", label) + require(multiplayer, "hasTourneyState[ MAX_CLIENTS ] = { false }", label) + require(multiplayer, "if ( msg.IsReadOverflowed() ) {\n\t\treturn;\n\t}\n\n\tisBuyingAllowedRightNow = decodedBuyingAllowed;", label) + require_before(multiplayer, "if ( msg.IsReadOverflowed() )", "playerState[ i ].ingame = decodedPlayerState", label) + require_before(multiplayer, "if ( msg.IsReadOverflowed() )", "ent->SetInstance( decodedInstance", label) + + for source, label in ( + (sp_player_physics, "SP player-physics snapshot"), + (mp_player_physics, "MP player-physics snapshot"), + ): + decode = function_body(source, "bool idPhysics_Player::DecodeSnapshotState(", label) + apply = function_body(source, "void idPhysics_Player::ApplySnapshotState(", label) + require(decode, "decoded = current;", label) + require(decode, "return !msg.IsReadOverflowed();", label) + require_before(apply, "current = decoded;", "clipModel->Link", label) + + for source, label in ( + (sp_entity, "SP bind snapshot"), + (mp_entity, "MP bind snapshot"), + ): + bind_reader = function_body(source, "void idEntity::ReadBindFromSnapshot(", label) + require_before(bind_reader, "msg.IsReadOverflowed()", "ApplyBindSnapshotInfo", label) + + for source, label in ( + (sp_weapon, "SP weapon snapshot"), + (mp_weapon, "MP weapon snapshot"), + ): + weapon_reader = function_body(source, "void rvWeapon::ReadFromSnapshot(", label) + require_before(weapon_reader, "msg.IsReadOverflowed()", "ApplySnapshotAmmo", label) + + for source, label in ( + (sp_particle, "SP particle-physics snapshot"), + (mp_particle, "MP particle-physics snapshot"), + ): + decode = function_body(source, "bool rvPhysics_Particle::DecodeSnapshotState(", label) + apply = function_body(source, "void rvPhysics_Particle::ApplySnapshotState(", label) + reader = function_body(source, "void rvPhysics_Particle::ReadFromSnapshot(", label) + require(decode, "decoded = current;", label) + require(decode, "return !msg.IsReadOverflowed();", label) + require_before(apply, "current = decoded;", "clipModel->Link", label) + require_before(reader, "DecodeSnapshotState", "ApplySnapshotState", label) + + for source, label in ( + (sp_projectile, "SP projectile snapshot"), + (mp_projectile, "MP projectile snapshot"), + ): + projectile = function_body(source, "void idProjectile::ReadFromSnapshot(", label) + require(projectile, "physicsObj.DecodeSnapshotState", label) + require(projectile, "newLaunchOrig", label) + require_before(projectile, "if ( msg.IsReadOverflowed() )", "physicsObj.ApplySnapshotState", label) + require_before(projectile, "if ( msg.IsReadOverflowed() )", "launchOrig = newLaunchOrig", label) + require_before(projectile, "if ( msg.IsReadOverflowed() )", "Create(", label) + + mp_projectile_reader = function_body( + mp_projectile, + "void idProjectile::ReadFromSnapshot(", + "MP projectile snapshot", + ) + require(mp_projectile, "msg.WriteBits( ownerNum, idMath::BitsForInteger(MAX_CLIENTS) );", "MP projectile owner wire slot") + require(mp_projectile_reader, "int ownerNum = MAX_CLIENTS;", "MP projectile owner sentinel") + require_before(mp_projectile_reader, "ownerNum >= 0", "gameLocal.entities[ ownerNum ]", "projectile owner bound") + + for source, label in ( + (sp_af, "SP articulated-figure snapshot"), + (mp_af, "MP articulated-figure snapshot"), + ): + af_reader = function_body(source, "void idPhysics_AF::ReadFromSnapshot(", label) + require(af_reader, "AFPState_t decodedCurrent = current;", label) + require(af_reader, "idList< AFBodyPState_t > decodedBodies;", label) + require(af_reader, "num != bodies.Num()", label) + require_before(af_reader, "if ( msg.IsReadOverflowed() )", "current = decodedCurrent;", label) + require_before(af_reader, "current = decodedCurrent;", "UpdateClipModels();", label) + + +def validate_rcon2_flow() -> None: + network = read("src/framework/async/AsyncNetwork.cpp") + server = read("src/framework/async/AsyncServer.cpp") + client = read("src/framework/async/AsyncClient.cpp") + + require(network, 'serverAllowLegacyRcon( "net_serverAllowLegacyRcon", "0"', "legacy server rcon default-off") + require(network, 'clientUseLegacyRcon( "net_clientUseLegacyRcon", "0"', "legacy client rcon default-off") + for name in ("serverRemoteConsolePassword", "clientRemoteConsolePassword"): + line = next((line for line in network.splitlines() if name in line and "idCVar" in line), "") + require(line, "CVAR_PRIVATE", f"{name} private flag") + require(line, "CVAR_CASE_SENSITIVE", f"{name} case-sensitive flag") + + refresh = function_body(server, "bool idAsyncServer::RefreshRcon2Verifier(", "cached rcon2 verifier") + challenge = function_body(server, "void idAsyncServer::ProcessRemoteConsole2ChallengeMessage(", "rcon2 challenge") + proof = function_body(server, "void idAsyncServer::ProcessRemoteConsole2Message(", "rcon2 proof") + legacy = function_body(server, "void idAsyncServer::ProcessRemoteConsoleMessage(", "legacy rcon server") + remote = function_body(client, "void idAsyncClient::RemoteConsole(", "rcon client") + client_reply = function_body(client, "void idAsyncClient::ConnectionlessMessage(", "client OOB source gate") + disconnect = function_body(client, "void idAsyncClient::ProcessDisconnectMessage(", "client disconnect handler") + clear = function_body(client, "void idAsyncClient::ClearRemoteConsoleRequest(", "rcon client cleanup") + + require_before(refresh, "rcon2VerifierInitialized", "DeriveVerifier", "cached verifier before expensive KDF") + require(refresh, "serverRemoteConsolePassword.IsModified()", "password-change verifier invalidation") + for token in ( + "AsyncServer_SameEndpoint( from, candidate.address )", + "RCON2_CHALLENGE_TIMEOUT_MSEC", + "Sys_GetSecureRandomBytes( randomValues", + "issued.clientNonce", + "issued.serverNonce", + "issued.endpointBinding", + "issued.requestDigest", + ): + require(challenge, token, "bound rcon2 challenge") + require_before(proof, "idCrypto::SecureZero( &rcon2Challenges[ slot ]", "msg.ReadString( command", "one-shot proof consumption") + require(proof, "AsyncServer_SameEndpoint( from, candidate.address )", "exact rcon2 proof endpoint") + require(proof, "idRcon2::HashRequest( command, requestDigest )", "rcon2 command binding") + require(proof, "idCrypto::ConstantTimeEquals( suppliedProof", "constant-time proof comparison") + require(proof, "RecordRconFailure( from )", "failed-proof throttling") + require(proof, "SendRemoteConsole2Complete", "transaction completion marker") + + require(legacy, "serverAllowLegacyRcon.GetBool()", "explicit server legacy opt-in") + require(legacy, "ConstantTimeEquals", "legacy password comparison") + require(remote, "clientUseLegacyRcon.GetBool()", "explicit client legacy opt-in") + require(remote, "Sys_GetSecureRandomBytes( rcon2Request.clientNonce", "client rcon2 nonce") + require(clear, "SecureZero( &rcon2Request", "rcon2 state wipe") + require(clear, "memset( &lastRconAddress", "rcon reply-window cleanup") + require(client_reply, "AsyncClient_SameEndpoint( from, rcon2Request.address )", "exact rcon reply source") + require(client_reply, "rcon2Request.state == RCON_REPLY_OUTPUT", "post-proof output window") + require(client_reply, "!fromCurrentServer && !fromPendingRconOutput", "pre-proof print rejection") + require_before( + client_reply, + 'if ( idStr::Icmp( string, "rcon2ChallengeResponse" ) == 0 )', + "if ( !fromCurrentServer )", + "rcon opcode dispatch before game endpoint gate", + ) + require_before( + client_reply, + "if ( !fromCurrentServer )", + 'if ( idStr::Icmp( string, "challengeResponse" ) == 0 )', + "exact game endpoint gate before game control dispatch", + ) + rcon_challenge_dispatch = client_reply[ + client_reply.index('if ( idStr::Icmp( string, "rcon2ChallengeResponse" ) == 0 )') : + client_reply.index('if ( idStr::Icmp( string, "rcon2Complete" ) == 0 )') + ] + require(rcon_challenge_dispatch, "if ( !fromPendingRcon )", "pending-rcon-only challenge reply") + rcon_complete_dispatch = client_reply[ + client_reply.index('if ( idStr::Icmp( string, "rcon2Complete" ) == 0 )') : + client_reply.index('if ( idStr::Icmp( string, "print" ) == 0 )') + ] + require(rcon_complete_dispatch, "if ( !fromPendingRcon )", "pending-rcon-only completion reply") + reject( + client_reply, + "if ( !fromCurrentServer && !fromPendingRcon )", + "union endpoint capability gate", + ) + require(disconnect, "AsyncClient_SameEndpoint( from, serverAddress )", "exact disconnect endpoint") + reject(disconnect, "Sys_CompareNetAdrBase( from, serverAddress )", "base-address-only disconnect gate") + reject(server, '"rcon from %s: %s', "remote command logging") + reject(server, '"bad rcon from %s: %s', "remote password logging") + + +def validate_pure_admission_fail_closed() -> None: + server = read("src/framework/async/AsyncServer.cpp") + connect = function_body(server, "void idAsyncServer::ProcessConnectMessage(", "connect admission") + map_change = function_body(server, "void idAsyncServer::ExecuteMapChange(", "map-change pure admission") + unreliable = function_body( + server, + "void idAsyncServer::ProcessUnreliableClientMessage(", + "wrong-gameinit pure admission", + ) + + resend = connect[ + connect.index("case CDK_PUREWAIT:") : connect.index("case CDK_ONLYLAN:") + ] + for token in ( + "if ( !SendPureServerMessage( from, OS ) )", + "AsyncServer_ClearChallenge( challenges[ ichallenge ] );", + "return;", + ): + require(resend, token, "fail-closed pure challenge resend") + + initial_start = connect.rindex( + 'if ( sessLocal.mapSpawnData.serverInfo.GetInt( "si_pure" ) && challenges[ ichallenge ].authState != CDK_PUREOK )' + ) + initial = connect[initial_start : connect.index("// push back decl checksum", initial_start)] + for token in ( + "if ( !SendPureServerMessage( from, OS ) )", + "AsyncServer_ClearChallenge( challenges[ ichallenge ] );", + "challenges[ ichallenge ].authState = CDK_PUREWAIT;", + "return;", + ): + require(initial, token, "fail-closed initial pure challenge") + require_before( + initial, + "if ( !SendPureServerMessage( from, OS ) )", + "challenges[ ichallenge ].authState = CDK_PUREWAIT;", + "pure send before wait-state admission", + ) + reject(initial, "if ( SendPureServerMessage( from, OS ) )", "fallthrough-capable initial pure send") + + for body, context in ( + (map_change, "map-change reliable pure send"), + (unreliable, "wrong-gameinit reliable pure send"), + ): + require(body, "if ( !SendReliablePureToClient(", context) + require(body, "DropClient(", context) + reject(body, "clientState = SCS_CONNECTED;\n\t\t\t\t}", f"{context} failure promotion") + + +def validate_private_cvar_handling() -> None: + header = read("src/framework/CVarSystem.h") + cvars = read("src/framework/CVarSystem.cpp") + matcher = read("src/idlib/PrivateCommand.h") + args = read("src/idlib/CmdArgs.cpp") + strings = read("src/idlib/Str.h") + commands = read("src/framework/CmdSystem.cpp") + console = read("src/framework/Console.cpp") + + require(header, "CVAR_PRIVATE", "private CVar flag") + require(cvars, '( internal->GetFlags() & CVAR_PRIVATE ) ? ""', "direct CVar query redaction") + require(cvars, "!( cvar->GetFlags() & CVAR_PRIVATE )", "private CVar serialization omission") + require(cvars, "ContainsBoundedCaseInsensitiveToken", "private command matcher integration") + require(cvars, "expandedArgs.TokenizeString( commandText, false )", "expanded private-target classification") + require(cvars, "expandedArgs.Argv( argIndex )", "expanded private-target token scan") + require(cvars, "expandedArgs.ClearSensitive()", "expanded command scratch wipe") + require(cvars, "( flags & CVAR_CASE_SENSITIVE ) ?", "case-sensitive CVar update selection") + require(cvars, "valueString.Cmp( newValue ) == 0", "case-only private password update") + require(cvars, "CVar_AssignString( valueString, newValue", "private CVar replacement wipe") + require(cvars, "valueString.SecureClear()", "private CVar destructor wipe") + require(cvars, "resetString.SecureClear()", "private CVar reset-value wipe") + require(cvars, "toggle is unavailable for private CVar", "private toggle rejection") + require(matcher, "nameBytes > commandBytes", "short-command matcher bound") + require(matcher, "offset <= commandBytes - nameBytes", "bounded private-token scan") + require(matcher, "rightOffset == commandBytes", "safe right-boundary check") + require(args, 'token = ""', "$ private-CVar expansion redaction") + require(args, "cmd_args.SecureClear()", "Args shared-scratch wipe") + args_body = function_body(args, "const char *idCmdArgs::Args(", "Args shared scratch") + require_before(args_body, "cmd_args.SecureClear()", "cmd_args +=", "scratch wipe before Args reuse") + require(strings, "ID_INLINE void idStr::SecureClear", "full idStr allocation wipe") + require(strings, "data != baseBuffer ? alloced", "dynamic idStr allocation wipe") + require(commands, "vstr is unavailable for private CVar", "private vstr rejection") + require(commands, "idCmdArgs::ClearArgsScratch()", "private command scratch cleanup") + completion_info = function_body( + console, + "bool idConsoleLocal::GetCompletionCvarInfo(", + "console completion CVar information", + ) + require( + completion_info, + '( cvar->GetFlags() & CVAR_PRIVATE ) ?\n\t\t\t"" : cvar->GetString()', + "private completion-popup value redaction", + ) + + evidence = { + "src/framework/Console.cpp": ( + "CommandContainsPrivateCVar", + "]", + "removedPrivateCommand", + ), + "src/sys/win32/win_syscon.cpp": ( + "CommandContainsPrivateCVar", + "]", + "if ( !privateCommand )", + ), + "src/sys/posix/posix_syscon.cpp": ( + "CommandContainsPrivateCVar", + "]", + "if ( !privateCommand )", + ), + "src/sys/posix/posix_main.cpp": ( + "CommandContainsPrivateCVar", + "memset( s, 0, len )", + "if ( !privateCommand )", + ), + "src/framework/Common.cpp": ( + "", + "com_consoleLines[ i ].ClearSensitive()", + ), + "src/framework/EventLoop.cpp": ( + "EventLoop_IsPrivateConsoleEvent", + "PRIVATE_EVENT_TEXT", + "memset( ev.evPtr, 0, ev.evPtrLength )", + ), + "src/framework/EditField.cpp": ( + "memset( buffer, 0, sizeof( buffer ) )", + "memset( &autoComplete, 0, sizeof( autoComplete ) )", + '"" : cvarSystem->GetCVarString( s )', + "autocomplete is unavailable for private CVar commands", + ), + "src/framework/CmdSystem.cpp": ( + "CommandContainsPrivateCVar", + "ClearSensitive()", + "memset( textBuf + textLength, 0", + ), + } + for path, tokens in evidence.items(): + source = read(path) + for token in tokens: + require(source, token, f"private-data lifecycle in {path}") + + +def validate_remote_dictionary_authority() -> None: + engine_header = read("src/framework/CVarSystem.h") + game_header = read_game("src/framework/CVarSystem.h") + cvars = read("src/framework/CVarSystem.cpp") + policy = read("src/framework/RemoteCVarPolicy.h") + native = read("tools/tests/native/CoreSafetyTest.cpp") + client = read("src/framework/async/AsyncClient.cpp") + server = read("src/framework/async/AsyncServer.cpp") + demo = read("src/framework/async/MultiViewDemo.cpp") + engine_bitmsg = read("src/idlib/BitMsg.cpp") + game_bitmsg = read_game("src/idlib/BitMsg.cpp") + sp_network = read_game("src/game/Game_network.cpp") + mp_network = read_game("src/mpgame/Game_network.cpp") + + for header, label in ( + (engine_header, "engine CVar interface"), + (game_header, "game CVar interface"), + ): + require(header, "SetCVarsFromDictByFlags", label) + require(header, "ABI rule: append new virtual methods here", label) + interface = header.split("class idCVarSystem {", 1)[1].split("};", 1)[0] + legacy_tail = interface.index("SetCVarsFromDict( const idDict &dict )") + flagged_slot = interface.index("SetCVarsFromDictByFlags") + private_slot = interface.index("CommandContainsPrivateCVar") + if not legacy_tail < flagged_slot < private_slot: + raise AssertionError(f"{label} security methods are not appended after the legacy v43 tail") + + require(policy, "IsSingleAllowedAuthority", "production remote-CVar policy") + require(policy, "CanApply", "production remote-CVar policy") + for token in ( + '"matching userinfo authority"', + '"matching serverinfo authority"', + '"matching networksync authority"', + '"cross-class authority"', + '"local-only CVar"', + '"private CVar"', + '"combined authority"', + '"unknown authority"', + ): + require(native, token, "native remote-CVar authority coverage") + + legacy_apply = function_body(cvars, "void idCVarSystemLocal::SetCVarsFromDict(", "legacy dictionary apply") + flagged_apply = function_body(cvars, "bool idCVarSystemLocal::SetCVarsFromDictByFlags(", "flagged dictionary apply") + require(legacy_apply, "CVAR_USERINFO | CVAR_SERVERINFO | CVAR_NETWORKSYNC", "legacy remote-class ceiling") + require(legacy_apply, "CVAR_PRIVATE", "legacy private-CVar exclusion") + require(flagged_apply, "CVAR_USERINFO | CVAR_SERVERINFO | CVAR_NETWORKSYNC", "remote-class allowlist") + require(flagged_apply, "idRemoteCVarPolicy::IsSingleAllowedAuthority", "single remote authority") + require(flagged_apply, "idRemoteCVarPolicy::CanApply", "per-CVar authority check") + require(flagged_apply, "CVAR_PRIVATE", "private-CVar exclusion") + + require(client, "SetCVarsFromDictByFlags( info, CVAR_USERINFO )", "client userinfo authority") + require(client, "SetCVarsFromDictByFlags( info, CVAR_NETWORKSYNC )", "client sync authority") + require(server, "SetCVarsFromDictByFlags( *gameInfo, CVAR_USERINFO )", "listen-server userinfo authority") + if demo.count("SetCVarsFromDictByFlags( sessLocal.mapSpawnData.syncedCVars, CVAR_NETWORKSYNC )") != 2: + raise AssertionError("MVD start/reset must both apply only NETWORKSYNC CVars") + + for source, label in ( + (engine_bitmsg, "engine delta dictionary"), + (game_bitmsg, "game delta dictionary"), + ): + decoder = function_body(source, "bool idBitMsg::ReadDeltaDict(", label) + require(decoder, "idDict\t\tdecoded;", f"{label} transaction scratch") + require(decoder, "if ( IsReadOverflowed() )", f"{label} underflow rejection") + require_before(decoder, "if ( IsReadOverflowed() )", "dict = decoded;", f"{label} commit after validation") + if decoder.count("return false;") < 3: + raise AssertionError(f"{label} does not reject every truncated key/value/tail boundary") + + reliable_client = function_body(client, "void idAsyncClient::ProcessReliableServerMessages(", "client reliable dispatcher") + clientinfo = reliable_client[ + reliable_client.index("case SERVER_RELIABLE_MESSAGE_CLIENTINFO") : + reliable_client.index("case SERVER_RELIABLE_MESSAGE_SYNCEDCVARS") + ] + synced = reliable_client[ + reliable_client.index("case SERVER_RELIABLE_MESSAGE_SYNCEDCVARS") : + reliable_client.index("case SERVER_RELIABLE_MESSAGE_PRINT") + ] + require_before(clientinfo, "msg.IsReadOverflowed()", "SetCVarsFromDictByFlags", "userinfo validation before CVar commit") + require_before(synced, "msg.IsReadOverflowed()", "SetCVarsFromDictByFlags", "sync validation before CVar commit") + + reliable_server = function_body(server, "void idAsyncServer::ProcessReliableClientMessages(", "server reliable dispatcher") + server_clientinfo = reliable_server[ + reliable_server.index("case CLIENT_RELIABLE_MESSAGE_CLIENTINFO") : + reliable_server.index("case CLIENT_RELIABLE_MESSAGE_PRINT") + ] + require_before(server_clientinfo, "msg.IsReadOverflowed()", "SendUserInfoBroadcast", "client userinfo validation before broadcast") + + for source, label in ((sp_network, "SP reliable server-info"), (mp_network, "MP reliable server-info")): + serverinfo = source[source.index("case GAME_RELIABLE_MESSAGE_SERVERINFO") :] + serverinfo = serverinfo[: serverinfo.index("case GAME_RELIABLE_MESSAGE_RESTART")] + require_before(serverinfo, "msg.IsReadOverflowed()", "SetServerInfo( info )", label) + + pure = function_body(server, "void idAsyncServer::ProcessReliablePure(", "reliable pure state recovery") + require(pure, "outMsg.WriteByte( SERVER_RELIABLE_MESSAGE_RELOAD )", "reliable reload opcode") + require(pure, "SendReliableMessage( clientNum, outMsg )", "reliable reload buffer identity") + reject(pure, "SendReliableMessage( clientNum, msg )", "client-controlled reliable echo") + + +def validate_documentation_and_registration() -> None: + guide = read("docs/user/server-security.md") + setup = read("docs/user/server-setup.md") + for token in ( + "not an encrypted transport", + "PBKDF2-throttled offline password guessing", + "24 or more characters", + "never launch with", + "net_serverAllowLegacyRcon 1", + "net_clientUseLegacyRcon 1", + "FIPS 180-4", + "RFC 2104", + "RFC 8018", + ): + require(guide, token, "rcon2 administrator guidance") + require(setup, "server-security.md", "server setup security-guide link") + + for path in ( + ".github/workflows/commit-validation.yml", + ".github/workflows/push-verification.yml", + ): + workflow = read(path) + require(workflow, "tools/tests/network_security.py \\", f"{path} syntax-check registration") + require(workflow, "python tools/tests/network_security.py", f"{path} execution registration") + require(read("tools/validation/openq4_validate.py"), '"network_security.py"', "local validation registration") + + +def main() -> None: + validate_crypto_and_native_vectors() + validate_secure_random_and_connection_challenges() + validate_oob_budgets() + validate_usercmd_packet_bounds() + validate_snapshot_decode_bounds() + validate_rcon2_flow() + validate_pure_admission_fail_closed() + validate_private_cvar_handling() + validate_remote_dictionary_authority() + validate_documentation_and_registration() + print("network_security: ok") + + +if __name__ == "__main__": + main() diff --git a/tools/tests/openq4_pure_pack.py b/tools/tests/openq4_pure_pack.py index 5edbdc17..dfe5c78f 100644 --- a/tools/tests/openq4_pure_pack.py +++ b/tools/tests/openq4_pure_pack.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Regression checks for openQ4 pure-pack handling.""" +"""Regression checks for openQ4 pure-pack and download handling.""" from __future__ import annotations @@ -128,11 +128,15 @@ def bounded_download_write_model( def validate_filesystem_pure_pack_contract() -> None: source = read("src/framework/FileSystem.cpp") + game_dir_policy = read("src/framework/GameDirPolicy.h") file_system_header = read("src/framework/FileSystem.h") md5_header = read("src/idlib/hashing/MD5.h") md5_source = read("src/idlib/hashing/MD5.cpp") async_client = read("src/framework/async/AsyncClient.cpp") + async_client_header = read("src/framework/async/AsyncClient.h") + async_network_header = read("src/framework/async/AsyncNetwork.h") async_server = read("src/framework/async/AsyncServer.cpp") + licensee_header = read("src/framework/licensee.h") session_menu = read("src/framework/Session_menu.cpp") curl_write = function_body( source, @@ -171,13 +175,60 @@ def validate_filesystem_pure_pack_contract() -> None: async_client, "void idAsyncClient::HandleDownloads(", ) + version_message = function_body( + async_client, + "void idAsyncClient::ProcessVersionMessage(", + ) + version_check = function_body( + async_client, + "void idAsyncClient::SendVersionCheck(", + ) helper = function_body(source, "bool idFileSystemLocal::IsOpenQ4PurePack(") checksum_validator = function_body(source, "bool idFileSystemLocal::ValidateOpenQ4Paks(") misplaced_validator = function_body(source, "bool idFileSystemLocal::FindMisplacedOfficialPaks(") startup = function_body(source, "void idFileSystemLocal::Startup(") status = function_body(source, "pureStatus_t idFileSystemLocal::GetPackStatus(") + update_game_pak_checksums = function_body( + source, + "bool idFileSystemLocal::UpdateGamePakChecksums(", + ) + validate_download_pak = function_body( + source, + "int idFileSystemLocal::ValidateDownloadPakForChecksum(", + ) + clear_pure_checksums = function_body( + source, + "void idFileSystemLocal::ClearPureChecksums(", + ) + set_pure_server_checksums = function_body( + source, + "fsPureReply_t idFileSystemLocal::SetPureServerChecksums(", + ) + get_pure_server_checksums = function_body( + source, + "void idFileSystemLocal::GetPureServerChecksums(", + ) + find_dll = function_body(source, "void idFileSystemLocal::FindDLL(") + get_mod_info = function_body(source, "bool idFileSystemLocal::GetModInfo(") + send_pure_server_message = function_body( + async_server, + "bool idAsyncServer::SendPureServerMessage(", + ) + send_reliable_pure_to_client = function_body( + async_server, + "bool idAsyncServer::SendReliablePureToClient(", + ) + process_connect_message = function_body( + async_server, + "void idAsyncServer::ProcessConnectMessage(", + ) + process_challenge_response = function_body( + async_client, + "void idAsyncClient::ProcessChallengeResponseMessage(", + ) require(source, '#include "openq4_paks_generated.h"', "filesystem generated pack checksum header") + require(source, '#include "GameDirPolicy.h"', "portable game-directory policy integration") require(source, "IsOpenQ4PurePack", "filesystem pure-pack declaration") require(source, "ValidateOpenQ4Paks", "filesystem pack checksum declaration") require(helper, "OPENQ4_GAMEDIR", "openQ4 pure-pack directory check") @@ -342,6 +393,262 @@ def validate_filesystem_pure_pack_contract() -> None: require(async_server, "Server decl checksum: 0x%08x", "server decl checksum diagnostic") require(async_server, "client=0x%08x server=0x%08x (non-pure)", "non-pure mismatch diagnostic") require(async_server, "client=0x%08x server=0x%08x (pure)", "pure mismatch diagnostic") + reject(async_server, 'serverInfo.SetInt( "si_pure", 0 )', "forced pure-server disable") + reject(async_server, "forcing si_pure 0", "forced pure-server disable diagnostic") + require( + async_network_header, + "const int ASYNC_PROTOCOL_MINOR\t\t= 41;", + "Quake 4 1.4.2 protocol compatibility", + ) + require( + source, + "static const int OPENQ4_Q4_142_GAME300_PAK_CHECKSUM = 0x68fb90b1;", + "platform-independent pure game-module compatibility token", + ) + require(file_system_header, "MAX_GAME_OS\t\t\t\t\t= 6;", "pure OS table capacity") + + for origin in ( + "GAME_MODULE_ORIGIN_NONE", + "GAME_MODULE_ORIGIN_ACTIVE_MOD", + "GAME_MODULE_ORIGIN_BASE_GAME", + "GAME_MODULE_ORIGIN_PACKAGE_ROOT", + ): + require(source, origin, "trusted game-module origin model") + require(find_dll, "gameModuleOrigin_t resolvedOrigin = GAME_MODULE_ORIGIN_NONE;", "module-origin reset") + require(find_dll, "GAME_MODULE_ORIGIN_ACTIVE_MOD", "active-mod module origin") + require(find_dll, "GAME_MODULE_ORIGIN_BASE_GAME", "base-game module origin") + require(find_dll, "GAME_MODULE_ORIGIN_PACKAGE_ROOT", "flat package-root module origin") + require(find_dll, "gameModuleOrigin = resolvedOrigin;", "resolved module-origin persistence") + require( + find_dll, + "gamePakChecksum = dllFile ? OPENQ4_Q4_142_GAME300_PAK_CHECKSUM : 0;", + "compatibility token only for a resolved trusted module", + ) + + for token in ( + "MAX_SEGMENT_BYTES = 255", + "IsWindowsDeviceName(", + "IsPortableSegment(", + "*scan == '/'", + "*scan == '\\\\'", + "*scan == ':'", + "segment[ 0 ] == '.'", + "segment[ 1 ] == '.'", + ): + require(game_dir_policy, token, "portable single-segment game-directory policy") + require( + get_mod_info, + "!idGameDirPolicy::IsPortableSegment( modDir )", + "mod manifest directory validation", + ) + require_order( + get_mod_info, + "!idGameDirPolicy::IsPortableSegment( modDir )", + "const char *search[ 3 ]", + "mod directory validation before manifest roots", + ) + require( + get_mod_info, + "mod directory must be one portable directory segment", + "actionable unsafe mod-directory failure", + ) + require( + find_dll, + "!idGameDirPolicy::IsPortableSegment( moduleGameDir )", + "trusted module game-directory validation", + ) + require_order( + find_dll, + "!idGameDirPolicy::IsPortableSegment( moduleGameDir )", + "FS_AppendGameModuleSearchPath(", + "game-directory validation before trusted module path construction", + ) + for token in ( + "gameDLLChecksum = 0;", + "gamePakChecksum = 0;", + "gameModuleOrigin = GAME_MODULE_ORIGIN_NONE;", + "_dllPath[ 0 ] = '\\0';", + "return;", + ): + require(find_dll, token, "unsafe fs_game module lookup fail-closed state") + require(process_challenge_response, "fileSystem->GetModInfo( serverGameBase", "server base-mod validation") + require(process_challenge_response, "fileSystem->GetModInfo( serverGame", "server mod validation") + require_order( + process_challenge_response, + "fileSystem->GetModInfo( serverGame", + 'cvarSystem->SetCVarString( "fs_game", serverGame );', + "server mod validation before fs_game mutation", + ) + + require(update_game_pak_checksums, "gameDLLChecksum == 0", "missing local game-module rejection") + require( + update_game_pak_checksums, + "gamePakChecksum != OPENQ4_Q4_142_GAME300_PAK_CHECKSUM", + "unexpected local compatibility-token rejection", + ) + require( + update_game_pak_checksums, + "gameModuleOrigin == GAME_MODULE_ORIGIN_NONE", + "untrusted module-origin rejection", + ) + require( + update_game_pak_checksums, + "gameModuleOrigin == GAME_MODULE_ORIGIN_ACTIVE_MOD", + "active-mod module-origin classification", + ) + require( + update_game_pak_checksums, + '!cvarSystem->GetCVarBool( "net_serverAllowServerMod" )', + "explicit server-mod opt-in gate", + ) + require( + update_game_pak_checksums, + "for ( int os = 0; os <= 2; ++os )", + "legacy Windows/Linux/macOS compatibility range", + ) + require_order( + update_game_pak_checksums, + "memset( gamePakForOS, 0, sizeof( gamePakForOS ) );", + "for ( int os = 0; os <= 2; ++os )", + "unsupported pure OS entries cleared before legacy OS mapping", + ) + require( + update_game_pak_checksums, + "gamePakForOS[ os ] = OPENQ4_Q4_142_GAME300_PAK_CHECKSUM;", + "platform-independent compatibility-token mapping", + ) + for obsolete_parser_symbol in ( + "BINARY_CONFIG", + '"binary.conf"', + "HashFileName(", + "ReadFile(", + "idLexer", + "ParseInt(", + "atoi(", + ): + reject( + update_game_pak_checksums, + obsolete_parser_symbol, + "pure game-module mapping without retail binary.conf parsing", + ) + + require(validate_download_pak, "pak->binary == BINARY_UNKNOWN", "lazy package binary classification") + require(validate_download_pak, "HashFileName( BINARY_CONFIG )", "binary.conf presence lookup") + require(validate_download_pak, "pak->binary = BINARY_NO", "content-only package classification") + require(validate_download_pak, "pak->binary = BINARY_YES", "binary-marked package classification") + reject(validate_download_pak, "ReadFile(", "package binary marker presence-only classification") + require(clear_pure_checksums, "memset( gamePakForOS, 0, sizeof( gamePakForOS ) );", "non-pure OS-mask reset") + + require( + set_pure_server_checksums, + "*missingGamePakChecksum = 0;", + "code-pak download output stays empty", + ) + if set_pure_server_checksums.count("*missingGamePakChecksum =") != 1: + raise AssertionError("Pure negotiation must never request a downloadable game-code pak") + require( + set_pure_server_checksums, + "_gamePakChecksum != OPENQ4_Q4_142_GAME300_PAK_CHECKSUM", + "server compatibility-token validation", + ) + require( + set_pure_server_checksums, + "gamePakChecksum != OPENQ4_Q4_142_GAME300_PAK_CHECKSUM", + "local compatibility-token validation", + ) + require(set_pure_server_checksums, "gameDLLChecksum == 0", "missing local module rejection") + require( + set_pure_server_checksums, + "gameModuleOrigin == GAME_MODULE_ORIGIN_NONE", + "untrusted local module rejection", + ) + require(set_pure_server_checksums, "return PURE_NODLL;", "unsupported token fail-closed result") + require_order( + set_pure_server_checksums, + "_gamePakChecksum != OPENQ4_Q4_142_GAME300_PAK_CHECKSUM", + "if ( pureChecksums[ 0 ] == 0 )", + "game-module token validation before asset-list processing", + ) + for forbidden_code_pak_path in ( + "GetPackForChecksum( _gamePakChecksum", + "GetPackForChecksum( gamePakChecksum", + "*missingGamePakChecksum = _gamePakChecksum", + "*missingGamePakChecksum = gamePakChecksum", + "gamePakChecksum = _gamePakChecksum", + "restartGamePakChecksum", + "BINARY_CONFIG", + '"binary.conf"', + ): + reject( + set_pure_server_checksums, + forbidden_code_pak_path, + "compatibility token cannot select, download, or restart into game code", + ) + + require(get_pure_server_checksums, "OS >= 0 && OS < MAX_GAME_OS", "pure checksum OS bounds") + require(get_pure_server_checksums, "else if ( OS == -1 )", "local pure-token sentinel") + require(get_pure_server_checksums, "*_gamePakChecksum = 0;", "invalid pure OS fail-closed token") + require_order( + get_pure_server_checksums, + "OS >= 0 && OS < MAX_GAME_OS", + "OS == -1", + "bounded remote OS lookup before local-only sentinel", + ) + + require(process_connect_message, "if ( OS < 0 || OS >= MAX_GAME_OS )", "pure client OS bounds") + require( + process_connect_message, + "const int osMask = fileSystem->GetOSMask();", + "pure supported-OS mask lookup", + ) + require( + process_connect_message, + "static_cast( osMask ) & ( 1u << OS )", + "defined unsigned pure OS-mask shift", + ) + require_order( + process_connect_message, + "if ( OS < 0 || OS >= MAX_GAME_OS )", + "const int osMask = fileSystem->GetOSMask();", + "OS range validation before OS-mask lookup and shift", + ) + require_order( + process_connect_message, + "const int osMask = fileSystem->GetOSMask();", + "ValidateChallenge( from, challenge, clientId )", + "pure supported-OS rejection before challenge state use", + ) + require_order( + process_connect_message, + "static_cast( osMask ) & ( 1u << OS )", + "ValidateChallenge( from, challenge, clientId )", + "pure supported-OS bit test before challenge state use", + ) + require_order( + process_connect_message, + "if ( OS < 0 || OS >= MAX_GAME_OS )", + "challenges[ ichallenge ].OS = OS;", + "OS range validation before challenge-state storage", + ) + + for send_body, message_init, context in ( + (send_pure_server_message, "outMsg.Init(", "connectionless pure send"), + (send_reliable_pure_to_client, "msg.Init(", "reliable pure send"), + ): + require( + send_body, + "if ( !serverChecksums[ 0 ] || !gamePakChecksum )", + f"{context} fail-closed prerequisites", + ) + require_order( + send_body, + "if ( !serverChecksums[ 0 ] || !gamePakChecksum )", + message_init, + f"{context} prerequisites before packet construction", + ) + guard_start = send_body.find("if ( !serverChecksums[ 0 ] || !gamePakChecksum )") + guard_end = send_body.find(message_init, guard_start) + require(send_body[guard_start:guard_end], "return false;", f"{context} fail-closed return") require( download_request, "dlSize[ MAX_PURE_PAKS ] = {};", @@ -431,11 +738,62 @@ def validate_filesystem_pure_pack_contract() -> None: require(invalid_download_type, "return;", "invalid download discriminator fail-closed return") reject(download_info, "assert( pakDl == SERVER_PAK_END );", "release-only download discriminator validation") - update_setup = download_handler[ - download_handler.find("fileSystem->OpenFileWrite( updateFile )") : - download_handler.find("updateState = UPDATE_DLING;") + require( + licensee_header, + '#define PROJECT_RELEASES_URL\t\t\tPROJECT_REPO "/releases"', + "compile-time openQ4 releases destination", + ) + require( + licensee_header, + '#define PROJECT_REPO\t\t\t\t\t"https://github.com/themuffinator/openQ4"', + "HTTPS openQ4 project destination", + ) + require( + version_message, + 'updateMSG = common->GetLanguageDict()->GetString( "#str_104330" );', + "fixed localized update notification", + ) + require( + version_message, + "ignoredNetworkField", + "legacy update fields are consumed without storage or display", + ) + if version_message.count("msg.ReadString(") != 3 or version_message.count("msg.ReadByte()") != 2: + raise AssertionError("Legacy update reply fields must remain wire-compatible and inert") + reject(version_message, "updateMSG = ignoredNetworkField", "network-controlled update instructions") + require(version_check, 'msg.WriteString( "" );', "empty legacy update identity field") + reject(version_check, 'GetCVarString( "com_guid" )', "update-check persistent GUID disclosure") + update_prompt = download_handler[ + download_handler.find("if ( updateState == UPDATE_READY )") : + download_handler.find("} else if ( dlList.Num() )") ] - require(update_setup, "backgroundDownload.url.expectedSize = 0;", "unrestricted updater download") + require(update_prompt, "sys->OpenURL( PROJECT_RELEASES_URL, false );", "fixed update information page") + require(update_prompt, "updateState = UPDATE_DONE;", "update notification terminal state") + require(update_prompt, "showUpdateMessage = false;", "update notification prompt cleanup") + for forbidden in ( + "BackgroundDownload", + "OpenFileWrite", + "StartProcess", + "DLTYPE_URL", + "file://", + "expectedSize", + "RemoveFile", + ): + reject(update_prompt, forbidden, "notification-only updater") + for removed_member in ( + "UPDATE_DLING", + "updateDirectDownload", + "updateURL", + "updateFile", + "updateMime", + "updateFallback", + "SendVersionDLUpdate", + ): + reject(async_client_header, removed_member, "legacy executable updater state") + reject(async_client, removed_member, "legacy executable updater implementation") + reject(file_system_header, "FILE_EXEC", "legacy executable updater MIME action") + reject(file_system_header, "dlMime_t", "legacy updater MIME type") + package_setup = download_handler[ download_handler.find("fileSystem->MakeTemporaryFile( )") : download_handler.find("session->DownloadProgressBox( &backgroundDownload, dltitle") @@ -467,7 +825,7 @@ def validate_filesystem_pure_pack_contract() -> None: if bounded_download_write_model(SIZE_T_MAX, 1, 1, 0) is not None: raise AssertionError("Bounded download model accepted overflowing cumulative bytes") if bounded_download_write_model(9, 1, 2, 0) != 11: - raise AssertionError("Updater download model unexpectedly applied a package-size limit") + raise AssertionError("Unbounded transfer model unexpectedly applied a package-size limit") require_order( download_handler, @@ -525,24 +883,6 @@ def validate_filesystem_pure_pack_contract() -> None: "checksum failure removes only the verified download destination", ) - update_destination = download_handler[ - download_handler.find("fileSystem->OpenFileWrite( updateFile )") : - download_handler.find("dltotal = 0;") - ] - require(update_destination, "if ( f == NULL )", "update destination null guard") - require(update_destination, "updateState = UPDATE_DONE;", "failed update state cleanup") - require(update_destination, "SendVersionDLUpdate( 2 );", "failed update telemetry") - require(update_destination, '"#str_04335"', "localized update failure message") - require(update_destination, "updateFallback.Length()", "failed update fallback") - require(update_destination, "return;", "failed update early return") - - update_download_result = download_handler[ - download_handler.find("if ( backgroundDownload.url.status == DL_DONE )") : - download_handler.find("} else if ( dlList.Num() )") - ] - if update_download_result.count("AsyncClient_CloseBackgroundDownloadFile( backgroundDownload );") != 2: - raise AssertionError("Update download success and failure paths must each release their owned file") - successful_copy_cleanup = download_handler[ download_handler.find("while ( remainlen )") : download_handler.find("fileSystem->CloseFile( saveas );") @@ -575,7 +915,7 @@ def validate_filesystem_pure_pack_contract() -> None: "direct background download member close", ) cleanup_call = "AsyncClient_CloseBackgroundDownloadFile( backgroundDownload );" - if download_handler.count(cleanup_call) != 6: + if download_handler.count(cleanup_call) != 4: raise AssertionError("Every background download completion path must release its file exactly once") valid_download_paths = ( diff --git a/tools/tests/openurl_security.py b/tools/tests/openurl_security.py new file mode 100644 index 00000000..0e6136a7 --- /dev/null +++ b/tools/tests/openurl_security.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Cross-platform source contracts for external URL handoff.""" + +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def read(relative_path: str) -> str: + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def require(haystack: str, needle: str, context: str) -> None: + if needle not in haystack: + raise AssertionError(f"Missing {needle!r} in {context}") + + +def reject(haystack: str, needle: str, context: str) -> None: + if needle in haystack: + raise AssertionError(f"Unexpected {needle!r} in {context}") + + +def require_before(haystack: str, first: str, second: str, context: str) -> None: + first_index = haystack.find(first) + second_index = haystack.find(second) + if first_index < 0 or second_index < 0: + raise AssertionError(f"Missing ordered tokens {first!r} and/or {second!r} in {context}") + if first_index >= second_index: + raise AssertionError(f"Expected {first!r} before {second!r} in {context}") + + +def function_body(source: str, signature: str) -> str: + start = source.find(signature) + if start < 0: + raise AssertionError(f"Missing function signature {signature!r}") + + depth = 0 + for index in range(start, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start : index + 1] + raise AssertionError(f"Could not find end of function {signature!r}") + + +def validate_shared_policy() -> None: + policy = read("src/sys/URLPolicy.h") + validator = function_body(policy, "inline bool IsAllowedHTTPURL( const char *url ) {") + authority = function_body(policy, "inline bool AuthorityHasHost( const char *begin, const char *end ) {") + port = function_body(policy, "inline bool ParsePort( const char *begin, const char *end ) {") + dns_host = function_body(policy, "inline bool IsDNSOrIPv4Host( const char *begin, const char *end ) {") + ipv4 = function_body(policy, "inline bool IsIPv4Literal( const char *begin, const char *end ) {") + ipv6 = function_body(policy, "inline bool IsIPv6Literal( const char *begin, const char *end ) {") + + require(policy, "const size_t MAX_URL_BYTES = 4096;", "bounded URL policy") + require(validator, "length < MAX_URL_BYTES", "bounded URL scan") + require(validator, "length == MAX_URL_BYTES", "unterminated/oversized URL rejection") + require(validator, "value <= 32 || value == 127", "URL whitespace and control-byte rejection") + require(validator, 'ASCIIStartsWith( url, "https://" )', "HTTPS scheme allowlist") + require(validator, 'ASCIIStartsWith( url, "http://" )', "HTTP scheme allowlist") + require(validator, "AuthorityHasHost( authority, authorityEnd )", "non-empty URL host policy") + require(authority, "begin == end", "empty URL authority rejection") + require(authority, "*cursor == '@' || *cursor == '%'", "ambiguous authority rejection") + require(authority, "IPv6 literals must use brackets", "unambiguous IPv6 authority policy") + require(authority, "hostEnd == begin", "empty URL host rejection") + require(authority, "IsDNSOrIPv4Host( begin, hostEnd )", "DNS/IPv4 host syntax validation") + require(authority, "IsIPv6Literal( begin + 1, closeBracket )", "IPv6 literal syntax validation") + require(dns_host, "hostLength > 253", "DNS host-length bound") + require(dns_host, "labelLength > 63", "DNS label-length bound") + require(dns_host, "IsASCIIAlphaNumeric( *labelBegin )", "DNS label boundary syntax") + require(ipv4, "value > 255", "IPv4 octet bound") + require(ipv6, "compressed ? groups < 8 : groups == 8", "IPv6 group/compression bound") + require(port, "port > 6553", "URL port overflow/range guard") + + +def validate_platform_integration() -> None: + platforms = ( + ( + "Windows", + "src/sys/win32/win_main.cpp", + "void idSysLocal::OpenURL(const char* url, bool doexit) {", + "ShellExecute(NULL, \"open\", url", + ), + ( + "Linux", + "src/sys/linux/main.cpp", + "void idSysLocal::OpenURL( const char *url, bool quit ) {", + 'Sys_FindExecutableOnPath( "xdg-open"', + ), + ( + "macOS", + "src/sys/osx/macosx_misc.mm", + "void idSysLocal::OpenURL( const char *url, bool doexit ) {", + "openURL: nsURL", + ), + ) + + for platform, path, signature, handoff in platforms: + source = read(path) + body = function_body(source, signature) + require(source, '#include "../URLPolicy.h"', f"{platform} shared URL policy include") + require(body, "idURLPolicy::IsAllowedHTTPURL( url )", f"{platform} URL guard") + require(body, "OpenURL rejected: expected a bounded HTTP or HTTPS URL with a host", f"{platform} rejection diagnostic") + require_before(body, "idURLPolicy::IsAllowedHTTPURL( url )", "Open URL:", f"{platform} guard before approved URL logging") + require_before(body, "idURLPolicy::IsAllowedHTTPURL( url )", handoff, f"{platform} guard before OS handoff") + reject(body, "rejected: %s", f"{platform} rejected URL raw logging") + reject(body, "ignoring %s", f"{platform} ignored URL raw logging") + + linux = read("src/sys/linux/main.cpp") + macos = read("src/sys/osx/macosx_misc.mm") + reject(linux, "static bool Sys_IsSafeURL", "Linux broad scheme policy") + reject(macos, "OSX_URLHasSafeSchemeSyntax", "macOS broad scheme policy") + reject(macos, "OSX_FileURLIsLocalRuntimeFile", "macOS file URL launcher") + + +def validate_runtime_coverage() -> None: + native = read("tools/tests/native/CoreSafetyTest.cpp") + require(native, '#include "src/sys/URLPolicy.h"', "native URL policy test") + require(native, "idURLPolicy::IsAllowedHTTPURL( url )", "native production-policy execution") + for case in ( + '"http://example.com", true', + '"HTTPS://example.com:443/releases?q=openq4#download", true', + '"https://[2001:db8::1]:65535/path", true', + '"file:///tmp/update", false', + '"javascript:alert(1)", false', + '"https://user@example.com/path", false', + '"https://example.com/line\\nbreak", false', + '"https://example.com\\\\attacker.invalid", false', + '"https://!/path", false', + '"https://999.1.2.3/path", false', + '"https://[not-an-ip]/path", false', + '"oversized URL"', + ): + require(native, case, "native URL policy boundary cases") + + +def validate_network_download_integration() -> None: + client = read("src/framework/async/AsyncClient.cpp") + file_system = read("src/framework/FileSystem.cpp") + download_info = function_body( + client, + "void idAsyncClient::ProcessDownloadInfoMessage( const netadr_t from, const idBitMsg &msg ) {", + ) + background_download = function_body( + file_system, + "void idFileSystemLocal::BackgroundDownload( backgroundDownload_t *bgl ) {", + ) + bounded_transfer = function_body( + file_system, + "static CURLcode FS_ConfigureBoundedHTTPTransfer( CURL *session ) {", + ) + download_worker = function_body( + file_system, + "dword BackgroundDownloadThread( void *parms ) {", + ) + curl_init = function_body( + file_system, + "static bool FS_InitializeCurl() {", + ) + curl_shutdown = function_body( + file_system, + "static void FS_ShutdownCurl() {", + ) + restart = function_body( + file_system, + "void idFileSystemLocal::Restart( void ) {", + ) + + require(client, '#include "../../sys/URLPolicy.h"', "async download URL policy include") + if download_info.count("idURLPolicy::IsAllowedHTTPURL( buf )") < 2: + raise AssertionError("Both redirect and package download URLs must use the shared HTTP/HTTPS policy") + require(file_system, '#include "../sys/URLPolicy.h"', "filesystem download URL policy include") + require(background_download, "bgl->opcode == DLTYPE_URL", "generic URL download discriminator") + require(background_download, "bgl->opcode != DLTYPE_FILE && bgl->opcode != DLTYPE_URL", "unknown download-opcode rejection") + require(background_download, "unsupported background download operation", "unknown download-opcode diagnostic") + require(background_download, "idURLPolicy::IsAllowedHTTPURL( bgl->url.url.c_str() )", "generic URL download guard") + require_before( + background_download, + "idURLPolicy::IsAllowedHTTPURL( bgl->url.url.c_str() )", + "backgroundDownloads = bgl", + "URL guard before background-queue handoff", + ) + require(download_worker, "else if ( bgl->opcode == DLTYPE_URL )", "worker URL opcode allowlist") + require(file_system, "OPENQ4_CURL_CAPABLE_BUILD", "compile-time libcurl capability gate") + require(curl_init, "curl_global_init( CURL_GLOBAL_DEFAULT )", "explicit libcurl global initialization") + require(curl_init, "versionInfo->version_num < 0x071304", "runtime libcurl version gate") + require(curl_init, "CURL_VERSION_ASYNCHDNS", "nonblocking DNS capability gate") + require(curl_init, "hasHTTP", "runtime HTTP protocol capability gate") + require(curl_shutdown, "curl_global_cleanup();", "libcurl global shutdown") + require(download_worker, "if ( !fsCurlHTTPReady )", "worker fail-closed libcurl gate") + require(restart, "StartBackgroundDownloadThread();", "background worker restart lifecycle") + require(bounded_transfer, "CURLOPT_PROTOCOLS", "libcurl protocol allowlist") + require(bounded_transfer, "CURLPROTO_HTTP | CURLPROTO_HTTPS", "libcurl HTTP/HTTPS-only policy") + require(bounded_transfer, "CURLOPT_FOLLOWLOCATION, 0L", "libcurl redirect disable") + require(bounded_transfer, "CURLOPT_CONNECTTIMEOUT, 15L", "bounded connect/DNS wait") + require(bounded_transfer, "CURLOPT_LOW_SPEED_LIMIT, 1024L", "stalled-transfer byte floor") + require(bounded_transfer, "CURLOPT_LOW_SPEED_TIME, 30L", "stalled-transfer timeout") + require(bounded_transfer, "CURLOPT_TIMEOUT, 3600L", "absolute transfer bound") + require(bounded_transfer, "CURLOPT_NOSIGNAL, 1L", "cross-thread libcurl timeout policy") + + +def validate_no_file_url_handoffs() -> None: + for path in (ROOT / "src").rglob("*"): + if path.suffix.lower() not in {".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".m", ".mm"}: + continue + text = path.read_text(encoding="utf-8", errors="ignore") + if '"file://' in text: + raise AssertionError(f"Active source still contains a file URL handoff: {path.relative_to(ROOT)}") + + +def validate_registration() -> None: + for path in ( + "tools/validation/openq4_validate.py", + ".github/workflows/commit-validation.yml", + ".github/workflows/push-verification.yml", + ): + require(read(path), "openurl_security.py", f"URL security test registration in {path}") + + +def main() -> None: + validate_shared_policy() + validate_platform_integration() + validate_runtime_coverage() + validate_network_download_integration() + validate_no_file_url_handoffs() + validate_registration() + print("openurl_security: ok") + + +if __name__ == "__main__": + main() diff --git a/tools/tests/p0_governance_evidence.py b/tools/tests/p0_governance_evidence.py new file mode 100644 index 00000000..615d7869 --- /dev/null +++ b/tools/tests/p0_governance_evidence.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Contract checks for P0 provenance and authoritative capability evidence.""" + +from __future__ import annotations + +import importlib.util +import copy +import hashlib +import sys +import tempfile +from pathlib import Path +from types import ModuleType + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_module(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise AssertionError(f"could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def read(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def require(text: str, snippet: str, context: str) -> None: + if snippet not in text: + raise AssertionError(f"missing {snippet!r} in {context}") + + +def test_provenance_inventory() -> None: + audit = load_module( + "openq4_source_provenance_audit", + ROOT / "tools" / "validation" / "audit_source_provenance.py", + ) + manifest = audit.load_manifest() + report = audit.inventory(ROOT, manifest) + failures = audit.validate(ROOT, manifest, report) + if failures: + raise AssertionError("provenance validation failed:\n" + "\n".join(failures)) + + assert report["families"]["doom3"]["count"] == 581 + bfg_files = report["families"]["doom3_bfg"]["files"] + assert len(bfg_files) == 37 + classifications: dict[str, int] = {} + for entry in bfg_files: + classification = entry.get("classification", "") + classifications[classification] = classifications.get(classification, 0) + 1 + assert entry.get("auditedPath") + assert entry.get("auditedCommit") + assert entry.get("repository") + assert classifications == { + "official-snapshot-path": 31, + "intermediate-fork-lineage": 6, + } + + (ROOT / ".tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(prefix="provenance-mutation-", dir=ROOT / ".tmp") as temp: + altered = Path(temp) / "altered-terms.txt" + original = ROOT / manifest["families"]["doom3"]["localAdditionalTerms"] + altered.write_bytes(original.read_bytes() + b"altered\n") + mutated_manifest = copy.deepcopy(manifest) + mutated_manifest["families"]["doom3"]["localAdditionalTerms"] = altered.relative_to(ROOT).as_posix() + mutation_failures = audit.validate(ROOT, mutated_manifest, report) + assert any("SHA-256 differs" in failure for failure in mutation_failures) + + official = Path(temp) / "official" + official.mkdir() + copying = official / "COPYING.txt" + copying.write_bytes(b"published copying bytes\r\n") + copying_spec = {"officialCopyingSha256": hashlib.sha256(copying.read_bytes()).hexdigest()} + assert audit.validate_official_copying("test", copying_spec, official) == [] + copying.write_bytes(copying.read_bytes() + b"tampered") + assert any( + "official COPYING.txt SHA-256 differs" in failure + for failure in audit.validate_official_copying("test", copying_spec, official) + ) + + provenance = read("docs/dev/source-provenance.md") + for snippet in ( + "not a legal opinion", + "Doom 3 GPL Source Code", + "Doom 3 BFG Edition GPL Source Code", + "intermediate lineage reference", + "audit_source_provenance.py --check", + ): + require(provenance, snippet, "source provenance documentation") + + +def test_accompanying_terms() -> None: + doom3 = read("LICENSES/DOOM-3-ADDITIONAL-TERMS.txt") + bfg = read("LICENSES/DOOM-3-BFG-ADDITIONAL-TERMS.txt") + require(doom3, "ADDITIONAL TERMS APPLICABLE TO THE DOOM 3 GPL SOURCE CODE", "Doom 3 Additional Terms") + require(bfg, "ADDITIONAL TERMS APPLICABLE TO THE Doom 3 BFG Edition GPL Source Code", "BFG Additional Terms") + for terms in (doom3, bfg): + for section in ( + "Replacement of Section 15", + "Replacement of Section 16", + "LEGAL NOTICES; NO TRADEMARK LICENSE; ORIGIN", + "INDEMNIFICATION", + ): + require(terms, section, "accompanying Additional Terms") + + +def test_capability_matrix_is_authoritative_and_scoped() -> None: + matrix = read("docs/dev/engine-capability-matrix.md") + require(matrix, "authoritative current-state index", "capability matrix") + for status in ("**Implemented**", "**Experimental**", "**Missing**"): + require(matrix, status, "capability matrix status vocabulary") + for capability in ( + "Network-driven executable updater", + "Server-supplied package transport", + "Pure multiplayer game-module boundary", + "Malformed network and snapshot input handling", + "Connection challenge entropy", + "Authenticated remote console (`rcon2`)", + "Rcon abuse limits and secret redaction", + "Legacy plaintext rcon", + "Doom 3 / Doom 3 BFG provenance inventory", + "Reproducible retail-PK4 SP/MP compatibility evidence", + "Modern visible lighting ownership", + "Vulkan renderer", + "GPU skeletal skinning", + "Automatic dynamic resolution", + "Namespaced PBR materials", + ): + require(matrix, capability, "capability matrix coverage") + require(matrix, "`net_clientUseLegacyRcon 1` / `net_serverAllowLegacyRcon 1`", "legacy rcon containment claim") + require(matrix, "The current proven-domain count is zero", "modern renderer qualification") + + renderer_matrix = read("docs/dev/renderer-validation-matrix.md") + proposal = read("docs/dev/proposals/rbdoom3-bfg-parity-modernization-plan.md") + readme = read("README.md") + require(renderer_matrix, "engine capability matrix", "renderer evidence federation") + require(proposal, "historical proposal, not a current-state inventory", "stale proposal warning") + require(readme, "engine capability matrix", "README capability link") + require(readme, "source-provenance inventory", "README provenance link") + + +def test_workflow_registration() -> None: + validator = read("tools/validation/openq4_validate.py") + for test_name in ("p0_governance_evidence.py", "stock_asset_baseline.py"): + require(validator, test_name, "local validation registration") + for workflow in ( + ".github/workflows/commit-validation.yml", + ".github/workflows/push-verification.yml", + ): + text = read(workflow) + require( + text, + "python tools/validation/audit_source_provenance.py --check", + workflow, + ) + for test_name in ("p0_governance_evidence.py", "stock_asset_baseline.py"): + require(text, f"python tools/tests/{test_name}", workflow) + + +def main() -> None: + test_provenance_inventory() + test_accompanying_terms() + test_capability_matrix_is_authoritative_and_scoped() + test_workflow_registration() + print("p0_governance_evidence: ok") + + +if __name__ == "__main__": + main() diff --git a/tools/tests/packaging_safety.py b/tools/tests/packaging_safety.py index fbb286bf..8a082a47 100644 --- a/tools/tests/packaging_safety.py +++ b/tools/tests/packaging_safety.py @@ -350,6 +350,7 @@ def validate_fast_stage_guards_and_copy() -> None: build_dir = source_root / "builddir" install_dir = source_root / ".install" write_file(build_dir / "openQ4-client_x64.exe", b"client\n") + write_file(build_dir / "renderer-gl_x64.dll", b"renderer\n") write_file(build_dir / "baseoq4" / "game-sp_x64.dll", b"game\n") write_file(build_dir / "baseoq4" / "pak0.pk4", b"pak0\n") @@ -394,7 +395,7 @@ def validate_fast_stage_guards_and_copy() -> None: "--install-dir", symlink_install_dir, ) - if symlink_result.returncode == 0 or "build directory must not be a symlink" not in symlink_result.stderr: + if symlink_result.returncode == 0 or "build directory must not be a link or junction" not in symlink_result.stderr: raise AssertionError(f"stage_fast_install.py accepted a symlinked build dir: {symlink_result.stderr}") result = run_script( @@ -412,6 +413,58 @@ def validate_fast_stage_guards_and_copy() -> None: raise AssertionError("fast stage did not copy root runtime binary") if not (install_dir / "baseoq4" / "game-sp_x64.dll").is_file(): raise AssertionError("fast stage did not copy game runtime binary") + if not (install_dir / "renderer-gl_x64.dll").is_file(): + raise AssertionError("fast stage did not copy renderer runtime binary") + + escaped_temporary = source_root / ".tmp" / "other-runtime" / "capture" + escaped_result = run_script( + BUILD_DIR / "stage_fast_install.py", + "--source-root", + source_root, + "--build-dir", + build_dir, + "--install-dir", + escaped_temporary, + "--temporary-runtime", + ) + if escaped_result.returncode == 0 or "must stay below" not in escaped_result.stderr: + raise AssertionError( + f"stage_fast_install.py accepted an escaped temporary runtime: {escaped_result.stderr}" + ) + + temporary_runtime = source_root / ".tmp" / "stock-runtime" / "capture" + temporary_result = run_script( + BUILD_DIR / "stage_fast_install.py", + "--source-root", + source_root, + "--build-dir", + build_dir, + "--install-dir", + temporary_runtime, + "--temporary-runtime", + ) + if temporary_result.returncode != 0: + raise AssertionError( + f"stage_fast_install.py failed isolated temporary staging: {temporary_result.stderr}" + ) + if not (temporary_runtime / "openQ4-client_x64.exe").is_file(): + raise AssertionError("temporary fast stage did not copy root runtime binary") + if not (temporary_runtime / "renderer-gl_x64.dll").is_file(): + raise AssertionError("temporary fast stage did not copy renderer runtime binary") + repeated_result = run_script( + BUILD_DIR / "stage_fast_install.py", + "--source-root", + source_root, + "--build-dir", + build_dir, + "--install-dir", + temporary_runtime, + "--temporary-runtime", + ) + if repeated_result.returncode == 0 or "must be new" not in repeated_result.stderr: + raise AssertionError( + f"stage_fast_install.py reused a temporary runtime: {repeated_result.stderr}" + ) def validate_stale_content_prune_symlink_handling() -> None: @@ -901,6 +954,98 @@ def validate_meson_source_symlink_guards() -> None: def validate_windows_runtime_staging_guards() -> None: + expected_stale_stage_files = { + f"openQ4-{kind}_{arch}" + for kind in ("client", "ded") + for arch in ("x86", "x64", "arm64") + } | { + f"renderer-{renderer}_{arch}.dll.mainbak" + for renderer in ("gl", "vk") + for arch in ("x86", "x64", "arm64") + } + if set(WINDOWS_RUNTIME.WINDOWS_STALE_STAGE_FILE_MANIFEST) != expected_stale_stage_files: + raise AssertionError("Windows stage cleanup must remain an exact stale-file allowlist") + if WINDOWS_RUNTIME.WINDOWS_EMPTY_STAGE_DIRECTORY_MANIFEST != ("baseoq4/skins",): + raise AssertionError("Windows stage cleanup must only prune the known empty skins directory") + + hygiene_root = WORK / "windows-runtime" / "stage-hygiene" + stale_client = hygiene_root / "openQ4-client_x64" + stale_dedicated = hygiene_root / "openQ4-ded_x64" + stale_backup = hygiene_root / "renderer-vk_x64.dll.mainbak" + current_client = hygiene_root / "openQ4-client_x64.exe" + current_renderer = hygiene_root / "renderer-vk_x64.dll" + unlisted_extensionless = hygiene_root / "openQ4-client_custom" + empty_skins = hygiene_root / "baseoq4" / "skins" + for path in (stale_client, stale_dedicated, stale_backup, current_client, current_renderer, unlisted_extensionless): + write_file(path) + empty_skins.mkdir(parents=True) + + hygiene_result = WINDOWS_RUNTIME.cleanup_windows_stage_target(hygiene_root) + if set(hygiene_result["removed_stale_files"]) != { + "openQ4-client_x64", + "openQ4-ded_x64", + "renderer-vk_x64.dll.mainbak", + }: + raise AssertionError(f"unexpected Windows stale-stage cleanup result: {hygiene_result!r}") + if hygiene_result["removed_empty_directories"] != ["baseoq4/skins"]: + raise AssertionError(f"empty Windows stage directory was not pruned: {hygiene_result!r}") + for removed_path in (stale_client, stale_dedicated, stale_backup, empty_skins): + if removed_path.exists() or removed_path.is_symlink(): + raise AssertionError(f"known stale Windows stage entry was retained: {removed_path}") + for preserved_path in (current_client, current_renderer, unlisted_extensionless): + if not preserved_path.is_file(): + raise AssertionError(f"Windows stage cleanup removed an unlisted runtime entry: {preserved_path}") + + populated_root = WORK / "windows-runtime" / "stage-hygiene-populated" + populated_skin = populated_root / "baseoq4" / "skins" / "custom.skin" + write_file(populated_skin) + populated_result = WINDOWS_RUNTIME.cleanup_windows_stage_target(populated_root) + if populated_result["removed_empty_directories"] or not populated_skin.is_file(): + raise AssertionError("Windows stage cleanup must preserve a populated baseoq4/skins directory") + + stale_directory_root = WORK / "windows-runtime" / "stage-hygiene-stale-directory" + (stale_directory_root / "openQ4-client_x64").mkdir(parents=True) + expect_runtime_error( + lambda: WINDOWS_RUNTIME.cleanup_windows_stage_target(stale_directory_root), + "not a regular file", + "Windows stage stale-file directory guard", + ) + + empty_path_file_root = WORK / "windows-runtime" / "stage-hygiene-empty-path-file" + write_file(empty_path_file_root / "baseoq4" / "skins") + expect_runtime_error( + lambda: WINDOWS_RUNTIME.cleanup_windows_stage_target(empty_path_file_root), + "is not a directory", + "Windows stage empty-directory path guard", + ) + + stale_link_root = WORK / "windows-runtime" / "stage-hygiene-stale-link" + stale_link_root.mkdir(parents=True) + stale_link_target = WORK / "windows-runtime" / "stage-hygiene-link-target" + write_file(stale_link_target, b"outside\n") + stale_link = stale_link_root / "renderer-vk_x64.dll.mainbak" + if make_symlink(stale_link_target, stale_link): + link_result = WINDOWS_RUNTIME.cleanup_windows_stage_target(stale_link_root) + if link_result["removed_stale_files"] != ["renderer-vk_x64.dll.mainbak"]: + raise AssertionError("Windows stage cleanup did not report the known stale symlink") + if stale_link.exists() or stale_link.is_symlink(): + raise AssertionError("Windows stage cleanup did not unlink the known stale symlink") + if stale_link_target.read_bytes() != b"outside\n": + raise AssertionError("Windows stage cleanup followed a known stale symlink target") + + linked_game_root = WORK / "windows-runtime" / "stage-hygiene-linked-game-root" + linked_game_target = WORK / "windows-runtime" / "stage-hygiene-linked-game-target" + linked_game_root.mkdir(parents=True) + (linked_game_target / "skins").mkdir(parents=True) + if make_symlink(linked_game_target, linked_game_root / "baseoq4", target_is_directory=True): + expect_runtime_error( + lambda: WINDOWS_RUNTIME.cleanup_windows_stage_target(linked_game_root), + "cleanup parent is a link or junction", + "Windows stage cleanup linked game-root guard", + ) + if not (linked_game_target / "skins").is_dir(): + raise AssertionError("Windows stage cleanup followed a linked baseoq4 directory") + malformed_pe = WORK / "windows-runtime" / "truncated.exe" malformed_pe.parent.mkdir(parents=True, exist_ok=True) data = bytearray(0x48) diff --git a/tools/tests/release_tooling_safety.py b/tools/tests/release_tooling_safety.py index 0f3b0801..c626f85a 100644 --- a/tools/tests/release_tooling_safety.py +++ b/tools/tests/release_tooling_safety.py @@ -1011,7 +1011,8 @@ def validate_manual_release_linux_runtime_gate() -> None: '["created OpenGL context"]', '["Shutting down OpenGL subsystem (SDL3 backend)"]', "executable = find_client_executable(root)", - "cwd=str(root / \".install\")", + 'runtime_dir = Path(args.runtime_dir).resolve() if args.runtime_dir else root / ".install"', + "cwd=str(runtime_dir)", "exit_code == 0 and not timed_out", ): if token not in renderer: diff --git a/tools/tests/renderer_gameplay_benchmark.py b/tools/tests/renderer_gameplay_benchmark.py index a63e1634..5cfca8a9 100644 --- a/tools/tests/renderer_gameplay_benchmark.py +++ b/tools/tests/renderer_gameplay_benchmark.py @@ -1246,6 +1246,7 @@ def run_mp_spec( ) append_set(server_args, "net_serverDedicated", "0") append_set(server_args, "net_port", str(port)) + append_set(server_args, "ui_autoJoin", "1") server_args += ["+seta", "si_pure", "0"] append_set(server_args, "net_serverAllowServerMod", "1") append_set(server_args, "sv_cheats", "1") @@ -1289,6 +1290,7 @@ def run_mp_spec( client_initial_autoexec_cfg, args.autoexec_delay_ms, ) + append_set(client_args, "ui_autoJoin", "1") append_set(client_args, "ui_name", "RendererBenchClient") client_postinit_connect_cfg = "" if spec.path_name == "postinit-connect": diff --git a/tools/tests/renderer_mp_flat_items.py b/tools/tests/renderer_mp_flat_items.py index 3d90de0a..3b9d705e 100644 --- a/tools/tests/renderer_mp_flat_items.py +++ b/tools/tests/renderer_mp_flat_items.py @@ -570,7 +570,7 @@ def test_render_entity_render_demo_and_module_abi_are_versioned(): require(gate, "ent.flatDiffuseFlags", "version 10 demos must restore flat flags") api = read(RENDERER / "RenderModuleAPI.h") - require(api, "#define RENDER_API_VERSION\t\t\t7", "the extended renderEntity ABI must be version 7") + require(api, "#define RENDER_API_VERSION\t\t\t8", "the resized-capture renderer ABI must be version 8") loader = read(RENDERER / "RendererModule.cpp") require( diff --git a/tools/tests/renderer_pbr_materials.py b/tools/tests/renderer_pbr_materials.py new file mode 100644 index 00000000..6b9635f8 --- /dev/null +++ b/tools/tests/renderer_pbr_materials.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""Compatibility contracts for opt-in PBR material metadata and resources.""" + +from __future__ import annotations + +import os +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +GAME_ROOT = Path( + os.environ.get("OPENQ4_GAMELIBS_REPO", ROOT.parent / "openQ4-game") +).resolve() + + +def read(path: Path) -> str: + if not path.is_file(): + raise AssertionError(f"required source is missing: {path}") + return path.read_text(encoding="utf-8") + + +def require(source: str, token: str, context: str) -> None: + if token not in source: + raise AssertionError(f"missing {token!r} in {context}") + + +def reject(source: str, token: str, context: str) -> None: + if token in source: + raise AssertionError(f"unexpected {token!r} in {context}") + + +def function_body(source: str, signature: str) -> str: + start = source.find(signature) + if start < 0: + raise AssertionError(f"missing function {signature!r}") + opening = source.find("{", start) + if opening < 0: + raise AssertionError(f"missing body for {signature!r}") + depth = 0 + for index in range(opening, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[opening + 1 : index] + raise AssertionError(f"unterminated body for {signature!r}") + + +def section(source: str, start_marker: str, end_marker: str) -> str: + start = source.find(start_marker) + end = source.find(end_marker, start + len(start_marker)) + if start < 0 or end < 0: + raise AssertionError(f"missing source section {start_marker!r}..{end_marker!r}") + return source[start:end] + + +def test_shared_material_abi_and_opt_in_defaults() -> None: + engine_header = read(ROOT / "src/renderer/Material.h") + game_header = read(GAME_ROOT / "src/renderer/Material.h") + if engine_header != game_header: + raise AssertionError("engine and companion Material.h must remain byte-identical") + + require(engine_header, "legacyFallbackMissing", "explicit missing-fallback state") + + init = read(ROOT / "src/renderer/RenderSystem_init.cpp") + for name, default in ( + ("r_pbrMaterials", "0"), + ("r_pbrGeneratedLegacyFallback", "1"), + ("r_pbrDebug", "0"), + ("r_pbrInferFromLegacyMaterials", "0"), + ): + require(init, f'idCVar {name}( "{name}", "{default}"', "PBR cvar defaults") + require(init, "PBR materials: parser=1 modernLighting=0", "honest gfxInfo capability") + + +def test_parser_is_namespaced_and_fail_closed() -> None: + material = read(ROOT / "src/renderer/Material.cpp") + parse_material = section(material, "void idMaterial::ParseMaterial( idLexer &src )", "void idMaterial::SetGui(") + require(parse_material, '!token.Icmp( "pbr" )', "top-level PBR namespace") + require(parse_material, '!token.Icmp( "physicallyBased" )', "PBR namespace alias") + reject(parse_material, "r_pbrInferFromLegacyMaterials", "classic parser path") + reject(parse_material, "pbrInfo.metallicRegister = GetExpressionConstant", "stock register allocation") + + parse_pbr = section(material, "bool idMaterial::ParsePBRBlock(", "void idMaterial::AddPBRLegacyFallbackStages(") + for token in ( + "workflow", + "albedoMap", + "normalMap", + "normalFormat", + "ormMap", + "metallicMap", + "roughnessMap", + "aoMap", + "emissiveMap", + "legacyBumpMap", + "legacyDiffuseMap", + "legacySpecularMap", + "legacyEmissiveMap", + "autoLegacyFallback", + ): + require(parse_pbr, f'!token.Icmp( "{token}" )', "PBR block parser") + require(parse_pbr, "unknown PBR parameter", "unknown-token rejection") + require(parse_pbr, "requires normalFormat", "normal encoding contract") + require(parse_pbr, "cannot combine ormMap", "packed/separate exclusivity") + require(parse_pbr, "pbrInfo.metallicRegister = GetExpressionConstant", "opt-in PBR register defaults") + require(parse_pbr, '( value != "0" && value != "1" )', "literal auto-fallback boolean grammar") + reject(parse_pbr, "value.GetIntValue()", "fractional auto-fallback coercion") + require(material, "autoLegacyFallback 0.5", "fractional auto-fallback negative self-test") + require(material, "albedoMap reflectionRenderMap", "dynamic image negative self-test") + require(material, "albedoMap glslProgram", "shader-token image negative self-test") + require(material, "albedoMap add( _white, reflectionRenderMap )", "nested dynamic image negative self-test") + require(material, "albedoMap add( _white, glslProgram )", "nested shader-token image negative self-test") + require(material, "albedoMap add( _white, _currentRender )", "nested scene-capture negative self-test") + require(material, "albedoMap add( _white, _reflectionRender )", "nested mutable-target negative self-test") + require(material, "albedoMap add( _white, alphaTest )", "nested stage-state negative self-test") + + unsupported_tokens = function_body(material, "static bool R_IsUnsupportedPBRImageProgramToken(") + parse_image = section(material, "bool idMaterial::ParsePBRImage(", "bool idMaterial::ParsePBRBlock(") + require(unsupported_tokens, "R_IsMutableRenderImageName( token.c_str() )", "central mutable-render-image rejection") + image_manager = read(ROOT / "src/renderer/ImageManager.cpp") + for mutable_name in ( + "_reflectionRender", + "_refractionRender", + "_currentRender", + "_forwardRenderResolved", + "_postProcessAlbedo", + "_shadowMap", + "_pointShadowMap", + "_hdrScene", + "_ssao", + "_bloom", + ): + require(image_manager, f'"{mutable_name}"', "mutable render-image classifier") + require(image_manager, "image->scratchImage = true;", "dynamic scratch-image classification") + for rejected in ( + "videoMap", + "soundMap", + "mirrorRenderMap", + "remoteRenderMap", + "reflectionRenderMap", + "refractionRenderMap", + "xrayRenderMap", + "cameraCubeMap", + "cubeMap", + "program", + "glslProgram", + "blend", + "map", + "screen", + "screen2", + "glassWarp", + "texGen", + "alphaTest", + "alphaFunc", + "translate", + "centerScale", + "shear", + "rotate", + "vertexColor", + "color", + "maskColor", + "privatePolygonOffset", + ): + require(unsupported_tokens, f'!token.Icmp( "{rejected}" )', "dynamic PBR image rejection") + require(parse_image, "R_IsUnsupportedPBRImageProgramToken( token )", "direct PBR image-token rejection") + require(parse_image, "R_IsUnsupportedPBRImageProgramToken( imageProgramToken )", "nested PBR image-token rejection") + require(parse_image, "R_IsMutableRenderImage( target.image )", "loaded mutable-image rejection") + require(parse_image, "while ( imageProgram.ReadToken", "complete PBR image-program validation") + for token in ( + "target.filter", + "target.repeat", + "target.allowPicmip", + "target.noMips", + "target.highQuality", + "target.forceHighQuality", + ): + require(parse_image, token, "retained PBR image options") + + fallback = section( + material, + "void idMaterial::AddPBRLegacyFallbackStages( const textureRepeat_t trpDefault )", + "void idMaterial::ParseDeform( idLexer &src )", + ) + require(fallback, "const bool hasUsableClassicInteraction = hasBump && hasDiffuse;", "complete classic fallback authority") + require(fallback, "!hasUsableClassicInteraction", "approximation requires incomplete classic interaction") + require(fallback, "const bool allowApproximate", "approximation gate") + require(fallback, "pbrInfo.normalFormat == PBR_NORMAL_QUAKE4_AGB", "classic normal encoding gate") + for token in ( + 'buffer.Append( "nearest\\n" )', + 'buffer.Append( "noclamp\\n" )', + 'buffer.Append( "nopicmip\\n" )', + 'buffer.Append( "nomips\\n" )', + 'buffer.Append( "forceHighQuality\\n" )', + ): + require(fallback, token, "classic fallback image-option replay") + require(material, "diffuseStage->texture.image == expectedDiffuse", "runtime fallback image identity assertion") + require(material, "pbrInfo.legacyFallbackMissing = !( hasBump && hasDiffuse );", "missing fallback classification") + require(material, "has no usable classic bump+diffuse fallback", "missing fallback diagnostic") + require(material, "_pbr_selftest_missing_fallback", "missing fallback runtime self-test") + require(material, "_pbr_selftest_tangent_normal_fallback", "tangent normal fallback runtime self-test") + require(material, "_pbr_selftest_quake4_normal_fallback", "Quake 4 normal fallback runtime self-test") + require(material, "bumpStage->texture.image == globalImages->flatNormalMap", "tangent normal neutral fallback assertion") + require(material, "bumpStage->texture.image == info.normal.image", "Quake 4 normal reuse assertion") + require(material, "const bool oldGeneratedFallback = r_pbrGeneratedLegacyFallback.GetBool();", "fallback self-test cvar save") + require(material, "r_pbrGeneratedLegacyFallback.SetBool( oldGeneratedFallback );", "fallback self-test cvar restore") + + +def test_texture_usage_and_lifecycle_contracts() -> None: + image_header = read(ROOT / "src/renderer/Image.h") + order = [image_header.find(name) for name in ("TD_HIGH_QUALITY", "TD_PBR_COLOR", "TD_MATERIAL_DATA")] + if any(position < 0 for position in order) or order != sorted(order): + raise AssertionError("PBR image usages must append after historical cache identities") + + image_load = read(ROOT / "src/renderer/Image_load.cpp") + require(image_load, "case TD_PBR_COLOR:", "PBR color derivation") + require(image_load, "opts.gammaMips = true;", "gamma-correct PBR color mips") + require(image_load, "case TD_MATERIAL_DATA:", "linear material-data derivation") + gamma_downsize = function_body(image_load, "static bool R_ImageUsageUsesGammaMips(") + require(gamma_downsize, "TD_PBR_COLOR", "gamma-correct PBR color prefiltering") + declared_usage = function_body(image_load, "static void R_LoadImageProgramForDeclaredUsage(") + require(declared_usage, "declaredUsage != TD_PBR_COLOR", "PBR color image-program usage preservation") + require(declared_usage, "declaredUsage != TD_MATERIAL_DATA", "PBR data image-program usage preservation") + load_image = function_body(image_load, "void idImage::ActuallyLoadImage(") + if load_image.count("R_LoadImageProgramForDeclaredUsage(") != 4: + raise AssertionError("every image-program decode in ActuallyLoadImage must preserve declared PBR usage") + + require(image_header, "textureUsage_t GetUsage() const", "PBR image usage runtime introspection") + material = read(ROOT / "src/renderer/Material.cpp") + require(material, "albedoMap heightmap( _white, 1 )", "PBR color usage mutation regression self-test") + require(material, "ormMap smoothnormals( _flat )", "PBR data usage mutation regression self-test") + require(material, "sameAlbedo == info.albedo.image", "PBR color cache identity assertion") + require(material, "sameORM == info.orm.image", "PBR data cache identity assertion") + require(material, "registerMatches( info.metallicRegister, 0.25f )", "PBR metallic register value assertion") + require(material, "registerMatches( info.emissiveColorRegisters[2], 0.3f )", "PBR emissive register value assertion") + + image_manager = read(ROOT / "src/renderer/ImageManager.cpp") + namespace_usage = function_body(image_manager, "static textureUsage_t R_ImageUsageForName(") + require(namespace_usage, "requestedUsage != TD_DEFAULT", "explicit image-usage namespace preservation") + require(namespace_usage, "return requestedUsage;", "explicit PBR image usage return") + if image_manager.count("usage = R_ImageUsageForName( _name, usage );") != 3: + raise AssertionError("all immediate, lookup, and deferred image paths must share namespace usage policy") + + for signature in ( + "void idMaterial::AddReference()", + "void idMaterial::ResolveUse()", + "void idMaterial::ReloadImages( bool force ) const", + ): + body = function_body(material, signature) + require(body, "pbrInfo.albedo", f"PBR lifecycle in {signature}") + require(body, "pbrInfo.legacyEmissive", f"complete PBR lifecycle in {signature}") + free_data = function_body(material, "void idMaterial::FreeData()") + require(free_data, "memset( &pbrInfo, 0, sizeof( pbrInfo ) );", "purged PBR metadata reset") + require(free_data, "pbrInfo.normalFormat = PBR_NORMAL_UNSPECIFIED;", "purged normal-format reset") + + +def test_scene_packet_and_resource_table_are_explicitly_non_visible() -> None: + packets_h = read(ROOT / "src/renderer/ScenePackets.h") + packets_cpp = read(ROOT / "src/renderer/ScenePackets.cpp") + table_h = read(ROOT / "src/renderer/MaterialResourceTable.h") + table_cpp = read(ROOT / "src/renderer/MaterialResourceTable.cpp") + + for token in ( + "hasPBR", + "pbrAlbedoImage", + "pbrNormalImage", + "pbrORMImage", + "pbrWorkflow", + "pbrNormalFormat", + "pbrUsesApproximateLegacyFallback", + "pbrLegacyFallbackMissing", + ): + require(packets_h, token, "scene-packet PBR record") + require(packets_cpp, token, "scene-packet PBR capture") + + for semantic in ( + "ALBEDO", + "NORMAL", + "ORM", + "METALLIC", + "ROUGHNESS", + "AO", + "EMISSIVE_PBR", + ): + require(table_h, f"MATERIAL_RESOURCE_TEXTURE_{semantic}", "PBR resource semantics") + require(table_h, "pbrResourceReady", "resource readiness split") + require(table_h, "pbrModernReady", "visible readiness split") + for token in ( + "classicRecords", + "pbrExplicitGeneratedFallbackRecords", + "pbrMissingLegacyFallbackRecords", + "pbrMissingAlbedoMapRecords", + "pbrMissingNormalMapRecords", + "pbrMissingORMMapRecords", + ): + require(table_h, token, "PBR fallback/map observability") + require(table_cpp, token, "PBR fallback/map metrics") + + texture_table = function_body( + table_cpp, + "static void R_MaterialResourceTable_BuildTextureArrayTable(", + ) + require(texture_table, "if ( record.hasPBR )", "PBR isolation from classic texture table") + require(texture_table, "binding.textureArrayLayer = -1;", "PBR texture-table layer reset") + finalize = section( + table_cpp, + "static void R_MaterialResourceTable_FinalizePBRContract(", + "static void R_MaterialResourceTable_FinalizeShadowContract(", + ) + require(finalize, "MATERIAL_RESOURCE_PBR_FALLBACK_SHADER_PATH_UNAVAILABLE", "Phase 3 visible fail-closed gate") + require(finalize, "record.pbrModernReady = false;", "no premature visible-PBR claim") + reject(finalize, "record.pbrModernReady = true", "premature visible-PBR ownership") + pbr_binding_ready = function_body( + table_cpp, + "static bool R_MaterialResourceTable_PBRBindingReady(", + ) + require(pbr_binding_ready, "!R_IsMutableRenderImage( binding.image )", "mutable PBR resource rejection") + + classic_gate = function_body( + table_cpp, + "bool R_MaterialResourceTable_ClassicModernPathEligible(", + ) + require(classic_gate, "return !record.hasPBR;", "classic-modern PBR exclusion") + require(table_cpp, "!R_MaterialResourceTable_ClassicModernPathEligible( *record )", "native PBR ownership self-test") + require(table_cpp, "pbrBindingsExcludedFromClassicTable", "native PBR texture-table isolation self-test") + require(table_cpp, "stats.textureArrayTableDescriptors == 0", "PBR classic-table descriptor isolation") + require(table_cpp, "stats.classicRecords == 0", "PBR records excluded from classic record count") + require(table_cpp, "stats.classicRecords != expectedRecords", "classic packet record count assertion") + require(table_cpp, 'RendererMaterialResourceTable PBR contract self-test passed', "native PBR table pass marker") + require(table_cpp, "savedMaxClassicTextureUnits", "uninitialized-backend PBR table unit budget scope") + require(table_cpp, "rg_materialResourceTable.maxClassicTextureUnits = savedMaxClassicTextureUnits;", "PBR table unit budget restoration") + for token in ( + "separateSource", + "scalarSource", + "explicitSource", + "unsupportedSource", + "missingFallbackSource", + "missingAlbedoSource", + "mutableImageSource", + "redundantExplicitSource", + ): + require(table_cpp, token, "PBR material-table layout self-tests") + require(table_cpp, "unsupportedRecord->pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_UNSUPPORTED_WORKFLOW", "unsupported workflow table assertion") + require(table_cpp, "missingAlbedoRecord->pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_ALBEDO", "missing albedo table assertion") + require(table_cpp, "mutableImageRecord->pbrFallbackReason == MATERIAL_RESOURCE_PBR_FALLBACK_MISSING_IMAGE", "mutable image table assertion") + require(table_cpp, "!redundantExplicitRecord->pbrUsesGeneratedLegacyFallback", "redundant explicit-map metric assertion") + require(table_cpp, "rg_materialResourceTable.records[i].material = NULL;", "self-test declaration lifetime cleanup") + + draw_plan = read(ROOT / "src/renderer/ModernGLDrawPlan.cpp") + submit_plan = read(ROOT / "src/renderer/ModernGLSubmitPlan.cpp") + executor = read(ROOT / "src/renderer/ModernGLExecutor.cpp") + require(draw_plan, "!R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord )", "draw-plan PBR exclusion") + require(submit_plan, "!R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord )", "submit-plan PBR exclusion") + if executor.count("!R_MaterialResourceTable_ClassicModernPathEligible( *materialRecord )") < 4: + raise AssertionError("modern executor must reject PBR ownership and all classic visible pipelines") + + scene_packet_selftest = function_body(packets_cpp, "bool RendererScenePacket_RunSelfTest(") + require(scene_packet_selftest, "pbrPacketFrame.AddDrawPacket", "native PBR scene-packet capture") + require(scene_packet_selftest, "pbrDrawPacket->materialRecord == pbrRecord", "public draw-packet material linkage") + require(scene_packet_selftest, "pbrRecord->pbrLegacyFallbackMissing", "PBR missing-fallback packet propagation assertion") + require(scene_packet_selftest, "validRegister( pbrRecord->pbrMetallicRegister )", "PBR packet register range assertion") + + +def test_runtime_selftest_is_registered_and_required() -> None: + init = read(ROOT / "src/renderer/RenderSystem_init.cpp") + material = read(ROOT / "src/renderer/Material.cpp") + table = read(ROOT / "src/renderer/MaterialResourceTable.cpp") + matrix = read(ROOT / "tools/tests/renderer_validation_matrix.py") + require(init, '"rendererPBRMaterialSelfTest"', "renderer command registration") + require(init, '"RendererPBRMaterial self-test passed\\n"', "runtime success marker") + require(matrix, '"RendererPBRMaterial self-test passed"', "matrix result contract") + require(matrix, '"+rendererPBRMaterialSelfTest"', "matrix command contract") + for source, context in ((material, "parser self-test"), (table, "resource-table self-test")): + require(source, "declManager->AllocateDecl( DECL_MATERIAL )", context) + require(source, "DeclManager_FreeAllocatedDecl", context) + reject(source, "idMaterial dual;", context) + + +def main() -> int: + tests = [value for name, value in sorted(globals().items()) if name.startswith("test_")] + for test in tests: + test() + print("renderer_pbr_materials: ok") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/renderer_screenshot_readback.py b/tools/tests/renderer_screenshot_readback.py index 93ca61b1..c0ea2cad 100644 --- a/tools/tests/renderer_screenshot_readback.py +++ b/tools/tests/renderer_screenshot_readback.py @@ -16,12 +16,34 @@ def require(source: str, snippet: str, context: str) -> None: raise AssertionError(f"Missing {snippet!r} in {context}") +def reject(source: str, snippet: str, context: str) -> None: + if snippet in source: + raise AssertionError(f"Unexpected {snippet!r} in {context}") + + def require_order(source: str, snippets: tuple[str, ...], context: str) -> None: positions = [source.find(snippet) for snippet in snippets] if any(position < 0 for position in positions) or positions != sorted(positions): raise AssertionError(f"Expected ordered snippets in {context}: {snippets!r}") +def function_body(source: str, signature: str) -> str: + start = source.find(signature) + if start < 0: + raise AssertionError(f"Missing function signature {signature!r}") + + depth = 0 + for index in range(start, len(source)): + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + if depth == 0: + return source[start : index + 1] + + raise AssertionError(f"Could not find end of function {signature!r}") + + def test_screenshot_reads_the_unpresented_back_buffer() -> None: init_cpp = read("src/renderer/RenderSystem_init.cpp") require_order( @@ -71,10 +93,158 @@ def test_vulkan_capture_resumes_the_acquired_back_buffer() -> None: ) +def test_save_preview_resamples_a_coherent_full_frame() -> None: + session_cpp = read("src/framework/Session.cpp") + save_game = function_body(session_cpp, "bool idSessionLocal::SaveGame(") + require_order( + save_game, + ( + "sessionRenderCropGuard_t previewCrop( renderSystem->GetScreenWidth(), renderSystem->GetScreenHeight() );", + "game->Draw( 0 );", + "renderSystem->CaptureRenderToFile( tempPreviewFile, true, 320, 240 );", + ), + "physical full-frame save-preview capture and CPU resize", + ) + guard = function_body(session_cpp, "class sessionRenderCropGuard_t") + require_order( + guard, + ( + "renderSystem->CropRenderSize( width, height, false, true );", + "active = true;", + "~sessionRenderCropGuard_t()", + "renderSystem->UnCrop();", + ), + "exception-safe physical render crop", + ) + + renderer_header = read("src/renderer/RenderSystem.h") + require_order( + renderer_header, + ( + "CaptureRenderToFile( const char *fileName, bool fixAlpha = false )", + "SetUnderwaterView( float amount, const idVec3 &tint, float fogDistance )", + "CaptureRenderToFile( const char *fileName, bool fixAlpha, int outputWidth, int outputHeight )", + ), + "append-only resized-capture renderer interface", + ) + + +def test_capture_skips_rgb_pack_padding() -> None: + renderer_cpp = read("src/renderer/RenderSystem.cpp") + capture = function_body( + renderer_cpp, + "void idRenderSystemLocal::CaptureRenderToFile( const char *fileName, bool fixAlpha,", + ) + require_order( + capture, + ( + "const int sourceStride = ( rc->width * 3 + 3 ) & ~3;", + "const size_t sourceBytes = (size_t)sourceStride * (size_t)rc->height;", + "R_StaticAlloc( sourceBytes )", + "memset( data, 0, sourceBytes );", + "glReadPixels(", + "const byte *sourceRow = data + (size_t)y * sourceStride;", + "byte *destinationRow = data2 + (size_t)y * rc->width * 4;", + "R_StaticFree( data );", + ), + "zeroed padded GL_RGB capture conversion", + ) + + width = 3 + rows = ( + bytes((1, 2, 3, 4, 5, 6, 7, 8, 9)), + bytes((11, 12, 13, 14, 15, 16, 17, 18, 19)), + ) + stride = (width * 3 + 3) & ~3 + packed = b"".join(row + bytes(stride - len(row)) for row in rows) + converted = b"".join( + packed[offset : offset + width * 3] + for offset in range(0, len(packed), stride) + ) + if converted != b"".join(rows): + raise AssertionError("row-stride model included GL_RGB pack padding") + + +def aspect_crop( + source_width: int, + source_height: int, + output_width: int, + output_height: int, +) -> tuple[int, int, int, int]: + crop_x = 0 + crop_y = 0 + crop_width = source_width + crop_height = source_height + source_product = source_width * output_height + output_product = source_height * output_width + if source_product > output_product: + crop_width = max(source_height * output_width // output_height, 1) + crop_x = (source_width - crop_width) // 2 + elif source_product < output_product: + crop_height = max(source_width * output_height // output_width, 1) + crop_y = (source_height - crop_height) // 2 + return crop_x, crop_y, crop_width, crop_height + + +def test_capture_center_crops_and_resamples_safely() -> None: + renderer_cpp = read("src/renderer/RenderSystem.cpp") + helper = function_body(renderer_cpp, "static byte *R_ResampleCaptureToAspectRGBA(") + capture = function_body( + renderer_cpp, + "void idRenderSystemLocal::CaptureRenderToFile( const char *fileName, bool fixAlpha,", + ) + + require_order( + helper, + ( + "const int64 sourceAspectProduct = (int64)sourceWidth * outputHeight;", + "cropX = ( sourceWidth - cropWidth ) / 2;", + "cropY = ( sourceHeight - cropHeight ) / 2;", + "const double sourceY = cropY +", + "byte *outputRow = output + (size_t)y * (size_t)outputWidth * 4;", + ), + "orientation-preserving center-aspect resize", + ) + require(capture, "outputWidth > MAX_CAPTURE_DIMENSION", "bounded capture output width") + require(capture, "outputHeight > MAX_CAPTURE_DIMENSION", "bounded capture output height") + require(renderer_cpp, "static const int64 MAX_CAPTURE_PIXELS = 33554432;", "capture pixel budget") + require(capture, "const int64 sourcePixelCount = (int64)rc->width * rc->height;", "64-bit source pixel count") + require(capture, "sourcePixelCount > MAX_CAPTURE_PIXELS", "bounded source pixel count") + require(capture, "const int64 outputPixelCount = (int64)outputWidth * outputHeight;", "64-bit output pixel count") + require(capture, "outputPixelCount > MAX_CAPTURE_PIXELS", "bounded output pixel count") + require(capture, "R_ResampleCaptureToAspectRGBA(", "CPU save-preview resample") + require_order( + capture, + ( + "R_StaticFree( data );", + "outputData = R_ResampleCaptureToAspectRGBA(", + "R_StaticFree( data2 );", + "R_WriteTGA(", + "R_StaticFree( outputData );", + ), + "bounded capture allocation lifetimes", + ) + + expected_crops = { + (1280, 720, 320, 240): (160, 0, 960, 720), + (1024, 768, 320, 240): (0, 0, 1024, 768), + (720, 1280, 320, 240): (0, 370, 720, 540), + } + for dimensions, expected in expected_crops.items(): + actual = aspect_crop(*dimensions) + if actual != expected: + raise AssertionError( + f"aspect crop for {dimensions!r} was {actual!r}, expected {expected!r}" + ) + + def main() -> None: test_screenshot_reads_the_unpresented_back_buffer() test_capture_defers_only_the_window_present() test_vulkan_capture_resumes_the_acquired_back_buffer() + test_save_preview_resamples_a_coherent_full_frame() + test_capture_skips_rgb_pack_padding() + test_capture_center_crops_and_resamples_safely() print("renderer_screenshot_readback: ok") diff --git a/tools/tests/renderer_validation_matrix.py b/tools/tests/renderer_validation_matrix.py index e65cea9d..0f62319b 100644 --- a/tools/tests/renderer_validation_matrix.py +++ b/tools/tests/renderer_validation_matrix.py @@ -39,6 +39,7 @@ ["RendererRenderGraph self-test passed"], ["RendererRenderGraphResource self-test passed", "RendererRenderGraphResource self-test skipped"], ["RendererMaterialResourceTable self-test passed", "RendererMaterialResourceTable self-test skipped"], + ["RendererPBRMaterial self-test passed"], ["RendererGeometryResource self-test passed"], ["RendererGLStateCache self-test passed", "RendererGLStateCache self-test skipped"], ["RendererModernGLShaderLibrary self-test passed"], @@ -456,7 +457,7 @@ def sanitize_case_id(case_id: str) -> str: def common_args( - root: Path, + runtime_dir: Path, case_id: str, basepath: str, savepath: Path, @@ -499,7 +500,7 @@ def common_args( str(savepath), "+set", "fs_devpath", - str(root / ".install"), + str(runtime_dir), "+set", "fs_game", "baseoq4", @@ -546,6 +547,7 @@ def build_safe_cases(tiers: tuple[str, ...]) -> list[dict[str, Any]]: "+rendererRenderGraphSelfTest", "+rendererRenderGraphResourceSelfTest", "+rendererMaterialResourceTableSelfTest", + "+rendererPBRMaterialSelfTest", "+rendererGeometryResourceSelfTest", "+rendererGLStateCacheSelfTest", "+rendererModernGLExecutorSelfTest", @@ -1567,23 +1569,23 @@ def filter_driver_specific_cases(cases: list[dict[str, Any]]) -> list[dict[str, ] -def vk_module_path(root: Path) -> Path: +def vk_module_path(runtime_dir: Path) -> Path: if os.name == "nt": suffix = ".dll" elif sys.platform == "darwin": suffix = ".dylib" else: suffix = ".so" - return root / ".install" / f"renderer-vk_{host_arch()}{suffix}" + return runtime_dir / f"renderer-vk_{host_arch()}{suffix}" -def filter_vulkan_module_cases(cases: list[dict[str, Any]], root: Path) -> list[dict[str, Any]]: +def filter_vulkan_module_cases(cases: list[dict[str, Any]], runtime_dir: Path) -> list[dict[str, Any]]: # the Vulkan cases need a staged renderer-vk module and a live Vulkan # driver. Headless Linux legs (Xvfb/WSL) offer neither, so they stay # dropped there. Windows has a native driver; macOS runs the module on # MoltenVK, which is bundled with the package, so both hosts qualify once # the module is staged next to the executable. - if (os.name == "nt" or sys.platform == "darwin") and vk_module_path(root).exists(): + if (os.name == "nt" or sys.platform == "darwin") and vk_module_path(runtime_dir).exists(): return cases dropped = [case["id"] for case in cases if case.get("requiresVulkanModule")] if dropped: @@ -1705,6 +1707,7 @@ def print_failure_details(result: dict[str, Any]) -> None: def run_case( root: Path, executable: Path, + runtime_dir: Path, output_dir: Path, savepath: Path, basepath: str, @@ -1723,7 +1726,7 @@ def run_case( case_assetless = bool(case.get("assetless", False)) case_basepath = "" if case_assetless else basepath case_skip_official_pak_validation = skip_official_pak_validation or case_assetless - args = common_args(root, case_id, case_basepath, savepath, case_skip_official_pak_validation) + case["args"] + ["+quit"] + args = common_args(runtime_dir, case_id, case_basepath, savepath, case_skip_official_pak_validation) + case["args"] + ["+quit"] startup_commands = sum(1 for arg in args if arg.startswith("+")) if startup_commands > ENGINE_MAX_STARTUP_COMMANDS: raise RuntimeError( @@ -1732,7 +1735,7 @@ def run_case( ) # drill lever: hide the staged renderer-vk module so the loader's # fallback ladder is exercised for real, restoring it afterwards - module_path = vk_module_path(root) + module_path = vk_module_path(runtime_dir) hidden_module_path = module_path.with_name(module_path.name + ".drill-hidden") hide_vk_module = bool(case.get("hideVkModule", False)) # cvars set on the command line are archived on exit; cases that opt @@ -1750,7 +1753,7 @@ def run_case( with stdout_path.open("w", encoding="utf-8", errors="replace") as stdout_file, stderr_path.open("w", encoding="utf-8", errors="replace") as stderr_file: process = subprocess.Popen( [str(executable)] + args, - cwd=str(root / ".install"), + cwd=str(runtime_dir), stdout=stdout_file, stderr=stderr_file, ) @@ -2030,6 +2033,11 @@ def parse_args(argv: list[str]) -> argparse.Namespace: default="", help="Explicit client or launcher to test. Defaults to the host-matching staged client.", ) + parser.add_argument( + "--runtime-dir", + default="", + help="Runtime root for working directory, renderer modules, and fs_devpath. Defaults to /.install.", + ) parser.add_argument( "--skip-official-pak-validation", action="store_true", @@ -2054,10 +2062,6 @@ def main(argv: list[str]) -> int: return 2 requested = set(requested_cases) safe_cases = [case for case in safe_cases if case["id"] in requested] - elif not args.list: - safe_cases = filter_driver_specific_cases(safe_cases) - safe_cases = filter_vulkan_module_cases(safe_cases, root) - if args.list: print("Automated safe cases:") for case in safe_cases: @@ -2104,6 +2108,13 @@ def main(argv: list[str]) -> int: return 2 else: executable = find_client_executable(root) + runtime_dir = Path(args.runtime_dir).resolve() if args.runtime_dir else root / ".install" + if not runtime_dir.is_dir(): + print(f"runtime directory does not exist: {runtime_dir}", file=sys.stderr) + return 2 + if not requested_cases: + safe_cases = filter_driver_specific_cases(safe_cases) + safe_cases = filter_vulkan_module_cases(safe_cases, runtime_dir) savepath = Path(args.savepath).resolve() if args.savepath else root / ".home" savepath.mkdir(parents=True, exist_ok=True) timestamp = time.strftime("%Y%m%d-%H%M%S") @@ -2121,6 +2132,7 @@ def main(argv: list[str]) -> int: result = run_case( root, executable, + runtime_dir, output_dir, savepath, basepath, @@ -2137,6 +2149,7 @@ def main(argv: list[str]) -> int: "generated": time.strftime("%Y-%m-%d %H:%M:%S %z"), "host": f"{platform.system()} {platform.release()} {platform.machine()}", "executable": str(executable), + "runtimeDir": str(runtime_dir), "savepath": str(savepath), "basepath": basepath, "skipOfficialPakValidation": args.skip_official_pak_validation, diff --git a/tools/tests/settings_menu_coverage.py b/tools/tests/settings_menu_coverage.py index f1e7ec00..1900f765 100644 --- a/tools/tests/settings_menu_coverage.py +++ b/tools/tests/settings_menu_coverage.py @@ -25,6 +25,29 @@ def reject(haystack: str, needle: str, context: str) -> None: raise AssertionError(f"Unexpected {needle!r} in {context}") +def cvar_statement(source_text: str, name: str, context: str) -> str: + match = re.search( + rf'^idCVar\s+\w+\(\s*"{re.escape(name)}".*?;\s*$', + source_text, + re.IGNORECASE | re.MULTILINE, + ) + if match is None: + raise AssertionError(f"Missing CVar declaration for {name!r} in {context}") + return " ".join(match.group(0).split()) + + +def cvar_signature(source_text: str, name: str, context: str) -> tuple[str, set[str]]: + statement = cvar_statement(source_text, name, context) + match = re.match( + rf'idCVar\s+\w+\(\s*"{re.escape(name)}"\s*,\s*"([^"]*)"\s*,\s*([^,]+),', + statement, + re.IGNORECASE, + ) + if match is None: + raise AssertionError(f"Cannot parse CVar declaration for {name!r} in {context}") + return match.group(1), {flag.strip() for flag in match.group(2).split("|")} + + def gui_block(source_text: str, widget_name: str) -> str: pattern = re.compile(rf"\b(?:bindDef|choiceDef|editDef|sliderDef|windowDef)\s+{re.escape(widget_name)}\b") match = pattern.search(source_text) @@ -626,10 +649,10 @@ def validate_controls_pane_extraction(mainmenu: str, controls_gui: str) -> None: '"loadgame quick"', "clientMessageMode", '"clientMessageMode 1"', - "voteyes", - "voteno", + "_impulse28", + "_impulse29", "_ingamestats", - "ready", + "_impulse17", '"emote salute"', '"emote cheer"', '"emote taunt"', @@ -962,6 +985,35 @@ def main() -> None: for text, context in ((sp_cvars, "SP game cvars"), (mp_cvars, "MP game cvars")): require(text, 'g_autoSkipCinematics",\t\t"0",\t\t\tCVAR_GAME | PC_CVAR_ARCHIVE | CVAR_BOOL', context) + # Stock multiplayer GUIs bind these names directly. Both game modules can + # be the first module loaded by the unified executable, so each must + # register an identical typed declaration rather than leaving a typeless + # GUI-created placeholder behind. + for name in ( + "net_menulanserver", + "net_serverMenuDedicated", + "s_voiceChatSend", + "s_voiceChatReceive", + "s_voiceChatEcho", + "s_voiceVolume", + "s_micInputLevel", + ): + sp_statement = cvar_statement(sp_cvars, name, "SP game cvars") + mp_statement = cvar_statement(mp_cvars, name, "MP game cvars") + if sp_statement != mp_statement: + raise AssertionError(f"SP/MP GUI CVar declarations differ for {name!r}") + + # The modules intentionally retain Quake 4's distinct SP (box) and MP + # (cylinder) defaults, but they share an integer type so an in-process + # module switch cannot register the same global CVar with two types. + sp_default, sp_flags = cvar_signature(sp_cvars, "pm_usecylinder", "SP game cvars") + mp_default, mp_flags = cvar_signature(mp_cvars, "pm_usecylinder", "MP game cvars") + if (sp_default, mp_default) != ("0", "1"): + raise AssertionError("pm_usecylinder no longer preserves the stock SP/MP defaults") + for flags, context in ((sp_flags, "SP"), (mp_flags, "MP")): + if "CVAR_INTEGER" not in flags or "CVAR_BOOL" in flags: + raise AssertionError(f"{context} pm_usecylinder must use the shared integer type") + for cvar in ("cl_gun_x", "cl_gun_y", "cl_gun_z"): require(mp_cvars, f'idCVar {cvar}', "MP game cvar definitions") require(mp_header, f"extern idCVar\t{cvar}", "MP game cvar declarations") diff --git a/tools/tests/stock_asset_baseline.py b/tools/tests/stock_asset_baseline.py new file mode 100644 index 00000000..a04387a2 --- /dev/null +++ b/tools/tests/stock_asset_baseline.py @@ -0,0 +1,1263 @@ +#!/usr/bin/env python3 +"""Focused unit/contract tests for the P0 stock-asset baseline harness.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import struct +import sys +import tempfile +from pathlib import Path +from types import ModuleType +from zipfile import ZipFile + + +ROOT = Path(__file__).resolve().parents[2] +TOOL_PATH = ROOT / "tools" / "validation" / "stock_asset_baseline.py" + + +def load_tool() -> ModuleType: + spec = importlib.util.spec_from_file_location("openq4_stock_asset_baseline", TOOL_PATH) + if spec is None or spec.loader is None: + raise AssertionError("could not load stock_asset_baseline.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def cvar_value(arguments: list[str], name: str) -> str | None: + for index in range(len(arguments) - 2): + if arguments[index] == "+set" and arguments[index + 1] == name: + return arguments[index + 2] + return None + + +def patterned_rgb(width: int, height: int) -> bytes: + pixels = bytearray(width * height * 3) + for y in range(height): + for x in range(width): + index = (y * width + x) * 3 + pixels[index] = (x * 17 + y * 43 + x * y * 3) & 0xFF + pixels[index + 1] = (x * 71 + y * 11 + x * y * 5) & 0xFF + pixels[index + 2] = (x * 29 + y * 97 + x * y * 7) & 0xFF + return bytes(pixels) + + +def write_rgb_tga(path: Path, width: int, height: int, rgb: bytes) -> None: + assert len(rgb) == width * height * 3 + header = bytearray(18) + header[2] = 2 + struct.pack_into(" None: + write_rgb_tga(path, width, height, patterned_rgb(width, height)) + + +def recursive_strip_rgb(width: int, height: int) -> bytes: + assert width % 4 == 0 and height % 8 == 0 + pixels = bytearray(patterned_rgb(width, height)) + for factor in (2, 4): + target_width = width // factor + target_height = (height // 2) // factor + target_y = height - height // factor + block_pixels = factor * factor + for y in range(target_height): + for x in range(target_width): + channels = [0, 0, 0] + for block_y in range(factor): + for block_x in range(factor): + source_index = ( + ((y * factor + block_y) * width + x * factor + block_x) * 3 + ) + for channel in range(3): + channels[channel] += pixels[source_index + channel] + target_index = ((target_y + y) * width + x) * 3 + for channel in range(3): + pixels[target_index + channel] = ( + channels[channel] + block_pixels // 2 + ) // block_pixels + return bytes(pixels) + + +def scene_rgb(width: int, height: int) -> bytes: + """Low-frequency synthetic game frame with geometry, lighting and HUD detail.""" + + pixels = bytearray(width * height * 3) + for y in range(height): + for x in range(width): + index = (y * width + x) * 3 + red = 18 + 72 * x // max(1, width - 1) + 24 * y // max(1, height - 1) + green = 28 + 45 * x // max(1, width - 1) + 36 * y // max(1, height - 1) + blue = 42 + 28 * x // max(1, width - 1) + 54 * y // max(1, height - 1) + if height * 5 // 9 < y: + floor_band = ((x // max(1, width // 12)) + (y // max(1, height // 10))) & 1 + red = 48 + floor_band * 16 + 18 * x // max(1, width - 1) + green = 42 + floor_band * 12 + blue = 38 + floor_band * 9 + if width * 7 // 16 < x < width * 9 // 16 and height // 5 < y < height * 4 // 5: + red += 50 + green += 25 + blue -= 12 + distance = (x - width * 11 // 16) ** 2 + (y - height * 2 // 5) ** 2 + if distance < max(1, width // 14) ** 2: + red += 95 + green += 70 + blue += 24 + if y >= height * 9 // 10 and width // 8 < x < width * 7 // 8: + red = 18 + 80 * x // max(1, width - 1) + green = 115 + 70 * x // max(1, width - 1) + blue = 72 + pixels[index] = min(255, max(0, red)) + pixels[index + 1] = min(255, max(0, green)) + pixels[index + 2] = min(255, max(0, blue)) + return bytes(pixels) + + +def postprocess_variant(rgb: bytes, width: int, height: int) -> bytes: + """Model mild exposure drift, a one-pixel scene shift and a changed effect.""" + + output = bytearray(len(rgb)) + for y in range(height): + for x in range(width): + source_x = max(0, x - 1) + source = (y * width + source_x) * 3 + target = (y * width + x) * 3 + output[target] = min(255, rgb[source] * 94 // 100 + 8) + output[target + 1] = min(255, rgb[source + 1] * 97 // 100 + 5) + output[target + 2] = min(255, rgb[source + 2] * 92 // 100 + 10) + if width * 2 // 3 < x < width * 3 // 4 and height // 3 < y < height // 2: + output[target] = min(255, output[target] + 20) + output[target + 1] = min(255, output[target + 1] + 12) + return bytes(output) + + +def compressed_exposure_variant(rgb: bytes) -> bytes: + """Model a bright, low-contrast frame that needs shared exposure correction.""" + + return bytes( + min(255, max(0, (channel + 192) * 1000 // 2040)) for channel in rgb + ) + + +def severe_color_defect(rgb: bytes) -> bytes: + """Apply a high-correlation red cast that exposure alone cannot repair.""" + + output = bytearray(len(rgb)) + for index in range(0, len(rgb), 3): + output[index] = min(255, rgb[index] + 90) + output[index + 1] = rgb[index + 1] // 2 + output[index + 2] = rgb[index + 2] // 2 + return bytes(output) + + +def feedback_corruption(tool: ModuleType, rgb: bytes, width: int, height: int) -> bytes: + """Overlay offset recursive views without matching the legacy strip signature.""" + + output = bytearray(rgb) + for target_x, target_y, target_width, target_height in ( + (5, height // 4, width * 3 // 4, height * 3 // 4), + (2, height * 11 // 16, width * 3 // 8, height * 5 // 16), + ): + scaled = tool.center_aspect_resample_rgb( + rgb, width, height, target_width, target_height + ) + for y in range(target_height): + if target_y + y >= height: + break + source = y * target_width * 3 + target = ((target_y + y) * width + target_x) * 3 + copy_width = min(target_width, width - target_x) + output[target : target + copy_width * 3] = scaled[ + source : source + copy_width * 3 + ] + return bytes(output) + + +def test_save_preview_coherence(tool: ModuleType, base: Path) -> None: + screenshot = base / "coherent-shot.tga" + preview = base / "coherent-preview.tga" + screenshot_width, screenshot_height = 640, 360 + preview_width, preview_height = tool.SP_SAVE_PREVIEW_DIMENSIONS + screenshot_rgb = scene_rgb(screenshot_width, screenshot_height) + preview_rgb = tool.center_aspect_resample_rgb( + screenshot_rgb, + screenshot_width, + screenshot_height, + preview_width, + preview_height, + ) + write_rgb_tga(screenshot, screenshot_width, screenshot_height, screenshot_rgb) + write_rgb_tga(preview, preview_width, preview_height, preview_rgb) + comparison, failure = tool.save_preview_comparison(screenshot, preview) + assert failure is None and comparison is not None + assert comparison["referenceArtifact"] == "saveReferenceScreenshot" + assert comparison["algorithm"].startswith("same-state-") + assert comparison["lumaCorrelation"] > 0.99 + + write_rgb_tga( + preview, + preview_width, + preview_height, + postprocess_variant(preview_rgb, preview_width, preview_height), + ) + variant_comparison, failure = tool.save_preview_comparison(screenshot, preview) + assert failure is None and variant_comparison is not None + assert variant_comparison["lumaCorrelation"] > tool.SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION + + write_rgb_tga( + preview, + preview_width, + preview_height, + compressed_exposure_variant(preview_rgb), + ) + exposure_comparison, failure = tool.save_preview_comparison(screenshot, preview) + assert failure is None and exposure_comparison is not None + assert ( + exposure_comparison["meanAbsoluteRgbError"] + > tool.SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR + ) + assert ( + exposure_comparison["exposureCompensatedMeanAbsoluteRgbError"] + <= tool.SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR + ) + + write_rgb_tga( + preview, + preview_width, + preview_height, + severe_color_defect(preview_rgb), + ) + color_comparison, failure = tool.save_preview_comparison(screenshot, preview) + assert color_comparison is not None + assert color_comparison["lumaCorrelation"] > tool.SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION + assert failure is not None and "exposure-compensated" in failure + + corrupted_rgb = feedback_corruption( + tool, preview_rgb, preview_width, preview_height + ) + write_rgb_tga(preview, preview_width, preview_height, corrupted_rgb) + assert tool.validate_tga(preview, tool.SP_SAVE_PREVIEW_DIMENSIONS) is None + corrupted_comparison, failure = tool.save_preview_comparison(screenshot, preview) + assert corrupted_comparison is not None + assert failure is not None and "luma correlation" in failure + + final6 = ( + ROOT + / ".tmp" + / "stock-baseline" + / "p0-20260819-save-preview-final6-sp-default" + / "savepaths" + / "sp" + / "baseoq4" + ) + final6_screenshot = final6 / "screenshots" / "stock-baseline" / "sp_after_load.tga" + final6_preview = final6 / "savegames" / "StockBaselineSP.tga" + final6_comparison = None + if final6_screenshot.is_file() and final6_preview.is_file(): + final6_comparison, failure = tool.save_preview_comparison( + final6_screenshot, final6_preview + ) + assert final6_comparison is not None and failure is None + assert final6_comparison["lumaCorrelation"] >= tool.SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION + assert ( + final6_comparison["exposureCompensatedMeanAbsoluteRgbError"] + <= tool.SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR + ) + + legacy_preview = ( + ROOT + / ".tmp" + / "stock-baseline" + / "p0-20260819-save-preview-final6-sp-legacy50" + / "savepaths" + / "sp" + / "baseoq4" + / "savegames" + / "StockBaselineSP.tga" + ) + if legacy_preview.is_file(): + default_decoded = tool.decode_tga_rgb(final6_preview.read_bytes()) + legacy_decoded = tool.decode_tga_rgb(legacy_preview.read_bytes()) + assert not isinstance(default_decoded, str) + assert not isinstance(legacy_decoded, str) + default_width, default_height, default_rgb = default_decoded + legacy_width, legacy_height, legacy_rgb = legacy_decoded + analysis_width, analysis_height = tool.SP_SAVE_PREVIEW_ANALYSIS_DIMENSIONS + default_normalized = tool.center_aspect_resample_rgb( + default_rgb, + default_width, + default_height, + analysis_width, + analysis_height, + ) + legacy_normalized = tool.center_aspect_resample_rgb( + legacy_rgb, + legacy_width, + legacy_height, + analysis_width, + analysis_height, + ) + preview_metrics = tool.rgb_similarity_metrics( + default_normalized, legacy_normalized + ) + assert not isinstance(preview_metrics, str) + assert preview_metrics[0] >= tool.SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION + compensated_metrics = tool.exposure_compensated_rgb_metrics( + default_normalized, legacy_normalized + ) + assert not isinstance(compensated_metrics, str) + assert compensated_metrics[2] <= tool.SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR + + final12 = ( + ROOT + / ".tmp" + / "stock-baseline" + / "p0-20260819-final12" + / "savepaths" + / "sp" + / "baseoq4" + ) + final12_screenshot = final12 / "screenshots" / "stock-baseline" / "sp_after_load.tga" + final12_preview = final12 / "savegames" / "StockBaselineSP.tga" + if final12_screenshot.is_file() and final12_preview.is_file(): + assert tool.validate_tga( + final12_preview, tool.SP_SAVE_PREVIEW_DIMENSIONS + ) is None + final12_comparison, failure = tool.save_preview_comparison( + final12_screenshot, final12_preview + ) + assert final12_comparison is not None and failure is None + assert final12_comparison["lumaCorrelation"] >= tool.SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION + assert ( + final12_comparison["meanAbsoluteRgbError"] + > tool.SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR + ) + assert ( + final12_comparison["exposureCompensatedMeanAbsoluteRgbError"] + <= tool.SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR + ) + + final11 = ( + ROOT + / ".tmp" + / "stock-baseline" + / "p0-20260819-final11" + / "savepaths" + / "sp" + / "baseoq4" + ) + final11_screenshot = final11 / "screenshots" / "stock-baseline" / "sp_after_load.tga" + final11_preview = final11 / "savegames" / "StockBaselineSP.tga" + if final11_screenshot.is_file() and final11_preview.is_file(): + final11_comparison, failure = tool.save_preview_comparison( + final11_screenshot, final11_preview + ) + assert final11_comparison is not None + assert final11_comparison["lumaCorrelation"] < tool.SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION + assert failure is not None and "luma correlation" in failure + if final6_comparison is not None: + assert ( + final6_comparison["lumaCorrelation"] + - final11_comparison["lumaCorrelation"] + >= 0.10 + ) + + +def test_asset_inventory_and_comparison(tool: ModuleType, base: Path) -> None: + asset_root = base / "assets" + q4base = asset_root / "q4base" + q4mp = asset_root / "q4mp" + q4base.mkdir(parents=True) + q4mp.mkdir() + (q4base / "pak001.pk4").write_bytes(b"stock-one") + (q4base / "zpak_english.pk4").write_bytes(b"language") + (q4mp / "game300.pk4").write_bytes(b"mp") + + records = tool.collect_pk4s(asset_root) + assert [item["path"] for item in records] == [ + "q4base/pak001.pk4", + "q4base/zpak_english.pk4", + "q4mp/game300.pk4", + ] + assert tool.compare_asset_records(records, records) == [] + changed = json.loads(json.dumps(records)) + changed[0]["sha256"] = "0" * 64 + failures = tool.compare_asset_records(changed, records) + assert len(failures) == 1 and "SHA" not in failures[0] and "sha256 differs" in failures[0] + (q4base / "generated.cfg").write_text("set test 1", encoding="utf-8") + loose = tool.collect_loose_asset_files(asset_root) + assert [item["path"] for item in loose] == ["q4base/generated.cfg"] + + contaminated = base / "contaminated" + (contaminated / "q4base").mkdir(parents=True) + (contaminated / "q4base" / "pak001.pk4").write_bytes(b"stock") + (contaminated / "baseoq4").mkdir() + (contaminated / "baseoq4" / "override.cfg").write_text("override", encoding="utf-8") + try: + tool.collect_pk4s(contaminated) + except ValueError as exc: + assert "non-empty" in str(exc) and "unapproved content" in str(exc) + else: + raise AssertionError("non-empty asset-root/baseoq4 must fail closed") + + original_link_check = tool.is_link_or_junction + tool.is_link_or_junction = lambda path: path.name == "q4base" + try: + try: + tool.validate_asset_root(asset_root) + except ValueError as exc: + assert "--allow-asset-dir-links" in str(exc) + else: + raise AssertionError("linked asset directories require explicit opt-in") + tool.validate_asset_root(asset_root, allow_asset_dir_links=True) + views = tool.asset_directory_views(asset_root) + assert next(view for view in views if view["path"] == "q4base")["linked"] is True + finally: + tool.is_link_or_junction = original_link_check + + stale = base / "stale-evidence" + stale.mkdir() + (stale / "old.log").write_text("stale", encoding="utf-8") + try: + tool.prepare_output_directory(stale) + except ValueError as exc: + assert "new or empty" in str(exc) + else: + raise AssertionError("non-empty evidence directories must be rejected") + + collision_assets = base / "collision-assets" + collision_runtime = base / "collision-runtime" + (collision_assets / "q4base").mkdir(parents=True) + (collision_runtime / "baseoq4").mkdir(parents=True) + with ZipFile(collision_assets / "q4base" / "pak001.pk4", "w") as archive: + archive.writestr("guis/mainmenu.gui", "retail") + archive.writestr("materials/retail_only.mtr", "retail") + with ZipFile(collision_runtime / "baseoq4" / "pak0.pk4", "w") as archive: + archive.writestr("GUIS/MainMenu.gui", "overlay") + archive.writestr("materials/openq4_only.mtr", "overlay") + collision_records = tool.collect_retail_path_collisions( + collision_assets, + tool.collect_pk4s(collision_assets), + collision_runtime, + tool.collect_overlay_pk4s(collision_runtime), + ) + assert collision_records == [ + { + "path": "guis/mainmenu.gui", + "retailPk4s": ["q4base/pak001.pk4"], + "openQ4OverlayPk4s": ["baseoq4/pak0.pk4"], + } + ] + report_dir = base / "collision-report" + report_dir.mkdir() + tool.write_reports( + report_dir, + { + "schemaVersion": tool.SCHEMA_VERSION, + "status": "pass", + "generatedUtc": "2026-08-19T00:00:00Z", + "git": tool.git_state(ROOT), + "mpPort": 28140, + "runtimeRoot": str(collision_runtime.resolve()), + "runtimeFiles": [], + "expectedAssets": {"supplied": True, "sha256": "1" * 64}, + "assets": { + "compatibilityModel": tool.ASSET_COMPATIBILITY_MODEL, + "root": str(collision_assets.resolve()), + "allowLinkedGameDirectories": False, + "directoryViews": tool.asset_directory_views(collision_assets), + "stockPk4s": tool.collect_pk4s(collision_assets), + "looseFiles": [], + "openQ4OverlayPk4s": tool.collect_overlay_pk4s(collision_runtime), + "openQ4OverlayLooseFiles": [], + "retailArchiveBytesMatchExpected": True, + "retailPathNamespaceUntouched": False, + "retailPathCollisionCount": 1, + "retailPathCollisions": collision_records, + }, + "assetComparisonFailures": [], + "preflightFailures": [], + "postCaptureVerificationFailures": [], + "results": [], + }, + ) + report_markdown = (report_dir / "stock_asset_baseline_report.md").read_text( + encoding="utf-8" + ) + assert "not an overlay-free or stock-only run" in report_markdown + assert "`baseoq4/pak0.pk4` supersedes 1 retail virtual path" in report_markdown + assert "clean logs" not in report_markdown.casefold() + assert "none of the harness denylist classes" in report_markdown + assert "engine `ERROR` records" in report_markdown + assert "shader compile/program-link failures" in report_markdown + assert "Vulkan validation messages or VUIDs" in report_markdown + assert "OpenGL errors" in report_markdown + assert "does not assert warning-free logs" in report_markdown + + +def test_plan_is_windowed_and_engine_only(tool: ModuleType, base: Path) -> None: + root = base / "repo" + output = base / "evidence" + asset_root = base / "assets" + (root / ".install").mkdir(parents=True) + plans = tool.prepare_plans(root, asset_root, output, 1280, 720, 28140) + assert set(plans) == {"sp-capture", "sp-demo-playback", "mp-server", "mp-client"} + for plan in plans.values(): + assert cvar_value(plan.args, "r_fullscreen") == "0" + assert cvar_value(plan.args, "r_borderless") == "0" + assert cvar_value(plan.args, "r_borderlessDefaultMigrated") == "1" + assert cvar_value(plan.args, "r_fullscreenDesktop") == "0" + assert cvar_value(plan.args, "r_windowWidth") == "1280" + assert cvar_value(plan.args, "r_windowHeight") == "720" + assert cvar_value(plan.args, "r_mode") == "-1" + assert cvar_value(plan.args, "r_customWidth") == "1280" + assert cvar_value(plan.args, "r_customHeight") == "720" + assert cvar_value(plan.args, "r_renderApi") == "gl" + assert cvar_value(plan.args, "fs_devpath") == str(plan.savepath) + assert cvar_value(plan.args, "fs_cdpath") is None + assert plan.args.count("+vid_restart") == 1 + assert plan.args.index("+vid_restart") > plan.args.index("r_windowHeight") + record = tool.plan_record(plan) + assert record["windowed"] is True + assert record["captureMethod"] == "engine screenshot command" + if plan.role_id.startswith("mp-"): + assert cvar_value(plan.args, "ui_autoJoin") == "1" + assert plan.args.index("ui_autoJoin") < plan.args.index("+vid_restart") + if plan.role_id == "mp-server": + assert cvar_value(plan.args, "si_pure") == "1" + assert cvar_value(plan.args, "net_serverAllowServerMod") == "0" + + sp_stage1 = (output / "savepaths" / "sp" / "baseoq4" / "stock-baseline" / "sp_stage1.cfg").read_text(encoding="utf-8") + sp_stage2 = (output / "savepaths" / "sp" / "baseoq4" / "stock-baseline" / "sp_after_load.cfg").read_text(encoding="utf-8") + for token in ("recordDemo stock_baseline_sp", "saveGame StockBaselineSP", "loadGame StockBaselineSP"): + assert token in sp_stage1 + sp_stage1_lines = sp_stage1.splitlines() + save_reference_line = sp_stage1_lines.index( + 'screenshot "screenshots/stock-baseline/sp_before_save.tga"' + ) + save_game_line = sp_stage1_lines.index("saveGame StockBaselineSP") + assert save_reference_line + 1 == save_game_line + assert plans["sp-capture"].expected.count( + ( + "saveReferenceScreenshot", + "baseoq4/screenshots/stock-baseline/sp_before_save.tga", + ) + ) == 1 + assert "OPENQ4_STOCK_BASELINE_SP_SAVE_LOAD_COMPLETE" in sp_stage2 + assert 'screenshot "screenshots/stock-baseline/sp_after_load.tga"' in sp_stage2 + mp_server_cfg = (output / "savepaths" / "mp-server" / "baseoq4" / "stock-baseline" / "server.cfg").read_text(encoding="utf-8") + mp_client_cfg = (output / "savepaths" / "mp-client" / "baseoq4" / "stock-baseline" / "client.cfg").read_text(encoding="utf-8") + assert "waitMsec 60000" in mp_server_cfg + assert "openq4_joinGame" not in mp_client_cfg + assert "openq4_assertMPClientActive" in mp_client_cfg + assert "openq4_assertMPGameplayView" in mp_client_cfg + mp_client_lines = mp_client_cfg.splitlines() + screenshot_line = mp_client_lines.index('screenshot "screenshots/stock-baseline/mp_client.tga"') + assertion_lines = [ + index for index, line in enumerate(mp_client_lines) if line == "openq4_assertMPClientActive" + ] + assert len(assertion_lines) == 3 + assert assertion_lines[-2] < screenshot_line < assertion_lines[-1] + view_assertion_lines = [ + index for index, line in enumerate(mp_client_lines) if line == "openq4_assertMPGameplayView" + ] + assert len(view_assertion_lines) == 3 + assert view_assertion_lines[-2] < screenshot_line < view_assertion_lines[-1] + assert mp_client_lines[view_assertion_lines[-1] + 1] == ( + "echo OPENQ4_STOCK_BASELINE_MP_CLIENT_COMPLETE" + ) + assert tool.runtime_window_failure( + " [0] * Test Display (contentScale 1.50)\nMODE: -1, 1920 x 1080 windowed hz:N/A\n", + 1280, + 720, + ) is None + assert "borderless" in tool.runtime_window_failure( + "MODE: 5, 2560 x 1440 borderless hz:N/A\n", 1280, 720 + ) + assert "missing" in tool.runtime_window_failure("no renderer evidence\n") + assert "screenshot-write" in tool.mp_client_active_proof_failure( + tool.MP_CLIENT_ACTIVE_MARKER + "\nOPENQ4_STOCK_BASELINE_MP_CLIENT_COMPLETE\n" + ) + legacy_active_line = ( + tool.MP_CLIENT_ACTIVE_MARKER + + " client=1 spectating=0 wantSpectate=0 ingame=1\n" + ) + view_line = tool.MP_CLIENT_VIEW_MARKER + " gui=0\n" + screenshot_write = "Wrote screenshots/stock-baseline/mp_client.tga\n" + assert "two exact" in tool.mp_client_active_proof_failure( + legacy_active_line + + view_line + + screenshot_write + + legacy_active_line + + view_line + + "OPENQ4_STOCK_BASELINE_MP_CLIENT_COMPLETE\n" + ) + active_line = ( + tool.MP_CLIENT_ACTIVE_MARKER + + " client=1 spectating=0 wantSpectate=0 ingame=1 menu=0 disableHud=0\n" + ) + completion_line = "OPENQ4_STOCK_BASELINE_MP_CLIENT_COMPLETE\n" + assert tool.mp_client_active_proof_failure( + active_line + view_line + screenshot_write + active_line + view_line + completion_line + ) is None + assert "bracket" in tool.mp_client_active_proof_failure( + active_line + view_line + active_line + view_line + screenshot_write + completion_line + ) + assert "bracket" in tool.mp_client_active_proof_failure( + screenshot_write + active_line + view_line + active_line + view_line + completion_line + ) + assert "2560x1440" in tool.runtime_window_failure( + " [0] * Test Display (contentScale 1.50)\nMODE: -1, 2560 x 1440 windowed hz:N/A\n", + 1280, + 720, + ) + + source = TOOL_PATH.read_text(encoding="utf-8") + for forbidden in ("pyautogui", "ImageGrab", "BitBlt", "PrintWindow", "mss.mss", "pynput"): + assert forbidden not in source + + +def test_diagnostic_authority_and_shared_mp_deadline( + tool: ModuleType, base: Path +) -> None: + log = base / "authoritative.log" + stdout = base / "mirrored.stdout.txt" + stderr = base / "mirrored.stderr.txt" + active_line = ( + tool.MP_CLIENT_ACTIVE_MARKER + + " client=1 spectating=0 wantSpectate=0 ingame=1 menu=0 disableHud=0\n" + ) + view_line = tool.MP_CLIENT_VIEW_MARKER + " gui=0\n" + screenshot_write = "Wrote screenshots/stock-baseline/mp_client.tga\n" + completion_line = "OPENQ4_STOCK_BASELINE_MP_CLIENT_COMPLETE\n" + valid_sequence = ( + active_line + + view_line + + screenshot_write + + active_line + + view_line + + completion_line + ) + log.write_text(valid_sequence, encoding="utf-8") + stdout.write_text(valid_sequence, encoding="utf-8") + stderr.write_text("", encoding="utf-8") + authoritative, all_diagnostics = tool.collect_role_diagnostics( + log, stdout, stderr + ) + assert authoritative == valid_sequence + assert all_diagnostics.count(screenshot_write.strip()) == 2 + assert tool.mp_client_active_proof_failure(authoritative) is None + assert "exactly one" in tool.mp_client_active_proof_failure(all_diagnostics) + + # A valid mirrored stdout sequence must not repair broken ordering in the + # authoritative engine log. + log.write_text( + active_line + view_line + active_line + view_line + screenshot_write + completion_line, + encoding="utf-8", + ) + authoritative, _ = tool.collect_role_diagnostics(log, stdout, stderr) + assert "bracket" in tool.mp_client_active_proof_failure(authoritative) + + clock = [100.0] + kill_times: list[float] = [] + + class HangingProcess: + def poll(self) -> None: + return None + + def kill(self) -> None: + kill_times.append(clock[0]) + + def wait(self, timeout: int) -> int: + assert timeout == 10 + return -9 + + original_monotonic = tool.time.monotonic + original_sleep = tool.time.sleep + tool.time.monotonic = lambda: clock[0] + tool.time.sleep = lambda seconds: clock.__setitem__(0, clock[0] + seconds) + try: + results = tool.wait_processes_until( + {"server": HangingProcess(), "client": HangingProcess()}, 101.0 + ) + finally: + tool.time.monotonic = original_monotonic + tool.time.sleep = original_sleep + assert results == {"server": (-9, True), "client": (-9, True)} + assert len(kill_times) == 2 + assert all(abs(kill_time - 101.0) < 1e-9 for kill_time in kill_times) + + +def make_runtime_package(tool: ModuleType, runtime_dir: Path) -> tuple[Path, list[dict[str, object]]]: + game_dir = runtime_dir / "baseoq4" + game_dir.mkdir(parents=True) + suffix = ".exe" if os.name == "nt" else "" + shared = ".dll" if os.name == "nt" else (".dylib" if sys.platform == "darwin" else ".so") + executable = runtime_dir / f"openQ4-client_{tool.host_arch()}{suffix}" + executable.write_bytes(b"client") + (runtime_dir / f"openQ4-ded_{tool.host_arch()}{suffix}").write_bytes(b"dedicated") + (runtime_dir / f"openQ4-client_{tool.host_arch()}.pdb").write_bytes(b"symbols") + (runtime_dir / f"renderer-gl_{tool.host_arch()}{shared}").write_bytes(b"renderer") + (runtime_dir / f"OpenAL32{shared}").write_bytes(b"audio") + (game_dir / f"game-sp_{tool.host_arch()}{shared}").write_bytes(b"sp") + (game_dir / f"game-mp_{tool.host_arch()}{shared}").write_bytes(b"mp") + (game_dir / "mod.json").write_text("{}", encoding="utf-8") + (game_dir / f"game-sp_{tool.host_arch()}.pdb").write_bytes(b"symbols") + records = tool.collect_runtime_files(runtime_dir, executable) + assert {item["kind"] for item in records} >= { + "clientExecutable", + "singlePlayerGameModule", + "multiplayerGameModule", + "dedicatedServerExecutable", + "diagnosticSymbols", + "runtimeLibrary", + } + loose, unexpected = tool.collect_overlay_loose_files(runtime_dir, records) + assert unexpected == [] and any(item["path"] == "baseoq4/mod.json" for item in loose) + unexpected_content = game_dir / "maps" / "override.map" + unexpected_content.parent.mkdir() + unexpected_content.write_text("override", encoding="utf-8") + _, unexpected = tool.collect_overlay_loose_files(runtime_dir, records) + assert any("override.map" in failure for failure in unexpected) + unexpected_content.unlink() + return executable, records + + +def test_tga_and_report_verification(tool: ModuleType, base: Path) -> None: + screenshot = base / "shot.tga" + write_minimal_tga(screenshot) + assert tool.validate_tga(screenshot) is None + assert tool.validate_tga(screenshot, (16, 8)) is None + assert "differ" in tool.validate_tga(screenshot, (32, 16)) + screenshot.write_bytes(b"short") + assert tool.validate_tga(screenshot) == "TGA header is truncated" + blank = base / "blank.tga" + write_rgb_tga(blank, 16, 8, bytes(16 * 8 * 3)) + assert "blank or near-solid" in tool.validate_tga(blank) + recursive = base / "recursive.tga" + write_rgb_tga(recursive, 64, 32, recursive_strip_rgb(64, 32)) + assert "recursive scaled-strip" in tool.validate_tga(recursive) + malformed = base / "malformed.tga" + write_minimal_tga(malformed) + malformed.write_bytes(malformed.read_bytes() + b"trailing") + assert "payload length" in tool.validate_tga(malformed) + + asset_root = base / "verify-assets" + (asset_root / "q4base").mkdir(parents=True) + stock = asset_root / "q4base" / "pak001.pk4" + stock.write_bytes(b"retail") + asset_records = tool.collect_pk4s(asset_root) + runtime_dir = base / "runtime" + runtime_executable, runtime_records = make_runtime_package(tool, runtime_dir) + overlay_loose, unexpected = tool.collect_overlay_loose_files(runtime_dir, runtime_records) + assert unexpected == [] + report_width, report_height = 640, 480 + expected_assets_path = base / "expected-assets.json" + expected_assets_path.write_text( + json.dumps( + { + "schemaVersion": 1, + "allowLinkedGameDirectories": False, + "directoryViews": tool.asset_directory_views(asset_root), + "stockPk4s": asset_records, + "looseFiles": [], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + report: dict[str, object] = { + "schemaVersion": tool.SCHEMA_VERSION, + "status": "pass", + "dryRun": False, + "git": tool.git_state(ROOT), + "mpPort": 28140, + "runtimeRoot": str(runtime_dir.resolve()), + "expectedAssets": { + "supplied": True, + "path": str(expected_assets_path.resolve()), + "sha256": tool.sha256_file(expected_assets_path), + }, + "runtimeFiles": runtime_records, + "safety": { + "windowedOnly": True, + "borderless": False, + "windowSize": {"width": report_width, "height": report_height}, + "engineScreenshotOnly": True, + "operatingSystemCapture": False, + "inputInjection": False, + }, + "assets": { + "compatibilityModel": tool.ASSET_COMPATIBILITY_MODEL, + "root": str(asset_root.resolve()), + "allowLinkedGameDirectories": False, + "directoryViews": tool.asset_directory_views(asset_root), + "stockPk4s": asset_records, + "looseFiles": [], + "openQ4OverlayPk4s": [], + "openQ4OverlayLooseFiles": overlay_loose, + "retailArchiveBytesMatchExpected": True, + "retailPathNamespaceUntouched": True, + "retailPathCollisionCount": 0, + "retailPathCollisions": [], + }, + "assetComparisonFailures": [], + "preflightFailures": [], + "postCaptureVerificationFailures": [], + "plan": [], + "results": [], + } + plans: list[dict[str, object]] = report["plan"] # type: ignore[assignment] + results: list[dict[str, object]] = report["results"] # type: ignore[assignment] + generated_plans = tool.prepare_plans( + runtime_dir, asset_root, base, report_width, report_height, 28140 + ) + for role, contract in tool.ROLE_EVIDENCE_CONTRACT.items(): + savepath = base / "savepaths" / contract["saveDir"] + plans.append(tool.plan_record(generated_plans[role])) + artifacts: list[dict[str, object]] = [] + log = base / "savepaths" / contract["saveDir"] / "baseoq4" / "logs" / contract["logName"] + log.parent.mkdir(parents=True, exist_ok=True) + active_proofs = "" + if role == "mp-client": + active_line = ( + tool.MP_CLIENT_ACTIVE_MARKER + + " client=1 spectating=0 wantSpectate=0 ingame=1 menu=0 disableHud=0\n" + ) + view_line = tool.MP_CLIENT_VIEW_MARKER + " gui=0\n" + active_proofs = ( + active_line + + view_line + + "Wrote screenshots/stock-baseline/mp_client.tga\n" + + active_line + + view_line + ) + log.write_text( + " [0] * Test Display (contentScale 1.00)\n" + f"MODE: -1, {report_width} x {report_height} windowed hz:N/A\n" + + active_proofs + + contract["marker"] + + "\n", + encoding="utf-8", + ) + artifacts.append({"kind": "engineLog", **tool.file_record(log, base)}) + for kind, stream in (("processStdout", "stdout"), ("processStderr", "stderr")): + path = base / f"{role}.{stream}.txt" + path.write_text("", encoding="utf-8") + artifacts.append({"kind": kind, **tool.file_record(path, base)}) + for kind, relative in contract["expected"].items(): + path = base / "savepaths" / contract["saveDir"] / relative + path.parent.mkdir(parents=True, exist_ok=True) + if kind == "screenshot": + if role == "sp-capture": + write_rgb_tga( + path, + report_width, + report_height, + patterned_rgb(report_width, report_height), + ) + else: + write_minimal_tga(path, report_width, report_height) + elif kind == "saveReferenceScreenshot": + write_rgb_tga( + path, + report_width, + report_height, + scene_rgb(report_width, report_height), + ) + elif kind == "savePreview": + reference_path = ( + base + / "savepaths" + / contract["saveDir"] + / contract["expected"]["saveReferenceScreenshot"] + ) + reference_decoded = tool.decode_tga_rgb(reference_path.read_bytes()) + assert not isinstance(reference_decoded, str) + reference_width, reference_height, reference_rgb = reference_decoded + preview_width, preview_height = tool.SP_SAVE_PREVIEW_DIMENSIONS + write_rgb_tga( + path, + preview_width, + preview_height, + tool.center_aspect_resample_rgb( + reference_rgb, + reference_width, + reference_height, + preview_width, + preview_height, + ), + ) + else: + path.write_bytes(b"evidence") + artifacts.append({"kind": kind, **tool.file_record(path, base)}) + result: dict[str, object] = { + "role": role, + "mode": generated_plans[role].mode, + "status": "pass", + "exitCode": 0, + "timedOut": False, + "failures": [], + "artifacts": artifacts, + } + if role == "sp-capture": + reference_path = ( + base + / "savepaths" + / contract["saveDir"] + / contract["expected"]["saveReferenceScreenshot"] + ) + preview_path = ( + base + / "savepaths" + / contract["saveDir"] + / contract["expected"]["savePreview"] + ) + comparison, failure = tool.save_preview_comparison( + reference_path, preview_path + ) + assert comparison is not None and failure is None + result["savePreviewComparison"] = comparison + results.append(result) + live_sp_result = tool.evaluate_role( + generated_plans["sp-capture"], base, 0, False + ) + assert live_sp_result["status"] == "pass" + assert live_sp_result["savePreviewComparison"] == next( + result for result in results if result["role"] == "sp-capture" + )["savePreviewComparison"] + assert tool.verify_recorded_files(report, base, asset_root, runtime_dir) == [] + assert tool.verify_recorded_files(report, base, asset_root) == [] + + missing_expected_binding = json.loads(json.dumps(report)) + missing_expected_binding["expectedAssets"]["path"] = str( + base / "missing-expected-assets.json" + ) + assert any( + "expected-assets manifest is missing" in failure + for failure in tool.verify_recorded_files( + missing_expected_binding, base, asset_root, runtime_dir + ) + ) + wrong_expected_hash = json.loads(json.dumps(report)) + wrong_expected_hash["expectedAssets"]["sha256"] = "0" * 64 + assert any( + "expected-assets manifest SHA-256 differs" in failure + for failure in tool.verify_recorded_files( + wrong_expected_hash, base, asset_root, runtime_dir + ) + ) + unrelated_expected_path = base / "unrelated-expected-assets.json" + unrelated_expected_path.write_text( + json.dumps( + { + "schemaVersion": 1, + "allowLinkedGameDirectories": False, + "directoryViews": tool.asset_directory_views(asset_root), + "stockPk4s": [], + "looseFiles": [], + } + ), + encoding="utf-8", + ) + unrelated_expected = json.loads(json.dumps(report)) + unrelated_expected["expectedAssets"]["path"] = str(unrelated_expected_path.resolve()) + unrelated_expected["expectedAssets"]["sha256"] = tool.sha256_file( + unrelated_expected_path + ) + assert any( + "inventory differs from the bound expected-assets manifest" in failure + for failure in tool.verify_recorded_files( + unrelated_expected, base, asset_root, runtime_dir + ) + ) + dry_run_pass = json.loads(json.dumps(report)) + dry_run_pass["dryRun"] = True + assert any( + "dryRun=false" in failure + for failure in tool.verify_recorded_files(dry_run_pass, base, asset_root, runtime_dir) + ) + wrong_revision = json.loads(json.dumps(report)) + wrong_revision["git"]["revision"] = "0" * 40 + assert any( + "not the current openQ4 HEAD" in failure + for failure in tool.verify_recorded_files( + wrong_revision, base, asset_root, runtime_dir + ) + ) + for failure_field in tool.TOP_LEVEL_FAILURE_ARRAYS: + recorded_failure = json.loads(json.dumps(report)) + recorded_failure[failure_field] = ["adversarial recorded failure"] + assert any( + f"{failure_field} field is nonempty" in failure + for failure in tool.verify_recorded_files( + recorded_failure, base, asset_root, runtime_dir + ) + ) + false_collision_inventory = json.loads(json.dumps(report)) + false_collision_inventory["assets"]["retailPathCollisions"] = [ + { + "path": "guis/fake.gui", + "retailPk4s": ["q4base/pak001.pk4"], + "openQ4OverlayPk4s": ["baseoq4/pak0.pk4"], + } + ] + false_collision_inventory["assets"]["retailPathCollisionCount"] = 1 + false_collision_inventory["assets"]["retailPathNamespaceUntouched"] = False + assert any( + "collision inventory differs" in failure + for failure in tool.verify_recorded_files( + false_collision_inventory, base, asset_root, runtime_dir + ) + ) + + def mutated_launch(role: str, token: str, value: str) -> dict[str, object]: + candidate = json.loads(json.dumps(report)) + role_plan = next(plan for plan in candidate["plan"] if plan["role"] == role) + token_index = role_plan["arguments"].index(token) + role_plan["arguments"][token_index + 1] = value + return candidate + + for role, token, value, expected_failure in ( + ("sp-capture", "fs_game", "wronggame", "launch CVar fs_game differs"), + ("sp-capture", "+map", "game/wrong", "launch command map differs"), + ("sp-capture", "si_gameType", "DM", "launch CVar si_gameType differs"), + ("mp-server", "+spawnServer", "mp/wrong", "launch command spawnServer differs"), + ("mp-server", "net_serverDedicated", "1", "launch CVar net_serverDedicated differs"), + ("mp-server", "si_gameType", "Tourney", "launch CVar si_gameType differs"), + ("mp-client", "+connect", "127.0.0.1:1", "launch command connect differs"), + ): + assert any( + expected_failure in failure + for failure in tool.verify_recorded_files( + mutated_launch(role, token, value), base, asset_root, runtime_dir + ) + ) + + mp_client_result = next( + result for result in results if result["role"] == "mp-client" + ) + mp_client_log_artifact = next( + artifact + for artifact in mp_client_result["artifacts"] # type: ignore[union-attr] + if artifact["kind"] == "engineLog" + ) + mp_client_stdout_artifact = next( + artifact + for artifact in mp_client_result["artifacts"] # type: ignore[union-attr] + if artifact["kind"] == "processStdout" + ) + mp_client_log = base / str(mp_client_log_artifact["path"]) + mp_client_stdout = base / str(mp_client_stdout_artifact["path"]) + mp_client_stdout.write_text( + mp_client_log.read_text(encoding="utf-8"), encoding="utf-8" + ) + mp_client_stdout_artifact.update(tool.file_record(mp_client_stdout, base)) + mirrored_live_result = tool.evaluate_role( + generated_plans["mp-client"], base, 0, False + ) + assert mirrored_live_result["status"] == "pass" + assert tool.verify_recorded_files(report, base, asset_root, runtime_dir) == [] + mp_client_stdout.write_text("", encoding="utf-8") + mp_client_stdout_artifact.update(tool.file_record(mp_client_stdout, base)) + save_preview_artifact = next( + artifact + for result in results + for artifact in result["artifacts"] # type: ignore[index] + if artifact["kind"] == "savePreview" + ) + save_preview_path = base / str(save_preview_artifact["path"]) + coherent_preview_bytes = save_preview_path.read_bytes() + sp_capture_result = next(result for result in results if result["role"] == "sp-capture") + coherent_comparison = sp_capture_result["savePreviewComparison"] + sp_reference_artifact = next( + artifact + for artifact in sp_capture_result["artifacts"] # type: ignore[union-attr] + if artifact["kind"] == "saveReferenceScreenshot" + ) + sp_reference_path = base / str(sp_reference_artifact["path"]) + preview_width, preview_height = tool.SP_SAVE_PREVIEW_DIMENSIONS + write_rgb_tga( + save_preview_path, + preview_width, + preview_height, + recursive_strip_rgb(preview_width, preview_height), + ) + save_preview_artifact.update(tool.file_record(save_preview_path, base)) + assert any( + "save preview: TGA contains recursive scaled-strip repetition" in failure + for failure in tool.verify_recorded_files(report, base, asset_root, runtime_dir) + ) + save_preview_path.write_bytes(coherent_preview_bytes) + save_preview_artifact.update(tool.file_record(save_preview_path, base)) + assert tool.verify_recorded_files(report, base, asset_root, runtime_dir) == [] + coherent_decoded = tool.decode_tga_rgb(coherent_preview_bytes) + assert not isinstance(coherent_decoded, str) + _, _, coherent_rgb = coherent_decoded + write_rgb_tga( + save_preview_path, + preview_width, + preview_height, + severe_color_defect(coherent_rgb), + ) + assert tool.validate_tga(save_preview_path, tool.SP_SAVE_PREVIEW_DIMENSIONS) is None + save_preview_artifact.update(tool.file_record(save_preview_path, base)) + corrupted_comparison, comparison_failure = tool.save_preview_comparison( + sp_reference_path, save_preview_path + ) + assert corrupted_comparison is not None and comparison_failure is not None + assert "exposure-compensated" in comparison_failure + assert corrupted_comparison["lumaCorrelation"] >= tool.SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION + live_corrupted_result = tool.evaluate_role( + generated_plans["sp-capture"], base, 0, False + ) + assert live_corrupted_result["status"] == "fail" + assert any( + "save preview differs from same-state save reference" in failure + for failure in live_corrupted_result["failures"] + ) + sp_capture_result["savePreviewComparison"] = corrupted_comparison + assert any( + "save preview differs from same-state save reference" in failure + for failure in tool.verify_recorded_files(report, base, asset_root, runtime_dir) + ) + save_preview_path.write_bytes(coherent_preview_bytes) + save_preview_artifact.update(tool.file_record(save_preview_path, base)) + sp_capture_result["savePreviewComparison"] = coherent_comparison + assert tool.verify_recorded_files(report, base, asset_root, runtime_dir) == [] + missing_comparison = json.loads(json.dumps(report)) + del next( + result for result in missing_comparison["results"] if result["role"] == "sp-capture" + )["savePreviewComparison"] + assert any( + "recorded save-preview comparison metrics differ" in failure + for failure in tool.verify_recorded_files( + missing_comparison, base, asset_root, runtime_dir + ) + ) + original_link_check = tool.is_link_or_junction + for linked_path in (runtime_dir, runtime_dir.parent): + tool.is_link_or_junction = lambda path, linked=linked_path: path == linked + try: + assert any( + "runtime directory ancestry must not contain a link or junction" in failure + for failure in tool.verify_recorded_files(report, base, asset_root) + ) + finally: + tool.is_link_or_junction = original_link_check + wrong_runtime_root = json.loads(json.dumps(report)) + wrong_runtime_root["runtimeRoot"] = str(base / "wrong-runtime") + assert any( + "runtime root differs" in failure + for failure in tool.verify_recorded_files( + wrong_runtime_root, base, asset_root, runtime_dir + ) + ) + missing_runtime_root = json.loads(json.dumps(report)) + del missing_runtime_root["runtimeRoot"] + assert any( + "does not record its runtime root" in failure + for failure in tool.verify_recorded_files( + missing_runtime_root, base, asset_root, runtime_dir + ) + ) + runtime_executable.write_bytes(b"mutated client") + assert any( + "runtime file" in failure and "differs" in failure + for failure in tool.verify_recorded_files(report, base, asset_root, runtime_dir) + ) + runtime_executable.write_bytes(b"client") + disabled_autojoin = json.loads(json.dumps(report)) + disabled_plan = next( + plan for plan in disabled_autojoin["plan"] if plan["role"] == "mp-client" + ) + disabled_index = disabled_plan["arguments"].index("ui_autoJoin") + disabled_plan["arguments"][disabled_index + 1] = "0" + assert any( + "mp-client: launch CVar ui_autoJoin differs" in failure + for failure in tool.verify_recorded_files( + disabled_autojoin, base, asset_root, runtime_dir + ) + ) + missing_autojoin = json.loads(json.dumps(report)) + missing_plan = next( + plan for plan in missing_autojoin["plan"] if plan["role"] == "mp-server" + ) + missing_index = missing_plan["arguments"].index("ui_autoJoin") + del missing_plan["arguments"][missing_index - 1 : missing_index + 2] + assert any( + "mp-server: launch CVar ui_autoJoin differs" in failure + for failure in tool.verify_recorded_files( + missing_autojoin, base, asset_root, runtime_dir + ) + ) + disabled_pure = json.loads(json.dumps(report)) + pure_plan = next( + plan for plan in disabled_pure["plan"] if plan["role"] == "mp-server" + ) + pure_index = pure_plan["arguments"].index("si_pure") + pure_plan["arguments"][pure_index + 1] = "0" + assert any( + "mp-server: launch CVar si_pure differs" in failure + for failure in tool.verify_recorded_files( + disabled_pure, base, asset_root, runtime_dir + ) + ) + empty_pass = dict(report) + empty_pass["results"] = [] + assert any( + "required role" in failure + for failure in tool.verify_recorded_files(empty_pass, base, asset_root, runtime_dir) + ) + missing_artifact = json.loads(json.dumps(report)) + missing_artifact["results"][0]["artifacts"] = [ + item + for item in missing_artifact["results"][0]["artifacts"] + if item["kind"] != "processStderr" + ] + assert any( + "required artifact kinds differ" in failure + for failure in tool.verify_recorded_files( + missing_artifact, base, asset_root, runtime_dir + ) + ) + stdout_artifact = next( + item for item in results[0]["artifacts"] if item["kind"] == "processStdout" # type: ignore[union-attr] + ) + artifact = base / str(stdout_artifact["path"]) + artifact.write_text("tampered stream", encoding="utf-8") + assert any( + "differs" in failure + for failure in tool.verify_recorded_files(report, base, asset_root, runtime_dir) + ) + + +def main() -> None: + tool = load_tool() + temp_parent = ROOT / ".tmp" / "stock-runtime" + temp_parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="stock-baseline-test-", dir=temp_parent) as temp: + base = Path(temp) + test_asset_inventory_and_comparison(tool, base) + test_plan_is_windowed_and_engine_only(tool, base) + test_diagnostic_authority_and_shared_mp_deadline(tool, base) + test_save_preview_coherence(tool, base) + test_tga_and_report_verification(tool, base) + print("stock_asset_baseline: ok") + + +if __name__ == "__main__": + main() diff --git a/tools/tests/vscode_fast_build.py b/tools/tests/vscode_fast_build.py index dd14e4a6..bdaa3713 100644 --- a/tools/tests/vscode_fast_build.py +++ b/tools/tests/vscode_fast_build.py @@ -71,16 +71,95 @@ def validate_wrapper() -> None: stager = read("tools/build/stage_fast_install.py") require(stager, "copy_file_if_changed", "fast install copy-if-changed behavior") + require( + stager, + "from windows_runtime import cleanup_windows_stage_target, is_windows_host", + "shared Windows stage hygiene helper", + ) + require( + stager, + "cleanup_windows_stage_target(install_dir)", + "fast-build Windows stale-runtime cleanup", + ) + require(stager, '"renderer-gl_*.dll"', "fast install renderer staging") + require(stager, '"--temporary-runtime"', "isolated compatibility runtime staging") + require(stager, 'source_root / ".tmp" / "stock-runtime"', "temporary runtime containment") require(stager, '"pak0.pk4"', "fast install stages pak0") require(stager, '"pak1.pk4"', "fast install stages pak1") reject(stager, '"*.lib",\n "pak0.pk4"', "fast install must not copy linker artifacts as runtime content") + windows_runtime = read("tools/build/windows_runtime.py") + require( + windows_runtime, + "WINDOWS_STALE_STAGE_FILE_MANIFEST", + "narrow Windows stale-runtime cleanup manifest", + ) + require( + windows_runtime, + '"baseoq4/skins"', + "known empty Windows stage directory manifest", + ) + require( + windows_runtime, + "cleanup_results[str(target)] = cleanup_windows_stage_target(target)", + "full-install Windows stale-runtime cleanup", + ) + def validate_launch_configs() -> None: launch = json.loads(read(".vscode/launch.json")) + mp_configs = [] for config in launch.get("configurations", []): if "preLaunchTask" in config: raise AssertionError(f"Launch config {config.get('name')!r} must not define preLaunchTask") + if "(MP)" in str(config.get("name", "")): + mp_configs.append(config) + args = config.get("args", []) + values = [ + str(args[index + 2]) + for index, token in enumerate(args[:-2]) + if token in ("+set", "+seta") and args[index + 1] == "ui_autoJoin" + ] + if values != ["1"]: + raise AssertionError( + f"MP launch config {config.get('name')!r} must set ui_autoJoin exactly once to 1" + ) + if not mp_configs: + raise AssertionError("Expected at least one VS Code MP launch configuration") + + +def validate_mp_autojoin_policy() -> None: + listen_script = read("tools/debug/start_listen_server_client.ps1") + if listen_script.count('"+set", "ui_autoJoin", "1"') != 2: + raise AssertionError("MP listen-server helper must enable auto-join for host and client") + + renderdoc_script = read("tools/debug/renderdoc_capture.ps1") + if renderdoc_script.count('"+set", "ui_autoJoin", "1"') != 1: + raise AssertionError("MP RenderDoc helper must enable auto-join for its listen host") + + benchmark = read("tools/tests/renderer_gameplay_benchmark.py") + for target in ("server_args", "client_args"): + require( + benchmark, + f'append_set({target}, "ui_autoJoin", "1")', + f"MP renderer benchmark {target}", + ) + + baseline = read("tools/validation/stock_asset_baseline.py") + require( + baseline, + 'args[restart_index:restart_index] = ("+set", "ui_autoJoin", "1")', + "stock baseline MP roles", + ) + require( + baseline, + 'launch_contract["ui_autoJoin"] = "1"', + "recorded stock baseline MP contract", + ) + + guide = read("AGENTS.md") + require(guide, "Keep `ui_autoJoin 1` enabled for multiplayer testing", "agent MP test policy") + require(guide, "explicit `+set ui_autoJoin 0`", "join-menu test exception") def validate_validation_coverage() -> None: @@ -99,6 +178,7 @@ def main() -> None: validate_tasks() validate_wrapper() validate_launch_configs() + validate_mp_autojoin_policy() validate_validation_coverage() print("vscode_fast_build: ok") diff --git a/tools/validation/audit_source_provenance.py b/tools/validation/audit_source_provenance.py new file mode 100644 index 00000000..d0b8b4d0 --- /dev/null +++ b/tools/validation/audit_source_provenance.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Inventory retained Doom 3 license-header families and verify their notices. + +The header is an attribution/license-family signal, not proof that the local +file is byte-identical to a file at the audited upstream commit. This tool is +offline and deliberately makes no legal-compatibility determination. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[2] +MANIFEST_PATH = ROOT / "docs" / "dev" / "source-provenance-manifest.json" +ADDITIONAL_TERMS_SIGNAL = "subject to certain additional terms" + + +def canonical_text_bytes(payload: bytes) -> bytes: + text = payload.decode("utf-8") + return text.replace("\r\n", "\n").replace("\r", "\n").encode("utf-8") + + +def load_manifest(path: Path = MANIFEST_PATH) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schemaVersion") != 1: + raise ValueError(f"unsupported provenance manifest schema: {payload.get('schemaVersion')!r}") + families = payload.get("families") + if not isinstance(families, dict) or not families: + raise ValueError("provenance manifest must define at least one header family") + return payload + + +def tracked_files(root: Path = ROOT) -> list[Path]: + try: + completed = subprocess.run( + ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], + cwd=root, + check=True, + capture_output=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise RuntimeError("source provenance audit requires a Git worktree") from exc + + paths: list[Path] = [] + for raw in completed.stdout.split(b"\0"): + if not raw: + continue + relative = Path(raw.decode("utf-8", errors="surrogateescape")) + path = root / relative + if path.is_file(): + paths.append(path) + return sorted(paths) + + +def read_header(path: Path, limit: int = 16 * 1024) -> str: + with path.open("rb") as stream: + return stream.read(limit).decode("utf-8", errors="replace") + + +def bfg_official_path(local_path: str) -> str: + if not local_path.startswith("src/"): + raise ValueError(f"BFG-marked path is outside src/: {local_path}") + relative = local_path.removeprefix("src/") + if relative.startswith("imagetools/"): + relative = "renderer/" + relative.removeprefix("imagetools/") + return "neo/" + relative + + +def bfg_origin(local_path: str, family_spec: dict[str, Any]) -> dict[str, str]: + intermediate_sources = family_spec.get("intermediateSources", {}) + for source_name, source in intermediate_sources.items(): + path = source.get("pathOverrides", {}).get(local_path) + if path: + return { + "classification": "intermediate-fork-lineage", + "source": source_name, + "repository": source["repository"], + "auditedCommit": source["auditedCommit"], + "auditedPath": path, + } + return { + "classification": "official-snapshot-path", + "source": "doom3_bfg_official", + "repository": family_spec["officialRepository"], + "auditedCommit": family_spec["auditedCommit"], + "auditedPath": bfg_official_path(local_path), + } + + +def inventory(root: Path, manifest: dict[str, Any]) -> dict[str, Any]: + family_specs = manifest["families"] + family_files: dict[str, list[dict[str, str]]] = {name: [] for name in family_specs} + unclassified: list[str] = [] + + # Match the more specific BFG marker before the Doom 3 marker. + ordered_families = sorted( + family_specs.items(), key=lambda item: len(item[1]["headerMarker"]), reverse=True + ) + for path in tracked_files(root): + relative = path.relative_to(root).as_posix() + if not relative.startswith("src/"): + continue + try: + header = read_header(path) + except OSError: + continue + matched = None + for family_name, spec in ordered_families: + if spec["headerMarker"].casefold() in header.casefold(): + matched = family_name + break + if matched is None: + if ADDITIONAL_TERMS_SIGNAL.casefold() in header.casefold(): + unclassified.append(relative) + continue + + entry = {"localPath": relative} + if matched == "doom3_bfg": + entry.update(bfg_origin(relative, family_specs[matched])) + family_files[matched].append(entry) + + return { + "schemaVersion": 1, + "manifest": MANIFEST_PATH.relative_to(root).as_posix(), + "families": { + name: { + "displayName": family_specs[name]["displayName"], + "auditedCommit": family_specs[name]["auditedCommit"], + "count": len(entries), + "files": entries, + } + for name, entries in family_files.items() + }, + "unclassifiedAdditionalTermsHeaders": unclassified, + } + + +def validate(root: Path, manifest: dict[str, Any], report: dict[str, Any]) -> list[str]: + failures: list[str] = [] + if manifest.get("textHashNormalization") != "UTF-8; CRLF and CR normalized to LF": + failures.append("manifest textHashNormalization is missing or unsupported") + for family_name, spec in manifest["families"].items(): + actual = report["families"].get(family_name, {}) + if actual.get("count") != spec.get("expectedFileCount"): + failures.append( + f"{family_name}: expected {spec.get('expectedFileCount')} marked files, " + f"found {actual.get('count')}" + ) + + commit = spec.get("auditedCommit", "") + if len(commit) != 40 or any(ch not in "0123456789abcdef" for ch in commit): + failures.append(f"{family_name}: auditedCommit is not a lowercase 40-digit Git object id") + copying_hash = spec.get("officialCopyingSha256", "") + if len(copying_hash) != 64 or any(ch not in "0123456789abcdef" for ch in copying_hash): + failures.append(f"{family_name}: officialCopyingSha256 is not a lowercase SHA-256 digest") + + local_terms_hash = spec.get("localAdditionalTermsSha256", "") + if len(local_terms_hash) != 64 or any(ch not in "0123456789abcdef" for ch in local_terms_hash): + failures.append( + f"{family_name}: localAdditionalTermsSha256 is not a lowercase SHA-256 digest" + ) + + terms_path = root / spec["localAdditionalTerms"] + if not terms_path.is_file(): + failures.append(f"{family_name}: missing {spec['localAdditionalTerms']}") + else: + terms_bytes = terms_path.read_bytes() + canonical_terms = canonical_text_bytes(terms_bytes) + actual_terms_hash = hashlib.sha256(canonical_terms).hexdigest() + if actual_terms_hash != local_terms_hash: + failures.append( + f"{family_name}: {spec['localAdditionalTerms']} SHA-256 differs: " + f"expected {local_terms_hash}, got {actual_terms_hash}" + ) + terms = canonical_terms.decode("utf-8") + if "ADDITIONAL TERMS APPLICABLE" not in terms: + failures.append(f"{family_name}: local Additional Terms heading is missing") + for section in ("Replacement of Section 15", "Replacement of Section 16", "LEGAL NOTICES", "INDEMNIFICATION"): + if section not in terms: + failures.append(f"{family_name}: local Additional Terms omit {section!r}") + + for source_name, source in spec.get("intermediateSources", {}).items(): + source_commit = source.get("auditedCommit", "") + if len(source_commit) != 40 or any(ch not in "0123456789abcdef" for ch in source_commit): + failures.append(f"{family_name}/{source_name}: auditedCommit is not a lowercase 40-digit Git object id") + overrides = source.get("pathOverrides", {}) + inventoried = {item["localPath"] for item in actual.get("files", [])} + unknown = sorted(set(overrides) - inventoried) + if unknown: + failures.append(f"{family_name}/{source_name}: overrides do not identify inventoried files: {', '.join(unknown)}") + + unclassified = report["unclassifiedAdditionalTermsHeaders"] + if unclassified: + failures.append( + "unclassified source headers refer to Additional Terms: " + ", ".join(unclassified) + ) + return failures + + +def validate_reference_tree( + report: dict[str, Any], + source_name: str, + source_root: Path | None, +) -> list[str]: + if source_root is None: + return [] + failures: list[str] = [] + if not source_root.is_dir(): + return [f"{source_name}: reference root does not exist: {source_root}"] + for entry in report["families"]["doom3_bfg"]["files"]: + if entry.get("source") != source_name: + continue + path = source_root / Path(entry["auditedPath"]) + if not path.is_file(): + failures.append( + f"{source_name}: configured audited path is absent: {entry['auditedPath']} " + f"(for {entry['localPath']})" + ) + return failures + + +def validate_official_copying( + family_name: str, + family_spec: dict[str, Any], + source_root: Path | None, +) -> list[str]: + if source_root is None: + return [] + if not source_root.is_dir(): + return [f"{family_name}: reference root does not exist: {source_root}"] + copying = source_root / "COPYING.txt" + if not copying.is_file(): + return [f"{family_name}: reference root is missing COPYING.txt: {source_root}"] + actual = hashlib.sha256(copying.read_bytes()).hexdigest() + expected = family_spec["officialCopyingSha256"] + if actual != expected: + return [ + f"{family_name}: official COPYING.txt SHA-256 differs: expected {expected}, got {actual}" + ] + return [] + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="Fail if counts, families, or accompanying terms differ from the audited manifest.") + parser.add_argument("--family", choices=("all", "doom3", "doom3_bfg"), default="all", help="Limit the displayed inventory; checks always cover every family.") + parser.add_argument("--format", choices=("text", "json"), default="text", help="Inventory output format.") + parser.add_argument("--output", default="", help="Optional output file. The default is stdout.") + parser.add_argument("--doom3-source", default="", help="Optional official DOOM-3 checkout used to verify COPYING.txt.") + parser.add_argument("--doom3-bfg-source", default="", help="Optional official DOOM-3-BFG checkout used to verify configured audited paths.") + parser.add_argument("--rbdoom3-bfg-source", default="", help="Optional RBDOOM-3-BFG checkout used to verify intermediate-fork audited paths.") + return parser.parse_args(argv) + + +def render_text(report: dict[str, Any], family_filter: str) -> str: + lines = ["openQ4 source provenance inventory"] + for family_name, family in report["families"].items(): + if family_filter != "all" and family_filter != family_name: + continue + lines.append(f"\n{family['displayName']}: {family['count']} files") + for entry in family["files"]: + upstream = entry.get("auditedPath") + source = entry.get("source") + suffix = f" -> {source}:{upstream}" if upstream else "" + lines.append(f" {entry['localPath']}{suffix}") + if report["unclassifiedAdditionalTermsHeaders"]: + lines.append("\nUnclassified Additional-Terms headers:") + lines.extend(f" {path}" for path in report["unclassifiedAdditionalTermsHeaders"]) + return "\n".join(lines) + "\n" + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + manifest = load_manifest() + report = inventory(ROOT, manifest) + failures = validate(ROOT, manifest, report) + doom3_source = Path(args.doom3_source).resolve() if args.doom3_source else None + bfg_source = Path(args.doom3_bfg_source).resolve() if args.doom3_bfg_source else None + failures += validate_official_copying( + "doom3", manifest["families"]["doom3"], doom3_source + ) + failures += validate_official_copying( + "doom3_bfg", manifest["families"]["doom3_bfg"], bfg_source + ) + failures += validate_reference_tree( + report, + "doom3_bfg_official", + bfg_source, + ) + failures += validate_reference_tree( + report, + "rbdoom3_bfg", + Path(args.rbdoom3_bfg_source).resolve() if args.rbdoom3_bfg_source else None, + ) + + if args.format == "json": + displayed = report + if args.family != "all": + displayed = dict(report) + displayed["families"] = {args.family: report["families"][args.family]} + output = json.dumps(displayed, indent=2) + "\n" + else: + output = render_text(report, args.family) + + if args.output: + output_path = Path(args.output).resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(output, encoding="utf-8") + else: + sys.stdout.write(output) + + if args.check and failures: + for failure in failures: + print(f"error: {failure}", file=sys.stderr) + return 1 + if args.check: + print("source_provenance: ok", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tools/validation/openq4_validate.py b/tools/validation/openq4_validate.py index f209e8ae..3604360c 100644 --- a/tools/validation/openq4_validate.py +++ b/tools/validation/openq4_validate.py @@ -481,7 +481,10 @@ def run_python_tests(args: argparse.Namespace, root: Path, env: dict[str, str]) root / "tools" / "tests" / "native_glx_shutdown.py", root / "tools" / "tests" / "network_ipv4_support.py", root / "tools" / "tests" / "network_ipv6_support.py", + root / "tools" / "tests" / "network_security.py", + root / "tools" / "tests" / "openurl_security.py", root / "tools" / "tests" / "openq4_pure_pack.py", + root / "tools" / "tests" / "p0_governance_evidence.py", root / "tools" / "tests" / "packaging_safety.py", root / "tools" / "tests" / "preprocessor_macro_safety.py", root / "tools" / "tests" / "posix_memory_management.py", @@ -491,6 +494,7 @@ def run_python_tests(args: argparse.Namespace, root: Path, env: dict[str, str]) root / "tools" / "tests" / "release_tooling_safety.py", root / "tools" / "tests" / "renderer_cel_shading.py", root / "tools" / "tests" / "renderer_mp_flat_items.py", + root / "tools" / "tests" / "renderer_pbr_materials.py", root / "tools" / "tests" / "renderer_msaa_cvar_safety.py", root / "tools" / "tests" / "renderer_picmip_policy.py", root / "tools" / "tests" / "renderer_player_visibility.py", @@ -512,6 +516,7 @@ def run_python_tests(args: argparse.Namespace, root: Path, env: dict[str, str]) root / "tools" / "tests" / "system_console_presentation.py", root / "tools" / "tests" / "lang_table_encoding.py", root / "tools" / "tests" / "startup_language_override.py", + root / "tools" / "tests" / "stock_asset_baseline.py", root / "tools" / "tests" / "ui_embedded_icons.py", root / "tools" / "tests" / "validation_hardening.py", root / "tools" / "tests" / "vk_shader_header_pin.py", diff --git a/tools/validation/stock_asset_baseline.py b/tools/validation/stock_asset_baseline.py new file mode 100644 index 00000000..bb01f320 --- /dev/null +++ b/tools/validation/stock_asset_baseline.py @@ -0,0 +1,2493 @@ +#!/usr/bin/env python3 +"""Capture and verify the retail-PK4 openQ4 SP/MP compatibility baseline. + +The harness is intentionally non-interactive. It launches only windowed +clients, drives registered console commands through generated cfg files, and +uses the engine's ``screenshot`` command. It never controls or captures host +mouse/keyboard input and never uses an operating-system screen-capture API. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import re +import stat +import struct +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from zipfile import BadZipFile, ZipFile + + +SCHEMA_VERSION = 8 +RETAIL_MANIFEST_SCHEMA_VERSION = 1 +RUNTIME_MANIFEST_SCHEMA_VERSION = 2 +ASSET_COMPATIBILITY_MODEL = "retail-pk4-fallback-with-packaged-openq4-overlays-v1" +GIT_PROVENANCE_POLICY = "current-openq4-head-and-dirty-state-v1" +TOP_LEVEL_FAILURE_ARRAYS = ( + "assetComparisonFailures", + "preflightFailures", + "postCaptureVerificationFailures", +) +SP_MAP = "game/storage1" +MP_MAP = "mp/q4dm1" +SP_SAVE_NAME = "StockBaselineSP" +SP_DEMO_NAME = "stock_baseline_sp" +BASELINE_DIR = "stock-baseline" +DEFAULT_WIDTH = 1280 +DEFAULT_HEIGHT = 720 +SP_SAVE_PREVIEW_DIMENSIONS = (320, 240) +SP_SAVE_PREVIEW_ANALYSIS_DIMENSIONS = (80, 60) +SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION = 0.75 +SP_SAVE_PREVIEW_MIN_EXPOSURE_GAIN = 0.25 +SP_SAVE_PREVIEW_MAX_EXPOSURE_GAIN = 4.0 +SP_SAVE_PREVIEW_MIN_EXPOSURE_BIAS = -192.0 +SP_SAVE_PREVIEW_MAX_EXPOSURE_BIAS = 192.0 +SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR = 48.0 +SP_SAVE_PREVIEW_COMPARISON_ALGORITHM = ( + "same-state-center-aspect-bilinear-luma-affine-rgb-v3" +) +MP_CLIENT_ACTIVE_MARKER = "OPENQ4_STOCK_BASELINE_MP_CLIENT_ACTIVE" +MP_CLIENT_ACTIVE_PATTERN = re.compile( + rf"^{MP_CLIENT_ACTIVE_MARKER} client=\d+ spectating=0 wantSpectate=0 ingame=1 menu=0 disableHud=0$", + re.MULTILINE, +) +MP_CLIENT_VIEW_MARKER = "OPENQ4_STOCK_BASELINE_MP_CLIENT_VIEW" +MP_CLIENT_VIEW_PATTERN = re.compile( + rf"^{MP_CLIENT_VIEW_MARKER} gui=0$", + re.MULTILINE, +) +MP_CLIENT_SCREENSHOT_WRITE_PATTERN = re.compile( + rf"^Wrote screenshots/{BASELINE_DIR}/mp_client\.tga$", + re.MULTILINE, +) +FATAL_PATTERNS = { + "fatal": re.compile(r"\bFatal Error\b|^[ \t]*(?:\*+[ \t]*)?FATAL[ \t]*:", re.IGNORECASE | re.MULTILINE), + "engineError": re.compile(r"^[ \t]*(?:\*+[ \t]*)?ERROR(?:[ \t]*:|[ \t]*$)", re.MULTILINE), + "shaderFailure": re.compile(r"(shader compile|program link).*(failed|error)|failed to compile", re.IGNORECASE), + "vulkanValidation": re.compile(r"\bVulkan validation:|\bVUID-[A-Za-z0-9_.-]+\b", re.IGNORECASE), + "graphicsError": re.compile(r"\bGL_(?:INVALID_[A-Z_]+|OUT_OF_MEMORY|CONTEXT_LOST)\b|OpenGL\s+error", re.IGNORECASE), +} +RUNTIME_WINDOW_MODE_PATTERN = re.compile( + r"^MODE:\s*[^,\r\n]+,\s*(\d+)\s+x\s+(\d+)\s+(windowed|borderless|fullscreen)\b", + re.IGNORECASE | re.MULTILINE, +) +ACTIVE_DISPLAY_SCALE_PATTERN = re.compile( + r"^\s*\[\d+\]\s+\*[^\r\n]*\bcontentScale\s+([0-9]+(?:\.[0-9]+)?)\b", + re.IGNORECASE | re.MULTILINE, +) + +ROLE_EVIDENCE_CONTRACT: dict[str, dict[str, Any]] = { + "sp-capture": { + "marker": "OPENQ4_STOCK_BASELINE_SP_SAVE_LOAD_COMPLETE", + "saveDir": "sp", + "logName": "stock_baseline_sp.log", + "expected": { + "screenshot": f"baseoq4/screenshots/{BASELINE_DIR}/sp_after_load.tga", + "saveReferenceScreenshot": ( + f"baseoq4/screenshots/{BASELINE_DIR}/sp_before_save.tga" + ), + "renderDemo": f"baseoq4/demos/{SP_DEMO_NAME}.demo", + "savePayload": f"baseoq4/savegames/{SP_SAVE_NAME}.save", + "savePreview": f"baseoq4/savegames/{SP_SAVE_NAME}.tga", + "saveDescription": f"baseoq4/savegames/{SP_SAVE_NAME}.txt", + }, + }, + "sp-demo-playback": { + "marker": "frames rendered in", + "saveDir": "sp", + "logName": "stock_baseline_demo_playback.log", + "expected": {}, + }, + "mp-server": { + "marker": "OPENQ4_STOCK_BASELINE_MP_SERVER_COMPLETE", + "saveDir": "mp-server", + "logName": "stock_baseline_server.log", + "expected": { + "screenshot": f"baseoq4/screenshots/{BASELINE_DIR}/mp_server.tga", + "renderDemo": f"baseoq4/demos/stock_baseline_mp_server.demo", + }, + }, + "mp-client": { + "marker": "OPENQ4_STOCK_BASELINE_MP_CLIENT_COMPLETE", + "requiredMarkers": [MP_CLIENT_ACTIVE_MARKER, MP_CLIENT_VIEW_MARKER], + "saveDir": "mp-client", + "logName": "stock_baseline_client.log", + "expected": { + "screenshot": f"baseoq4/screenshots/{BASELINE_DIR}/mp_client.tga", + "renderDemo": f"baseoq4/demos/stock_baseline_mp_client.demo", + }, + }, +} + + +@dataclass +class RolePlan: + role_id: str + mode: str + savepath: Path + log_name: str + marker: str + args: list[str] + expected: list[tuple[str, str]] = field(default_factory=list) + stdout_path: Path | None = None + stderr_path: Path | None = None + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def prepare_output_directory(output_dir: Path) -> None: + if output_dir.exists(): + if not output_dir.is_dir(): + raise ValueError(f"evidence output path is not a directory: {output_dir}") + if is_link_or_junction(output_dir): + raise ValueError(f"evidence output directory must not be a link or junction: {output_dir}") + if any(output_dir.iterdir()): + raise ValueError(f"evidence output directory must be new or empty: {output_dir}") + else: + output_dir.mkdir(parents=True) + + +def host_arch() -> str: + machine = platform.machine().lower() + if machine in {"amd64", "x86_64"}: + return "x64" + if machine in {"arm64", "aarch64"}: + return "arm64" + return machine + + +def default_asset_root() -> str: + if os.name == "nt": + return r"C:\Program Files (x86)\Steam\steamapps\common\Quake 4" + return "" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def file_record(path: Path, root: Path) -> dict[str, Any]: + return { + "path": path.relative_to(root).as_posix(), + "size": path.stat().st_size, + "sha256": sha256_file(path), + } + + +def is_link_or_junction(path: Path) -> bool: + is_junction = getattr(path, "is_junction", None) + if path.is_symlink() or bool(is_junction and is_junction()): + return True + try: + # Python before 3.12 has no Path.is_junction(). Inspect the path's own + # Windows reparse-point bit instead of comparing resolved paths: every + # ordinary child below an explicitly allowed junction resolves through + # that parent and must not be misclassified as a nested link. + attributes = getattr(path.lstat(), "st_file_attributes", 0) + return bool(attributes & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + except OSError: + return False + + +def validate_asset_root(asset_root: Path, allow_asset_dir_links: bool = False) -> None: + if not asset_root.is_dir(): + raise FileNotFoundError(f"Quake 4 asset root does not exist: {asset_root}") + direct_game_dir = asset_root / "baseoq4" + if direct_game_dir.is_dir() and any(direct_game_dir.iterdir()): + raise ValueError( + f"asset root contains non-empty {direct_game_dir}; fs_game=baseoq4 would load " + "that unapproved content ahead of the stock q4base fallback. Use a clean asset " + "view containing q4base/q4mp only." + ) + for game_dir_name in ("q4base", "q4mp"): + game_dir = asset_root / game_dir_name + if game_dir.exists() and is_link_or_junction(game_dir) and not allow_asset_dir_links: + raise ValueError( + f"asset directory is a symlink or junction: {game_dir}; pass " + "--allow-asset-dir-links only for an intentionally constructed clean view" + ) + + +def asset_directory_views(asset_root: Path) -> list[dict[str, Any]]: + views: list[dict[str, Any]] = [] + for game_dir_name in ("q4base", "q4mp"): + game_dir = asset_root / game_dir_name + if game_dir.exists(): + views.append( + { + "path": game_dir_name, + "linked": is_link_or_junction(game_dir), + "resolvedTarget": str(game_dir.resolve()), + } + ) + return views + + +def collect_pk4s( + asset_root: Path, allow_asset_dir_links: bool = False +) -> list[dict[str, Any]]: + validate_asset_root(asset_root, allow_asset_dir_links) + files: list[Path] = [] + for game_dir_name in ("q4base", "q4mp"): + game_dir = asset_root / game_dir_name + if game_dir.is_dir(): + files.extend(path for path in game_dir.iterdir() if path.is_file() and path.suffix.casefold() == ".pk4") + files.sort(key=lambda path: path.relative_to(asset_root).as_posix().casefold()) + if not files: + raise FileNotFoundError(f"no retail PK4 files found under {asset_root}/q4base or q4mp") + if not any(item.relative_to(asset_root).as_posix().casefold() == "q4base/pak001.pk4" for item in files): + raise FileNotFoundError("retail asset set is missing q4base/pak001.pk4") + return [file_record(path, asset_root) for path in files] + + +def collect_loose_asset_files( + asset_root: Path, allow_asset_dir_links: bool = False +) -> list[dict[str, Any]]: + """Hash every non-PK4 file below q4base/q4mp that the fallback can see.""" + validate_asset_root(asset_root, allow_asset_dir_links) + files: list[Path] = [] + for game_dir_name in ("q4base", "q4mp"): + game_dir = asset_root / game_dir_name + if not game_dir.is_dir(): + continue + for parent_name, dir_names, file_names in os.walk(game_dir, followlinks=False): + parent = Path(parent_name) + for dir_name in list(dir_names): + directory = parent / dir_name + if is_link_or_junction(directory): + raise ValueError(f"loose asset tree contains a symlink or junction: {directory}") + for file_name in file_names: + path = parent / file_name + if is_link_or_junction(path): + raise ValueError(f"loose asset tree contains a symlink or junction: {path}") + if path.suffix.casefold() != ".pk4": + files.append(path) + files.sort(key=lambda path: path.relative_to(asset_root).as_posix().casefold()) + return [file_record(path, asset_root) for path in files] + + +def collect_overlay_pk4s(runtime_dir: Path) -> list[dict[str, Any]]: + game_dir = runtime_dir / "baseoq4" + if not game_dir.is_dir(): + return [] + files = sorted( + (path for path in game_dir.iterdir() if path.is_file() and path.suffix.casefold() == ".pk4"), + key=lambda path: path.name.casefold(), + ) + return [file_record(path, runtime_dir) for path in files] + + +def normalized_pk4_member_path(name: str, archive_path: Path) -> str: + """Return the case-preserving VFS path for one ordinary PK4 member.""" + + normalized = name.replace("\\", "/") + parts = normalized.split("/") + if ( + not normalized + or normalized.startswith("/") + or any(part in {"", ".", ".."} for part in parts) + or ":" in parts[0] + ): + raise ValueError(f"PK4 contains an unsafe or ambiguous member path: {archive_path}: {name!r}") + return "/".join(parts) + + +def pk4_member_index( + root: Path, records: list[dict[str, Any]], label: str +) -> dict[str, dict[str, Any]]: + """Index archive members by the case-insensitive idTech virtual path.""" + + index: dict[str, dict[str, Any]] = {} + for record in records: + relative = str(record.get("path", "")) + relative_path = Path(relative) + if not relative or relative_path.is_absolute() or ".." in relative_path.parts: + raise ValueError(f"{label} manifest contains an unsafe PK4 path: {relative!r}") + archive_path = root / relative_path + if not archive_path.is_file(): + raise FileNotFoundError(f"{label} PK4 is missing: {archive_path}") + try: + with ZipFile(archive_path) as archive: + for member in archive.infolist(): + if member.is_dir(): + continue + member_path = normalized_pk4_member_path(member.filename, archive_path) + key = member_path.casefold() + entry = index.setdefault( + key, + {"path": member_path, "archives": set()}, + ) + if member_path.casefold() == str(entry["path"]).casefold(): + entry["path"] = min(str(entry["path"]), member_path) + entry["archives"].add(relative.replace("\\", "/")) + except BadZipFile as exc: + raise ValueError(f"{label} PK4 is not a readable ZIP archive: {archive_path}") from exc + return index + + +def collect_retail_path_collisions( + asset_root: Path, + stock_pk4s: list[dict[str, Any]], + runtime_dir: Path, + overlay_pk4s: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Inventory packaged openQ4 members that supersede retail virtual paths.""" + + if not overlay_pk4s: + return [] + retail_index = pk4_member_index(asset_root, stock_pk4s, "retail") + overlay_index = pk4_member_index(runtime_dir, overlay_pk4s, "openQ4 overlay") + collisions: list[dict[str, Any]] = [] + for key in sorted(set(retail_index) & set(overlay_index)): + retail = retail_index[key] + overlay = overlay_index[key] + collisions.append( + { + "path": str(retail["path"]), + "retailPk4s": sorted(retail["archives"], key=str.casefold), + "openQ4OverlayPk4s": sorted(overlay["archives"], key=str.casefold), + } + ) + collisions.sort(key=lambda item: str(item["path"]).casefold()) + return collisions + + +def is_shared_library(path: Path) -> bool: + name = path.name.casefold() + return name.endswith((".dll", ".dylib")) or name.endswith(".so") or ".so." in name + + +def runtime_kind(path: Path, executable: Path) -> str: + if path.resolve() == executable.resolve(): + return "clientExecutable" + name = path.name.casefold() + if name.startswith("openq4-ded_"): + return "dedicatedServerExecutable" if path.suffix.casefold() != ".pdb" else "diagnosticSymbols" + if name.startswith("openq4-client_") and path.suffix.casefold() == ".pdb": + return "diagnosticSymbols" + if name.startswith("renderer-gl_"): + return "rendererModuleOpenGL" if path.suffix.casefold() != ".pdb" else "diagnosticSymbols" + if name.startswith("renderer-vk_"): + return "rendererModuleVulkan" if path.suffix.casefold() != ".pdb" else "diagnosticSymbols" + if name.startswith("game-sp_"): + return "singlePlayerGameModule" if path.suffix.casefold() != ".pdb" else "diagnosticSymbols" + if name.startswith("game-mp_"): + return "multiplayerGameModule" if path.suffix.casefold() != ".pdb" else "diagnosticSymbols" + if path.suffix.casefold() == ".pdb": + return "diagnosticSymbols" + return "runtimeLibrary" + + +def collect_runtime_files(runtime_dir: Path, executable: Path) -> list[dict[str, Any]]: + """Hash the selected client and every packaged shared library it may load.""" + candidates = {executable.resolve(): executable} + for parent_name, dir_names, file_names in os.walk(runtime_dir, followlinks=False): + parent = Path(parent_name) + for dir_name in list(dir_names): + directory = parent / dir_name + if is_link_or_junction(directory): + raise ValueError(f"runtime tree contains a symlink or junction: {directory}") + for file_name in file_names: + path = parent / file_name + if is_link_or_junction(path): + raise ValueError(f"runtime tree contains a symlink or junction: {path}") + lower_name = path.name.casefold() + if ( + is_shared_library(path) + or lower_name.startswith("openq4-client_") + or lower_name.startswith("openq4-ded_") + or path.suffix.casefold() == ".pdb" + ): + candidates[path.resolve()] = path + + records = [ + {"kind": runtime_kind(path, executable), **file_record(path, runtime_dir)} + for path in candidates.values() + ] + records.sort(key=lambda item: (item["kind"], item["path"].casefold())) + kinds = {item["kind"] for item in records} + required = {"clientExecutable", "singlePlayerGameModule", "multiplayerGameModule"} + if sys.platform != "darwin": + required.add("rendererModuleOpenGL") + missing = sorted(required - kinds) + if missing: + raise FileNotFoundError("runtime package is missing: " + ", ".join(missing)) + return records + + +def collect_overlay_loose_files( + runtime_dir: Path, runtime_files: list[dict[str, Any]] +) -> tuple[list[dict[str, Any]], list[str]]: + """Hash loose fs_game files and reject content outside the minimal package surface.""" + game_dir = runtime_dir / "baseoq4" + if not game_dir.is_dir(): + return [], [f"openQ4 overlay directory is missing: {game_dir}"] + runtime_paths = {item["path"].casefold() for item in runtime_files} + files: list[Path] = [] + for parent_name, dir_names, file_names in os.walk(game_dir, followlinks=False): + parent = Path(parent_name) + for dir_name in list(dir_names): + directory = parent / dir_name + if is_link_or_junction(directory): + raise ValueError(f"openQ4 overlay contains a symlink or junction: {directory}") + for file_name in file_names: + path = parent / file_name + if is_link_or_junction(path): + raise ValueError(f"openQ4 overlay contains a symlink or junction: {path}") + if path.suffix.casefold() != ".pk4": + files.append(path) + files.sort(key=lambda path: path.relative_to(runtime_dir).as_posix().casefold()) + records = [file_record(path, runtime_dir) for path in files] + allowed = runtime_paths | {"baseoq4/mod.json"} + unexpected = [ + f"unexpected loose openQ4 overlay file: {item['path']}" + for item in records + if item["path"].casefold() not in allowed + and not ( + Path(item["path"]).parent.as_posix().casefold() == "baseoq4" + and Path(item["path"]).suffix.casefold() == ".pdb" + ) + ] + return records, unexpected + + +def asset_manifest_from_payload(payload: dict[str, Any]) -> list[dict[str, Any]]: + records = payload.get("stockPk4s") + if records is None and isinstance(payload.get("assets"), dict): + records = payload["assets"].get("stockPk4s") + if not isinstance(records, list): + raise ValueError("expected asset manifest does not contain a stockPk4s array") + return records + + +def loose_manifest_from_payload(payload: dict[str, Any]) -> list[dict[str, Any]]: + records = payload.get("looseFiles") + if records is None and isinstance(payload.get("assets"), dict): + records = payload["assets"].get("looseFiles") + if records is None: + return [] + if not isinstance(records, list): + raise ValueError("expected asset manifest looseFiles field is not an array") + return records + + +def compare_file_records( + expected: list[dict[str, Any]], actual: list[dict[str, Any]], label: str +) -> list[str]: + failures: list[str] = [] + expected_by_path = {str(item.get("path", "")).casefold(): item for item in expected} + actual_by_path = {str(item.get("path", "")).casefold(): item for item in actual} + for path in sorted(set(expected_by_path) - set(actual_by_path)): + failures.append(f"missing {label}: {expected_by_path[path].get('path', path)}") + for path in sorted(set(actual_by_path) - set(expected_by_path)): + failures.append(f"unexpected {label}: {actual_by_path[path].get('path', path)}") + for path in sorted(set(expected_by_path) & set(actual_by_path)): + expected_item = expected_by_path[path] + actual_item = actual_by_path[path] + for key in ("size", "sha256"): + if expected_item.get(key) != actual_item.get(key): + failures.append( + f"{label} {actual_item.get('path', path)} {key} differs: " + f"expected {expected_item.get(key)!r}, got {actual_item.get(key)!r}" + ) + return failures + + +def compare_asset_records(expected: list[dict[str, Any]], actual: list[dict[str, Any]]) -> list[str]: + return compare_file_records(expected, actual, "retail PK4") + + +def find_client(runtime_dir: Path) -> Path: + suffix = ".exe" if os.name == "nt" else "" + preferred = runtime_dir / f"openQ4-client_{host_arch()}{suffix}" + if preferred.is_file(): + return preferred + candidates = sorted(runtime_dir.glob(f"openQ4-client_*{suffix}")) + for candidate in candidates: + if candidate.is_file(): + return candidate + raise FileNotFoundError(f"openQ4 client executable not found under {runtime_dir}") + + +def validate_runtime_dir(runtime_dir: Path, source_root: Path) -> Path: + """Require canonical .install or an ordinary isolated temporary package.""" + source_absolute = source_root.absolute() + runtime_absolute = runtime_dir.absolute() + if is_link_or_junction(source_absolute): + raise ValueError(f"source root must not be a link or junction: {source_absolute}") + try: + relative_to_source = runtime_absolute.relative_to(source_absolute) + except ValueError as exc: + raise ValueError( + f"runtime directory must stay below the source root {source_absolute}: {runtime_absolute}" + ) from exc + current = source_absolute + for part in relative_to_source.parts: + current /= part + if current.exists() and is_link_or_junction(current): + raise ValueError( + f"runtime directory ancestry must not contain a link or junction: {current}" + ) + resolved = runtime_absolute.resolve() + if not resolved.is_dir(): + raise FileNotFoundError(f"runtime directory does not exist: {resolved}") + canonical = (source_root / ".install").resolve() + temporary_parent = (source_root / ".tmp" / "stock-runtime").resolve() + if resolved != canonical: + try: + relative = resolved.relative_to(temporary_parent) + except ValueError as exc: + raise ValueError( + f"alternate runtime directory must stay below {temporary_parent}: {resolved}" + ) from exc + if not relative.parts: + raise ValueError( + f"alternate runtime directory must be a named child below {temporary_parent}" + ) + return resolved + + +def add_set(args: list[str], name: str, value: Any) -> None: + args.extend(("+set", name, str(value))) + + +def add_command(args: list[str], name: str, *values: Any) -> None: + args.append("+" + name) + args.extend(str(value) for value in values) + + +def write_cfg(savepath: Path, relative: str, lines: list[str]) -> None: + payload = "\n".join(lines) + "\n" + # baseoq4 is authoritative; q4base mirrors the file for legacy path + # diagnostics without adding any asset override to the repository. + for game_dir in ("baseoq4", "q4base"): + path = savepath / game_dir / Path(relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(payload, encoding="utf-8") + + +def common_args( + runtime_dir: Path, + asset_root: Path, + savepath: Path, + log_name: str, + cfg_path: str | None, + width: int, + height: int, +) -> list[str]: + args: list[str] = [] + add_set(args, "win_allowMultipleInstances" if os.name == "nt" else "sys_allowMultipleInstances", 1) + add_set(args, "logFile", 2) + add_set(args, "logFileName", f"logs/{log_name}") + add_set(args, "developer", 1) + add_set(args, "r_ignoreGLErrors", 0) + add_set(args, "r_fullscreen", 0) + add_set(args, "r_borderless", 0) + # Suppress the one-time legacy migration before the renderer starts. A + # queued r_borderless=0 alone is too late: the migration otherwise creates + # the initial window as desktop-sized borderless before command execution. + add_set(args, "r_borderlessDefaultMigrated", 1) + add_set(args, "r_fullscreenDesktop", 0) + add_set(args, "r_windowWidth", width) + add_set(args, "r_windowHeight", height) + add_set(args, "r_mode", -1) + add_set(args, "r_customWidth", width) + add_set(args, "r_customHeight", height) + add_set(args, "r_swapInterval", 0) + add_set(args, "r_renderApi", "gl") + add_set(args, "com_maxfps", 240) + add_set(args, "com_skipLoadingContinue", 1) + add_set(args, "com_loadingContinueAutoAdvance", 1) + add_set(args, "g_autoSkipCinematics", 1) + add_set(args, "g_autoScreenshot", 0) + add_set(args, "sv_cheats", 1) + if cfg_path: + add_set(args, "g_autoExecAfterMapLoad", cfg_path) + add_set(args, "g_autoExecAfterMapLoadDelayMs", 1000) + add_set(args, "fs_basepath", asset_root) + add_set(args, "fs_savepath", savepath) + # The process is launched with this exact recorded runtime directory as its + # working directory, so the engine's locked fs_cdpath mounts that staged + # package. Keep fs_devpath in the isolated evidence tree: retail-style + # generated collision caches and other developer outputs must never mutate + # the package after it is hashed. + add_set(args, "fs_devpath", savepath) + add_set(args, "fs_game", "baseoq4") + # Re-apply the complete display contract after queued +set commands. This + # makes the renderer state, not only the eventual CVar values, authoritative. + add_command(args, "vid_restart") + return args + + +def expected_role_arguments( + role: str, + runtime_dir: Path, + asset_root: Path, + output_dir: Path, + width: int, + height: int, + mp_port: int, +) -> list[str]: + """Build the one canonical command line accepted for an evidence role.""" + + if role not in ROLE_EVIDENCE_CONTRACT: + raise ValueError(f"unknown baseline role: {role}") + contract = ROLE_EVIDENCE_CONTRACT[role] + savepath = output_dir / "savepaths" / contract["saveDir"] + cfg_by_role = { + "sp-capture": f"{BASELINE_DIR}/sp_stage1.cfg", + "sp-demo-playback": None, + "mp-server": f"{BASELINE_DIR}/server.cfg", + "mp-client": f"{BASELINE_DIR}/client.cfg", + } + args = common_args( + runtime_dir, + asset_root, + savepath, + contract["logName"], + cfg_by_role[role], + width, + height, + ) + if role == "sp-capture": + add_set(args, "si_gameType", "singleplayer") + add_command(args, "map", SP_MAP) + elif role == "sp-demo-playback": + add_command(args, "gfxInfo") + add_command(args, "timeDemoQuit", SP_DEMO_NAME) + else: + # Automated MP evidence always exercises the archived auto-join path. + # Join/spectator-menu tests are separate and explicitly set this to 0. + restart_index = args.index("+vid_restart") + args[restart_index:restart_index] = ("+set", "ui_autoJoin", "1") + if role == "mp-server": + add_set(args, "net_serverDedicated", 0) + add_set(args, "net_port", mp_port) + add_set(args, "si_pure", 1) + add_set(args, "net_serverAllowServerMod", 0) + add_set(args, "si_gameType", "DM") + add_command(args, "spawnServer", MP_MAP) + else: + add_set(args, "ui_name", "StockBaselineClient") + add_command(args, "connect", f"127.0.0.1:{mp_port}") + return args + + +def prepare_plans( + runtime_dir: Path, + asset_root: Path, + output_dir: Path, + width: int, + height: int, + mp_port: int, +) -> dict[str, RolePlan]: + plans: dict[str, RolePlan] = {} + + sp_savepath = output_dir / "savepaths" / "sp" + sp_stage1 = f"{BASELINE_DIR}/sp_stage1.cfg" + sp_stage2 = f"{BASELINE_DIR}/sp_after_load.cfg" + write_cfg( + sp_savepath, + sp_stage1, + [ + "waitMsec 5000", + "god", + "notarget", + f"recordDemo {SP_DEMO_NAME}", + "waitMsec 2500", + "stopRecording", + f'screenshot "screenshots/{BASELINE_DIR}/sp_before_save.tga"', + f"saveGame {SP_SAVE_NAME}", + "waitMsec 1000", + f'set g_autoExecAfterMapLoad "{sp_stage2}"', + "set g_autoExecAfterMapLoadDelayMs 1000", + f"loadGame {SP_SAVE_NAME}", + ], + ) + write_cfg( + sp_savepath, + sp_stage2, + [ + "echo OPENQ4_STOCK_BASELINE_SP_SAVE_LOAD_COMPLETE", + "waitMsec 2500", + "framePacingReset", + "r_rendererMetrics 1", + "waitMsec 2500", + "rendererBenchmarkCapture", + "r_rendererMetrics 0", + "framePacingSnapshot", + "gfxInfo", + f'screenshot "screenshots/{BASELINE_DIR}/sp_after_load.tga"', + "wait 5", + "quit", + ], + ) + sp_args = expected_role_arguments( + "sp-capture", runtime_dir, asset_root, output_dir, width, height, mp_port + ) + plans["sp-capture"] = RolePlan( + "sp-capture", + "SP", + sp_savepath, + "stock_baseline_sp.log", + "OPENQ4_STOCK_BASELINE_SP_SAVE_LOAD_COMPLETE", + sp_args, + [ + ("screenshot", f"baseoq4/screenshots/{BASELINE_DIR}/sp_after_load.tga"), + ( + "saveReferenceScreenshot", + f"baseoq4/screenshots/{BASELINE_DIR}/sp_before_save.tga", + ), + ("renderDemo", f"baseoq4/demos/{SP_DEMO_NAME}.demo"), + ("savePayload", f"baseoq4/savegames/{SP_SAVE_NAME}.save"), + ("savePreview", f"baseoq4/savegames/{SP_SAVE_NAME}.tga"), + ("saveDescription", f"baseoq4/savegames/{SP_SAVE_NAME}.txt"), + ], + ) + + playback_args = expected_role_arguments( + "sp-demo-playback", + runtime_dir, + asset_root, + output_dir, + width, + height, + mp_port, + ) + plans["sp-demo-playback"] = RolePlan( + "sp-demo-playback", + "SP demo playback", + sp_savepath, + "stock_baseline_demo_playback.log", + "frames rendered in", + playback_args, + ) + + for role_id, role_name, wait_msec in ( + # Leave enough headroom for a completely cold loopback client to build + # its isolated binary image/animation and collision caches before the + # listen server records its own evidence and exits. + ("mp-server", "server", 60000), + ("mp-client", "client", 5000), + ): + savepath = output_dir / "savepaths" / role_id + cfg = f"{BASELINE_DIR}/{role_name}.cfg" + marker = f"OPENQ4_STOCK_BASELINE_MP_{role_name.upper()}_COMPLETE" + demo_name = f"stock_baseline_mp_{role_name}" + role_commands = [f"waitMsec {wait_msec}"] + if role_name == "client": + role_commands.extend( + ( + "openq4_assertMPClientActive", + "openq4_assertMPGameplayView", + ) + ) + role_commands.extend( + ( + f"recordDemo {demo_name}", + "waitMsec 2500", + "stopRecording", + "framePacingSnapshot", + "gfxInfo", + ) + ) + if role_name == "client": + role_commands.extend( + ("openq4_assertMPClientActive", "openq4_assertMPGameplayView") + ) + role_commands.append(f'screenshot "screenshots/{BASELINE_DIR}/mp_{role_name}.tga"') + if role_name == "client": + role_commands.extend( + ("openq4_assertMPClientActive", "openq4_assertMPGameplayView") + ) + role_commands.extend((f"echo {marker}", "wait 5", "quit")) + write_cfg( + savepath, + cfg, + role_commands, + ) + log_name = f"stock_baseline_{role_name}.log" + role_args = expected_role_arguments( + role_id, runtime_dir, asset_root, output_dir, width, height, mp_port + ) + plans[role_id] = RolePlan( + role_id, + "MP", + savepath, + log_name, + marker, + role_args, + [ + ("screenshot", f"baseoq4/screenshots/{BASELINE_DIR}/mp_{role_name}.tga"), + ("renderDemo", f"baseoq4/demos/{demo_name}.demo"), + ], + ) + + for plan in plans.values(): + plan.stdout_path = output_dir / f"{plan.role_id}.stdout.txt" + plan.stderr_path = output_dir / f"{plan.role_id}.stderr.txt" + return plans + + +def plan_record(plan: RolePlan) -> dict[str, Any]: + return { + "role": plan.role_id, + "mode": plan.mode, + "windowed": True, + "captureMethod": "engine screenshot command", + "savepath": str(plan.savepath), + "logName": plan.log_name, + "marker": plan.marker, + "requiredMarkers": ROLE_EVIDENCE_CONTRACT[plan.role_id].get("requiredMarkers", []), + "arguments": plan.args, + "expected": [{"kind": kind, "path": path} for kind, path in plan.expected], + } + + +def planned_cvar_values(arguments: Any, name: str) -> list[str]: + if not isinstance(arguments, list): + return [] + values: list[str] = [] + for index in range(len(arguments) - 2): + if arguments[index] in {"+set", "+seta"} and arguments[index + 1] == name: + values.append(str(arguments[index + 2])) + return values + + +def planned_command_values(arguments: Any, name: str) -> list[list[str]]: + if not isinstance(arguments, list): + return [] + values: list[list[str]] = [] + command = "+" + name + for index, argument in enumerate(arguments): + if argument != command: + continue + command_values: list[str] = [] + for value in arguments[index + 1 :]: + if isinstance(value, str) and value.startswith("+"): + break + command_values.append(str(value)) + values.append(command_values) + return values + + +def launch(executable: Path, plan: RolePlan, cwd: Path) -> subprocess.Popen[Any]: + assert plan.stdout_path is not None and plan.stderr_path is not None + stdout = plan.stdout_path.open("w", encoding="utf-8", errors="replace") + stderr = plan.stderr_path.open("w", encoding="utf-8", errors="replace") + try: + process = subprocess.Popen([str(executable), *plan.args], cwd=cwd, stdout=stdout, stderr=stderr) + finally: + stdout.close() + stderr.close() + return process + + +def wait_process(process: subprocess.Popen[Any], timeout: int) -> tuple[int, bool]: + try: + return process.wait(timeout=timeout), False + except subprocess.TimeoutExpired: + process.kill() + return process.wait(timeout=10), True + + +def wait_processes_until( + processes: dict[str, subprocess.Popen[Any]], deadline: float +) -> dict[str, tuple[int, bool]]: + """Wait for a process group without giving sequential waits fresh budgets.""" + + pending = dict(processes) + results: dict[str, tuple[int, bool]] = {} + while pending: + for name, process in list(pending.items()): + exit_code = process.poll() + if exit_code is not None: + results[name] = (exit_code, False) + del pending[name] + if not pending: + break + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.05, remaining)) + + # Kill every process still pending before reaping any of them. This keeps + # the absolute deadline common to the whole group even if process teardown + # takes time on one platform. + for process in pending.values(): + try: + process.kill() + except OSError: + # The process may have exited between the final poll and kill. + pass + for name, process in pending.items(): + results[name] = (process.wait(timeout=10), True) + return results + + +def find_log(plan: RolePlan) -> Path | None: + for game_dir in ("baseoq4", "q4base"): + candidate = plan.savepath / game_dir / "logs" / plan.log_name + if candidate.is_file(): + return candidate + return None + + +def read_text(path: Path | None) -> str: + if path is None or not path.is_file(): + return "" + return path.read_text(encoding="utf-8", errors="replace") + + +def collect_role_diagnostics( + log_path: Path | None, + stdout_path: Path | None, + stderr_path: Path | None, +) -> tuple[str, str]: + """Return authoritative engine diagnostics and all captured diagnostics. + + On POSIX, the engine can mirror the same diagnostic line to both its + logfile and stdout. Ordered and exact lifecycle proofs therefore use only + the engine logfile, while the combined text remains useful for fail-closed + fatal/error scanning across every captured channel. + """ + + engine_log = read_text(log_path) + all_diagnostics = "\n".join( + (engine_log, read_text(stdout_path), read_text(stderr_path)) + ) + return engine_log, all_diagnostics + + +def runtime_window_evidence(diagnostics: str) -> tuple[int, int, str] | None: + modes = RUNTIME_WINDOW_MODE_PATTERN.findall(diagnostics) + if not modes: + return None + width, height, mode = modes[-1] + return int(width), int(height), mode.casefold() + + +def mp_client_active_proof_failure(diagnostics: str) -> str | None: + active_proofs = list(MP_CLIENT_ACTIVE_PATTERN.finditer(diagnostics)) + view_proofs = list(MP_CLIENT_VIEW_PATTERN.finditer(diagnostics)) + screenshot_writes = list(MP_CLIENT_SCREENSHOT_WRITE_PATTERN.finditer(diagnostics)) + if len(screenshot_writes) != 1: + return "exactly one MP client screenshot-write marker is required" + completion_pattern = re.compile( + rf"^{re.escape(ROLE_EVIDENCE_CONTRACT['mp-client']['marker'])}[ \t]*$", + re.MULTILINE, + ) + completions = list(completion_pattern.finditer(diagnostics)) + if len(completions) != 1: + return "exactly one MP client completion marker is required" + screenshot_write = screenshot_writes[0] + completion = completions[0] + if completion.start() <= screenshot_write.end(): + return "MP client completion marker must follow the screenshot write" + if len(active_proofs) < 2 or len(view_proofs) < 2: + return ( + "two exact active-player and gameplay-view proofs " + "(before and after screenshot) are required" + ) + for label, proofs in (("active-player", active_proofs), ("gameplay-view", view_proofs)): + before = [proof for proof in proofs if proof.end() <= screenshot_write.start()] + after = [ + proof + for proof in proofs + if proof.start() >= screenshot_write.end() and proof.end() <= completion.start() + ] + if not before or not after: + return f"exact {label} proofs must bracket the MP client screenshot write" + if completion.start() - after[-1].end() > 512: + return f"final {label} proof is not immediately before the completion marker" + return None + + +def runtime_window_failure( + diagnostics: str, + requested_width: int | None = None, + requested_height: int | None = None, +) -> str | None: + evidence = runtime_window_evidence(diagnostics) + if evidence is None: + return "runtime display mode evidence missing" + actual_width, actual_height, actual_mode = evidence + if actual_mode != "windowed": + return f"runtime display mode is {actual_mode}, not bordered windowed" + if requested_width is not None and requested_height is not None: + allowed_sizes = {(requested_width, requested_height)} + scales = ACTIVE_DISPLAY_SCALE_PATTERN.findall(diagnostics) + if scales: + scale = float(scales[-1]) + allowed_sizes.add((round(requested_width * scale), round(requested_height * scale))) + if (actual_width, actual_height) not in allowed_sizes: + expected = " or ".join(f"{width}x{height}" for width, height in sorted(allowed_sizes)) + return ( + f"runtime window/render size is {actual_width}x{actual_height}, " + f"expected {expected} from the requested logical size and active display scale" + ) + return None + + +def decode_tga_rgb(data: bytes) -> tuple[int, int, bytes] | str: + """Decode the uncompressed true-colour TGA subset written by openQ4.""" + + if len(data) < 18: + return "TGA header is truncated" + id_length = data[0] + color_map_type = data[1] + image_type = data[2] + width, height = struct.unpack_from(" tuple[float, float] | None: + """Compare the top half with the recursive lower-left strip signature.""" + + if factor not in (2, 4) or width % factor or (height // 2) % factor: + return None + target_width = width // factor + target_height = (height // 2) // factor + if target_width < 1 or target_height < 1: + return None + target_y = height - height // factor + if target_y + target_height > height: + return None + + count = target_width * target_height + source_luma_sum = 0 + target_luma_sum = 0 + source_luma_squared = 0 + target_luma_squared = 0 + luma_products = 0 + absolute_rgb_error = 0 + block_pixels = factor * factor + + for y in range(target_height): + for x in range(target_width): + source_rgb = [0, 0, 0] + for block_y in range(factor): + source_y = y * factor + block_y + for block_x in range(factor): + source_x = x * factor + block_x + source_index = (source_y * width + source_x) * 3 + source_rgb[0] += rgb[source_index] + source_rgb[1] += rgb[source_index + 1] + source_rgb[2] += rgb[source_index + 2] + source_rgb = [ + (channel + block_pixels // 2) // block_pixels for channel in source_rgb + ] + target_index = ((target_y + y) * width + x) * 3 + target_rgb = rgb[target_index : target_index + 3] + absolute_rgb_error += sum( + abs(source_rgb[channel] - target_rgb[channel]) for channel in range(3) + ) + source_luma = ( + 77 * source_rgb[0] + 150 * source_rgb[1] + 29 * source_rgb[2] + ) >> 8 + target_luma = ( + 77 * target_rgb[0] + 150 * target_rgb[1] + 29 * target_rgb[2] + ) >> 8 + source_luma_sum += source_luma + target_luma_sum += target_luma + source_luma_squared += source_luma * source_luma + target_luma_squared += target_luma * target_luma + luma_products += source_luma * target_luma + + source_variance_term = count * source_luma_squared - source_luma_sum**2 + target_variance_term = count * target_luma_squared - target_luma_sum**2 + if source_variance_term <= count * count or target_variance_term <= count * count: + return None + correlation = ( + count * luma_products - source_luma_sum * target_luma_sum + ) / (source_variance_term * target_variance_term) ** 0.5 + mean_absolute_error = absolute_rgb_error / (count * 3) + return correlation, mean_absolute_error + + +def recursive_scaled_strip_failure(rgb: bytes, width: int, height: int) -> str | None: + if width < 16 or height < 16 or width % 4 or height % 8: + return None + metrics = [recursive_scaled_strip_metrics(rgb, width, height, factor) for factor in (2, 4)] + if all( + metric is not None and metric[0] >= 0.98 and metric[1] <= 3.0 + for metric in metrics + ): + return "TGA contains recursive scaled-strip repetition" + return None + + +def validate_tga(path: Path, expected_dimensions: tuple[int, int] | None = None) -> str | None: + decoded = decode_tga_rgb(path.read_bytes()) + if isinstance(decoded, str): + return decoded + width, height, rgb = decoded + if expected_dimensions is not None and (width, height) != expected_dimensions: + return ( + f"TGA dimensions {width}x{height} differ from expected " + f"{expected_dimensions[0]}x{expected_dimensions[1]}" + ) + pixel_count = width * height + sample_step = max(1, pixel_count // 4096) + sampled_luma = [ + (77 * rgb[index] + 150 * rgb[index + 1] + 29 * rgb[index + 2]) >> 8 + for index in range(0, len(rgb), sample_step * 3) + ] + if sampled_luma and max(sampled_luma) - min(sampled_luma) <= 2: + return "TGA image is blank or near-solid" + strip_failure = recursive_scaled_strip_failure(rgb, width, height) + if strip_failure: + return strip_failure + return None + + +def center_aspect_resample_rgb( + rgb: bytes, + source_width: int, + source_height: int, + output_width: int, + output_height: int, +) -> bytes: + """Centre-crop and bilinearly resize RGB pixels using integer arithmetic.""" + + if ( + source_width < 1 + or source_height < 1 + or output_width < 1 + or output_height < 1 + or len(rgb) != source_width * source_height * 3 + ): + raise ValueError("invalid RGB dimensions or payload for centre-aspect resampling") + + crop_x = 0 + crop_y = 0 + crop_width = source_width + crop_height = source_height + source_aspect_product = source_width * output_height + output_aspect_product = source_height * output_width + if source_aspect_product > output_aspect_product: + crop_width = max(1, source_height * output_width // output_height) + crop_x = (source_width - crop_width) // 2 + elif source_aspect_product < output_aspect_product: + crop_height = max(1, source_width * output_height // output_width) + crop_y = (source_height - crop_height) // 2 + + output = bytearray(output_width * output_height * 3) + crop_max_x = crop_x + crop_width - 1 + crop_max_y = crop_y + crop_height - 1 + x_denominator = 2 * output_width + y_denominator = 2 * output_height + interpolation_denominator = x_denominator * y_denominator + + for output_y in range(output_height): + source_y_numerator = (2 * output_y + 1) * crop_height - output_height + source_y = crop_y + source_y_numerator // y_denominator + y_fraction = source_y_numerator % y_denominator + if source_y < crop_y: + source_y = crop_y + y_fraction = 0 + elif source_y >= crop_max_y: + source_y = crop_max_y + y_fraction = 0 + next_y = min(source_y + 1, crop_max_y) + + for output_x in range(output_width): + source_x_numerator = (2 * output_x + 1) * crop_width - output_width + source_x = crop_x + source_x_numerator // x_denominator + x_fraction = source_x_numerator % x_denominator + if source_x < crop_x: + source_x = crop_x + x_fraction = 0 + elif source_x >= crop_max_x: + source_x = crop_max_x + x_fraction = 0 + next_x = min(source_x + 1, crop_max_x) + + top_left = (source_y * source_width + source_x) * 3 + top_right = (source_y * source_width + next_x) * 3 + bottom_left = (next_y * source_width + source_x) * 3 + bottom_right = (next_y * source_width + next_x) * 3 + target = (output_y * output_width + output_x) * 3 + for channel in range(3): + top = ( + rgb[top_left + channel] * (x_denominator - x_fraction) + + rgb[top_right + channel] * x_fraction + ) + bottom = ( + rgb[bottom_left + channel] * (x_denominator - x_fraction) + + rgb[bottom_right + channel] * x_fraction + ) + output[target + channel] = ( + top * (y_denominator - y_fraction) + + bottom * y_fraction + + interpolation_denominator // 2 + ) // interpolation_denominator + return bytes(output) + + +def rgb_similarity_metrics(reference_rgb: bytes, candidate_rgb: bytes) -> tuple[float, float] | str: + """Return Pearson luma correlation and mean absolute RGB error.""" + + if not reference_rgb or len(reference_rgb) != len(candidate_rgb) or len(reference_rgb) % 3: + return "RGB comparison payloads differ or are empty" + pixel_count = len(reference_rgb) // 3 + reference_luma_sum = 0 + candidate_luma_sum = 0 + reference_luma_squared = 0 + candidate_luma_squared = 0 + luma_products = 0 + absolute_rgb_error = 0 + for index in range(0, len(reference_rgb), 3): + reference_luma = ( + 77 * reference_rgb[index] + + 150 * reference_rgb[index + 1] + + 29 * reference_rgb[index + 2] + ) >> 8 + candidate_luma = ( + 77 * candidate_rgb[index] + + 150 * candidate_rgb[index + 1] + + 29 * candidate_rgb[index + 2] + ) >> 8 + reference_luma_sum += reference_luma + candidate_luma_sum += candidate_luma + reference_luma_squared += reference_luma * reference_luma + candidate_luma_squared += candidate_luma * candidate_luma + luma_products += reference_luma * candidate_luma + absolute_rgb_error += ( + abs(reference_rgb[index] - candidate_rgb[index]) + + abs(reference_rgb[index + 1] - candidate_rgb[index + 1]) + + abs(reference_rgb[index + 2] - candidate_rgb[index + 2]) + ) + + reference_variance_term = ( + pixel_count * reference_luma_squared - reference_luma_sum**2 + ) + candidate_variance_term = ( + pixel_count * candidate_luma_squared - candidate_luma_sum**2 + ) + if reference_variance_term <= 0 or candidate_variance_term <= 0: + return "RGB comparison has insufficient luma variance" + correlation = ( + pixel_count * luma_products - reference_luma_sum * candidate_luma_sum + ) / (reference_variance_term * candidate_variance_term) ** 0.5 + mean_absolute_rgb_error = absolute_rgb_error / (pixel_count * 3) + return correlation, mean_absolute_rgb_error + + +def exposure_compensated_rgb_metrics( + reference_rgb: bytes, candidate_rgb: bytes +) -> tuple[float, float, float] | str: + """Fit one bounded luma exposure transform and return its RGB residual.""" + + if not reference_rgb or len(reference_rgb) != len(candidate_rgb) or len(reference_rgb) % 3: + return "RGB comparison payloads differ or are empty" + pixel_count = len(reference_rgb) // 3 + reference_luma_sum = 0 + candidate_luma_sum = 0 + candidate_luma_squared = 0 + luma_products = 0 + for index in range(0, len(reference_rgb), 3): + reference_luma = ( + 77 * reference_rgb[index] + + 150 * reference_rgb[index + 1] + + 29 * reference_rgb[index + 2] + ) >> 8 + candidate_luma = ( + 77 * candidate_rgb[index] + + 150 * candidate_rgb[index + 1] + + 29 * candidate_rgb[index + 2] + ) >> 8 + reference_luma_sum += reference_luma + candidate_luma_sum += candidate_luma + candidate_luma_squared += candidate_luma * candidate_luma + luma_products += reference_luma * candidate_luma + + candidate_variance_term = ( + pixel_count * candidate_luma_squared - candidate_luma_sum**2 + ) + if candidate_variance_term <= 0: + return "RGB comparison has insufficient luma variance" + covariance_term = ( + pixel_count * luma_products - reference_luma_sum * candidate_luma_sum + ) + exposure_gain = covariance_term / candidate_variance_term + exposure_gain = max( + SP_SAVE_PREVIEW_MIN_EXPOSURE_GAIN, + min(SP_SAVE_PREVIEW_MAX_EXPOSURE_GAIN, exposure_gain), + ) + reference_luma_mean = reference_luma_sum / pixel_count + candidate_luma_mean = candidate_luma_sum / pixel_count + exposure_bias = reference_luma_mean - exposure_gain * candidate_luma_mean + exposure_bias = max( + SP_SAVE_PREVIEW_MIN_EXPOSURE_BIAS, + min(SP_SAVE_PREVIEW_MAX_EXPOSURE_BIAS, exposure_bias), + ) + + absolute_rgb_error = 0 + for reference_channel, candidate_channel in zip(reference_rgb, candidate_rgb): + compensated_channel = int(exposure_gain * candidate_channel + exposure_bias + 0.5) + compensated_channel = max(0, min(255, compensated_channel)) + absolute_rgb_error += abs(reference_channel - compensated_channel) + mean_absolute_rgb_error = absolute_rgb_error / len(reference_rgb) + return exposure_gain, exposure_bias, mean_absolute_rgb_error + + +def save_preview_comparison( + reference_path: Path, preview_path: Path +) -> tuple[dict[str, Any] | None, str | None]: + """Compare the save preview with the adjacent pre-save frame from the same SP run.""" + + reference = decode_tga_rgb(reference_path.read_bytes()) + if isinstance(reference, str): + return None, f"same-state save-reference screenshot cannot be compared: {reference}" + preview = decode_tga_rgb(preview_path.read_bytes()) + if isinstance(preview, str): + return None, f"save preview cannot be compared: {preview}" + reference_width, reference_height, reference_rgb = reference + preview_width, preview_height, preview_rgb = preview + analysis_width, analysis_height = SP_SAVE_PREVIEW_ANALYSIS_DIMENSIONS + reference_rgb = center_aspect_resample_rgb( + reference_rgb, + reference_width, + reference_height, + analysis_width, + analysis_height, + ) + candidate_rgb = center_aspect_resample_rgb( + preview_rgb, + preview_width, + preview_height, + analysis_width, + analysis_height, + ) + metrics = rgb_similarity_metrics(reference_rgb, candidate_rgb) + if isinstance(metrics, str): + return None, metrics + luma_correlation, mean_absolute_rgb_error = metrics + compensated_metrics = exposure_compensated_rgb_metrics(reference_rgb, candidate_rgb) + if isinstance(compensated_metrics, str): + return None, compensated_metrics + exposure_gain, exposure_bias, compensated_mean_absolute_rgb_error = compensated_metrics + record = { + "algorithm": SP_SAVE_PREVIEW_COMPARISON_ALGORITHM, + "referenceArtifact": "saveReferenceScreenshot", + "analysisDimensions": {"width": analysis_width, "height": analysis_height}, + "lumaCorrelation": round(luma_correlation, 6), + "meanAbsoluteRgbError": round(mean_absolute_rgb_error, 3), + "minimumLumaCorrelation": SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION, + "exposureCompensation": { + "model": "shared-luma-affine", + "gain": round(exposure_gain, 6), + "bias": round(exposure_bias, 3), + "minimumGain": SP_SAVE_PREVIEW_MIN_EXPOSURE_GAIN, + "maximumGain": SP_SAVE_PREVIEW_MAX_EXPOSURE_GAIN, + "minimumBias": SP_SAVE_PREVIEW_MIN_EXPOSURE_BIAS, + "maximumBias": SP_SAVE_PREVIEW_MAX_EXPOSURE_BIAS, + }, + "exposureCompensatedMeanAbsoluteRgbError": round( + compensated_mean_absolute_rgb_error, 3 + ), + "maximumExposureCompensatedMeanAbsoluteRgbError": ( + SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR + ), + } + policy_failures: list[str] = [] + if luma_correlation < SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION: + policy_failures.append( + f"luma correlation {luma_correlation:.6f} is below " + f"{SP_SAVE_PREVIEW_MIN_LUMA_CORRELATION:.6f}" + ) + if compensated_mean_absolute_rgb_error > SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR: + policy_failures.append( + "exposure-compensated mean RGB error " + f"{compensated_mean_absolute_rgb_error:.3f} exceeds " + f"{SP_SAVE_PREVIEW_MAX_COMPENSATED_RGB_ERROR:.3f}" + ) + failure = "; ".join(policy_failures) if policy_failures else None + return record, failure + + +def evaluate_role( + plan: RolePlan, + output_dir: Path, + exit_code: int, + timed_out: bool, +) -> dict[str, Any]: + log_path = find_log(plan) + authoritative_diagnostics, all_diagnostics = collect_role_diagnostics( + log_path, plan.stdout_path, plan.stderr_path + ) + failures: list[str] = [] + if timed_out: + failures.append("process timeout") + if exit_code != 0: + failures.append(f"exit code {exit_code}") + if log_path is None: + failures.append("engine log missing") + if plan.marker.casefold() not in authoritative_diagnostics.casefold(): + failures.append(f"completion marker missing: {plan.marker}") + for marker in ROLE_EVIDENCE_CONTRACT[plan.role_id].get("requiredMarkers", []): + if marker.casefold() not in authoritative_diagnostics.casefold(): + failures.append(f"required gameplay marker missing: {marker}") + if plan.role_id == "mp-client": + active_failure = mp_client_active_proof_failure(authoritative_diagnostics) + if active_failure: + failures.append(active_failure) + width_values = planned_cvar_values(plan.args, "r_windowWidth") + height_values = planned_cvar_values(plan.args, "r_windowHeight") + requested_width = int(width_values[-1]) if len(width_values) == 1 else None + requested_height = int(height_values[-1]) if len(height_values) == 1 else None + display_failure = runtime_window_failure( + authoritative_diagnostics, requested_width, requested_height + ) + if display_failure: + failures.append(display_failure) + for name, pattern in FATAL_PATTERNS.items(): + count = len(pattern.findall(all_diagnostics)) + if count: + failures.append(f"{name} diagnostics={count}") + + artifacts: list[dict[str, Any]] = [] + if log_path is not None: + artifacts.append({"kind": "engineLog", **file_record(log_path, output_dir)}) + for kind, path in (("processStdout", plan.stdout_path), ("processStderr", plan.stderr_path)): + if path is None or not path.is_file(): + failures.append(f"{kind} capture missing") + else: + artifacts.append({"kind": kind, **file_record(path, output_dir)}) + for kind, relative in plan.expected: + path = plan.savepath / Path(relative) + if not path.is_file() or path.stat().st_size == 0: + failures.append(f"missing or empty {kind}: {relative}") + continue + if kind in ("screenshot", "saveReferenceScreenshot", "savePreview"): + runtime_evidence = runtime_window_evidence(authoritative_diagnostics) + expected_dimensions = SP_SAVE_PREVIEW_DIMENSIONS if kind == "savePreview" else ( + (runtime_evidence[0], runtime_evidence[1]) if runtime_evidence is not None else None + ) + tga_failure = validate_tga(path, expected_dimensions) + if tga_failure: + failures.append(f"{relative}: {tga_failure}") + artifacts.append({"kind": kind, **file_record(path, output_dir)}) + + result = { + "role": plan.role_id, + "mode": plan.mode, + "status": "pass" if not failures else "fail", + "exitCode": exit_code, + "timedOut": timed_out, + "failures": failures, + "artifacts": artifacts, + } + if plan.role_id == "sp-capture": + expected_by_kind = {kind: plan.savepath / Path(relative) for kind, relative in plan.expected} + reference_path = expected_by_kind.get("saveReferenceScreenshot") + preview_path = expected_by_kind.get("savePreview") + if ( + reference_path is not None + and preview_path is not None + and reference_path.is_file() + and preview_path.is_file() + ): + comparison, comparison_failure = save_preview_comparison( + reference_path, preview_path + ) + if comparison is not None: + result["savePreviewComparison"] = comparison + if comparison_failure: + failures.append( + "save preview differs from same-state save reference: " + f"{comparison_failure}" + ) + else: + failures.append("save preview comparison artifacts are unavailable") + result["status"] = "pass" if not failures else "fail" + return result + + +def run_capture( + runtime_dir: Path, + executable: Path, + output_dir: Path, + plans: dict[str, RolePlan], + timeout: int, + mp_client_delay: int, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + + sp_plan = plans["sp-capture"] + sp_process = launch(executable, sp_plan, runtime_dir) + exit_code, timed_out = wait_process(sp_process, timeout) + results.append(evaluate_role(sp_plan, output_dir, exit_code, timed_out)) + if results[-1]["status"] == "pass": + playback_plan = plans["sp-demo-playback"] + playback_process = launch(executable, playback_plan, runtime_dir) + exit_code, timed_out = wait_process(playback_process, timeout) + results.append(evaluate_role(playback_plan, output_dir, exit_code, timed_out)) + else: + results.append({ + "role": "sp-demo-playback", + "mode": "SP demo playback", + "status": "fail", + "exitCode": None, + "timedOut": False, + "failures": ["not run because SP capture/save-load failed"], + "artifacts": [], + }) + + server_plan = plans["mp-server"] + client_plan = plans["mp-client"] + mp_deadline = time.monotonic() + timeout + server_process = launch(executable, server_plan, runtime_dir) + time.sleep(max(1, mp_client_delay)) + client_process = launch(executable, client_plan, runtime_dir) + mp_results = wait_processes_until( + {"server": server_process, "client": client_process}, mp_deadline + ) + server_exit, server_timeout = mp_results["server"] + client_exit, client_timeout = mp_results["client"] + results.append(evaluate_role(server_plan, output_dir, server_exit, server_timeout)) + results.append(evaluate_role(client_plan, output_dir, client_exit, client_timeout)) + return results + + +def git_state(root: Path) -> dict[str, Any]: + def run(*args: str) -> str: + completed = subprocess.run(["git", *args], cwd=root, capture_output=True, text=True, check=False) + return completed.stdout.strip() if completed.returncode == 0 else "" + + return { + "policy": GIT_PROVENANCE_POLICY, + "root": str(root.resolve()), + "revision": run("rev-parse", "HEAD"), + "dirty": bool(run("status", "--porcelain")), + } + + +def verify_recorded_files( + report: dict[str, Any], + report_dir: Path, + asset_root: Path, + runtime_dir: Path | None = None, +) -> list[str]: + failures: list[str] = [] + if report.get("status") != "pass": + failures.append(f"baseline report status is {report.get('status')!r}, not 'pass'") + if report.get("dryRun") is not False: + failures.append("passing baseline must record dryRun=false") + for field_name in TOP_LEVEL_FAILURE_ARRAYS: + field_value = report.get(field_name) + if not isinstance(field_value, list): + failures.append(f"baseline {field_name} field is missing or is not an array") + elif field_value: + failures.append(f"baseline {field_name} field is nonempty") + + current_git = git_state(repo_root()) + recorded_git = report.get("git") + if not isinstance(recorded_git, dict): + failures.append("baseline report does not contain git provenance") + else: + revision = recorded_git.get("revision") + if ( + recorded_git.get("policy") != GIT_PROVENANCE_POLICY + or recorded_git.get("root") != current_git.get("root") + or not isinstance(revision, str) + or re.fullmatch(r"[0-9a-f]{40}", revision) is None + or not isinstance(recorded_git.get("dirty"), bool) + ): + failures.append("baseline git provenance policy/shape differs") + if not current_git.get("revision"): + failures.append("current openQ4 git revision is unavailable") + elif revision != current_git.get("revision"): + failures.append( + "baseline git revision is not the current openQ4 HEAD: " + f"recorded {revision!r}, current {current_git.get('revision')!r}" + ) + if isinstance(recorded_git.get("dirty"), bool) and ( + recorded_git.get("dirty") != current_git.get("dirty") + ): + failures.append("baseline git dirty state differs from the current checkout") + + assets_value = report.get("assets") + if not isinstance(assets_value, dict): + failures.append("baseline report assets field is missing or is not an object") + assets: dict[str, Any] = {} + else: + assets = assets_value + selected_asset_root = asset_root.resolve() + recorded_asset_root = assets.get("root") + if not isinstance(recorded_asset_root, str) or not recorded_asset_root: + failures.append("baseline report does not record its retail asset root") + elif Path(recorded_asset_root).resolve() != selected_asset_root: + failures.append( + "retail asset root differs: " + f"recorded {Path(recorded_asset_root).resolve()}, selected {selected_asset_root}" + ) + allow_value = assets.get("allowLinkedGameDirectories") + if not isinstance(allow_value, bool): + failures.append("allowLinkedGameDirectories must be a boolean") + allow_asset_dir_links = False + else: + allow_asset_dir_links = allow_value + + expected_binding = report.get("expectedAssets", {}) + if ( + not isinstance(expected_binding, dict) + or expected_binding.get("supplied") is not True + or not isinstance(expected_binding.get("path"), str) + or not expected_binding.get("path") + or not isinstance(expected_binding.get("sha256"), str) + or re.fullmatch(r"[0-9a-f]{64}", expected_binding.get("sha256", "")) is None + ): + failures.append("passing baseline is not bound to a recorded --expected-assets manifest") + else: + expected_path = Path(expected_binding["path"]) + if not expected_path.is_absolute(): + failures.append("recorded expected-assets manifest path is not absolute") + elif not expected_path.is_file(): + failures.append(f"recorded expected-assets manifest is missing: {expected_path}") + elif is_link_or_junction(expected_path): + failures.append(f"recorded expected-assets manifest must not be a link: {expected_path}") + else: + actual_expected_sha256 = sha256_file(expected_path) + if actual_expected_sha256 != expected_binding["sha256"]: + failures.append( + "recorded expected-assets manifest SHA-256 differs: " + f"expected {expected_binding['sha256']}, got {actual_expected_sha256}" + ) + try: + expected_payload = json.loads(expected_path.read_text(encoding="utf-8")) + if not isinstance(expected_payload, dict): + raise ValueError("expected asset manifest root is not an object") + expected_records = asset_manifest_from_payload(expected_payload) + recorded_records = asset_manifest_from_payload(assets) + if expected_records != recorded_records: + failures.append( + "recorded retail PK4 inventory differs from the bound expected-assets manifest" + ) + expected_loose = loose_manifest_from_payload(expected_payload) + recorded_loose = loose_manifest_from_payload(assets) + if expected_loose != recorded_loose: + failures.append( + "recorded loose retail inventory differs from the bound expected-assets manifest" + ) + except (json.JSONDecodeError, OSError, ValueError) as exc: + failures.append(f"recorded expected-assets manifest is invalid: {exc}") + + current_assets: list[dict[str, Any]] | None = None + try: + current_assets = collect_pk4s(selected_asset_root, allow_asset_dir_links) + failures.extend(compare_asset_records(asset_manifest_from_payload(assets), current_assets)) + if assets.get("directoryViews") != asset_directory_views(selected_asset_root): + failures.append("recorded retail directory views differ") + current_loose = collect_loose_asset_files(selected_asset_root, allow_asset_dir_links) + failures.extend( + compare_file_records(loose_manifest_from_payload(assets), current_loose, "loose asset file") + ) + if current_loose: + failures.append( + f"retail fallback asset root contains {len(current_loose)} loose q4base/q4mp files" + ) + if loose_manifest_from_payload(assets): + failures.append("recorded baseline includes loose q4base/q4mp files") + except (OSError, ValueError) as exc: + failures.append(str(exc)) + + recorded_runtime_root = report.get("runtimeRoot") + selected_runtime_dir: Path | None = runtime_dir + if not isinstance(recorded_runtime_root, str) or not recorded_runtime_root: + failures.append("baseline report does not record its runtime root") + else: + recorded_runtime_dir = Path(recorded_runtime_root).absolute() + if selected_runtime_dir is None: + selected_runtime_dir = recorded_runtime_dir + elif selected_runtime_dir.resolve() != recorded_runtime_dir: + failures.append( + "runtime root differs: " + f"recorded {recorded_runtime_dir}, selected {selected_runtime_dir.resolve()}" + ) + if selected_runtime_dir is None: + selected_runtime_dir = repo_root() / ".install" + current_overlay_pk4s: list[dict[str, Any]] | None = None + try: + selected_runtime_dir = validate_runtime_dir(selected_runtime_dir, repo_root()) + executable = find_client(selected_runtime_dir) + current_runtime = collect_runtime_files(selected_runtime_dir, executable) + failures.extend( + compare_file_records(report.get("runtimeFiles", []), current_runtime, "runtime file") + ) + current_overlay_pk4s = collect_overlay_pk4s(selected_runtime_dir) + failures.extend( + compare_file_records( + assets.get("openQ4OverlayPk4s", []), current_overlay_pk4s, "openQ4 overlay PK4" + ) + ) + current_overlay_loose, unexpected = collect_overlay_loose_files( + selected_runtime_dir, current_runtime + ) + failures.extend( + compare_file_records( + assets.get("openQ4OverlayLooseFiles", []), + current_overlay_loose, + "openQ4 overlay loose file", + ) + ) + failures.extend(unexpected) + except (OSError, ValueError) as exc: + failures.append(str(exc)) + + recorded_collisions = assets.get("retailPathCollisions") + if assets.get("compatibilityModel") != ASSET_COMPATIBILITY_MODEL: + failures.append("asset compatibility model differs") + if assets.get("retailArchiveBytesMatchExpected") is not True: + failures.append("report does not establish that retail archive bytes match the expected manifest") + if not isinstance(recorded_collisions, list): + failures.append("retailPathCollisions field is missing or is not an array") + recorded_collisions = [] + collision_count = assets.get("retailPathCollisionCount") + if not isinstance(collision_count, int) or isinstance(collision_count, bool): + failures.append("retailPathCollisionCount field is missing or is not an integer") + elif collision_count != len(recorded_collisions): + failures.append("retailPathCollisionCount does not match the recorded collision inventory") + namespace_untouched = assets.get("retailPathNamespaceUntouched") + if namespace_untouched is not (len(recorded_collisions) == 0): + failures.append("retailPathNamespaceUntouched claim differs from the collision inventory") + if current_assets is not None and current_overlay_pk4s is not None: + try: + current_collisions = collect_retail_path_collisions( + selected_asset_root, + current_assets, + selected_runtime_dir, + current_overlay_pk4s, + ) + if recorded_collisions != current_collisions: + failures.append( + "recorded retail/openQ4 virtual-path collision inventory differs" + ) + except (OSError, ValueError) as exc: + failures.append(str(exc)) + + safety = report.get("safety", {}) + window_size = safety.get("windowSize", {}) if isinstance(safety, dict) else {} + width = window_size.get("width") if isinstance(window_size, dict) else None + height = window_size.get("height") if isinstance(window_size, dict) else None + if ( + not isinstance(safety, dict) + or safety.get("windowedOnly") is not True + or safety.get("borderless") is not False + or safety.get("engineScreenshotOnly") is not True + or safety.get("operatingSystemCapture") is not False + or safety.get("inputInjection") is not False + or not isinstance(width, int) + or not isinstance(height, int) + or width < 640 + or height < 480 + ): + failures.append("baseline safety/window contract differs") + + mp_port_value = report.get("mpPort") + if ( + not isinstance(mp_port_value, int) + or isinstance(mp_port_value, bool) + or not (1024 <= mp_port_value <= 65535) + ): + failures.append("baseline mpPort is missing or outside 1024..65535") + mp_port = 28140 + else: + mp_port = mp_port_value + + plans_value = report.get("plan") + if not isinstance(plans_value, list): + failures.append("baseline plan field is missing or is not an array") + plans: list[Any] = [] + else: + plans = plans_value + plan_by_role = { + plan.get("role"): plan for plan in plans if isinstance(plan, dict) and plan.get("role") + } + if len(plans) != len(ROLE_EVIDENCE_CONTRACT) or set(plan_by_role) != set(ROLE_EVIDENCE_CONTRACT): + failures.append("baseline plan does not contain each required role exactly once") + for role, contract in ROLE_EVIDENCE_CONTRACT.items(): + plan = plan_by_role.get(role, {}) + expected_mode = "SP demo playback" if role == "sp-demo-playback" else ( + "SP" if role == "sp-capture" else "MP" + ) + canonical_savepath = report_dir / "savepaths" / contract["saveDir"] + if plan.get("mode") != expected_mode: + failures.append(f"{role}: plan mode differs") + if plan.get("savepath") != str(canonical_savepath): + failures.append(f"{role}: plan savepath differs") + if plan.get("logName") != contract["logName"]: + failures.append(f"{role}: plan log name differs") + if plan.get("marker") != contract["marker"]: + failures.append(f"{role}: plan completion marker differs") + if plan.get("requiredMarkers", []) != contract.get("requiredMarkers", []): + failures.append(f"{role}: plan required gameplay markers differ") + if plan.get("windowed") is not True or plan.get("captureMethod") != "engine screenshot command": + failures.append(f"{role}: plan safety/capture contract differs") + launch_contract = { + "r_fullscreen": "0", + "r_borderless": "0", + "r_borderlessDefaultMigrated": "1", + "r_fullscreenDesktop": "0", + "r_windowWidth": str(width), + "r_windowHeight": str(height), + "r_renderApi": "gl", + "fs_basepath": str(selected_asset_root), + "fs_savepath": str(canonical_savepath), + "fs_devpath": str(canonical_savepath), + "fs_game": "baseoq4", + } + if role in ("mp-server", "mp-client"): + launch_contract["ui_autoJoin"] = "1" + if role == "sp-capture": + launch_contract["si_gameType"] = "singleplayer" + elif role == "mp-server": + launch_contract.update( + { + "net_serverDedicated": "0", + "net_port": str(mp_port), + "si_pure": "1", + "net_serverAllowServerMod": "0", + "si_gameType": "DM", + } + ) + for cvar, expected_value in launch_contract.items(): + if planned_cvar_values(plan.get("arguments"), cvar) != [expected_value]: + failures.append(f"{role}: launch CVar {cvar} differs") + command_contract: dict[str, list[list[str]]] = {} + if role == "sp-capture": + command_contract["map"] = [[SP_MAP]] + elif role == "sp-demo-playback": + command_contract["timeDemoQuit"] = [[SP_DEMO_NAME]] + elif role == "mp-server": + command_contract["spawnServer"] = [[MP_MAP]] + else: + command_contract["connect"] = [[f"127.0.0.1:{mp_port}"]] + for command, expected_values in command_contract.items(): + if planned_command_values(plan.get("arguments"), command) != expected_values: + failures.append(f"{role}: launch command {command} differs") + arguments = plan.get("arguments", []) + if not isinstance(arguments, list) or arguments.count("+vid_restart") != 1: + failures.append(f"{role}: launch must contain exactly one post-CVar vid_restart") + elif any( + cvar in arguments and arguments.index(cvar) > arguments.index("+vid_restart") + for cvar in ( + "r_fullscreen", + "r_borderless", + "r_borderlessDefaultMigrated", + "r_fullscreenDesktop", + "r_windowWidth", + "r_windowHeight", + "r_renderApi", + "fs_basepath", + "fs_savepath", + "fs_devpath", + "fs_game", + *(("ui_autoJoin",) if role in ("mp-server", "mp-client") else ()), + ) + ): + failures.append(f"{role}: vid_restart must follow the display/filesystem launch CVars") + if planned_cvar_values(plan.get("arguments"), "fs_cdpath"): + failures.append(f"{role}: launch must not override locked fs_cdpath") + expected_arguments = expected_role_arguments( + role, + selected_runtime_dir, + selected_asset_root, + report_dir, + width if isinstance(width, int) else DEFAULT_WIDTH, + height if isinstance(height, int) else DEFAULT_HEIGHT, + mp_port, + ) + if arguments != expected_arguments: + failures.append(f"{role}: launch arguments differ from the exact role contract") + expected_artifacts = [ + {"kind": kind, "path": path} for kind, path in contract["expected"].items() + ] + if plan.get("expected") != expected_artifacts: + failures.append(f"{role}: plan expected-artifact contract differs") + + results_value = report.get("results") + if not isinstance(results_value, list): + failures.append("baseline results field is missing or is not an array") + results: list[Any] = [] + else: + results = results_value + result_by_role = { + result.get("role"): result + for result in results + if isinstance(result, dict) and result.get("role") + } + if len(results) != len(ROLE_EVIDENCE_CONTRACT) or set(result_by_role) != set(ROLE_EVIDENCE_CONTRACT): + failures.append("baseline results do not contain each required role exactly once") + + for result in results: + for artifact in result.get("artifacts", []): + path = report_dir / Path(str(artifact.get("path", ""))) + if not path.is_file(): + failures.append(f"recorded artifact is missing: {artifact.get('path')}") + continue + if path.stat().st_size != artifact.get("size"): + failures.append(f"recorded artifact size differs: {artifact.get('path')}") + elif sha256_file(path) != artifact.get("sha256"): + failures.append(f"recorded artifact SHA-256 differs: {artifact.get('path')}") + for role, contract in ROLE_EVIDENCE_CONTRACT.items(): + result = result_by_role.get(role, {}) + expected_mode = "SP demo playback" if role == "sp-demo-playback" else ( + "SP" if role == "sp-capture" else "MP" + ) + if ( + result.get("status") != "pass" + or result.get("mode") != expected_mode + or result.get("exitCode") != 0 + or result.get("timedOut") is not False + or result.get("failures") != [] + ): + failures.append(f"{role}: passing result lifecycle contract is not satisfied") + artifacts = result.get("artifacts", []) + artifact_by_kind = { + item.get("kind"): item for item in artifacts if isinstance(item, dict) and item.get("kind") + } + required_kinds = {"engineLog", "processStdout", "processStderr", *contract["expected"]} + if len(artifacts) != len(required_kinds) or set(artifact_by_kind) != required_kinds: + failures.append(f"{role}: required artifact kinds differ") + continue + for kind, relative in contract["expected"].items(): + expected_suffix = f"savepaths/{'sp' if role.startswith('sp-') else role}/{relative}" + if str(artifact_by_kind[kind].get("path", "")).replace("\\", "/") != expected_suffix: + failures.append(f"{role}: {kind} artifact path differs") + for kind in ("processStdout", "processStderr"): + if artifact_by_kind[kind].get("path") != f"{role}.{'stdout' if kind == 'processStdout' else 'stderr'}.txt": + failures.append(f"{role}: {kind} artifact path differs") + log_path = str(artifact_by_kind["engineLog"].get("path", "")).replace("\\", "/") + allowed_log_paths = { + f"savepaths/{contract['saveDir']}/{game_dir}/logs/{contract['logName']}" + for game_dir in ("baseoq4", "q4base") + } + if log_path not in allowed_log_paths: + failures.append(f"{role}: engineLog artifact path differs") + authoritative_diagnostics, all_diagnostics = collect_role_diagnostics( + report_dir / Path(str(artifact_by_kind["engineLog"].get("path", ""))), + report_dir / Path(str(artifact_by_kind["processStdout"].get("path", ""))), + report_dir / Path(str(artifact_by_kind["processStderr"].get("path", ""))), + ) + if contract["marker"].casefold() not in authoritative_diagnostics.casefold(): + failures.append(f"{role}: completion marker is absent from verified diagnostics") + for marker in contract.get("requiredMarkers", []): + if marker.casefold() not in authoritative_diagnostics.casefold(): + failures.append(f"{role}: required gameplay marker is absent: {marker}") + if role == "mp-client": + active_failure = mp_client_active_proof_failure(authoritative_diagnostics) + if active_failure: + failures.append(f"{role}: {active_failure}") + for name, pattern in FATAL_PATTERNS.items(): + count = len(pattern.findall(all_diagnostics)) + if count: + failures.append(f"{role}: {name} diagnostics={count}") + display_failure = runtime_window_failure(authoritative_diagnostics, width, height) + if display_failure: + failures.append(f"{role}: {display_failure}") + runtime_evidence = runtime_window_evidence(authoritative_diagnostics) + if runtime_evidence is not None: + for screenshot_kind in ("screenshot", "saveReferenceScreenshot"): + if screenshot_kind not in artifact_by_kind: + continue + screenshot_path = report_dir / Path( + str(artifact_by_kind[screenshot_kind].get("path", "")) + ) + if not screenshot_path.is_file(): + continue + tga_failure = validate_tga( + screenshot_path, (runtime_evidence[0], runtime_evidence[1]) + ) + if tga_failure: + failures.append(f"{role}: {screenshot_kind}: {tga_failure}") + if "savePreview" in artifact_by_kind: + preview_path = report_dir / Path( + str(artifact_by_kind["savePreview"].get("path", "")) + ) + if preview_path.is_file(): + tga_failure = validate_tga(preview_path, SP_SAVE_PREVIEW_DIMENSIONS) + if tga_failure: + failures.append(f"{role}: save preview: {tga_failure}") + if role == "sp-capture": + reference_path = report_dir / Path( + str(artifact_by_kind["saveReferenceScreenshot"].get("path", "")) + ) + preview_path = report_dir / Path( + str(artifact_by_kind["savePreview"].get("path", "")) + ) + if reference_path.is_file() and preview_path.is_file(): + comparison, comparison_failure = save_preview_comparison( + reference_path, preview_path + ) + if comparison is None: + failures.append( + f"{role}: save preview comparison could not be computed: " + f"{comparison_failure or 'unknown error'}" + ) + else: + if result.get("savePreviewComparison") != comparison: + failures.append( + f"{role}: recorded save-preview comparison metrics differ" + ) + if comparison_failure: + failures.append( + f"{role}: save preview differs from same-state save reference: " + f"{comparison_failure}" + ) + else: + failures.append(f"{role}: save preview comparison artifacts are unavailable") + return failures + + +def write_reports(output_dir: Path, payload: dict[str, Any]) -> tuple[Path, Path, Path, Path]: + report_json = output_dir / "stock_asset_baseline_report.json" + report_md = output_dir / "stock_asset_baseline_report.md" + asset_json = output_dir / "stock_pk4_manifest.json" + runtime_json = output_dir / "openq4_runtime_manifest.json" + report_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + asset_json.write_text( + json.dumps( + { + "schemaVersion": RETAIL_MANIFEST_SCHEMA_VERSION, + "allowLinkedGameDirectories": payload["assets"]["allowLinkedGameDirectories"], + "directoryViews": payload["assets"]["directoryViews"], + "stockPk4s": payload["assets"]["stockPk4s"], + "looseFiles": payload["assets"]["looseFiles"], + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + runtime_json.write_text( + json.dumps( + { + "schemaVersion": RUNTIME_MANIFEST_SCHEMA_VERSION, + "runtimeRoot": payload["runtimeRoot"], + "runtimeFiles": payload["runtimeFiles"], + "openQ4OverlayPk4s": payload["assets"]["openQ4OverlayPk4s"], + "openQ4OverlayLooseFiles": payload["assets"]["openQ4OverlayLooseFiles"], + "compatibilityModel": payload["assets"]["compatibilityModel"], + "retailPathCollisionCount": payload["assets"]["retailPathCollisionCount"], + "retailPathCollisions": payload["assets"]["retailPathCollisions"], + }, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + results = payload.get("results", []) + collisions = payload["assets"].get("retailPathCollisions", []) + collision_counts_by_overlay: dict[str, int] = {} + for collision in collisions: + for archive in collision.get("openQ4OverlayPk4s", []): + collision_counts_by_overlay[archive] = collision_counts_by_overlay.get(archive, 0) + 1 + lines = [ + "# Retail-PK4 Compatibility Baseline With Packaged openQ4 Overlays", + "", + f"- Status: **{payload['status']}**", + f"- Generated: {payload['generatedUtc']}", + f"- Compatibility model: `{payload['assets']['compatibilityModel']}`", + f"- Git revision: `{payload['git']['revision'] or 'unavailable'}` (dirty: `{str(payload['git']['dirty']).lower()}`)", + f"- Git provenance policy: `{payload['git']['policy']}`", + f"- Retail fallback asset root: `{payload['assets']['root']}`", + f"- Linked q4base/q4mp view allowed: `{str(payload['assets']['allowLinkedGameDirectories']).lower()}`", + f"- Retail PK4s: {len(payload['assets']['stockPk4s'])}", + f"- Loose q4base/q4mp files: {len(payload['assets']['looseFiles'])} (must be zero)", + "- Retail archive bytes match bound expected manifest: " + f"`{str(payload['assets']['retailArchiveBytesMatchExpected']).lower()}`", + f"- Expected-assets manifest bound: `{str(payload['expectedAssets']['supplied']).lower()}`", + f"- Expected-assets SHA-256: `{payload['expectedAssets']['sha256'] or 'not supplied'}`", + f"- openQ4 runtime root: `{payload['runtimeRoot']}`", + f"- openQ4 runtime files: {len(payload['runtimeFiles'])}", + f"- openQ4 overlay PK4s: {len(payload['assets']['openQ4OverlayPk4s'])}", + f"- openQ4 loose overlay files: {len(payload['assets']['openQ4OverlayLooseFiles'])}", + f"- Retail virtual paths superseded by packaged overlays: {len(collisions)}", + "- Retail virtual-path namespace untouched: " + f"`{str(payload['assets']['retailPathNamespaceUntouched']).lower()}`", + f"- MP loopback port: {payload['mpPort']}", + "- Display: forced windowed", + "- Screenshots: engine `screenshot` command only", + "- Input automation: none", + "", + "## Asset directory view", + "", + "| Path | Linked | Resolved target |", + "|---|---|---|", + ] + for view in payload["assets"]["directoryViews"]: + lines.append( + f"| `{view['path']}` | `{str(view['linked']).lower()}` | `{view['resolvedTarget']}` |" + ) + lines += [ + "", + "## Packaged-overlay precedence", + "", + ] + if collisions: + lines.append( + "The verified retail PK4 archive bytes are unchanged, but packaged openQ4 " + f"overlays supersede **{len(collisions)}** retail virtual paths. This is a " + "retail-asset compatibility run, not an overlay-free or stock-only run." + ) + lines.append("") + for archive, count in sorted(collision_counts_by_overlay.items(), key=lambda item: item[0].casefold()): + noun = "path" if count == 1 else "paths" + lines.append(f"- `{archive}` supersedes {count} retail virtual {noun}.") + lines.append("") + lines.append( + "The complete path-by-path collision inventory is recorded in the JSON report." + ) + else: + lines.append( + "No member of a packaged openQ4 overlay PK4 supersedes a path present in the " + "verified retail PK4 set." + ) + lines += [ + "", + "## Results", + "", + "| Status | Role | Mode | Artifacts | Failures |", + "|---|---|---|---:|---|", + ] + for result in results: + failures = "; ".join(result.get("failures", [])) or "none" + lines.append( + f"| {result.get('status')} | `{result.get('role')}` | {result.get('mode')} | " + f"{len(result.get('artifacts', []))} | {failures} |" + ) + sp_capture = next( + (result for result in results if result.get("role") == "sp-capture"), None + ) + comparison = ( + sp_capture.get("savePreviewComparison", {}) + if isinstance(sp_capture, dict) + else {} + ) + if comparison: + dimensions = comparison.get("analysisDimensions", {}) + exposure = comparison.get("exposureCompensation", {}) + lines += [ + "", + "## SP save-preview coherence", + "", + f"- Algorithm: `{comparison.get('algorithm')}`", + f"- Reference artifact: `{comparison.get('referenceArtifact')}`", + f"- Analysis size: {dimensions.get('width')}x{dimensions.get('height')}", + f"- Luma correlation: {comparison.get('lumaCorrelation')} " + f"(minimum {comparison.get('minimumLumaCorrelation')})", + f"- Raw mean absolute RGB error: {comparison.get('meanAbsoluteRgbError')}", + f"- Exposure compensation: gain {exposure.get('gain')}, " + f"bias {exposure.get('bias')}", + "- Exposure-compensated mean absolute RGB error: " + f"{comparison.get('exposureCompensatedMeanAbsoluteRgbError')} " + "(maximum " + f"{comparison.get('maximumExposureCompensatedMeanAbsoluteRgbError')})", + ] + if payload.get("assetComparisonFailures"): + lines += ["", "## Asset comparison failures", ""] + lines.extend(f"- {failure}" for failure in payload["assetComparisonFailures"]) + if payload.get("preflightFailures"): + lines += ["", "## Runtime/overlay preflight failures", ""] + lines.extend(f"- {failure}" for failure in payload["preflightFailures"]) + if payload.get("postCaptureVerificationFailures"): + lines += ["", "## Post-capture verification failures", ""] + lines.extend(f"- {failure}" for failure in payload["postCaptureVerificationFailures"]) + lines += [ + "", + "## Manual review still required", + "", + "Automated pass status proves file identity, map lifecycle, render-demo playback, save/restore completion, artifact integrity, broad save-preview coherence with the same-state pre-save frame, and that captured diagnostics contain none of the harness denylist classes: fatal errors, engine `ERROR` records, shader compile/program-link failures, Vulkan validation messages or VUIDs, and OpenGL errors. It does not assert warning-free logs; other warnings remain retained for manual review. A human must still review the engine screenshots for visual correctness and play representative SP/MP sequences for behavior, audio, and input feel.", + ] + report_md.write_text("\n".join(lines) + "\n", encoding="utf-8") + return report_json, report_md, asset_json, runtime_json + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--asset-root", default=default_asset_root(), help="Quake 4 installation root containing q4base/ and optionally q4mp/.") + parser.add_argument( + "--runtime-dir", + default="", + help=( + "Resolved openQ4 package directory containing the client and baseoq4/. " + "Defaults to repository .install; alternates must be fresh ordinary directories below .tmp/stock-runtime/." + ), + ) + parser.add_argument("--output-dir", default="", help="Evidence directory; defaults to .tmp/stock-baseline/.") + parser.add_argument("--expected-assets", default="", help="Optional prior stock_pk4_manifest.json or baseline report. Any mismatch fails before launch.") + parser.add_argument( + "--allow-asset-dir-links", + action="store_true", + help="Permit q4base/q4mp symlinks or junctions in an intentionally clean asset view; resolved targets are recorded and nested links remain forbidden.", + ) + parser.add_argument("--verify-report", default="", help="Verify an existing report's PK4 and artifact hashes without launching openQ4.") + parser.add_argument("--dry-run", action="store_true", help="Write the exact windowed launch/config plan and manifests without launching openQ4.") + parser.add_argument("--list", action="store_true", help="List baseline cases and safety invariants without hashing or launching.") + parser.add_argument( + "--timeout", + type=int, + default=180, + help="Maximum seconds for each SP role and for the shared MP server/client session.", + ) + parser.add_argument("--mp-client-delay", type=int, default=8, help="Seconds between starting the listen server and loopback client.") + parser.add_argument("--mp-port", type=int, default=28140, help="Loopback listen-server UDP port.") + parser.add_argument("--width", type=int, default=DEFAULT_WIDTH) + parser.add_argument("--height", type=int, default=DEFAULT_HEIGHT) + args = parser.parse_args(argv) + if args.width < 640 or args.height < 480: + parser.error("baseline window must be at least 640x480") + if not (1024 <= args.mp_port <= 65535): + parser.error("--mp-port must be between 1024 and 65535") + if args.timeout < 30: + parser.error("--timeout must be at least 30 seconds") + return args + + +def print_list() -> None: + print( + f"SP: {SP_MAP} -> render demo -> save-reference screenshot -> save -> " + "load -> post-load engine screenshot" + ) + print(f"SP demo: play back {SP_DEMO_NAME}.demo with timeDemoQuit") + print(f"MP: {MP_MAP} listen server + loopback client -> demos + engine screenshots") + print("Safety: r_fullscreen=0, fixed window, no OS capture, no mouse/keyboard injection") + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + if args.list: + print_list() + return 0 + if not args.asset_root: + raise ValueError("--asset-root is required on this platform") + asset_root = Path(args.asset_root).resolve() + source_root = repo_root() + requested_runtime_dir = Path(args.runtime_dir).absolute() if args.runtime_dir else None + + if args.verify_report: + report_path = Path(args.verify_report).resolve() + report = json.loads(report_path.read_text(encoding="utf-8")) + if report.get("schemaVersion") != SCHEMA_VERSION: + raise ValueError(f"unsupported baseline report schema: {report.get('schemaVersion')!r}") + failures = verify_recorded_files( + report, report_path.parent, asset_root, requested_runtime_dir + ) + for failure in failures: + print(f"error: {failure}", file=sys.stderr) + if not failures: + print("stock_asset_baseline verification: pass") + return 1 if failures else 0 + + runtime_dir = validate_runtime_dir( + requested_runtime_dir or source_root / ".install", source_root + ) + executable = find_client(runtime_dir) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%SZ") + output_dir = Path(args.output_dir).resolve() if args.output_dir else source_root / ".tmp" / "stock-baseline" / timestamp + prepare_output_directory(output_dir) + + stock_pk4s = collect_pk4s(asset_root, args.allow_asset_dir_links) + loose_files = collect_loose_asset_files(asset_root, args.allow_asset_dir_links) + runtime_files = collect_runtime_files(runtime_dir, executable) + overlay_pk4s = collect_overlay_pk4s(runtime_dir) + overlay_loose_files, preflight_failures = collect_overlay_loose_files( + runtime_dir, runtime_files + ) + try: + retail_path_collisions = collect_retail_path_collisions( + asset_root, stock_pk4s, runtime_dir, overlay_pk4s + ) + except (OSError, ValueError) as exc: + retail_path_collisions = [] + preflight_failures.append(str(exc)) + asset_failures: list[str] = [] + expected_assets_path: Path | None = None + expected_assets_sha256 = "" + directory_views = asset_directory_views(asset_root) + if loose_files: + asset_failures.append( + f"retail fallback asset root contains {len(loose_files)} loose q4base/q4mp files; " + "use a PK4-only asset view" + ) + if args.expected_assets: + expected_assets_path = Path(args.expected_assets).resolve() + expected_assets_sha256 = sha256_file(expected_assets_path) + expected_payload = json.loads(expected_assets_path.read_text(encoding="utf-8")) + if not isinstance(expected_payload, dict): + raise ValueError("expected asset manifest root is not an object") + asset_failures += compare_asset_records(asset_manifest_from_payload(expected_payload), stock_pk4s) + asset_failures += compare_file_records( + loose_manifest_from_payload(expected_payload), loose_files, "loose asset file" + ) + elif not args.dry_run: + asset_failures.append( + "passing capture requires --expected-assets from a separately generated PK4-only manifest" + ) + plans = prepare_plans( + runtime_dir, asset_root, output_dir, args.width, args.height, args.mp_port + ) + + if asset_failures or preflight_failures: + results: list[dict[str, Any]] = [] + elif args.dry_run: + results = [ + {"role": plan.role_id, "mode": plan.mode, "status": "planned", "failures": [], "artifacts": []} + for plan in plans.values() + ] + else: + results = run_capture( + runtime_dir, + executable, + output_dir, + plans, + args.timeout, + args.mp_client_delay, + ) + + passed = bool(results) and all(result["status"] == "pass" for result in results) + status = ( + "planned" + if args.dry_run and not asset_failures and not preflight_failures + else ("pass" if passed and not asset_failures and not preflight_failures else "fail") + ) + payload = { + "schemaVersion": SCHEMA_VERSION, + "generatedUtc": utc_now(), + "status": status, + "dryRun": args.dry_run, + "git": git_state(source_root), + "mpPort": args.mp_port, + "runtimeRoot": str(runtime_dir), + "host": { + "platform": platform.platform(), + "architecture": platform.machine(), + }, + "runtimeFiles": runtime_files, + "expectedAssets": { + "supplied": expected_assets_path is not None, + "path": str(expected_assets_path) if expected_assets_path is not None else "", + "sha256": expected_assets_sha256, + }, + "assets": { + "compatibilityModel": ASSET_COMPATIBILITY_MODEL, + "root": str(asset_root), + "allowLinkedGameDirectories": args.allow_asset_dir_links, + "directoryViews": directory_views, + "stockPk4s": stock_pk4s, + "looseFiles": loose_files, + "openQ4OverlayPk4s": overlay_pk4s, + "openQ4OverlayLooseFiles": overlay_loose_files, + "retailArchiveBytesMatchExpected": ( + expected_assets_path is not None and not asset_failures + ), + "retailPathNamespaceUntouched": not retail_path_collisions, + "retailPathCollisionCount": len(retail_path_collisions), + "retailPathCollisions": retail_path_collisions, + }, + "assetComparisonFailures": asset_failures, + "preflightFailures": preflight_failures, + "postCaptureVerificationFailures": [], + "safety": { + "windowedOnly": True, + "borderless": False, + "windowSize": {"width": args.width, "height": args.height}, + "engineScreenshotOnly": True, + "operatingSystemCapture": False, + "inputInjection": False, + }, + "plan": [plan_record(plan) for plan in plans.values()], + "results": results, + } + if status == "pass": + # Re-hash the retail assets, staged runtime/overlays, and every captured + # artifact before declaring success. This catches a launch that writes + # through an incorrectly mounted package path as well as any concurrent + # or stale-file mutation during the run. + post_capture_failures = verify_recorded_files( + payload, output_dir, asset_root, runtime_dir + ) + payload["postCaptureVerificationFailures"] = post_capture_failures + if post_capture_failures: + status = "fail" + payload["status"] = status + report_json, report_md, asset_json, runtime_json = write_reports(output_dir, payload) + print(f"wrote {report_json}") + print(f"wrote {report_md}") + print(f"wrote {asset_json}") + print(f"wrote {runtime_json}") + for failure in asset_failures: + print(f"error: {failure}", file=sys.stderr) + for failure in preflight_failures: + print(f"error: {failure}", file=sys.stderr) + for failure in payload["postCaptureVerificationFailures"]: + print(f"error: {failure}", file=sys.stderr) + if args.dry_run: + return 1 if asset_failures or preflight_failures else 0 + return 0 if status == "pass" else 1 + + +if __name__ == "__main__": + try: + raise SystemExit(main(sys.argv[1:])) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + raise SystemExit(1) from None From a454178bd7dd8545fd3a9125ace803170d8ad82c Mon Sep 17 00:00:00 2001 From: themuffinator Date: Wed, 19 Aug 2026 23:48:00 +0100 Subject: [PATCH 02/35] modernization: complete milestone A foundation --- .github/workflows/commit-validation.yml | 6 + .github/workflows/push-verification.yml | 6 + docs/dev/engine-capability-matrix.md | 4 +- docs/dev/gl-renderer-modernization.md | 6 +- docs/dev/idtech5-modernization-roadmap.md | 127 +- docs/dev/parallel-job-system.md | 141 +++ docs/dev/release-completion.md | 7 + docs/dev/releases/v0.12.0.md | 4 + docs/dev/renderer-validation-matrix.md | 87 +- docs/dev/stock-asset-baseline.md | 56 +- meson.build | 28 + meson_options.txt | 2 +- src/framework/Common.cpp | 157 +++ src/framework/ParallelJobSystem.cpp | 1045 +++++++++++++++++ src/framework/ParallelJobSystem.h | 203 ++++ src/framework/Session.cpp | 3 + src/idlib/precompiled.h | 13 + src/renderer/GpuFrameTimingCore.h | 116 ++ src/renderer/RenderModuleAPI.h | 4 +- src/renderer/RenderSystem.cpp | 17 + src/renderer/RenderSystem.h | 33 + src/renderer/RenderSystem_init.cpp | 34 +- src/renderer/RendererBenchmarks.cpp | 214 +++- src/renderer/RendererBenchmarks.h | 8 + src/renderer/RendererMetrics.cpp | 355 +++++- src/renderer/RendererMetrics.h | 14 + src/renderer/Vulkan/VulkanDevice.cpp | 10 +- src/renderer/Vulkan/VulkanDevice.h | 1 + src/renderer/Vulkan/VulkanGpuFrameTiming.cpp | 195 +++ src/renderer/Vulkan/VulkanGpuFrameTiming.h | 17 + src/renderer/Vulkan/vk_Backend.cpp | 2 + src/renderer/Vulkan/vk_GuiExecutor.cpp | 29 +- src/renderer/draw_common.cpp | 6 +- src/renderer/tr_local.h | 4 +- tools/build/meson_sources.py | 13 + tools/tests/cmdargs_append_contract.py | 7 +- tools/tests/native/GpuFrameTimingTest.cpp | 72 ++ tools/tests/native/ParallelJobSystemTest.cpp | 514 ++++++++ tools/tests/network_ipv4_support.py | 6 +- tools/tests/parallel_job_system.py | 157 +++ tools/tests/renderer_budget_contract.py | 593 ++++++++++ tools/tests/renderer_gameplay_benchmark.py | 743 +++++++++++- tools/tests/renderer_gpu_frame_timing.py | 189 +++ tools/tests/renderer_mp_flat_items.py | 2 +- tools/tests/renderer_picmip_policy.py | 4 +- tools/tests/renderer_validation_matrix.py | 38 +- .../renderer_vulkan_shadow_compatibility.py | 2 + ..._vulkan_world_interaction_compatibility.py | 16 + tools/tests/stock_asset_baseline.py | 217 +++- tools/validation/openq4_validate.py | 3 + tools/validation/renderer_budget_contract.py | 407 +++++++ .../validation/renderer_per_map_budgets.json | 24 + tools/validation/stock_asset_baseline.py | 220 +++- 53 files changed, 5989 insertions(+), 192 deletions(-) create mode 100644 docs/dev/parallel-job-system.md create mode 100644 src/framework/ParallelJobSystem.cpp create mode 100644 src/framework/ParallelJobSystem.h create mode 100644 src/renderer/GpuFrameTimingCore.h create mode 100644 src/renderer/Vulkan/VulkanGpuFrameTiming.cpp create mode 100644 src/renderer/Vulkan/VulkanGpuFrameTiming.h create mode 100644 tools/tests/native/GpuFrameTimingTest.cpp create mode 100644 tools/tests/native/ParallelJobSystemTest.cpp create mode 100644 tools/tests/parallel_job_system.py create mode 100644 tools/tests/renderer_budget_contract.py create mode 100644 tools/tests/renderer_gpu_frame_timing.py create mode 100644 tools/validation/renderer_budget_contract.py create mode 100644 tools/validation/renderer_per_map_budgets.json diff --git a/.github/workflows/commit-validation.yml b/.github/workflows/commit-validation.yml index aa1d1e21..b8b26ecf 100644 --- a/.github/workflows/commit-validation.yml +++ b/.github/workflows/commit-validation.yml @@ -164,13 +164,16 @@ jobs: tools/tests/openq4_pure_pack.py \ tools/tests/p0_governance_evidence.py \ tools/tests/packaging_safety.py \ + tools/tests/parallel_job_system.py \ tools/tests/preprocessor_macro_safety.py \ tools/tests/posix_memory_management.py \ tools/tests/posix_monotonic_time.py \ tools/tests/posix_network_resolution.py \ tools/tests/posix_thread_shutdown.py \ tools/tests/release_tooling_safety.py \ + tools/tests/renderer_budget_contract.py \ tools/tests/renderer_cel_shading.py \ + tools/tests/renderer_gpu_frame_timing.py \ tools/tests/renderer_mp_flat_items.py \ tools/tests/renderer_pbr_materials.py \ tools/tests/renderer_msaa_cvar_safety.py \ @@ -297,13 +300,16 @@ jobs: python tools/tests/openq4_pure_pack.py python tools/tests/p0_governance_evidence.py python tools/tests/packaging_safety.py + python tools/tests/parallel_job_system.py python tools/tests/preprocessor_macro_safety.py python tools/tests/release_tooling_safety.py python tools/tests/posix_memory_management.py python tools/tests/posix_monotonic_time.py python tools/tests/posix_network_resolution.py python tools/tests/posix_thread_shutdown.py + python tools/tests/renderer_budget_contract.py python tools/tests/renderer_cel_shading.py + python tools/tests/renderer_gpu_frame_timing.py python tools/tests/renderer_mp_flat_items.py python tools/tests/renderer_pbr_materials.py python tools/tests/renderer_msaa_cvar_safety.py diff --git a/.github/workflows/push-verification.yml b/.github/workflows/push-verification.yml index bb969efc..31c1db54 100644 --- a/.github/workflows/push-verification.yml +++ b/.github/workflows/push-verification.yml @@ -164,13 +164,16 @@ jobs: tools/tests/openq4_pure_pack.py \ tools/tests/p0_governance_evidence.py \ tools/tests/packaging_safety.py \ + tools/tests/parallel_job_system.py \ tools/tests/preprocessor_macro_safety.py \ tools/tests/posix_memory_management.py \ tools/tests/posix_monotonic_time.py \ tools/tests/posix_network_resolution.py \ tools/tests/posix_thread_shutdown.py \ tools/tests/release_tooling_safety.py \ + tools/tests/renderer_budget_contract.py \ tools/tests/renderer_cel_shading.py \ + tools/tests/renderer_gpu_frame_timing.py \ tools/tests/renderer_mp_flat_items.py \ tools/tests/renderer_pbr_materials.py \ tools/tests/renderer_msaa_cvar_safety.py \ @@ -297,13 +300,16 @@ jobs: python tools/tests/openq4_pure_pack.py python tools/tests/p0_governance_evidence.py python tools/tests/packaging_safety.py + python tools/tests/parallel_job_system.py python tools/tests/preprocessor_macro_safety.py python tools/tests/release_tooling_safety.py python tools/tests/posix_memory_management.py python tools/tests/posix_monotonic_time.py python tools/tests/posix_network_resolution.py python tools/tests/posix_thread_shutdown.py + python tools/tests/renderer_budget_contract.py python tools/tests/renderer_cel_shading.py + python tools/tests/renderer_gpu_frame_timing.py python tools/tests/renderer_mp_flat_items.py python tools/tests/renderer_pbr_materials.py python tools/tests/renderer_msaa_cvar_safety.py diff --git a/docs/dev/engine-capability-matrix.md b/docs/dev/engine-capability-matrix.md index dd842edf..a7c216c6 100644 --- a/docs/dev/engine-capability-matrix.md +++ b/docs/dev/engine-capability-matrix.md @@ -36,7 +36,7 @@ Any change that moves a row between these states must update this file and its e | Render demos and multiview demos | **Implemented** | Render-demo record/playback and server multiview recording/playback exist with versioned compatibility checks. | [`Session.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/Session.cpp), [`MultiViewDemo.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/async/MultiViewDemo.cpp), [`demo_playback.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/demo_playback.py), [`multiview_demo.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/multiview_demo.py) | | Fixed 60 Hz simulation with high-refresh presentation | **Implemented** | Authoritative simulation remains 60 Hz while presentation/interpolation and frame-pacing diagnostics support higher display rates. This does not change save/demo/network cadence. | [`Session.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/Session.cpp), [`renderer_gameplay_benchmark.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_gameplay_benchmark.py) | | Dedicated server | **Implemented** | Headless dedicated builds avoid client renderer/BSE presentation and have stock-map smoke coverage. | [`dedicated.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/sys/linux/dedicated.cpp), [`linux_dedicated_stock_map_smoke.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/linux_dedicated_stock_map_smoke.py) | -| Background job system for general engine work | **Missing** | There is no portable idTech 5-style dependency/job-list substrate for renderer, animation, streaming, or archive work. Existing local worker uses do not provide that contract. | [`tr_local.h`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/tr_local.h), [idTech 5-level roadmap](idtech5-modernization-roadmap.md) | +| Background job system for general engine work | **Implemented** | Client, tool, and dedicated builds own one portable service with bounded list/job/dependency admission, sleepable waits, low/normal/high priority aging, dependency ordering, cooperative cancellation, deterministic inline fallback, shutdown joining, and observable counters. Dedicated builds default inline, and `jobs_enable 0` executes rather than drops work. Current-build validation has identical jobs-on/off storage1 TGA and game-state evidence, clean jobs-on/off OpenGL and jobs-on Vulkan repeated-map campaigns, and five deterministic synchronous dedicated exits. No async asset or renderer consumer has migrated yet, and clean final-package recapture remains required for release promotion. | [`ParallelJobSystem.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/framework/ParallelJobSystem.cpp), [`ParallelJobSystemTest.cpp`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/native/ParallelJobSystemTest.cpp), [portable job-system contract](parallel-job-system.md) | | Pipelined async asset streaming and learned preload manifests | **Missing** | Level image loading and most PK4/decode/upload work remain synchronous; cache features do not yet form a cancellable read→decompress→decode→upload pipeline. | [`ImageManager.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ImageManager.cpp), [level-load cache](../user/level-load-cache.md), [idTech 5-level roadmap](idtech5-modernization-roadmap.md) | ## Rendering @@ -48,6 +48,8 @@ Any change that moves a row between these states must update this file and its e | Modern visible lighting ownership | **Experimental** | Interaction capability coverage exists, but exact math parity is not proven; ambient, fog, blend, stage-condition/color, and deform coverage still block production ownership. The current proven-domain count is zero, so ARB2 owns lit stock frames. | [Modern visible-lighting ownership](plans/2026-08-16-modern-visible-lighting-ownership.md), [renderer validation](renderer-validation-matrix.md) | | GPU-driven GL submission / clustered Forward+ | **Experimental** | SSBO/compute/MDI, clustered-lighting, Hi-Z, and persistent/DSA paths exist at capable tiers, but remain opt-in and depend on the incomplete modern-visible path. | [`ModernClusteredLighting.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernClusteredLighting.cpp), [`ModernGLSubmitPlan.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernGLSubmitPlan.cpp), [`ModernGLExecutor.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/ModernGLExecutor.cpp) | | Vulkan renderer | **Experimental** | Broad stock material, interaction, decal, GUI, MD5R, and shadow support exists and can reach gameplay. It remains non-default while cubemap render targets, soft particles, debug tooling, SMP, custom programs, and other long-tail parity gaps remain. | [`vk_Backend.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Vulkan/vk_Backend.cpp), [`renderer_vulkan_world_interaction_compatibility.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_vulkan_world_interaction_compatibility.py), [`renderer_vulkan_shadow_compatibility.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_vulkan_shadow_compatibility.py) | +| Backend-neutral whole-frame GPU timing | **Implemented** | OpenGL uses a delayed four-slot timestamp ring and Vulkan resolves per-slot timestamps only after its existing frame fence retires. Renderer ABI v9 exposes one common microsecond sample with backend/frame/generation identity and cumulative availability, drop, and reset diagnostics; map/device/context discontinuities invalidate old generations. Benchmark capture pairs unique valid GPU frames with high-resolution whole-renderer CPU samples without current-frame query waits. | [`GpuFrameTimingCore.h`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/GpuFrameTimingCore.h), [`RendererMetrics.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/RendererMetrics.cpp), [`VulkanGpuFrameTiming.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Vulkan/VulkanGpuFrameTiming.cpp), [`renderer_gpu_frame_timing.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_gpu_frame_timing.py) | +| Replay-verifiable per-map CPU/GPU budgets | **Implemented** (locally validated; release promotion pending) | A versioned contract selects exact map/backend/profile rows, requires independent CPU/GPU samples and percentiles, and fails closed on missing GPU timing, identity drift, threshold failure, changed contracts/runtimes/artifacts, or replay mismatch. Promotion captures also bind an exact bordered-window 1280x720 display contract so archived settings cannot change the measured workload. Gameplay reports can bind either GL or Vulkan; the fixed retail baseline binds OpenGL SP plus pure, auto-joined MP roles. A schema-10 four-role stock capture and replay pass, current-build storage/campaign evidence exercises both timing backends, and the final immutable development runtime passes and replay-verifies all eight OpenGL and all eight Vulkan required-profile cases. Clean committed-source and final-package capture plus platform/driver qualification remain open. The v1 20/28 ms values are initial target ceilings, not universal performance claims. | [`renderer_per_map_budgets.json`](https://github.com/themuffinator/openQ4/blob/master/tools/validation/renderer_per_map_budgets.json), [`renderer_budget_contract.py`](https://github.com/themuffinator/openQ4/blob/master/tools/validation/renderer_budget_contract.py), [renderer validation](renderer-validation-matrix.md), [retail baseline](stock-asset-baseline.md) | | Shadow maps | **Experimental** | Projected/point maps, CSM, cutout handling, caching, debug views, and stencil fallback exist, but `r_useShadowMap` remains opt-in/default-off pending complete promotion evidence. | [Shadow mapping](../user/shadow-mapping.md), [`Interaction.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Interaction.cpp), [`renderer_gameplay_benchmark.py`](https://github.com/themuffinator/openQ4/blob/master/tools/tests/renderer_gameplay_benchmark.py) | | Baked light grids | **Experimental** | Bake, packed atlas, visibility/distance moments, portal-aware sampling, streaming controls, and worker-assisted baking exist. They require generated per-map data and are not a stock-asset default. | [Light grids](../user/light-grids.md), [`RenderWorld_lightgrid.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/RenderWorld_lightgrid.cpp), [`draw_common.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_common.cpp) | | SMAA post-process anti-aliasing | **Implemented** | Supported post-AA path for current renderers; it remains the compatibility/low-cost choice for future temporal work. | [`draw_common.cpp`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/draw_common.cpp), [`material_smaa_edge.frag`](https://github.com/themuffinator/openQ4/blob/master/src/renderer/Vulkan/shaders/material_smaa_edge.frag) | diff --git a/docs/dev/gl-renderer-modernization.md b/docs/dev/gl-renderer-modernization.md index 6d65c964..36a27c21 100644 --- a/docs/dev/gl-renderer-modernization.md +++ b/docs/dev/gl-renderer-modernization.md @@ -257,7 +257,9 @@ Use `rendererUploadSelfTest` to run the ring, allocator, and static-buffer tests The metrics layer records front-end time, visibility time, scene-packet build time, render-graph build time, submit time, back-end time, present/swap time, view/entity/light counts, draw/surface/vertex/index counts, upload bytes, buffer stalls, upload-stream high-water/overflow data, scene-packet counts, packet material/resource/geometry/instance coverage, packet category and sort-key validation counters, packet-driven render-graph counts, modern-executor preparation coverage, modern shader-library readiness, modern draw-plan coverage, modern submit-plan readiness, modern clustered-light readiness, modern deferred-resolve readiness, modern forward+ readiness, modern visible-frame readiness, and selected renderer tier. -`r_rendererGpuTimers 1` samples GL timer queries when `r_rendererMetrics` is enabled and the driver exposes timer-query support. Samples are resolved on a delayed, nonblocking path; unavailable results are reported as `not-sampled` or dropped instead of stalling the CPU. Detail mode reports resolved GPU timing for the current compatibility backend command categories: +`r_rendererGpuTimers 1` enables a backend-neutral whole-frame timestamp sample on supported OpenGL and Vulkan renderers. OpenGL resolves a four-slot timestamp ring only after both query objects report availability; Vulkan reads a reused slot only after its ordinary frame fence has already retired and never requests `VK_QUERY_RESULT_WAIT_BIT`. Renderer ABI v9 reports validity, backend, source frame, reset generation, microseconds, frame latency, and cumulative resolved/unavailable/dropped/reset counters. Map loads and renderer/context/device discontinuities invalidate the old generation. The benchmark capture window records high-resolution `BeginFrame` to `EndFrame` CPU microseconds and counts each delayed GPU `(generation, frame)` only once in `OPENQ4_FRAME_TIMING_V1`. + +OpenGL detail mode continues to report the compatibility backend command categories when `r_rendererMetrics` is enabled: - 3D views - 2D/GUI views @@ -267,7 +269,7 @@ The metrics layer records front-end time, visibility time, scene-packet build ti - buffer switches - swap/present -Use `rendererGpuTimerSelfTest` to verify live timer-query support. `gfxInfo` reports whether renderer GPU timers are available and whether the cvar is enabled. +Use `rendererGpuTimerSelfTest` to verify live timestamp support without forcing completion. `gfxInfo` reports backend support, latest validity/frame/generation/microseconds/latency, cumulative availability/drop/reset counters, and the no-extra-wait contract. Metrics now also include the front-end scene-packet stream, resource-backed render graph, graph resource owner, modern GL executor path, modern clustered-light data model, deferred-lite resolve bridge, clustered forward+ bridge, visible-frame composition bridge, modern GL state-cache counters, and benchmark capture summaries. Completed `RenderWorld` views emit `ScenePacket`, `PassPacket`, and `DrawPacket` records after portal/area/scissor culling, surface extraction, special-effect surface submission, subview generation, and draw-surface sorting. Full-screen GUI views emit the same packet contract from the GUI model builder. Draw packets carry legacy sort keys, material records, first bump/diffuse/specular stage images where available, geometry counts, scissor data, shader-register availability, and cache availability. The old ARB2 command-stream translator remains as a backend fallback for direct legacy command flushes, but normal frames report `packets=frontend` in `r_rendererMetrics`. The render graph now preserves ordered packet-pass nodes and attaches explicit virtual resources such as `sceneColor`, `sceneDepth`, G-buffer attachments, `deferredLight`, `hybridSceneColor`, `postA`, `backBuffer`, and imported light-grid/cluster data. It records per-pass read/write/clear/resolve/invalidate/present edges, transient/imported resource counts, first/last resource lifetimes, and aliasable transient groups while ARB2 still owns visible pass execution by default. When `r_rendererModernExecutor 1` is enabled on a GL 3.3+ capable tier, the modern executor consumes that packet/graph data, keeps a starter VAO plus frame-constants UBO alive, validates the internal shader-library variants, builds a draw plan with program selections and state-batch counts, derives a submit plan from vertex/index cache state, builds CPU clustered-light records and UBOs for the hybrid-lighting milestones, can resolve the G-buffer subset into graph-owned deferred lighting, can submit graph-backed clustered forward+ opaque/alpha/transparent side-path draws, can compose a controlled modern visible frame from `deferredLight` and `sceneColor` through graph-owned `hybridSceneColor`, updates the frame UBO once per backend frame through the shared `GLStateCache`, and reports prepared pass/draw/plan/submit/cluster/deferred/forward+/visible-frame coverage. When `r_rendererModernSubmit 1` is also enabled, the executor issues diagnostic GL 3.3 draw calls before the legacy backend runs while masking color/depth writes so ARB2 remains the visible renderer. diff --git a/docs/dev/idtech5-modernization-roadmap.md b/docs/dev/idtech5-modernization-roadmap.md index 895e9383..20336331 100644 --- a/docs/dev/idtech5-modernization-roadmap.md +++ b/docs/dev/idtech5-modernization-roadmap.md @@ -77,18 +77,22 @@ when a status differs or a narrower qualification is needed. | Stock-compatibility and security foundation | **Implemented** | Protocol 2.41 preservation, pure-MP game-module containment, bounded malformed-input handling with immediate session teardown, challenge entropy, rcon2, private-CVar redaction/remote authority, HTTP(S)-only transfer policy, source-provenance auditing, archived MP auto-join test policy, and the four-role retail-PK4 evidence harness are present. Release promotion still requires a clean source pair, final-package capture, and retained human review. | | Audited BFG-lineage image, sound, and idlib work | **Implemented** | The existing 37-file BFG inventory is tracked with source lineage and Additional Terms. Further imports must update the same manifest and notices. | | PBR material authoring/resource foundation | **Implemented foundation; visible capability missing** | Namespaced parsing, typed color/data image usage, classic ARB2 fallbacks, scene-packet metadata, resource-table diagnostics, and fail-closed exclusion from unsupported modern-visible paths cover Phases 0-3 of the PBR plan. PBR shaders, direct lighting, visible ownership, IBL, and specular probes do not exist yet. | -| GPU measurement and dynamic resolution | **Partial** | OpenGL has a delayed four-frame non-blocking timer-query ring and the engine has manual render scaling. A backend-neutral full-frame result, Vulkan timestamp ring, automatic controller, discontinuity resets, and promotion evidence remain Milestones A/E. | -| General job system | **Planned** | Local workers do not provide a bounded dependency/job-list substrate. Milestone A begins with portable sleepable synchronization, cancellation, deterministic synchronous execution, and dedicated-server-safe ownership. | +| GPU measurement and dynamic resolution | **Partial** | OpenGL and Vulkan publish the same delayed, non-blocking whole-frame microsecond result through renderer ABI v9, including frame/generation identity and availability/drop/reset counters. Map loads, context/device changes, swapchain recreation, capture discontinuities, and shutdown invalidate the timing generation. High-resolution CPU and unique GPU samples feed the versioned benchmark marker, and the gameplay/stock tools enforce and replay exact map/backend/profile CPU/GPU budgets under a fixed bordered-window 1280x720 promotion contract. Current-build storage, repeated-map, and complete required-profile captures have exercised both timing backends; all eight OpenGL and all eight Vulkan cases pass and replay-verify. The initial target rows still need release-candidate/platform qualification before they support universal performance claims. The automatic controller remains Milestone E. | +| General job system | **Implemented foundation** | The engine-owned [portable bounded job service](parallel-job-system.md) provides sleepable workers and waits, bounded list/job/dependency admission, low/normal/high priority aging, dependency ordering, cooperative cancellation, deterministic inline execution, metrics, and dedicated-safe lifecycle ownership. Threaded and synchronous native coverage passes. Current-build stock validation also produced identical jobs-on/off storage1 screenshots and game state, completed jobs-on/off OpenGL plus jobs-on Vulkan repeated-map campaigns with clean shutdown markers, and recorded five deterministic synchronous dedicated-server exits. No production loading or renderer consumer has migrated yet; clean final-package recapture remains a separate release-promotion gate. | | Generated caches, streaming, and learned preload manifests | **Partial** | Binary images and generated-animation patterns exist, but model/world/collision caches and a cancellable read -> decompress -> decode -> upload pipeline do not. Retail PK4 resolution remains authoritative. | | Shared renderer contracts and GPU skinning | **Partial** | Scene packets, resource tables, upload infrastructure, and CPU skinning provide inputs, but there is no backend-neutral material/pass IR or supported joint-buffer/GPU deformation path. CPU deformation remains authoritative. | | Modern classic-frame ownership | **Experimental** | Render-graph, modern OpenGL submission, clustered/MDI infrastructure, shadow maps, light grids, and Vulkan coverage exist, but no complete stock visible-lighting domain is promoted. ARB2 remains the supported/default owner. | | Temporal presentation | **Planned** | Complete motion vectors, history ownership, TAA/TAAU, reactive/disocclusion handling, and dynamic-resolution integration are absent. SMAA remains the compatibility path. | | Modern PBR lighting and idTech 6-like follow-ons | **Planned** | GGX/IBL, reflection probes, clustered decals/probes, froxel volumetrics, SSR/SSGI, GPU-driven visible ownership, and optional sparse residency all remain after the shared-contract and temporal gates. | -The practical next target is **Milestone A**. It unlocks safe parallel loading, -cache generation, renderer-front-end work, and trustworthy GPU-budget feedback -without changing stock content interpretation. The PBR Phase 0-3 foundation is -intentionally not a reason to skip ahead to visible PBR lighting. +Milestone A's implementation and local integration gate are complete. The next +recommended implementation target is **Milestone B**, beginning with +immutable-input loading and cache consumers. Release qualification remains a +separate track: repeat and retain the Milestone A acceptance set from clean +committed source and a freshly staged final package, with the required platform +and driver coverage. Later renderer-front-end consumers still require the +module boundary described below. The PBR Phase 0-3 foundation is intentionally +not a reason to skip ahead to visible PBR lighting. ## Best official Doom 3 BFG candidates @@ -99,7 +103,7 @@ snapshot. |---|---|---|---|---| | Parallel job substrate | `idlib/ParallelJobList.*`, `idlib/Thread.*`, renderer consumers in `tr_frontend_addmodels.cpp` and `tr_frontend_addlights.cpp` | A bounded, dependency-aware worker pool for renderer front-end work, archive/decode jobs, animation work, and cache generation | Adapt architecture; replace platform primitives and spin waits with SDL3/portable C++, and retain deterministic single-thread fallback | **P1** | | GPU skeletal skinning | `renderer/BufferObject.*`, `VertexCache.*`, `Model_md5.cpp`, `tr_frontend_addmodels.cpp`, `tr_backend_draw.cpp`, `RenderProgs*` | Joint-buffer uploads and optional four-weight GPU deformation for rendered MD5/MD5R draw surfaces and shadow-map casters while preserving CPU consumers | Port the algorithm into dedicated backend-neutral skin attributes and buffers; do not import BFG's GL backend or blindly reuse its vertex-color packing | **P1** | -| GPU timing and automatic resolution scaling | `renderer/ResolutionScale.*`, the timer query in `RenderSystem.cpp`, CPU profiling blocks in `RenderLog.*` | Feed a backend-neutral full-frame result from openQ4's existing non-blocking GL query ring and a new Vulkan timestamp path into a bounded controller | Reuse the controller logic, not BFG's single-query blocking readback | **P1** | +| GPU timing and automatic resolution scaling | `renderer/ResolutionScale.*`, the timer query in `RenderSystem.cpp`, CPU profiling blocks in `RenderLog.*` | Feed openQ4's implemented backend-neutral, non-blocking GL/Vulkan whole-frame timing result into a bounded controller | Reuse the controller logic, not BFG's single-query blocking readback | **P1** | | Generated model, render-world, and collision caches | `renderer/Model.cpp`, `Model_md5.cpp`, `ModelManager.cpp`, `RenderWorld_load.cpp`, `cm/CollisionModel_files.cpp` | Cache parsed static/MD5 geometry, `.proc` world data, and collision data after first trusted-source load | Design a hardened openQ4 format; follow the generated-animation cache contract and include Quake 4 MD5R/source-PK4 identity | **P1** | | Preload manifests | `framework/File_Manifest.*` and resource-type discovery in `FileSystem.cpp` | Record actual per-map image/model/animation/sample/collision use and replay it through a cancellable preload queue | Reuse the manifest concept, not BFG's retail manifest contents | **P1** | | Resource containers | `framework/File_Resource.*` | Optional developer-generated, sequential cache containers for derived data | Reuse the access-order concept only; BFG uses 32-bit offsets and trusted tables, so prefer individual cache files or a new bounded 64-bit format and never require a BFG `.resources` package | **P2** | @@ -118,19 +122,26 @@ boundary. Its real BFG renderer users are deliberately coarse: add visible models, add lights, and build shadow work. That is a better starting point than spawning ad-hoc threads throughout openQ4. -The API should be adapted, not copied blindly: - -- back it with SDL3 threads/condition variables or a small portable C++ core; -- replace BFG's spinning `Wait()` behavior and fixed platform processing-unit - assumptions with blocking waits and explicit worker limits; -- make cancellation and shutdown explicit; -- make job payload ownership and lifetime visible in the type/API contract; -- provide a synchronous implementation used by dedicated builds, tests, and - deterministic debugging; -- bound queues and allocations; report saturation instead of silently growing; -- collect queue, execution, wait, and critical-path timings; -- prohibit renderer API calls from arbitrary workers unless the backend - explicitly owns that queue. +The landed openQ4 service adapts that architecture rather than copying it: + +- portable C++ threads and condition variables provide bounded workers and + blocking waits instead of BFG's spinning `Wait()` behavior and fixed + processing-unit assumptions; +- cancellation, shutdown, payload ownership, and lifetime are explicit in the + service contract; +- deterministic synchronous mode is available for dedicated builds, tests, + and debugging; +- list, job, and dependency admission is bounded, and saturation is reported + instead of growing or dropping work silently; +- queue, execution, wait, high-water, rejection, and starvation-aging metrics + are observable; dependency critical-path aggregation remains for real + consumer graphs; +- arbitrary worker-side renderer calls remain prohibited unless a future + backend-owned queue defines that boundary. + +The production contract, controls, saturation behavior, native coverage, and +remaining consumer/promotion boundary are documented in the +[portable job-system guide](parallel-job-system.md). First consumers should be work that already has a clean join point: learned preload discovery, image decode/transcode, generated-cache writes, and then @@ -201,14 +212,17 @@ BFG's resolution controller is compact and readily adaptable. It lowers resolution quickly when GPU time exceeds a threshold and raises it more slowly after several under-budget frames, avoiding constant oscillation. openQ4 already has render scaling, renderer metrics, high-refresh presentation, and a -four-frame, non-blocking GL timer-query ring. What is missing is a -backend-neutral total-frame timing result, an equivalent Vulkan timestamp path, -the feedback controller, and complete promotion evidence. +four-slot, non-blocking GL timestamp ring. Milestone A exposes that ring +as a backend-neutral whole-frame result and provides the equivalent Vulkan +timestamp-query path. Both backends resolve only retired/available slots, reset +their generation at workload discontinuities, and feed high-resolution CPU plus +de-duplicated GPU samples into `OPENQ4_FRAME_TIMING_V1`. What remains is the +feedback controller and complete promotion evidence. -The production version should improve on the 2012 implementation: +The future controller should build on this implemented timing foundation: -- extend the existing delayed GL query ring and add a Vulkan timestamp-query - ring; never wait on the current frame's result; +- preserve the non-blocking GL/Vulkan timestamp contract and never wait on a + current-frame result; - target a user/display frame budget and account for VRR; - quantize dimensions to backend-friendly alignments; - expose minimum scale, response rate, and a conservative default-off rollout; @@ -295,24 +309,65 @@ temporal or PBR work multiplies the parity surface. | Milestone | Current state | Dependency that prevents promotion | |---|---|---| -| A. Foundation and measurement | **Partial** | The GL timing ring exists; the portable job substrate, backend-neutral timing, Vulkan timestamps, and recorded budgets do not. | +| A. Foundation and measurement | **Implemented and locally validated; release promotion pending** | The portable bounded job substrate, backend-neutral delayed GL/Vulkan whole-frame timing, and versioned, replay-verifiable per-map CPU/GPU budget tooling are implemented. Current-build jobs-on/off parity, repeated map-change shutdown, deterministic dedicated exits, schema-10 stock capture/replay, and complete replay-verified 8/8 OpenGL plus 8/8 Vulkan required profiles have passed. Promotion still requires the same evidence retained from clean committed source and a freshly staged final package, plus release platform/driver qualification. | | B. Loading and cache modernization | **Partial** | Existing binary-image/generated-animation patterns do not yet form learned manifests, bounded pipeline stages, or model/world/collision caches. | | C. Shared renderer contracts and GPU animation | **Partial** | Packet/resource infrastructure exists, but shared GL/Vulkan pass semantics and GPU skinning parity are missing. | | D. Modern classic-frame ownership | **Experimental** | Individual modern paths exist; no complete classic-visible domain has satisfied the cross-backend parity exit gate. | -| E. Temporal presentation | **Planned** | Depends on Milestones A, C, and D for timing, motion/resource contracts, and complete frame ownership. | +| E. Temporal presentation | **Planned** | Milestone A now supplies the timing prerequisite; incomplete Milestones C and D still block motion/resource contracts and complete frame ownership. | | F. Modern materials and advanced lighting | **Foundation only** | PBR authoring/resource Phases 0-3 exist, but visible PBR/IBL and advanced-lighting ownership must wait for Milestones C-E. | ### Milestone A: foundation and measurement -1. Land a portable bounded job manager with synchronous mode, dependency tests, +1. **Implemented:** the engine-owned portable bounded job manager provides + synchronous mode, starvation-safe priorities, dependency tests, shutdown/cancellation tests, and timing counters. -2. Expose the existing delayed GL timer ring through a backend-neutral - total-frame timing result and add the equivalent Vulkan timestamp-query ring. -3. Establish per-map CPU/GPU budgets in the existing benchmark and stock - evidence tools. - -Exit gate: identical stock screenshots/game state with jobs on/off, clean -shutdown under repeated map changes, and trustworthy non-blocking timing. +2. **Implemented:** the delayed GL timestamp result is exposed through a + backend-neutral whole-frame timing contract, and the equivalent Vulkan + timestamp path is integrated; both are generation-aware and never wait for a + current-frame result. +3. **Implemented and locally validated; release promotion pending:** enforce versioned, + configurable map/backend/profile CPU and GPU percentile budgets in the + gameplay benchmark and stock baseline; bind contract/runtime/artifact + provenance and replay measurements fail-closed. The initial repeated 20/28 + ms rows are target ceilings until complete GL/Vulkan captures calibrate and, + where justified, tighten each explicit identity. + +The 2026-08-19 current-build evidence snapshot closes the local job lifecycle +portion of this gate: + +- jobs-on and jobs-off `game/storage1` runs produced identical engine TGA bytes + and matching game-state evidence; +- jobs-on and jobs-off OpenGL campaigns, plus a jobs-on Vulkan campaign, crossed + `game/mcc_2` -> `game/storage1` -> `game/storage2` -> `game/storage1` -> + `game/tram1` and ended with + `jobsShutdown PASS v1 initialized=0 queued=0 running=0`; +- five dedicated-server runs exited normally with one synchronous self-test and + one clean shutdown marker each; +- the schema-10 four-role retail-PK4 baseline passed capture and immediate + replay under the canonical display/budget contract, and its engine screenshots + and save preview passed local human review; +- storage and repeated-map runs exercised nonblocking OpenGL and Vulkan timing; + the final immutable development runtime then passed and replay-verified all + eight OpenGL and all eight Vulkan required-profile cases. The earlier + `game/medlabs` failure was fixed by ordering and clamping depth bounds before + the OpenGL call; its debug-context rerun records zero GL errors. + +This evidence set culminated in the immutable development runtime +`milestone-a-20260819-final3`; its complete required-profile reports are +`ma-a-gl3` and `ma-a-vk3`. It came from an uncommitted current source tree and is +not a retained release artifact. It does not replace clean-source provenance, a +freshly staged final package, platform/driver qualification, or retained release +review. + +Exit gate: replay-valid exact bordered-window 1280x720 GL/Vulkan captures for +the required budget identities; identical stock screenshots and game state with +jobs on/off; repeated map changes ending in +`jobsShutdown PASS v1 initialized=0 queued=0 running=0`; deterministic +dedicated-server exit; and the general four-role retail-PK4 and human-review +promotion evidence. The current-build job, lifecycle, timing-path, required-map, +and stock-baseline checks above satisfy the local implementation gate. Only the +general clean-source, final-package, retained-review, and platform/driver gates +keep release promotion open. ### Milestone B: loading and cache modernization diff --git a/docs/dev/parallel-job-system.md b/docs/dev/parallel-job-system.md new file mode 100644 index 00000000..758ff062 --- /dev/null +++ b/docs/dev/parallel-job-system.md @@ -0,0 +1,141 @@ +# Portable Job System + +openQ4 now owns a portable bounded job service for coarse engine work. It is +available in client, tool, and dedicated builds without changing Quake 4 asset, +save, demo, or network formats. The service is a foundation for later loading, +cache, animation, and renderer-front-end consumers; no stock asset work has +been moved off-thread yet. + +The architecture was informed by id Software's GPLv3 +[Doom 3 BFG `idParallelJobList`](https://github.com/id-Software/DOOM-3-BFG/blob/1caba1979589971b5ed44e315d9ead30b278d8b4/neo/idlib/ParallelJobList.h). +The implementation is original openQ4 code. It deliberately replaces BFG's +platform-specific worker assumptions and spinning list wait with portable C++ +threads and condition variables, explicit saturation, and cooperative +cancellation. Both projects use GPLv3-compatible licensing; no BFG source file +was copied into this subsystem, so the audited BFG-lineage source-file count is +unchanged. + +## Runtime policy and lifecycle + +`idCommon` initializes the engine-owned service before commands and higher-level +runtime systems can submit work. Shutdown stops admission, requests cancellation, +waits for running callers and workers, and joins every worker before network, +session, renderer, or other subsystem teardown begins. + +The startup-only controls are: + +- `jobs_enable 1` enables threaded scheduling. Setting it to `0` never discards + work; accepted lists execute inline in deterministic insertion order. +- `jobs_deterministic 1` selects the same synchronous implementation for tests + and debugging. Dedicated builds default this setting to `1`, so they create no + background job workers unless explicitly configured otherwise; threaded + client execution requires `jobs_enable 1` with `jobs_deterministic 0`. +- `jobs_numThreads 0` reserves one hardware thread when possible and clamps the + worker count to 1-32. An explicit value selects that bounded count. +- `jobs_queueCapacity 64` bounds admitted lists and accepts values from 1-1024. + +Synchronous work participates in the same active-list and quiescence contract as +threaded work. A concurrent `CancelAll`, `WaitAll`, or shutdown therefore sees +and joins an inline list running on another submitting thread. Concurrent inline +submissions sleep until their submit sequence reaches the front rather than +spinning. + +## Submission contract + +Each `idJobList` declares fixed job and dependency capacities when it is created. +The implementation also fails closed above 1,048,576 jobs or 256 dependencies +per list. Combined with the configured active-list bound, this places a finite +upper limit on admitted work. Scheduler list and worker storage is reserved +before workers start; admission, dependency refresh, cancellation, and teardown +do not grow scheduler-owned containers. Job functions receive their payload, +worker index, and an `idJobCancellationToken`; payload ownership and lifetime +remain the caller's responsibility through the list's terminal state. + +Admission returns an explicit `idJobSubmitResult`. In particular, +`QUEUE_FULL` leaves the list in its building state so the owner can wait for its +own safe join point and retry. Work is never silently dropped or placed in an +unbounded overflow queue. Callers must handle every non-success result; the +service does not guess whether blocking, inline fallback, or cancellation is +safe for a consumer. + +Dependencies are deliberately chronological: a list may depend only on a list +already submitted to the same service. Self-dependencies, duplicate edges, +foreign-service lists, unsubmitted dependencies, and reverse edges into an +already-submitted list fail closed. This makes a dependency cycle +unrepresentable. A dependent list sleeps until all prerequisites complete and +is cancelled if a prerequisite fails or is cancelled. + +Lists have low, normal, or high priority. Higher priority wins initially, while +every skipped runnable list gains a scheduling age. At the starvation threshold, +aged lists enter a primary oldest-age selection class which ignores base +priority; equal ages use the monotonic submit sequence. A continuously runnable +low-priority list therefore runs after a bounded number of dispatches even when +nine or more earlier high-priority lists remain runnable. Selection is +deterministic even though completion timing with multiple workers is necessarily +concurrent. + +Cancellation is cooperative. Jobs that have not started are accounted for and +never invoked; running jobs must poll `context.IsCancellationRequested()` and +leave their own data in a safe state. List waits, worker waits, dependency waits, +and shutdown waits all use condition variables. There is no spin or sleep-poll +loop in the scheduler. + +## Diagnostics and validation + +`jobsStats` prints current admission, queue high-water, running/sleeping worker, +completion, rejection, cancellation, failure, wake, priority-promotion, +execution-time, and wait-time counters. Per-list status and timing counters are +also available through the C++ API. + +`jobsSelfTest` exercises engine-service execution, dependency ordering, batch +work, and counters in either threaded or synchronous mode. Automation should +require the exact versioned success prefix: + +```text +jobsSelfTest PASS v1 +``` + +Engine shutdown emits this exact clean-quiescence marker only after all accepted +work has reached a terminal state and every worker has joined: + +```text +jobsShutdown PASS v1 initialized=0 queued=0 running=0 +``` + +The focused native test covers deterministic inline execution, worker +parallelism, fixed capacities, retryable saturation, dependency and failure +propagation, self/cycle rejection, cooperative cancellation, low-priority aging, +threaded shutdown, and cancellation/join of an inline job submitted from another +thread. `tools/tests/parallel_job_system.py` keeps the production lifecycle, +public contract, build registration, and documentation wired into the default +validation suite. + +## Consumer boundary and promotion evidence + +The first consumers should have immutable inputs, caller-owned result storage, +and an explicit main-thread join point: learned preload discovery, decode or +transcode stages, and generated-cache preparation are suitable candidates. +Archive mutation, live game-state mutation, and renderer API calls from arbitrary +workers are not. + +The current light-grid CPU integration lives inside a dynamically loaded renderer +module. That module does not import the engine's `jobSystem` through +`RenderModuleAPI`, so wiring it directly would create an unresolved/duplicate +service boundary and unclear shutdown ownership. Light-grid work should adopt +the service only after a narrow renderer job import or engine-owned work packet +contract is designed; it is not a safe first consumer in this change. + +Because no production consumer has migrated yet, `jobs_enable 1` and +`jobs_enable 0` execute the same stock workload outside the self-test. The +2026-08-19 current-build validation closed the local Milestone A lifecycle gate: +jobs-on and jobs-off `game/storage1` produced identical engine TGA bytes and +matching game state; both OpenGL modes and jobs-on Vulkan completed the repeated +`game/mcc_2` -> `game/storage1` -> `game/storage2` -> `game/storage1` -> +`game/tram1` campaign with clean shutdown markers; and five dedicated-server +runs each exited normally with exactly one synchronous self-test and one clean +shutdown marker. + +Those runs used an immutable development runtime built from an uncommitted +source tree. Release promotion still requires the same evidence to be retained +from clean source and a freshly staged final package. Multiplayer evidence must +retain `ui_autoJoin 1` unless the join flow itself is under test. diff --git a/docs/dev/release-completion.md b/docs/dev/release-completion.md index f2232c6b..0454ee63 100644 --- a/docs/dev/release-completion.md +++ b/docs/dev/release-completion.md @@ -36,6 +36,8 @@ is `docs/dev/macos-moltenvk-decision.md`. ## Ready For Changelog +- [x] openQ4 now has a portable job service for future parallel loading, cache, animation, and renderer-front-end work without changing stock assets or compatibility formats. It bounds admitted lists and per-list work, supports dependency ordering and starvation-safe low/normal/high priorities, exposes cooperative cancellation and diagnostics, and uses sleepable waits instead of spinning. Dedicated servers default to deterministic inline execution, and `jobs_enable 0` still executes every accepted job instead of dropping it. The foundation is integrated and native-tested; production renderer and asset consumers plus retained jobs-on/off gameplay evidence remain follow-up promotion work. +- [x] Renderer performance capture now uses one backend-neutral whole-frame GPU timing contract on OpenGL and Vulkan. Both paths resolve delayed timestamps without adding a current-frame wait, invalidate old generations across map/session and renderer/device discontinuities, and expose validity plus availability/drop/reset diagnostics through renderer ABI v9. `rendererBenchmarkCapture` pairs unique resolved GPU frames with high-resolution whole-renderer CPU samples in the versioned budget marker; target-hardware GL/Vulkan evidence is still required before performance promotion claims. - [x] Internet-server administration and package negotiation now fail closed at their remaining legacy edges. `rcon2` keeps remote-console passwords out of packets, applies bounded challenge and reply budgets, and redacts private settings from console output, history, journals, configuration serialization, command expansion, and completion previews; plaintext rcon is disabled unless both relevant sides explicitly opt into it. Server-originated userinfo and synchronized-CVar dictionaries decode transactionally and can update only the CVar class owned by that wire message, so they cannot borrow authority over private or unrelated settings. Server-provided redirects and PK4 entries accept only bounded HTTP or HTTPS URLs with a syntactically valid DNS, IPv4, or bracketed IPv6 host. Standard Meson packages keep in-process direct PK4 transfer disabled while preserving validated web-redirect prompts; a separately integrated curl-enabled build additionally rechecks the URL, refuses redirects, applies connect/stall/total-duration limits, and retains package path, size, and checksum validation. The obsolete network-driven executable updater no longer downloads or runs code and ignores server-controlled release text and links. - [x] Pure multiplayer can remain enabled without giving a server control over executable code. `si_pure 1` is no longer silently disabled; the ordered asset-PK4 list is still enforced, while the protocol 2.41 game-code field uses the stock 1.4.2 `game300.pk4` checksum only as a platform-independent compatibility token for an already loaded module from trusted local openQ4 package/module roots. Missing or unexpected tokens, modules, asset lists, and platform IDs fail closed, code-bearing server mods require the explicit `net_serverAllowServerMod 1` opt-in, and neither pure negotiation nor package download can install or restart into game code. The token is not a cryptographic measurement of the loaded module, a module-equality proof, or an anti-cheat guarantee. - [x] Truncated and malformed multiplayer state now fails at an explicit safety boundary. Bit-message underflow is tracked across normal and delta reads, queued messages and delta user commands are length-checked, and audited SP/MP leaf readers stage decoded fields until their payload is valid while validating referenced slots, types, ranges, and remaining data. Invalid covered traffic is dropped; a late malformed top-level snapshot tears down the affected session before another game or presentation frame. The legacy entity lifecycle is not a whole-snapshot transaction, and this targeted hardening does not claim formal verification of every parser. @@ -906,6 +908,11 @@ is `docs/dev/macos-moltenvk-decision.md`. - [x] Accessibility: text can be given a solid black backing so it stays readable over the world and over busy panel artwork. `gui_textBackground` sets how opaque the backing is (0 is off, 1 is fully opaque; around 0.75 keeps the artwork visible while making text easy to read) and `gui_textBackgroundPadding` controls how far it extends past the text. It covers menu, HUD and in-game GUI text, applies immediately with no restart, and works with either the scalable or the original bitmap fonts. The backing is drawn once per line, so it never shows through the gaps between characters, and every line of a given font gets the same height whatever characters it contains. - [x] A texture quality control in the style of Quake 3's `r_picmip` is available for players who need to trade sharpness for memory and bandwidth. `image_picmip` drops whole mip levels, so every step halves a texture no matter how large it started, but unlike the original it only reduces the **diffuse layer** of a material — normal maps, specular maps, lighting, skies, decals, fonts, and the whole HUD and menu keep their authored resolution, so surfaces stay correctly lit and the interface stays sharp. `image_picmipFilter` restricts it to world surfaces (the default), models, or both, `image_picmipMinSize` stops small textures from turning to mush, and a material that declares `nopicmip` opts out. Textures reload as soon as one of these changes, with no `vid_restart` needed, and the generated texture cache is keyed by the active reduction so switching settings never leaves a stale size behind. +- [x] Renderer performance evidence now binds exact stock-map, OpenGL/Vulkan backend, benchmark profile, and bordered-window 1280x720 launch state to a versioned CPU/GPU percentile contract. Gameplay and retail-baseline reports retain the selected thresholds, measured samples, runtime/artifact hashes, and source provenance, then replay them fail-closed; unavailable GPU timing, archived display drift, fullscreen/noncanonical budget runs, or changed evidence cannot pass. The initial 20/28 ms rows are target ceilings rather than universal measured claims. +- [x] Milestone A's current-build integration gate has exercised identical jobs-on/off storage output, clean OpenGL and Vulkan repeated-map job shutdown, five deterministic dedicated exits, delayed whole-frame timing on both backends, a replay-valid schema-10 four-role retail baseline with local human image review, and replay-verified 8/8 OpenGL plus 8/8 Vulkan required profiles from the final immutable development runtime. This validates the development build, not a release package or universal performance level. +- [x] OpenGL depth-bounds submission now orders and clamps both values to the legal range before calling the driver, eliminating the two `GL_INVALID_VALUE` records that previously made `game/medlabs` the lone required-profile failure; its debug-context rerun is error-free. +- [ ] Finish Milestone A release qualification by repeating and retaining the full job, renderer-budget, retail-baseline, and human-review evidence from clean committed source and a freshly staged final package, then recording the required release platform/driver coverage. + ## macOS Evidence Gate Complete this section before release notes claim macOS support beyond the current experimental Apple Silicon/arm64 status. diff --git a/docs/dev/releases/v0.12.0.md b/docs/dev/releases/v0.12.0.md index 38ee2bb2..adc51b45 100644 --- a/docs/dev/releases/v0.12.0.md +++ b/docs/dev/releases/v0.12.0.md @@ -47,3 +47,7 @@ - Brightened clear-water visibility, removed hall-like underwater reverb tails, and debounced rapid surface sounds without suppressing their splash effects. - Added localized drowning, slime, and lava obituaries with distinct graphical death-feed icons and normal burn feedback for slime. - Strengthened the experimental modern OpenGL lighting path with per-light image ownership and missing-binding fallbacks. +- Added a bounded, portable background-job foundation with deterministic fallback, cooperative cancellation, dependency ordering, starvation protection, and clean-shutdown diagnostics. Current-build stock checks produced identical jobs-on/off output and clean repeated-map and dedicated exits; stock gameplay remains unchanged until individual workloads are deliberately migrated. +- Added non-blocking whole-frame GPU timing for OpenGL and Vulkan, together with backend-neutral high-resolution renderer CPU timing. Delayed results avoid forcing the GPU to idle, reset safely across renderer, map, and capture transitions, and have been exercised on both backends in current-build gameplay. +- Added replayable per-map renderer budget contracts and exact bordered-window 1280x720 evidence checks for both gameplay benchmarks and the stock-asset baseline. Current-build required profiles pass on all eight OpenGL and all eight Vulkan roles. The checked-in limits are initial target ceilings rather than a performance guarantee; clean final-package and platform qualification remain separate release gates. +- Prevented invalid OpenGL depth-bounds values in effects-heavy scenes by ordering and clamping the submitted range, making the `game/medlabs` validation run error-free. diff --git a/docs/dev/renderer-validation-matrix.md b/docs/dev/renderer-validation-matrix.md index cf61761a..6f653aeb 100644 --- a/docs/dev/renderer-validation-matrix.md +++ b/docs/dev/renderer-validation-matrix.md @@ -1,6 +1,6 @@ # Renderer Validation Matrix -This matrix is the validation source of truth for the staged GL renderer work. It separates safe automated startup/self-test coverage from gameplay smoke coverage that must be run manually with the mode-specific SP/MP launch tasks. +This matrix is the validation source of truth for staged renderer work. The safe tier and self-test matrix remains GL-focused, while replayable budget evidence covers both OpenGL and Vulkan. Supervised gameplay can use the mode-specific SP/MP launch tasks or the noninteractive gameplay harness described below. For cross-engine feature status, use the [engine capability matrix](engine-capability-matrix.md). This document owns renderer acceptance evidence and promotion gates; it does not turn an experimental renderer capability into a supported/default one by itself. @@ -180,6 +180,56 @@ These are manual long-run sign-off loops. They are intentionally outside the saf | `modern` | 16 ms | 24 ms | 100% | 8x6x16 | 96 | 64 | 1024 px / every frame | 2 | | `high-end` | 12 ms | 18 ms | 100% | 8x6x16 | 128 | 96 | 2048 px / every frame | 3 | +These preset values are renderer workload/scalability defaults, not per-map +measurements. Promotion evidence uses the separate, versioned +`tools/validation/renderer_per_map_budgets.json` contract. Each row is selected +by the exact active map, launch-derived backend (`opengl` or `vulkan`), and +benchmark profile, and carries minimum sample counts plus independent CPU and +whole-frame GPU P95/P99 ceilings in integer microseconds. The initial v1 rows +deliberately apply the baseline 20/28 ms target to every listed stock scene on +both backends. They are cross-map target ceilings, not a claim that every row +was measured at those values; a captured report retains each row's actual +samples and percentiles so confirmed target-machine results can tighten a row +without changing the schema. + +`rendererBenchmarkCapture` supplies the budget tools with one backend-neutral +line of this exact shape: + +```text +OPENQ4_FRAME_TIMING_V1 map=game/storage1 backend=opengl profile=baseline cpuSamples=256 cpuP50Us=7000 cpuP95Us=11000 cpuP99Us=14000 gpuAvailable=1 gpuSamples=252 gpuP50Us=5000 gpuP95Us=8000 gpuP99Us=10000 +``` + +CPU and GPU percentiles must be monotonic. GPU values come from nonblocking +whole-frame backend timestamps rather than a sum of overlapping pass timers. +An unsupported/unresolved backend reports `gpuAvailable=0`, zero samples, and +`-1` for all three GPU percentiles; because promotion rows require GPU timing, +that representation fails closed. The verifier rejects malformed lines, +multiple markers in one source, conflicting mirrored sources, identity or +sample-count mismatches, missing exact budget rows, exceeded ceilings, changed +contract hashes, altered runtime/artifact bytes, and recorded measurements that +do not recompute from the retained log streams. + +### Milestone A current-build evidence snapshot + +The 2026-08-19 development snapshot verifies that the implementation works as +an integrated system, but it is not final-package or universal performance +evidence: + +| Gate | Current development-build result | +|---|---| +| Job parity | PASS: jobs-on and jobs-off `game/storage1` produced identical engine TGA bytes and matching game-state evidence. | +| Repeated lifecycle | PASS: jobs-on/off OpenGL and jobs-on Vulkan completed the five-map campaign through `game/tram1`; each run ended with zero initialized, queued, or running jobs. | +| Dedicated lifecycle | PASS: five independent dedicated-server runs exited normally, each with one synchronous job self-test and one clean shutdown marker. | +| Backend timing | PASS for the exercised storage and campaign captures: OpenGL and Vulkan returned delayed whole-frame GPU samples without a current-frame wait. | +| Retail baseline | PASS locally: the schema-10 four-role OpenGL capture and its immediate replay retained pure MP, `ui_autoJoin 1`, canonical display/budget evidence, artifacts, and source/runtime identities; engine screenshots and the save preview also passed local human review. | +| Required map budgets | PASS locally: the immutable `milestone-a-20260819-final3` runtime passes and replay-verifies all eight OpenGL cases in `ma-a-gl3` and all eight Vulkan cases in `ma-a-vk3`. The corrected `game/medlabs` debug-context run records zero GL errors after depth bounds are ordered and clamped before submission. | + +The renderer sweep uses the immutable current-build runtime named above, staged +from an uncommitted development tree. Promotion still requires clean committed +source provenance, a freshly staged final package, retained reports/artifacts +and release review, and separate platform/driver qualification. The local 8/8 +results do not establish a universal performance level. + ## Manual Gameplay Matrix Gameplay validation remains mandatory before renderer release sign-off, but it is not run by the safe matrix by default because map loads need target-hardware supervision. Use the SP launch task for single-player maps, the MP launch task or `tools\debug\start_listen_server_client.ps1` for multiplayer, or the opt-in gameplay benchmark harness below when you want a repeatable logged capture set. @@ -209,26 +259,51 @@ After each gameplay smoke, inspect the configured log file under `fs_savepath\