From 678e7d56180edee331e7ab2639411a03a7b2dbad Mon Sep 17 00:00:00 2001 From: schlegp Date: Tue, 28 Jul 2026 08:17:49 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20AV1=20RADV=20support=20=E2=80=94=20layer?= =?UTF-8?q?ed=20DPB,=20converter=20layout,=20GOP=20clamping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes required for correct AV1 encoding on AMD RADV (and generally correct Vulkan usage): 1. Converter target image barrier always used UNDEFINED as the old layout, but the encoder's clear_input_image() already transitions the image to VIDEO_ENCODE_SRC_KHR during init. Correct the old layout and source access mask so the barrier is spec-compliant on every frame. 2. Allow the AV1 encoder to use a layered DPB image when the driver does not advertise VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES. AMD RADV requires this; without it the encode command faults the GPU (GCVM_L2_PROTECTION_FAULT / VK_ERROR_DEVICE_LOST). 3. Clamp gop_frame_count and key_frame_period to at least 1 in the AV1 rate-control info. Moonshine (and other users) set GOP size to 0 to signal "manual IDR only"; passing 0 to the driver is undefined and triggers undefined behaviour on some drivers. --- src/converter/mod.rs | 9 ++++++--- src/encoder/av1/init.rs | 6 ++++-- src/encoder/av1/record.rs | 7 +++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/converter/mod.rs b/src/converter/mod.rs index 7ddf47b..4aeae1f 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -770,10 +770,13 @@ impl ColorConverter { ); // Transition target image (encoder's input) to TRANSFER_DST layout. + // The encoder's init clears the input image and leaves it in + // VIDEO_ENCODE_SRC_KHR, so we must use that (not UNDEFINED) as the + // old layout on every frame, including the first. let target_barrier_to_transfer = vk::ImageMemoryBarrier::default() - .src_access_mask(vk::AccessFlags::empty()) + .src_access_mask(vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE) .dst_access_mask(vk::AccessFlags::TRANSFER_WRITE) - .old_layout(vk::ImageLayout::UNDEFINED) + .old_layout(vk::ImageLayout::VIDEO_ENCODE_SRC_KHR) .new_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) .image(target_image) .subresource_range(vk::ImageSubresourceRange { @@ -786,7 +789,7 @@ impl ColorConverter { device.cmd_pipeline_barrier( self.command_buffer, - vk::PipelineStageFlags::TOP_OF_PIPE, + vk::PipelineStageFlags::ALL_COMMANDS, vk::PipelineStageFlags::TRANSFER, vk::DependencyFlags::empty(), &[], diff --git a/src/encoder/av1/init.rs b/src/encoder/av1/init.rs index 559b340..b408257 100644 --- a/src/encoder/av1/init.rs +++ b/src/encoder/av1/init.rs @@ -77,8 +77,10 @@ impl Av1 { // array and `1 << slot` refresh mask can represent). max_active_refs_cap: 7, bitstream_buffer_size, - // AV1 reference handling here does not use a layered DPB. - allow_layered_dpb: false, + // Allow the shared infrastructure to fall back to a layered DPB + // when the driver does not support separate reference images + // (required for AMD RADV). + allow_layered_dpb: true, })?; let active_reference_count = init.active_reference_count; let common = init.common; diff --git a/src/encoder/av1/record.rs b/src/encoder/av1/record.rs index 02db29d..622a3d5 100644 --- a/src/encoder/av1/record.rs +++ b/src/encoder/av1/record.rs @@ -324,9 +324,12 @@ impl Av1 { all_reference_slots.extend_from_slice(&reference_slots); let is_first_frame = plan.is_first_frame(); + // Clamp GOP values to at least 1; a value of 0 is undefined in + // Vulkan and causes undefined behavior on some drivers (RADV). + let gop_frames = common.config.gop_size.max(1); let mut av1_rc_info = vk::VideoEncodeAV1RateControlInfoKHR::default() - .gop_frame_count(common.config.gop_size) - .key_frame_period(common.config.gop_size) + .gop_frame_count(gop_frames) + .key_frame_period(gop_frames) .consecutive_bipredictive_frame_count(0) .temporal_layer_count(1);