fix: AMD AV1 RADV support — layered DPB, converter layout, GOP clamping

src/converter/mod.rs

@@ -770,10 +770,13 @@ impl ColorConverter {
             );
 
             // Transition target image (encoder's input) to TRANSFER_DST layout.
+            // The encoder's init clears the input image and leaves it in
+            // VIDEO_ENCODE_SRC_KHR, so we must use that (not UNDEFINED) as the
+            // old layout on every frame, including the first.
             let target_barrier_to_transfer = vk::ImageMemoryBarrier::default()
-                .src_access_mask(vk::AccessFlags::empty())
+                .src_access_mask(vk::AccessFlags::MEMORY_READ | vk::AccessFlags::MEMORY_WRITE)
                 .dst_access_mask(vk::AccessFlags::TRANSFER_WRITE)
-                .old_layout(vk::ImageLayout::UNDEFINED)
+                .old_layout(vk::ImageLayout::VIDEO_ENCODE_SRC_KHR)
                 .new_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL)
                 .image(target_image)
                 .subresource_range(vk::ImageSubresourceRange {
@@ -786,7 +789,7 @@ impl ColorConverter {
 
             device.cmd_pipeline_barrier(
                 self.command_buffer,
-                vk::PipelineStageFlags::TOP_OF_PIPE,
+                vk::PipelineStageFlags::ALL_COMMANDS,
                 vk::PipelineStageFlags::TRANSFER,
                 vk::DependencyFlags::empty(),
                 &[],

src/encoder/av1/init.rs

@@ -77,8 +77,10 @@ impl Av1 {
             // array and `1 << slot` refresh mask can represent).
             max_active_refs_cap: 7,
             bitstream_buffer_size,
-            // AV1 reference handling here does not use a layered DPB.
-            allow_layered_dpb: false,
+            // Allow the shared infrastructure to fall back to a layered DPB
+            // when the driver does not support separate reference images
+            // (required for AMD RADV).
+            allow_layered_dpb: true,
         })?;
         let active_reference_count = init.active_reference_count;
         let common = init.common;

src/encoder/av1/record.rs

@@ -324,9 +324,12 @@ impl Av1 {
         all_reference_slots.extend_from_slice(&reference_slots);
 
         let is_first_frame = plan.is_first_frame();
+        // Clamp GOP values to at least 1; a value of 0 is undefined in
+        // Vulkan and causes undefined behavior on some drivers (RADV).
+        let gop_frames = common.config.gop_size.max(1);
         let mut av1_rc_info = vk::VideoEncodeAV1RateControlInfoKHR::default()
-            .gop_frame_count(common.config.gop_size)
-            .key_frame_period(common.config.gop_size)
+            .gop_frame_count(gop_frames)
+            .key_frame_period(gop_frames)
             .consecutive_bipredictive_frame_count(0)
             .temporal_layer_count(1);