fix(linux): skip h264_vulkan when the device has no H.264 encode queue

electron/native/pipewire-capture/src/encoder.rs

@@ -177,6 +177,98 @@ fn vaapi_is_safe_to_probe() -> bool {
     }
 }
 
+/// True when the Vulkan device behind `device` has a queue family that encodes
+/// H.264 — the exact lookup `h264_vulkan` makes in `avcodec_open2` (issue #1340).
+///
+/// WHY ASK BEFORE OPENING. A driver can advertise VK_KHR_video_encode_queue on
+/// a GPU with no H.264 encode queue (RADV on an AMD Carrizo, in the report).
+/// ffmpeg 8.1 then fails with "Encoding of h264 is not supported by this device" and
+/// segfaults while freeing the half-built encoder, so the ladder never reaches
+/// libopenh264. Asking Vulkan first turns that crash into a skipped rung.
+///
+/// Read through Vulkan itself rather than `AVVulkanDeviceContext::qf`, because
+/// that struct needs the Vulkan headers to bind and the build does not have
+/// them. Only its first four pointer-sized fields are read: `alloc`,
+/// `get_proc_addr`, `inst` and `phys_dev`, in that order in the vendored 8.1.
+///
+/// SAFETY: `device` is a live AV_HWDEVICE_TYPE_VULKAN device context.
+unsafe fn vulkan_can_encode_h264(device: *mut ff::AVBufferRef) -> bool {
+    use std::ffi::c_void;
+
+    #[repr(C)]
+    struct DeviceHead {
+        alloc: *const c_void,
+        get_proc_addr: Option<GetInstanceProcAddr>,
+        instance: *mut c_void,
+        physical_device: *mut c_void,
+    }
+    #[repr(C)]
+    struct QueueFamilyVideoProperties {
+        s_type: i32,
+        p_next: *mut c_void,
+        codec_operations: u32,
+    }
+    #[repr(C)]
+    struct QueueFamilyProperties2 {
+        s_type: i32,
+        p_next: *mut c_void,
+        queue_flags: u32,
+        queue_count: u32,
+        timestamp_valid_bits: u32,
+        min_image_transfer_granularity: [u32; 3],
+    }
+    type GetInstanceProcAddr =
+        unsafe extern "C" fn(*mut c_void, *const std::ffi::c_char) -> *const c_void;
+    type GetQueueFamilyProperties2 =
+        unsafe extern "C" fn(*mut c_void, *mut u32, *mut QueueFamilyProperties2);
+
+    const STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2: i32 = 1_000_059_005;
+    const STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR: i32 = 1_000_023_012;
+    const QUEUE_VIDEO_ENCODE_BIT_KHR: u32 = 0x40;
+    const VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: u32 = 0x0001_0000;
+
+    let context = (*device).data as *const ff::AVHWDeviceContext;
+    let head = (*context).hwctx as *const DeviceHead;
+    let Some(get_proc_addr) = (*head).get_proc_addr else {
+        return false;
+    };
+    let query = get_proc_addr(
+        (*head).instance,
+        c"vkGetPhysicalDeviceQueueFamilyProperties2".as_ptr(),
+    );
+    if query.is_null() {
+        return false;
+    }
+    let query: GetQueueFamilyProperties2 = std::mem::transmute(query);
+
+    let mut count = 0u32;
+    query((*head).physical_device, &mut count, ptr::null_mut());
+    let mut video: Vec<QueueFamilyVideoProperties> = (0..count)
+        .map(|_| QueueFamilyVideoProperties {
+            s_type: STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
+            p_next: ptr::null_mut(),
+            codec_operations: 0,
+        })
+        .collect();
+    let mut families: Vec<QueueFamilyProperties2> = video
+        .iter_mut()
+        .map(|video| QueueFamilyProperties2 {
+            s_type: STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
+            p_next: (video as *mut QueueFamilyVideoProperties).cast(),
+            queue_flags: 0,
+            queue_count: 0,
+            timestamp_valid_bits: 0,
+            min_image_transfer_granularity: [0; 3],
+        })
+        .collect();
+    query((*head).physical_device, &mut count, families.as_mut_ptr());
+
+    families.iter().zip(&video).take(count as usize).any(|(family, video)| {
+        family.queue_flags & QUEUE_VIDEO_ENCODE_BIT_KHR != 0
+            && video.codec_operations & VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR != 0
+    })
+}
+
 const RTLD_LAZY: i32 = 1;
 const RTLD_LOCAL: i32 = 0;
 
@@ -497,6 +589,9 @@ impl VideoEncoder {
                 ff::err_to_string(created)
             ));
         }
+        if device_type == ff::AV_HWDEVICE_TYPE_VULKAN && !vulkan_can_encode_h264(self.hw_device) {
+            return Err("the Vulkan device has no H.264 encode queue".to_owned());
+        }
 
         self.hw_frames = ff::av_hwframe_ctx_alloc(self.hw_device);
         if self.hw_frames.is_null() {