drm/asahi: Handle channel errors

Signed-off-by: Asahi Lina <lina@asahilina.net>

Author: hoshinolina

drivers/gpu/drm/asahi/channel.rs

@@ -392,6 +392,31 @@ impl EventChannel::ver {
                                 )
                             }
                         },
+                        EventMsg::ChannelError {
+                            error_type,
+                            pipe_type,
+                            event_slot,
+                            event_value,
+                        } => match self.gpu.as_ref() {
+                            Some(gpu) => {
+                                let error_type = match error_type {
+                                    0 => ChannelErrorType::MemoryError,
+                                    1 => ChannelErrorType::DMKill,
+                                    2 => ChannelErrorType::Aborted,
+                                    3 => ChannelErrorType::Unk3,
+                                    a => ChannelErrorType::Unknown(a),
+                                };
+                                gpu.handle_channel_error(
+                                    error_type,
+                                    pipe_type,
+                                    event_slot,
+                                    event_value,
+                                );
+                            }
+                            None => {
+                                dev_crit!(self.dev, "EventChannel: No GPU manager available!\n")
+                            }
+                        },
                         msg => {
                             dev_crit!(self.dev.as_ref(), "Unknown event message: {:?}\n", msg);
                         }

drivers/gpu/drm/asahi/debug.rs

@@ -66,6 +66,7 @@ pub(crate) enum DebugFlags {
     Debug6 = 54,
     Debug7 = 55,
 
+    VerboseFaults = 61,
     AllowUnknownOverrides = 62,
     OopsOnGpuCrash = 63,
 }

drivers/gpu/drm/asahi/event.rs

@@ -216,6 +216,16 @@ impl EventManager {
         }
     }
 
+    /// Returns a reference to the workqueue owning an event.
+    pub(crate) fn get_owner(
+        &self,
+        slot: u32,
+    ) -> Option<Arc<dyn workqueue::WorkQueue + Send + Sync>> {
+        self.alloc
+            .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned())
+            .map(|a| a.clone())
+    }
+
     /// Fail all commands, used when the GPU crashes.
     pub(crate) fn fail_all(&self, error: workqueue::WorkError) {
         let mut owners: KVec<Arc<dyn workqueue::WorkQueue + Send + Sync>> = KVec::new();

drivers/gpu/drm/asahi/fw/channels.rs

@@ -184,8 +184,16 @@ pub(crate) enum DeviceControlMsg {
         halt_count: U64,
         __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x1c }>,
     },
-    Unk0e(Array<DEVICECONTROL_SZ::ver, u8>),
-    Unk0f(Array<DEVICECONTROL_SZ::ver, u8>),
+    RecoverChannel {
+        pipe_type: u32,
+        work_queue: GpuWeakPointer<super::workqueue::QueueInfo::ver>,
+        event_value: u32,
+        __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x10 }>,
+    },
+    IdlePowerOff {
+        val: u32,
+        __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x4 }>,
+    },
     Unk10(Array<DEVICECONTROL_SZ::ver, u8>),
     Unk11(Array<DEVICECONTROL_SZ::ver, u8>),
     Unk12(Array<DEVICECONTROL_SZ::ver, u8>),
@@ -236,6 +244,17 @@ pub(crate) struct FwCtlMsg {
 
 pub(crate) const EVENT_SZ: usize = 0x34;
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[repr(C, u32)]
+#[allow(dead_code)]
+pub(crate) enum ChannelErrorType {
+    MemoryError,
+    DMKill,
+    Aborted,
+    Unk3,
+    Unknown(u32),
+}
+
 #[derive(Debug, Copy, Clone)]
 #[repr(C, u32)]
 #[allow(dead_code)]
@@ -258,12 +277,19 @@ pub(crate) enum EventMsg {
         vm_slot: u32,
         buffer_slot: u32,
         counter: u32,
-    }, // Max discriminant: 0x7
+    },
+    ChannelError {
+        error_type: u32,
+        pipe_type: u32,
+        event_slot: u32,
+        event_value: u32,
+    },
+    // Max discriminant: 0x8
 }
 
 static_assert!(core::mem::size_of::<EventMsg>() == 4 + EVENT_SZ);
 
-pub(crate) const EVENT_MAX: u32 = 0x7;
+pub(crate) const EVENT_MAX: u32 = 0x8;
 
 #[derive(Copy, Clone)]
 #[repr(C)]

drivers/gpu/drm/asahi/fw/workqueue.rs

@@ -135,7 +135,7 @@ pub(crate) mod raw {
         #[ver(V >= V13_2 && G < G14X)]
         pub(crate) unk_84_0: u32,
         pub(crate) unk_84_state: AtomicU32,
-        pub(crate) unk_88: u32,
+        pub(crate) error_count: AtomicU32,
         pub(crate) unk_8c: u32,
         pub(crate) unk_90: u32,
         pub(crate) unk_94: u32,

drivers/gpu/drm/asahi/gpu.rs

@@ -34,7 +34,7 @@ use kernel::{
 use crate::alloc::Allocator;
 use crate::debug::*;
 use crate::driver::{AsahiDevRef, AsahiDevice};
-use crate::fw::channels::PipeType;
+use crate::fw::channels::{ChannelErrorType, PipeType};
 use crate::fw::types::{U32, U64};
 use crate::{
     alloc, buffer, channel, event, fw, gem, hw, initdata, mem, mmu, queue, regs, workqueue,
@@ -255,6 +255,14 @@ pub(crate) trait GpuManager: Send + Sync {
     fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32);
     /// Handle a GPU fault event.
     fn handle_fault(&self);
+    /// Handle a channel error event.
+    fn handle_channel_error(
+        &self,
+        error_type: ChannelErrorType,
+        pipe_type: u32,
+        event_slot: u32,
+        event_value: u32,
+    );
     /// Acknowledge a Buffer grow op.
     fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32);
     /// Wait for the GPU to become idle and power off.
@@ -1344,6 +1352,82 @@ impl GpuManager for GpuManager::ver {
         self.recover();
     }
 
+    fn handle_channel_error(
+        &self,
+        error_type: ChannelErrorType,
+        pipe_type: u32,
+        event_slot: u32,
+        event_value: u32,
+    ) {
+        dev_err!(self.dev, " (\\________/) \n");
+        dev_err!(self.dev, "  |        |  \n");
+        dev_err!(self.dev, "'.| \\  , / |.'\n");
+        dev_err!(self.dev, "--| / (( \\ |--\n");
+        dev_err!(self.dev, ".'|  _-_-  |'.\n");
+        dev_err!(self.dev, "  |________|  \n");
+        dev_err!(self.dev, "GPU channel error nya~!!!!!\n");
+        dev_err!(self.dev, "  Error type: {:?}\n", error_type);
+        dev_err!(self.dev, "  Pipe type: {}\n", pipe_type);
+        dev_err!(self.dev, "  Event slot: {}\n", event_slot);
+        dev_err!(self.dev, "  Event value: {:#x?}\n", event_value);
+
+        self.event_manager.mark_error(
+            event_slot,
+            event_value,
+            workqueue::WorkError::ChannelError(error_type),
+        );
+
+        let wq = match self.event_manager.get_owner(event_slot) {
+            Some(wq) => wq,
+            None => {
+                dev_err!(self.dev, "Workqueue not found for this event slot!\n");
+                return;
+            }
+        };
+
+        let wq = match wq.as_any().downcast_ref::<workqueue::WorkQueue::ver>() {
+            Some(wq) => wq,
+            None => {
+                dev_crit!(self.dev, "GpuManager mismatched with WorkQueue!\n");
+                return;
+            }
+        };
+
+        if debug_enabled(DebugFlags::VerboseFaults) {
+            wq.dump_info();
+        }
+
+        let dc = fw::channels::DeviceControlMsg::ver::RecoverChannel {
+            pipe_type,
+            work_queue: wq.info_pointer(),
+            event_value,
+            __pad: Default::default(),
+        };
+
+        mod_dev_dbg!(self.dev, "Recover Channel command: {:?}\n", &dc);
+        let mut txch = self.tx_channels.lock();
+
+        let token = txch.device_control.send(&dc);
+        {
+            let mut guard = self.rtkit.lock();
+            let rtk = guard.as_mut().unwrap();
+            if rtk
+                .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)
+                .is_err()
+            {
+                dev_err!(self.dev, "Failed to send Recover Channel command\n");
+            }
+        }
+
+        if txch.device_control.wait_for(token).is_err() {
+            dev_err!(self.dev, "Timed out waiting for Recover Channel command\n");
+        }
+
+        if debug_enabled(DebugFlags::VerboseFaults) {
+            wq.dump_info();
+        }
+    }
+
     fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32) {
         let halt_count = self
             .initdata

drivers/gpu/drm/asahi/workqueue.rs

@@ -14,13 +14,14 @@
 //! up its associated event.
 
 use crate::debug::*;
-use crate::fw::channels::PipeType;
+use crate::fw::channels::{ChannelErrorType, PipeType};
 use crate::fw::types::*;
 use crate::fw::workqueue::*;
 use crate::no_debug;
 use crate::object::OpaqueGpuObject;
 use crate::regs::FaultReason;
 use crate::{channel, driver, event, fw, gpu, object, regs};
+use core::any::Any;
 use core::num::NonZeroU64;
 use core::sync::atomic::Ordering;
 use kernel::{
@@ -47,6 +48,8 @@ pub(crate) enum WorkError {
     Fault(regs::FaultInfo),
     /// Work failed due to an error caused by other concurrent GPU work.
     Killed,
+    /// Channel error
+    ChannelError(ChannelErrorType),
     /// The GPU crashed.
     NoDevice,
     /// Unknown reason.
@@ -78,6 +81,9 @@ impl From<WorkError> for uapi::drm_asahi_result_info {
                 status: match a {
                     WorkError::Timeout => uapi::drm_asahi_status_DRM_ASAHI_STATUS_TIMEOUT,
                     WorkError::Killed => uapi::drm_asahi_status_DRM_ASAHI_STATUS_KILLED,
+                    WorkError::ChannelError(_) => {
+                        uapi::drm_asahi_status_DRM_ASAHI_STATUS_CHANNEL_ERROR
+                    }
                     WorkError::NoDevice => uapi::drm_asahi_status_DRM_ASAHI_STATUS_NO_DEVICE,
                     _ => uapi::drm_asahi_status_DRM_ASAHI_STATUS_UNKNOWN_ERROR,
                 },
@@ -96,6 +102,7 @@ impl From<WorkError> for kernel::error::Error {
             WorkError::Unknown => ENODATA,
             WorkError::Killed => ECANCELED,
             WorkError::NoDevice => ENODEV,
+            WorkError::ChannelError(_) => EIO,
         }
     }
 }
@@ -600,20 +607,26 @@ impl WorkQueue::ver {
         size: u32,
     ) -> Result<Arc<WorkQueue::ver>> {
         let gpu_buf = alloc.private.array_empty_tagged(0x2c18, b"GPBF")?;
-        let shared = &mut alloc.shared;
+        let mut state = alloc.shared.new_default::<RingState>()?;
+        let ring = alloc.shared.array_empty(size as usize)?;
         let inner = WorkQueueInner::ver {
             dev: dev.into(),
             event_manager,
-            info: alloc.private.new_init(
+            // Use shared (coherent) state with verbose faults so we can dump state correctly
+            info: if debug_enabled(DebugFlags::VerboseFaults) {
+                &mut alloc.shared
+            } else {
+                &mut alloc.private
+            }
+            .new_init(
                 try_init!(QueueInfo::ver {
                     state: {
-                        let mut s = shared.new_default::<RingState>()?;
-                        s.with_mut(|raw, _inner| {
+                        state.with_mut(|raw, _inner| {
                             raw.rb_size = size;
                         });
-                        s
+                        state
                     },
-                    ring: shared.array_empty(size as usize)?,
+                    ring,
                     gpu_buf,
                     notifier_list: notifier_list,
                     gpu_context: gpu_context,
@@ -638,7 +651,7 @@ impl WorkQueue::ver {
                         #[ver(V >= V13_2 && G < G14X)]
                         unk_84_0: 0,
                         unk_84_state: Default::default(),
-                        unk_88: 0,
+                        error_count: Default::default(),
                         unk_8c: 0,
                         unk_90: 0,
                         unk_94: 0,
@@ -743,18 +756,46 @@ impl WorkQueue::ver {
     pub(crate) fn pipe_type(&self) -> PipeType {
         self.inner.lock().pipe_type
     }
+
+    pub(crate) fn dump_info(&self) {
+        pr_info!("WorkQueue @ {:?}:", self.info_pointer);
+        self.inner.lock().info.with(|raw, _inner| {
+            pr_info!("  GPU rptr1: {:#x}", raw.gpu_rptr1.load(Ordering::Relaxed));
+            pr_info!("  GPU rptr1: {:#x}", raw.gpu_rptr2.load(Ordering::Relaxed));
+            pr_info!("  GPU rptr1: {:#x}", raw.gpu_rptr3.load(Ordering::Relaxed));
+            pr_info!("  Event ID: {:#x}", raw.event_id.load(Ordering::Relaxed));
+            pr_info!("  Busy: {:#x}", raw.busy.load(Ordering::Relaxed));
+            pr_info!("  Unk 84: {:#x}", raw.unk_84_state.load(Ordering::Relaxed));
+            pr_info!(
+                "  Error count: {:#x}",
+                raw.error_count.load(Ordering::Relaxed)
+            );
+            pr_info!("  Pending: {:#x}", raw.pending.load(Ordering::Relaxed));
+        });
+    }
+
+    pub(crate) fn info_pointer(&self) -> GpuWeakPointer<QueueInfo::ver> {
+        self.info_pointer
+    }
 }
 
 /// Trait used to erase the version-specific type of WorkQueues, to avoid leaking
 /// version-specificity into the event module.
 pub(crate) trait WorkQueue {
+    /// Cast as an Any type.
+    fn as_any(&self) -> &dyn Any;
+
     fn signal(&self) -> bool;
     fn mark_error(&self, value: event::EventValue, error: WorkError);
     fn fail_all(&self, error: WorkError);
 }
 
 #[versions(AGX)]
 impl WorkQueue for WorkQueue::ver {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
     /// Signal a workqueue that some work was completed.
     ///
     /// This will check the event stamp value to find out exactly how many commands were processed.