rust: drm: gpuvm: Switch to DRM_GPUVM_IMMEDIATE_MODE

DRM_GPUVM_IMMEDIATE_MODE allows for deferred gpuva unlink and gpuvm bo
release.

Signed-off-by: Janne Grunau <j@jannau.net>

Author: jannau

rust/helpers/drm_gpuvm.c

@@ -25,6 +25,11 @@ struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo)
 	return drm_gpuvm_bo_get(vm_bo);
 }
 
+bool rust_helper_drm_gpuvm_immediate_mode(struct drm_gpuvm *gpuvm)
+{
+	return drm_gpuvm_immediate_mode(gpuvm);
+}
+
 bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj)
 {
 	return drm_gpuvm_is_extobj(gpuvm, obj);

rust/kernel/drm/gpuvm.rs

@@ -8,7 +8,7 @@
 
 use crate::{
     bindings, drm,
-    drm::device,
+    drm::{device, gem::BaseObject},
     error::{
         code::{EINVAL, ENOMEM},
         from_result, to_result, Error, Result,
@@ -177,7 +177,9 @@ impl<T: DriverGpuVm> OpMap<T> {
                 return Err(Pin::new_unchecked(KBox::from_raw(p)));
             };
             // SAFETY: This takes a new reference to the gpuvmbo.
+            gpuvmbo.lock_gpuva();
             bindings::drm_gpuva_link(&mut p.gpuva, &gpuvmbo.bo as *const _ as *mut _);
+            gpuvmbo.unlock_gpuva();
         }
         Ok(())
     }
@@ -194,6 +196,12 @@ impl<T: DriverGpuVm> OpUnMap<T> {
         Some(unsafe { &*p })
     }
     pub fn unmap_and_unlink_va(&mut self) -> Option<Pin<KBox<GpuVa<T>>>> {
+        self.do_unmap_and_unlink_va(false)
+    }
+    pub fn unmap_and_unlink_va_defer(&mut self) -> Option<Pin<KBox<GpuVa<T>>>> {
+        self.do_unmap_and_unlink_va(true)
+    }
+    fn do_unmap_and_unlink_va(&mut self, defer: bool) -> Option<Pin<KBox<GpuVa<T>>>> {
         if self.0.va.is_null() {
             return None;
         }
@@ -203,7 +211,11 @@ impl<T: DriverGpuVm> OpUnMap<T> {
         // SAFETY: The GpuVa object reference is valid per the op_unmap contract
         unsafe {
             bindings::drm_gpuva_unmap(&mut self.0);
-            bindings::drm_gpuva_unlink(self.0.va);
+            if defer {
+                bindings::drm_gpuva_unlink_defer(self.0.va);
+            } else {
+                bindings::drm_gpuva_unlink(self.0.va);
+            }
         }
 
         // Unlinking/unmapping relinquishes ownership of the GpuVa object,
@@ -287,6 +299,20 @@ impl<T: DriverGpuVm> GpuVmBo<T> {
     pub fn inner(&self) -> &T::GpuVmBo {
         &self.inner
     }
+    /// Lock the GpuVmBo's gem boject gpuva lock
+    pub fn lock_gpuva(&self) {
+        unsafe {
+            let lock = &raw mut (*self.bo.obj).gpuva.lock;
+            bindings::mutex_lock(lock);
+        }
+    }
+    /// Unlock the GpuVmBo's gem boject gpuva lock
+    pub fn unlock_gpuva(&self) {
+        unsafe {
+            let lock = &raw mut (*self.bo.obj).gpuva.lock;
+            bindings::mutex_unlock(lock);
+        }
+    }
 }
 
 // SAFETY: DRM GpuVmBo objects are always reference counted and the get/put functions
@@ -302,10 +328,9 @@ unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVmBo<T> {
         // The drm_gpuvm_put function satisfies the requirements for dec_ref().
         // (We do not support custom locks yet.)
         unsafe {
-            let resv = (*obj.as_mut().bo.obj).resv;
-            bindings::dma_resv_lock(resv, core::ptr::null_mut());
+            obj.as_mut().lock_gpuva();
             bindings::drm_gpuvm_bo_put(&mut obj.as_mut().bo);
-            bindings::dma_resv_unlock(resv);
+            obj.as_mut().unlock_gpuva();
         }
     }
 }
@@ -414,7 +439,7 @@ pub(super) unsafe extern "C" fn step_remap_callback<T: DriverGpuVm>(
 
     // SAFETY: We incremented the refcount above, and the Rust reference we took is
     // no longer in scope.
-    unsafe { bindings::drm_gpuvm_bo_put(p_vm_bo) };
+    unsafe { bindings::drm_gpuvm_bo_put_deferred(p_vm_bo) };
 
     res
 }
@@ -461,6 +486,7 @@ impl<T: DriverGpuVm> GpuVm<T> {
 
     pub fn new<E>(
         name: &'static CStr,
+        flags: bindings::drm_gpuvm_flags,
         dev: &device::Device<T::Driver>,
         r_obj: ARef<<<T::Driver as drm::Driver>::Object as BaseDriverObject>::Object>,
         range: Range<u64>,
@@ -480,7 +506,7 @@ impl<T: DriverGpuVm> GpuVm<T> {
                         bindings::drm_gpuvm_init(
                             Opaque::cast_into(slot),
                             name.as_char_ptr(),
-                            0,
+                            flags,
                             dev.as_raw(),
                             r_obj.as_raw() as *const _ as *mut _,
                             range.start,
@@ -560,6 +586,66 @@ impl<T: DriverGpuVm> GpuVm<T> {
         // SAFETY: This is safe to call as long as the arguments are valid pointers.
         unsafe { bindings::drm_gpuvm_is_extobj(self.gpuvm() as *mut _, gem) }
     }
+
+    pub fn bo_deferred_cleanup(&self) {
+        unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.gpuvm() as *mut _) }
+    }
+
+    pub fn find_bo(& self,
+        obj: &<<T::Driver as drm::Driver>::Object as BaseDriverObject>::Object
+    ) -> Option<ARef<GpuVmBo<T>>> {
+        obj.lock_gpuva();
+        // SAFETY: drm_gem_object.gpuva.lock was just locked.
+        let p = unsafe {
+            bindings::drm_gpuvm_bo_find(
+                self.gpuvm() as *mut _,
+                obj.as_raw() as *const _ as *mut _,
+            )
+        };
+        obj.unlock_gpuva();
+        if p.is_null() {
+            None
+        } else {
+            // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo<T>.
+            let p = unsafe { crate::container_of!(p, GpuVmBo<T>, bo) as *mut GpuVmBo<T> };
+            // SAFETY: We checked for NULL above, and the types ensure that
+            // this object was created by vm_bo_alloc_callback<T>.
+            Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) })
+        }
+    }
+
+    pub fn obtain_bo(& self,
+        obj: &<<T::Driver as drm::Driver>::Object as BaseDriverObject>::Object) -> Result<ARef<GpuVmBo<T>>> {
+        obj.lock_gpuva();
+        // SAFETY: drm_gem_object.gpuva.lock was just locked.
+        let p = unsafe {
+            bindings::drm_gpuvm_bo_obtain(
+                self.gpuvm() as *mut _,
+                obj.as_raw() as *const _ as *mut _,
+            )
+        };
+        obj.unlock_gpuva();
+        if p.is_null() {
+            Err(ENOMEM)
+        } else {
+            // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm.
+            let p = unsafe { crate::container_of!(p, GpuVmBo<T>, bo) as *mut GpuVmBo<T> };
+            // SAFETY: We checked for NULL above, and the types ensure that
+            // this object was created by vm_bo_alloc_callback<T>.
+            Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) })
+        }
+    }
+
+    pub fn bo_unmap(& self, ctx: &mut T::StepContext, bo: &GpuVmBo<T>) -> Result {
+        let mut ctx = StepContext {
+            ctx,
+            gpuvm: self,
+        };
+        // SAFETY: LockedGpuVm implies the right locks are held.
+        to_result(unsafe {
+            bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _)
+        })
+    }
 }
 
 // SAFETY: DRM GpuVm objects are always reference counted and the get/put functions