drm/asahi: Convert to GPUVM and implement more VM_BIND ops

Signed-off-by: Asahi Lina <lina@asahilina.net>

Author: hoshinolina

drivers/gpu/drm/asahi/alloc.rs

@@ -444,7 +444,7 @@ pub(crate) struct SimpleAllocation {
     ptr: Option<NonNull<u8>>,
     gpu_ptr: u64,
     size: usize,
-    vm: mmu::Vm,
+    _mapping: mmu::KernelMapping,
     obj: crate::gem::ObjectRef,
 }
 
@@ -464,7 +464,6 @@ impl Drop for SimpleAllocation {
                 vmap.as_mut_slice().fill(0x42);
             }
         }
-        self.obj.drop_vm_mappings(self.vm.id());
     }
 }
 
@@ -566,7 +565,7 @@ impl Allocator for SimpleAllocator {
         if debug_enabled(DebugFlags::FillAllocations) {
             obj.vmap()?.as_mut_slice().fill(0xde);
         }
-        let iova = obj.map_into_range(
+        let mapping = obj.map_into_range(
             &self.vm,
             self.start,
             self.end,
@@ -575,6 +574,8 @@ impl Allocator for SimpleAllocator {
             true,
         )?;
 
+        let iova = mapping.iova();
+
         let ptr = unsafe { p.add(offset) };
         let gpu_ptr = (iova + offset) as u64;
 
@@ -592,7 +593,7 @@ impl Allocator for SimpleAllocator {
             ptr: NonNull::new(ptr),
             gpu_ptr,
             size,
-            vm: self.vm.clone(),
+            _mapping: mapping,
             obj,
         })
     }
@@ -696,11 +697,10 @@ impl RawAllocation for HeapAllocation {
 struct HeapAllocatorInner {
     dev: AsahiDevRef,
     allocated: usize,
-    backing_objects: Vec<(crate::gem::ObjectRef, u64)>,
+    backing_objects: Vec<(crate::gem::ObjectRef, mmu::KernelMapping, u64)>,
     garbage: Option<Vec<mm::Node<HeapAllocatorInner, HeapAllocationInner>>>,
     total_garbage: usize,
     name: CString,
-    vm_id: u64,
 }
 
 /// A heap allocator which uses the DRM MM range allocator to manage its objects.
@@ -754,7 +754,6 @@ impl HeapAllocator {
             backing_objects: Vec::new(),
             // TODO: This clearly needs a try_clone() or similar
             name: CString::try_from_fmt(fmt!("{}", &*name))?,
-            vm_id: vm.id(),
             garbage: if keep_garbage { Some(Vec::new()) } else { None },
             total_garbage: 0,
         };
@@ -817,16 +816,18 @@ impl HeapAllocator {
         }
 
         let gpu_ptr = self.top;
-        if let Err(e) = obj.map_at(&self.vm, gpu_ptr, self.prot, self.cpu_maps) {
-            dev_err!(
-                &self.dev,
-                "HeapAllocator[{}]::add_block: Failed to map at {:#x} ({:?})\n",
-                &*self.name,
-                gpu_ptr,
-                e
-            );
-            return Err(e);
-        }
+        let mapping = obj
+            .map_at(&self.vm, gpu_ptr, self.prot, self.cpu_maps)
+            .map_err(|err| {
+                dev_err!(
+                    &self.dev,
+                    "HeapAllocator[{}]::add_block: Failed to map at {:#x} ({:?})\n",
+                    &*self.name,
+                    gpu_ptr,
+                    err
+                );
+                err
+            })?;
 
         self.mm
             .with_inner(|inner| inner.backing_objects.reserve(1, GFP_KERNEL))?;
@@ -874,8 +875,11 @@ impl HeapAllocator {
             new_top
         );
 
-        self.mm
-            .with_inner(|inner| inner.backing_objects.try_push((obj, gpu_ptr)))?;
+        self.mm.with_inner(|inner| {
+            inner
+                .backing_objects
+                .push((obj, mapping, gpu_ptr), GFP_KERNEL)
+        })?;
 
         self.top = new_top;
 
@@ -896,8 +900,8 @@ impl HeapAllocator {
             inner
                 .backing_objects
                 .binary_search_by(|obj| {
-                    let start = obj.1;
-                    let end = obj.1 + obj.0.size() as u64;
+                    let start = obj.2;
+                    let end = obj.2 + obj.0.size() as u64;
                     if start > addr {
                         Ordering::Greater
                     } else if end <= addr {
@@ -1013,7 +1017,7 @@ impl Allocator for HeapAllocator {
                 let idx = idx.unwrap_or(inner.backing_objects.len() - 1);
                 let obj = &mut inner.backing_objects[idx];
                 let p = obj.0.vmap()?.as_mut_ptr() as *mut u8;
-                Ok((obj.1, obj.0.size(), p))
+                Ok((obj.2, obj.0.size(), p))
             })?;
             assert!(obj_start <= start);
             assert!(obj_start + obj_size as u64 >= end);
@@ -1091,10 +1095,6 @@ impl Drop for HeapAllocatorInner {
                 &*self.name,
                 self.allocated
             );
-        } else {
-            for mut obj in self.backing_objects.drain(..) {
-                obj.0.drop_vm_mappings(self.vm_id);
-            }
         }
     }
 }

drivers/gpu/drm/asahi/driver.rs

@@ -52,8 +52,11 @@ impl drv::Driver for AsahiDriver {
     type Object = gem::Object;
 
     const INFO: drv::DriverInfo = INFO;
-    const FEATURES: u32 =
-        drv::FEAT_GEM | drv::FEAT_RENDER | drv::FEAT_SYNCOBJ | drv::FEAT_SYNCOBJ_TIMELINE;
+    const FEATURES: u32 = drv::FEAT_GEM
+        | drv::FEAT_RENDER
+        | drv::FEAT_SYNCOBJ
+        | drv::FEAT_SYNCOBJ_TIMELINE
+        | drv::FEAT_GEM_GPUVA;
 
     kernel::declare_drm_ioctls! {
         (ASAHI_GET_PARAMS,      drm_asahi_get_params,

drivers/gpu/drm/asahi/file.rs

@@ -30,13 +30,19 @@ struct Vm {
     ualloc: Arc<Mutex<alloc::DefaultAllocator>>,
     ualloc_priv: Arc<Mutex<alloc::DefaultAllocator>>,
     vm: mmu::Vm,
-    dummy_obj: gem::ObjectRef,
+    _dummy_mapping: mmu::KernelMapping,
 }
 
 impl Drop for Vm {
     fn drop(&mut self) {
-        // Mappings create a reference loop, make sure to break it.
-        self.dummy_obj.drop_vm_mappings(self.vm.id());
+        // When the user Vm is dropped, unmap everything in the user range
+        if self
+            .vm
+            .unmap_range(mmu::IOVA_USER_BASE as u64, VM_USER_END)
+            .is_err()
+        {
+            pr_err!("Vm::Drop: vm.unmap_range() failed\n");
+        }
     }
 }
 
@@ -154,16 +160,16 @@ const VM_SHADER_END: u64 = 0x11_ffffffff;
 /// Start address of the general user mapping region.
 const VM_USER_START: u64 = 0x20_00000000;
 /// End address of the general user mapping region.
-const VM_USER_END: u64 = 0x5f_ffffffff;
+const VM_USER_END: u64 = 0x6f_ffff0000;
 
 /// Start address of the kernel-managed GPU-only mapping region.
-const VM_DRV_GPU_START: u64 = 0x60_00000000;
+const VM_DRV_GPU_START: u64 = 0x70_00000000;
 /// End address of the kernel-managed GPU-only mapping region.
-const VM_DRV_GPU_END: u64 = 0x60_ffffffff;
+const VM_DRV_GPU_END: u64 = 0x70_ffffffff;
 /// Start address of the kernel-managed GPU/FW shared mapping region.
-const VM_DRV_GPUFW_START: u64 = 0x61_00000000;
+const VM_DRV_GPUFW_START: u64 = 0x71_00000000;
 /// End address of the kernel-managed GPU/FW shared mapping region.
-const VM_DRV_GPUFW_END: u64 = 0x61_ffffffff;
+const VM_DRV_GPUFW_END: u64 = 0x71_ffffffff;
 /// Address of a special dummy page?
 const VM_UNK_PAGE: u64 = 0x6f_ffff8000;
 
@@ -298,7 +304,7 @@ impl File {
 
         let gpu = &device.data().gpu;
         let file_id = file.inner().id;
-        let vm = gpu.new_vm(file_id)?;
+        let vm = gpu.new_vm()?;
 
         let resv = file.inner().vms().reserve()?;
         let id: u32 = resv.index().try_into()?;
@@ -343,15 +349,15 @@ impl File {
         );
         let mut dummy_obj = gem::new_kernel_object(device, 0x4000)?;
         dummy_obj.vmap()?.as_mut_slice().fill(0);
-        dummy_obj.map_at(&vm, VM_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?;
+        let dummy_mapping = dummy_obj.map_at(&vm, VM_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?;
 
         mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id);
         resv.store(Box::new(Vm {
             ualloc,
             ualloc_priv,
             vm,
-            dummy_obj,
-        })?)?;
+            _dummy_mapping: dummy_mapping,
+        }, GFP_KERNEL,)?)?;
 
         data.vm_id = id;
 
@@ -492,15 +498,10 @@ impl File {
         data: &mut uapi::drm_asahi_gem_bind,
         file: &DrmFile,
     ) -> Result<u32> {
-        if data.offset != 0 {
-            pr_err!("gem_bind: Offset not supported yet\n");
-            return Err(EINVAL); // Not supported yet
-        }
-
-        if (data.addr | data.range) as usize & mmu::UAT_PGMSK != 0 {
+        if (data.addr | data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 {
             cls_pr_debug!(
                 Errors,
-                "gem_bind: Addr/range not page aligned: {:#x} {:#x}\n",
+                "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n",
                 data.addr,
                 data.range
             );
@@ -512,12 +513,7 @@ impl File {
             return Err(EINVAL);
         }
 
-        let mut bo = gem::lookup_handle(file, data.handle)?;
-
-        if data.range != bo.size().try_into()? {
-            pr_err!("gem_bind: Partial maps not supported yet\n");
-            return Err(EINVAL); // Not supported yet
-        }
+        let bo = gem::lookup_handle(file, data.handle)?;
 
         let start = data.addr;
         let end = data.addr + data.range - 1;
@@ -590,11 +586,36 @@ impl File {
             .vm
             .clone();
 
-        bo.map_at(&vm, start, prot, true)?;
+        vm.bind_object(&bo.gem, data.addr, data.range, data.offset, prot)?;
 
         Ok(0)
     }
 
+    pub(crate) fn unbind_gem_object(file: &DrmFile, bo: &gem::Object) -> Result {
+        let mut index = 0;
+        loop {
+            let item = file
+                .inner()
+                .vms()
+                .find(index, xarray::XArray::<Box<Vm>>::MAX);
+            match item {
+                Some((idx, file_vm)) => {
+                    // Clone since we can't hold the xarray spinlock while
+                    // calling drop_mappings()
+                    let vm = file_vm.borrow().vm.clone();
+                    core::mem::drop(file_vm);
+                    vm.drop_mappings(bo)?;
+                    if idx == xarray::XArray::<Box<Vm>>::MAX {
+                        break;
+                    }
+                    index = idx + 1;
+                }
+                None => break,
+            }
+        }
+        Ok(())
+    }
+
     pub(crate) fn do_gem_unbind_all(
         _device: &AsahiDevice,
         data: &mut uapi::drm_asahi_gem_bind,
@@ -605,20 +626,18 @@ impl File {
             return Err(EINVAL);
         }
 
-        let mut bo = gem::lookup_handle(file, data.handle)?;
+        let bo = gem::lookup_handle(file, data.handle)?;
 
         if data.vm_id == 0 {
-            bo.drop_file_mappings(file.inner().id);
+            Self::unbind_gem_object(file, &bo.gem)?;
         } else {
-            let vm_id = file
-                .inner()
+            file.inner()
                 .vms()
                 .get(data.vm_id.try_into()?)
                 .ok_or(ENOENT)?
                 .borrow()
                 .vm
-                .id();
-            bo.drop_vm_mappings(vm_id);
+                .drop_mappings(&bo.gem)?;
         }
 
         Ok(0)
@@ -828,11 +847,6 @@ impl File {
             Ok(_) => Ok(0),
         }
     }
-
-    /// Returns the unique file ID for this `File`.
-    pub(crate) fn file_id(&self) -> u64 {
-        self.id
-    }
 }
 
 impl Drop for File {

drivers/gpu/drm/asahi/fw/initdata.rs

@@ -4,7 +4,7 @@
 
 use super::channels;
 use super::types::*;
-use crate::{default_zeroed, gem, no_debug, trivial_gpustruct};
+use crate::{default_zeroed, gem, mmu, no_debug, trivial_gpustruct};
 
 pub(crate) mod raw {
     use super::*;
@@ -1326,6 +1326,8 @@ pub(crate) struct RuntimePointers {
     pub(crate) unkptr_1c8: GpuArray<u8>,
 
     pub(crate) buffer_mgr_ctl: gem::ObjectRef,
+    pub(crate) buffer_mgr_ctl_low_mapping: Option<mmu::KernelMapping>,
+    pub(crate) buffer_mgr_ctl_high_mapping: Option<mmu::KernelMapping>,
 }
 
 #[versions(AGX)]