fix(uvm): skip RM calls during cleanup when GPU is surprise removed

Author: bdandy

kernel-open/nvidia-uvm/uvm_channel.c

@@ -27,6 +27,8 @@
 #include "uvm_common.h"
 #include "uvm_global.h"
 #include "uvm_hal.h"
+#include "uvm_gpu.h"
+#include "uvm_gpu_isr.h"
 #include "uvm_procfs.h"
 #include "uvm_push.h"
 #include "uvm_gpu_semaphore.h"
@@ -2310,10 +2312,14 @@ static void channel_destroy(uvm_channel_pool_t *pool, uvm_channel_t *channel)
             free_conf_computing_buffers(channel);
     }
 
-    if (uvm_channel_is_proxy(channel))
-        uvm_rm_locked_call_void(nvUvmInterfacePagingChannelDestroy(channel->proxy.handle));
-    else
-        uvm_rm_locked_call_void(nvUvmInterfaceChannelDestroy(channel->handle));
+    // Skip RM calls if GPU has been surprise removed. Calling RM with stale
+    // handles will result in NV_ERR_INVALID_OBJECT_HANDLE errors.
+    if (uvm_parent_gpu_is_accessible(pool->manager->gpu->parent)) {
+        if (uvm_channel_is_proxy(channel))
+            uvm_rm_locked_call_void(nvUvmInterfacePagingChannelDestroy(channel->proxy.handle));
+        else
+            uvm_rm_locked_call_void(nvUvmInterfaceChannelDestroy(channel->handle));
+    }
 
     uvm_gpu_tracking_semaphore_free(&channel->tracking_sem);
 
@@ -2657,7 +2663,11 @@ static void tsg_destroy(uvm_channel_pool_t *pool, uvmGpuTsgHandle tsg_handle)
 {
     UVM_ASSERT(pool->num_tsgs > 0);
 
-    uvm_rm_locked_call_void(nvUvmInterfaceTsgDestroy(tsg_handle));
+    // Skip RM call if GPU has been surprise removed. Calling RM with stale
+    // handles will result in NV_ERR_INVALID_OBJECT_HANDLE errors.
+    if (uvm_parent_gpu_is_accessible(pool->manager->gpu->parent))
+        uvm_rm_locked_call_void(nvUvmInterfaceTsgDestroy(tsg_handle));
+
     pool->num_tsgs--;
 }
 

kernel-open/nvidia-uvm/uvm_rm_mem.c

@@ -23,6 +23,7 @@
 
 #include "uvm_rm_mem.h"
 #include "uvm_gpu.h"
+#include "uvm_gpu_isr.h"
 #include "uvm_global.h"
 #include "uvm_kvmalloc.h"
 #include "uvm_linux.h"
@@ -298,8 +299,11 @@ void uvm_rm_mem_unmap_cpu(uvm_rm_mem_t *rm_mem)
     if (!uvm_rm_mem_mapped_on_cpu(rm_mem))
         return;
 
-    uvm_rm_locked_call_void(nvUvmInterfaceMemoryCpuUnMap(rm_mem->gpu_owner->rm_address_space,
-                                                         uvm_rm_mem_get_cpu_va(rm_mem)));
+    // Skip RM call if GPU has been surprise removed. Calling RM with stale
+    // handles will result in NV_ERR_INVALID_OBJECT_HANDLE errors.
+    if (uvm_parent_gpu_is_accessible(rm_mem->gpu_owner->parent))
+        uvm_rm_locked_call_void(nvUvmInterfaceMemoryCpuUnMap(rm_mem->gpu_owner->rm_address_space,
+                                                             uvm_rm_mem_get_cpu_va(rm_mem)));
 
     rm_mem_clear_cpu_va(rm_mem);
 }
@@ -355,7 +359,12 @@ static void rm_mem_unmap_gpu(uvm_rm_mem_t *rm_mem, uvm_gpu_t *gpu)
     rm_mem_unmap_gpu_proxy(rm_mem, gpu);
 
     va = uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu);
-    uvm_rm_locked_call_void(nvUvmInterfaceMemoryFree(gpu->rm_address_space, va));
+
+    // Skip RM call if GPU has been surprise removed. Calling RM with stale
+    // handles will result in NV_ERR_INVALID_OBJECT_HANDLE errors.
+    if (uvm_parent_gpu_is_accessible(gpu->parent))
+        uvm_rm_locked_call_void(nvUvmInterfaceMemoryFree(gpu->rm_address_space, va));
+
     rm_mem_clear_gpu_va(rm_mem, gpu);
 }
 

kernel-open/nvidia-uvm/uvm_user_channel.c

@@ -32,6 +32,7 @@
 #include "uvm_kvmalloc.h"
 #include "uvm_api.h"
 #include "uvm_gpu.h"
+#include "uvm_gpu_isr.h"
 #include "uvm_tracker.h"
 #include "uvm_map_external.h"
 #include "nv_uvm_interface.h"
@@ -782,6 +783,14 @@ void uvm_user_channel_stop(uvm_user_channel_t *user_channel)
     //       write mode.
     uvm_assert_rwsem_locked_read(&va_space->lock);
 
+    // Skip RM call if GPU has been surprise removed. Calling RM with stale
+    // client handles will result in repeated NV_ERR_INVALID_OBJECT_HANDLE
+    // errors during teardown.
+    if (!uvm_parent_gpu_is_accessible(user_channel->gpu->parent)) {
+        atomic_set(&user_channel->is_bound, 0);
+        return;
+    }
+
     // TODO: Bug 1737765. This doesn't stop the user from putting the
     //       channel back on the runlist, which could put stale instance
     //       pointers back in the fault buffer.
@@ -854,7 +863,9 @@ void uvm_user_channel_destroy_detached(uvm_user_channel_t *user_channel)
         uvm_kvfree(user_channel->resources);
     }
 
-    if (user_channel->rm_retained_channel)
+    // Skip RM call if GPU has been surprise removed. Calling RM with stale
+    // handles will result in NV_ERR_INVALID_OBJECT_HANDLE errors.
+    if (user_channel->rm_retained_channel && uvm_parent_gpu_is_accessible(user_channel->gpu->parent))
         uvm_rm_locked_call_void(nvUvmInterfaceReleaseChannel(user_channel->rm_retained_channel));
 
     uvm_user_channel_release(user_channel);

kernel-open/nvidia-uvm/uvm_va_space.c

@@ -32,6 +32,7 @@
 #include "uvm_tools.h"
 #include "uvm_thread_context.h"
 #include "uvm_hal.h"
+#include "uvm_gpu_isr.h"
 #include "uvm_map_external.h"
 #include "uvm_ats.h"
 #include "uvm_gpu_access_counters.h"
@@ -1436,6 +1437,13 @@ void uvm_gpu_va_space_unset_page_dir(uvm_gpu_va_space_t *gpu_va_space)
     if (gpu_va_space->did_set_page_directory) {
         NV_STATUS status;
 
+        // Skip RM call if GPU has been surprise removed. Calling RM with stale
+        // handles will result in NV_ERR_INVALID_OBJECT_HANDLE errors.
+        if (!uvm_parent_gpu_is_accessible(gpu_va_space->gpu->parent)) {
+            gpu_va_space->did_set_page_directory = false;
+            return;
+        }
+
         status = uvm_rm_locked_call(nvUvmInterfaceUnsetPageDirectory(gpu_va_space->duped_gpu_va_space));
         UVM_ASSERT_MSG(status == NV_OK,
                        "nvUvmInterfaceUnsetPageDirectory() failed: %s, GPU %s\n",
@@ -1487,7 +1495,9 @@ static void destroy_gpu_va_space(uvm_gpu_va_space_t *gpu_va_space)
     if (gpu_va_space->page_tables.root)
         uvm_page_tree_deinit(&gpu_va_space->page_tables);
 
-    if (gpu_va_space->duped_gpu_va_space)
+    // Skip RM call if GPU has been surprise removed. Calling RM with stale
+    // handles will result in NV_ERR_INVALID_OBJECT_HANDLE errors.
+    if (gpu_va_space->duped_gpu_va_space && uvm_parent_gpu_is_accessible(gpu_va_space->gpu->parent))
         uvm_rm_locked_call_void(nvUvmInterfaceAddressSpaceDestroy(gpu_va_space->duped_gpu_va_space));
 
     // If the state is DEAD, then this GPU VA space is tracked in