Skip to content

Commit fb544b8

Browse files
mbrost05mbrost05
committed
drm/xe: Implement xe_pagefault_queue_work
Implement a worker that services page faults, using the same implementation as in xe_gt_pagefault.c. v2: - Rebase on exhaustive eviction changes - Include engine instance in debug prints (Stuart) Signed-off-by: Matthew Brost <matthew.brost@intel.com> Reviewed-by: Stuart Summers <stuart.summers@intel.com> Reviewed-by: Lucas De Marchi <lucas.demarchi@intel.com> Tested-by: Francois Dugast <francois.dugast@intel.com> Link: https://patch.msgid.link/20251031165416.2871503-6-matthew.brost@intel.com
1 parent 143aa16 commit fb544b8

1 file changed

Lines changed: 234 additions & 1 deletion

File tree

‎drivers/gpu/drm/xe/xe_pagefault.c‎

Lines changed: 234 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,20 @@
55

66
#include <linux/circ_buf.h>
77

8+
#include <drm/drm_exec.h>
89
#include <drm/drm_managed.h>
910

11+
#include "xe_bo.h"
1012
#include "xe_device.h"
13+
#include "xe_gt_printk.h"
1114
#include "xe_gt_types.h"
15+
#include "xe_gt_stats.h"
16+
#include "xe_hw_engine.h"
1217
#include "xe_pagefault.h"
1318
#include "xe_pagefault_types.h"
19+
#include "xe_svm.h"
20+
#include "xe_trace_bo.h"
21+
#include "xe_vm.h"
1422

1523
/**
1624
* DOC: Xe page faults
@@ -37,9 +45,234 @@ static int xe_pagefault_entry_size(void)
3745
return roundup_pow_of_two(sizeof(struct xe_pagefault));
3846
}
3947

48+
static int xe_pagefault_begin(struct drm_exec *exec, struct xe_vma *vma,
49+
struct xe_vram_region *vram, bool need_vram_move)
50+
{
51+
struct xe_bo *bo = xe_vma_bo(vma);
52+
struct xe_vm *vm = xe_vma_vm(vma);
53+
int err;
54+
55+
err = xe_vm_lock_vma(exec, vma);
56+
if (err)
57+
return err;
58+
59+
if (!bo)
60+
return 0;
61+
62+
return need_vram_move ? xe_bo_migrate(bo, vram->placement, NULL, exec) :
63+
xe_bo_validate(bo, vm, true, exec);
64+
}
65+
66+
static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma,
67+
bool atomic)
68+
{
69+
struct xe_vm *vm = xe_vma_vm(vma);
70+
struct xe_tile *tile = gt_to_tile(gt);
71+
struct xe_validation_ctx ctx;
72+
struct drm_exec exec;
73+
struct dma_fence *fence;
74+
int err, needs_vram;
75+
76+
lockdep_assert_held_write(&vm->lock);
77+
78+
needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic);
79+
if (needs_vram < 0 || (needs_vram && xe_vma_is_userptr(vma)))
80+
return needs_vram < 0 ? needs_vram : -EACCES;
81+
82+
xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1);
83+
xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB,
84+
xe_vma_size(vma) / SZ_1K);
85+
86+
trace_xe_vma_pagefault(vma);
87+
88+
/* Check if VMA is valid, opportunistic check only */
89+
if (xe_vm_has_valid_gpu_mapping(tile, vma->tile_present,
90+
vma->tile_invalidated) && !atomic)
91+
return 0;
92+
93+
retry_userptr:
94+
if (xe_vma_is_userptr(vma) &&
95+
xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
96+
struct xe_userptr_vma *uvma = to_userptr_vma(vma);
97+
98+
err = xe_vma_userptr_pin_pages(uvma);
99+
if (err)
100+
return err;
101+
}
102+
103+
/* Lock VM and BOs dma-resv */
104+
xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {});
105+
drm_exec_init(&exec, 0, 0);
106+
drm_exec_until_all_locked(&exec) {
107+
err = xe_pagefault_begin(&exec, vma, tile->mem.vram,
108+
needs_vram == 1);
109+
drm_exec_retry_on_contention(&exec);
110+
xe_validation_retry_on_oom(&ctx, &err);
111+
if (err)
112+
goto unlock_dma_resv;
113+
114+
/* Bind VMA only to the GT that has faulted */
115+
trace_xe_vma_pf_bind(vma);
116+
xe_vm_set_validation_exec(vm, &exec);
117+
fence = xe_vma_rebind(vm, vma, BIT(tile->id));
118+
xe_vm_set_validation_exec(vm, NULL);
119+
if (IS_ERR(fence)) {
120+
err = PTR_ERR(fence);
121+
xe_validation_retry_on_oom(&ctx, &err);
122+
goto unlock_dma_resv;
123+
}
124+
}
125+
126+
dma_fence_wait(fence, false);
127+
dma_fence_put(fence);
128+
129+
unlock_dma_resv:
130+
xe_validation_ctx_fini(&ctx);
131+
if (err == -EAGAIN)
132+
goto retry_userptr;
133+
134+
return err;
135+
}
136+
137+
static bool
138+
xe_pagefault_access_is_atomic(enum xe_pagefault_access_type access_type)
139+
{
140+
return access_type == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC;
141+
}
142+
143+
static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid)
144+
{
145+
struct xe_vm *vm;
146+
147+
down_read(&xe->usm.lock);
148+
vm = xa_load(&xe->usm.asid_to_vm, asid);
149+
if (vm && xe_vm_in_fault_mode(vm))
150+
xe_vm_get(vm);
151+
else
152+
vm = ERR_PTR(-EINVAL);
153+
up_read(&xe->usm.lock);
154+
155+
return vm;
156+
}
157+
158+
static int xe_pagefault_service(struct xe_pagefault *pf)
159+
{
160+
struct xe_gt *gt = pf->gt;
161+
struct xe_device *xe = gt_to_xe(gt);
162+
struct xe_vm *vm;
163+
struct xe_vma *vma = NULL;
164+
int err;
165+
bool atomic;
166+
167+
/* Producer flagged this fault to be nacked */
168+
if (pf->consumer.fault_level == XE_PAGEFAULT_LEVEL_NACK)
169+
return -EFAULT;
170+
171+
vm = xe_pagefault_asid_to_vm(xe, pf->consumer.asid);
172+
if (IS_ERR(vm))
173+
return PTR_ERR(vm);
174+
175+
/*
176+
* TODO: Change to read lock? Using write lock for simplicity.
177+
*/
178+
down_write(&vm->lock);
179+
180+
if (xe_vm_is_closed(vm)) {
181+
err = -ENOENT;
182+
goto unlock_vm;
183+
}
184+
185+
vma = xe_vm_find_vma_by_addr(vm, pf->consumer.page_addr);
186+
if (!vma) {
187+
err = -EINVAL;
188+
goto unlock_vm;
189+
}
190+
191+
atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type);
192+
193+
if (xe_vma_is_cpu_addr_mirror(vma))
194+
err = xe_svm_handle_pagefault(vm, vma, gt,
195+
pf->consumer.page_addr, atomic);
196+
else
197+
err = xe_pagefault_handle_vma(gt, vma, atomic);
198+
199+
unlock_vm:
200+
if (!err)
201+
vm->usm.last_fault_vma = vma;
202+
up_write(&vm->lock);
203+
xe_vm_put(vm);
204+
205+
return err;
206+
}
207+
208+
static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue,
209+
struct xe_pagefault *pf)
210+
{
211+
bool found_fault = false;
212+
213+
spin_lock_irq(&pf_queue->lock);
214+
if (pf_queue->tail != pf_queue->head) {
215+
memcpy(pf, pf_queue->data + pf_queue->tail, sizeof(*pf));
216+
pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) %
217+
pf_queue->size;
218+
found_fault = true;
219+
}
220+
spin_unlock_irq(&pf_queue->lock);
221+
222+
return found_fault;
223+
}
224+
225+
static void xe_pagefault_print(struct xe_pagefault *pf)
226+
{
227+
xe_gt_dbg(pf->gt, "\n\tASID: %d\n"
228+
"\tFaulted Address: 0x%08x%08x\n"
229+
"\tFaultType: %d\n"
230+
"\tAccessType: %d\n"
231+
"\tFaultLevel: %d\n"
232+
"\tEngineClass: %d %s\n"
233+
"\tEngineInstance: %d\n",
234+
pf->consumer.asid,
235+
upper_32_bits(pf->consumer.page_addr),
236+
lower_32_bits(pf->consumer.page_addr),
237+
pf->consumer.fault_type,
238+
pf->consumer.access_type,
239+
pf->consumer.fault_level,
240+
pf->consumer.engine_class,
241+
xe_hw_engine_class_to_str(pf->consumer.engine_class),
242+
pf->consumer.engine_instance);
243+
}
244+
40245
static void xe_pagefault_queue_work(struct work_struct *w)
41246
{
42-
/* TODO: Implement */
247+
struct xe_pagefault_queue *pf_queue =
248+
container_of(w, typeof(*pf_queue), worker);
249+
struct xe_pagefault pf;
250+
unsigned long threshold;
251+
252+
#define USM_QUEUE_MAX_RUNTIME_MS 20
253+
threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
254+
255+
while (xe_pagefault_queue_pop(pf_queue, &pf)) {
256+
int err;
257+
258+
if (!pf.gt) /* Fault squashed during reset */
259+
continue;
260+
261+
err = xe_pagefault_service(&pf);
262+
if (err) {
263+
xe_pagefault_print(&pf);
264+
xe_gt_dbg(pf.gt, "Fault response: Unsuccessful %pe\n",
265+
ERR_PTR(err));
266+
}
267+
268+
pf.producer.ops->ack_fault(&pf, err);
269+
270+
if (time_after(jiffies, threshold)) {
271+
queue_work(gt_to_xe(pf.gt)->usm.pf_wq, w);
272+
break;
273+
}
274+
}
275+
#undef USM_QUEUE_MAX_RUNTIME_MS
43276
}
44277

45278
static int xe_pagefault_queue_init(struct xe_device *xe,

0 commit comments

Comments
 (0)