Skip to content

Commit c376644

Browse files
isilenceaxboe
isilence
authored and
axboe
committed
io_uring/rsrc: merge nodes and io_rsrc_put
struct io_rsrc_node carries a number of resources represented by struct io_rsrc_put. That was handy before for sync overhead ammortisation, but all complexity is gone and nodes are simple and lightweight. Let's allocate a separate node for each resource. Nodes and io_rsrc_put and not much different in size, and former are cached, so node allocation should work better. That also removes some overhead for nested iteration in io_rsrc_node_ref_zero() / __io_rsrc_put_work(). Another reason for the patch is that it greatly reduces complexity by moving io_rsrc_node_switch[_start]() inside io_queue_rsrc_removal(), so users don't have to care about it. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Link: https://lore.kernel.org/r/c7d3a45b30cc14cd93700a710dd112edc703db98.1681822823.git.asml.silence@gmail.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent 63fea89 commit c376644

3 files changed

Lines changed: 25 additions & 97 deletions

File tree

‎io_uring/filetable.c‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -80,18 +80,13 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
8080
if (file_slot->file_ptr) {
8181
struct file *old_file;
8282

83-
ret = io_rsrc_node_switch_start(ctx);
84-
if (ret)
85-
return ret;
86-
8783
old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
8884
ret = io_queue_rsrc_removal(ctx->file_data, slot_index, old_file);
8985
if (ret)
9086
return ret;
9187

9288
file_slot->file_ptr = 0;
9389
io_file_bitmap_clear(&ctx->file_table, slot_index);
94-
io_rsrc_node_switch(ctx, ctx->file_data);
9590
}
9691

9792
ret = io_scm_file_account(ctx, file);
@@ -152,9 +147,6 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
152147
return -ENXIO;
153148
if (offset >= ctx->nr_user_files)
154149
return -EINVAL;
155-
ret = io_rsrc_node_switch_start(ctx);
156-
if (ret)
157-
return ret;
158150

159151
offset = array_index_nospec(offset, ctx->nr_user_files);
160152
file_slot = io_fixed_file_slot(&ctx->file_table, offset);
@@ -168,7 +160,6 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
168160

169161
file_slot->file_ptr = 0;
170162
io_file_bitmap_clear(&ctx->file_table, offset);
171-
io_rsrc_node_switch(ctx, ctx->file_data);
172163
return 0;
173164
}
174165

‎io_uring/rsrc.c‎

Lines changed: 23 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -153,17 +153,10 @@ static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
153153
static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
154154
{
155155
struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
156-
struct io_rsrc_put *prsrc, *tmp;
157156

158-
if (ref_node->inline_items)
157+
if (likely(ref_node->inline_items))
159158
io_rsrc_put_work_one(rsrc_data, &ref_node->item);
160159

161-
list_for_each_entry_safe(prsrc, tmp, &ref_node->item_list, list) {
162-
list_del(&prsrc->list);
163-
io_rsrc_put_work_one(rsrc_data, prsrc);
164-
kfree(prsrc);
165-
}
166-
167160
io_rsrc_node_destroy(rsrc_data->ctx, ref_node);
168161
}
169162

@@ -206,53 +199,29 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
206199
}
207200

208201
ref_node->rsrc_data = NULL;
209-
ref_node->refs = 1;
210-
INIT_LIST_HEAD(&ref_node->node);
211-
INIT_LIST_HEAD(&ref_node->item_list);
212202
ref_node->inline_items = 0;
203+
ref_node->refs = 1;
213204
return ref_node;
214205
}
215206

216-
void io_rsrc_node_switch(struct io_ring_ctx *ctx,
217-
struct io_rsrc_data *data_to_kill)
218-
__must_hold(&ctx->uring_lock)
219-
{
220-
struct io_rsrc_node *node = ctx->rsrc_node;
221-
struct io_rsrc_node *backup = io_rsrc_node_alloc(ctx);
222-
223-
if (WARN_ON_ONCE(!backup))
224-
return;
225-
226-
node->rsrc_data = data_to_kill;
227-
list_add_tail(&node->node, &ctx->rsrc_ref_list);
228-
/* put master ref */
229-
io_put_rsrc_node(ctx, node);
230-
ctx->rsrc_node = backup;
231-
}
232-
233-
int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
234-
{
235-
struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);
236-
237-
if (!node)
238-
return -ENOMEM;
239-
io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);
240-
return 0;
241-
}
242-
243207
__cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
244208
struct io_ring_ctx *ctx)
245209
{
210+
struct io_rsrc_node *backup;
246211
DEFINE_WAIT(we);
247212
int ret;
248213

249-
/* As we may drop ->uring_lock, other task may have started quiesce */
214+
/* As We may drop ->uring_lock, other task may have started quiesce */
250215
if (data->quiesce)
251216
return -ENXIO;
252-
ret = io_rsrc_node_switch_start(ctx);
253-
if (ret)
254-
return ret;
255-
io_rsrc_node_switch(ctx, data);
217+
218+
backup = io_rsrc_node_alloc(ctx);
219+
if (!backup)
220+
return -ENOMEM;
221+
ctx->rsrc_node->rsrc_data = data;
222+
list_add_tail(&ctx->rsrc_node->node, &ctx->rsrc_ref_list);
223+
io_put_rsrc_node(ctx, ctx->rsrc_node);
224+
ctx->rsrc_node = backup;
256225

257226
if (list_empty(&ctx->rsrc_ref_list))
258227
return 0;
@@ -382,7 +351,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
382351
struct file *file;
383352
int fd, i, err = 0;
384353
unsigned int done;
385-
bool needs_switch = false;
386354

387355
if (!ctx->file_data)
388356
return -ENXIO;
@@ -414,7 +382,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
414382
break;
415383
file_slot->file_ptr = 0;
416384
io_file_bitmap_clear(&ctx->file_table, i);
417-
needs_switch = true;
418385
}
419386
if (fd != -1) {
420387
file = fget(fd);
@@ -445,9 +412,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
445412
io_file_bitmap_set(&ctx->file_table, i);
446413
}
447414
}
448-
449-
if (needs_switch)
450-
io_rsrc_node_switch(ctx, data);
451415
return done ? done : err;
452416
}
453417

@@ -458,7 +422,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
458422
u64 __user *tags = u64_to_user_ptr(up->tags);
459423
struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
460424
struct page *last_hpage = NULL;
461-
bool needs_switch = false;
462425
__u32 done;
463426
int i, err;
464427

@@ -498,15 +461,11 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
498461
break;
499462
}
500463
ctx->user_bufs[i] = ctx->dummy_ubuf;
501-
needs_switch = true;
502464
}
503465

504466
ctx->user_bufs[i] = imu;
505467
*io_get_tag_slot(ctx->buf_data, i) = tag;
506468
}
507-
508-
if (needs_switch)
509-
io_rsrc_node_switch(ctx, ctx->buf_data);
510469
return done ? done : err;
511470
}
512471

@@ -515,15 +474,11 @@ static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
515474
unsigned nr_args)
516475
{
517476
__u32 tmp;
518-
int err;
519477

520478
lockdep_assert_held(&ctx->uring_lock);
521479

522480
if (check_add_overflow(up->offset, nr_args, &tmp))
523481
return -EOVERFLOW;
524-
err = io_rsrc_node_switch_start(ctx);
525-
if (err)
526-
return err;
527482

528483
switch (type) {
529484
case IORING_RSRC_FILE:
@@ -685,21 +640,21 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc)
685640
struct io_ring_ctx *ctx = data->ctx;
686641
struct io_rsrc_node *node = ctx->rsrc_node;
687642
u64 *tag_slot = io_get_tag_slot(data, idx);
688-
struct io_rsrc_put *prsrc;
689643

690-
if (!node->inline_items) {
691-
prsrc = &node->item;
692-
node->inline_items++;
693-
} else {
694-
prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
695-
if (!prsrc)
696-
return -ENOMEM;
697-
list_add(&prsrc->list, &node->item_list);
644+
ctx->rsrc_node = io_rsrc_node_alloc(ctx);
645+
if (unlikely(!ctx->rsrc_node)) {
646+
ctx->rsrc_node = node;
647+
return -ENOMEM;
698648
}
699649

700-
prsrc->tag = *tag_slot;
650+
node->item.rsrc = rsrc;
651+
node->item.tag = *tag_slot;
652+
node->inline_items = 1;
701653
*tag_slot = 0;
702-
prsrc->rsrc = rsrc;
654+
655+
node->rsrc_data = data;
656+
list_add_tail(&node->node, &ctx->rsrc_ref_list);
657+
io_put_rsrc_node(ctx, node);
703658
return 0;
704659
}
705660

‎io_uring/rsrc.h‎

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ enum {
1818
};
1919

2020
struct io_rsrc_put {
21-
struct list_head list;
2221
u64 tag;
2322
union {
2423
void *rsrc;
@@ -43,17 +42,10 @@ struct io_rsrc_node {
4342
struct io_cache_entry cache;
4443
struct io_rsrc_data *rsrc_data;
4544
};
46-
struct list_head node;
4745
int refs;
48-
49-
/*
50-
* Keeps a list of struct io_rsrc_put to be completed. Each entry
51-
* represents one rsrc (e.g. file or buffer), but all of them should've
52-
* came from the same table and so are of the same type.
53-
*/
54-
struct list_head item_list;
55-
struct io_rsrc_put item;
5646
int inline_items;
47+
struct list_head node;
48+
struct io_rsrc_put item;
5749
};
5850

5951
struct io_mapped_ubuf {
@@ -68,11 +60,8 @@ void io_rsrc_put_tw(struct callback_head *cb);
6860
void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
6961
void io_rsrc_put_work(struct work_struct *work);
7062
void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
71-
int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
7263
struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
7364
int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc);
74-
void io_rsrc_node_switch(struct io_ring_ctx *ctx,
75-
struct io_rsrc_data *data_to_kill);
7665

7766
int io_import_fixed(int ddir, struct iov_iter *iter,
7867
struct io_mapped_ubuf *imu,
@@ -109,13 +98,6 @@ static inline int io_scm_file_account(struct io_ring_ctx *ctx,
10998
return __io_scm_file_account(ctx, file);
11099
}
111100

112-
static inline int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
113-
{
114-
if (unlikely(io_alloc_cache_empty(&ctx->rsrc_node_cache)))
115-
return __io_rsrc_node_switch_start(ctx);
116-
return 0;
117-
}
118-
119101
int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
120102
unsigned nr_args);
121103
int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,

0 commit comments

Comments
 (0)