Skip to content

Commit 001b76b

Browse files
isilenceaxboe
isilence
authored and
axboe
committed
io_uring: keep ring laoyut in a structure
Add a structure keeping SQ/CQ sizes and offsets. For now it only records data previously returned from rings_size and the SQ size. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent 0f4b537 commit 001b76b

3 files changed

Lines changed: 45 additions & 45 deletions

File tree

‎io_uring/io_uring.c‎

Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2757,47 +2757,57 @@ static void io_rings_free(struct io_ring_ctx *ctx)
27572757
ctx->sq_sqes = NULL;
27582758
}
27592759

2760-
unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
2761-
unsigned int cq_entries, size_t *sq_offset)
2760+
int rings_size(unsigned int flags, unsigned int sq_entries,
2761+
unsigned int cq_entries, struct io_rings_layout *rl)
27622762
{
27632763
struct io_rings *rings;
2764+
size_t sqe_size;
27642765
size_t off;
27652766

2766-
*sq_offset = SIZE_MAX;
2767-
27682767
if (flags & IORING_SETUP_CQE_MIXED) {
27692768
if (cq_entries < 2)
2770-
return SIZE_MAX;
2769+
return -EOVERFLOW;
27712770
}
27722771
if (flags & IORING_SETUP_SQE_MIXED) {
27732772
if (sq_entries < 2)
2774-
return SIZE_MAX;
2773+
return -EOVERFLOW;
27752774
}
27762775

2776+
rl->sq_array_offset = SIZE_MAX;
2777+
2778+
sqe_size = sizeof(struct io_uring_sqe);
2779+
if (flags & IORING_SETUP_SQE128)
2780+
sqe_size *= 2;
2781+
2782+
rl->sq_size = array_size(sqe_size, sq_entries);
2783+
if (rl->sq_size == SIZE_MAX)
2784+
return -EOVERFLOW;
2785+
27772786
off = struct_size(rings, cqes, cq_entries);
27782787
if (flags & IORING_SETUP_CQE32)
27792788
off = size_mul(off, 2);
27802789
if (off == SIZE_MAX)
2781-
return SIZE_MAX;
2790+
return -EOVERFLOW;
27822791

27832792
#ifdef CONFIG_SMP
27842793
off = ALIGN(off, SMP_CACHE_BYTES);
27852794
if (off == 0)
2786-
return SIZE_MAX;
2795+
return -EOVERFLOW;
27872796
#endif
27882797

27892798
if (!(flags & IORING_SETUP_NO_SQARRAY)) {
27902799
size_t sq_array_size;
27912800

2792-
*sq_offset = off;
2801+
rl->sq_array_offset = off;
27932802

27942803
sq_array_size = array_size(sizeof(u32), sq_entries);
27952804
off = size_add(off, sq_array_size);
27962805
if (off == SIZE_MAX)
2797-
return SIZE_MAX;
2806+
return -EOVERFLOW;
27982807
}
27992808

2800-
return off;
2809+
rl->rings_size = off;
2810+
return 0;
28012811
}
28022812

28032813
static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@@ -3346,28 +3356,20 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
33463356
struct io_uring_params *p)
33473357
{
33483358
struct io_uring_region_desc rd;
3359+
struct io_rings_layout __rl, *rl = &__rl;
33493360
struct io_rings *rings;
3350-
size_t sq_array_offset;
3351-
size_t sq_size, cq_size, sqe_size;
33523361
int ret;
33533362

33543363
/* make sure these are sane, as we already accounted them */
33553364
ctx->sq_entries = p->sq_entries;
33563365
ctx->cq_entries = p->cq_entries;
33573366

3358-
sqe_size = sizeof(struct io_uring_sqe);
3359-
if (p->flags & IORING_SETUP_SQE128)
3360-
sqe_size *= 2;
3361-
sq_size = array_size(sqe_size, p->sq_entries);
3362-
if (sq_size == SIZE_MAX)
3363-
return -EOVERFLOW;
3364-
cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,
3365-
&sq_array_offset);
3366-
if (cq_size == SIZE_MAX)
3367-
return -EOVERFLOW;
3367+
ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
3368+
if (ret)
3369+
return ret;
33683370

33693371
memset(&rd, 0, sizeof(rd));
3370-
rd.size = PAGE_ALIGN(cq_size);
3372+
rd.size = PAGE_ALIGN(rl->rings_size);
33713373
if (ctx->flags & IORING_SETUP_NO_MMAP) {
33723374
rd.user_addr = p->cq_off.user_addr;
33733375
rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3378,10 +3380,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
33783380
ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
33793381

33803382
if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
3381-
ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
3383+
ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
33823384

33833385
memset(&rd, 0, sizeof(rd));
3384-
rd.size = PAGE_ALIGN(sq_size);
3386+
rd.size = PAGE_ALIGN(rl->sq_size);
33853387
if (ctx->flags & IORING_SETUP_NO_MMAP) {
33863388
rd.user_addr = p->sq_off.user_addr;
33873389
rd.flags |= IORING_MEM_REGION_TYPE_USER;

‎io_uring/io_uring.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,14 @@
1717
#include <trace/events/io_uring.h>
1818
#endif
1919

20+
struct io_rings_layout {
21+
/* size of CQ + headers + SQ offset array */
22+
size_t rings_size;
23+
size_t sq_size;
24+
25+
size_t sq_array_offset;
26+
};
27+
2028
struct io_ctx_config {
2129
struct io_uring_params p;
2230
struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
139147
#define IORING_MAX_ENTRIES 32768
140148
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
141149

142-
unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
143-
unsigned int cq_entries, size_t *sq_offset);
150+
int rings_size(unsigned int flags, unsigned int sq_entries,
151+
unsigned int cq_entries, struct io_rings_layout *rl);
144152
int io_prepare_config(struct io_ctx_config *config);
145153

146154
bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

‎io_uring/register.c‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -401,9 +401,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
401401
struct io_ctx_config config;
402402
struct io_uring_region_desc rd;
403403
struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
404-
size_t size, sq_array_offset;
405404
unsigned i, tail, old_head;
406405
struct io_uring_params *p = &config.p;
406+
struct io_rings_layout __rl, *rl = &__rl;
407407
int ret;
408408

409409
memset(&config, 0, sizeof(config));
@@ -423,13 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
423423
if (unlikely(ret))
424424
return ret;
425425

426-
size = rings_size(p->flags, p->sq_entries, p->cq_entries,
427-
&sq_array_offset);
428-
if (size == SIZE_MAX)
429-
return -EOVERFLOW;
426+
ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
427+
if (ret)
428+
return ret;
430429

431430
memset(&rd, 0, sizeof(rd));
432-
rd.size = PAGE_ALIGN(size);
431+
rd.size = PAGE_ALIGN(rl->rings_size);
433432
if (p->flags & IORING_SETUP_NO_MMAP) {
434433
rd.user_addr = p->cq_off.user_addr;
435434
rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -458,17 +457,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
458457
return -EFAULT;
459458
}
460459

461-
if (p->flags & IORING_SETUP_SQE128)
462-
size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
463-
else
464-
size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
465-
if (size == SIZE_MAX) {
466-
io_register_free_rings(ctx, &n);
467-
return -EOVERFLOW;
468-
}
469-
470460
memset(&rd, 0, sizeof(rd));
471-
rd.size = PAGE_ALIGN(size);
461+
rd.size = PAGE_ALIGN(rl->sq_size);
472462
if (p->flags & IORING_SETUP_NO_MMAP) {
473463
rd.user_addr = p->sq_off.user_addr;
474464
rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -551,7 +541,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
551541

552542
/* all done, store old pointers and assign new ones */
553543
if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
554-
ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);
544+
ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
555545

556546
ctx->sq_entries = p->sq_entries;
557547
ctx->cq_entries = p->cq_entries;

0 commit comments

Comments
 (0)