Skip to content

Commit 9d2789a

Browse files
axboeaxboe
committed
block/io_uring: pass in issue_flags for uring_cmd task_work handling
io_uring_cmd_done() currently assumes that the uring_lock is held when invoked, and while it generally is, this is not guaranteed. Pass in the issue_flags associated with it, so that we have IO_URING_F_UNLOCKED available to be able to lock the CQ ring appropriately when completing events. Cc: stable@vger.kernel.org Fixes: ee692a2 ("fs,io_uring: add infrastructure for uring-cmd") Signed-off-by: Jens Axboe <axboe@kernel.dk>
1 parent 4985e7b commit 9d2789a

4 files changed

Lines changed: 38 additions & 28 deletions

File tree

‎drivers/block/ublk_drv.c‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -715,7 +715,8 @@ static void __ublk_fail_req(struct ublk_queue *ubq, struct ublk_io *io,
715715
}
716716
}
717717

718-
static void ubq_complete_io_cmd(struct ublk_io *io, int res)
718+
static void ubq_complete_io_cmd(struct ublk_io *io, int res,
719+
unsigned issue_flags)
719720
{
720721
/* mark this cmd owned by ublksrv */
721722
io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV;
@@ -727,7 +728,7 @@ static void ubq_complete_io_cmd(struct ublk_io *io, int res)
727728
io->flags &= ~UBLK_IO_FLAG_ACTIVE;
728729

729730
/* tell ublksrv one io request is coming */
730-
io_uring_cmd_done(io->cmd, res, 0);
731+
io_uring_cmd_done(io->cmd, res, 0, issue_flags);
731732
}
732733

733734
#define UBLK_REQUEUE_DELAY_MS 3
@@ -744,7 +745,8 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq,
744745
mod_delayed_work(system_wq, &ubq->dev->monitor_work, 0);
745746
}
746747

747-
static inline void __ublk_rq_task_work(struct request *req)
748+
static inline void __ublk_rq_task_work(struct request *req,
749+
unsigned issue_flags)
748750
{
749751
struct ublk_queue *ubq = req->mq_hctx->driver_data;
750752
int tag = req->tag;
@@ -782,7 +784,7 @@ static inline void __ublk_rq_task_work(struct request *req)
782784
pr_devel("%s: need get data. op %d, qid %d tag %d io_flags %x\n",
783785
__func__, io->cmd->cmd_op, ubq->q_id,
784786
req->tag, io->flags);
785-
ubq_complete_io_cmd(io, UBLK_IO_RES_NEED_GET_DATA);
787+
ubq_complete_io_cmd(io, UBLK_IO_RES_NEED_GET_DATA, issue_flags);
786788
return;
787789
}
788790
/*
@@ -820,17 +822,18 @@ static inline void __ublk_rq_task_work(struct request *req)
820822
mapped_bytes >> 9;
821823
}
822824

823-
ubq_complete_io_cmd(io, UBLK_IO_RES_OK);
825+
ubq_complete_io_cmd(io, UBLK_IO_RES_OK, issue_flags);
824826
}
825827

826-
static inline void ublk_forward_io_cmds(struct ublk_queue *ubq)
828+
static inline void ublk_forward_io_cmds(struct ublk_queue *ubq,
829+
unsigned issue_flags)
827830
{
828831
struct llist_node *io_cmds = llist_del_all(&ubq->io_cmds);
829832
struct ublk_rq_data *data, *tmp;
830833

831834
io_cmds = llist_reverse_order(io_cmds);
832835
llist_for_each_entry_safe(data, tmp, io_cmds, node)
833-
__ublk_rq_task_work(blk_mq_rq_from_pdu(data));
836+
__ublk_rq_task_work(blk_mq_rq_from_pdu(data), issue_flags);
834837
}
835838

836839
static inline void ublk_abort_io_cmds(struct ublk_queue *ubq)
@@ -842,12 +845,12 @@ static inline void ublk_abort_io_cmds(struct ublk_queue *ubq)
842845
__ublk_abort_rq(ubq, blk_mq_rq_from_pdu(data));
843846
}
844847

845-
static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd)
848+
static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd, unsigned issue_flags)
846849
{
847850
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
848851
struct ublk_queue *ubq = pdu->ubq;
849852

850-
ublk_forward_io_cmds(ubq);
853+
ublk_forward_io_cmds(ubq, issue_flags);
851854
}
852855

853856
static void ublk_rq_task_work_fn(struct callback_head *work)
@@ -856,8 +859,9 @@ static void ublk_rq_task_work_fn(struct callback_head *work)
856859
struct ublk_rq_data, work);
857860
struct request *req = blk_mq_rq_from_pdu(data);
858861
struct ublk_queue *ubq = req->mq_hctx->driver_data;
862+
unsigned issue_flags = IO_URING_F_UNLOCKED;
859863

860-
ublk_forward_io_cmds(ubq);
864+
ublk_forward_io_cmds(ubq, issue_flags);
861865
}
862866

863867
static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
@@ -1111,7 +1115,8 @@ static void ublk_cancel_queue(struct ublk_queue *ubq)
11111115
struct ublk_io *io = &ubq->ios[i];
11121116

11131117
if (io->flags & UBLK_IO_FLAG_ACTIVE)
1114-
io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, 0);
1118+
io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, 0,
1119+
IO_URING_F_UNLOCKED);
11151120
}
11161121

11171122
/* all io commands are canceled */
@@ -1351,7 +1356,7 @@ static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
13511356
return -EIOCBQUEUED;
13521357

13531358
out:
1354-
io_uring_cmd_done(cmd, ret, 0);
1359+
io_uring_cmd_done(cmd, ret, 0, issue_flags);
13551360
pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n",
13561361
__func__, cmd_op, tag, ret, io->flags);
13571362
return -EIOCBQUEUED;
@@ -2234,7 +2239,7 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd,
22342239
if (ub)
22352240
ublk_put_device(ub);
22362241
out:
2237-
io_uring_cmd_done(cmd, ret, 0);
2242+
io_uring_cmd_done(cmd, ret, 0, issue_flags);
22382243
pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n",
22392244
__func__, ret, cmd->cmd_op, header->dev_id, header->queue_id);
22402245
return -EIOCBQUEUED;

‎drivers/nvme/host/ioctl.c‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,8 @@ static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
464464
return (struct nvme_uring_cmd_pdu *)&ioucmd->pdu;
465465
}
466466

467-
static void nvme_uring_task_meta_cb(struct io_uring_cmd *ioucmd)
467+
static void nvme_uring_task_meta_cb(struct io_uring_cmd *ioucmd,
468+
unsigned issue_flags)
468469
{
469470
struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
470471
struct request *req = pdu->req;
@@ -485,17 +486,18 @@ static void nvme_uring_task_meta_cb(struct io_uring_cmd *ioucmd)
485486
blk_rq_unmap_user(req->bio);
486487
blk_mq_free_request(req);
487488

488-
io_uring_cmd_done(ioucmd, status, result);
489+
io_uring_cmd_done(ioucmd, status, result, issue_flags);
489490
}
490491

491-
static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd)
492+
static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,
493+
unsigned issue_flags)
492494
{
493495
struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
494496

495497
if (pdu->bio)
496498
blk_rq_unmap_user(pdu->bio);
497499

498-
io_uring_cmd_done(ioucmd, pdu->nvme_status, pdu->u.result);
500+
io_uring_cmd_done(ioucmd, pdu->nvme_status, pdu->u.result, issue_flags);
499501
}
500502

501503
static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
@@ -517,7 +519,7 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
517519
* Otherwise, move the completion to task work.
518520
*/
519521
if (cookie != NULL && blk_rq_is_poll(req))
520-
nvme_uring_task_cb(ioucmd);
522+
nvme_uring_task_cb(ioucmd, IO_URING_F_UNLOCKED);
521523
else
522524
io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb);
523525

@@ -539,7 +541,7 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io_meta(struct request *req,
539541
* Otherwise, move the completion to task work.
540542
*/
541543
if (cookie != NULL && blk_rq_is_poll(req))
542-
nvme_uring_task_meta_cb(ioucmd);
544+
nvme_uring_task_meta_cb(ioucmd, IO_URING_F_UNLOCKED);
543545
else
544546
io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_meta_cb);
545547

‎include/linux/io_uring.h‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ struct io_uring_cmd {
2727
const void *cmd;
2828
union {
2929
/* callback to defer completions to task context */
30-
void (*task_work_cb)(struct io_uring_cmd *cmd);
30+
void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned);
3131
/* used for polled completion */
3232
void *cookie;
3333
};
@@ -39,9 +39,10 @@ struct io_uring_cmd {
3939
#if defined(CONFIG_IO_URING)
4040
int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
4141
struct iov_iter *iter, void *ioucmd);
42-
void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t res2);
42+
void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t res2,
43+
unsigned issue_flags);
4344
void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
44-
void (*task_work_cb)(struct io_uring_cmd *));
45+
void (*task_work_cb)(struct io_uring_cmd *, unsigned));
4546
struct sock *io_uring_get_socket(struct file *file);
4647
void __io_uring_cancel(bool cancel_all);
4748
void __io_uring_free(struct task_struct *tsk);
@@ -72,11 +73,11 @@ static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
7273
return -EOPNOTSUPP;
7374
}
7475
static inline void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret,
75-
ssize_t ret2)
76+
ssize_t ret2, unsigned issue_flags)
7677
{
7778
}
7879
static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
79-
void (*task_work_cb)(struct io_uring_cmd *))
80+
void (*task_work_cb)(struct io_uring_cmd *, unsigned))
8081
{
8182
}
8283
static inline struct sock *io_uring_get_socket(struct file *file)

‎io_uring/uring_cmd.c‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@
1515
static void io_uring_cmd_work(struct io_kiocb *req, bool *locked)
1616
{
1717
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
18+
unsigned issue_flags = *locked ? 0 : IO_URING_F_UNLOCKED;
1819

19-
ioucmd->task_work_cb(ioucmd);
20+
ioucmd->task_work_cb(ioucmd, issue_flags);
2021
}
2122

2223
void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
23-
void (*task_work_cb)(struct io_uring_cmd *))
24+
void (*task_work_cb)(struct io_uring_cmd *, unsigned))
2425
{
2526
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
2627

@@ -42,7 +43,8 @@ static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
4243
* Called by consumers of io_uring_cmd, if they originally returned
4344
* -EIOCBQUEUED upon receiving the command.
4445
*/
45-
void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
46+
void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2,
47+
unsigned issue_flags)
4648
{
4749
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
4850

@@ -56,7 +58,7 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2)
5658
/* order with io_iopoll_req_issued() checking ->iopoll_complete */
5759
smp_store_release(&req->iopoll_completed, 1);
5860
else
59-
io_req_complete_post(req, 0);
61+
io_req_complete_post(req, issue_flags);
6062
}
6163
EXPORT_SYMBOL_GPL(io_uring_cmd_done);
6264

0 commit comments

Comments
 (0)