5757#define UBLK_PARAM_TYPE_ALL (UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD)
5858
5959struct ublk_rq_data {
60- union {
61- struct callback_head work ;
62- struct llist_node node ;
63- };
60+ struct llist_node node ;
61+ struct callback_head work ;
6462};
6563
6664struct ublk_uring_cmd_pdu {
@@ -766,30 +764,52 @@ static inline void __ublk_rq_task_work(struct request *req)
766764 ubq_complete_io_cmd (io , UBLK_IO_RES_OK );
767765}
768766
767+ static inline void ublk_forward_io_cmds (struct ublk_queue * ubq )
768+ {
769+ struct llist_node * io_cmds = llist_del_all (& ubq -> io_cmds );
770+ struct ublk_rq_data * data , * tmp ;
771+
772+ io_cmds = llist_reverse_order (io_cmds );
773+ llist_for_each_entry_safe (data , tmp , io_cmds , node )
774+ __ublk_rq_task_work (blk_mq_rq_from_pdu (data ));
775+ }
776+
777+ static inline void ublk_abort_io_cmds (struct ublk_queue * ubq )
778+ {
779+ struct llist_node * io_cmds = llist_del_all (& ubq -> io_cmds );
780+ struct ublk_rq_data * data , * tmp ;
781+
782+ llist_for_each_entry_safe (data , tmp , io_cmds , node )
783+ __ublk_abort_rq (ubq , blk_mq_rq_from_pdu (data ));
784+ }
785+
769786static void ublk_rq_task_work_cb (struct io_uring_cmd * cmd )
770787{
771788 struct ublk_uring_cmd_pdu * pdu = ublk_get_uring_cmd_pdu (cmd );
772789 struct ublk_queue * ubq = pdu -> ubq ;
773- struct llist_node * io_cmds = llist_del_all (& ubq -> io_cmds );
774- struct ublk_rq_data * data ;
775790
776- llist_for_each_entry (data , io_cmds , node )
777- __ublk_rq_task_work (blk_mq_rq_from_pdu (data ));
791+ ublk_forward_io_cmds (ubq );
778792}
779793
780794static void ublk_rq_task_work_fn (struct callback_head * work )
781795{
782796 struct ublk_rq_data * data = container_of (work ,
783797 struct ublk_rq_data , work );
784798 struct request * req = blk_mq_rq_from_pdu (data );
799+ struct ublk_queue * ubq = req -> mq_hctx -> driver_data ;
785800
786- __ublk_rq_task_work ( req );
801+ ublk_forward_io_cmds ( ubq );
787802}
788803
789- static void ublk_submit_cmd (struct ublk_queue * ubq , const struct request * rq )
804+ static void ublk_queue_cmd (struct ublk_queue * ubq , struct request * rq )
790805{
791- struct ublk_io * io = & ubq -> ios [rq -> tag ];
806+ struct ublk_rq_data * data = blk_mq_rq_to_pdu (rq );
807+ struct ublk_io * io ;
792808
809+ if (!llist_add (& data -> node , & ubq -> io_cmds ))
810+ return ;
811+
812+ io = & ubq -> ios [rq -> tag ];
793813 /*
794814 * If the check pass, we know that this is a re-issued request aborted
795815 * previously in monitor_work because the ubq_daemon(cmd's task) is
@@ -803,11 +823,11 @@ static void ublk_submit_cmd(struct ublk_queue *ubq, const struct request *rq)
803823 * guarantees that here is a re-issued request aborted previously.
804824 */
805825 if (unlikely (io -> flags & UBLK_IO_FLAG_ABORTED )) {
806- struct llist_node * io_cmds = llist_del_all ( & ubq -> io_cmds );
807- struct ublk_rq_data * data ;
808-
809- llist_for_each_entry ( data , io_cmds , node )
810- __ublk_abort_rq (ubq , blk_mq_rq_from_pdu ( data ) );
826+ ublk_abort_io_cmds ( ubq );
827+ } else if ( ublk_can_use_task_work ( ubq )) {
828+ if ( task_work_add ( ubq -> ubq_daemon , & data -> work ,
829+ TWA_SIGNAL_NO_IPI ) )
830+ ublk_abort_io_cmds (ubq );
811831 } else {
812832 struct io_uring_cmd * cmd = io -> cmd ;
813833 struct ublk_uring_cmd_pdu * pdu = ublk_get_uring_cmd_pdu (cmd );
@@ -817,23 +837,6 @@ static void ublk_submit_cmd(struct ublk_queue *ubq, const struct request *rq)
817837 }
818838}
819839
820- static void ublk_queue_cmd (struct ublk_queue * ubq , struct request * rq ,
821- bool last )
822- {
823- struct ublk_rq_data * data = blk_mq_rq_to_pdu (rq );
824-
825- if (ublk_can_use_task_work (ubq )) {
826- enum task_work_notify_mode notify_mode = last ?
827- TWA_SIGNAL_NO_IPI : TWA_NONE ;
828-
829- if (task_work_add (ubq -> ubq_daemon , & data -> work , notify_mode ))
830- __ublk_abort_rq (ubq , rq );
831- } else {
832- if (llist_add (& data -> node , & ubq -> io_cmds ))
833- ublk_submit_cmd (ubq , rq );
834- }
835- }
836-
837840static blk_status_t ublk_queue_rq (struct blk_mq_hw_ctx * hctx ,
838841 const struct blk_mq_queue_data * bd )
839842{
@@ -865,19 +868,11 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
865868 return BLK_STS_OK ;
866869 }
867870
868- ublk_queue_cmd (ubq , rq , bd -> last );
871+ ublk_queue_cmd (ubq , rq );
869872
870873 return BLK_STS_OK ;
871874}
872875
873- static void ublk_commit_rqs (struct blk_mq_hw_ctx * hctx )
874- {
875- struct ublk_queue * ubq = hctx -> driver_data ;
876-
877- if (ublk_can_use_task_work (ubq ))
878- __set_notify_signal (ubq -> ubq_daemon );
879- }
880-
881876static int ublk_init_hctx (struct blk_mq_hw_ctx * hctx , void * driver_data ,
882877 unsigned int hctx_idx )
883878{
@@ -899,7 +894,6 @@ static int ublk_init_rq(struct blk_mq_tag_set *set, struct request *req,
899894
900895static const struct blk_mq_ops ublk_mq_ops = {
901896 .queue_rq = ublk_queue_rq ,
902- .commit_rqs = ublk_commit_rqs ,
903897 .init_hctx = ublk_init_hctx ,
904898 .init_request = ublk_init_rq ,
905899};
@@ -1197,7 +1191,7 @@ static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id,
11971191 struct ublk_queue * ubq = ublk_get_queue (ub , q_id );
11981192 struct request * req = blk_mq_tag_to_rq (ub -> tag_set .tags [q_id ], tag );
11991193
1200- ublk_queue_cmd (ubq , req , true );
1194+ ublk_queue_cmd (ubq , req );
12011195}
12021196
12031197static int ublk_ch_uring_cmd (struct io_uring_cmd * cmd , unsigned int issue_flags )
0 commit comments