@@ -129,9 +129,6 @@ struct smb_direct_transport {
129129 spinlock_t recvmsg_queue_lock ;
130130 struct list_head recvmsg_queue ;
131131
132- spinlock_t empty_recvmsg_queue_lock ;
133- struct list_head empty_recvmsg_queue ;
134-
135132 int send_credit_target ;
136133 atomic_t send_credits ;
137134 spinlock_t lock_new_recv_credits ;
@@ -268,40 +265,19 @@ smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t)
268265static void put_recvmsg (struct smb_direct_transport * t ,
269266 struct smb_direct_recvmsg * recvmsg )
270267{
271- ib_dma_unmap_single (t -> cm_id -> device , recvmsg -> sge .addr ,
272- recvmsg -> sge .length , DMA_FROM_DEVICE );
268+ if (likely (recvmsg -> sge .length != 0 )) {
269+ ib_dma_unmap_single (t -> cm_id -> device ,
270+ recvmsg -> sge .addr ,
271+ recvmsg -> sge .length ,
272+ DMA_FROM_DEVICE );
273+ recvmsg -> sge .length = 0 ;
274+ }
273275
274276 spin_lock (& t -> recvmsg_queue_lock );
275277 list_add (& recvmsg -> list , & t -> recvmsg_queue );
276278 spin_unlock (& t -> recvmsg_queue_lock );
277279}
278280
279- static struct
280- smb_direct_recvmsg * get_empty_recvmsg (struct smb_direct_transport * t )
281- {
282- struct smb_direct_recvmsg * recvmsg = NULL ;
283-
284- spin_lock (& t -> empty_recvmsg_queue_lock );
285- if (!list_empty (& t -> empty_recvmsg_queue )) {
286- recvmsg = list_first_entry (& t -> empty_recvmsg_queue ,
287- struct smb_direct_recvmsg , list );
288- list_del (& recvmsg -> list );
289- }
290- spin_unlock (& t -> empty_recvmsg_queue_lock );
291- return recvmsg ;
292- }
293-
294- static void put_empty_recvmsg (struct smb_direct_transport * t ,
295- struct smb_direct_recvmsg * recvmsg )
296- {
297- ib_dma_unmap_single (t -> cm_id -> device , recvmsg -> sge .addr ,
298- recvmsg -> sge .length , DMA_FROM_DEVICE );
299-
300- spin_lock (& t -> empty_recvmsg_queue_lock );
301- list_add_tail (& recvmsg -> list , & t -> empty_recvmsg_queue );
302- spin_unlock (& t -> empty_recvmsg_queue_lock );
303- }
304-
305281static void enqueue_reassembly (struct smb_direct_transport * t ,
306282 struct smb_direct_recvmsg * recvmsg ,
307283 int data_length )
@@ -386,9 +362,6 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
386362 spin_lock_init (& t -> recvmsg_queue_lock );
387363 INIT_LIST_HEAD (& t -> recvmsg_queue );
388364
389- spin_lock_init (& t -> empty_recvmsg_queue_lock );
390- INIT_LIST_HEAD (& t -> empty_recvmsg_queue );
391-
392365 init_waitqueue_head (& t -> wait_send_pending );
393366 atomic_set (& t -> send_pending , 0 );
394367
@@ -548,13 +521,13 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
548521 t = recvmsg -> transport ;
549522
550523 if (wc -> status != IB_WC_SUCCESS || wc -> opcode != IB_WC_RECV ) {
524+ put_recvmsg (t , recvmsg );
551525 if (wc -> status != IB_WC_WR_FLUSH_ERR ) {
552526 pr_err ("Recv error. status='%s (%d)' opcode=%d\n" ,
553527 ib_wc_status_msg (wc -> status ), wc -> status ,
554528 wc -> opcode );
555529 smb_direct_disconnect_rdma_connection (t );
556530 }
557- put_empty_recvmsg (t , recvmsg );
558531 return ;
559532 }
560533
@@ -568,15 +541,16 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
568541 switch (recvmsg -> type ) {
569542 case SMB_DIRECT_MSG_NEGOTIATE_REQ :
570543 if (wc -> byte_len < sizeof (struct smb_direct_negotiate_req )) {
571- put_empty_recvmsg (t , recvmsg );
544+ put_recvmsg (t , recvmsg );
545+ smb_direct_disconnect_rdma_connection (t );
572546 return ;
573547 }
574548 t -> negotiation_requested = true;
575549 t -> full_packet_received = true;
576550 t -> status = SMB_DIRECT_CS_CONNECTED ;
577551 enqueue_reassembly (t , recvmsg , 0 );
578552 wake_up_interruptible (& t -> wait_status );
579- break ;
553+ return ;
580554 case SMB_DIRECT_MSG_DATA_TRANSFER : {
581555 struct smb_direct_data_transfer * data_transfer =
582556 (struct smb_direct_data_transfer * )recvmsg -> packet ;
@@ -585,15 +559,17 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
585559
586560 if (wc -> byte_len <
587561 offsetof(struct smb_direct_data_transfer , padding )) {
588- put_empty_recvmsg (t , recvmsg );
562+ put_recvmsg (t , recvmsg );
563+ smb_direct_disconnect_rdma_connection (t );
589564 return ;
590565 }
591566
592567 data_length = le32_to_cpu (data_transfer -> data_length );
593568 if (data_length ) {
594569 if (wc -> byte_len < sizeof (struct smb_direct_data_transfer ) +
595570 (u64 )data_length ) {
596- put_empty_recvmsg (t , recvmsg );
571+ put_recvmsg (t , recvmsg );
572+ smb_direct_disconnect_rdma_connection (t );
597573 return ;
598574 }
599575
@@ -605,16 +581,11 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
605581 else
606582 t -> full_packet_received = true;
607583
608- enqueue_reassembly (t , recvmsg , (int )data_length );
609- wake_up_interruptible (& t -> wait_reassembly_queue );
610-
611584 spin_lock (& t -> receive_credit_lock );
612585 receive_credits = -- (t -> recv_credits );
613586 avail_recvmsg_count = t -> count_avail_recvmsg ;
614587 spin_unlock (& t -> receive_credit_lock );
615588 } else {
616- put_empty_recvmsg (t , recvmsg );
617-
618589 spin_lock (& t -> receive_credit_lock );
619590 receive_credits = -- (t -> recv_credits );
620591 avail_recvmsg_count = ++ (t -> count_avail_recvmsg );
@@ -636,11 +607,23 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
636607 if (is_receive_credit_post_required (receive_credits , avail_recvmsg_count ))
637608 mod_delayed_work (smb_direct_wq ,
638609 & t -> post_recv_credits_work , 0 );
639- break ;
610+
611+ if (data_length ) {
612+ enqueue_reassembly (t , recvmsg , (int )data_length );
613+ wake_up_interruptible (& t -> wait_reassembly_queue );
614+ } else
615+ put_recvmsg (t , recvmsg );
616+
617+ return ;
640618 }
641- default :
642- break ;
643619 }
620+
621+ /*
622+ * This is an internal error!
623+ */
624+ WARN_ON_ONCE (recvmsg -> type != SMB_DIRECT_MSG_DATA_TRANSFER );
625+ put_recvmsg (t , recvmsg );
626+ smb_direct_disconnect_rdma_connection (t );
644627}
645628
646629static int smb_direct_post_recv (struct smb_direct_transport * t ,
@@ -670,6 +653,7 @@ static int smb_direct_post_recv(struct smb_direct_transport *t,
670653 ib_dma_unmap_single (t -> cm_id -> device ,
671654 recvmsg -> sge .addr , recvmsg -> sge .length ,
672655 DMA_FROM_DEVICE );
656+ recvmsg -> sge .length = 0 ;
673657 smb_direct_disconnect_rdma_connection (t );
674658 return ret ;
675659 }
@@ -811,26 +795,16 @@ static void smb_direct_post_recv_credits(struct work_struct *work)
811795 struct smb_direct_recvmsg * recvmsg ;
812796 int receive_credits , credits = 0 ;
813797 int ret ;
814- int use_free = 1 ;
815798
816799 spin_lock (& t -> receive_credit_lock );
817800 receive_credits = t -> recv_credits ;
818801 spin_unlock (& t -> receive_credit_lock );
819802
820803 if (receive_credits < t -> recv_credit_target ) {
821804 while (true) {
822- if (use_free )
823- recvmsg = get_free_recvmsg (t );
824- else
825- recvmsg = get_empty_recvmsg (t );
826- if (!recvmsg ) {
827- if (use_free ) {
828- use_free = 0 ;
829- continue ;
830- } else {
831- break ;
832- }
833- }
805+ recvmsg = get_free_recvmsg (t );
806+ if (!recvmsg )
807+ break ;
834808
835809 recvmsg -> type = SMB_DIRECT_MSG_DATA_TRANSFER ;
836810 recvmsg -> first_segment = false;
@@ -1806,8 +1780,6 @@ static void smb_direct_destroy_pools(struct smb_direct_transport *t)
18061780
18071781 while ((recvmsg = get_free_recvmsg (t )))
18081782 mempool_free (recvmsg , t -> recvmsg_mempool );
1809- while ((recvmsg = get_empty_recvmsg (t )))
1810- mempool_free (recvmsg , t -> recvmsg_mempool );
18111783
18121784 mempool_destroy (t -> recvmsg_mempool );
18131785 t -> recvmsg_mempool = NULL ;
@@ -1863,6 +1835,7 @@ static int smb_direct_create_pools(struct smb_direct_transport *t)
18631835 if (!recvmsg )
18641836 goto err ;
18651837 recvmsg -> transport = t ;
1838+ recvmsg -> sge .length = 0 ;
18661839 list_add (& recvmsg -> list , & t -> recvmsg_queue );
18671840 }
18681841 t -> count_avail_recvmsg = t -> recv_credit_max ;
0 commit comments