Skip to content

Commit 80a85a7

Browse files
metze-sambarleon
metze-samba
authored and
rleon
committed
RDMA/rxe: reclassify sockets in order to avoid false positives from lockdep
While developing IPPROTO_SMBDIRECT support for the code under fs/smb/common/smbdirect [1], I noticed false positives like this: [+0,003927] ============================================ [+0,000532] WARNING: possible recursive locking detected [+0,000611] 6.18.0-rc5-metze-kasan-lockdep.02+ #1 Tainted: G OE [+0,000835] -------------------------------------------- [+0,000729] ksmbd:r5445/3609 is trying to acquire lock: [+0,000709] ffff88800b9570f8 (k-sk_lock-AF_INET){+.+.}-{0:0}, at: inet_shutdown+0x52/0x360 [+0,000831] but task is already holding lock: [+0,000684] ffff88800654af78 (k-sk_lock-AF_INET){+.+.}-{0:0}, at: smbdirect_sk_close+0x122/0x790 [smbdirect] [+0,000928] other info that might help us debug this: [+0,005552] Possible unsafe locking scenario: [+0,000723] CPU0 [+0,000359] ---- [+0,000377] lock(k-sk_lock-AF_INET); [+0,000478] lock(k-sk_lock-AF_INET); [+0,000498] *** DEADLOCK *** [+0,001012] May be due to missing lock nesting notation [+0,000831] 3 locks held by ksmbd:r5445/3609: [+0,000484] #0: ffff88800654af78 (k-sk_lock-AF_INET){+.+.}-{0:0}, at: smbdirect_sk_close+0x122/0x790 [smbdirect] [+0,001000] #1: ffff888020a40458 (&id_priv->handler_mutex){+.+.}-{4:4}, at: rdma_lock_handler+0x17/0x30 [rdma_cm] [+0,000982] #2: ffff888020a40350 (&id_priv->qp_mutex){+.+.}-{4:4}, at: rdma_destroy_qp+0x5d/0x1f0 [rdma_cm] [+0,000934] stack backtrace: [+0,000589] CPU: 0 UID: 0 PID: 3609 Comm: ksmbd:r5445 Kdump: loaded Tainted: G OE 6.18.0-rc5-metze-kasan-lockdep.02+ #1 PREEMPT(voluntary) [+0,000023] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE [+0,000004] Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006 ... [+0,000010] print_deadlock_bug+0x245/0x330 [+0,000014] validate_chain+0x32a/0x590 [+0,000012] __lock_acquire+0x535/0xc30 [+0,000013] lock_acquire.part.0+0xb3/0x240 [+0,000017] ? inet_shutdown+0x52/0x360 [+0,000013] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000007] ? mark_held_locks+0x46/0x90 [+0,000012] lock_acquire+0x60/0x140 [+0,000006] ? inet_shutdown+0x52/0x360 [+0,000028] lock_sock_nested+0x3b/0xf0 [+0,000009] ? inet_shutdown+0x52/0x360 [+0,000008] inet_shutdown+0x52/0x360 [+0,000010] kernel_sock_shutdown+0x5b/0x90 [+0,000011] rxe_qp_do_cleanup+0x4ef/0x810 [rdma_rxe] [+0,000043] ? __pfx_rxe_qp_do_cleanup+0x10/0x10 [rdma_rxe] [+0,000030] execute_in_process_context+0x2b/0x170 [+0,000013] rxe_qp_cleanup+0x1c/0x30 [rdma_rxe] [+0,000021] __rxe_cleanup+0x1cf/0x2e0 [rdma_rxe] [+0,000036] ? __pfx___rxe_cleanup+0x10/0x10 [rdma_rxe] [+0,000020] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000006] ? __kasan_check_read+0x11/0x20 [+0,000012] rxe_destroy_qp+0xe1/0x230 [rdma_rxe] [+0,000035] ib_destroy_qp_user+0x217/0x450 [ib_core] [+0,000074] rdma_destroy_qp+0x83/0x1f0 [rdma_cm] [+0,000034] smbdirect_connection_destroy_qp+0x98/0x2e0 [smbdirect] [+0,000017] ? __pfx_smb_direct_logging_needed+0x10/0x10 [ksmbd] [+0,000044] smbdirect_connection_destroy+0x698/0xed0 [smbdirect] [+0,000023] ? __pfx_smbdirect_connection_destroy+0x10/0x10 [smbdirect] [+0,000033] ? __pfx_smb_direct_logging_needed+0x10/0x10 [ksmbd] [+0,000031] smbdirect_connection_destroy_sync+0x42b/0x9f0 [smbdirect] [+0,000029] ? mark_held_locks+0x46/0x90 [+0,000012] ? __pfx_smbdirect_connection_destroy_sync+0x10/0x10 [smbdirect] [+0,000019] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000007] ? trace_hardirqs_on+0x64/0x70 [+0,000029] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000010] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000006] ? __smbdirect_connection_schedule_disconnect+0x339/0x4b0 [+0,000021] smbdirect_sk_destroy+0xb0/0x680 [smbdirect] [+0,000024] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000006] ? trace_hardirqs_on+0x64/0x70 [+0,000006] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000005] ? __local_bh_enable_ip+0xba/0x150 [+0,000011] sk_common_release+0x66/0x340 [+0,000010] smbdirect_sk_close+0x12a/0x790 [smbdirect] [+0,000023] ? ip_mc_drop_socket+0x1e/0x240 [+0,000013] inet_release+0x10a/0x240 [+0,000011] smbdirect_sock_release+0x502/0xe80 [smbdirect] [+0,000015] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000024] sock_release+0x91/0x1c0 [+0,000010] smb_direct_free_transport+0x31/0x50 [ksmbd] [+0,000025] ksmbd_conn_free+0x1d0/0x240 [ksmbd] [+0,000040] smb_direct_disconnect+0xb2/0x120 [ksmbd] [+0,000023] ? srso_alias_return_thunk+0x5/0xfbef5 [+0,000018] ksmbd_conn_handler_loop+0x94e/0xf10 [ksmbd] ... I'll also add reclassify to the smbdirect socket code [1], but I think it's better to have it in both direction (below and above the RDMA layer). [1] https://git.samba.org/?p=metze/linux/wip.git;a=shortlog;h=refs/heads/master-ipproto-smbdirect Cc: Zhu Yanjun <zyjzyj2000@gmail.com> Reviewed-by: Zhu Yanjun <yanjun.zhu@linux.dev> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: Leon Romanovsky <leon@kernel.org> Cc: linux-rdma@vger.kernel.org Cc: netdev@vger.kernel.org Cc: linux-cifs@vger.kernel.org Signed-off-by: Stefan Metzmacher <metze@samba.org> Link: https://patch.msgid.link/20251127105614.2040922-1-metze@samba.org Signed-off-by: Leon Romanovsky <leon@kernel.org>
1 parent 3a2c32d commit 80a85a7

2 files changed

Lines changed: 98 additions & 0 deletions

File tree

‎drivers/infiniband/sw/rxe/rxe_net.c‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,54 @@
2020

2121
static struct rxe_recv_sockets recv_sockets;
2222

23+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
24+
/*
25+
* lockdep can detect false positive circular dependencies
26+
* when there are user-space socket API users or in kernel
27+
* users switching between a tcp and rdma transport.
28+
* Maybe also switching between siw and rxe may cause
29+
* problems as per default sockets are only classified
30+
* by family and not by ip protocol. And there might
31+
* be different locks used between the application
32+
* and the low level sockets.
33+
*
34+
* Problems were seen with ksmbd.ko and cifs.ko,
35+
* switching transports, use git blame to find
36+
* more details.
37+
*/
38+
static struct lock_class_key rxe_recv_sk_key[2];
39+
static struct lock_class_key rxe_recv_slock_key[2];
40+
#endif /* CONFIG_DEBUG_LOCK_ALLOC */
41+
42+
static inline void rxe_reclassify_recv_socket(struct socket *sock)
43+
{
44+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
45+
struct sock *sk = sock->sk;
46+
47+
if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
48+
return;
49+
50+
switch (sk->sk_family) {
51+
case AF_INET:
52+
sock_lock_init_class_and_name(sk,
53+
"slock-AF_INET-RDMA-RXE-RECV",
54+
&rxe_recv_slock_key[0],
55+
"sk_lock-AF_INET-RDMA-RXE-RECV",
56+
&rxe_recv_sk_key[0]);
57+
break;
58+
case AF_INET6:
59+
sock_lock_init_class_and_name(sk,
60+
"slock-AF_INET6-RDMA-RXE-RECV",
61+
&rxe_recv_slock_key[1],
62+
"sk_lock-AF_INET6-RDMA-RXE-RECV",
63+
&rxe_recv_sk_key[1]);
64+
break;
65+
default:
66+
WARN_ON_ONCE(1);
67+
}
68+
#endif /* CONFIG_DEBUG_LOCK_ALLOC */
69+
}
70+
2371
static struct dst_entry *rxe_find_route4(struct rxe_qp *qp,
2472
struct net_device *ndev,
2573
struct in_addr *saddr,
@@ -192,6 +240,7 @@ static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port,
192240
err = udp_sock_create(net, &udp_cfg, &sock);
193241
if (err < 0)
194242
return ERR_PTR(err);
243+
rxe_reclassify_recv_socket(sock);
195244

196245
tnl_cfg.encap_type = 1;
197246
tnl_cfg.encap_rcv = rxe_udp_encap_recv;

‎drivers/infiniband/sw/rxe/rxe_qp.c‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,54 @@
1515
#include "rxe_queue.h"
1616
#include "rxe_task.h"
1717

18+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
19+
/*
20+
* lockdep can detect false positive circular dependencies
21+
* when there are user-space socket API users or in kernel
22+
* users switching between a tcp and rdma transport.
23+
* Maybe also switching between siw and rxe may cause
24+
* problems as per default sockets are only classified
25+
* by family and not by ip protocol. And there might
26+
* be different locks used between the application
27+
* and the low level sockets.
28+
*
29+
* Problems were seen with ksmbd.ko and cifs.ko,
30+
* switching transports, use git blame to find
31+
* more details.
32+
*/
33+
static struct lock_class_key rxe_send_sk_key[2];
34+
static struct lock_class_key rxe_send_slock_key[2];
35+
#endif /* CONFIG_DEBUG_LOCK_ALLOC */
36+
37+
static inline void rxe_reclassify_send_socket(struct socket *sock)
38+
{
39+
#ifdef CONFIG_DEBUG_LOCK_ALLOC
40+
struct sock *sk = sock->sk;
41+
42+
if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
43+
return;
44+
45+
switch (sk->sk_family) {
46+
case AF_INET:
47+
sock_lock_init_class_and_name(sk,
48+
"slock-AF_INET-RDMA-RXE-SEND",
49+
&rxe_send_slock_key[0],
50+
"sk_lock-AF_INET-RDMA-RXE-SEND",
51+
&rxe_send_sk_key[0]);
52+
break;
53+
case AF_INET6:
54+
sock_lock_init_class_and_name(sk,
55+
"slock-AF_INET6-RDMA-RXE-SEND",
56+
&rxe_send_slock_key[1],
57+
"sk_lock-AF_INET6-RDMA-RXE-SEND",
58+
&rxe_send_sk_key[1]);
59+
break;
60+
default:
61+
WARN_ON_ONCE(1);
62+
}
63+
#endif /* CONFIG_DEBUG_LOCK_ALLOC */
64+
}
65+
1866
static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap,
1967
int has_srq)
2068
{
@@ -244,6 +292,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
244292
err = sock_create_kern(&init_net, AF_INET, SOCK_DGRAM, 0, &qp->sk);
245293
if (err < 0)
246294
return err;
295+
rxe_reclassify_send_socket(qp->sk);
247296
qp->sk->sk->sk_user_data = qp;
248297

249298
/* pick a source UDP port number for this QP based on

0 commit comments

Comments
 (0)