Skip to content

Commit 70f1fbe

Browse files
kuba-mookuba-moo
committed
Merge branch 'ipv6-tcp-no-longer-rebuild-fl6-at-each-transmit'
Eric Dumazet says: ==================== ipv6: tcp: no longer rebuild fl6 at each transmit TCP v6 spends a good amount of time rebuilding a fresh fl6 at each transmit in inet6_csk_xmit()/inet6_csk_route_socket(). TCP v4 caches the information in inet->cork.fl.u.ip4 instead. This series changes TCP v6 to behave the same, saving cpu cycles and reducing cache line misses and stack use. ==================== Link: https://patch.msgid.link/20260206173426.1638518-1-edumazet@google.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2 parents 792aaea + 97d7ae6 commit 70f1fbe

6 files changed

Lines changed: 82 additions & 72 deletions

File tree

‎include/linux/ipv6.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,10 @@ struct ipv6_fl_socklist;
230230
struct ipv6_pinfo {
231231
/* Used in tx path (inet6_csk_route_socket(), ip6_xmit()) */
232232
struct in6_addr saddr;
233+
union {
234+
struct in6_addr daddr;
235+
struct in6_addr final;
236+
};
233237
__be32 flow_label;
234238
u32 dst_cookie;
235239
struct ipv6_txoptions __rcu *opt;

‎include/net/inet6_connection_sock.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@ struct sk_buff;
1818
struct sock;
1919
struct sockaddr;
2020

21-
struct dst_entry *inet6_csk_route_req(const struct sock *sk, struct flowi6 *fl6,
21+
struct dst_entry *inet6_csk_route_req(const struct sock *sk,
22+
struct dst_entry *dst,
23+
struct flowi6 *fl6,
2224
const struct request_sock *req, u8 proto);
2325

2426
int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl);

‎net/ipv6/af_inet6.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -825,7 +825,7 @@ int inet6_sk_rebuild_header(struct sock *sk)
825825
{
826826
struct ipv6_pinfo *np = inet6_sk(sk);
827827
struct inet_sock *inet = inet_sk(sk);
828-
struct in6_addr *final_p, final;
828+
struct in6_addr *final_p;
829829
struct dst_entry *dst;
830830
struct flowi6 *fl6;
831831

@@ -847,7 +847,7 @@ int inet6_sk_rebuild_header(struct sock *sk)
847847
security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));
848848

849849
rcu_read_lock();
850-
final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &final);
850+
final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &np->final);
851851
rcu_read_unlock();
852852

853853
dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);

‎net/ipv6/datagram.c‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,12 @@ static void ip6_datagram_flow_key_init(struct flowi6 *fl6,
7272
int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr)
7373
{
7474
struct ip6_flowlabel *flowlabel = NULL;
75-
struct in6_addr *final_p, final;
76-
struct ipv6_txoptions *opt;
77-
struct dst_entry *dst;
7875
struct inet_sock *inet = inet_sk(sk);
7976
struct ipv6_pinfo *np = inet6_sk(sk);
80-
struct flowi6 fl6;
77+
struct ipv6_txoptions *opt;
78+
struct in6_addr *final_p;
79+
struct dst_entry *dst;
80+
struct flowi6 *fl6;
8181
int err = 0;
8282

8383
if (inet6_test_bit(SNDFLOW, sk) &&
@@ -86,32 +86,33 @@ int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr)
8686
if (IS_ERR(flowlabel))
8787
return -EINVAL;
8888
}
89-
ip6_datagram_flow_key_init(&fl6, sk);
89+
fl6 = &inet_sk(sk)->cork.fl.u.ip6;
90+
ip6_datagram_flow_key_init(fl6, sk);
9091

9192
rcu_read_lock();
9293
opt = flowlabel ? flowlabel->opt : rcu_dereference(np->opt);
93-
final_p = fl6_update_dst(&fl6, opt, &final);
94+
final_p = fl6_update_dst(fl6, opt, &np->final);
9495
rcu_read_unlock();
9596

96-
dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl6, final_p);
97+
dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);
9798
if (IS_ERR(dst)) {
9899
err = PTR_ERR(dst);
99100
goto out;
100101
}
101102

102103
if (fix_sk_saddr) {
103104
if (ipv6_addr_any(&np->saddr))
104-
np->saddr = fl6.saddr;
105+
np->saddr = fl6->saddr;
105106

106107
if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {
107-
sk->sk_v6_rcv_saddr = fl6.saddr;
108+
sk->sk_v6_rcv_saddr = fl6->saddr;
108109
inet->inet_rcv_saddr = LOOPBACK4_IPV6;
109110
if (sk->sk_prot->rehash)
110111
sk->sk_prot->rehash(sk);
111112
}
112113
}
113114

114-
ip6_sk_dst_store_flow(sk, dst, &fl6);
115+
ip6_sk_dst_store_flow(sk, dst, fl6);
115116

116117
out:
117118
fl6_sock_release(flowlabel);

‎net/ipv6/inet6_connection_sock.c‎

Lines changed: 30 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,14 @@
2525
#include <net/sock_reuseport.h>
2626

2727
struct dst_entry *inet6_csk_route_req(const struct sock *sk,
28+
struct dst_entry *dst,
2829
struct flowi6 *fl6,
2930
const struct request_sock *req,
3031
u8 proto)
3132
{
3233
const struct inet_request_sock *ireq = inet_rsk(req);
3334
const struct ipv6_pinfo *np = inet6_sk(sk);
3435
struct in6_addr *final_p, final;
35-
struct dst_entry *dst;
3636

3737
memset(fl6, 0, sizeof(*fl6));
3838
fl6->flowi6_proto = proto;
@@ -48,10 +48,11 @@ struct dst_entry *inet6_csk_route_req(const struct sock *sk,
4848
fl6->flowi6_uid = sk_uid(sk);
4949
security_req_classify_flow(req, flowi6_to_flowi_common(fl6));
5050

51-
dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);
52-
if (IS_ERR(dst))
53-
return NULL;
54-
51+
if (!dst) {
52+
dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);
53+
if (IS_ERR(dst))
54+
return NULL;
55+
}
5556
return dst;
5657
}
5758

@@ -60,7 +61,7 @@ static struct dst_entry *inet6_csk_route_socket(struct sock *sk,
6061
{
6162
struct inet_sock *inet = inet_sk(sk);
6263
struct ipv6_pinfo *np = inet6_sk(sk);
63-
struct in6_addr *final_p, final;
64+
struct in6_addr *final_p;
6465
struct dst_entry *dst;
6566

6667
memset(fl6, 0, sizeof(*fl6));
@@ -77,41 +78,41 @@ static struct dst_entry *inet6_csk_route_socket(struct sock *sk,
7778
security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));
7879

7980
rcu_read_lock();
80-
final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &final);
81+
final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &np->final);
8182
rcu_read_unlock();
8283

83-
dst = __sk_dst_check(sk, np->dst_cookie);
84-
if (!dst) {
85-
dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);
84+
dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p);
85+
86+
if (!IS_ERR(dst))
87+
ip6_dst_store(sk, dst, false, false);
8688

87-
if (!IS_ERR(dst))
88-
ip6_dst_store(sk, dst, false, false);
89-
}
9089
return dst;
9190
}
9291

9392
int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl_unused)
9493
{
94+
struct flowi6 *fl6 = &inet_sk(sk)->cork.fl.u.ip6;
9595
struct ipv6_pinfo *np = inet6_sk(sk);
96-
struct flowi6 fl6;
9796
struct dst_entry *dst;
9897
int res;
9998

100-
dst = inet6_csk_route_socket(sk, &fl6);
101-
if (IS_ERR(dst)) {
102-
WRITE_ONCE(sk->sk_err_soft, -PTR_ERR(dst));
103-
sk->sk_route_caps = 0;
104-
kfree_skb(skb);
105-
return PTR_ERR(dst);
99+
dst = __sk_dst_check(sk, np->dst_cookie);
100+
if (unlikely(!dst)) {
101+
dst = inet6_csk_route_socket(sk, fl6);
102+
if (IS_ERR(dst)) {
103+
WRITE_ONCE(sk->sk_err_soft, -PTR_ERR(dst));
104+
sk->sk_route_caps = 0;
105+
kfree_skb(skb);
106+
return PTR_ERR(dst);
107+
}
108+
/* Restore final destination back after routing done */
109+
fl6->daddr = sk->sk_v6_daddr;
106110
}
107111

108112
rcu_read_lock();
109113
skb_dst_set_noref(skb, dst);
110114

111-
/* Restore final destination back after routing done */
112-
fl6.daddr = sk->sk_v6_daddr;
113-
114-
res = ip6_xmit(sk, skb, &fl6, sk->sk_mark, rcu_dereference(np->opt),
115+
res = ip6_xmit(sk, skb, fl6, sk->sk_mark, rcu_dereference(np->opt),
115116
np->tclass, READ_ONCE(sk->sk_priority));
116117
rcu_read_unlock();
117118
return res;
@@ -120,13 +121,15 @@ EXPORT_SYMBOL_GPL(inet6_csk_xmit);
120121

121122
struct dst_entry *inet6_csk_update_pmtu(struct sock *sk, u32 mtu)
122123
{
123-
struct flowi6 fl6;
124-
struct dst_entry *dst = inet6_csk_route_socket(sk, &fl6);
124+
struct flowi6 *fl6 = &inet_sk(sk)->cork.fl.u.ip6;
125+
struct dst_entry *dst;
126+
127+
dst = inet6_csk_route_socket(sk, fl6);
125128

126129
if (IS_ERR(dst))
127130
return NULL;
128131
dst->ops->update_pmtu(dst, sk, NULL, mtu, true);
129132

130-
dst = inet6_csk_route_socket(sk, &fl6);
133+
dst = inet6_csk_route_socket(sk, fl6);
131134
return IS_ERR(dst) ? NULL : dst;
132135
}

‎net/ipv6/tcp_ipv6.c‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -138,15 +138,15 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
138138
{
139139
struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
140140
struct inet_connection_sock *icsk = inet_csk(sk);
141-
struct in6_addr *saddr = NULL, *final_p, final;
142141
struct inet_timewait_death_row *tcp_death_row;
143142
struct ipv6_pinfo *np = tcp_inet6_sk(sk);
143+
struct in6_addr *saddr = NULL, *final_p;
144144
struct inet_sock *inet = inet_sk(sk);
145145
struct tcp_sock *tp = tcp_sk(sk);
146146
struct net *net = sock_net(sk);
147147
struct ipv6_txoptions *opt;
148148
struct dst_entry *dst;
149-
struct flowi6 fl6;
149+
struct flowi6 *fl6;
150150
int addr_type;
151151
int err;
152152

@@ -156,14 +156,15 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
156156
if (usin->sin6_family != AF_INET6)
157157
return -EAFNOSUPPORT;
158158

159-
memset(&fl6, 0, sizeof(fl6));
159+
fl6 = &inet_sk(sk)->cork.fl.u.ip6;
160+
memset(fl6, 0, sizeof(*fl6));
160161

161162
if (inet6_test_bit(SNDFLOW, sk)) {
162-
fl6.flowlabel = usin->sin6_flowinfo&IPV6_FLOWINFO_MASK;
163-
IP6_ECN_flow_init(fl6.flowlabel);
164-
if (fl6.flowlabel&IPV6_FLOWLABEL_MASK) {
163+
fl6->flowlabel = usin->sin6_flowinfo & IPV6_FLOWINFO_MASK;
164+
IP6_ECN_flow_init(fl6->flowlabel);
165+
if (fl6->flowlabel & IPV6_FLOWLABEL_MASK) {
165166
struct ip6_flowlabel *flowlabel;
166-
flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);
167+
flowlabel = fl6_sock_lookup(sk, fl6->flowlabel);
167168
if (IS_ERR(flowlabel))
168169
return -EINVAL;
169170
fl6_sock_release(flowlabel);
@@ -212,7 +213,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
212213
}
213214

214215
sk->sk_v6_daddr = usin->sin6_addr;
215-
np->flow_label = fl6.flowlabel;
216+
np->flow_label = fl6->flowlabel;
216217

217218
/*
218219
* TCP over IPv4
@@ -260,24 +261,24 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
260261
if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
261262
saddr = &sk->sk_v6_rcv_saddr;
262263

263-
fl6.flowi6_proto = IPPROTO_TCP;
264-
fl6.daddr = sk->sk_v6_daddr;
265-
fl6.saddr = saddr ? *saddr : np->saddr;
266-
fl6.flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
267-
fl6.flowi6_oif = sk->sk_bound_dev_if;
268-
fl6.flowi6_mark = sk->sk_mark;
269-
fl6.fl6_dport = usin->sin6_port;
270-
fl6.fl6_sport = inet->inet_sport;
271-
if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6.fl6_sport)
272-
fl6.flowi6_flags = FLOWI_FLAG_ANY_SPORT;
273-
fl6.flowi6_uid = sk_uid(sk);
264+
fl6->flowi6_proto = IPPROTO_TCP;
265+
fl6->daddr = sk->sk_v6_daddr;
266+
fl6->saddr = saddr ? *saddr : np->saddr;
267+
fl6->flowlabel = ip6_make_flowinfo(np->tclass, np->flow_label);
268+
fl6->flowi6_oif = sk->sk_bound_dev_if;
269+
fl6->flowi6_mark = sk->sk_mark;
270+
fl6->fl6_dport = usin->sin6_port;
271+
fl6->fl6_sport = inet->inet_sport;
272+
if (IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) && !fl6->fl6_sport)
273+
fl6->flowi6_flags = FLOWI_FLAG_ANY_SPORT;
274+
fl6->flowi6_uid = sk_uid(sk);
274275

275276
opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk));
276-
final_p = fl6_update_dst(&fl6, opt, &final);
277+
final_p = fl6_update_dst(fl6, opt, &np->final);
277278

278-
security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
279+
security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6));
279280

280-
dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
281+
dst = ip6_dst_lookup_flow(net, sk, fl6, final_p);
281282
if (IS_ERR(dst)) {
282283
err = PTR_ERR(dst);
283284
goto failure;
@@ -287,7 +288,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
287288
tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
288289

289290
if (!saddr) {
290-
saddr = &fl6.saddr;
291+
saddr = &fl6->saddr;
291292

292293
err = inet_bhash2_update_saddr(sk, saddr, AF_INET6);
293294
if (err)
@@ -538,7 +539,7 @@ static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
538539
u8 tclass;
539540

540541
/* First, grab a route. */
541-
if (!dst && (dst = inet6_csk_route_req(sk, fl6, req,
542+
if (!dst && (dst = inet6_csk_route_req(sk, NULL, fl6, req,
542543
IPPROTO_TCP)) == NULL)
543544
goto done;
544545

@@ -788,7 +789,7 @@ static struct dst_entry *tcp_v6_route_req(const struct sock *sk,
788789
if (security_inet_conn_request(sk, skb, req))
789790
return NULL;
790791

791-
return inet6_csk_route_req(sk, &fl->u.ip6, req, IPPROTO_TCP);
792+
return inet6_csk_route_req(sk, NULL, &fl->u.ip6, req, IPPROTO_TCP);
792793
}
793794

794795
struct request_sock_ops tcp6_request_sock_ops __read_mostly = {
@@ -1317,12 +1318,12 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
13171318
struct request_sock *req_unhash,
13181319
bool *own_req)
13191320
{
1320-
struct inet_request_sock *ireq;
1321-
struct ipv6_pinfo *newnp;
13221321
const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
1322+
struct inet_request_sock *ireq;
13231323
struct ipv6_txoptions *opt;
13241324
struct inet_sock *newinet;
13251325
bool found_dup_sk = false;
1326+
struct ipv6_pinfo *newnp;
13261327
struct tcp_sock *newtp;
13271328
struct sock *newsk;
13281329
#ifdef CONFIG_TCP_MD5SIG
@@ -1391,11 +1392,9 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
13911392
if (sk_acceptq_is_full(sk))
13921393
goto exit_overflow;
13931394

1394-
if (!dst) {
1395-
dst = inet6_csk_route_req(sk, &fl6, req, IPPROTO_TCP);
1396-
if (!dst)
1397-
goto exit;
1398-
}
1395+
dst = inet6_csk_route_req(sk, dst, &fl6, req, IPPROTO_TCP);
1396+
if (!dst)
1397+
goto exit;
13991398

14001399
newsk = tcp_create_openreq_child(sk, req, skb);
14011400
if (!newsk)
@@ -1411,6 +1410,7 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
14111410
inet6_sk_rx_dst_set(newsk, skb);
14121411

14131412
newinet = inet_sk(newsk);
1413+
newinet->cork.fl.u.ip6 = fl6;
14141414
newinet->pinet6 = tcp_inet6_sk(newsk);
14151415
newinet->ipv6_fl_list = NULL;
14161416
newinet->inet_opt = NULL;

0 commit comments

Comments
 (0)