Skip to content

Commit ac60031

Browse files
davem330davem330
committed
Merge branch 'udp-4tuple-hash'
Philo Lu says: ==================== udp: Add 4-tuple hash for connected sockets This patchset introduces 4-tuple hash for connected udp sockets, to make connected udp lookup faster. Stress test results (with 1 cpu fully used) are shown below, in pps: (1) _un-connected_ socket as server [a] w/o hash4: 1,825176 [b] w/ hash4: 1,831750 (+0.36%) (2) 500 _connected_ sockets as server [c] w/o hash4: 290860 (only 16% of [a]) [d] w/ hash4: 1,889658 (+3.1% compared with [b]) With hash4, compute_score is skipped when lookup, so [d] is slightly better than [b]. Patch1: Add a new counter for hslot2 named hash4_cnt, to avoid cache line miss when lookup. Patch2: Add hslot/hlist_nulls for 4-tuple hash. Patch3 and 4: Implement 4-tuple hash for ipv4 and ipv6. The detailed motivation is described in Patch 3. The 4-tuple hash increases the size of udp_sock and udp_hslot. Thus add it with CONFIG_BASE_SMALL, i.e., it's a no op with CONFIG_BASE_SMALL. Intentionally, the feature is not available for udplite. Though udplite shares some structs and functions with udp, its connect() keeps unchanged. So all udplite sockets perform the same as un-connected udp sockets. Besides, udplite also shares the additional memory consumption in udp_sock and udptable. changelogs: v8 -> v9 (Paolo Abeni): - Add explanation about udplite in cover letter - Update tags for co-developers - Add acked-by tags of Paolo and Willem v7 -> v8: - add EXPORT_SYMBOL for ipv6.ko build v6 -> v7 (Kuniyuki Iwashima): - export udp_ehashfn to be used by udpv6 rehash v5 -> v6 (Paolo Abeni): - move udp_table_hash4_init from patch2 to patch1 - use hlist_nulls for lookup-rehash race - add test results in commit log - add more comment, e.g., for rehash4 used in hash4 - add ipv6 support (Patch4), and refactor some functions for better sharing, without functionality change v4 -> v5 (Paolo Abeni): - add CONFIG_BASE_SMALL with which udp hash4 does nothing v3 -> v4 (Willem de Bruijn): - fix mistakes in udp_pernet_table_alloc() RFCv2 -> v3 (Gur Stavi): - minor fix in udp_hashslot2() and udp_table_init() - add rcu sync in rehash4() RFCv1 -> RFCv2: - add a new struct for hslot2 - remove the sockopt UDP_HASH4 because it has little side effect for unconnected sockets - add rehash in connect() - re-organize the patch into 3 smaller ones - other minor fix v8: https://lore.kernel.org/all/20241108054836.123484-1-lulie@linux.alibaba.com/ v7: https://lore.kernel.org/all/20241105121225.12513-1-lulie@linux.alibaba.com/ v6: https://lore.kernel.org/all/20241031124550.20227-1-lulie@linux.alibaba.com/ v5: https://lore.kernel.org/all/20241018114535.35712-1-lulie@linux.alibaba.com/ v4: https://lore.kernel.org/all/20241012012918.70888-1-lulie@linux.alibaba.com/ v3: https://lore.kernel.org/all/20241010090351.79698-1-lulie@linux.alibaba.com/ RFCv2: https://lore.kernel.org/all/20240924110414.52618-1-lulie@linux.alibaba.com/ RFCv1: https://lore.kernel.org/all/20240913100941.8565-1-lulie@linux.alibaba.com/ ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
2 parents 296a681 + 1b29a73 commit ac60031

4 files changed

Lines changed: 468 additions & 42 deletions

File tree

‎include/linux/udp.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ struct udp_sock {
5656
int pending; /* Any pending frames ? */
5757
__u8 encap_type; /* Is this an Encapsulation socket? */
5858

59+
#if !IS_ENABLED(CONFIG_BASE_SMALL)
60+
/* For UDP 4-tuple hash */
61+
__u16 udp_lrpa_hash;
62+
struct hlist_nulls_node udp_lrpa_node;
63+
#endif
64+
5965
/*
6066
* Following member retains the information to create a UDP header
6167
* when the socket is uncorked.
@@ -206,6 +212,11 @@ static inline void udp_allow_gso(struct sock *sk)
206212
#define udp_portaddr_for_each_entry_rcu(__sk, list) \
207213
hlist_for_each_entry_rcu(__sk, list, __sk_common.skc_portaddr_node)
208214

215+
#if !IS_ENABLED(CONFIG_BASE_SMALL)
216+
#define udp_lrpa_for_each_entry_rcu(__up, node, list) \
217+
hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node)
218+
#endif
219+
209220
#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)
210221

211222
#endif /* _LINUX_UDP_H */

‎include/net/udp.h‎

Lines changed: 131 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,29 +50,56 @@ struct udp_skb_cb {
5050
#define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb))
5151

5252
/**
53-
* struct udp_hslot - UDP hash slot
53+
* struct udp_hslot - UDP hash slot used by udp_table.hash/hash4
5454
*
5555
* @head: head of list of sockets
56+
* @nulls_head: head of list of sockets, only used by hash4
5657
* @count: number of sockets in 'head' list
5758
* @lock: spinlock protecting changes to head/count
5859
*/
5960
struct udp_hslot {
60-
struct hlist_head head;
61+
union {
62+
struct hlist_head head;
63+
/* hash4 uses hlist_nulls to avoid moving wrongly onto another
64+
* hlist, because rehash() can happen with lookup().
65+
*/
66+
struct hlist_nulls_head nulls_head;
67+
};
6168
int count;
6269
spinlock_t lock;
63-
} __attribute__((aligned(2 * sizeof(long))));
70+
} __aligned(2 * sizeof(long));
71+
72+
/**
73+
* struct udp_hslot_main - UDP hash slot used by udp_table.hash2
74+
*
75+
* @hslot: basic hash slot
76+
* @hash4_cnt: number of sockets in hslot4 of the same
77+
* (local port, local address)
78+
*/
79+
struct udp_hslot_main {
80+
struct udp_hslot hslot; /* must be the first member */
81+
#if !IS_ENABLED(CONFIG_BASE_SMALL)
82+
u32 hash4_cnt;
83+
#endif
84+
} __aligned(2 * sizeof(long));
85+
#define UDP_HSLOT_MAIN(__hslot) ((struct udp_hslot_main *)(__hslot))
6486

6587
/**
6688
* struct udp_table - UDP table
6789
*
6890
* @hash: hash table, sockets are hashed on (local port)
6991
* @hash2: hash table, sockets are hashed on (local port, local address)
92+
* @hash4: hash table, connected sockets are hashed on
93+
* (local port, local address, remote port, remote address)
7094
* @mask: number of slots in hash tables, minus 1
7195
* @log: log2(number of slots in hash table)
7296
*/
7397
struct udp_table {
7498
struct udp_hslot *hash;
75-
struct udp_hslot *hash2;
99+
struct udp_hslot_main *hash2;
100+
#if !IS_ENABLED(CONFIG_BASE_SMALL)
101+
struct udp_hslot *hash4;
102+
#endif
76103
unsigned int mask;
77104
unsigned int log;
78105
};
@@ -84,15 +111,97 @@ static inline struct udp_hslot *udp_hashslot(struct udp_table *table,
84111
{
85112
return &table->hash[udp_hashfn(net, num, table->mask)];
86113
}
114+
87115
/*
88116
* For secondary hash, net_hash_mix() is performed before calling
89117
* udp_hashslot2(), this explains difference with udp_hashslot()
90118
*/
91119
static inline struct udp_hslot *udp_hashslot2(struct udp_table *table,
92120
unsigned int hash)
93121
{
94-
return &table->hash2[hash & table->mask];
122+
return &table->hash2[hash & table->mask].hslot;
123+
}
124+
125+
#if IS_ENABLED(CONFIG_BASE_SMALL)
126+
static inline void udp_table_hash4_init(struct udp_table *table)
127+
{
128+
}
129+
130+
static inline struct udp_hslot *udp_hashslot4(struct udp_table *table,
131+
unsigned int hash)
132+
{
133+
BUILD_BUG();
134+
return NULL;
135+
}
136+
137+
static inline bool udp_hashed4(const struct sock *sk)
138+
{
139+
return false;
140+
}
141+
142+
static inline unsigned int udp_hash4_slot_size(void)
143+
{
144+
return 0;
145+
}
146+
147+
static inline bool udp_has_hash4(const struct udp_hslot *hslot2)
148+
{
149+
return false;
150+
}
151+
152+
static inline void udp_hash4_inc(struct udp_hslot *hslot2)
153+
{
154+
}
155+
156+
static inline void udp_hash4_dec(struct udp_hslot *hslot2)
157+
{
95158
}
159+
#else /* !CONFIG_BASE_SMALL */
160+
161+
/* Must be called with table->hash2 initialized */
162+
static inline void udp_table_hash4_init(struct udp_table *table)
163+
{
164+
table->hash4 = (void *)(table->hash2 + (table->mask + 1));
165+
for (int i = 0; i <= table->mask; i++) {
166+
table->hash2[i].hash4_cnt = 0;
167+
168+
INIT_HLIST_NULLS_HEAD(&table->hash4[i].nulls_head, i);
169+
table->hash4[i].count = 0;
170+
spin_lock_init(&table->hash4[i].lock);
171+
}
172+
}
173+
174+
static inline struct udp_hslot *udp_hashslot4(struct udp_table *table,
175+
unsigned int hash)
176+
{
177+
return &table->hash4[hash & table->mask];
178+
}
179+
180+
static inline bool udp_hashed4(const struct sock *sk)
181+
{
182+
return !hlist_nulls_unhashed(&udp_sk(sk)->udp_lrpa_node);
183+
}
184+
185+
static inline unsigned int udp_hash4_slot_size(void)
186+
{
187+
return sizeof(struct udp_hslot);
188+
}
189+
190+
static inline bool udp_has_hash4(const struct udp_hslot *hslot2)
191+
{
192+
return UDP_HSLOT_MAIN(hslot2)->hash4_cnt;
193+
}
194+
195+
static inline void udp_hash4_inc(struct udp_hslot *hslot2)
196+
{
197+
UDP_HSLOT_MAIN(hslot2)->hash4_cnt++;
198+
}
199+
200+
static inline void udp_hash4_dec(struct udp_hslot *hslot2)
201+
{
202+
UDP_HSLOT_MAIN(hslot2)->hash4_cnt--;
203+
}
204+
#endif /* CONFIG_BASE_SMALL */
96205

97206
extern struct proto udp_prot;
98207

@@ -193,13 +302,29 @@ static inline int udp_lib_hash(struct sock *sk)
193302
}
194303

195304
void udp_lib_unhash(struct sock *sk);
196-
void udp_lib_rehash(struct sock *sk, u16 new_hash);
305+
void udp_lib_rehash(struct sock *sk, u16 new_hash, u16 new_hash4);
306+
u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
307+
const __be32 faddr, const __be16 fport);
197308

198309
static inline void udp_lib_close(struct sock *sk, long timeout)
199310
{
200311
sk_common_release(sk);
201312
}
202313

314+
/* hash4 routines shared between UDPv4/6 */
315+
#if IS_ENABLED(CONFIG_BASE_SMALL)
316+
static inline void udp_lib_hash4(struct sock *sk, u16 hash)
317+
{
318+
}
319+
320+
static inline void udp4_hash4(struct sock *sk)
321+
{
322+
}
323+
#else /* !CONFIG_BASE_SMALL */
324+
void udp_lib_hash4(struct sock *sk, u16 hash);
325+
void udp4_hash4(struct sock *sk);
326+
#endif /* CONFIG_BASE_SMALL */
327+
203328
int udp_lib_get_port(struct sock *sk, unsigned short snum,
204329
unsigned int hash2_nulladdr);
205330

0 commit comments

Comments
 (0)