Skip to content

Commit ee90848

Browse files
paulmckrcuFrederic Weisbecker
paulmckrcu
authored and
Frederic Weisbecker
committed
srcu: Create a DEFINE_SRCU_FAST()
This commit creates DEFINE_SRCU_FAST() and DEFINE_STATIC_SRCU_FAST() macros that are similar to DEFINE_SRCU() and DEFINE_STATIC_SRCU(), but which create srcu_struct structures that are usable only by readers initiated by srcu_read_lock_fast() and friends. This commit does make DEFINE_SRCU_FAST() available to modules, in which case the per-CPU srcu_data structures are not created at compile time, but rather at module-load time. This means that the >srcu_reader_flavor field of the srcu_data structure is not available. Therefore, this commit instead creates an ->srcu_reader_flavor field in the srcu_struct structure, adds arguments to the DEFINE_SRCU()-related macros to initialize this new field, and extends the checks in the __srcu_check_read_flavor() function to include this new field. This commit also allows dynamically allocated srcu_struct structure to be marked for SRCU-fast readers. It does so by defining a new init_srcu_struct_fast() function that marks the specified srcu_struct structure for use by srcu_read_lock_fast() and friends. Signed-off-by: Paul E. McKenney <paulmck@kernel.org> Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Cc: <bpf@vger.kernel.org> Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
1 parent 950063c commit ee90848

5 files changed

Lines changed: 78 additions & 19 deletions

File tree

‎include/linux/notifier.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
109109
.mutex = __MUTEX_INITIALIZER(name.mutex), \
110110
.head = NULL, \
111111
.srcuu = __SRCU_USAGE_INIT(name.srcuu), \
112-
.srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \
112+
.srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu, 0), \
113113
}
114114

115115
#define ATOMIC_NOTIFIER_HEAD(name) \

‎include/linux/srcu.h‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@ struct srcu_struct;
2525

2626
#ifdef CONFIG_DEBUG_LOCK_ALLOC
2727

28-
int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
29-
struct lock_class_key *key);
28+
int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
29+
#ifndef CONFIG_TINY_SRCU
30+
int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
31+
#endif // #ifndef CONFIG_TINY_SRCU
3032

3133
#define init_srcu_struct(ssp) \
3234
({ \
@@ -35,10 +37,20 @@ int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
3537
__init_srcu_struct((ssp), #ssp, &__srcu_key); \
3638
})
3739

40+
#define init_srcu_struct_fast(ssp) \
41+
({ \
42+
static struct lock_class_key __srcu_key; \
43+
\
44+
__init_srcu_struct_fast((ssp), #ssp, &__srcu_key); \
45+
})
46+
3847
#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },
3948
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
4049

4150
int init_srcu_struct(struct srcu_struct *ssp);
51+
#ifndef CONFIG_TINY_SRCU
52+
int init_srcu_struct_fast(struct srcu_struct *ssp);
53+
#endif // #ifndef CONFIG_TINY_SRCU
4254

4355
#define __SRCU_DEP_MAP_INIT(srcu_name)
4456
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */

‎include/linux/srcutiny.h‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ struct srcu_struct {
3131

3232
void srcu_drive_gp(struct work_struct *wp);
3333

34-
#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored) \
34+
#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored) \
3535
{ \
3636
.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \
3737
.srcu_cb_tail = &name.srcu_cb_head, \
@@ -44,13 +44,20 @@ void srcu_drive_gp(struct work_struct *wp);
4444
* Tree SRCU, which needs some per-CPU data.
4545
*/
4646
#define DEFINE_SRCU(name) \
47-
struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
47+
struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
4848
#define DEFINE_STATIC_SRCU(name) \
49-
static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
49+
static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
50+
#define DEFINE_SRCU_FAST(name) DEFINE_SRCU(name)
51+
#define DEFINE_STATIC_SRCU_FAST(name) \
52+
static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
5053

5154
// Dummy structure for srcu_notifier_head.
5255
struct srcu_usage { };
5356
#define __SRCU_USAGE_INIT(name) { }
57+
#define __init_srcu_struct_fast __init_srcu_struct
58+
#ifndef CONFIG_DEBUG_LOCK_ALLOC
59+
#define init_srcu_struct_fast init_srcu_struct
60+
#endif // #ifndef CONFIG_DEBUG_LOCK_ALLOC
5461

5562
void synchronize_srcu(struct srcu_struct *ssp);
5663

‎include/linux/srcutree.h‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ struct srcu_usage {
104104
struct srcu_struct {
105105
struct srcu_ctr __percpu *srcu_ctrp;
106106
struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */
107+
u8 srcu_reader_flavor;
107108
struct lockdep_map dep_map;
108109
struct srcu_usage *srcu_sup; /* Update-side data. */
109110
};
@@ -162,20 +163,21 @@ struct srcu_struct {
162163
.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \
163164
}
164165

165-
#define __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
166+
#define __SRCU_STRUCT_INIT_COMMON(name, usage_name, fast) \
166167
.srcu_sup = &usage_name, \
168+
.srcu_reader_flavor = fast, \
167169
__SRCU_DEP_MAP_INIT(name)
168170

169-
#define __SRCU_STRUCT_INIT_MODULE(name, usage_name) \
171+
#define __SRCU_STRUCT_INIT_MODULE(name, usage_name, fast) \
170172
{ \
171-
__SRCU_STRUCT_INIT_COMMON(name, usage_name) \
173+
__SRCU_STRUCT_INIT_COMMON(name, usage_name, fast) \
172174
}
173175

174-
#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name) \
176+
#define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name, fast) \
175177
{ \
176178
.sda = &pcpu_name, \
177179
.srcu_ctrp = &pcpu_name.srcu_ctrs[0], \
178-
__SRCU_STRUCT_INIT_COMMON(name, usage_name) \
180+
__SRCU_STRUCT_INIT_COMMON(name, usage_name, fast) \
179181
}
180182

181183
/*
@@ -196,23 +198,29 @@ struct srcu_struct {
196198
* init_srcu_struct(&my_srcu);
197199
*
198200
* See include/linux/percpu-defs.h for the rules on per-CPU variables.
201+
*
202+
* DEFINE_SRCU_FAST() creates an srcu_struct and associated structures
203+
* whose readers must be of the SRCU-fast variety.
199204
*/
200205
#ifdef MODULE
201-
# define __DEFINE_SRCU(name, is_static) \
206+
# define __DEFINE_SRCU(name, fast, is_static) \
202207
static struct srcu_usage name##_srcu_usage = __SRCU_USAGE_INIT(name##_srcu_usage); \
203-
is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage); \
208+
is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage, \
209+
fast); \
204210
extern struct srcu_struct * const __srcu_struct_##name; \
205211
struct srcu_struct * const __srcu_struct_##name \
206212
__section("___srcu_struct_ptrs") = &name
207213
#else
208-
# define __DEFINE_SRCU(name, is_static) \
214+
# define __DEFINE_SRCU(name, fast, is_static) \
209215
static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data); \
210216
static struct srcu_usage name##_srcu_usage = __SRCU_USAGE_INIT(name##_srcu_usage); \
211217
is_static struct srcu_struct name = \
212-
__SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data)
218+
__SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data, fast)
213219
#endif
214-
#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
215-
#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
220+
#define DEFINE_SRCU(name) __DEFINE_SRCU(name, 0, /* not static */)
221+
#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, 0, static)
222+
#define DEFINE_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* not static */)
223+
#define DEFINE_STATIC_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, static)
216224

217225
int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
218226
void synchronize_srcu_expedited(struct srcu_struct *ssp);

‎kernel/rcu/srcutree.c‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,16 +286,29 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
286286

287287
#ifdef CONFIG_DEBUG_LOCK_ALLOC
288288

289-
int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
290-
struct lock_class_key *key)
289+
static int
290+
__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
291291
{
292292
/* Don't re-initialize a lock while it is held. */
293293
debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
294294
lockdep_init_map(&ssp->dep_map, name, key, 0);
295295
return init_srcu_struct_fields(ssp, false);
296296
}
297+
298+
int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
299+
{
300+
ssp->srcu_reader_flavor = 0;
301+
return __init_srcu_struct_common(ssp, name, key);
302+
}
297303
EXPORT_SYMBOL_GPL(__init_srcu_struct);
298304

305+
int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
306+
{
307+
ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST;
308+
return __init_srcu_struct_common(ssp, name, key);
309+
}
310+
EXPORT_SYMBOL_GPL(__init_srcu_struct_fast);
311+
299312
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
300313

301314
/**
@@ -308,10 +321,26 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct);
308321
*/
309322
int init_srcu_struct(struct srcu_struct *ssp)
310323
{
324+
ssp->srcu_reader_flavor = 0;
311325
return init_srcu_struct_fields(ssp, false);
312326
}
313327
EXPORT_SYMBOL_GPL(init_srcu_struct);
314328

329+
/**
330+
* init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure
331+
* @ssp: structure to initialize.
332+
*
333+
* Must invoke this on a given srcu_struct before passing that srcu_struct
334+
* to any other function. Each srcu_struct represents a separate domain
335+
* of SRCU protection.
336+
*/
337+
int init_srcu_struct_fast(struct srcu_struct *ssp)
338+
{
339+
ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST;
340+
return init_srcu_struct_fields(ssp, false);
341+
}
342+
EXPORT_SYMBOL_GPL(init_srcu_struct_fast);
343+
315344
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
316345

317346
/*
@@ -734,6 +763,9 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
734763

735764
sdp = raw_cpu_ptr(ssp->sda);
736765
old_read_flavor = READ_ONCE(sdp->srcu_reader_flavor);
766+
WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor);
767+
WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor &&
768+
old_read_flavor != ssp->srcu_reader_flavor);
737769
if (!old_read_flavor) {
738770
old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor);
739771
if (!old_read_flavor)

0 commit comments

Comments
 (0)