Skip to content

Commit 90450a0

Browse files
torvaldstorvalds
committed
Merge tag 'rcu-fixes-v6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks
Pull RCU fixes from Frederic Weisbecker: - Fix a lock inversion between scheduler and RCU introduced in v6.2-rc4. The scenario could trigger on any user of RCU_NOCB (mostly Android but also nohz_full) - Fix PF_IDLE semantic changes introduced in v6.6-rc3 breaking some RCU-Tasks and RCU-Tasks-Trace expectations as to what exactly is an idle task. This resulted in potential spurious stalls and warnings. * tag 'rcu-fixes-v6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks: rcu/tasks-trace: Handle new PF_IDLE semantics rcu/tasks: Handle new PF_IDLE semantics rcu: Introduce rcu_cpu_online() rcu: Break rcu_node_0 --> &rq->__lock order
2 parents 447cec0 + a80712b commit 90450a0

3 files changed

Lines changed: 63 additions & 14 deletions

File tree

‎kernel/rcu/rcu.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,7 @@ static inline void rcu_expedite_gp(void) { }
500500
static inline void rcu_unexpedite_gp(void) { }
501501
static inline void rcu_async_hurry(void) { }
502502
static inline void rcu_async_relax(void) { }
503+
static inline bool rcu_cpu_online(int cpu) { return true; }
503504
#else /* #ifdef CONFIG_TINY_RCU */
504505
bool rcu_gp_is_normal(void); /* Internal RCU use. */
505506
bool rcu_gp_is_expedited(void); /* Internal RCU use. */
@@ -509,6 +510,7 @@ void rcu_unexpedite_gp(void);
509510
void rcu_async_hurry(void);
510511
void rcu_async_relax(void);
511512
void rcupdate_announce_bootup_oddness(void);
513+
bool rcu_cpu_online(int cpu);
512514
#ifdef CONFIG_TASKS_RCU_GENERIC
513515
void show_rcu_tasks_gp_kthreads(void);
514516
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */

‎kernel/rcu/tasks.h‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -895,10 +895,36 @@ static void rcu_tasks_pregp_step(struct list_head *hop)
895895
synchronize_rcu();
896896
}
897897

898+
/* Check for quiescent states since the pregp's synchronize_rcu() */
899+
static bool rcu_tasks_is_holdout(struct task_struct *t)
900+
{
901+
int cpu;
902+
903+
/* Has the task been seen voluntarily sleeping? */
904+
if (!READ_ONCE(t->on_rq))
905+
return false;
906+
907+
/*
908+
* Idle tasks (or idle injection) within the idle loop are RCU-tasks
909+
* quiescent states. But CPU boot code performed by the idle task
910+
* isn't a quiescent state.
911+
*/
912+
if (is_idle_task(t))
913+
return false;
914+
915+
cpu = task_cpu(t);
916+
917+
/* Idle tasks on offline CPUs are RCU-tasks quiescent states. */
918+
if (t == idle_task(cpu) && !rcu_cpu_online(cpu))
919+
return false;
920+
921+
return true;
922+
}
923+
898924
/* Per-task initial processing. */
899925
static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop)
900926
{
901-
if (t != current && READ_ONCE(t->on_rq) && !is_idle_task(t)) {
927+
if (t != current && rcu_tasks_is_holdout(t)) {
902928
get_task_struct(t);
903929
t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);
904930
WRITE_ONCE(t->rcu_tasks_holdout, true);
@@ -947,7 +973,7 @@ static void check_holdout_task(struct task_struct *t,
947973

948974
if (!READ_ONCE(t->rcu_tasks_holdout) ||
949975
t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) ||
950-
!READ_ONCE(t->on_rq) ||
976+
!rcu_tasks_is_holdout(t) ||
951977
(IS_ENABLED(CONFIG_NO_HZ_FULL) &&
952978
!is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {
953979
WRITE_ONCE(t->rcu_tasks_holdout, false);
@@ -1525,7 +1551,7 @@ static int trc_inspect_reader(struct task_struct *t, void *bhp_in)
15251551
} else {
15261552
// The task is not running, so C-language access is safe.
15271553
nesting = t->trc_reader_nesting;
1528-
WARN_ON_ONCE(ofl && task_curr(t) && !is_idle_task(t));
1554+
WARN_ON_ONCE(ofl && task_curr(t) && (t != idle_task(task_cpu(t))));
15291555
if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && ofl)
15301556
n_heavy_reader_ofl_updates++;
15311557
}

‎kernel/rcu/tree.c‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -755,14 +755,19 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp)
755755
}
756756

757757
/*
758-
* Return true if the specified CPU has passed through a quiescent
759-
* state by virtue of being in or having passed through an dynticks
760-
* idle state since the last call to dyntick_save_progress_counter()
761-
* for this same CPU, or by virtue of having been offline.
758+
* Returns positive if the specified CPU has passed through a quiescent state
759+
* by virtue of being in or having passed through an dynticks idle state since
760+
* the last call to dyntick_save_progress_counter() for this same CPU, or by
761+
* virtue of having been offline.
762+
*
763+
* Returns negative if the specified CPU needs a force resched.
764+
*
765+
* Returns zero otherwise.
762766
*/
763767
static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
764768
{
765769
unsigned long jtsq;
770+
int ret = 0;
766771
struct rcu_node *rnp = rdp->mynode;
767772

768773
/*
@@ -848,8 +853,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
848853
(time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) ||
849854
rcu_state.cbovld)) {
850855
WRITE_ONCE(rdp->rcu_urgent_qs, true);
851-
resched_cpu(rdp->cpu);
852856
WRITE_ONCE(rdp->last_fqs_resched, jiffies);
857+
ret = -1;
853858
}
854859

855860
/*
@@ -862,8 +867,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
862867
if (time_after(jiffies, rcu_state.jiffies_resched)) {
863868
if (time_after(jiffies,
864869
READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
865-
resched_cpu(rdp->cpu);
866870
WRITE_ONCE(rdp->last_fqs_resched, jiffies);
871+
ret = -1;
867872
}
868873
if (IS_ENABLED(CONFIG_IRQ_WORK) &&
869874
!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
@@ -892,7 +897,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
892897
}
893898
}
894899

895-
return 0;
900+
return ret;
896901
}
897902

898903
/* Trace-event wrapper function for trace_rcu_future_grace_period. */
@@ -2271,15 +2276,15 @@ static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
22712276
{
22722277
int cpu;
22732278
unsigned long flags;
2274-
unsigned long mask;
2275-
struct rcu_data *rdp;
22762279
struct rcu_node *rnp;
22772280

22782281
rcu_state.cbovld = rcu_state.cbovldnext;
22792282
rcu_state.cbovldnext = false;
22802283
rcu_for_each_leaf_node(rnp) {
2284+
unsigned long mask = 0;
2285+
unsigned long rsmask = 0;
2286+
22812287
cond_resched_tasks_rcu_qs();
2282-
mask = 0;
22832288
raw_spin_lock_irqsave_rcu_node(rnp, flags);
22842289
rcu_state.cbovldnext |= !!rnp->cbovldmask;
22852290
if (rnp->qsmask == 0) {
@@ -2297,11 +2302,17 @@ static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
22972302
continue;
22982303
}
22992304
for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) {
2305+
struct rcu_data *rdp;
2306+
int ret;
2307+
23002308
rdp = per_cpu_ptr(&rcu_data, cpu);
2301-
if (f(rdp)) {
2309+
ret = f(rdp);
2310+
if (ret > 0) {
23022311
mask |= rdp->grpmask;
23032312
rcu_disable_urgency_upon_qs(rdp);
23042313
}
2314+
if (ret < 0)
2315+
rsmask |= rdp->grpmask;
23052316
}
23062317
if (mask != 0) {
23072318
/* Idle/offline CPUs, report (releases rnp->lock). */
@@ -2310,6 +2321,9 @@ static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
23102321
/* Nothing to do here, so just drop the lock. */
23112322
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
23122323
}
2324+
2325+
for_each_leaf_node_cpu_mask(rnp, cpu, rsmask)
2326+
resched_cpu(cpu);
23132327
}
23142328
}
23152329

@@ -4195,6 +4209,13 @@ static bool rcu_rdp_cpu_online(struct rcu_data *rdp)
41954209
return !!(rdp->grpmask & rcu_rnp_online_cpus(rdp->mynode));
41964210
}
41974211

4212+
bool rcu_cpu_online(int cpu)
4213+
{
4214+
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
4215+
4216+
return rcu_rdp_cpu_online(rdp);
4217+
}
4218+
41984219
#if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
41994220

42004221
/*

0 commit comments

Comments
 (0)