88 *
99 * Started by: Thomas Gleixner and Ingo Molnar
1010 */
11+ #include <linux/compiler.h>
1112#include <linux/cpu.h>
1213#include <linux/err.h>
1314#include <linux/hrtimer.h>
@@ -204,7 +205,7 @@ static inline void tick_sched_flag_clear(struct tick_sched *ts,
204205
205206static void tick_sched_do_timer (struct tick_sched * ts , ktime_t now )
206207{
207- int cpu = smp_processor_id ();
208+ int tick_cpu , cpu = smp_processor_id ();
208209
209210 /*
210211 * Check if the do_timer duty was dropped. We don't care about
@@ -216,16 +217,18 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
216217 * If nohz_full is enabled, this should not happen because the
217218 * 'tick_do_timer_cpu' CPU never relinquishes.
218219 */
219- if (IS_ENABLED (CONFIG_NO_HZ_COMMON ) &&
220- unlikely (tick_do_timer_cpu == TICK_DO_TIMER_NONE )) {
220+ tick_cpu = READ_ONCE (tick_do_timer_cpu );
221+
222+ if (IS_ENABLED (CONFIG_NO_HZ_COMMON ) && unlikely (tick_cpu == TICK_DO_TIMER_NONE )) {
221223#ifdef CONFIG_NO_HZ_FULL
222224 WARN_ON_ONCE (tick_nohz_full_running );
223225#endif
224- tick_do_timer_cpu = cpu ;
226+ WRITE_ONCE (tick_do_timer_cpu , cpu );
227+ tick_cpu = cpu ;
225228 }
226229
227230 /* Check if jiffies need an update */
228- if (tick_do_timer_cpu == cpu )
231+ if (tick_cpu == cpu )
229232 tick_do_update_jiffies64 (now );
230233
231234 /*
@@ -610,7 +613,7 @@ bool tick_nohz_cpu_hotpluggable(unsigned int cpu)
610613 * timers, workqueues, timekeeping, ...) on behalf of full dynticks
611614 * CPUs. It must remain online when nohz full is enabled.
612615 */
613- if (tick_nohz_full_running && tick_do_timer_cpu == cpu )
616+ if (tick_nohz_full_running && READ_ONCE ( tick_do_timer_cpu ) == cpu )
614617 return false;
615618 return true;
616619}
@@ -891,6 +894,7 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
891894{
892895 u64 basemono , next_tick , delta , expires ;
893896 unsigned long basejiff ;
897+ int tick_cpu ;
894898
895899 basemono = get_jiffies_update (& basejiff );
896900 ts -> last_jiffies = basejiff ;
@@ -947,9 +951,9 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
947951 * Otherwise we can sleep as long as we want.
948952 */
949953 delta = timekeeping_max_deferment ();
950- if ( cpu != tick_do_timer_cpu &&
951- ( tick_do_timer_cpu != TICK_DO_TIMER_NONE ||
952- !tick_sched_flag_test (ts , TS_FLAG_DO_TIMER_LAST )))
954+ tick_cpu = READ_ONCE ( tick_do_timer_cpu );
955+ if ( tick_cpu != cpu &&
956+ ( tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test (ts , TS_FLAG_DO_TIMER_LAST )))
953957 delta = KTIME_MAX ;
954958
955959 /* Calculate the next expiry time */
@@ -970,6 +974,7 @@ static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
970974 unsigned long basejiff = ts -> last_jiffies ;
971975 u64 basemono = ts -> timer_expires_base ;
972976 bool timer_idle = tick_sched_flag_test (ts , TS_FLAG_STOPPED );
977+ int tick_cpu ;
973978 u64 expires ;
974979
975980 /* Make sure we won't be trying to stop it twice in a row. */
@@ -1007,10 +1012,11 @@ static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
10071012 * do_timer() never gets invoked. Keep track of the fact that it
10081013 * was the one which had the do_timer() duty last.
10091014 */
1010- if (cpu == tick_do_timer_cpu ) {
1011- tick_do_timer_cpu = TICK_DO_TIMER_NONE ;
1015+ tick_cpu = READ_ONCE (tick_do_timer_cpu );
1016+ if (tick_cpu == cpu ) {
1017+ WRITE_ONCE (tick_do_timer_cpu , TICK_DO_TIMER_NONE );
10121018 tick_sched_flag_set (ts , TS_FLAG_DO_TIMER_LAST );
1013- } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE ) {
1019+ } else if (tick_cpu != TICK_DO_TIMER_NONE ) {
10141020 tick_sched_flag_clear (ts , TS_FLAG_DO_TIMER_LAST );
10151021 }
10161022
@@ -1173,15 +1179,17 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
11731179 return false;
11741180
11751181 if (tick_nohz_full_enabled ()) {
1182+ int tick_cpu = READ_ONCE (tick_do_timer_cpu );
1183+
11761184 /*
11771185 * Keep the tick alive to guarantee timekeeping progression
11781186 * if there are full dynticks CPUs around
11791187 */
1180- if (tick_do_timer_cpu == cpu )
1188+ if (tick_cpu == cpu )
11811189 return false;
11821190
11831191 /* Should not happen for nohz-full */
1184- if (WARN_ON_ONCE (tick_do_timer_cpu == TICK_DO_TIMER_NONE ))
1192+ if (WARN_ON_ONCE (tick_cpu == TICK_DO_TIMER_NONE ))
11851193 return false;
11861194 }
11871195
0 commit comments