Skip to content

Commit 05c58a6

Browse files
author
Vasily Gorbik
committed
Merge branch 'idle-vtime-fixes-cleanups' into fixes
Pull a small idle/vtime series with two cputime accounting fixes plus related cleanups and minor improvements. * idle-vtime-fixes-cleanups: s390/idle: Remove psw_idle() prototype s390/vtime: Use lockdep_assert_irqs_disabled() instead of BUG_ON() s390/vtime: Use __this_cpu_read() / get rid of READ_ONCE() s390/irq/idle: Remove psw bits early s390/idle: Inline update_timer_idle() s390/idle: Slightly optimize idle time accounting s390/idle: Add comment for non obvious code s390/vtime: Fix virtual timer forwarding s390/idle: Fix cpu idle exit cpu time accounting Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>
2 parents 6de23f8 + acbc043 commit 05c58a6

6 files changed

Lines changed: 63 additions & 64 deletions

File tree

‎arch/s390/include/asm/idle.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ struct s390_idle_data {
1919
unsigned long mt_cycles_enter[8];
2020
};
2121

22+
DECLARE_PER_CPU(struct s390_idle_data, s390_idle);
23+
2224
extern struct device_attribute dev_attr_idle_count;
2325
extern struct device_attribute dev_attr_idle_time_us;
2426

25-
void psw_idle(struct s390_idle_data *data, unsigned long psw_mask);
26-
2727
#endif /* _S390_IDLE_H */

‎arch/s390/include/asm/vtime.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
#ifndef _S390_VTIME_H
33
#define _S390_VTIME_H
44

5+
#include <asm/lowcore.h>
6+
#include <asm/cpu_mf.h>
7+
#include <asm/idle.h>
8+
9+
DECLARE_PER_CPU(u64, mt_cycles[8]);
10+
511
static inline void update_timer_sys(void)
612
{
713
struct lowcore *lc = get_lowcore();
@@ -20,4 +26,32 @@ static inline void update_timer_mcck(void)
2026
lc->last_update_timer = lc->mcck_enter_timer;
2127
}
2228

29+
static inline void update_timer_idle(void)
30+
{
31+
struct s390_idle_data *idle = this_cpu_ptr(&s390_idle);
32+
struct lowcore *lc = get_lowcore();
33+
u64 cycles_new[8];
34+
int i, mtid;
35+
36+
mtid = smp_cpu_mtid;
37+
if (mtid) {
38+
stcctm(MT_DIAG, mtid, cycles_new);
39+
for (i = 0; i < mtid; i++)
40+
__this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]);
41+
}
42+
/*
43+
* This is a bit subtle: Forward last_update_clock so it excludes idle
44+
* time. For correct steal time calculation in do_account_vtime() add
45+
* passed wall time before idle_enter to steal_timer:
46+
* During the passed wall time before idle_enter CPU time may have
47+
* been accounted to system, hardirq, softirq, etc. lowcore fields.
48+
* The accounted CPU times will be subtracted again from steal_timer
49+
* when accumulated steal time is calculated in do_account_vtime().
50+
*/
51+
lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock;
52+
lc->last_update_clock = lc->int_clock;
53+
lc->system_timer += lc->last_update_timer - idle->timer_idle_enter;
54+
lc->last_update_timer = lc->sys_enter_timer;
55+
}
56+
2357
#endif /* _S390_VTIME_H */

‎arch/s390/kernel/entry.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ long sys_s390_pci_mmio_write(unsigned long, const void __user *, size_t);
5656
long sys_s390_pci_mmio_read(unsigned long, void __user *, size_t);
5757
long sys_s390_sthyi(unsigned long function_code, void __user *buffer, u64 __user *return_code, unsigned long flags);
5858

59-
DECLARE_PER_CPU(u64, mt_cycles[8]);
60-
6159
unsigned long stack_alloc(void);
6260
void stack_free(unsigned long stack);
6361

‎arch/s390/kernel/idle.c‎

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,37 +15,22 @@
1515
#include <trace/events/power.h>
1616
#include <asm/cpu_mf.h>
1717
#include <asm/cputime.h>
18+
#include <asm/idle.h>
1819
#include <asm/nmi.h>
1920
#include <asm/smp.h>
20-
#include "entry.h"
2121

22-
static DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
22+
DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
2323

2424
void account_idle_time_irq(void)
2525
{
2626
struct s390_idle_data *idle = this_cpu_ptr(&s390_idle);
27-
struct lowcore *lc = get_lowcore();
2827
unsigned long idle_time;
29-
u64 cycles_new[8];
30-
int i;
3128

32-
if (smp_cpu_mtid) {
33-
stcctm(MT_DIAG, smp_cpu_mtid, cycles_new);
34-
for (i = 0; i < smp_cpu_mtid; i++)
35-
this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]);
36-
}
37-
38-
idle_time = lc->int_clock - idle->clock_idle_enter;
39-
40-
lc->steal_timer += idle->clock_idle_enter - lc->last_update_clock;
41-
lc->last_update_clock = lc->int_clock;
42-
43-
lc->system_timer += lc->last_update_timer - idle->timer_idle_enter;
44-
lc->last_update_timer = lc->sys_enter_timer;
29+
idle_time = get_lowcore()->int_clock - idle->clock_idle_enter;
4530

4631
/* Account time spent with enabled wait psw loaded as idle time. */
47-
WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time);
48-
WRITE_ONCE(idle->idle_count, READ_ONCE(idle->idle_count) + 1);
32+
__atomic64_add(idle_time, &idle->idle_time);
33+
__atomic64_add_const(1, &idle->idle_count);
4934
account_idle_time(cputime_to_nsecs(idle_time));
5035
}
5136

‎arch/s390/kernel/irq.c‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,12 @@ void noinstr do_io_irq(struct pt_regs *regs)
146146
struct pt_regs *old_regs = set_irq_regs(regs);
147147
bool from_idle;
148148

149+
from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT);
150+
if (from_idle) {
151+
update_timer_idle();
152+
regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT);
153+
}
154+
149155
irq_enter_rcu();
150156

151157
if (user_mode(regs)) {
@@ -154,7 +160,6 @@ void noinstr do_io_irq(struct pt_regs *regs)
154160
current->thread.last_break = regs->last_break;
155161
}
156162

157-
from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT);
158163
if (from_idle)
159164
account_idle_time_irq();
160165

@@ -171,9 +176,6 @@ void noinstr do_io_irq(struct pt_regs *regs)
171176

172177
set_irq_regs(old_regs);
173178
irqentry_exit(regs, state);
174-
175-
if (from_idle)
176-
regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT);
177179
}
178180

179181
void noinstr do_ext_irq(struct pt_regs *regs)
@@ -182,6 +184,12 @@ void noinstr do_ext_irq(struct pt_regs *regs)
182184
struct pt_regs *old_regs = set_irq_regs(regs);
183185
bool from_idle;
184186

187+
from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT);
188+
if (from_idle) {
189+
update_timer_idle();
190+
regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT);
191+
}
192+
185193
irq_enter_rcu();
186194

187195
if (user_mode(regs)) {
@@ -194,7 +202,6 @@ void noinstr do_ext_irq(struct pt_regs *regs)
194202
regs->int_parm = get_lowcore()->ext_params;
195203
regs->int_parm_long = get_lowcore()->ext_params2;
196204

197-
from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT);
198205
if (from_idle)
199206
account_idle_time_irq();
200207

@@ -203,9 +210,6 @@ void noinstr do_ext_irq(struct pt_regs *regs)
203210
irq_exit_rcu();
204211
set_irq_regs(old_regs);
205212
irqentry_exit(regs, state);
206-
207-
if (from_idle)
208-
regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT);
209213
}
210214

211215
static void show_msi_interrupt(struct seq_file *p, int irq)

‎arch/s390/kernel/vtime.c‎

Lines changed: 10 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,7 @@ static inline void set_vtimer(u64 expires)
4848

4949
static inline int virt_timer_forward(u64 elapsed)
5050
{
51-
BUG_ON(!irqs_disabled());
52-
51+
lockdep_assert_irqs_disabled();
5352
if (list_empty(&virt_timer_list))
5453
return 0;
5554
elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
@@ -137,23 +136,16 @@ static int do_account_vtime(struct task_struct *tsk)
137136
lc->system_timer += timer;
138137

139138
/* Update MT utilization calculation */
140-
if (smp_cpu_mtid &&
141-
time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
139+
if (smp_cpu_mtid && time_after64(jiffies_64, __this_cpu_read(mt_scaling_jiffies)))
142140
update_mt_scaling();
143141

144142
/* Calculate cputime delta */
145-
user = update_tsk_timer(&tsk->thread.user_timer,
146-
READ_ONCE(lc->user_timer));
147-
guest = update_tsk_timer(&tsk->thread.guest_timer,
148-
READ_ONCE(lc->guest_timer));
149-
system = update_tsk_timer(&tsk->thread.system_timer,
150-
READ_ONCE(lc->system_timer));
151-
hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
152-
READ_ONCE(lc->hardirq_timer));
153-
softirq = update_tsk_timer(&tsk->thread.softirq_timer,
154-
READ_ONCE(lc->softirq_timer));
155-
lc->steal_timer +=
156-
clock - user - guest - system - hardirq - softirq;
143+
user = update_tsk_timer(&tsk->thread.user_timer, lc->user_timer);
144+
guest = update_tsk_timer(&tsk->thread.guest_timer, lc->guest_timer);
145+
system = update_tsk_timer(&tsk->thread.system_timer, lc->system_timer);
146+
hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, lc->hardirq_timer);
147+
softirq = update_tsk_timer(&tsk->thread.softirq_timer, lc->softirq_timer);
148+
lc->steal_timer += clock - user - guest - system - hardirq - softirq;
157149

158150
/* Push account value */
159151
if (user) {
@@ -225,10 +217,6 @@ static u64 vtime_delta(void)
225217
return timer - lc->last_update_timer;
226218
}
227219

228-
/*
229-
* Update process times based on virtual cpu times stored by entry.S
230-
* to the lowcore fields user_timer, system_timer & steal_clock.
231-
*/
232220
void vtime_account_kernel(struct task_struct *tsk)
233221
{
234222
struct lowcore *lc = get_lowcore();
@@ -238,27 +226,17 @@ void vtime_account_kernel(struct task_struct *tsk)
238226
lc->guest_timer += delta;
239227
else
240228
lc->system_timer += delta;
241-
242-
virt_timer_forward(delta);
243229
}
244230
EXPORT_SYMBOL_GPL(vtime_account_kernel);
245231

246232
void vtime_account_softirq(struct task_struct *tsk)
247233
{
248-
u64 delta = vtime_delta();
249-
250-
get_lowcore()->softirq_timer += delta;
251-
252-
virt_timer_forward(delta);
234+
get_lowcore()->softirq_timer += vtime_delta();
253235
}
254236

255237
void vtime_account_hardirq(struct task_struct *tsk)
256238
{
257-
u64 delta = vtime_delta();
258-
259-
get_lowcore()->hardirq_timer += delta;
260-
261-
virt_timer_forward(delta);
239+
get_lowcore()->hardirq_timer += vtime_delta();
262240
}
263241

264242
/*

0 commit comments

Comments
 (0)