Skip to content

Commit b9f5621

Browse files
Like XuPeter Zijlstra
Like Xu
authored and
Peter Zijlstra
committed
perf/core: Rework guest callbacks to prepare for static_call support
To prepare for using static_calls to optimize perf's guest callbacks, replace ->is_in_guest and ->is_user_mode with a new multiplexed hook ->state, tweak ->handle_intel_pt_intr to play nice with being called when there is no active guest, and drop "guest" from ->get_guest_ip. Return '0' from ->state and ->handle_intel_pt_intr to indicate "not in guest" so that DEFINE_STATIC_CALL_RET0 can be used to define the static calls, i.e. no callback == !guest. [sean: extracted from static_call patch, fixed get_ip() bug, wrote changelog] Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org> Originally-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Like Xu <like.xu@linux.intel.com> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com> Signed-off-by: Sean Christopherson <seanjc@google.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Boris Ostrovsky <boris.ostrovsky@oracle.com> Reviewed-by: Paolo Bonzini <pbonzini@redhat.com> Link: https://lore.kernel.org/r/20211111020738.2512932-7-seanjc@google.com
1 parent 84af21d commit b9f5621

9 files changed

Lines changed: 73 additions & 79 deletions

File tree

‎arch/arm64/kernel/perf_callchain.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
104104
{
105105
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
106106

107-
if (guest_cbs && guest_cbs->is_in_guest()) {
107+
if (guest_cbs && guest_cbs->state()) {
108108
/* We don't support guest os callchain now */
109109
return;
110110
}
@@ -152,7 +152,7 @@ void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
152152
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
153153
struct stackframe frame;
154154

155-
if (guest_cbs && guest_cbs->is_in_guest()) {
155+
if (guest_cbs && guest_cbs->state()) {
156156
/* We don't support guest os callchain now */
157157
return;
158158
}
@@ -165,19 +165,20 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs)
165165
{
166166
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
167167

168-
if (guest_cbs && guest_cbs->is_in_guest())
169-
return guest_cbs->get_guest_ip();
168+
if (guest_cbs && guest_cbs->state())
169+
return guest_cbs->get_ip();
170170

171171
return instruction_pointer(regs);
172172
}
173173

174174
unsigned long perf_misc_flags(struct pt_regs *regs)
175175
{
176176
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
177+
unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0;
177178
int misc = 0;
178179

179-
if (guest_cbs && guest_cbs->is_in_guest()) {
180-
if (guest_cbs->is_user_mode())
180+
if (guest_state) {
181+
if (guest_state & PERF_GUEST_USER)
181182
misc |= PERF_RECORD_MISC_GUEST_USER;
182183
else
183184
misc |= PERF_RECORD_MISC_GUEST_KERNEL;

‎arch/arm64/kvm/perf.c‎

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -13,39 +13,34 @@
1313

1414
DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available);
1515

16-
static int kvm_is_in_guest(void)
16+
static unsigned int kvm_guest_state(void)
1717
{
18-
return kvm_get_running_vcpu() != NULL;
19-
}
20-
21-
static int kvm_is_user_mode(void)
22-
{
23-
struct kvm_vcpu *vcpu;
18+
struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
19+
unsigned int state;
2420

25-
vcpu = kvm_get_running_vcpu();
21+
if (!vcpu)
22+
return 0;
2623

27-
if (vcpu)
28-
return !vcpu_mode_priv(vcpu);
24+
state = PERF_GUEST_ACTIVE;
25+
if (!vcpu_mode_priv(vcpu))
26+
state |= PERF_GUEST_USER;
2927

30-
return 0;
28+
return state;
3129
}
3230

3331
static unsigned long kvm_get_guest_ip(void)
3432
{
35-
struct kvm_vcpu *vcpu;
36-
37-
vcpu = kvm_get_running_vcpu();
33+
struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
3834

39-
if (vcpu)
40-
return *vcpu_pc(vcpu);
35+
if (WARN_ON_ONCE(!vcpu))
36+
return 0;
4137

42-
return 0;
38+
return *vcpu_pc(vcpu);
4339
}
4440

4541
static struct perf_guest_info_callbacks kvm_guest_cbs = {
46-
.is_in_guest = kvm_is_in_guest,
47-
.is_user_mode = kvm_is_user_mode,
48-
.get_guest_ip = kvm_get_guest_ip,
42+
.state = kvm_guest_state,
43+
.get_ip = kvm_get_guest_ip,
4944
};
5045

5146
void kvm_perf_init(void)

‎arch/x86/events/core.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2772,7 +2772,7 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re
27722772
struct unwind_state state;
27732773
unsigned long addr;
27742774

2775-
if (guest_cbs && guest_cbs->is_in_guest()) {
2775+
if (guest_cbs && guest_cbs->state()) {
27762776
/* TODO: We don't support guest os callchain now */
27772777
return;
27782778
}
@@ -2876,7 +2876,7 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs
28762876
struct stack_frame frame;
28772877
const struct stack_frame __user *fp;
28782878

2879-
if (guest_cbs && guest_cbs->is_in_guest()) {
2879+
if (guest_cbs && guest_cbs->state()) {
28802880
/* TODO: We don't support guest os callchain now */
28812881
return;
28822882
}
@@ -2955,19 +2955,20 @@ unsigned long perf_instruction_pointer(struct pt_regs *regs)
29552955
{
29562956
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
29572957

2958-
if (guest_cbs && guest_cbs->is_in_guest())
2959-
return guest_cbs->get_guest_ip();
2958+
if (guest_cbs && guest_cbs->state())
2959+
return guest_cbs->get_ip();
29602960

29612961
return regs->ip + code_segment_base(regs);
29622962
}
29632963

29642964
unsigned long perf_misc_flags(struct pt_regs *regs)
29652965
{
29662966
struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
2967+
unsigned int guest_state = guest_cbs ? guest_cbs->state() : 0;
29672968
int misc = 0;
29682969

2969-
if (guest_cbs && guest_cbs->is_in_guest()) {
2970-
if (guest_cbs->is_user_mode())
2970+
if (guest_state) {
2971+
if (guest_state & PERF_GUEST_USER)
29712972
misc |= PERF_RECORD_MISC_GUEST_USER;
29722973
else
29732974
misc |= PERF_RECORD_MISC_GUEST_KERNEL;

‎arch/x86/events/intel/core.c‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2906,10 +2906,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
29062906
handled++;
29072907

29082908
guest_cbs = perf_get_guest_cbs();
2909-
if (unlikely(guest_cbs && guest_cbs->is_in_guest() &&
2910-
guest_cbs->handle_intel_pt_intr))
2911-
guest_cbs->handle_intel_pt_intr();
2912-
else
2909+
if (likely(!guest_cbs || !guest_cbs->handle_intel_pt_intr()))
29132910
intel_pt_interrupt();
29142911
}
29152912

‎arch/x86/include/asm/kvm_host.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1895,7 +1895,7 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
18951895
int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
18961896
void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu);
18971897

1898-
int kvm_is_in_guest(void);
1898+
unsigned int kvm_guest_state(void);
18991899

19001900
void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
19011901
u32 size);

‎arch/x86/kvm/pmu.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ static void kvm_perf_overflow_intr(struct perf_event *perf_event,
8787
* woken up. So we should wake it, but this is impossible from
8888
* NMI context. Do it from irq work instead.
8989
*/
90-
if (!kvm_is_in_guest())
90+
if (!kvm_guest_state())
9191
irq_work_queue(&pmc_to_pmu(pmc)->irq_work);
9292
else
9393
kvm_make_request(KVM_REQ_PMI, pmc->vcpu);

‎arch/x86/kvm/x86.c‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8472,44 +8472,48 @@ static void kvm_timer_init(void)
84728472
DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
84738473
EXPORT_PER_CPU_SYMBOL_GPL(current_vcpu);
84748474

8475-
int kvm_is_in_guest(void)
8475+
unsigned int kvm_guest_state(void)
84768476
{
8477-
return __this_cpu_read(current_vcpu) != NULL;
8478-
}
8477+
struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
8478+
unsigned int state;
84798479

8480-
static int kvm_is_user_mode(void)
8481-
{
8482-
int user_mode = 3;
8480+
if (!vcpu)
8481+
return 0;
84838482

8484-
if (__this_cpu_read(current_vcpu))
8485-
user_mode = static_call(kvm_x86_get_cpl)(__this_cpu_read(current_vcpu));
8483+
state = PERF_GUEST_ACTIVE;
8484+
if (static_call(kvm_x86_get_cpl)(vcpu))
8485+
state |= PERF_GUEST_USER;
84868486

8487-
return user_mode != 0;
8487+
return state;
84888488
}
84898489

8490-
static unsigned long kvm_get_guest_ip(void)
8490+
static unsigned long kvm_guest_get_ip(void)
84918491
{
8492-
unsigned long ip = 0;
8492+
struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
84938493

8494-
if (__this_cpu_read(current_vcpu))
8495-
ip = kvm_rip_read(__this_cpu_read(current_vcpu));
8494+
if (WARN_ON_ONCE(!vcpu))
8495+
return 0;
84968496

8497-
return ip;
8497+
return kvm_rip_read(vcpu);
84988498
}
84998499

8500-
static void kvm_handle_intel_pt_intr(void)
8500+
static unsigned int kvm_handle_intel_pt_intr(void)
85018501
{
85028502
struct kvm_vcpu *vcpu = __this_cpu_read(current_vcpu);
85038503

8504+
/* '0' on failure so that the !PT case can use a RET0 static call. */
8505+
if (!vcpu)
8506+
return 0;
8507+
85048508
kvm_make_request(KVM_REQ_PMI, vcpu);
85058509
__set_bit(MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT,
85068510
(unsigned long *)&vcpu->arch.pmu.global_status);
8511+
return 1;
85078512
}
85088513

85098514
static struct perf_guest_info_callbacks kvm_guest_cbs = {
8510-
.is_in_guest = kvm_is_in_guest,
8511-
.is_user_mode = kvm_is_user_mode,
8512-
.get_guest_ip = kvm_get_guest_ip,
8515+
.state = kvm_guest_state,
8516+
.get_ip = kvm_guest_get_ip,
85138517
.handle_intel_pt_intr = NULL,
85148518
};
85158519

‎arch/x86/xen/pmu.c‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -413,34 +413,29 @@ int pmu_apic_update(uint32_t val)
413413
}
414414

415415
/* perf callbacks */
416-
static int xen_is_in_guest(void)
416+
static unsigned int xen_guest_state(void)
417417
{
418418
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
419+
unsigned int state = 0;
419420

420421
if (!xenpmu_data) {
421422
pr_warn_once("%s: pmudata not initialized\n", __func__);
422-
return 0;
423+
return state;
423424
}
424425

425426
if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
426-
return 0;
427+
return state;
427428

428-
return 1;
429-
}
430-
431-
static int xen_is_user_mode(void)
432-
{
433-
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
429+
state |= PERF_GUEST_ACTIVE;
434430

435-
if (!xenpmu_data) {
436-
pr_warn_once("%s: pmudata not initialized\n", __func__);
437-
return 0;
431+
if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) {
432+
if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER)
433+
state |= PERF_GUEST_USER;
434+
} else if (xenpmu_data->pmu.r.regs.cpl & 3) {
435+
state |= PERF_GUEST_USER;
438436
}
439437

440-
if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV)
441-
return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
442-
else
443-
return !!(xenpmu_data->pmu.r.regs.cpl & 3);
438+
return state;
444439
}
445440

446441
static unsigned long xen_get_guest_ip(void)
@@ -456,9 +451,8 @@ static unsigned long xen_get_guest_ip(void)
456451
}
457452

458453
static struct perf_guest_info_callbacks xen_guest_cbs = {
459-
.is_in_guest = xen_is_in_guest,
460-
.is_user_mode = xen_is_user_mode,
461-
.get_guest_ip = xen_get_guest_ip,
454+
.state = xen_guest_state,
455+
.get_ip = xen_get_guest_ip,
462456
};
463457

464458
/* Convert registers from Xen's format to Linux' */

‎include/linux/perf_event.h‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@
2626
# include <asm/local64.h>
2727
#endif
2828

29+
#define PERF_GUEST_ACTIVE 0x01
30+
#define PERF_GUEST_USER 0x02
31+
2932
struct perf_guest_info_callbacks {
30-
int (*is_in_guest)(void);
31-
int (*is_user_mode)(void);
32-
unsigned long (*get_guest_ip)(void);
33-
void (*handle_intel_pt_intr)(void);
33+
unsigned int (*state)(void);
34+
unsigned long (*get_ip)(void);
35+
unsigned int (*handle_intel_pt_intr)(void);
3436
};
3537

3638
#ifdef CONFIG_HAVE_HW_BREAKPOINT

0 commit comments

Comments
 (0)