Skip to content

Commit e1e315c

Browse files
ouptonoupton
committed
Merge branch kvm-arm64/misc into kvmarm/next
* kvm-arm64/misc: : Miscellaneous updates : : - Avoid trapping CTR_EL0 on systems with FEAT_EVT, as the register is : commonly read by userspace : : - Make use of FEAT_BTI at hyp stage-1, setting the Guard Page bit to 1 : for executable mappings : : - Use a separate set of pointer authentication keys for the hypervisor : when running in protected mode (i.e. pKVM) : : - Plug a few holes in timer initialization where KVM fails to free the : timer IRQ(s) KVM: arm64: Use different pointer authentication keys for pKVM KVM: arm64: timers: Fix resource leaks in kvm_timer_hyp_init() KVM: arm64: Use BTI for nvhe KVM: arm64: Relax trapping of CTR_EL0 when FEAT_EVT is available Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
2 parents 89a734b + 8c15c2a commit e1e315c

10 files changed

Lines changed: 109 additions & 8 deletions

File tree

‎arch/arm64/include/asm/kvm_arm.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#define HCR_ATA_SHIFT 56
1919
#define HCR_ATA (UL(1) << HCR_ATA_SHIFT)
2020
#define HCR_AMVOFFEN (UL(1) << 51)
21+
#define HCR_TID4 (UL(1) << 49)
2122
#define HCR_FIEN (UL(1) << 47)
2223
#define HCR_FWB (UL(1) << 46)
2324
#define HCR_API (UL(1) << 41)
@@ -86,7 +87,7 @@
8687
#define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \
8788
HCR_BSU_IS | HCR_FB | HCR_TACR | \
8889
HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \
89-
HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 | HCR_TID2)
90+
HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3)
9091
#define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)
9192
#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA)
9293
#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC)

‎arch/arm64/include/asm/kvm_emulate.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,12 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
9090
vcpu->arch.hcr_el2 |= HCR_TVM;
9191
}
9292

93+
if (cpus_have_final_cap(ARM64_HAS_EVT) &&
94+
!cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE))
95+
vcpu->arch.hcr_el2 |= HCR_TID4;
96+
else
97+
vcpu->arch.hcr_el2 |= HCR_TID2;
98+
9399
if (vcpu_el1_is_32bit(vcpu))
94100
vcpu->arch.hcr_el2 &= ~HCR_RW;
95101

‎arch/arm64/include/asm/sysreg.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,7 @@
558558
(BIT(18)) | (BIT(22)) | (BIT(23)) | (BIT(28)) | \
559559
(BIT(29)))
560560

561+
#define SCTLR_EL2_BT (BIT(36))
561562
#ifdef CONFIG_CPU_BIG_ENDIAN
562563
#define ENDIAN_SET_EL2 SCTLR_ELx_EE
563564
#else

‎arch/arm64/kernel/cpufeature.c‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2662,6 +2662,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
26622662
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
26632663
.matches = hvhe_possible,
26642664
},
2665+
{
2666+
.desc = "Enhanced Virtualization Traps",
2667+
.capability = ARM64_HAS_EVT,
2668+
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2669+
.sys_reg = SYS_ID_AA64MMFR2_EL1,
2670+
.sign = FTR_UNSIGNED,
2671+
.field_pos = ID_AA64MMFR2_EL1_EVT_SHIFT,
2672+
.field_width = 4,
2673+
.min_field_value = ID_AA64MMFR2_EL1_EVT_IMP,
2674+
.matches = has_cpuid_feature,
2675+
},
26652676
{},
26662677
};
26672678

‎arch/arm64/kvm/arch_timer.c‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1406,7 +1406,7 @@ int __init kvm_timer_hyp_init(bool has_gic)
14061406
kvm_get_running_vcpus());
14071407
if (err) {
14081408
kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1409-
goto out_free_irq;
1409+
goto out_free_vtimer_irq;
14101410
}
14111411

14121412
static_branch_enable(&has_gic_active_state);
@@ -1422,15 +1422,15 @@ int __init kvm_timer_hyp_init(bool has_gic)
14221422
if (err) {
14231423
kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n",
14241424
host_ptimer_irq, err);
1425-
return err;
1425+
goto out_free_vtimer_irq;
14261426
}
14271427

14281428
if (has_gic) {
14291429
err = irq_set_vcpu_affinity(host_ptimer_irq,
14301430
kvm_get_running_vcpus());
14311431
if (err) {
14321432
kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
1433-
goto out_free_irq;
1433+
goto out_free_ptimer_irq;
14341434
}
14351435
}
14361436

@@ -1439,11 +1439,15 @@ int __init kvm_timer_hyp_init(bool has_gic)
14391439
kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n",
14401440
info->physical_irq);
14411441
err = -ENODEV;
1442-
goto out_free_irq;
1442+
goto out_free_vtimer_irq;
14431443
}
14441444

14451445
return 0;
1446-
out_free_irq:
1446+
1447+
out_free_ptimer_irq:
1448+
if (info->physical_irq > 0)
1449+
free_percpu_irq(host_ptimer_irq, kvm_get_running_vcpus());
1450+
out_free_vtimer_irq:
14471451
free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
14481452
return err;
14491453
}

‎arch/arm64/kvm/arm.c‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
5151
DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
5252
DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
5353

54+
DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
55+
5456
static bool vgic_present;
5557

5658
static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
@@ -2140,6 +2142,26 @@ static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
21402142
return 0;
21412143
}
21422144

2145+
static void pkvm_hyp_init_ptrauth(void)
2146+
{
2147+
struct kvm_cpu_context *hyp_ctxt;
2148+
int cpu;
2149+
2150+
for_each_possible_cpu(cpu) {
2151+
hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu);
2152+
hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long();
2153+
hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long();
2154+
hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long();
2155+
hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long();
2156+
hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long();
2157+
hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long();
2158+
hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long();
2159+
hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long();
2160+
hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long();
2161+
hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long();
2162+
}
2163+
}
2164+
21432165
/* Inits Hyp-mode on all online CPUs */
21442166
static int __init init_hyp_mode(void)
21452167
{
@@ -2301,6 +2323,10 @@ static int __init init_hyp_mode(void)
23012323
kvm_hyp_init_symbols();
23022324

23032325
if (is_protected_kvm_enabled()) {
2326+
if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) &&
2327+
cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH))
2328+
pkvm_hyp_init_ptrauth();
2329+
23042330
init_cpu_logical_map();
23052331

23062332
if (!init_psci_relay()) {

‎arch/arm64/kvm/hyp/nvhe/host.S‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <asm/kvm_arm.h>
1111
#include <asm/kvm_asm.h>
1212
#include <asm/kvm_mmu.h>
13+
#include <asm/kvm_ptrauth.h>
1314

1415
.text
1516

@@ -37,10 +38,43 @@ SYM_FUNC_START(__host_exit)
3738

3839
/* Save the host context pointer in x29 across the function call */
3940
mov x29, x0
41+
42+
#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
43+
alternative_if_not ARM64_HAS_ADDRESS_AUTH
44+
b __skip_pauth_save
45+
alternative_else_nop_endif
46+
47+
alternative_if ARM64_KVM_PROTECTED_MODE
48+
/* Save kernel ptrauth keys. */
49+
add x18, x29, #CPU_APIAKEYLO_EL1
50+
ptrauth_save_state x18, x19, x20
51+
52+
/* Use hyp keys. */
53+
adr_this_cpu x18, kvm_hyp_ctxt, x19
54+
add x18, x18, #CPU_APIAKEYLO_EL1
55+
ptrauth_restore_state x18, x19, x20
56+
isb
57+
alternative_else_nop_endif
58+
__skip_pauth_save:
59+
#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */
60+
4061
bl handle_trap
4162

42-
/* Restore host regs x0-x17 */
4363
__host_enter_restore_full:
64+
/* Restore kernel keys. */
65+
#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
66+
alternative_if_not ARM64_HAS_ADDRESS_AUTH
67+
b __skip_pauth_restore
68+
alternative_else_nop_endif
69+
70+
alternative_if ARM64_KVM_PROTECTED_MODE
71+
add x18, x29, #CPU_APIAKEYLO_EL1
72+
ptrauth_restore_state x18, x19, x20
73+
alternative_else_nop_endif
74+
__skip_pauth_restore:
75+
#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */
76+
77+
/* Restore host regs x0-x17 */
4478
ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)]
4579
ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)]
4680
ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)]

‎arch/arm64/kvm/hyp/nvhe/hyp-init.S‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,13 @@ alternative_if ARM64_HAS_ADDRESS_AUTH
141141
SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)
142142
orr x0, x0, x1
143143
alternative_else_nop_endif
144+
145+
#ifdef CONFIG_ARM64_BTI_KERNEL
146+
alternative_if ARM64_BTI
147+
orr x0, x0, #SCTLR_EL2_BT
148+
alternative_else_nop_endif
149+
#endif /* CONFIG_ARM64_BTI_KERNEL */
150+
144151
msr sctlr_el2, x0
145152
isb
146153

@@ -210,6 +217,11 @@ SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu)
210217
SYM_CODE_END(__kvm_hyp_init_cpu)
211218

212219
SYM_CODE_START(__kvm_handle_stub_hvc)
220+
/*
221+
* __kvm_handle_stub_hvc called from __host_hvc through branch instruction(br) so
222+
* we need bti j at beginning.
223+
*/
224+
bti j
213225
cmp x0, #HVC_SOFT_RESTART
214226
b.ne 1f
215227

‎arch/arm64/kvm/hyp/pgtable.c‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,16 @@
3636
#define KVM_PTE_LEAF_ATTR_LO_S2_SH_IS 3
3737
#define KVM_PTE_LEAF_ATTR_LO_S2_AF BIT(10)
3838

39-
#define KVM_PTE_LEAF_ATTR_HI GENMASK(63, 51)
39+
#define KVM_PTE_LEAF_ATTR_HI GENMASK(63, 50)
4040

4141
#define KVM_PTE_LEAF_ATTR_HI_SW GENMASK(58, 55)
4242

4343
#define KVM_PTE_LEAF_ATTR_HI_S1_XN BIT(54)
4444

4545
#define KVM_PTE_LEAF_ATTR_HI_S2_XN BIT(54)
4646

47+
#define KVM_PTE_LEAF_ATTR_HI_S1_GP BIT(50)
48+
4749
#define KVM_PTE_LEAF_ATTR_S2_PERMS (KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R | \
4850
KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | \
4951
KVM_PTE_LEAF_ATTR_HI_S2_XN)
@@ -386,6 +388,9 @@ static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)
386388

387389
if (device)
388390
return -EINVAL;
391+
392+
if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) && system_supports_bti())
393+
attr |= KVM_PTE_LEAF_ATTR_HI_S1_GP;
389394
} else {
390395
attr |= KVM_PTE_LEAF_ATTR_HI_S1_XN;
391396
}

‎arch/arm64/tools/cpucaps‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ HAS_E0PD
2525
HAS_ECV
2626
HAS_ECV_CNTPOFF
2727
HAS_EPAN
28+
HAS_EVT
2829
HAS_GENERIC_AUTH
2930
HAS_GENERIC_AUTH_ARCH_QARMA3
3031
HAS_GENERIC_AUTH_ARCH_QARMA5

0 commit comments

Comments
 (0)