Skip to content

Commit fc5e5c5

Browse files
torvaldstorvalds
committed
Merge tag 'x86_paravirt_for_v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 paravirt updates from Borislav Petkov: - Replace the paravirt patching functionality using the alternatives infrastructure and remove the former - Misc other improvements * tag 'x86_paravirt_for_v6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/alternative: Correct feature bit debug output x86/paravirt: Remove no longer needed paravirt patching code x86/paravirt: Switch mixed paravirt/alternative calls to alternatives x86/alternative: Add indirect call patching x86/paravirt: Move some functions and defines to alternative.c x86/paravirt: Introduce ALT_NOT_XEN x86/paravirt: Make the struct paravirt_patch_site packed x86/paravirt: Use relative reference for the original instruction offset
2 parents 41a80ca + 7991ed4 commit fc5e5c5

13 files changed

Lines changed: 169 additions & 289 deletions

File tree

‎arch/x86/include/asm/alternative.h‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010

1111
#define ALT_FLAG_NOT (1 << 0)
1212
#define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature))
13+
#define ALT_FLAG_DIRECT_CALL (1 << 1)
14+
#define ALT_DIRECT_CALL(feature) ((ALT_FLAG_DIRECT_CALL << ALT_FLAGS_SHIFT) | (feature))
15+
#define ALT_CALL_ALWAYS ALT_DIRECT_CALL(X86_FEATURE_ALWAYS)
1316

1417
#ifndef __ASSEMBLY__
1518

@@ -86,6 +89,8 @@ struct alt_instr {
8689
u8 replacementlen; /* length of new instruction */
8790
} __packed;
8891

92+
extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
93+
8994
/*
9095
* Debug flag that can be tested to see whether alternative
9196
* instructions were patched in already:
@@ -101,11 +106,10 @@ extern void apply_fineibt(s32 *start_retpoline, s32 *end_retpoine,
101106
s32 *start_cfi, s32 *end_cfi);
102107

103108
struct module;
104-
struct paravirt_patch_site;
105109

106110
struct callthunk_sites {
107111
s32 *call_start, *call_end;
108-
struct paravirt_patch_site *pv_start, *pv_end;
112+
struct alt_instr *alt_start, *alt_end;
109113
};
110114

111115
#ifdef CONFIG_CALL_THUNKS
@@ -150,6 +154,8 @@ static inline int alternatives_text_reserved(void *start, void *end)
150154
}
151155
#endif /* CONFIG_SMP */
152156

157+
#define ALT_CALL_INSTR "call BUG_func"
158+
153159
#define b_replacement(num) "664"#num
154160
#define e_replacement(num) "665"#num
155161

@@ -330,6 +336,22 @@ static inline int alternatives_text_reserved(void *start, void *end)
330336
*/
331337
#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
332338

339+
/* Macro for creating assembler functions avoiding any C magic. */
340+
#define DEFINE_ASM_FUNC(func, instr, sec) \
341+
asm (".pushsection " #sec ", \"ax\"\n" \
342+
".global " #func "\n\t" \
343+
".type " #func ", @function\n\t" \
344+
ASM_FUNC_ALIGN "\n" \
345+
#func ":\n\t" \
346+
ASM_ENDBR \
347+
instr "\n\t" \
348+
ASM_RET \
349+
".size " #func ", . - " #func "\n\t" \
350+
".popsection")
351+
352+
void BUG_func(void);
353+
void nop_func(void);
354+
333355
#else /* __ASSEMBLY__ */
334356

335357
#ifdef CONFIG_SMP
@@ -370,6 +392,10 @@ static inline int alternatives_text_reserved(void *start, void *end)
370392
.byte \alt_len
371393
.endm
372394

395+
.macro ALT_CALL_INSTR
396+
call BUG_func
397+
.endm
398+
373399
/*
374400
* Define an alternative between two instructions. If @feature is
375401
* present, early code in apply_alternatives() replaces @oldinstr with

‎arch/x86/include/asm/paravirt.h‎

Lines changed: 23 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,7 @@ static inline void write_cr0(unsigned long x)
142142
static __always_inline unsigned long read_cr2(void)
143143
{
144144
return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2,
145-
"mov %%cr2, %%rax;",
146-
ALT_NOT(X86_FEATURE_XENPV));
145+
"mov %%cr2, %%rax;", ALT_NOT_XEN);
147146
}
148147

149148
static __always_inline void write_cr2(unsigned long x)
@@ -154,13 +153,12 @@ static __always_inline void write_cr2(unsigned long x)
154153
static inline unsigned long __read_cr3(void)
155154
{
156155
return PVOP_ALT_CALL0(unsigned long, mmu.read_cr3,
157-
"mov %%cr3, %%rax;", ALT_NOT(X86_FEATURE_XENPV));
156+
"mov %%cr3, %%rax;", ALT_NOT_XEN);
158157
}
159158

160159
static inline void write_cr3(unsigned long x)
161160
{
162-
PVOP_ALT_VCALL1(mmu.write_cr3, x,
163-
"mov %%rdi, %%cr3", ALT_NOT(X86_FEATURE_XENPV));
161+
PVOP_ALT_VCALL1(mmu.write_cr3, x, "mov %%rdi, %%cr3", ALT_NOT_XEN);
164162
}
165163

166164
static inline void __write_cr4(unsigned long x)
@@ -182,7 +180,7 @@ extern noinstr void pv_native_wbinvd(void);
182180

183181
static __always_inline void wbinvd(void)
184182
{
185-
PVOP_ALT_VCALL0(cpu.wbinvd, "wbinvd", ALT_NOT(X86_FEATURE_XENPV));
183+
PVOP_ALT_VCALL0(cpu.wbinvd, "wbinvd", ALT_NOT_XEN);
186184
}
187185

188186
static inline u64 paravirt_read_msr(unsigned msr)
@@ -390,27 +388,25 @@ static inline void paravirt_release_p4d(unsigned long pfn)
390388
static inline pte_t __pte(pteval_t val)
391389
{
392390
return (pte_t) { PVOP_ALT_CALLEE1(pteval_t, mmu.make_pte, val,
393-
"mov %%rdi, %%rax",
394-
ALT_NOT(X86_FEATURE_XENPV)) };
391+
"mov %%rdi, %%rax", ALT_NOT_XEN) };
395392
}
396393

397394
static inline pteval_t pte_val(pte_t pte)
398395
{
399396
return PVOP_ALT_CALLEE1(pteval_t, mmu.pte_val, pte.pte,
400-
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
397+
"mov %%rdi, %%rax", ALT_NOT_XEN);
401398
}
402399

403400
static inline pgd_t __pgd(pgdval_t val)
404401
{
405402
return (pgd_t) { PVOP_ALT_CALLEE1(pgdval_t, mmu.make_pgd, val,
406-
"mov %%rdi, %%rax",
407-
ALT_NOT(X86_FEATURE_XENPV)) };
403+
"mov %%rdi, %%rax", ALT_NOT_XEN) };
408404
}
409405

410406
static inline pgdval_t pgd_val(pgd_t pgd)
411407
{
412408
return PVOP_ALT_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd,
413-
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
409+
"mov %%rdi, %%rax", ALT_NOT_XEN);
414410
}
415411

416412
#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
@@ -444,14 +440,13 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
444440
static inline pmd_t __pmd(pmdval_t val)
445441
{
446442
return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, mmu.make_pmd, val,
447-
"mov %%rdi, %%rax",
448-
ALT_NOT(X86_FEATURE_XENPV)) };
443+
"mov %%rdi, %%rax", ALT_NOT_XEN) };
449444
}
450445

451446
static inline pmdval_t pmd_val(pmd_t pmd)
452447
{
453448
return PVOP_ALT_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd,
454-
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
449+
"mov %%rdi, %%rax", ALT_NOT_XEN);
455450
}
456451

457452
static inline void set_pud(pud_t *pudp, pud_t pud)
@@ -464,15 +459,15 @@ static inline pud_t __pud(pudval_t val)
464459
pudval_t ret;
465460

466461
ret = PVOP_ALT_CALLEE1(pudval_t, mmu.make_pud, val,
467-
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
462+
"mov %%rdi, %%rax", ALT_NOT_XEN);
468463

469464
return (pud_t) { ret };
470465
}
471466

472467
static inline pudval_t pud_val(pud_t pud)
473468
{
474469
return PVOP_ALT_CALLEE1(pudval_t, mmu.pud_val, pud.pud,
475-
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
470+
"mov %%rdi, %%rax", ALT_NOT_XEN);
476471
}
477472

478473
static inline void pud_clear(pud_t *pudp)
@@ -492,16 +487,15 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
492487
static inline p4d_t __p4d(p4dval_t val)
493488
{
494489
p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val,
495-
"mov %%rdi, %%rax",
496-
ALT_NOT(X86_FEATURE_XENPV));
490+
"mov %%rdi, %%rax", ALT_NOT_XEN);
497491

498492
return (p4d_t) { ret };
499493
}
500494

501495
static inline p4dval_t p4d_val(p4d_t p4d)
502496
{
503497
return PVOP_ALT_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d,
504-
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
498+
"mov %%rdi, %%rax", ALT_NOT_XEN);
505499
}
506500

507501
static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd)
@@ -687,17 +681,17 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
687681
static __always_inline unsigned long arch_local_save_flags(void)
688682
{
689683
return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;",
690-
ALT_NOT(X86_FEATURE_XENPV));
684+
ALT_NOT_XEN);
691685
}
692686

693687
static __always_inline void arch_local_irq_disable(void)
694688
{
695-
PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV));
689+
PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT_XEN);
696690
}
697691

698692
static __always_inline void arch_local_irq_enable(void)
699693
{
700-
PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV));
694+
PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT_XEN);
701695
}
702696

703697
static __always_inline unsigned long arch_local_irq_save(void)
@@ -726,52 +720,25 @@ static __always_inline unsigned long arch_local_irq_save(void)
726720
#undef PVOP_VCALL4
727721
#undef PVOP_CALL4
728722

729-
#define DEFINE_PARAVIRT_ASM(func, instr, sec) \
730-
asm (".pushsection " #sec ", \"ax\"\n" \
731-
".global " #func "\n\t" \
732-
".type " #func ", @function\n\t" \
733-
ASM_FUNC_ALIGN "\n" \
734-
#func ":\n\t" \
735-
ASM_ENDBR \
736-
instr "\n\t" \
737-
ASM_RET \
738-
".size " #func ", . - " #func "\n\t" \
739-
".popsection")
740-
741723
extern void default_banner(void);
742724
void native_pv_lock_init(void) __init;
743725

744726
#else /* __ASSEMBLY__ */
745727

746-
#define _PVSITE(ptype, ops, word, algn) \
747-
771:; \
748-
ops; \
749-
772:; \
750-
.pushsection .parainstructions,"a"; \
751-
.align algn; \
752-
word 771b; \
753-
.byte ptype; \
754-
.byte 772b-771b; \
755-
_ASM_ALIGN; \
756-
.popsection
757-
758-
759728
#ifdef CONFIG_X86_64
760729
#ifdef CONFIG_PARAVIRT_XXL
730+
#ifdef CONFIG_DEBUG_ENTRY
761731

762-
#define PARA_PATCH(off) ((off) / 8)
763-
#define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .quad, 8)
764732
#define PARA_INDIRECT(addr) *addr(%rip)
765733

766-
#ifdef CONFIG_DEBUG_ENTRY
767734
.macro PARA_IRQ_save_fl
768-
PARA_SITE(PARA_PATCH(PV_IRQ_save_fl),
769-
ANNOTATE_RETPOLINE_SAFE;
770-
call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);)
735+
ANNOTATE_RETPOLINE_SAFE;
736+
call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);
771737
.endm
772738

773-
#define SAVE_FLAGS ALTERNATIVE "PARA_IRQ_save_fl;", "pushf; pop %rax;", \
774-
ALT_NOT(X86_FEATURE_XENPV)
739+
#define SAVE_FLAGS ALTERNATIVE_2 "PARA_IRQ_save_fl;", \
740+
"ALT_CALL_INSTR;", ALT_CALL_ALWAYS, \
741+
"pushf; pop %rax;", ALT_NOT_XEN
775742
#endif
776743
#endif /* CONFIG_PARAVIRT_XXL */
777744
#endif /* CONFIG_X86_64 */

0 commit comments

Comments
 (0)