Skip to content

Commit 200b0d2

Browse files
kevin-brodsky-armwilldeacon
kevin-brodsky-arm
authored and
willdeacon
committed
arm64: mm: Move KPTI helpers to mmu.c
create_kpti_ng_temp_pgd() is currently defined (as an alias) in mmu.c without matching declaration in a header; instead cpufeature.c makes its own declaration. This is clearly not pretty, and as commit ceca927 ("arm64: mm: Fix CFI failure due to kpti_ng_pgd_alloc function signature") showed, it also makes it very easy for the prototypes to go out of sync. All this would be much simpler if kpti_install_ng_mappings() and associated functions lived in mmu.c, where they logically belong. This is what this patch does: - Move kpti_install_ng_mappings() and associated functions from cpufeature.c to mmu.c, add a declaration to <asm/mmu.h> - Remove create_kpti_ng_temp_pgd() and just call __create_pgd_mapping_locked() directly instead - Mark all these functions __init - Move __initdata after kpti_ng_temp_alloc (as suggested by checkpatch) Reviewed-by: Ryan Roberts <ryan.roberts@arm.com> Reviewed-by: Ard Biesheuvel <ardb@kernel.org> Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com> Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com> Reviewed-by: Yeoreum Yun <yeoreum.yun@arm.com> [will: Fix conflicts with init_idmap_kpti_bbml2_flag()] Signed-off-by: Will Deacon <will@kernel.org>
1 parent fd2f74f commit 200b0d2

3 files changed

Lines changed: 94 additions & 109 deletions

File tree

‎arch/arm64/include/asm/mmu.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
7979
extern void *fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot);
8080
extern void mark_linear_text_alias_ro(void);
8181
extern int split_kernel_leaf_mapping(unsigned long start, unsigned long end);
82-
extern void init_idmap_kpti_bbml2_flag(void);
8382
extern void linear_map_maybe_split_to_ptes(void);
8483

8584
/*
@@ -107,5 +106,11 @@ static inline bool kaslr_requires_kpti(void)
107106
return true;
108107
}
109108

109+
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
110+
void kpti_install_ng_mappings(void);
111+
#else
112+
static inline void kpti_install_ng_mappings(void) {}
113+
#endif
114+
110115
#endif /* !__ASSEMBLY__ */
111116
#endif

‎arch/arm64/kernel/cpufeature.c‎

Lines changed: 0 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -1941,104 +1941,6 @@ static bool has_pmuv3(const struct arm64_cpu_capabilities *entry, int scope)
19411941
}
19421942
#endif
19431943

1944-
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1945-
#define KPTI_NG_TEMP_VA (-(1UL << PMD_SHIFT))
1946-
1947-
extern
1948-
void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
1949-
phys_addr_t size, pgprot_t prot,
1950-
phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
1951-
1952-
static phys_addr_t __initdata kpti_ng_temp_alloc;
1953-
1954-
static phys_addr_t __init kpti_ng_pgd_alloc(enum pgtable_type type)
1955-
{
1956-
kpti_ng_temp_alloc -= PAGE_SIZE;
1957-
return kpti_ng_temp_alloc;
1958-
}
1959-
1960-
static int __init __kpti_install_ng_mappings(void *__unused)
1961-
{
1962-
typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
1963-
extern kpti_remap_fn idmap_kpti_install_ng_mappings;
1964-
kpti_remap_fn *remap_fn;
1965-
1966-
int cpu = smp_processor_id();
1967-
int levels = CONFIG_PGTABLE_LEVELS;
1968-
int order = order_base_2(levels);
1969-
u64 kpti_ng_temp_pgd_pa = 0;
1970-
pgd_t *kpti_ng_temp_pgd;
1971-
u64 alloc = 0;
1972-
1973-
if (levels == 5 && !pgtable_l5_enabled())
1974-
levels = 4;
1975-
else if (levels == 4 && !pgtable_l4_enabled())
1976-
levels = 3;
1977-
1978-
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
1979-
1980-
if (!cpu) {
1981-
alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
1982-
kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);
1983-
kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd);
1984-
1985-
//
1986-
// Create a minimal page table hierarchy that permits us to map
1987-
// the swapper page tables temporarily as we traverse them.
1988-
//
1989-
// The physical pages are laid out as follows:
1990-
//
1991-
// +--------+-/-------+-/------ +-/------ +-\\\--------+
1992-
// : PTE[] : | PMD[] : | PUD[] : | P4D[] : ||| PGD[] :
1993-
// +--------+-\-------+-\------ +-\------ +-///--------+
1994-
// ^
1995-
// The first page is mapped into this hierarchy at a PMD_SHIFT
1996-
// aligned virtual address, so that we can manipulate the PTE
1997-
// level entries while the mapping is active. The first entry
1998-
// covers the PTE[] page itself, the remaining entries are free
1999-
// to be used as a ad-hoc fixmap.
2000-
//
2001-
create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc),
2002-
KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,
2003-
kpti_ng_pgd_alloc, 0);
2004-
}
2005-
2006-
cpu_install_idmap();
2007-
remap_fn(cpu, num_online_cpus(), kpti_ng_temp_pgd_pa, KPTI_NG_TEMP_VA);
2008-
cpu_uninstall_idmap();
2009-
2010-
if (!cpu) {
2011-
free_pages(alloc, order);
2012-
arm64_use_ng_mappings = true;
2013-
}
2014-
2015-
return 0;
2016-
}
2017-
2018-
static void __init kpti_install_ng_mappings(void)
2019-
{
2020-
/* Check whether KPTI is going to be used */
2021-
if (!arm64_kernel_unmapped_at_el0())
2022-
return;
2023-
2024-
/*
2025-
* We don't need to rewrite the page-tables if either we've done
2026-
* it already or we have KASLR enabled and therefore have not
2027-
* created any global mappings at all.
2028-
*/
2029-
if (arm64_use_ng_mappings)
2030-
return;
2031-
2032-
init_idmap_kpti_bbml2_flag();
2033-
stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask);
2034-
}
2035-
2036-
#else
2037-
static inline void kpti_install_ng_mappings(void)
2038-
{
2039-
}
2040-
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
2041-
20421944
static void cpu_enable_kpti(struct arm64_cpu_capabilities const *cap)
20431945
{
20441946
if (__this_cpu_read(this_cpu_vector) == vectors) {

‎arch/arm64/mm/mmu.c‎

Lines changed: 88 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -470,14 +470,6 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
470470
mutex_unlock(&fixmap_lock);
471471
}
472472

473-
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
474-
extern __alias(__create_pgd_mapping_locked)
475-
void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
476-
phys_addr_t size, pgprot_t prot,
477-
phys_addr_t (*pgtable_alloc)(enum pgtable_type),
478-
int flags);
479-
#endif
480-
481473
#define INVALID_PHYS_ADDR (-1ULL)
482474

483475
static phys_addr_t __pgd_pgtable_alloc(struct mm_struct *mm, gfp_t gfp,
@@ -823,7 +815,7 @@ static bool linear_map_requires_bbml2 __initdata;
823815

824816
u32 idmap_kpti_bbml2_flag;
825817

826-
void __init init_idmap_kpti_bbml2_flag(void)
818+
static void __init init_idmap_kpti_bbml2_flag(void)
827819
{
828820
WRITE_ONCE(idmap_kpti_bbml2_flag, 1);
829821
/* Must be visible to other CPUs before stop_machine() is called. */
@@ -1135,7 +1127,93 @@ static void __init declare_vma(struct vm_struct *vma,
11351127
}
11361128

11371129
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1138-
static pgprot_t kernel_exec_prot(void)
1130+
#define KPTI_NG_TEMP_VA (-(1UL << PMD_SHIFT))
1131+
1132+
static phys_addr_t kpti_ng_temp_alloc __initdata;
1133+
1134+
static phys_addr_t __init kpti_ng_pgd_alloc(enum pgtable_type type)
1135+
{
1136+
kpti_ng_temp_alloc -= PAGE_SIZE;
1137+
return kpti_ng_temp_alloc;
1138+
}
1139+
1140+
static int __init __kpti_install_ng_mappings(void *__unused)
1141+
{
1142+
typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
1143+
extern kpti_remap_fn idmap_kpti_install_ng_mappings;
1144+
kpti_remap_fn *remap_fn;
1145+
1146+
int cpu = smp_processor_id();
1147+
int levels = CONFIG_PGTABLE_LEVELS;
1148+
int order = order_base_2(levels);
1149+
u64 kpti_ng_temp_pgd_pa = 0;
1150+
pgd_t *kpti_ng_temp_pgd;
1151+
u64 alloc = 0;
1152+
1153+
if (levels == 5 && !pgtable_l5_enabled())
1154+
levels = 4;
1155+
else if (levels == 4 && !pgtable_l4_enabled())
1156+
levels = 3;
1157+
1158+
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
1159+
1160+
if (!cpu) {
1161+
alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
1162+
kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);
1163+
kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd);
1164+
1165+
//
1166+
// Create a minimal page table hierarchy that permits us to map
1167+
// the swapper page tables temporarily as we traverse them.
1168+
//
1169+
// The physical pages are laid out as follows:
1170+
//
1171+
// +--------+-/-------+-/------ +-/------ +-\\\--------+
1172+
// : PTE[] : | PMD[] : | PUD[] : | P4D[] : ||| PGD[] :
1173+
// +--------+-\-------+-\------ +-\------ +-///--------+
1174+
// ^
1175+
// The first page is mapped into this hierarchy at a PMD_SHIFT
1176+
// aligned virtual address, so that we can manipulate the PTE
1177+
// level entries while the mapping is active. The first entry
1178+
// covers the PTE[] page itself, the remaining entries are free
1179+
// to be used as a ad-hoc fixmap.
1180+
//
1181+
__create_pgd_mapping_locked(kpti_ng_temp_pgd, __pa(alloc),
1182+
KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,
1183+
kpti_ng_pgd_alloc, 0);
1184+
}
1185+
1186+
cpu_install_idmap();
1187+
remap_fn(cpu, num_online_cpus(), kpti_ng_temp_pgd_pa, KPTI_NG_TEMP_VA);
1188+
cpu_uninstall_idmap();
1189+
1190+
if (!cpu) {
1191+
free_pages(alloc, order);
1192+
arm64_use_ng_mappings = true;
1193+
}
1194+
1195+
return 0;
1196+
}
1197+
1198+
void __init kpti_install_ng_mappings(void)
1199+
{
1200+
/* Check whether KPTI is going to be used */
1201+
if (!arm64_kernel_unmapped_at_el0())
1202+
return;
1203+
1204+
/*
1205+
* We don't need to rewrite the page-tables if either we've done
1206+
* it already or we have KASLR enabled and therefore have not
1207+
* created any global mappings at all.
1208+
*/
1209+
if (arm64_use_ng_mappings)
1210+
return;
1211+
1212+
init_idmap_kpti_bbml2_flag();
1213+
stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask);
1214+
}
1215+
1216+
static pgprot_t __init kernel_exec_prot(void)
11391217
{
11401218
return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
11411219
}

0 commit comments

Comments
 (0)