LoongArch: Adjust memory management for 32BIT/64BIT

Adjust memory management for both 32BIT and 64BIT, including: address
space definition, DMW CSR definition, page table bits definition, boot
time detection of VA/PA bits, page table init, tlb exception handling,
copy_page/clear_page/dump_tlb libraries, etc.

Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
Signed-off-by: Yawei Li <liyawei@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>

Author: chenhuacai

arch/loongarch/include/asm/addrspace.h

@@ -38,11 +38,20 @@ extern unsigned long vm_map_base;
 #endif
 
 #ifndef WRITECOMBINE_BASE
+#ifdef CONFIG_32BIT
+#define WRITECOMBINE_BASE	CSR_DMW0_BASE
+#else
 #define WRITECOMBINE_BASE	CSR_DMW2_BASE
 #endif
+#endif
 
+#ifdef CONFIG_32BIT
+#define DMW_PABITS	29
+#define TO_PHYS_MASK	((_UL(1) << _UL(DMW_PABITS)) - 1)
+#else
 #define DMW_PABITS	48
 #define TO_PHYS_MASK	((_ULL(1) << _ULL(DMW_PABITS)) - 1)
+#endif
 
 /*
  * Memory above this physical address will be considered highmem.
@@ -112,7 +121,11 @@ extern unsigned long vm_map_base;
 /*
  * Returns the physical address of a KPRANGEx / XKPRANGE address
  */
+#ifdef CONFIG_32BIT
+#define PHYSADDR(a)		((_ACAST32_(a)) & TO_PHYS_MASK)
+#else
 #define PHYSADDR(a)		((_ACAST64_(a)) & TO_PHYS_MASK)
+#endif
 
 /*
  * On LoongArch, I/O ports mappring is following:

arch/loongarch/include/asm/cpu-features.h

@@ -20,16 +20,13 @@
 #define cpu_has_loongarch64		(cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT)
 
 #ifdef CONFIG_32BIT
-# define cpu_has_64bits			(cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT)
 # define cpu_vabits			31
 # define cpu_pabits			31
 #endif
 
 #ifdef CONFIG_64BIT
-# define cpu_has_64bits			1
 # define cpu_vabits			cpu_data[0].vabits
 # define cpu_pabits			cpu_data[0].pabits
-# define __NEED_ADDRBITS_PROBE
 #endif
 
 /*

arch/loongarch/include/asm/loongarch.h

@@ -912,6 +912,26 @@
 #define LOONGARCH_CSR_DMWIN3		0x183	/* 64 direct map win3: MEM */
 
 /* Direct Map window 0/1/2/3 */
+
+#ifdef CONFIG_32BIT
+
+#define CSR_DMW0_PLV0		(1 << 0)
+#define CSR_DMW0_VSEG		(0x4)
+#define CSR_DMW0_BASE		(CSR_DMW0_VSEG << DMW_PABITS)
+#define CSR_DMW0_INIT		(CSR_DMW0_BASE | CSR_DMW0_PLV0)
+
+#define CSR_DMW1_PLV0		(1 << 0)
+#define CSR_DMW1_MAT		(1 << 4)
+#define CSR_DMW1_VSEG		(0x5)
+#define CSR_DMW1_BASE		(CSR_DMW1_VSEG << DMW_PABITS)
+#define CSR_DMW1_INIT		(CSR_DMW1_BASE | CSR_DMW1_MAT | CSR_DMW1_PLV0)
+
+#define CSR_DMW2_INIT		0x0
+
+#define CSR_DMW3_INIT		0x0
+
+#else
+
 #define CSR_DMW0_PLV0		_CONST64_(1 << 0)
 #define CSR_DMW0_VSEG		_CONST64_(0x8000)
 #define CSR_DMW0_BASE		(CSR_DMW0_VSEG << DMW_PABITS)
@@ -931,6 +951,8 @@
 
 #define CSR_DMW3_INIT		0x0
 
+#endif
+
 /* Performance Counter registers */
 #define LOONGARCH_CSR_PERFCTRL0		0x200	/* 32 perf event 0 config */
 #define LOONGARCH_CSR_PERFCNTR0		0x201	/* 64 perf event 0 count value */
@@ -1388,8 +1410,10 @@ __BUILD_CSR_OP(tlbidx)
 #define ENTRYLO_C_SHIFT		4
 #define ENTRYLO_C		(_ULCAST_(3) << ENTRYLO_C_SHIFT)
 #define ENTRYLO_G		(_ULCAST_(1) << 6)
+#ifdef CONFIG_64BIT
 #define ENTRYLO_NR		(_ULCAST_(1) << 61)
 #define ENTRYLO_NX		(_ULCAST_(1) << 62)
+#endif
 
 /* Values for PageSize register */
 #define PS_4K		0x0000000c

arch/loongarch/include/asm/page.h

@@ -10,7 +10,7 @@
 
 #include <vdso/page.h>
 
-#define HPAGE_SHIFT	(PAGE_SHIFT + PAGE_SHIFT - 3)
+#define HPAGE_SHIFT	(PAGE_SHIFT + PAGE_SHIFT - PTRLOG)
 #define HPAGE_SIZE	(_AC(1, UL) << HPAGE_SHIFT)
 #define HPAGE_MASK	(~(HPAGE_SIZE - 1))
 #define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)

arch/loongarch/include/asm/pgtable-bits.h

@@ -6,6 +6,26 @@
 #define _ASM_PGTABLE_BITS_H
 
 /* Page table bits */
+
+#ifdef CONFIG_32BIT
+#define	_PAGE_VALID_SHIFT	0
+#define	_PAGE_ACCESSED_SHIFT	0  /* Reuse Valid for Accessed */
+#define	_PAGE_DIRTY_SHIFT	1
+#define	_PAGE_PLV_SHIFT		2  /* 2~3, two bits */
+#define	_CACHE_SHIFT		4  /* 4~5, two bits */
+#define	_PAGE_GLOBAL_SHIFT	6
+#define	_PAGE_HUGE_SHIFT	6  /* HUGE is a PMD bit */
+#define	_PAGE_PRESENT_SHIFT	7
+#define	_PAGE_PFN_SHIFT		8
+#define	_PAGE_HGLOBAL_SHIFT	12 /* HGlobal is a PMD bit */
+#define	_PAGE_SWP_EXCLUSIVE_SHIFT 13
+#define	_PAGE_PFN_END_SHIFT	28
+#define	_PAGE_WRITE_SHIFT	29
+#define	_PAGE_MODIFIED_SHIFT	30
+#define	_PAGE_PRESENT_INVALID_SHIFT 31
+#endif
+
+#ifdef CONFIG_64BIT
 #define	_PAGE_VALID_SHIFT	0
 #define	_PAGE_ACCESSED_SHIFT	0  /* Reuse Valid for Accessed */
 #define	_PAGE_DIRTY_SHIFT	1
@@ -18,25 +38,31 @@
 #define	_PAGE_MODIFIED_SHIFT	9
 #define	_PAGE_PROTNONE_SHIFT	10
 #define	_PAGE_SPECIAL_SHIFT	11
-#define	_PAGE_HGLOBAL_SHIFT	12 /* HGlobal is a PMD bit */
 #define	_PAGE_PFN_SHIFT		12
+#define	_PAGE_HGLOBAL_SHIFT	12 /* HGlobal is a PMD bit */
 #define	_PAGE_SWP_EXCLUSIVE_SHIFT 23
 #define	_PAGE_PFN_END_SHIFT	48
 #define	_PAGE_PRESENT_INVALID_SHIFT 60
 #define	_PAGE_NO_READ_SHIFT	61
 #define	_PAGE_NO_EXEC_SHIFT	62
 #define	_PAGE_RPLV_SHIFT	63
+#endif
 
 /* Used by software */
 #define _PAGE_PRESENT		(_ULCAST_(1) << _PAGE_PRESENT_SHIFT)
 #define _PAGE_PRESENT_INVALID	(_ULCAST_(1) << _PAGE_PRESENT_INVALID_SHIFT)
 #define _PAGE_WRITE		(_ULCAST_(1) << _PAGE_WRITE_SHIFT)
 #define _PAGE_ACCESSED		(_ULCAST_(1) << _PAGE_ACCESSED_SHIFT)
 #define _PAGE_MODIFIED		(_ULCAST_(1) << _PAGE_MODIFIED_SHIFT)
+#ifdef CONFIG_32BIT
+#define _PAGE_PROTNONE		0
+#define _PAGE_SPECIAL		0
+#else
 #define _PAGE_PROTNONE		(_ULCAST_(1) << _PAGE_PROTNONE_SHIFT)
 #define _PAGE_SPECIAL		(_ULCAST_(1) << _PAGE_SPECIAL_SHIFT)
+#endif
 
-/* We borrow bit 23 to store the exclusive marker in swap PTEs. */
+/* We borrow bit 13/23 to store the exclusive marker in swap PTEs. */
 #define _PAGE_SWP_EXCLUSIVE	(_ULCAST_(1) << _PAGE_SWP_EXCLUSIVE_SHIFT)
 
 /* Used by TLB hardware (placed in EntryLo*) */
@@ -46,9 +72,15 @@
 #define _PAGE_GLOBAL		(_ULCAST_(1) << _PAGE_GLOBAL_SHIFT)
 #define _PAGE_HUGE		(_ULCAST_(1) << _PAGE_HUGE_SHIFT)
 #define _PAGE_HGLOBAL		(_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT)
+#ifdef CONFIG_32BIT
+#define _PAGE_NO_READ		0
+#define _PAGE_NO_EXEC		0
+#define _PAGE_RPLV		0
+#else
 #define _PAGE_NO_READ		(_ULCAST_(1) << _PAGE_NO_READ_SHIFT)
 #define _PAGE_NO_EXEC		(_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT)
 #define _PAGE_RPLV		(_ULCAST_(1) << _PAGE_RPLV_SHIFT)
+#endif
 #define _CACHE_MASK		(_ULCAST_(3) << _CACHE_SHIFT)
 #define PFN_PTE_SHIFT		(PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT)
 

arch/loongarch/include/asm/pgtable.h

@@ -11,6 +11,7 @@
 
 #include <linux/compiler.h>
 #include <asm/addrspace.h>
+#include <asm/asm.h>
 #include <asm/page.h>
 #include <asm/pgtable-bits.h>
 
@@ -23,37 +24,45 @@
 #endif
 
 #if CONFIG_PGTABLE_LEVELS == 2
-#define PGDIR_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT - 3))
+#define PGDIR_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT - PTRLOG))
 #elif CONFIG_PGTABLE_LEVELS == 3
-#define PMD_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT - 3))
+#define PMD_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT - PTRLOG))
 #define PMD_SIZE	(1UL << PMD_SHIFT)
 #define PMD_MASK	(~(PMD_SIZE-1))
-#define PGDIR_SHIFT	(PMD_SHIFT + (PAGE_SHIFT - 3))
+#define PGDIR_SHIFT	(PMD_SHIFT + (PAGE_SHIFT - PTRLOG))
 #elif CONFIG_PGTABLE_LEVELS == 4
-#define PMD_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT - 3))
+#define PMD_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT - PTRLOG))
 #define PMD_SIZE	(1UL << PMD_SHIFT)
 #define PMD_MASK	(~(PMD_SIZE-1))
-#define PUD_SHIFT	(PMD_SHIFT + (PAGE_SHIFT - 3))
+#define PUD_SHIFT	(PMD_SHIFT + (PAGE_SHIFT - PTRLOG))
 #define PUD_SIZE	(1UL << PUD_SHIFT)
 #define PUD_MASK	(~(PUD_SIZE-1))
-#define PGDIR_SHIFT	(PUD_SHIFT + (PAGE_SHIFT - 3))
+#define PGDIR_SHIFT	(PUD_SHIFT + (PAGE_SHIFT - PTRLOG))
 #endif
 
 #define PGDIR_SIZE	(1UL << PGDIR_SHIFT)
 #define PGDIR_MASK	(~(PGDIR_SIZE-1))
 
-#define VA_BITS		(PGDIR_SHIFT + (PAGE_SHIFT - 3))
+#ifdef CONFIG_32BIT
+#define VA_BITS		32
+#else
+#define VA_BITS		(PGDIR_SHIFT + (PAGE_SHIFT - PTRLOG))
+#endif
 
-#define PTRS_PER_PGD	(PAGE_SIZE >> 3)
+#define PTRS_PER_PGD	(PAGE_SIZE >> PTRLOG)
 #if CONFIG_PGTABLE_LEVELS > 3
-#define PTRS_PER_PUD	(PAGE_SIZE >> 3)
+#define PTRS_PER_PUD	(PAGE_SIZE >> PTRLOG)
 #endif
 #if CONFIG_PGTABLE_LEVELS > 2
-#define PTRS_PER_PMD	(PAGE_SIZE >> 3)
+#define PTRS_PER_PMD	(PAGE_SIZE >> PTRLOG)
 #endif
-#define PTRS_PER_PTE	(PAGE_SIZE >> 3)
+#define PTRS_PER_PTE	(PAGE_SIZE >> PTRLOG)
 
+#ifdef CONFIG_32BIT
+#define USER_PTRS_PER_PGD       (TASK_SIZE / PGDIR_SIZE)
+#else
 #define USER_PTRS_PER_PGD       ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1)
+#endif
 
 #ifndef __ASSEMBLER__
 
@@ -74,11 +83,15 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 
 #define ZERO_PAGE(vaddr)	virt_to_page(empty_zero_page)
 
-/*
- * TLB refill handlers may also map the vmalloc area into xkvrange.
- * Avoid the first couple of pages so NULL pointer dereferences will
- * still reliably trap.
- */
+#ifdef CONFIG_32BIT
+
+#define VMALLOC_START	(vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
+#define VMALLOC_END	(FIXADDR_START - (2 * PAGE_SIZE))
+
+#endif
+
+#ifdef CONFIG_64BIT
+
 #define MODULES_VADDR	(vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE))
 #define MODULES_END	(MODULES_VADDR + SZ_256M)
 
@@ -106,6 +119,8 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 #define KFENCE_AREA_START	(VMEMMAP_END + 1)
 #define KFENCE_AREA_END		(KFENCE_AREA_START + KFENCE_AREA_SIZE - 1)
 
+#endif
+
 #define ptep_get(ptep) READ_ONCE(*(ptep))
 #define pmdp_get(pmdp) READ_ONCE(*(pmdp))
 
@@ -277,7 +292,16 @@ extern void kernel_pte_init(void *addr);
  * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
  * are !pte_none() && !pte_present().
  *
- * Format of swap PTEs:
+ * Format of 32bit swap PTEs:
+ *
+ *   3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
+ *   1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
+ *   <------------ offset -------------> E <- type -> <-- zeroes -->
+ *
+ *   E is the exclusive marker that is not stored in swap entries.
+ *   The zero'ed bits include _PAGE_PRESENT.
+ *
+ * Format of 64bit swap PTEs:
  *
  *   6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3
  *   3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2
@@ -290,16 +314,27 @@ extern void kernel_pte_init(void *addr);
  *   E is the exclusive marker that is not stored in swap entries.
  *   The zero'ed bits include _PAGE_PRESENT and _PAGE_PROTNONE.
  */
+
+#define __SWP_TYPE_BITS		(IS_ENABLED(CONFIG_32BIT) ? 5 : 7)
+#define __SWP_TYPE_MASK		((1UL << __SWP_TYPE_BITS) - 1)
+#define __SWP_TYPE_SHIFT	(IS_ENABLED(CONFIG_32BIT) ? 8 : 16)
+#define __SWP_OFFSET_SHIFT	(__SWP_TYPE_BITS + __SWP_TYPE_SHIFT + 1)
+
 static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
-{ pte_t pte; pte_val(pte) = ((type & 0x7f) << 16) | (offset << 24); return pte; }
+{
+	pte_t pte;
+	pte_val(pte) = ((type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT) | (offset << __SWP_OFFSET_SHIFT);
+	return pte;
+}
 
-#define __swp_type(x)		(((x).val >> 16) & 0x7f)
-#define __swp_offset(x)		((x).val >> 24)
+#define __swp_type(x)		(((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
+#define __swp_offset(x)		((x).val >> __SWP_OFFSET_SHIFT)
 #define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) })
-#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
+
 #define __swp_entry_to_pte(x)	__pte((x).val)
-#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
 #define __swp_entry_to_pmd(x)	__pmd((x).val | _PAGE_HUGE)
+#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
+#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
 
 static inline bool pte_swp_exclusive(pte_t pte)
 {

arch/loongarch/kernel/cpu-probe.c

@@ -106,7 +106,11 @@ EXPORT_SYMBOL(vm_map_base);
 
 static void cpu_probe_addrbits(struct cpuinfo_loongarch *c)
 {
-#ifdef __NEED_ADDRBITS_PROBE
+#ifdef CONFIG_32BIT
+	c->pabits = cpu_pabits;
+	c->vabits = cpu_vabits;
+	vm_map_base = KVRANGE;
+#else
 	c->pabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_PABITS) >> 4;
 	c->vabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_VABITS) >> 12;
 	vm_map_base = 0UL - (1UL << c->vabits);

arch/loongarch/lib/dump_tlb.c

@@ -73,24 +73,32 @@ static void dump_tlb(int first, int last)
 			vwidth, (entryhi & ~0x1fffUL), asidwidth, asid & asidmask);
 
 		/* NR/NX are in awkward places, so mask them off separately */
+#ifdef CONFIG_64BIT
 		pa = entrylo0 & ~(ENTRYLO_NR | ENTRYLO_NX);
+#endif
 		pa = pa & PAGE_MASK;
 		pr_cont("\n\t[");
+#ifdef CONFIG_64BIT
 		pr_cont("nr=%d nx=%d ",
 			(entrylo0 & ENTRYLO_NR) ? 1 : 0,
 			(entrylo0 & ENTRYLO_NX) ? 1 : 0);
+#endif
 		pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld] [",
 			pwidth, pa, c0,
 			(entrylo0 & ENTRYLO_D) ? 1 : 0,
 			(entrylo0 & ENTRYLO_V) ? 1 : 0,
 			(entrylo0 & ENTRYLO_G) ? 1 : 0,
 			(entrylo0 & ENTRYLO_PLV) >> ENTRYLO_PLV_SHIFT);
 		/* NR/NX are in awkward places, so mask them off separately */
+#ifdef CONFIG_64BIT
 		pa = entrylo1 & ~(ENTRYLO_NR | ENTRYLO_NX);
+#endif
 		pa = pa & PAGE_MASK;
+#ifdef CONFIG_64BIT
 		pr_cont("nr=%d nx=%d ",
 			(entrylo1 & ENTRYLO_NR) ? 1 : 0,
 			(entrylo1 & ENTRYLO_NX) ? 1 : 0);
+#endif
 		pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld]\n",
 			pwidth, pa, c1,
 			(entrylo1 & ENTRYLO_D) ? 1 : 0,

arch/loongarch/mm/init.c

@@ -224,7 +224,7 @@ EXPORT_SYMBOL(invalid_pmd_table);
 pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
 EXPORT_SYMBOL(invalid_pte_table);
 
-#ifdef CONFIG_EXECMEM
+#if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR)
 static struct execmem_info execmem_info __ro_after_init;
 
 struct execmem_info __init *execmem_arch_setup(void)
@@ -242,4 +242,4 @@ struct execmem_info __init *execmem_arch_setup(void)
 
 	return &execmem_info;
 }
-#endif /* CONFIG_EXECMEM */
+#endif /* CONFIG_EXECMEM && MODULES_VADDR */