arm64: Implement Apple IMPDEF TSO memory model control

Apple CPUs may implement the TSO memory model as an optional
configurable mode. This allows x86 emulators to simplify their
load/store handling, greatly increasing performance.

Expose this via the prctl PR_SET_MEM_MODEL_TSO mechanism. We use the
Apple IMPDEF AIDR_EL1 register to check for the availability of TSO
mode, and enable this codepath on all CPUs with an Apple implementer.

This relies on the ACTLR_EL1 thread state scaffolding introduced
earlier.

Signed-off-by: Hector Martin <marcan@marcan.st>
Reviewed-by: Neal Gompa <neal@gompa.dev>

Author: marcan

arch/arm64/Kconfig

@@ -2299,6 +2299,7 @@ endif # ARM64_PSEUDO_NMI
 
 config ARM64_MEMORY_MODEL_CONTROL
 	bool "Runtime memory model control"
+	select ARM64_ACTLR_STATE
 	help
 	  Some ARM64 CPUs support runtime switching of the CPU memory
 	  model, which can be useful to emulate other CPU architectures

arch/arm64/include/asm/apple_cpufeature.h

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#ifndef __ASM_APPLE_CPUFEATURES_H
+#define __ASM_APPLE_CPUFEATURES_H
+
+#include <linux/bits.h>
+#include <asm/sysreg.h>
+
+#define AIDR_APPLE_TSO_SHIFT	9
+#define AIDR_APPLE_TSO		BIT(9)
+
+#define ACTLR_APPLE_TSO_SHIFT	1
+#define ACTLR_APPLE_TSO		BIT(1)
+
+#endif

arch/arm64/include/asm/cpufeature.h

@@ -957,7 +957,8 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
 
 static __always_inline bool system_has_actlr_state(void)
 {
-	return false;
+	return IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) &&
+		alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE);
 }
 
 s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur);

arch/arm64/kernel/cpufeature_impdef.c

@@ -3,9 +3,51 @@
  * Contains implementation-defined CPU feature definitions.
  */
 
+#define pr_fmt(fmt) "CPU features: " fmt
+
 #include <asm/cpufeature.h>
+#include <asm/apple_cpufeature.h>
+#include <linux/irqflags.h>
+#include <linux/preempt.h>
+#include <linux/printk.h>
 
 #ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL
+static bool has_apple_feature(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u64 val;
+	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
+
+	if (read_cpuid_implementor() != ARM_CPU_IMP_APPLE)
+		return false;
+
+	val = read_sysreg(aidr_el1);
+	return cpufeature_matches(val, entry);
+}
+
+static bool has_apple_tso(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u64 val;
+
+	if (!has_apple_feature(entry, scope))
+		return false;
+
+	/*
+	 * KVM and old versions of the macOS hypervisor will advertise TSO in
+	 * AIDR_EL1, but then ignore writes to ACTLR_EL1. Test that the bit is
+	 * actually writable before enabling TSO.
+	 */
+
+	val = read_sysreg(actlr_el1);
+	write_sysreg(val ^ ACTLR_APPLE_TSO, actlr_el1);
+	if (!((val ^ read_sysreg(actlr_el1)) & ACTLR_APPLE_TSO)) {
+		pr_info_once("CPU advertises Apple TSO but it is broken, ignoring\n");
+		return false;
+	}
+
+	write_sysreg(val, actlr_el1);
+	return true;
+}
+
 static bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope)
 {
 	/* List of CPUs that always use the TSO memory model */
@@ -22,6 +64,17 @@ static bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope)
 
 static const struct arm64_cpu_capabilities arm64_impdef_features[] = {
 #ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL
+	{
+		.desc = "TSO memory model (Apple)",
+		.capability = ARM64_HAS_TSO_APPLE,
+		.type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE,
+		.matches = has_apple_tso,
+		.field_pos = AIDR_APPLE_TSO_SHIFT,
+		.field_width = 1,
+		.sign = FTR_UNSIGNED,
+		.min_field_value = 1,
+		.max_field_value = 1,
+	},
 	{
 		.desc = "TSO memory model (Fixed)",
 		.capability = ARM64_HAS_TSO_FIXED,

arch/arm64/kernel/process.c

@@ -44,6 +44,7 @@
 #include <linux/memory_ordering_model.h>
 
 #include <asm/alternative.h>
+#include <asm/apple_cpufeature.h>
 #include <asm/arch_timer.h>
 #include <asm/compat.h>
 #include <asm/cpufeature.h>
@@ -708,6 +709,10 @@ void update_sctlr_el1(u64 sctlr)
 #ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL
 int arch_prctl_mem_model_get(struct task_struct *t)
 {
+	if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE) &&
+		t->thread.actlr & ACTLR_APPLE_TSO)
+		return PR_SET_MEM_MODEL_TSO;
+
 	return PR_SET_MEM_MODEL_DEFAULT;
 }
 
@@ -717,6 +722,23 @@ int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val)
 	    val == PR_SET_MEM_MODEL_TSO)
 		return 0;
 
+	if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE)) {
+		WARN_ON(!system_has_actlr_state());
+
+		switch (val) {
+		case PR_SET_MEM_MODEL_TSO:
+			t->thread.actlr |= ACTLR_APPLE_TSO;
+			break;
+		case PR_SET_MEM_MODEL_DEFAULT:
+			t->thread.actlr &= ~ACTLR_APPLE_TSO;
+			break;
+		default:
+			return -EINVAL;
+		}
+		write_sysreg(t->thread.actlr, actlr_el1);
+		return 0;
+	}
+
 	if (val == PR_SET_MEM_MODEL_DEFAULT)
 		return 0;
 

arch/arm64/tools/cpucaps

@@ -62,6 +62,7 @@ HAS_STAGE2_FWB
 HAS_TCR2
 HAS_TIDCP1
 HAS_TLB_RANGE
+HAS_TSO_APPLE
 HAS_TSO_FIXED
 HAS_VA52
 HAS_VIRT_HOST_EXTN