Skip to content

Commit 18fdb63

Browse files
mrutland-armctmarinas
mrutland-arm
authored and
ctmarinas
committed
arm64: irqchip/gic-v3: Select priorities at boot time
The distributor and PMR/RPR can present different views of the interrupt priority space dependent upon the values of GICD_CTLR.DS and SCR_EL3.FIQ. Currently we treat the distributor's view of the priority space as canonical, and when the two differ we change the way we handle values in the PMR/RPR, using the `gic_nonsecure_priorities` static key to decide what to do. This approach works, but it's sub-optimal. When using pseudo-NMI we manipulate the distributor rarely, and we manipulate the PMR/RPR registers very frequently in code spread out throughout the kernel (e.g. local_irq_{save,restore}()). It would be nicer if we could use fixed values for the PMR/RPR, and dynamically choose the values programmed into the distributor. This patch changes the GICv3 driver and arm64 code accordingly. PMR values are chosen at compile time, and the GICv3 driver determines the appropriate values to program into the distributor at boot time. This removes the need for the `gic_nonsecure_priorities` static key and results in smaller and better generated code for saving/restoring the irqflags. Before this patch, local_irq_disable() compiles to: | 0000000000000000 <outlined_local_irq_disable>: | 0: d503201f nop | 4: d50343df msr daifset, #0x3 | 8: d65f03c0 ret | c: d503201f nop | 10: d2800c00 mov x0, #0x60 // #96 | 14: d5184600 msr icc_pmr_el1, x0 | 18: d65f03c0 ret | 1c: d2801400 mov x0, #0xa0 // #160 | 20: 17fffffd b 14 <outlined_local_irq_disable+0x14> After this patch, local_irq_disable() compiles to: | 0000000000000000 <outlined_local_irq_disable>: | 0: d503201f nop | 4: d50343df msr daifset, #0x3 | 8: d65f03c0 ret | c: d2801800 mov x0, #0xc0 // #192 | 10: d5184600 msr icc_pmr_el1, x0 | 14: d65f03c0 ret ... with 3 fewer instructions per call. For defconfig + CONFIG_PSEUDO_NMI=y, this results in a minor saving of ~4K of text, and will make it easier to make further improvements to the way we manipulate irqflags and DAIF bits. Signed-off-by: Mark Rutland <mark.rutland@arm.com> Cc: Alexandru Elisei <alexandru.elisei@arm.com> Cc: Marc Zyngier <maz@kernel.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Will Deacon <will@kernel.org> Reviewed-by: Marc Zyngier <maz@kernel.org> Tested-by: Marc Zyngier <maz@kernel.org> Link: https://lore.kernel.org/r/20240617111841.2529370-6-mark.rutland@arm.com Signed-off-by: Catalin Marinas <catalin.marinas@arm.com> Acked-by: Thomas Gleixner <tglx@linutronix.de>
1 parent d447bf0 commit 18fdb63

5 files changed

Lines changed: 97 additions & 106 deletions

File tree

‎arch/arm64/include/asm/arch_gicv3.h‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -175,21 +175,6 @@ static inline bool gic_prio_masking_enabled(void)
175175

176176
static inline void gic_pmr_mask_irqs(void)
177177
{
178-
BUILD_BUG_ON(GICD_INT_DEF_PRI < (__GIC_PRIO_IRQOFF |
179-
GIC_PRIO_PSR_I_SET));
180-
BUILD_BUG_ON(GICD_INT_DEF_PRI >= GIC_PRIO_IRQON);
181-
/*
182-
* Need to make sure IRQON allows IRQs when SCR_EL3.FIQ is cleared
183-
* and non-secure PMR accesses are not subject to the shifts that
184-
* are applied to IRQ priorities
185-
*/
186-
BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) >= GIC_PRIO_IRQON);
187-
/*
188-
* Same situation as above, but now we make sure that we can mask
189-
* regular interrupts.
190-
*/
191-
BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) < (__GIC_PRIO_IRQOFF_NS |
192-
GIC_PRIO_PSR_I_SET));
193178
gic_write_pmr(GIC_PRIO_IRQOFF);
194179
}
195180

‎arch/arm64/include/asm/ptrace.h‎

Lines changed: 6 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -21,35 +21,12 @@
2121
#define INIT_PSTATE_EL2 \
2222
(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h)
2323

24-
/*
25-
* PMR values used to mask/unmask interrupts.
26-
*
27-
* GIC priority masking works as follows: if an IRQ's priority is a higher value
28-
* than the value held in PMR, that IRQ is masked. Lowering the value of PMR
29-
* means masking more IRQs (or at least that the same IRQs remain masked).
30-
*
31-
* To mask interrupts, we clear the most significant bit of PMR.
32-
*
33-
* Some code sections either automatically switch back to PSR.I or explicitly
34-
* require to not use priority masking. If bit GIC_PRIO_PSR_I_SET is included
35-
* in the priority mask, it indicates that PSR.I should be set and
36-
* interrupt disabling temporarily does not rely on IRQ priorities.
37-
*/
38-
#define GIC_PRIO_IRQON 0xe0
39-
#define __GIC_PRIO_IRQOFF (GIC_PRIO_IRQON & ~0x80)
40-
#define __GIC_PRIO_IRQOFF_NS 0xa0
41-
#define GIC_PRIO_PSR_I_SET (1 << 4)
42-
43-
#define GIC_PRIO_IRQOFF \
44-
({ \
45-
extern struct static_key_false gic_nonsecure_priorities;\
46-
u8 __prio = __GIC_PRIO_IRQOFF; \
47-
\
48-
if (static_branch_unlikely(&gic_nonsecure_priorities)) \
49-
__prio = __GIC_PRIO_IRQOFF_NS; \
50-
\
51-
__prio; \
52-
})
24+
#include <linux/irqchip/arm-gic-v3-prio.h>
25+
26+
#define GIC_PRIO_IRQON GICV3_PRIO_UNMASKED
27+
#define GIC_PRIO_IRQOFF GICV3_PRIO_IRQ
28+
29+
#define GIC_PRIO_PSR_I_SET GICV3_PRIO_PSR_I_SET
5330

5431
/* Additional SPSR bits not exposed in the UABI */
5532
#define PSR_MODE_THREAD_BIT (1 << 0)

‎arch/arm64/kernel/image-vars.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -105,11 +105,6 @@ KVM_NVHE_ALIAS(__hyp_stub_vectors);
105105
KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);
106106
KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
107107

108-
#ifdef CONFIG_ARM64_PSEUDO_NMI
109-
/* Static key checked in GIC_PRIO_IRQOFF. */
110-
KVM_NVHE_ALIAS(gic_nonsecure_priorities);
111-
#endif
112-
113108
/* EL2 exception handling */
114109
KVM_NVHE_ALIAS(__start___kvm_ex_table);
115110
KVM_NVHE_ALIAS(__stop___kvm_ex_table);

‎drivers/irqchip/irq-gic-v3.c‎

Lines changed: 39 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include <linux/irqchip.h>
2626
#include <linux/irqchip/arm-gic-common.h>
2727
#include <linux/irqchip/arm-gic-v3.h>
28+
#include <linux/irqchip/arm-gic-v3-prio.h>
2829
#include <linux/irqchip/irq-partition-percpu.h>
2930
#include <linux/bitfield.h>
3031
#include <linux/bits.h>
@@ -37,8 +38,8 @@
3738

3839
#include "irq-gic-common.h"
3940

40-
static u8 dist_prio_irq __ro_after_init = GICD_INT_DEF_PRI;
41-
static u8 dist_prio_nmi __ro_after_init = GICD_INT_DEF_PRI & ~0x80;
41+
static u8 dist_prio_irq __ro_after_init = GICV3_PRIO_IRQ;
42+
static u8 dist_prio_nmi __ro_after_init = GICV3_PRIO_NMI;
4243

4344
#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
4445
#define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1)
@@ -110,30 +111,6 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
110111
*/
111112
static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
112113

113-
DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities);
114-
EXPORT_SYMBOL(gic_nonsecure_priorities);
115-
116-
/*
117-
* When the Non-secure world has access to group 0 interrupts (as a
118-
* consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will
119-
* return the Distributor's view of the interrupt priority.
120-
*
121-
* When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority
122-
* written by software is moved to the Non-secure range by the Distributor.
123-
*
124-
* If both are true (which is when gic_nonsecure_priorities gets enabled),
125-
* we need to shift down the priority programmed by software to match it
126-
* against the value returned by ICC_RPR_EL1.
127-
*/
128-
#define GICD_INT_RPR_PRI(priority) \
129-
({ \
130-
u32 __priority = (priority); \
131-
if (static_branch_unlikely(&gic_nonsecure_priorities)) \
132-
__priority = 0x80 | (__priority >> 1); \
133-
\
134-
__priority; \
135-
})
136-
137114
static u32 gic_get_pribits(void)
138115
{
139116
u32 pribits;
@@ -185,6 +162,41 @@ static void __init gic_prio_init(void)
185162
cpus_have_security_disabled = gic_dist_security_disabled();
186163
cpus_have_group0 = gic_has_group0();
187164

165+
/*
166+
* How priority values are used by the GIC depends on two things:
167+
* the security state of the GIC (controlled by the GICD_CTRL.DS bit)
168+
* and if Group 0 interrupts can be delivered to Linux in the non-secure
169+
* world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
170+
* way priorities are presented in ICC_PMR_EL1 and in the distributor:
171+
*
172+
* GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Distributor
173+
* -------------------------------------------------------
174+
* 1 | - | unchanged | unchanged
175+
* -------------------------------------------------------
176+
* 0 | 1 | non-secure | non-secure
177+
* -------------------------------------------------------
178+
* 0 | 0 | unchanged | non-secure
179+
*
180+
* In the non-secure view reads and writes are modified:
181+
*
182+
* - A value written is right-shifted by one and the MSB is set,
183+
* forcing the priority into the non-secure range.
184+
*
185+
* - A value read is left-shifted by one.
186+
*
187+
* In the first two cases, where ICC_PMR_EL1 and the interrupt priority
188+
* are both either modified or unchanged, we can use the same set of
189+
* priorities.
190+
*
191+
* In the last case, where only the interrupt priorities are modified to
192+
* be in the non-secure range, we program the non-secure values into
193+
* the distributor to match the PMR values we want.
194+
*/
195+
if (cpus_have_group0 & !cpus_have_security_disabled) {
196+
dist_prio_irq = __gicv3_prio_to_ns(dist_prio_irq);
197+
dist_prio_nmi = __gicv3_prio_to_ns(dist_prio_nmi);
198+
}
199+
188200
pr_info("GICD_CTRL.DS=%d, SCR_EL3.FIQ=%d\n",
189201
cpus_have_security_disabled,
190202
!cpus_have_group0);
@@ -811,7 +823,7 @@ static bool gic_rpr_is_nmi_prio(void)
811823
if (!gic_supports_nmi())
812824
return false;
813825

814-
return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(dist_prio_nmi));
826+
return unlikely(gic_read_rpr() == GICV3_PRIO_NMI);
815827
}
816828

817829
static bool gic_irqnr_is_special(u32 irqnr)
@@ -1960,36 +1972,6 @@ static void gic_enable_nmi_support(void)
19601972
pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
19611973
gic_has_relaxed_pmr_sync() ? "relaxed" : "forced");
19621974

1963-
/*
1964-
* How priority values are used by the GIC depends on two things:
1965-
* the security state of the GIC (controlled by the GICD_CTRL.DS bit)
1966-
* and if Group 0 interrupts can be delivered to Linux in the non-secure
1967-
* world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
1968-
* ICC_PMR_EL1 register and the priority that software assigns to
1969-
* interrupts:
1970-
*
1971-
* GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority
1972-
* -----------------------------------------------------------
1973-
* 1 | - | unchanged | unchanged
1974-
* -----------------------------------------------------------
1975-
* 0 | 1 | non-secure | non-secure
1976-
* -----------------------------------------------------------
1977-
* 0 | 0 | unchanged | non-secure
1978-
*
1979-
* where non-secure means that the value is right-shifted by one and the
1980-
* MSB bit set, to make it fit in the non-secure priority range.
1981-
*
1982-
* In the first two cases, where ICC_PMR_EL1 and the interrupt priority
1983-
* are both either modified or unchanged, we can use the same set of
1984-
* priorities.
1985-
*
1986-
* In the last case, where only the interrupt priorities are modified to
1987-
* be in the non-secure range, we use a different PMR value to mask IRQs
1988-
* and the rest of the values that we use remain unchanged.
1989-
*/
1990-
if (gic_has_group0() && !gic_dist_security_disabled())
1991-
static_branch_enable(&gic_nonsecure_priorities);
1992-
19931975
static_branch_enable(&supports_pseudo_nmis);
19941976

19951977
if (static_branch_likely(&supports_deactivate_key))

‎include/linux/irqchip/arm-gic-v3-prio.h‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/* SPDX-License-Identifier: GPL-2.0-only */
2+
3+
#ifndef __LINUX_IRQCHIP_ARM_GIC_V3_PRIO_H
4+
#define __LINUX_IRQCHIP_ARM_GIC_V3_PRIO_H
5+
6+
/*
7+
* GIC priorities from the view of the PMR/RPR.
8+
*
9+
* These values are chosen to be valid in either the absolute priority space or
10+
* the NS view of the priority space. The value programmed into the distributor
11+
* and ITS will be chosen at boot time such that these values appear in the
12+
* PMR/RPR.
13+
*
14+
* GICV3_PRIO_UNMASKED is the PMR view of the priority to use to permit both
15+
* IRQs and pseudo-NMIs.
16+
*
17+
* GICV3_PRIO_IRQ is the PMR view of the priority of regular interrupts. This
18+
* can be written to the PMR to mask regular IRQs.
19+
*
20+
* GICV3_PRIO_NMI is the PMR view of the priority of pseudo-NMIs. This can be
21+
* written to the PMR to mask pseudo-NMIs.
22+
*
23+
* On arm64 some code sections either automatically switch back to PSR.I or
24+
* explicitly require to not use priority masking. If bit GICV3_PRIO_PSR_I_SET
25+
* is included in the priority mask, it indicates that PSR.I should be set and
26+
* interrupt disabling temporarily does not rely on IRQ priorities.
27+
*/
28+
#define GICV3_PRIO_UNMASKED 0xe0
29+
#define GICV3_PRIO_IRQ 0xc0
30+
#define GICV3_PRIO_NMI 0x80
31+
32+
#define GICV3_PRIO_PSR_I_SET (1 << 4)
33+
34+
#ifndef __ASSEMBLER__
35+
36+
#define __gicv3_prio_to_ns(p) (0xff & ((p) << 1))
37+
#define __gicv3_ns_to_prio(ns) (0x80 | ((ns) >> 1))
38+
39+
#define __gicv3_prio_valid_ns(p) \
40+
(__gicv3_ns_to_prio(__gicv3_prio_to_ns(p)) == (p))
41+
42+
static_assert(__gicv3_prio_valid_ns(GICV3_PRIO_NMI));
43+
static_assert(__gicv3_prio_valid_ns(GICV3_PRIO_IRQ));
44+
45+
static_assert(GICV3_PRIO_NMI < GICV3_PRIO_IRQ);
46+
static_assert(GICV3_PRIO_IRQ < GICV3_PRIO_UNMASKED);
47+
48+
static_assert(GICV3_PRIO_IRQ < (GICV3_PRIO_IRQ | GICV3_PRIO_PSR_I_SET));
49+
50+
#endif /* __ASSEMBLER */
51+
52+
#endif /* __LINUX_IRQCHIP_ARM_GIC_V3_PRIO_H */

0 commit comments

Comments
 (0)