Skip to content

Commit a89dfde

Browse files
Peter Zijlstrasuryasaimadhu
Peter Zijlstra
authored and
suryasaimadhu
committed
x86: Remove dynamic NOP selection
This ensures that a NOP is a NOP and not a random other instruction that is also a NOP. It allows simplification of dynamic code patching that wants to verify existing code before writing new instructions (ftrace, jump_label, static_call, etc..). Differentiating on NOPs is not a feature. This pessimises 32bit (DONTCARE) and 32bit on 64bit CPUs (CARELESS). 32bit is not a performance target. Everything x86_64 since AMD K10 (2007) and Intel IvyBridge (2012) is fine with using NOPL (as opposed to prefix NOP). And per FEATURE_NOPL being required for x86_64, all x86_64 CPUs can use NOPL. So stop caring about NOPs, simplify things and get on with life. [ The problem seems to be that some uarchs can only decode NOPL on a single front-end port while others have severe decode penalties for excessive prefixes. All modern uarchs can handle both, except Atom, which has prefix penalties. ] [ Also, much doubt you can actually measure any of this on normal workloads. ] After this, FEATURE_NOPL is unused except for required-features for x86_64. FEATURE_K8 is only used for PTI. [ bp: Kernel build measurements showed ~0.3s slowdown on Sandybridge which is hardly a slowdown. Get rid of X86_FEATURE_K7, while at it. ] Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Borislav Petkov <bp@suse.de> Acked-by: Alexei Starovoitov <alexei.starovoitov@gmail.com> # bpf Acked-by: Linus Torvalds <torvalds@linuxfoundation.org> Link: https://lkml.kernel.org/r/20210312115749.065275711@infradead.org
1 parent 59eca2f commit a89dfde

12 files changed

Lines changed: 97 additions & 351 deletions

File tree

‎arch/x86/include/asm/cpufeatures.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@
8484

8585
/* CPU types for specific tunings: */
8686
#define X86_FEATURE_K8 ( 3*32+ 4) /* "" Opteron, Athlon64 */
87-
#define X86_FEATURE_K7 ( 3*32+ 5) /* "" Athlon */
87+
/* FREE, was #define X86_FEATURE_K7 ( 3*32+ 5) "" Athlon */
8888
#define X86_FEATURE_P3 ( 3*32+ 6) /* "" P3 */
8989
#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */
9090
#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */

‎arch/x86/include/asm/jump_label.h‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,6 @@
66

77
#define JUMP_LABEL_NOP_SIZE 5
88

9-
#ifdef CONFIG_X86_64
10-
# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
11-
#else
12-
# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
13-
#endif
14-
159
#include <asm/asm.h>
1610
#include <asm/nops.h>
1711

@@ -23,7 +17,7 @@
2317
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
2418
{
2519
asm_volatile_goto("1:"
26-
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
20+
".byte " __stringify(BYTES_NOP5) "\n\t"
2721
".pushsection __jump_table, \"aw\" \n\t"
2822
_ASM_ALIGN "\n\t"
2923
".long 1b - ., %l[l_yes] - . \n\t"
@@ -63,7 +57,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
6357
.long \target - .Lstatic_jump_after_\@
6458
.Lstatic_jump_after_\@:
6559
.else
66-
.byte STATIC_KEY_INIT_NOP
60+
.byte BYTES_NOP5
6761
.endif
6862
.pushsection __jump_table, "aw"
6963
_ASM_ALIGN
@@ -75,7 +69,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
7569
.macro STATIC_JUMP_IF_FALSE target, key, def
7670
.Lstatic_jump_\@:
7771
.if \def
78-
.byte STATIC_KEY_INIT_NOP
72+
.byte BYTES_NOP5
7973
.else
8074
/* Equivalent to "jmp.d32 \target" */
8175
.byte 0xe9

‎arch/x86/include/asm/nops.h‎

Lines changed: 55 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -4,144 +4,78 @@
44

55
/*
66
* Define nops for use with alternative() and for tracing.
7-
*
8-
* *_NOP5_ATOMIC must be a single instruction.
97
*/
108

11-
#define NOP_DS_PREFIX 0x3e
9+
#ifndef CONFIG_64BIT
1210

13-
/* generic versions from gas
14-
1: nop
15-
the following instructions are NOT nops in 64-bit mode,
16-
for 64-bit mode use K8 or P6 nops instead
17-
2: movl %esi,%esi
18-
3: leal 0x00(%esi),%esi
19-
4: leal 0x00(,%esi,1),%esi
20-
6: leal 0x00000000(%esi),%esi
21-
7: leal 0x00000000(,%esi,1),%esi
22-
*/
23-
#define GENERIC_NOP1 0x90
24-
#define GENERIC_NOP2 0x89,0xf6
25-
#define GENERIC_NOP3 0x8d,0x76,0x00
26-
#define GENERIC_NOP4 0x8d,0x74,0x26,0x00
27-
#define GENERIC_NOP5 GENERIC_NOP1,GENERIC_NOP4
28-
#define GENERIC_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00
29-
#define GENERIC_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00
30-
#define GENERIC_NOP8 GENERIC_NOP1,GENERIC_NOP7
31-
#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4
11+
/*
12+
* Generic 32bit nops from GAS:
13+
*
14+
* 1: nop
15+
* 2: movl %esi,%esi
16+
* 3: leal 0x0(%esi),%esi
17+
* 4: leal 0x0(%esi,%eiz,1),%esi
18+
* 5: leal %ds:0x0(%esi,%eiz,1),%esi
19+
* 6: leal 0x0(%esi),%esi
20+
* 7: leal 0x0(%esi,%eiz,1),%esi
21+
* 8: leal %ds:0x0(%esi,%eiz,1),%esi
22+
*
23+
* Except 5 and 8, which are DS prefixed 4 and 7 resp, where GAS would emit 2
24+
* nop instructions.
25+
*/
26+
#define BYTES_NOP1 0x90
27+
#define BYTES_NOP2 0x89,0xf6
28+
#define BYTES_NOP3 0x8d,0x76,0x00
29+
#define BYTES_NOP4 0x8d,0x74,0x26,0x00
30+
#define BYTES_NOP5 0x3e,BYTES_NOP4
31+
#define BYTES_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00
32+
#define BYTES_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00
33+
#define BYTES_NOP8 0x3e,BYTES_NOP7
3234

33-
/* Opteron 64bit nops
34-
1: nop
35-
2: osp nop
36-
3: osp osp nop
37-
4: osp osp osp nop
38-
*/
39-
#define K8_NOP1 GENERIC_NOP1
40-
#define K8_NOP2 0x66,K8_NOP1
41-
#define K8_NOP3 0x66,K8_NOP2
42-
#define K8_NOP4 0x66,K8_NOP3
43-
#define K8_NOP5 K8_NOP3,K8_NOP2
44-
#define K8_NOP6 K8_NOP3,K8_NOP3
45-
#define K8_NOP7 K8_NOP4,K8_NOP3
46-
#define K8_NOP8 K8_NOP4,K8_NOP4
47-
#define K8_NOP5_ATOMIC 0x66,K8_NOP4
35+
#else
4836

49-
/* K7 nops
50-
uses eax dependencies (arbitrary choice)
51-
1: nop
52-
2: movl %eax,%eax
53-
3: leal (,%eax,1),%eax
54-
4: leal 0x00(,%eax,1),%eax
55-
6: leal 0x00000000(%eax),%eax
56-
7: leal 0x00000000(,%eax,1),%eax
57-
*/
58-
#define K7_NOP1 GENERIC_NOP1
59-
#define K7_NOP2 0x8b,0xc0
60-
#define K7_NOP3 0x8d,0x04,0x20
61-
#define K7_NOP4 0x8d,0x44,0x20,0x00
62-
#define K7_NOP5 K7_NOP4,K7_NOP1
63-
#define K7_NOP6 0x8d,0x80,0,0,0,0
64-
#define K7_NOP7 0x8D,0x04,0x05,0,0,0,0
65-
#define K7_NOP8 K7_NOP7,K7_NOP1
66-
#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4
37+
/*
38+
* Generic 64bit nops from GAS:
39+
*
40+
* 1: nop
41+
* 2: osp nop
42+
* 3: nopl (%eax)
43+
* 4: nopl 0x00(%eax)
44+
* 5: nopl 0x00(%eax,%eax,1)
45+
* 6: osp nopl 0x00(%eax,%eax,1)
46+
* 7: nopl 0x00000000(%eax)
47+
* 8: nopl 0x00000000(%eax,%eax,1)
48+
*/
49+
#define BYTES_NOP1 0x90
50+
#define BYTES_NOP2 0x66,BYTES_NOP1
51+
#define BYTES_NOP3 0x0f,0x1f,0x00
52+
#define BYTES_NOP4 0x0f,0x1f,0x40,0x00
53+
#define BYTES_NOP5 0x0f,0x1f,0x44,0x00,0x00
54+
#define BYTES_NOP6 0x66,BYTES_NOP5
55+
#define BYTES_NOP7 0x0f,0x1f,0x80,0x00,0x00,0x00,0x00
56+
#define BYTES_NOP8 0x0f,0x1f,0x84,0x00,0x00,0x00,0x00,0x00
6757

68-
/* P6 nops
69-
uses eax dependencies (Intel-recommended choice)
70-
1: nop
71-
2: osp nop
72-
3: nopl (%eax)
73-
4: nopl 0x00(%eax)
74-
5: nopl 0x00(%eax,%eax,1)
75-
6: osp nopl 0x00(%eax,%eax,1)
76-
7: nopl 0x00000000(%eax)
77-
8: nopl 0x00000000(%eax,%eax,1)
78-
Note: All the above are assumed to be a single instruction.
79-
There is kernel code that depends on this.
80-
*/
81-
#define P6_NOP1 GENERIC_NOP1
82-
#define P6_NOP2 0x66,0x90
83-
#define P6_NOP3 0x0f,0x1f,0x00
84-
#define P6_NOP4 0x0f,0x1f,0x40,0
85-
#define P6_NOP5 0x0f,0x1f,0x44,0x00,0
86-
#define P6_NOP6 0x66,0x0f,0x1f,0x44,0x00,0
87-
#define P6_NOP7 0x0f,0x1f,0x80,0,0,0,0
88-
#define P6_NOP8 0x0f,0x1f,0x84,0x00,0,0,0,0
89-
#define P6_NOP5_ATOMIC P6_NOP5
58+
#endif /* CONFIG_64BIT */
9059

9160
#ifdef __ASSEMBLY__
9261
#define _ASM_MK_NOP(x) .byte x
9362
#else
9463
#define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n"
9564
#endif
9665

97-
#if defined(CONFIG_MK7)
98-
#define ASM_NOP1 _ASM_MK_NOP(K7_NOP1)
99-
#define ASM_NOP2 _ASM_MK_NOP(K7_NOP2)
100-
#define ASM_NOP3 _ASM_MK_NOP(K7_NOP3)
101-
#define ASM_NOP4 _ASM_MK_NOP(K7_NOP4)
102-
#define ASM_NOP5 _ASM_MK_NOP(K7_NOP5)
103-
#define ASM_NOP6 _ASM_MK_NOP(K7_NOP6)
104-
#define ASM_NOP7 _ASM_MK_NOP(K7_NOP7)
105-
#define ASM_NOP8 _ASM_MK_NOP(K7_NOP8)
106-
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K7_NOP5_ATOMIC)
107-
#elif defined(CONFIG_X86_P6_NOP)
108-
#define ASM_NOP1 _ASM_MK_NOP(P6_NOP1)
109-
#define ASM_NOP2 _ASM_MK_NOP(P6_NOP2)
110-
#define ASM_NOP3 _ASM_MK_NOP(P6_NOP3)
111-
#define ASM_NOP4 _ASM_MK_NOP(P6_NOP4)
112-
#define ASM_NOP5 _ASM_MK_NOP(P6_NOP5)
113-
#define ASM_NOP6 _ASM_MK_NOP(P6_NOP6)
114-
#define ASM_NOP7 _ASM_MK_NOP(P6_NOP7)
115-
#define ASM_NOP8 _ASM_MK_NOP(P6_NOP8)
116-
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(P6_NOP5_ATOMIC)
117-
#elif defined(CONFIG_X86_64)
118-
#define ASM_NOP1 _ASM_MK_NOP(K8_NOP1)
119-
#define ASM_NOP2 _ASM_MK_NOP(K8_NOP2)
120-
#define ASM_NOP3 _ASM_MK_NOP(K8_NOP3)
121-
#define ASM_NOP4 _ASM_MK_NOP(K8_NOP4)
122-
#define ASM_NOP5 _ASM_MK_NOP(K8_NOP5)
123-
#define ASM_NOP6 _ASM_MK_NOP(K8_NOP6)
124-
#define ASM_NOP7 _ASM_MK_NOP(K8_NOP7)
125-
#define ASM_NOP8 _ASM_MK_NOP(K8_NOP8)
126-
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K8_NOP5_ATOMIC)
127-
#else
128-
#define ASM_NOP1 _ASM_MK_NOP(GENERIC_NOP1)
129-
#define ASM_NOP2 _ASM_MK_NOP(GENERIC_NOP2)
130-
#define ASM_NOP3 _ASM_MK_NOP(GENERIC_NOP3)
131-
#define ASM_NOP4 _ASM_MK_NOP(GENERIC_NOP4)
132-
#define ASM_NOP5 _ASM_MK_NOP(GENERIC_NOP5)
133-
#define ASM_NOP6 _ASM_MK_NOP(GENERIC_NOP6)
134-
#define ASM_NOP7 _ASM_MK_NOP(GENERIC_NOP7)
135-
#define ASM_NOP8 _ASM_MK_NOP(GENERIC_NOP8)
136-
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(GENERIC_NOP5_ATOMIC)
137-
#endif
66+
#define ASM_NOP1 _ASM_MK_NOP(BYTES_NOP1)
67+
#define ASM_NOP2 _ASM_MK_NOP(BYTES_NOP2)
68+
#define ASM_NOP3 _ASM_MK_NOP(BYTES_NOP3)
69+
#define ASM_NOP4 _ASM_MK_NOP(BYTES_NOP4)
70+
#define ASM_NOP5 _ASM_MK_NOP(BYTES_NOP5)
71+
#define ASM_NOP6 _ASM_MK_NOP(BYTES_NOP6)
72+
#define ASM_NOP7 _ASM_MK_NOP(BYTES_NOP7)
73+
#define ASM_NOP8 _ASM_MK_NOP(BYTES_NOP8)
13874

13975
#define ASM_NOP_MAX 8
140-
#define NOP_ATOMIC5 (ASM_NOP_MAX+1) /* Entry for the 5-byte atomic NOP */
14176

14277
#ifndef __ASSEMBLY__
143-
extern const unsigned char * const *ideal_nops;
144-
extern void arch_init_ideal_nops(void);
78+
extern const unsigned char * const x86_nops[];
14579
#endif
14680

14781
#endif /* _ASM_X86_NOPS_H */

‎arch/x86/include/asm/special_insns.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ static inline void clflush(volatile void *__p)
214214

215215
static inline void clflushopt(volatile void *__p)
216216
{
217-
alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0",
217+
alternative_io(".byte 0x3e; clflush %P0",
218218
".byte 0x66; clflush %P0",
219219
X86_FEATURE_CLFLUSHOPT,
220220
"+m" (*(volatile char __force *)__p));
@@ -225,7 +225,7 @@ static inline void clwb(volatile void *__p)
225225
volatile struct { char x[64]; } *p = __p;
226226

227227
asm volatile(ALTERNATIVE_2(
228-
".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])",
228+
".byte 0x3e; clflush (%[pax])",
229229
".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */
230230
X86_FEATURE_CLFLUSHOPT,
231231
".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */

0 commit comments

Comments
 (0)