Skip to content

Commit 884f058

Browse files
hcahcaAlexander Gordeev
hcahca
authored and
Alexander Gordeev
committed
s390/uaccess: Remove INLINE_COPY_FROM_USER and INLINE_COPY_TO_USER
The s390 implementations of raw_copy_from_user() and raw_copy_to_user() are never inlined. However INLINE_COPY_FROM_USER and INLINE_COPY_TO_USER are still set. This leads to the odd situation that only the error handling (memset to zero of the not copied bytes) of copy_from_user() is inlined, while the actual fast path code is out-of-line. This would make sense if raw_copy_from_user() and raw_copy_to_user() were implemented in assembler files, where inlining is not possible. But the current s390 setup does not make any sense. Address this by moving the raw uaccess copy inline assemblies to the uaccess header file, and remove INLINE_COPY_FROM_USER and INLINE_COPY_TO_USER definitions. This way the uaccess code, but now including error handling, is still out-of-line with the common code _copy_from_user() and _copy_to_user() variants, which inline the raw uaccess functions via _inline_copy_from_user() and _inline_copy_to_user(). This reduces the size of the kernel image by ~17kb. (defconfig, gcc 14.2.0) Acked-by: Alexander Gordeev <agordeev@linux.ibm.com> Signed-off-by: Heiko Carstens <hca@linux.ibm.com> Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>
1 parent ea5ae3a commit 884f058

2 files changed

Lines changed: 109 additions & 120 deletions

File tree

‎arch/s390/include/asm/uaccess.h‎

Lines changed: 109 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,117 @@
2222

2323
void debug_user_asce(int exit);
2424

25-
unsigned long __must_check
26-
raw_copy_from_user(void *to, const void __user *from, unsigned long n);
25+
union oac {
26+
unsigned int val;
27+
struct {
28+
struct {
29+
unsigned short key : 4;
30+
unsigned short : 4;
31+
unsigned short as : 2;
32+
unsigned short : 4;
33+
unsigned short k : 1;
34+
unsigned short a : 1;
35+
} oac1;
36+
struct {
37+
unsigned short key : 4;
38+
unsigned short : 4;
39+
unsigned short as : 2;
40+
unsigned short : 4;
41+
unsigned short k : 1;
42+
unsigned short a : 1;
43+
} oac2;
44+
};
45+
};
2746

28-
unsigned long __must_check
29-
raw_copy_to_user(void __user *to, const void *from, unsigned long n);
47+
static __always_inline __must_check unsigned long
48+
raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key)
49+
{
50+
unsigned long rem;
51+
union oac spec = {
52+
.oac2.key = key,
53+
.oac2.as = PSW_BITS_AS_SECONDARY,
54+
.oac2.k = 1,
55+
.oac2.a = 1,
56+
};
3057

31-
#ifndef CONFIG_KASAN
32-
#define INLINE_COPY_FROM_USER
33-
#define INLINE_COPY_TO_USER
34-
#endif
58+
asm_inline volatile(
59+
" lr %%r0,%[spec]\n"
60+
"0: mvcos 0(%[to]),0(%[from]),%[size]\n"
61+
"1: jz 5f\n"
62+
" algr %[size],%[val]\n"
63+
" slgr %[from],%[val]\n"
64+
" slgr %[to],%[val]\n"
65+
" j 0b\n"
66+
"2: la %[rem],4095(%[from])\n" /* rem = from + 4095 */
67+
" nr %[rem],%[val]\n" /* rem = (from + 4095) & -4096 */
68+
" slgr %[rem],%[from]\n"
69+
" clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
70+
" jnh 6f\n"
71+
"3: mvcos 0(%[to]),0(%[from]),%[rem]\n"
72+
"4: slgr %[size],%[rem]\n"
73+
" j 6f\n"
74+
"5: lghi %[size],0\n"
75+
"6:\n"
76+
EX_TABLE(0b, 2b)
77+
EX_TABLE(1b, 2b)
78+
EX_TABLE(3b, 6b)
79+
EX_TABLE(4b, 6b)
80+
: [size] "+&a" (size), [from] "+&a" (from), [to] "+&a" (to), [rem] "=&a" (rem)
81+
: [val] "a" (-4096UL), [spec] "d" (spec.val)
82+
: "cc", "memory", "0");
83+
return size;
84+
}
85+
86+
static __always_inline __must_check unsigned long
87+
raw_copy_from_user(void *to, const void __user *from, unsigned long n)
88+
{
89+
return raw_copy_from_user_key(to, from, n, 0);
90+
}
91+
92+
static __always_inline __must_check unsigned long
93+
raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key)
94+
{
95+
unsigned long rem;
96+
union oac spec = {
97+
.oac1.key = key,
98+
.oac1.as = PSW_BITS_AS_SECONDARY,
99+
.oac1.k = 1,
100+
.oac1.a = 1,
101+
};
102+
103+
asm_inline volatile(
104+
" lr %%r0,%[spec]\n"
105+
"0: mvcos 0(%[to]),0(%[from]),%[size]\n"
106+
"1: jz 5f\n"
107+
" algr %[size],%[val]\n"
108+
" slgr %[to],%[val]\n"
109+
" slgr %[from],%[val]\n"
110+
" j 0b\n"
111+
"2: la %[rem],4095(%[to])\n" /* rem = to + 4095 */
112+
" nr %[rem],%[val]\n" /* rem = (to + 4095) & -4096 */
113+
" slgr %[rem],%[to]\n"
114+
" clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
115+
" jnh 6f\n"
116+
"3: mvcos 0(%[to]),0(%[from]),%[rem]\n"
117+
"4: slgr %[size],%[rem]\n"
118+
" j 6f\n"
119+
"5: lghi %[size],0\n"
120+
"6:\n"
121+
EX_TABLE(0b, 2b)
122+
EX_TABLE(1b, 2b)
123+
EX_TABLE(3b, 6b)
124+
EX_TABLE(4b, 6b)
125+
: [size] "+&a" (size), [to] "+&a" (to), [from] "+&a" (from), [rem] "=&a" (rem)
126+
: [val] "a" (-4096UL), [spec] "d" (spec.val)
127+
: "cc", "memory", "0");
128+
return size;
129+
}
130+
131+
static __always_inline __must_check unsigned long
132+
raw_copy_to_user(void __user *to, const void *from, unsigned long n)
133+
{
134+
return raw_copy_to_user_key(to, from, n, 0);
135+
}
35136

36137
unsigned long __must_check
37138
_copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key);
@@ -55,28 +156,6 @@ copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned lo
55156
return n;
56157
}
57158

58-
union oac {
59-
unsigned int val;
60-
struct {
61-
struct {
62-
unsigned short key : 4;
63-
unsigned short : 4;
64-
unsigned short as : 2;
65-
unsigned short : 4;
66-
unsigned short k : 1;
67-
unsigned short a : 1;
68-
} oac1;
69-
struct {
70-
unsigned short key : 4;
71-
unsigned short : 4;
72-
unsigned short as : 2;
73-
unsigned short : 4;
74-
unsigned short k : 1;
75-
unsigned short a : 1;
76-
} oac2;
77-
};
78-
};
79-
80159
int __noreturn __put_user_bad(void);
81160

82161
#ifdef CONFIG_KMSAN

‎arch/s390/lib/uaccess.c‎

Lines changed: 0 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -31,51 +31,6 @@ void debug_user_asce(int exit)
3131
}
3232
#endif /*CONFIG_DEBUG_ENTRY */
3333

34-
static unsigned long raw_copy_from_user_key(void *to, const void __user *from,
35-
unsigned long size, unsigned long key)
36-
{
37-
unsigned long rem;
38-
union oac spec = {
39-
.oac2.key = key,
40-
.oac2.as = PSW_BITS_AS_SECONDARY,
41-
.oac2.k = 1,
42-
.oac2.a = 1,
43-
};
44-
45-
asm volatile(
46-
" lr 0,%[spec]\n"
47-
"0: mvcos 0(%[to]),0(%[from]),%[size]\n"
48-
"1: jz 5f\n"
49-
" algr %[size],%[val]\n"
50-
" slgr %[from],%[val]\n"
51-
" slgr %[to],%[val]\n"
52-
" j 0b\n"
53-
"2: la %[rem],4095(%[from])\n" /* rem = from + 4095 */
54-
" nr %[rem],%[val]\n" /* rem = (from + 4095) & -4096 */
55-
" slgr %[rem],%[from]\n"
56-
" clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
57-
" jnh 6f\n"
58-
"3: mvcos 0(%[to]),0(%[from]),%[rem]\n"
59-
"4: slgr %[size],%[rem]\n"
60-
" j 6f\n"
61-
"5: slgr %[size],%[size]\n"
62-
"6:\n"
63-
EX_TABLE(0b, 2b)
64-
EX_TABLE(1b, 2b)
65-
EX_TABLE(3b, 6b)
66-
EX_TABLE(4b, 6b)
67-
: [size] "+&a" (size), [from] "+&a" (from), [to] "+&a" (to), [rem] "=&a" (rem)
68-
: [val] "a" (-4096UL), [spec] "d" (spec.val)
69-
: "cc", "memory", "0");
70-
return size;
71-
}
72-
73-
unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
74-
{
75-
return raw_copy_from_user_key(to, from, n, 0);
76-
}
77-
EXPORT_SYMBOL(raw_copy_from_user);
78-
7934
unsigned long _copy_from_user_key(void *to, const void __user *from,
8035
unsigned long n, unsigned long key)
8136
{
@@ -93,51 +48,6 @@ unsigned long _copy_from_user_key(void *to, const void __user *from,
9348
}
9449
EXPORT_SYMBOL(_copy_from_user_key);
9550

96-
static unsigned long raw_copy_to_user_key(void __user *to, const void *from,
97-
unsigned long size, unsigned long key)
98-
{
99-
unsigned long rem;
100-
union oac spec = {
101-
.oac1.key = key,
102-
.oac1.as = PSW_BITS_AS_SECONDARY,
103-
.oac1.k = 1,
104-
.oac1.a = 1,
105-
};
106-
107-
asm volatile(
108-
" lr 0,%[spec]\n"
109-
"0: mvcos 0(%[to]),0(%[from]),%[size]\n"
110-
"1: jz 5f\n"
111-
" algr %[size],%[val]\n"
112-
" slgr %[to],%[val]\n"
113-
" slgr %[from],%[val]\n"
114-
" j 0b\n"
115-
"2: la %[rem],4095(%[to])\n" /* rem = to + 4095 */
116-
" nr %[rem],%[val]\n" /* rem = (to + 4095) & -4096 */
117-
" slgr %[rem],%[to]\n"
118-
" clgr %[size],%[rem]\n" /* copy crosses next page boundary? */
119-
" jnh 6f\n"
120-
"3: mvcos 0(%[to]),0(%[from]),%[rem]\n"
121-
"4: slgr %[size],%[rem]\n"
122-
" j 6f\n"
123-
"5: slgr %[size],%[size]\n"
124-
"6:\n"
125-
EX_TABLE(0b, 2b)
126-
EX_TABLE(1b, 2b)
127-
EX_TABLE(3b, 6b)
128-
EX_TABLE(4b, 6b)
129-
: [size] "+&a" (size), [to] "+&a" (to), [from] "+&a" (from), [rem] "=&a" (rem)
130-
: [val] "a" (-4096UL), [spec] "d" (spec.val)
131-
: "cc", "memory", "0");
132-
return size;
133-
}
134-
135-
unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
136-
{
137-
return raw_copy_to_user_key(to, from, n, 0);
138-
}
139-
EXPORT_SYMBOL(raw_copy_to_user);
140-
14151
unsigned long _copy_to_user_key(void __user *to, const void *from,
14252
unsigned long n, unsigned long key)
14353
{

0 commit comments

Comments
 (0)