uaccess_64.h 6.77 KB
Newer Older
1 2
#ifndef _ASM_X86_UACCESS_64_H
#define _ASM_X86_UACCESS_64_H
Linus Torvalds's avatar
Linus Torvalds committed
3 4 5 6 7 8

/*
 * User space memory access functions
 */
#include <linux/compiler.h>
#include <linux/errno.h>
9
#include <linux/lockdep.h>
10 11
#include <asm/alternative.h>
#include <asm/cpufeature.h>
Linus Torvalds's avatar
Linus Torvalds committed
12 13 14 15 16 17 18
#include <asm/page.h>

/*
 * Copy To/From Userspace
 */

/* Handles exceptions in both to and from, but doesn't do access_ok */
19
__must_check unsigned long
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
copy_user_generic_string(void *to, const void *from, unsigned len);
__must_check unsigned long
copy_user_generic_unrolled(void *to, const void *from, unsigned len);

static __always_inline __must_check unsigned long
copy_user_generic(void *to, const void *from, unsigned len)
{
	unsigned ret;

	alternative_call(copy_user_generic_unrolled,
			 copy_user_generic_string,
			 X86_FEATURE_REP_GOOD,
			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
				     "=d" (len)),
			 "1" (to), "2" (from), "3" (len)
			 : "memory", "rcx", "r8", "r9", "r10", "r11");
	return ret;
}
38 39

__must_check unsigned long
40
_copy_to_user(void __user *to, const void *from, unsigned len);
41
__must_check unsigned long
42
_copy_from_user(void *to, const void __user *from, unsigned len);
43 44 45
__must_check unsigned long
copy_in_user(void __user *to, const void __user *from, unsigned len);

46 47 48 49 50 51
static inline unsigned long __must_check copy_from_user(void *to,
					  const void __user *from,
					  unsigned long n)
{
	int sz = __compiletime_object_size(to);

52
	might_fault();
53
	if (likely(sz == -1 || sz >= n))
54
		n = _copy_from_user(to, from, n);
55 56 57 58
#ifdef CONFIG_DEBUG_VM
	else
		WARN(1, "Buffer overflow detected!\n");
#endif
59
	return n;
60 61
}

62 63 64 65 66 67 68
static __always_inline __must_check
int copy_to_user(void __user *dst, const void *src, unsigned size)
{
	might_fault();

	return _copy_to_user(dst, src, size);
}
69

70 71
static __always_inline __must_check
int __copy_from_user(void *dst, const void __user *src, unsigned size)
72
{
73
	int ret = 0;
74

75
	might_fault();
Linus Torvalds's avatar
Linus Torvalds committed
76
	if (!__builtin_constant_p(size))
77 78 79 80
		return copy_user_generic(dst, (__force void *)src, size);
	switch (size) {
	case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
			      ret, "b", "b", "=q", 1);
Linus Torvalds's avatar
Linus Torvalds committed
81
		return ret;
82 83
	case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
			      ret, "w", "w", "=r", 2);
Linus Torvalds's avatar
Linus Torvalds committed
84
		return ret;
85 86 87 88 89
	case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
			      ret, "l", "k", "=r", 4);
		return ret;
	case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			      ret, "q", "", "=r", 8);
Linus Torvalds's avatar
Linus Torvalds committed
90 91
		return ret;
	case 10:
92
		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
93
			       ret, "q", "", "=r", 10);
94 95 96 97 98 99
		if (unlikely(ret))
			return ret;
		__get_user_asm(*(u16 *)(8 + (char *)dst),
			       (u16 __user *)(8 + (char __user *)src),
			       ret, "w", "w", "=r", 2);
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
100
	case 16:
101 102 103 104 105 106 107 108
		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			       ret, "q", "", "=r", 16);
		if (unlikely(ret))
			return ret;
		__get_user_asm(*(u64 *)(8 + (char *)dst),
			       (u64 __user *)(8 + (char __user *)src),
			       ret, "q", "", "=r", 8);
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
109
	default:
110
		return copy_user_generic(dst, (__force void *)src, size);
Linus Torvalds's avatar
Linus Torvalds committed
111
	}
112
}
Linus Torvalds's avatar
Linus Torvalds committed
113

114 115
static __always_inline __must_check
int __copy_to_user(void __user *dst, const void *src, unsigned size)
116
{
117
	int ret = 0;
118

119
	might_fault();
Linus Torvalds's avatar
Linus Torvalds committed
120
	if (!__builtin_constant_p(size))
121 122 123 124
		return copy_user_generic((__force void *)dst, src, size);
	switch (size) {
	case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
			      ret, "b", "b", "iq", 1);
Linus Torvalds's avatar
Linus Torvalds committed
125
		return ret;
126 127
	case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
			      ret, "w", "w", "ir", 2);
Linus Torvalds's avatar
Linus Torvalds committed
128
		return ret;
129 130 131 132
	case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
			      ret, "l", "k", "ir", 4);
		return ret;
	case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
133
			      ret, "q", "", "er", 8);
Linus Torvalds's avatar
Linus Torvalds committed
134 135
		return ret;
	case 10:
136
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
137
			       ret, "q", "", "er", 10);
138 139
		if (unlikely(ret))
			return ret;
Linus Torvalds's avatar
Linus Torvalds committed
140
		asm("":::"memory");
141 142 143
		__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
			       ret, "w", "w", "ir", 2);
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
144
	case 16:
145
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
146
			       ret, "q", "", "er", 16);
147 148
		if (unlikely(ret))
			return ret;
Linus Torvalds's avatar
Linus Torvalds committed
149
		asm("":::"memory");
150
		__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
151
			       ret, "q", "", "er", 8);
152
		return ret;
Linus Torvalds's avatar
Linus Torvalds committed
153
	default:
154
		return copy_user_generic((__force void *)dst, src, size);
Linus Torvalds's avatar
Linus Torvalds committed
155
	}
156
}
Linus Torvalds's avatar
Linus Torvalds committed
157

158 159
static __always_inline __must_check
int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
160
{
161
	int ret = 0;
162

163
	might_fault();
Linus Torvalds's avatar
Linus Torvalds committed
164
	if (!__builtin_constant_p(size))
165 166 167 168
		return copy_user_generic((__force void *)dst,
					 (__force void *)src, size);
	switch (size) {
	case 1: {
Linus Torvalds's avatar
Linus Torvalds committed
169
		u8 tmp;
170 171
		__get_user_asm(tmp, (u8 __user *)src,
			       ret, "b", "b", "=q", 1);
Linus Torvalds's avatar
Linus Torvalds committed
172
		if (likely(!ret))
173 174
			__put_user_asm(tmp, (u8 __user *)dst,
				       ret, "b", "b", "iq", 1);
Linus Torvalds's avatar
Linus Torvalds committed
175 176
		return ret;
	}
177
	case 2: {
Linus Torvalds's avatar
Linus Torvalds committed
178
		u16 tmp;
179 180
		__get_user_asm(tmp, (u16 __user *)src,
			       ret, "w", "w", "=r", 2);
Linus Torvalds's avatar
Linus Torvalds committed
181
		if (likely(!ret))
182 183
			__put_user_asm(tmp, (u16 __user *)dst,
				       ret, "w", "w", "ir", 2);
Linus Torvalds's avatar
Linus Torvalds committed
184 185 186
		return ret;
	}

187
	case 4: {
Linus Torvalds's avatar
Linus Torvalds committed
188
		u32 tmp;
189 190
		__get_user_asm(tmp, (u32 __user *)src,
			       ret, "l", "k", "=r", 4);
Linus Torvalds's avatar
Linus Torvalds committed
191
		if (likely(!ret))
192 193
			__put_user_asm(tmp, (u32 __user *)dst,
				       ret, "l", "k", "ir", 4);
Linus Torvalds's avatar
Linus Torvalds committed
194 195
		return ret;
	}
196
	case 8: {
Linus Torvalds's avatar
Linus Torvalds committed
197
		u64 tmp;
198 199
		__get_user_asm(tmp, (u64 __user *)src,
			       ret, "q", "", "=r", 8);
Linus Torvalds's avatar
Linus Torvalds committed
200
		if (likely(!ret))
201
			__put_user_asm(tmp, (u64 __user *)dst,
202
				       ret, "q", "", "er", 8);
Linus Torvalds's avatar
Linus Torvalds committed
203 204 205
		return ret;
	}
	default:
206 207
		return copy_user_generic((__force void *)dst,
					 (__force void *)src, size);
Linus Torvalds's avatar
Linus Torvalds committed
208
	}
209
}
Linus Torvalds's avatar
Linus Torvalds committed
210

211
__must_check long
212
strncpy_from_user(char *dst, const char __user *src, long count);
213
__must_check long
214 215 216 217 218 219 220
__strncpy_from_user(char *dst, const char __user *src, long count);
__must_check long strnlen_user(const char __user *str, long n);
__must_check long __strnlen_user(const char __user *str, long n);
__must_check long strlen_user(const char __user *str);
__must_check unsigned long clear_user(void __user *mem, unsigned long len);
__must_check unsigned long __clear_user(void __user *mem, unsigned long len);

221 222 223 224 225
static __must_check __always_inline int
__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
{
	return copy_user_generic(dst, (__force const void *)src, size);
}
226 227 228 229 230 231

static __must_check __always_inline int
__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
{
	return copy_user_generic((__force void *)dst, src, size);
}
Linus Torvalds's avatar
Linus Torvalds committed
232

233 234
extern long __copy_user_nocache(void *dst, const void __user *src,
				unsigned size, int zerorest);
235

236 237
static inline int
__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
238 239
{
	might_sleep();
240
	return __copy_user_nocache(dst, src, size, 1);
241 242
}

243 244 245
static inline int
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
				  unsigned size)
246
{
247
	return __copy_user_nocache(dst, src, size, 0);
248 249
}

250 251 252
unsigned long
copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest);

253
#endif /* _ASM_X86_UACCESS_64_H */