fault.c 25.8 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/*
 *  Copyright (C) 1995  Linus Torvalds
 *  Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
 */

#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/tty.h>
#include <linux/vt_kern.h>		/* For unblank_screen() */
#include <linux/compiler.h>
Harvey Harrison's avatar
Harvey Harrison committed
21
22
#include <linux/highmem.h>
#include <linux/bootmem.h>		/* for max_low_pfn */
23
#include <linux/vmalloc.h>
Linus Torvalds's avatar
Linus Torvalds committed
24
#include <linux/module.h>
25
#include <linux/kprobes.h>
26
#include <linux/uaccess.h>
27
#include <linux/kdebug.h>
Linus Torvalds's avatar
Linus Torvalds committed
28
29

#include <asm/system.h>
Harvey Harrison's avatar
Harvey Harrison committed
30
31
#include <asm/desc.h>
#include <asm/segment.h>
Linus Torvalds's avatar
Linus Torvalds committed
32
33
34
35
36
37
#include <asm/pgalloc.h>
#include <asm/smp.h>
#include <asm/tlbflush.h>
#include <asm/proto.h>
#include <asm-generic/sections.h>

38
39
40
41
42
43
44
45
/*
 * Page fault error code bits
 *	bit 0 == 0 means no page found, 1 means protection fault
 *	bit 1 == 0 means read, 1 means write
 *	bit 2 == 0 means kernel, 1 means user-mode
 *	bit 3 == 1 means use of reserved bit detected
 *	bit 4 == 1 means fault was an instruction fetch
 */
46
#define PF_PROT		(1<<0)
47
#define PF_WRITE	(1<<1)
48
49
#define PF_USER		(1<<2)
#define PF_RSVD		(1<<3)
50
51
#define PF_INSTR	(1<<4)

52
53
54
#ifdef CONFIG_MMIOTRACE_HOOKS
static pf_handler_func mmiotrace_pf_handler; /* protected by RCU */
static DEFINE_SPINLOCK(mmiotrace_handler_lock);
55

56
int mmiotrace_register_pf(pf_handler_func new_pfh)
57
{
58
	int ret = 0;
59
	unsigned long flags;
60
61
62
63
64
65
66
	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
	if (mmiotrace_pf_handler)
		ret = -EBUSY;
	else
		mmiotrace_pf_handler = new_pfh;
	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
	return ret;
67
}
68
EXPORT_SYMBOL_GPL(mmiotrace_register_pf);
69
70

/**
71
 * mmiotrace_unregister_pf:
72
 * The caller must ensure @old_pfh is not in use anymore before freeing it.
73
74
 * This function does not guarantee it. The handler function pointer is
 * protected by RCU, so you can do this by e.g. calling synchronize_rcu().
75
 */
76
int mmiotrace_unregister_pf(pf_handler_func old_pfh)
77
{
78
	int ret = 0;
79
	unsigned long flags;
80
81
82
83
84
85
86
	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
	if (mmiotrace_pf_handler != old_pfh)
		ret = -EPERM;
	else
		mmiotrace_pf_handler = NULL;
	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
	return ret;
87
}
88
89
EXPORT_SYMBOL_GPL(mmiotrace_unregister_pf);
#endif /* CONFIG_MMIOTRACE_HOOKS */
90
91

/* returns non-zero if do_page_fault() should return */
92
93
94
static inline int call_mmiotrace(struct pt_regs *regs,
					unsigned long error_code,
					unsigned long address)
95
{
96
#ifdef CONFIG_MMIOTRACE_HOOKS
97
98
	int ret = 0;
	rcu_read_lock();
99
100
	if (mmiotrace_pf_handler)
		ret = mmiotrace_pf_handler(regs, error_code, address);
101
102
103
104
105
106
107
	rcu_read_unlock();
	return ret;
#else
	return 0;
#endif
}

108
static inline int notify_page_fault(struct pt_regs *regs)
109
{
110
#ifdef CONFIG_KPROBES
111
112
113
	int ret = 0;

	/* kprobe_running() needs smp_processor_id() */
114
115
116
#ifdef CONFIG_X86_32
	if (!user_mode_vm(regs)) {
#else
117
	if (!user_mode(regs)) {
118
#endif
119
120
121
122
123
		preempt_disable();
		if (kprobe_running() && kprobe_fault_handler(regs, 14))
			ret = 1;
		preempt_enable();
	}
124

125
126
127
128
	return ret;
#else
	return 0;
#endif
129
}
130

131
132
133
134
135
136
137
138
139
140
141
142
143
/*
 * X86_32
 * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
 * Check that here and ignore it.
 *
 * X86_64
 * Sometimes the CPU reports invalid exceptions on prefetch.
 * Check that here and ignore it.
 *
 * Opcode checker based on code by Richard Brunner
 */
static int is_prefetch(struct pt_regs *regs, unsigned long addr,
		       unsigned long error_code)
144
{
145
	unsigned char *instr;
Linus Torvalds's avatar
Linus Torvalds committed
146
	int scan_more = 1;
147
	int prefetch = 0;
148
	unsigned char *max_instr;
Linus Torvalds's avatar
Linus Torvalds committed
149

Ingo Molnar's avatar
Ingo Molnar committed
150
151
152
153
	/*
	 * If it was a exec (instruction fetch) fault on NX page, then
	 * do not ignore the fault:
	 */
154
	if (error_code & PF_INSTR)
Linus Torvalds's avatar
Linus Torvalds committed
155
		return 0;
156

157
	instr = (unsigned char *)convert_ip_to_linear(current, regs);
158
	max_instr = instr + 15;
Linus Torvalds's avatar
Linus Torvalds committed
159

160
	if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
161
162
		return 0;

163
	while (scan_more && instr < max_instr) {
Linus Torvalds's avatar
Linus Torvalds committed
164
165
166
167
		unsigned char opcode;
		unsigned char instr_hi;
		unsigned char instr_lo;

168
		if (probe_kernel_address(instr, opcode))
169
			break;
Linus Torvalds's avatar
Linus Torvalds committed
170

171
172
		instr_hi = opcode & 0xf0;
		instr_lo = opcode & 0x0f;
Linus Torvalds's avatar
Linus Torvalds committed
173
174
		instr++;

175
		switch (instr_hi) {
Linus Torvalds's avatar
Linus Torvalds committed
176
177
		case 0x20:
		case 0x30:
178
179
180
181
182
183
			/*
			 * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
			 * In X86_64 long mode, the CPU will signal invalid
			 * opcode if some of these prefixes are present so
			 * X86_64 will never get here anyway
			 */
Linus Torvalds's avatar
Linus Torvalds committed
184
185
			scan_more = ((instr_lo & 7) == 0x6);
			break;
186
#ifdef CONFIG_X86_64
Linus Torvalds's avatar
Linus Torvalds committed
187
		case 0x40:
188
189
190
191
192
193
194
			/*
			 * In AMD64 long mode 0x40..0x4F are valid REX prefixes
			 * Need to figure out under what instruction mode the
			 * instruction was issued. Could check the LDT for lm,
			 * but for now it's good enough to assume that long
			 * mode only uses well known segments or kernel.
			 */
195
			scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
Linus Torvalds's avatar
Linus Torvalds committed
196
			break;
197
#endif
Linus Torvalds's avatar
Linus Torvalds committed
198
199
200
		case 0x60:
			/* 0x64 thru 0x67 are valid prefixes in all modes. */
			scan_more = (instr_lo & 0xC) == 0x4;
201
			break;
Linus Torvalds's avatar
Linus Torvalds committed
202
		case 0xF0:
203
			/* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
Linus Torvalds's avatar
Linus Torvalds committed
204
			scan_more = !instr_lo || (instr_lo>>1) == 1;
205
			break;
Linus Torvalds's avatar
Linus Torvalds committed
206
207
208
		case 0x00:
			/* Prefetch instruction is 0x0F0D or 0x0F18 */
			scan_more = 0;
209

210
			if (probe_kernel_address(instr, opcode))
Linus Torvalds's avatar
Linus Torvalds committed
211
212
213
				break;
			prefetch = (instr_lo == 0xF) &&
				(opcode == 0x0D || opcode == 0x18);
214
			break;
Linus Torvalds's avatar
Linus Torvalds committed
215
216
217
		default:
			scan_more = 0;
			break;
218
		}
Linus Torvalds's avatar
Linus Torvalds committed
219
220
221
222
	}
	return prefetch;
}

223
224
225
226
227
228
229
230
231
232
233
234
static void force_sig_info_fault(int si_signo, int si_code,
	unsigned long address, struct task_struct *tsk)
{
	siginfo_t info;

	info.si_signo = si_signo;
	info.si_errno = 0;
	info.si_code = si_code;
	info.si_addr = (void __user *)address;
	force_sig_info(si_signo, &info, tsk);
}

235
#ifdef CONFIG_X86_64
236
237
static int bad_address(void *p)
{
Linus Torvalds's avatar
Linus Torvalds committed
238
	unsigned long dummy;
239
	return probe_kernel_address((unsigned long *)p, dummy);
240
}
241
#endif
Linus Torvalds's avatar
Linus Torvalds committed
242

243
static void dump_pagetable(unsigned long address)
Linus Torvalds's avatar
Linus Torvalds committed
244
{
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
#ifdef CONFIG_X86_32
	__typeof__(pte_val(__pte(0))) page;

	page = read_cr3();
	page = ((__typeof__(page) *) __va(page))[address >> PGDIR_SHIFT];
#ifdef CONFIG_X86_PAE
	printk("*pdpt = %016Lx ", page);
	if ((page >> PAGE_SHIFT) < max_low_pfn
	    && page & _PAGE_PRESENT) {
		page &= PAGE_MASK;
		page = ((__typeof__(page) *) __va(page))[(address >> PMD_SHIFT)
		                                         & (PTRS_PER_PMD - 1)];
		printk(KERN_CONT "*pde = %016Lx ", page);
		page &= ~_PAGE_NX;
	}
#else
	printk("*pde = %08lx ", page);
#endif

	/*
	 * We must not directly access the pte in the highpte
	 * case if the page table is located in highmem.
	 * And let's rather not kmap-atomic the pte, just in case
	 * it's allocated already.
	 */
	if ((page >> PAGE_SHIFT) < max_low_pfn
	    && (page & _PAGE_PRESENT)
	    && !(page & _PAGE_PSE)) {
		page &= PAGE_MASK;
		page = ((__typeof__(page) *) __va(page))[(address >> PAGE_SHIFT)
		                                         & (PTRS_PER_PTE - 1)];
		printk("*pte = %0*Lx ", sizeof(page)*2, (u64)page);
	}

	printk("\n");
#else /* CONFIG_X86_64 */
Linus Torvalds's avatar
Linus Torvalds committed
281
282
283
284
285
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

286
	pgd = (pgd_t *)read_cr3();
Linus Torvalds's avatar
Linus Torvalds committed
287

288
	pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
Linus Torvalds's avatar
Linus Torvalds committed
289
290
	pgd += pgd_index(address);
	if (bad_address(pgd)) goto bad;
291
	printk("PGD %lx ", pgd_val(*pgd));
292
	if (!pgd_present(*pgd)) goto ret;
Linus Torvalds's avatar
Linus Torvalds committed
293

294
	pud = pud_offset(pgd, address);
Linus Torvalds's avatar
Linus Torvalds committed
295
296
	if (bad_address(pud)) goto bad;
	printk("PUD %lx ", pud_val(*pud));
297
298
	if (!pud_present(*pud) || pud_large(*pud))
		goto ret;
Linus Torvalds's avatar
Linus Torvalds committed
299
300
301
302

	pmd = pmd_offset(pud, address);
	if (bad_address(pmd)) goto bad;
	printk("PMD %lx ", pmd_val(*pmd));
303
	if (!pmd_present(*pmd) || pmd_large(*pmd)) goto ret;
Linus Torvalds's avatar
Linus Torvalds committed
304
305
306

	pte = pte_offset_kernel(pmd, address);
	if (bad_address(pte)) goto bad;
307
	printk("PTE %lx", pte_val(*pte));
Linus Torvalds's avatar
Linus Torvalds committed
308
309
310
311
312
ret:
	printk("\n");
	return;
bad:
	printk("BAD\n");
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
#endif
}

#ifdef CONFIG_X86_32
static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
{
	unsigned index = pgd_index(address);
	pgd_t *pgd_k;
	pud_t *pud, *pud_k;
	pmd_t *pmd, *pmd_k;

	pgd += index;
	pgd_k = init_mm.pgd + index;

	if (!pgd_present(*pgd_k))
		return NULL;

	/*
	 * set_pgd(pgd, *pgd_k); here would be useless on PAE
	 * and redundant with the set_pmd() on non-PAE. As would
	 * set_pud.
	 */

	pud = pud_offset(pgd, address);
	pud_k = pud_offset(pgd_k, address);
	if (!pud_present(*pud_k))
		return NULL;

	pmd = pmd_offset(pud, address);
	pmd_k = pmd_offset(pud_k, address);
	if (!pmd_present(*pmd_k))
		return NULL;
	if (!pmd_present(*pmd)) {
		set_pmd(pmd, *pmd_k);
		arch_flush_lazy_mmu_mode();
	} else
		BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
	return pmd_k;
Linus Torvalds's avatar
Linus Torvalds committed
351
}
352
#endif
Linus Torvalds's avatar
Linus Torvalds committed
353

354
#ifdef CONFIG_X86_64
355
static const char errata93_warning[] =
Linus Torvalds's avatar
Linus Torvalds committed
356
357
358
359
KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
KERN_ERR "******* Please consider a BIOS update.\n"
KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
360
#endif
Linus Torvalds's avatar
Linus Torvalds committed
361
362
363

/* Workaround for K8 erratum #93 & buggy BIOS.
   BIOS SMM functions are required to use a specific workaround
364
365
   to avoid corruption of the 64bit RIP register on C stepping K8.
   A lot of BIOS that didn't get tested properly miss this.
Linus Torvalds's avatar
Linus Torvalds committed
366
367
   The OS sees this as a page fault with the upper 32bits of RIP cleared.
   Try to work around it here.
368
369
370
   Note we only handle faults in kernel here.
   Does nothing for X86_32
 */
371
static int is_errata93(struct pt_regs *regs, unsigned long address)
Linus Torvalds's avatar
Linus Torvalds committed
372
{
373
#ifdef CONFIG_X86_64
Linus Torvalds's avatar
Linus Torvalds committed
374
	static int warned;
375
	if (address != regs->ip)
Linus Torvalds's avatar
Linus Torvalds committed
376
		return 0;
377
	if ((address >> 32) != 0)
Linus Torvalds's avatar
Linus Torvalds committed
378
379
		return 0;
	address |= 0xffffffffUL << 32;
380
381
	if ((address >= (u64)_stext && address <= (u64)_etext) ||
	    (address >= MODULES_VADDR && address <= MODULES_END)) {
Linus Torvalds's avatar
Linus Torvalds committed
382
		if (!warned) {
383
			printk(errata93_warning);
Linus Torvalds's avatar
Linus Torvalds committed
384
385
			warned = 1;
		}
386
		regs->ip = address;
Linus Torvalds's avatar
Linus Torvalds committed
387
388
		return 1;
	}
389
#endif
Linus Torvalds's avatar
Linus Torvalds committed
390
	return 0;
391
}
Linus Torvalds's avatar
Linus Torvalds committed
392

393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
/*
 * Work around K8 erratum #100 K8 in compat mode occasionally jumps to illegal
 * addresses >4GB.  We catch this in the page fault handler because these
 * addresses are not reachable. Just detect this case and return.  Any code
 * segment in LDT is compatibility mode.
 */
static int is_errata100(struct pt_regs *regs, unsigned long address)
{
#ifdef CONFIG_X86_64
	if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
	    (address >> 32))
		return 1;
#endif
	return 0;
}

409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
void do_invalid_op(struct pt_regs *, unsigned long);

static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
{
#ifdef CONFIG_X86_F00F_BUG
	unsigned long nr;
	/*
	 * Pentium F0 0F C7 C8 bug workaround.
	 */
	if (boot_cpu_data.f00f_bug) {
		nr = (address - idt_descr.address) >> 3;

		if (nr == 6) {
			do_invalid_op(regs, 0);
			return 1;
		}
	}
#endif
	return 0;
}

430
431
432
static void show_fault_oops(struct pt_regs *regs, unsigned long error_code,
			    unsigned long address)
{
433
434
435
#ifdef CONFIG_X86_32
	if (!oops_may_print())
		return;
436
#endif
437
438
439

#ifdef CONFIG_X86_PAE
	if (error_code & PF_INSTR) {
440
		unsigned int level;
441
442
443
444
445
446
447
448
449
		pte_t *pte = lookup_address(address, &level);

		if (pte && pte_present(*pte) && !pte_exec(*pte))
			printk(KERN_CRIT "kernel tried to execute "
				"NX-protected page - exploit attempt? "
				"(uid: %d)\n", current->uid);
	}
#endif

450
	printk(KERN_ALERT "BUG: unable to handle kernel ");
451
	if (address < PAGE_SIZE)
452
		printk(KERN_CONT "NULL pointer dereference");
453
	else
454
		printk(KERN_CONT "paging request");
455
456
457
#ifdef CONFIG_X86_32
	printk(KERN_CONT " at %08lx\n", address);
#else
458
	printk(KERN_CONT " at %016lx\n", address);
459
#endif
460
	printk(KERN_ALERT "IP:");
461
462
463
464
	printk_address(regs->ip, 1);
	dump_pagetable(address);
}

465
#ifdef CONFIG_X86_64
Linus Torvalds's avatar
Linus Torvalds committed
466
467
468
static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
				 unsigned long error_code)
{
469
	unsigned long flags = oops_begin();
470
	struct task_struct *tsk;
471

Linus Torvalds's avatar
Linus Torvalds committed
472
473
474
	printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
	       current->comm, address);
	dump_pagetable(address);
475
476
477
478
	tsk = current;
	tsk->thread.cr2 = address;
	tsk->thread.trap_no = 14;
	tsk->thread.error_code = error_code;
479
480
481
	if (__die("Bad pagetable", regs, error_code))
		regs = NULL;
	oops_end(flags, regs, SIGKILL);
Linus Torvalds's avatar
Linus Torvalds committed
482
}
483
#endif
Linus Torvalds's avatar
Linus Torvalds committed
484

485
486
487
488
489
490
491
492
493
494
static int spurious_fault_check(unsigned long error_code, pte_t *pte)
{
	if ((error_code & PF_WRITE) && !pte_write(*pte))
		return 0;
	if ((error_code & PF_INSTR) && !pte_exec(*pte))
		return 0;

	return 1;
}

495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
/*
 * Handle a spurious fault caused by a stale TLB entry.  This allows
 * us to lazily refresh the TLB when increasing the permissions of a
 * kernel page (RO -> RW or NX -> X).  Doing it eagerly is very
 * expensive since that implies doing a full cross-processor TLB
 * flush, even if no stale TLB entries exist on other processors.
 * There are no security implications to leaving a stale TLB when
 * increasing the permissions on a page.
 */
static int spurious_fault(unsigned long address,
			  unsigned long error_code)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	/* Reserved-bit violation or user access to kernel space? */
	if (error_code & (PF_USER | PF_RSVD))
		return 0;

	pgd = init_mm.pgd + pgd_index(address);
	if (!pgd_present(*pgd))
		return 0;

	pud = pud_offset(pgd, address);
	if (!pud_present(*pud))
		return 0;

524
525
526
	if (pud_large(*pud))
		return spurious_fault_check(error_code, (pte_t *) pud);

527
528
529
530
	pmd = pmd_offset(pud, address);
	if (!pmd_present(*pmd))
		return 0;

531
532
533
	if (pmd_large(*pmd))
		return spurious_fault_check(error_code, (pte_t *) pmd);

534
535
536
537
	pte = pte_offset_kernel(pmd, address);
	if (!pte_present(*pte))
		return 0;

538
	return spurious_fault_check(error_code, pte);
539
540
}

Linus Torvalds's avatar
Linus Torvalds committed
541
/*
542
543
544
545
 * X86_32
 * Handle a fault on the vmalloc or module mapping area
 *
 * X86_64
546
 * Handle a fault on the vmalloc area
547
548
 *
 * This assumes no large pages in there.
Linus Torvalds's avatar
Linus Torvalds committed
549
550
551
 */
static int vmalloc_fault(unsigned long address)
{
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
#ifdef CONFIG_X86_32
	unsigned long pgd_paddr;
	pmd_t *pmd_k;
	pte_t *pte_k;
	/*
	 * Synchronize this task's top level page-table
	 * with the 'reference' page table.
	 *
	 * Do _not_ use "current" here. We might be inside
	 * an interrupt in the middle of a task switch..
	 */
	pgd_paddr = read_cr3();
	pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
	if (!pmd_k)
		return -1;
	pte_k = pte_offset_kernel(pmd_k, address);
	if (!pte_present(*pte_k))
		return -1;
	return 0;
#else
Linus Torvalds's avatar
Linus Torvalds committed
572
573
574
575
576
	pgd_t *pgd, *pgd_ref;
	pud_t *pud, *pud_ref;
	pmd_t *pmd, *pmd_ref;
	pte_t *pte, *pte_ref;

577
578
579
580
	/* Make sure we are in vmalloc area */
	if (!(address >= VMALLOC_START && address < VMALLOC_END))
		return -1;

Linus Torvalds's avatar
Linus Torvalds committed
581
582
583
584
585
586
587
588
589
590
	/* Copy kernel mappings over when needed. This can also
	   happen within a race in page table update. In the later
	   case just flush. */

	pgd = pgd_offset(current->mm ?: &init_mm, address);
	pgd_ref = pgd_offset_k(address);
	if (pgd_none(*pgd_ref))
		return -1;
	if (pgd_none(*pgd))
		set_pgd(pgd, *pgd_ref);
591
	else
592
		BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
Linus Torvalds's avatar
Linus Torvalds committed
593
594
595
596
597
598
599
600

	/* Below here mismatches are bugs because these lower tables
	   are shared */

	pud = pud_offset(pgd, address);
	pud_ref = pud_offset(pgd_ref, address);
	if (pud_none(*pud_ref))
		return -1;
601
	if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
Linus Torvalds's avatar
Linus Torvalds committed
602
603
604
605
606
607
608
609
610
611
612
		BUG();
	pmd = pmd_offset(pud, address);
	pmd_ref = pmd_offset(pud_ref, address);
	if (pmd_none(*pmd_ref))
		return -1;
	if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
		BUG();
	pte_ref = pte_offset_kernel(pmd_ref, address);
	if (!pte_present(*pte_ref))
		return -1;
	pte = pte_offset_kernel(pmd, address);
613
614
615
616
	/* Don't use pte_page here, because the mappings can point
	   outside mem_map, and the NUMA hash lookup cannot handle
	   that. */
	if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
Linus Torvalds's avatar
Linus Torvalds committed
617
618
		BUG();
	return 0;
619
#endif
Linus Torvalds's avatar
Linus Torvalds committed
620
621
}

622
int show_unhandled_signals = 1;
Linus Torvalds's avatar
Linus Torvalds committed
623
624
625
626
627
628

/*
 * This routine handles page faults.  It determines the address,
 * and the problem, and then passes it off to one of the appropriate
 * routines.
 */
629
630
631
632
#ifdef CONFIG_X86_64
asmlinkage
#endif
void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
Linus Torvalds's avatar
Linus Torvalds committed
633
634
635
{
	struct task_struct *tsk;
	struct mm_struct *mm;
636
	struct vm_area_struct *vma;
Linus Torvalds's avatar
Linus Torvalds committed
637
	unsigned long address;
638
639
640
	int write, si_code;
	int fault;
#ifdef CONFIG_X86_64
641
	unsigned long flags;
642
#endif
Linus Torvalds's avatar
Linus Torvalds committed
643

Peter Zijlstra's avatar
Peter Zijlstra committed
644
645
646
647
648
	/*
	 * We can fault from pretty much anywhere, with unknown IRQ state.
	 */
	trace_hardirqs_fixup();

649
650
651
652
	tsk = current;
	mm = tsk->mm;
	prefetchw(&mm->mmap_sem);

Linus Torvalds's avatar
Linus Torvalds committed
653
	/* get the address */
654
	address = read_cr2();
Linus Torvalds's avatar
Linus Torvalds committed
655

656
	si_code = SEGV_MAPERR;
Linus Torvalds's avatar
Linus Torvalds committed
657

658
659
	if (notify_page_fault(regs))
		return;
660
	if (call_mmiotrace(regs, error_code, address))
661
		return;
Linus Torvalds's avatar
Linus Torvalds committed
662
663
664
665
666
667
668
669
670
671
672
673

	/*
	 * We fault-in kernel-space virtual memory on-demand. The
	 * 'reference' page table is init_mm.pgd.
	 *
	 * NOTE! We MUST NOT take any locks for this case. We may
	 * be in an interrupt or a critical region, and should
	 * only copy the information from the master page table,
	 * nothing more.
	 *
	 * This verifies that the fault happens in kernel space
	 * (error_code & 4) == 0, and that the fault was not a
674
	 * protection error (error_code & 9) == 0.
Linus Torvalds's avatar
Linus Torvalds committed
675
	 */
676
677
#ifdef CONFIG_X86_32
	if (unlikely(address >= TASK_SIZE)) {
678
679
680
#else
	if (unlikely(address >= TASK_SIZE64)) {
#endif
681
682
683
		if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
		    vmalloc_fault(address) >= 0)
			return;
684
685
686
687
688

		/* Can handle a stale RO->RW TLB */
		if (spurious_fault(address, error_code))
			return;

689
690
691
692
693
694
695
		/*
		 * Don't take the mm semaphore here. If we fixup a prefetch
		 * fault we could otherwise deadlock.
		 */
		goto bad_area_nosemaphore;
	}

696
697

#ifdef CONFIG_X86_32
698
699
	/* It's safe to allow irq's after cr2 has been saved and the vmalloc
	   fault has been handled. */
700
	if (regs->flags & (X86_EFLAGS_IF | X86_VM_MASK))
701
702
703
704
705
706
707
708
709
		local_irq_enable();

	/*
	 * If we're in an interrupt, have no user context or are running in an
	 * atomic region then we must not take the fault.
	 */
	if (in_atomic() || !mm)
		goto bad_area_nosemaphore;
#else /* CONFIG_X86_64 */
710
	if (likely(regs->flags & X86_EFLAGS_IF))
711
712
		local_irq_enable();

713
	if (unlikely(error_code & PF_RSVD))
Linus Torvalds's avatar
Linus Torvalds committed
714
715
716
		pgtable_bad(address, regs, error_code);

	/*
717
718
	 * If we're in an interrupt, have no user context or are running in an
	 * atomic region then we must not take the fault.
Linus Torvalds's avatar
Linus Torvalds committed
719
720
721
722
	 */
	if (unlikely(in_atomic() || !mm))
		goto bad_area_nosemaphore;

723
724
725
726
727
728
	/*
	 * User-mode registers count as a user access even for any
	 * potential system fault or CPU buglet.
	 */
	if (user_mode_vm(regs))
		error_code |= PF_USER;
729
730
again:
#endif
Linus Torvalds's avatar
Linus Torvalds committed
731
732
	/* When running in the kernel we expect faults to occur only to
	 * addresses in user space.  All other faults represent errors in the
Simon Arlott's avatar
Simon Arlott committed
733
	 * kernel and should generate an OOPS.  Unfortunately, in the case of an
734
	 * erroneous fault occurring in a code path which already holds mmap_sem
Linus Torvalds's avatar
Linus Torvalds committed
735
736
737
738
739
740
	 * we will deadlock attempting to validate the fault against the
	 * address space.  Luckily the kernel only validly references user
	 * space from well defined areas of code, which are listed in the
	 * exceptions table.
	 *
	 * As the vast majority of faults will be valid we will only perform
Simon Arlott's avatar
Simon Arlott committed
741
	 * the source reference check when there is a possibility of a deadlock.
Linus Torvalds's avatar
Linus Torvalds committed
742
743
744
745
746
	 * Attempt to lock the address space, if we cannot we then validate the
	 * source.  If this is invalid we can skip the address space check,
	 * thus avoiding the deadlock.
	 */
	if (!down_read_trylock(&mm->mmap_sem)) {
747
		if ((error_code & PF_USER) == 0 &&
748
		    !search_exception_tables(regs->ip))
Linus Torvalds's avatar
Linus Torvalds committed
749
750
751
752
753
754
755
			goto bad_area_nosemaphore;
		down_read(&mm->mmap_sem);
	}

	vma = find_vma(mm, address);
	if (!vma)
		goto bad_area;
756
	if (vma->vm_start <= address)
Linus Torvalds's avatar
Linus Torvalds committed
757
758
759
		goto good_area;
	if (!(vma->vm_flags & VM_GROWSDOWN))
		goto bad_area;
760
	if (error_code & PF_USER) {
761
762
763
764
765
		/*
		 * Accessing the stack below %sp is always a bug.
		 * The large cushion allows instructions like enter
		 * and pusha to work.  ("enter $65535,$31" pushes
		 * 32 pointers and then decrements %sp by 65535.)
766
		 */
767
		if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
Linus Torvalds's avatar
Linus Torvalds committed
768
769
770
771
772
773
774
775
776
			goto bad_area;
	}
	if (expand_stack(vma, address))
		goto bad_area;
/*
 * Ok, we have a good vm_area for this memory access, so
 * we can handle it..
 */
good_area:
777
	si_code = SEGV_ACCERR;
Linus Torvalds's avatar
Linus Torvalds committed
778
	write = 0;
779
	switch (error_code & (PF_PROT|PF_WRITE)) {
780
781
782
783
784
785
786
787
788
789
790
	default:	/* 3: write, present */
		/* fall through */
	case PF_WRITE:		/* write, not present */
		if (!(vma->vm_flags & VM_WRITE))
			goto bad_area;
		write++;
		break;
	case PF_PROT:		/* read, present */
		goto bad_area;
	case 0:			/* read, not present */
		if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
Linus Torvalds's avatar
Linus Torvalds committed
791
792
793
			goto bad_area;
	}

794
795
796
#ifdef CONFIG_X86_32
survive:
#endif
Linus Torvalds's avatar
Linus Torvalds committed
797
798
799
800
801
	/*
	 * If for any reason at all we couldn't handle the fault,
	 * make sure we exit gracefully rather than endlessly redo
	 * the fault.
	 */
Nick Piggin's avatar
Nick Piggin committed
802
803
804
805
806
807
808
	fault = handle_mm_fault(mm, vma, address, write);
	if (unlikely(fault & VM_FAULT_ERROR)) {
		if (fault & VM_FAULT_OOM)
			goto out_of_memory;
		else if (fault & VM_FAULT_SIGBUS)
			goto do_sigbus;
		BUG();
Linus Torvalds's avatar
Linus Torvalds committed
809
	}
Nick Piggin's avatar
Nick Piggin committed
810
811
812
813
	if (fault & VM_FAULT_MAJOR)
		tsk->maj_flt++;
	else
		tsk->min_flt++;
814
815
816
817
818
819
820
821
822
823
824

#ifdef CONFIG_X86_32
	/*
	 * Did it hit the DOS screen memory VA from vm86 mode?
	 */
	if (v8086_mode(regs)) {
		unsigned long bit = (address - 0xA0000) >> PAGE_SHIFT;
		if (bit < 32)
			tsk->thread.screen_bitmap |= 1 << bit;
	}
#endif
Linus Torvalds's avatar
Linus Torvalds committed
825
826
827
828
829
830
831
832
833
834
835
836
	up_read(&mm->mmap_sem);
	return;

/*
 * Something tried to access memory that isn't in our memory map..
 * Fix it, but check if it's kernel or user first..
 */
bad_area:
	up_read(&mm->mmap_sem);

bad_area_nosemaphore:
	/* User mode accesses just cause a SIGSEGV */
837
	if (error_code & PF_USER) {
838
839
840
841
842
		/*
		 * It's possible to have interrupts off here.
		 */
		local_irq_enable();

843
844
845
846
		/*
		 * Valid to do another page fault here because this one came
		 * from user space.
		 */
Linus Torvalds's avatar
Linus Torvalds committed
847
848
849
		if (is_prefetch(regs, address, error_code))
			return;

850
		if (is_errata100(regs, address))
Linus Torvalds's avatar
Linus Torvalds committed
851
852
			return;

853
854
		if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
		    printk_ratelimit()) {
Linus Torvalds's avatar
Linus Torvalds committed
855
			printk(
856
#ifdef CONFIG_X86_32
857
			"%s%s[%d]: segfault at %lx ip %08lx sp %08lx error %lx",
858
#else
859
			"%s%s[%d]: segfault at %lx ip %lx sp %lx error %lx",
860
861
862
863
#endif
			task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
			tsk->comm, task_pid_nr(tsk), address, regs->ip,
			regs->sp, error_code);
864
865
			print_vma_addr(" in ", regs->ip);
			printk("\n");
Linus Torvalds's avatar
Linus Torvalds committed
866
		}
867

Linus Torvalds's avatar
Linus Torvalds committed
868
869
870
871
		tsk->thread.cr2 = address;
		/* Kernel addresses are always protection faults */
		tsk->thread.error_code = error_code | (address >= TASK_SIZE);
		tsk->thread.trap_no = 14;
872
		force_sig_info_fault(SIGSEGV, si_code, address, tsk);
Linus Torvalds's avatar
Linus Torvalds committed
873
874
875
		return;
	}

876
877
878
	if (is_f00f_bug(regs, address))
		return;

Linus Torvalds's avatar
Linus Torvalds committed
879
880
no_context:
	/* Are we prepared to handle this kernel fault?  */
881
	if (fixup_exception(regs))
Linus Torvalds's avatar
Linus Torvalds committed
882
883
		return;

884
	/*
885
886
887
888
889
890
	 * X86_32
	 * Valid to do another page fault here, because if this fault
	 * had been triggered by is_prefetch fixup_exception would have
	 * handled it.
	 *
	 * X86_64
Linus Torvalds's avatar
Linus Torvalds committed
891
892
	 * Hall of shame of CPU/BIOS bugs.
	 */
893
894
	if (is_prefetch(regs, address, error_code))
		return;
Linus Torvalds's avatar
Linus Torvalds committed
895
896

	if (is_errata93(regs, address))
897
		return;
Linus Torvalds's avatar
Linus Torvalds committed
898
899
900
901
902

/*
 * Oops. The kernel tried to access some bad page. We'll have to
 * terminate things with extreme prejudice.
 */
903
904
#ifdef CONFIG_X86_32
	bust_spinlocks(1);
905
906
907
#else
	flags = oops_begin();
#endif
908
909

	show_fault_oops(regs, error_code, address);
Linus Torvalds's avatar
Linus Torvalds committed
910

911
912
913
	tsk->thread.cr2 = address;
	tsk->thread.trap_no = 14;
	tsk->thread.error_code = error_code;
914
915

#ifdef CONFIG_X86_32
916
917
918
	die("Oops", regs, error_code);
	bust_spinlocks(0);
	do_exit(SIGKILL);
919
#else
920
921
	if (__die("Oops", regs, error_code))
		regs = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
922
923
	/* Executive summary in case the body of the oops scrolled away */
	printk(KERN_EMERG "CR2: %016lx\n", address);
924
	oops_end(flags, regs, SIGKILL);
925
#endif
Linus Torvalds's avatar
Linus Torvalds committed
926
927
928
929
930
931
932

/*
 * We ran out of memory, or some other thing happened to us that made
 * us unable to handle the page fault gracefully.
 */
out_of_memory:
	up_read(&mm->mmap_sem);
933
934
	if (is_global_init(tsk)) {
		yield();
935
#ifdef CONFIG_X86_32
936
937
938
		down_read(&mm->mmap_sem);
		goto survive;
#else
Linus Torvalds's avatar
Linus Torvalds committed
939
		goto again;
940
#endif
941
942
	}

Linus Torvalds's avatar
Linus Torvalds committed
943
	printk("VM: killing process %s\n", tsk->comm);
944
	if (error_code & PF_USER)
945
		do_group_exit(SIGKILL);
Linus Torvalds's avatar
Linus Torvalds committed
946
947
948
949
950
951
	goto no_context;

do_sigbus:
	up_read(&mm->mmap_sem);

	/* Kernel mode? Handle exceptions or die */
952
	if (!(error_code & PF_USER))
Linus Torvalds's avatar
Linus Torvalds committed
953
		goto no_context;
954
955
956
957
958
#ifdef CONFIG_X86_32
	/* User space => ok to do another page fault */
	if (is_prefetch(regs, address, error_code))
		return;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
959
960
961
	tsk->thread.cr2 = address;
	tsk->thread.error_code = error_code;
	tsk->thread.trap_no = 14;
962
	force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
Linus Torvalds's avatar
Linus Torvalds committed
963
}
964

965
DEFINE_SPINLOCK(pgd_lock);
966
LIST_HEAD(pgd_list);
967
968
969

void vmalloc_sync_all(void)
{
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
#ifdef CONFIG_X86_32
	/*
	 * Note that races in the updates of insync and start aren't
	 * problematic: insync can only get set bits added, and updates to
	 * start are only improving performance (without affecting correctness
	 * if undone).
	 */
	static DECLARE_BITMAP(insync, PTRS_PER_PGD);
	static unsigned long start = TASK_SIZE;
	unsigned long address;

	if (SHARED_KERNEL_PMD)
		return;

	BUILD_BUG_ON(TASK_SIZE & ~PGDIR_MASK);
	for (address = start; address >= TASK_SIZE; address += PGDIR_SIZE) {
		if (!test_bit(pgd_index(address), insync)) {
			unsigned long flags;
			struct page *page;

			spin_lock_irqsave(&pgd_lock, flags);
991
			list_for_each_entry(page, &pgd_list, lru) {
992
				if (!vmalloc_sync_one(page_address(page),
993
						      address))
994
					break;
995
			}
996
997
998
999
1000
1001
1002
1003
			spin_unlock_irqrestore(&pgd_lock, flags);
			if (!page)
				set_bit(pgd_index(address), insync);
		}
		if (address == start && test_bit(pgd_index(address), insync))
			start = address + PGDIR_SIZE;
	}
#else /* CONFIG_X86_64 */
1004
1005
1006
1007
1008
1009
	/*
	 * Note that races in the updates of insync and start aren't
	 * problematic: insync can only get set bits added, and updates to
	 * start are only improving performance (without affecting correctness
	 * if undone).
	 */
1010
1011
1012
1013
1014
1015
1016
	static DECLARE_BITMAP(insync, PTRS_PER_PGD);
	static unsigned long start = VMALLOC_START & PGDIR_MASK;
	unsigned long address;

	for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
		if (!test_bit(pgd_index(address), insync)) {
			const pgd_t *pgd_ref = pgd_offset_k(address);
1017
			unsigned long flags;
1018
1019
1020
1021
			struct page *page;

			if (pgd_none(*pgd_ref))
				continue;
1022
			spin_lock_irqsave(&pgd_lock, flags);
1023
			list_for_each_entry(page, &pgd_list, lru) {
1024
1025
1026
1027
1028
				pgd_t *pgd;
				pgd = (pgd_t *)page_address(page) + pgd_index(address);
				if (pgd_none(*pgd))
					set_pgd(pgd, *pgd_ref);
				else
1029
					BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
1030
			}
1031
			spin_unlock_irqrestore(&pgd_lock, flags);
1032
1033
1034
1035
1036
			set_bit(pgd_index(address), insync);
		}
		if (address == start)
			start = address + PGDIR_SIZE;
	}
1037
#endif
1038
}