fault.c 25.8 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/*
 *  Copyright (C) 1995  Linus Torvalds
 *  Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
 */

#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/tty.h>
#include <linux/vt_kern.h>		/* For unblank_screen() */
#include <linux/compiler.h>
Harvey Harrison's avatar
Harvey Harrison committed
21 22
#include <linux/highmem.h>
#include <linux/bootmem.h>		/* for max_low_pfn */
23
#include <linux/vmalloc.h>
Linus Torvalds's avatar
Linus Torvalds committed
24
#include <linux/module.h>
25
#include <linux/kprobes.h>
26
#include <linux/uaccess.h>
27
#include <linux/kdebug.h>
Linus Torvalds's avatar
Linus Torvalds committed
28 29

#include <asm/system.h>
Harvey Harrison's avatar
Harvey Harrison committed
30 31
#include <asm/desc.h>
#include <asm/segment.h>
Linus Torvalds's avatar
Linus Torvalds committed
32 33 34 35 36 37
#include <asm/pgalloc.h>
#include <asm/smp.h>
#include <asm/tlbflush.h>
#include <asm/proto.h>
#include <asm-generic/sections.h>

38 39 40 41 42 43 44 45
/*
 * Page fault error code bits
 *	bit 0 == 0 means no page found, 1 means protection fault
 *	bit 1 == 0 means read, 1 means write
 *	bit 2 == 0 means kernel, 1 means user-mode
 *	bit 3 == 1 means use of reserved bit detected
 *	bit 4 == 1 means fault was an instruction fetch
 */
46
#define PF_PROT		(1<<0)
47
#define PF_WRITE	(1<<1)
48 49
#define PF_USER		(1<<2)
#define PF_RSVD		(1<<3)
50 51
#define PF_INSTR	(1<<4)

52 53 54
#ifdef CONFIG_MMIOTRACE_HOOKS
static pf_handler_func mmiotrace_pf_handler; /* protected by RCU */
static DEFINE_SPINLOCK(mmiotrace_handler_lock);
55

56
int mmiotrace_register_pf(pf_handler_func new_pfh)
57
{
58
	int ret = 0;
59
	unsigned long flags;
60 61 62 63 64 65 66
	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
	if (mmiotrace_pf_handler)
		ret = -EBUSY;
	else
		mmiotrace_pf_handler = new_pfh;
	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
	return ret;
67
}
68
EXPORT_SYMBOL_GPL(mmiotrace_register_pf);
69 70

/**
71
 * mmiotrace_unregister_pf:
72
 * The caller must ensure @old_pfh is not in use anymore before freeing it.
73 74
 * This function does not guarantee it. The handler function pointer is
 * protected by RCU, so you can do this by e.g. calling synchronize_rcu().
75
 */
76
int mmiotrace_unregister_pf(pf_handler_func old_pfh)
77
{
78
	int ret = 0;
79
	unsigned long flags;
80 81 82 83 84 85 86
	spin_lock_irqsave(&mmiotrace_handler_lock, flags);
	if (mmiotrace_pf_handler != old_pfh)
		ret = -EPERM;
	else
		mmiotrace_pf_handler = NULL;
	spin_unlock_irqrestore(&mmiotrace_handler_lock, flags);
	return ret;
87
}
88 89
EXPORT_SYMBOL_GPL(mmiotrace_unregister_pf);
#endif /* CONFIG_MMIOTRACE_HOOKS */
90 91

/* returns non-zero if do_page_fault() should return */
92 93 94
static inline int call_mmiotrace(struct pt_regs *regs,
					unsigned long error_code,
					unsigned long address)
95
{
96
#ifdef CONFIG_MMIOTRACE_HOOKS
97 98
	int ret = 0;
	rcu_read_lock();
99 100
	if (mmiotrace_pf_handler)
		ret = mmiotrace_pf_handler(regs, error_code, address);
101 102 103 104 105 106 107
	rcu_read_unlock();
	return ret;
#else
	return 0;
#endif
}

108
static inline int notify_page_fault(struct pt_regs *regs)
109
{
110
#ifdef CONFIG_KPROBES
111 112 113
	int ret = 0;

	/* kprobe_running() needs smp_processor_id() */
114 115 116
#ifdef CONFIG_X86_32
	if (!user_mode_vm(regs)) {
#else
117
	if (!user_mode(regs)) {
118
#endif
119 120 121 122 123
		preempt_disable();
		if (kprobe_running() && kprobe_fault_handler(regs, 14))
			ret = 1;
		preempt_enable();
	}
124

125 126 127 128
	return ret;
#else
	return 0;
#endif
129
}
130

131 132 133 134 135 136 137 138 139 140 141 142 143
/*
 * X86_32
 * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
 * Check that here and ignore it.
 *
 * X86_64
 * Sometimes the CPU reports invalid exceptions on prefetch.
 * Check that here and ignore it.
 *
 * Opcode checker based on code by Richard Brunner
 */
static int is_prefetch(struct pt_regs *regs, unsigned long addr,
		       unsigned long error_code)
144
{
145
	unsigned char *instr;
Linus Torvalds's avatar
Linus Torvalds committed
146
	int scan_more = 1;
147
	int prefetch = 0;
148
	unsigned char *max_instr;
Linus Torvalds's avatar
Linus Torvalds committed
149

Ingo Molnar's avatar
Ingo Molnar committed
150 151 152 153
	/*
	 * If it was a exec (instruction fetch) fault on NX page, then
	 * do not ignore the fault:
	 */
154
	if (error_code & PF_INSTR)
Linus Torvalds's avatar
Linus Torvalds committed
155
		return 0;
156

157
	instr = (unsigned char *)convert_ip_to_linear(current, regs);
158
	max_instr = instr + 15;
Linus Torvalds's avatar
Linus Torvalds committed
159

160
	if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
161 162
		return 0;

163
	while (scan_more && instr < max_instr) {
Linus Torvalds's avatar
Linus Torvalds committed
164 165 166 167
		unsigned char opcode;
		unsigned char instr_hi;
		unsigned char instr_lo;

168
		if (probe_kernel_address(instr, opcode))
169
			break;
Linus Torvalds's avatar
Linus Torvalds committed
170

171 172
		instr_hi = opcode & 0xf0;
		instr_lo = opcode & 0x0f;
Linus Torvalds's avatar
Linus Torvalds committed
173 174
		instr++;

175
		switch (instr_hi) {
Linus Torvalds's avatar
Linus Torvalds committed
176 177
		case 0x20:
		case 0x30:
178 179 180 181 182 183
			/*
			 * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
			 * In X86_64 long mode, the CPU will signal invalid
			 * opcode if some of these prefixes are present so
			 * X86_64 will never get here anyway
			 */
Linus Torvalds's avatar
Linus Torvalds committed
184 185
			scan_more = ((instr_lo & 7) == 0x6);
			break;
186
#ifdef CONFIG_X86_64
Linus Torvalds's avatar
Linus Torvalds committed
187
		case 0x40:
188 189 190 191 192 193 194
			/*
			 * In AMD64 long mode 0x40..0x4F are valid REX prefixes
			 * Need to figure out under what instruction mode the
			 * instruction was issued. Could check the LDT for lm,
			 * but for now it's good enough to assume that long
			 * mode only uses well known segments or kernel.
			 */
195
			scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
Linus Torvalds's avatar
Linus Torvalds committed
196
			break;
197
#endif
Linus Torvalds's avatar
Linus Torvalds committed
198 199 200
		case 0x60:
			/* 0x64 thru 0x67 are valid prefixes in all modes. */
			scan_more = (instr_lo & 0xC) == 0x4;
201
			break;
Linus Torvalds's avatar
Linus Torvalds committed
202
		case 0xF0:
203
			/* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
Linus Torvalds's avatar
Linus Torvalds committed
204
			scan_more = !instr_lo || (instr_lo>>1) == 1;
205
			break;
Linus Torvalds's avatar
Linus Torvalds committed
206 207 208
		case 0x00:
			/* Prefetch instruction is 0x0F0D or 0x0F18 */
			scan_more = 0;
209

210
			if (probe_kernel_address(instr, opcode))
Linus Torvalds's avatar
Linus Torvalds committed
211 212 213
				break;
			prefetch = (instr_lo == 0xF) &&
				(opcode == 0x0D || opcode == 0x18);
214
			break;
Linus Torvalds's avatar
Linus Torvalds committed
215 216 217
		default:
			scan_more = 0;
			break;
218
		}
Linus Torvalds's avatar
Linus Torvalds committed
219 220 221 222
	}
	return prefetch;
}

223 224 225 226 227 228 229 230 231 232 233 234
static void force_sig_info_fault(int si_signo, int si_code,
	unsigned long address, struct task_struct *tsk)
{
	siginfo_t info;

	info.si_signo = si_signo;
	info.si_errno = 0;
	info.si_code = si_code;
	info.si_addr = (void __user *)address;
	force_sig_info(si_signo, &info, tsk);
}

235
#ifdef CONFIG_X86_64
236 237
static int bad_address(void *p)
{
Linus Torvalds's avatar
Linus Torvalds committed
238
	unsigned long dummy;
239
	return probe_kernel_address((unsigned long *)p, dummy);
240
}
241
#endif
Linus Torvalds's avatar
Linus Torvalds committed
242

243
static void dump_pagetable(unsigned long address)
Linus Torvalds's avatar
Linus Torvalds committed
244
{
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
#ifdef CONFIG_X86_32
	__typeof__(pte_val(__pte(0))) page;

	page = read_cr3();
	page = ((__typeof__(page) *) __va(page))[address >> PGDIR_SHIFT];
#ifdef CONFIG_X86_PAE
	printk("*pdpt = %016Lx ", page);
	if ((page >> PAGE_SHIFT) < max_low_pfn
	    && page & _PAGE_PRESENT) {
		page &= PAGE_MASK;
		page = ((__typeof__(page) *) __va(page))[(address >> PMD_SHIFT)
		                                         & (PTRS_PER_PMD - 1)];
		printk(KERN_CONT "*pde = %016Lx ", page);
		page &= ~_PAGE_NX;
	}
#else
	printk("*pde = %08lx ", page);
#endif

	/*
	 * We must not directly access the pte in the highpte
	 * case if the page table is located in highmem.
	 * And let's rather not kmap-atomic the pte, just in case
	 * it's allocated already.
	 */
	if ((page >> PAGE_SHIFT) < max_low_pfn
	    && (page & _PAGE_PRESENT)
	    && !(page & _PAGE_PSE)) {
		page &= PAGE_MASK;
		page = ((__typeof__(page) *) __va(page))[(address >> PAGE_SHIFT)
		                                         & (PTRS_PER_PTE - 1)];
		printk("*pte = %0*Lx ", sizeof(page)*2, (u64)page);
	}

	printk("\n");
#else /* CONFIG_X86_64 */
Linus Torvalds's avatar
Linus Torvalds committed
281 282 283 284 285
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

286
	pgd = (pgd_t *)read_cr3();
Linus Torvalds's avatar
Linus Torvalds committed
287

288
	pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
Linus Torvalds's avatar
Linus Torvalds committed
289 290
	pgd += pgd_index(address);
	if (bad_address(pgd)) goto bad;
291
	printk("PGD %lx ", pgd_val(*pgd));
292
	if (!pgd_present(*pgd)) goto ret;
Linus Torvalds's avatar
Linus Torvalds committed
293

294
	pud = pud_offset(pgd, address);
Linus Torvalds's avatar
Linus Torvalds committed
295 296
	if (bad_address(pud)) goto bad;
	printk("PUD %lx ", pud_val(*pud));
297 298
	if (!pud_present(*pud) || pud_large(*pud))
		goto ret;
Linus Torvalds's avatar
Linus Torvalds committed
299 300 301 302

	pmd = pmd_offset(pud, address);
	if (bad_address(pmd)) goto bad;
	printk("PMD %lx ", pmd_val(*pmd));
303
	if (!pmd_present(*pmd) || pmd_large(*pmd)) goto ret;
Linus Torvalds's avatar
Linus Torvalds committed
304 305 306

	pte = pte_offset_kernel(pmd, address);
	if (bad_address(pte)) goto bad;
307
	printk("PTE %lx", pte_val(*pte));
Linus Torvalds's avatar
Linus Torvalds committed
308 309 310 311 312
ret:
	printk("\n");
	return;
bad:
	printk("BAD\n");
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
#endif
}

#ifdef CONFIG_X86_32
static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
{
	unsigned index = pgd_index(address);
	pgd_t *pgd_k;
	pud_t *pud, *pud_k;
	pmd_t *pmd, *pmd_k;

	pgd += index;
	pgd_k = init_mm.pgd + index;

	if (!pgd_present(*pgd_k))
		return NULL;

	/*
	 * set_pgd(pgd, *pgd_k); here would be useless on PAE
	 * and redundant with the set_pmd() on non-PAE. As would
	 * set_pud.
	 */

	pud = pud_offset(pgd, address);
	pud_k = pud_offset(pgd_k, address);
	if (!pud_present(*pud_k))
		return NULL;

	pmd = pmd_offset(pud, address);
	pmd_k = pmd_offset(pud_k, address);
	if (!pmd_present(*pmd_k))
		return NULL;
	if (!pmd_present(*pmd)) {
		set_pmd(pmd, *pmd_k);
		arch_flush_lazy_mmu_mode();
	} else
		BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
	return pmd_k;
Linus Torvalds's avatar
Linus Torvalds committed
351
}
352
#endif
Linus Torvalds's avatar
Linus Torvalds committed
353

354
#ifdef CONFIG_X86_64
355
static const char errata93_warning[] =
Linus Torvalds's avatar
Linus Torvalds committed
356 357 358 359
KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
KERN_ERR "******* Please consider a BIOS update.\n"
KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
360
#endif
Linus Torvalds's avatar
Linus Torvalds committed
361 362 363

/* Workaround for K8 erratum #93 & buggy BIOS.
   BIOS SMM functions are required to use a specific workaround
364 365
   to avoid corruption of the 64bit RIP register on C stepping K8.
   A lot of BIOS that didn't get tested properly miss this.
Linus Torvalds's avatar
Linus Torvalds committed
366 367
   The OS sees this as a page fault with the upper 32bits of RIP cleared.
   Try to work around it here.
368 369 370
   Note we only handle faults in kernel here.
   Does nothing for X86_32
 */
371
static int is_errata93(struct pt_regs *regs, unsigned long address)
Linus Torvalds's avatar
Linus Torvalds committed
372
{
373
#ifdef CONFIG_X86_64
Linus Torvalds's avatar
Linus Torvalds committed
374
	static int warned;
375
	if (address != regs->ip)
Linus Torvalds's avatar
Linus Torvalds committed
376
		return 0;
377
	if ((address >> 32) != 0)
Linus Torvalds's avatar
Linus Torvalds committed
378 379
		return 0;
	address |= 0xffffffffUL << 32;
380 381
	if ((address >= (u64)_stext && address <= (u64)_etext) ||
	    (address >= MODULES_VADDR && address <= MODULES_END)) {
Linus Torvalds's avatar
Linus Torvalds committed
382
		if (!warned) {
383
			printk(errata93_warning);
Linus Torvalds's avatar
Linus Torvalds committed
384 385
			warned = 1;
		}
386
		regs->ip = address;
Linus Torvalds's avatar
Linus Torvalds committed
387 388
		return 1;
	}
389
#endif
Linus Torvalds's avatar
Linus Torvalds committed
390
	return 0;
391
}
Linus Torvalds's avatar
Linus Torvalds committed
392

393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
/*
 * Work around K8 erratum #100 K8 in compat mode occasionally jumps to illegal
 * addresses >4GB.  We catch this in the page fault handler because these
 * addresses are not reachable. Just detect this case and return.  Any code
 * segment in LDT is compatibility mode.
 */
static int is_errata100(struct pt_regs *regs, unsigned long address)
{
#ifdef CONFIG_X86_64
	if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
	    (address >> 32))
		return 1;
#endif
	return 0;
}

409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
void do_invalid_op(struct pt_regs *, unsigned long);

static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
{
#ifdef CONFIG_X86_F00F_BUG
	unsigned long nr;
	/*
	 * Pentium F0 0F C7 C8 bug workaround.
	 */
	if (boot_cpu_data.f00f_bug) {
		nr = (address - idt_descr.address) >> 3;

		if (nr == 6) {
			do_invalid_op(regs, 0);
			return 1;
		}
	}
#endif
	return 0;
}

430 431 432
static void show_fault_oops(struct pt_regs *regs, unsigned long error_code,
			    unsigned long address)
{
433 434 435
#ifdef CONFIG_X86_32
	if (!oops_may_print())
		return;
436
#endif
437 438 439

#ifdef CONFIG_X86_PAE
	if (error_code & PF_INSTR) {
440
		unsigned int level;
441 442 443 444 445 446 447 448 449
		pte_t *pte = lookup_address(address, &level);

		if (pte && pte_present(*pte) && !pte_exec(*pte))
			printk(KERN_CRIT "kernel tried to execute "
				"NX-protected page - exploit attempt? "
				"(uid: %d)\n", current->uid);
	}
#endif

450
	printk(KERN_ALERT "BUG: unable to handle kernel ");
451
	if (address < PAGE_SIZE)
452
		printk(KERN_CONT "NULL pointer dereference");
453
	else
454
		printk(KERN_CONT "paging request");
455 456 457
#ifdef CONFIG_X86_32
	printk(KERN_CONT " at %08lx\n", address);
#else
458
	printk(KERN_CONT " at %016lx\n", address);
459
#endif
460
	printk(KERN_ALERT "IP:");
461 462 463 464
	printk_address(regs->ip, 1);
	dump_pagetable(address);
}

465
#ifdef CONFIG_X86_64
Linus Torvalds's avatar
Linus Torvalds committed
466 467 468
static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
				 unsigned long error_code)
{
469
	unsigned long flags = oops_begin();
470
	struct task_struct *tsk;
471

Linus Torvalds's avatar
Linus Torvalds committed
472 473 474
	printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
	       current->comm, address);
	dump_pagetable(address);
475 476 477 478
	tsk = current;
	tsk->thread.cr2 = address;
	tsk->thread.trap_no = 14;
	tsk->thread.error_code = error_code;
479 480 481
	if (__die("Bad pagetable", regs, error_code))
		regs = NULL;
	oops_end(flags, regs, SIGKILL);
Linus Torvalds's avatar
Linus Torvalds committed
482
}
483
#endif
Linus Torvalds's avatar
Linus Torvalds committed
484

485 486 487 488 489 490 491 492 493 494
static int spurious_fault_check(unsigned long error_code, pte_t *pte)
{
	if ((error_code & PF_WRITE) && !pte_write(*pte))
		return 0;
	if ((error_code & PF_INSTR) && !pte_exec(*pte))
		return 0;

	return 1;
}

495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
/*
 * Handle a spurious fault caused by a stale TLB entry.  This allows
 * us to lazily refresh the TLB when increasing the permissions of a
 * kernel page (RO -> RW or NX -> X).  Doing it eagerly is very
 * expensive since that implies doing a full cross-processor TLB
 * flush, even if no stale TLB entries exist on other processors.
 * There are no security implications to leaving a stale TLB when
 * increasing the permissions on a page.
 */
static int spurious_fault(unsigned long address,
			  unsigned long error_code)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	/* Reserved-bit violation or user access to kernel space? */
	if (error_code & (PF_USER | PF_RSVD))
		return 0;

	pgd = init_mm.pgd + pgd_index(address);
	if (!pgd_present(*pgd))
		return 0;

	pud = pud_offset(pgd, address);
	if (!pud_present(*pud))
		return 0;

524 525 526
	if (pud_large(*pud))
		return spurious_fault_check(error_code, (pte_t *) pud);

527 528 529 530
	pmd = pmd_offset(pud, address);
	if (!pmd_present(*pmd))
		return 0;

531 532 533
	if (pmd_large(*pmd))
		return spurious_fault_check(error_code, (pte_t *) pmd);

534 535 536 537
	pte = pte_offset_kernel(pmd, address);
	if (!pte_present(*pte))
		return 0;

538
	return spurious_fault_check(error_code, pte);
539 540
}

Linus Torvalds's avatar
Linus Torvalds committed
541
/*
542 543 544 545
 * X86_32
 * Handle a fault on the vmalloc or module mapping area
 *
 * X86_64
546
 * Handle a fault on the vmalloc area
547 548
 *
 * This assumes no large pages in there.
Linus Torvalds's avatar
Linus Torvalds committed
549 550 551
 */
static int vmalloc_fault(unsigned long address)
{
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
#ifdef CONFIG_X86_32
	unsigned long pgd_paddr;
	pmd_t *pmd_k;
	pte_t *pte_k;
	/*
	 * Synchronize this task's top level page-table
	 * with the 'reference' page table.
	 *
	 * Do _not_ use "current" here. We might be inside
	 * an interrupt in the middle of a task switch..
	 */
	pgd_paddr = read_cr3();
	pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
	if (!pmd_k)
		return -1;
	pte_k = pte_offset_kernel(pmd_k, address);
	if (!pte_present(*pte_k))
		return -1;
	return 0;
#else
Linus Torvalds's avatar
Linus Torvalds committed
572 573 574 575 576
	pgd_t *pgd, *pgd_ref;
	pud_t *pud, *pud_ref;
	pmd_t *pmd, *pmd_ref;
	pte_t *pte, *pte_ref;

577 578 579 580
	/* Make sure we are in vmalloc area */
	if (!(address >= VMALLOC_START && address < VMALLOC_END))
		return -1;

Linus Torvalds's avatar
Linus Torvalds committed
581 582 583 584 585 586 587 588 589 590
	/* Copy kernel mappings over when needed. This can also
	   happen within a race in page table update. In the later
	   case just flush. */

	pgd = pgd_offset(current->mm ?: &init_mm, address);
	pgd_ref = pgd_offset_k(address);
	if (pgd_none(*pgd_ref))
		return -1;
	if (pgd_none(*pgd))
		set_pgd(pgd, *pgd_ref);
591
	else
592
		BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
Linus Torvalds's avatar
Linus Torvalds committed
593 594 595 596 597 598 599 600

	/* Below here mismatches are bugs because these lower tables
	   are shared */

	pud = pud_offset(pgd, address);
	pud_ref = pud_offset(pgd_ref, address);
	if (pud_none(*pud_ref))
		return -1;
601
	if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
Linus Torvalds's avatar
Linus Torvalds committed
602 603 604 605 606 607 608 609 610 611 612
		BUG();
	pmd = pmd_offset(pud, address);
	pmd_ref = pmd_offset(pud_ref, address);
	if (pmd_none(*pmd_ref))
		return -1;
	if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
		BUG();
	pte_ref = pte_offset_kernel(pmd_ref, address);
	if (!pte_present(*pte_ref))
		return -1;
	pte = pte_offset_kernel(pmd, address);
613 614 615 616
	/* Don't use pte_page here, because the mappings can point
	   outside mem_map, and the NUMA hash lookup cannot handle
	   that. */
	if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
Linus Torvalds's avatar
Linus Torvalds committed
617 618
		BUG();
	return 0;
619
#endif
Linus Torvalds's avatar
Linus Torvalds committed
620 621
}

622
int show_unhandled_signals = 1;
Linus Torvalds's avatar
Linus Torvalds committed
623 624 625 626 627 628

/*
 * This routine handles page faults.  It determines the address,
 * and the problem, and then passes it off to one of the appropriate
 * routines.
 */
629 630 631 632
#ifdef CONFIG_X86_64
asmlinkage
#endif
void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
Linus Torvalds's avatar
Linus Torvalds committed
633 634 635
{
	struct task_struct *tsk;
	struct mm_struct *mm;
636
	struct vm_area_struct *vma;
Linus Torvalds's avatar
Linus Torvalds committed
637
	unsigned long address;
638 639 640
	int write, si_code;
	int fault;
#ifdef CONFIG_X86_64
641
	unsigned long flags;
642
#endif
Linus Torvalds's avatar
Linus Torvalds committed
643

Peter Zijlstra's avatar
Peter Zijlstra committed
644 645 646 647 648
	/*
	 * We can fault from pretty much anywhere, with unknown IRQ state.
	 */
	trace_hardirqs_fixup();

649 650 651 652
	tsk = current;
	mm = tsk->mm;
	prefetchw(&mm->mmap_sem);

Linus Torvalds's avatar
Linus Torvalds committed
653
	/* get the address */
654
	address = read_cr2();
Linus Torvalds's avatar
Linus Torvalds committed
655

656
	si_code = SEGV_MAPERR;
Linus Torvalds's avatar
Linus Torvalds committed
657

658 659
	if (notify_page_fault(regs))
		return;
660
	if (call_mmiotrace(regs, error_code, address))
661
		return;
Linus Torvalds's avatar
Linus Torvalds committed
662 663 664 665 666 667 668 669 670 671 672 673

	/*
	 * We fault-in kernel-space virtual memory on-demand. The
	 * 'reference' page table is init_mm.pgd.
	 *
	 * NOTE! We MUST NOT take any locks for this case. We may
	 * be in an interrupt or a critical region, and should
	 * only copy the information from the master page table,
	 * nothing more.
	 *
	 * This verifies that the fault happens in kernel space
	 * (error_code & 4) == 0, and that the fault was not a
674
	 * protection error (error_code & 9) == 0.
Linus Torvalds's avatar
Linus Torvalds committed
675
	 */
676 677
#ifdef CONFIG_X86_32
	if (unlikely(address >= TASK_SIZE)) {
678 679 680
#else
	if (unlikely(address >= TASK_SIZE64)) {
#endif
681 682 683
		if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
		    vmalloc_fault(address) >= 0)
			return;
684 685 686 687 688

		/* Can handle a stale RO->RW TLB */
		if (spurious_fault(address, error_code))
			return;

689 690 691 692 693 694 695
		/*
		 * Don't take the mm semaphore here. If we fixup a prefetch
		 * fault we could otherwise deadlock.
		 */
		goto bad_area_nosemaphore;
	}

696 697

#ifdef CONFIG_X86_32
698 699
	/* It's safe to allow irq's after cr2 has been saved and the vmalloc
	   fault has been handled. */
700
	if (regs->flags & (X86_EFLAGS_IF | X86_VM_MASK))
701 702 703 704 705 706 707 708 709
		local_irq_enable();

	/*
	 * If we're in an interrupt, have no user context or are running in an
	 * atomic region then we must not take the fault.
	 */
	if (in_atomic() || !mm)
		goto bad_area_nosemaphore;
#else /* CONFIG_X86_64 */
710
	if (likely(regs->flags & X86_EFLAGS_IF))
711 712
		local_irq_enable();

713
	if (unlikely(error_code & PF_RSVD))
Linus Torvalds's avatar
Linus Torvalds committed
714 715 716
		pgtable_bad(address, regs, error_code);

	/*
717 718
	 * If we're in an interrupt, have no user context or are running in an
	 * atomic region then we must not take the fault.
Linus Torvalds's avatar
Linus Torvalds committed
719 720 721 722
	 */
	if (unlikely(in_atomic() || !mm))
		goto bad_area_nosemaphore;

723 724 725 726 727 728
	/*
	 * User-mode registers count as a user access even for any
	 * potential system fault or CPU buglet.
	 */
	if (user_mode_vm(regs))
		error_code |= PF_USER;
729 730
again:
#endif
Linus Torvalds's avatar
Linus Torvalds committed
731 732
	/* When running in the kernel we expect faults to occur only to
	 * addresses in user space.  All other faults represent errors in the
Simon Arlott's avatar
Simon Arlott committed
733
	 * kernel and should generate an OOPS.  Unfortunately, in the case of an
734
	 * erroneous fault occurring in a code path which already holds mmap_sem
Linus Torvalds's avatar
Linus Torvalds committed
735 736 737 738 739 740
	 * we will deadlock attempting to validate the fault against the
	 * address space.  Luckily the kernel only validly references user
	 * space from well defined areas of code, which are listed in the
	 * exceptions table.
	 *
	 * As the vast majority of faults will be valid we will only perform
Simon Arlott's avatar
Simon Arlott committed
741
	 * the source reference check when there is a possibility of a deadlock.
Linus Torvalds's avatar
Linus Torvalds committed
742 743 744 745 746
	 * Attempt to lock the address space, if we cannot we then validate the
	 * source.  If this is invalid we can skip the address space check,
	 * thus avoiding the deadlock.
	 */
	if (!down_read_trylock(&mm->mmap_sem)) {
747
		if ((error_code & PF_USER) == 0 &&
748
		    !search_exception_tables(regs->ip))
Linus Torvalds's avatar
Linus Torvalds committed
749 750 751 752 753 754 755
			goto bad_area_nosemaphore;
		down_read(&mm->mmap_sem);
	}

	vma = find_vma(mm, address);
	if (!vma)
		goto bad_area;
756
	if (vma->vm_start <= address)
Linus Torvalds's avatar
Linus Torvalds committed
757 758 759
		goto good_area;
	if (!(vma->vm_flags & VM_GROWSDOWN))
		goto bad_area;
760
	if (error_code & PF_USER) {
761 762 763 764 765
		/*
		 * Accessing the stack below %sp is always a bug.
		 * The large cushion allows instructions like enter
		 * and pusha to work.  ("enter $65535,$31" pushes
		 * 32 pointers and then decrements %sp by 65535.)
766
		 */
767
		if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
Linus Torvalds's avatar
Linus Torvalds committed
768 769 770 771 772 773 774 775 776
			goto bad_area;
	}
	if (expand_stack(vma, address))
		goto bad_area;
/*
 * Ok, we have a good vm_area for this memory access, so
 * we can handle it..
 */
good_area:
777
	si_code = SEGV_ACCERR;
Linus Torvalds's avatar
Linus Torvalds committed
778
	write = 0;
779
	switch (error_code & (PF_PROT|PF_WRITE)) {
780 781 782 783 784 785 786 787 788 789 790
	default:	/* 3: write, present */
		/* fall through */
	case PF_WRITE:		/* write, not present */
		if (!(vma->vm_flags & VM_WRITE))
			goto bad_area;
		write++;
		break;
	case PF_PROT:		/* read, present */
		goto bad_area;
	case 0:			/* read, not present */
		if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
Linus Torvalds's avatar
Linus Torvalds committed
791 792 793
			goto bad_area;
	}

794 795 796
#ifdef CONFIG_X86_32
survive:
#endif
Linus Torvalds's avatar
Linus Torvalds committed
797 798 799 800 801
	/*
	 * If for any reason at all we couldn't handle the fault,
	 * make sure we exit gracefully rather than endlessly redo
	 * the fault.
	 */
Nick Piggin's avatar
Nick Piggin committed
802 803 804 805 806 807 808
	fault = handle_mm_fault(mm, vma, address, write);
	if (unlikely(fault & VM_FAULT_ERROR)) {
		if (fault & VM_FAULT_OOM)
			goto out_of_memory;
		else if (fault & VM_FAULT_SIGBUS)
			goto do_sigbus;
		BUG();
Linus Torvalds's avatar
Linus Torvalds committed
809
	}
Nick Piggin's avatar
Nick Piggin committed
810 811 812 813
	if (fault & VM_FAULT_MAJOR)
		tsk->maj_flt++;
	else
		tsk->min_flt++;
814 815 816 817 818 819 820 821 822 823 824

#ifdef CONFIG_X86_32
	/*
	 * Did it hit the DOS screen memory VA from vm86 mode?
	 */
	if (v8086_mode(regs)) {
		unsigned long bit = (address - 0xA0000) >> PAGE_SHIFT;
		if (bit < 32)
			tsk->thread.screen_bitmap |= 1 << bit;
	}
#endif
Linus Torvalds's avatar
Linus Torvalds committed
825 826 827 828 829 830 831 832 833 834 835 836
	up_read(&mm->mmap_sem);
	return;

/*
 * Something tried to access memory that isn't in our memory map..
 * Fix it, but check if it's kernel or user first..
 */
bad_area:
	up_read(&mm->mmap_sem);

bad_area_nosemaphore:
	/* User mode accesses just cause a SIGSEGV */
837
	if (error_code & PF_USER) {
838 839 840 841 842
		/*
		 * It's possible to have interrupts off here.
		 */
		local_irq_enable();

843 844 845 846
		/*
		 * Valid to do another page fault here because this one came
		 * from user space.
		 */
Linus Torvalds's avatar
Linus Torvalds committed
847 848 849
		if (is_prefetch(regs, address, error_code))
			return;

850
		if (is_errata100(regs, address))
Linus Torvalds's avatar
Linus Torvalds committed
851 852
			return;

853 854
		if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
		    printk_ratelimit()) {
Linus Torvalds's avatar
Linus Torvalds committed
855
			printk(
856
#ifdef CONFIG_X86_32
857
			"%s%s[%d]: segfault at %lx ip %08lx sp %08lx error %lx",
858
#else
859
			"%s%s[%d]: segfault at %lx ip %lx sp %lx error %lx",
860 861 862 863
#endif
			task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
			tsk->comm, task_pid_nr(tsk), address, regs->ip,
			regs->sp, error_code);
864 865
			print_vma_addr(" in ", regs->ip);
			printk("\n");
Linus Torvalds's avatar
Linus Torvalds committed
866
		}
867

Linus Torvalds's avatar
Linus Torvalds committed
868 869 870 871
		tsk->thread.cr2 = address;
		/* Kernel addresses are always protection faults */
		tsk->thread.error_code = error_code | (address >= TASK_SIZE);
		tsk->thread.trap_no = 14;
872
		force_sig_info_fault(SIGSEGV, si_code, address, tsk);
Linus Torvalds's avatar
Linus Torvalds committed
873 874 875
		return;
	}

876 877 878
	if (is_f00f_bug(regs, address))
		return;

Linus Torvalds's avatar
Linus Torvalds committed
879 880
no_context:
	/* Are we prepared to handle this kernel fault?  */
881
	if (fixup_exception(regs))
Linus Torvalds's avatar
Linus Torvalds committed
882 883
		return;

884
	/*
885 886 887 888 889 890
	 * X86_32
	 * Valid to do another page fault here, because if this fault
	 * had been triggered by is_prefetch fixup_exception would have
	 * handled it.
	 *
	 * X86_64
Linus Torvalds's avatar
Linus Torvalds committed
891 892
	 * Hall of shame of CPU/BIOS bugs.
	 */
893 894
	if (is_prefetch(regs, address, error_code))
		return;
Linus Torvalds's avatar
Linus Torvalds committed
895 896

	if (is_errata93(regs, address))
897
		return;
Linus Torvalds's avatar
Linus Torvalds committed
898 899 900 901 902

/*
 * Oops. The kernel tried to access some bad page. We'll have to
 * terminate things with extreme prejudice.
 */
903 904
#ifdef CONFIG_X86_32
	bust_spinlocks(1);
905 906 907
#else
	flags = oops_begin();
#endif
908 909

	show_fault_oops(regs, error_code, address);
Linus Torvalds's avatar
Linus Torvalds committed
910

911 912 913
	tsk->thread.cr2 = address;
	tsk->thread.trap_no = 14;
	tsk->thread.error_code = error_code;
914 915

#ifdef CONFIG_X86_32
916 917 918
	die("Oops", regs, error_code);
	bust_spinlocks(0);
	do_exit(SIGKILL);
919
#else
920 921
	if (__die("Oops", regs, error_code))
		regs = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
922 923
	/* Executive summary in case the body of the oops scrolled away */
	printk(KERN_EMERG "CR2: %016lx\n", address);
924
	oops_end(flags, regs, SIGKILL);
925
#endif
Linus Torvalds's avatar
Linus Torvalds committed
926 927 928 929 930 931 932

/*
 * We ran out of memory, or some other thing happened to us that made
 * us unable to handle the page fault gracefully.
 */
out_of_memory:
	up_read(&mm->mmap_sem);
933 934
	if (is_global_init(tsk)) {
		yield();
935
#ifdef CONFIG_X86_32
936 937 938
		down_read(&mm->mmap_sem);
		goto survive;
#else
Linus Torvalds's avatar
Linus Torvalds committed
939
		goto again;
940
#endif
941 942
	}

Linus Torvalds's avatar
Linus Torvalds committed
943
	printk("VM: killing process %s\n", tsk->comm);
944
	if (error_code & PF_USER)
945
		do_group_exit(SIGKILL);
Linus Torvalds's avatar
Linus Torvalds committed
946 947 948 949 950 951
	goto no_context;

do_sigbus:
	up_read(&mm->mmap_sem);

	/* Kernel mode? Handle exceptions or die */
952
	if (!(error_code & PF_USER))
Linus Torvalds's avatar
Linus Torvalds committed
953
		goto no_context;
954 955 956 957 958
#ifdef CONFIG_X86_32
	/* User space => ok to do another page fault */
	if (is_prefetch(regs, address, error_code))
		return;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
959 960 961
	tsk->thread.cr2 = address;
	tsk->thread.error_code = error_code;
	tsk->thread.trap_no = 14;
962
	force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
Linus Torvalds's avatar
Linus Torvalds committed
963
}
964

965
DEFINE_SPINLOCK(pgd_lock);
966
LIST_HEAD(pgd_list);
967 968 969

void vmalloc_sync_all(void)
{
970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
#ifdef CONFIG_X86_32
	/*
	 * Note that races in the updates of insync and start aren't
	 * problematic: insync can only get set bits added, and updates to
	 * start are only improving performance (without affecting correctness
	 * if undone).
	 */
	static DECLARE_BITMAP(insync, PTRS_PER_PGD);
	static unsigned long start = TASK_SIZE;
	unsigned long address;

	if (SHARED_KERNEL_PMD)
		return;

	BUILD_BUG_ON(TASK_SIZE & ~PGDIR_MASK);
	for (address = start; address >= TASK_SIZE; address += PGDIR_SIZE) {
		if (!test_bit(pgd_index(address), insync)) {
			unsigned long flags;
			struct page *page;

			spin_lock_irqsave(&pgd_lock, flags);
991
			list_for_each_entry(page, &pgd_list, lru) {
992
				if (!vmalloc_sync_one(page_address(page),
993
						      address))
994
					break;
995
			}
996 997 998 999 1000 1001 1002 1003
			spin_unlock_irqrestore(&pgd_lock, flags);
			if (!page)
				set_bit(pgd_index(address), insync);
		}
		if (address == start && test_bit(pgd_index(address), insync))
			start = address + PGDIR_SIZE;
	}
#else /* CONFIG_X86_64 */
1004 1005 1006 1007 1008 1009
	/*
	 * Note that races in the updates of insync and start aren't
	 * problematic: insync can only get set bits added, and updates to
	 * start are only improving performance (without affecting correctness
	 * if undone).
	 */
1010 1011 1012 1013 1014 1015 1016
	static DECLARE_BITMAP(insync, PTRS_PER_PGD);
	static unsigned long start = VMALLOC_START & PGDIR_MASK;
	unsigned long address;

	for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
		if (!test_bit(pgd_index(address), insync)) {
			const pgd_t *pgd_ref = pgd_offset_k(address);
1017
			unsigned long flags;
1018 1019 1020 1021
			struct page *page;

			if (pgd_none(*pgd_ref))
				continue;
1022
			spin_lock_irqsave(&pgd_lock, flags);
1023
			list_for_each_entry(page, &pgd_list, lru) {
1024 1025 1026 1027 1028
				pgd_t *pgd;
				pgd = (pgd_t *)page_address(page) + pgd_index(address);
				if (pgd_none(*pgd))
					set_pgd(pgd, *pgd_ref);
				else
1029
					BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
1030
			}
1031
			spin_unlock_irqrestore(&pgd_lock, flags);
1032 1033 1034 1035 1036
			set_bit(pgd_index(address), insync);
		}
		if (address == start)
			start = address + PGDIR_SIZE;
	}
1037
#endif
1038
}