mmap.c 69.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
/*
 * mm/mmap.c
 *
 * Written by obz.
 *
Alan Cox's avatar
Alan Cox committed
6
 * Address space accounting code	<alan@lxorguk.ukuu.org.uk>
Linus Torvalds's avatar
Linus Torvalds committed
7
8
9
 */

#include <linux/slab.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
10
#include <linux/backing-dev.h>
Linus Torvalds's avatar
Linus Torvalds committed
11
12
13
14
15
16
#include <linux/mm.h>
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/syscalls.h>
17
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
19
20
21
22
23
24
25
26
27
28
#include <linux/init.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/personality.h>
#include <linux/security.h>
#include <linux/hugetlb.h>
#include <linux/profile.h>
#include <linux/module.h>
#include <linux/mount.h>
#include <linux/mempolicy.h>
#include <linux/rmap.h>
Andrea Arcangeli's avatar
Andrea Arcangeli committed
29
#include <linux/mmu_notifier.h>
30
#include <linux/perf_event.h>
Al Viro's avatar
Al Viro committed
31
#include <linux/audit.h>
Andrea Arcangeli's avatar
Andrea Arcangeli committed
32
#include <linux/khugepaged.h>
Linus Torvalds's avatar
Linus Torvalds committed
33
34
35
36

#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlb.h>
37
#include <asm/mmu_context.h>
Linus Torvalds's avatar
Linus Torvalds committed
38

39
40
#include "internal.h"

41
42
43
44
#ifndef arch_mmap_check
#define arch_mmap_check(addr, len, flags)	(0)
#endif

45
46
47
48
#ifndef arch_rebalance_pgtables
#define arch_rebalance_pgtables(addr, len)		(addr)
#endif

49
50
51
52
static void unmap_region(struct mm_struct *mm,
		struct vm_area_struct *vma, struct vm_area_struct *prev,
		unsigned long start, unsigned long end);

Linus Torvalds's avatar
Linus Torvalds committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
/*
 * WARNING: the debugging will use recursive algorithms so never enable this
 * unless you know what you are doing.
 */
#undef DEBUG_MM_RB

/* description of effects of mapping type and prot in current implementation.
 * this is due to the limited x86 page protection hardware.  The expected
 * behavior is in parens:
 *
 * map_type	prot
 *		PROT_NONE	PROT_READ	PROT_WRITE	PROT_EXEC
 * MAP_SHARED	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (yes) yes	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
 *		
 * MAP_PRIVATE	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (copy) copy	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
 *
 */
pgprot_t protection_map[16] = {
	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
};

79
80
pgprot_t vm_get_page_prot(unsigned long vm_flags)
{
81
82
83
	return __pgprot(pgprot_val(protection_map[vm_flags &
				(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
			pgprot_val(arch_vm_get_page_prot(vm_flags)));
84
85
86
}
EXPORT_SYMBOL(vm_get_page_prot);

87
88
int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;  /* heuristic overcommit */
int sysctl_overcommit_ratio __read_mostly = 50;	/* default is 50% */
89
int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
90
91
92
93
94
/*
 * Make sure vm_committed_as in one cacheline and not cacheline shared with
 * other variables. It can be updated by several CPUs frequently.
 */
struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
Linus Torvalds's avatar
Linus Torvalds committed
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

/*
 * Check that a process has enough memory to allocate a new virtual
 * mapping. 0 means there is enough memory for the allocation to
 * succeed and -ENOMEM implies there is not.
 *
 * We currently support three overcommit policies, which are set via the
 * vm.overcommit_memory sysctl.  See Documentation/vm/overcommit-accounting
 *
 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
 * Additional code 2002 Jul 20 by Robert Love.
 *
 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
 *
 * Note this is a helper function intended to be used by LSMs which
 * wish to use this logic.
 */
112
int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
Linus Torvalds's avatar
Linus Torvalds committed
113
114
115
116
117
118
119
120
121
122
123
124
{
	unsigned long free, allowed;

	vm_acct_memory(pages);

	/*
	 * Sometimes we want to use more memory than we have
	 */
	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
		return 0;

	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
125
126
127
128
129
130
131
132
133
134
		free = global_page_state(NR_FREE_PAGES);
		free += global_page_state(NR_FILE_PAGES);

		/*
		 * shmem pages shouldn't be counted as free in this
		 * case, they can't be purged, only swapped out, and
		 * that won't affect the overall amount of available
		 * memory in the system.
		 */
		free -= global_page_state(NR_SHMEM);
Linus Torvalds's avatar
Linus Torvalds committed
135
136
137
138
139
140
141
142
143

		free += nr_swap_pages;

		/*
		 * Any slabs which are created with the
		 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
		 * which are reclaimable, under pressure.  The dentry
		 * cache and most inode caches should fall into this
		 */
144
		free += global_page_state(NR_SLAB_RECLAIMABLE);
Linus Torvalds's avatar
Linus Torvalds committed
145

146
147
148
		/*
		 * Leave reserved pages. The pages are not for anonymous pages.
		 */
149
		if (free <= totalreserve_pages)
150
151
			goto error;
		else
152
			free -= totalreserve_pages;
153
154
155
156

		/*
		 * Leave the last 3% for root
		 */
Linus Torvalds's avatar
Linus Torvalds committed
157
		if (!cap_sys_admin)
158
			free -= free / 32;
Linus Torvalds's avatar
Linus Torvalds committed
159
160
161

		if (free > pages)
			return 0;
162
163

		goto error;
Linus Torvalds's avatar
Linus Torvalds committed
164
165
166
167
168
169
170
171
172
173
174
175
176
	}

	allowed = (totalram_pages - hugetlb_total_pages())
	       	* sysctl_overcommit_ratio / 100;
	/*
	 * Leave the last 3% for root
	 */
	if (!cap_sys_admin)
		allowed -= allowed / 32;
	allowed += total_swap_pages;

	/* Don't let a single process grow too big:
	   leave 3% of the size of this process for other processes */
Alan Cox's avatar
Alan Cox committed
177
178
	if (mm)
		allowed -= mm->total_vm / 32;
Linus Torvalds's avatar
Linus Torvalds committed
179

180
	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
Linus Torvalds's avatar
Linus Torvalds committed
181
		return 0;
182
error:
Linus Torvalds's avatar
Linus Torvalds committed
183
184
185
186
187
188
	vm_unacct_memory(pages);

	return -ENOMEM;
}

/*
189
 * Requires inode->i_mapping->i_mmap_mutex
Linus Torvalds's avatar
Linus Torvalds committed
190
191
192
193
194
 */
static void __remove_shared_vm_struct(struct vm_area_struct *vma,
		struct file *file, struct address_space *mapping)
{
	if (vma->vm_flags & VM_DENYWRITE)
195
		atomic_inc(&file->f_path.dentry->d_inode->i_writecount);
Linus Torvalds's avatar
Linus Torvalds committed
196
197
198
199
200
201
202
203
204
205
206
207
	if (vma->vm_flags & VM_SHARED)
		mapping->i_mmap_writable--;

	flush_dcache_mmap_lock(mapping);
	if (unlikely(vma->vm_flags & VM_NONLINEAR))
		list_del_init(&vma->shared.vm_set.list);
	else
		vma_prio_tree_remove(vma, &mapping->i_mmap);
	flush_dcache_mmap_unlock(mapping);
}

/*
208
209
 * Unlink a file-based vm structure from its prio_tree, to hide
 * vma from rmap and vmtruncate before freeing its page tables.
Linus Torvalds's avatar
Linus Torvalds committed
210
 */
211
void unlink_file_vma(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
212
213
214
215
216
{
	struct file *file = vma->vm_file;

	if (file) {
		struct address_space *mapping = file->f_mapping;
217
		mutex_lock(&mapping->i_mmap_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
218
		__remove_shared_vm_struct(vma, file, mapping);
219
		mutex_unlock(&mapping->i_mmap_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
220
	}
221
222
223
224
225
226
227
228
229
230
}

/*
 * Close a vm structure and free it, returning the next.
 */
static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
{
	struct vm_area_struct *next = vma->vm_next;

	might_sleep();
Linus Torvalds's avatar
Linus Torvalds committed
231
232
	if (vma->vm_ops && vma->vm_ops->close)
		vma->vm_ops->close(vma);
Matt Helsley's avatar
Matt Helsley committed
233
	if (vma->vm_file) {
234
		fput(vma->vm_file);
Matt Helsley's avatar
Matt Helsley committed
235
236
237
		if (vma->vm_flags & VM_EXECUTABLE)
			removed_exe_file_vma(vma->vm_mm);
	}
238
	mpol_put(vma_policy(vma));
Linus Torvalds's avatar
Linus Torvalds committed
239
	kmem_cache_free(vm_area_cachep, vma);
240
	return next;
Linus Torvalds's avatar
Linus Torvalds committed
241
242
}

243
SYSCALL_DEFINE1(brk, unsigned long, brk)
Linus Torvalds's avatar
Linus Torvalds committed
244
245
246
247
{
	unsigned long rlim, retval;
	unsigned long newbrk, oldbrk;
	struct mm_struct *mm = current->mm;
248
	unsigned long min_brk;
Linus Torvalds's avatar
Linus Torvalds committed
249
250
251

	down_write(&mm->mmap_sem);

252
#ifdef CONFIG_COMPAT_BRK
253
254
255
256
257
	/*
	 * CONFIG_COMPAT_BRK can still be overridden by setting
	 * randomize_va_space to 2, which will still cause mm->start_brk
	 * to be arbitrarily shifted
	 */
258
	if (current->brk_randomized)
259
260
261
		min_brk = mm->start_brk;
	else
		min_brk = mm->end_data;
262
263
264
265
#else
	min_brk = mm->start_brk;
#endif
	if (brk < min_brk)
Linus Torvalds's avatar
Linus Torvalds committed
266
		goto out;
Ram Gupta's avatar
Ram Gupta committed
267
268
269
270
271
272
273

	/*
	 * Check against rlimit here. If this check is done later after the test
	 * of oldbrk with newbrk then it can escape the test and let the data
	 * segment grow beyond its set limit the in case where the limit is
	 * not page aligned -Ram Gupta
	 */
Jiri Slaby's avatar
Jiri Slaby committed
274
	rlim = rlimit(RLIMIT_DATA);
Jiri Kosina's avatar
Jiri Kosina committed
275
276
	if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
			(mm->end_data - mm->start_data) > rlim)
Ram Gupta's avatar
Ram Gupta committed
277
278
		goto out;

Linus Torvalds's avatar
Linus Torvalds committed
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
	newbrk = PAGE_ALIGN(brk);
	oldbrk = PAGE_ALIGN(mm->brk);
	if (oldbrk == newbrk)
		goto set_brk;

	/* Always allow shrinking brk. */
	if (brk <= mm->brk) {
		if (!do_munmap(mm, newbrk, oldbrk-newbrk))
			goto set_brk;
		goto out;
	}

	/* Check against existing mmap mappings. */
	if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
		goto out;

	/* Ok, looks good - let it rip. */
	if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
		goto out;
set_brk:
	mm->brk = brk;
out:
	retval = mm->brk;
	up_write(&mm->mmap_sem);
	return retval;
}

#ifdef DEBUG_MM_RB
static int browse_rb(struct rb_root *root)
{
	int i = 0, j;
	struct rb_node *nd, *pn = NULL;
	unsigned long prev = 0, pend = 0;

	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
		struct vm_area_struct *vma;
		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
		if (vma->vm_start < prev)
			printk("vm_start %lx prev %lx\n", vma->vm_start, prev), i = -1;
		if (vma->vm_start < pend)
			printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
		if (vma->vm_start > vma->vm_end)
			printk("vm_end %lx < vm_start %lx\n", vma->vm_end, vma->vm_start);
		i++;
		pn = nd;
David Miller's avatar
David Miller committed
324
325
		prev = vma->vm_start;
		pend = vma->vm_end;
Linus Torvalds's avatar
Linus Torvalds committed
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
	}
	j = 0;
	for (nd = pn; nd; nd = rb_prev(nd)) {
		j++;
	}
	if (i != j)
		printk("backwards %d, forwards %d\n", j, i), i = 0;
	return i;
}

void validate_mm(struct mm_struct *mm)
{
	int bug = 0;
	int i = 0;
	struct vm_area_struct *tmp = mm->mmap;
	while (tmp) {
		tmp = tmp->vm_next;
		i++;
	}
	if (i != mm->map_count)
		printk("map_count %d vm_next %d\n", mm->map_count, i), bug = 1;
	i = browse_rb(&mm->mm_rb);
	if (i != mm->map_count)
		printk("map_count %d rb %d\n", mm->map_count, i), bug = 1;
350
	BUG_ON(bug);
Linus Torvalds's avatar
Linus Torvalds committed
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
}
#else
#define validate_mm(mm) do { } while (0)
#endif

static struct vm_area_struct *
find_vma_prepare(struct mm_struct *mm, unsigned long addr,
		struct vm_area_struct **pprev, struct rb_node ***rb_link,
		struct rb_node ** rb_parent)
{
	struct vm_area_struct * vma;
	struct rb_node ** __rb_link, * __rb_parent, * rb_prev;

	__rb_link = &mm->mm_rb.rb_node;
	rb_prev = __rb_parent = NULL;
	vma = NULL;

	while (*__rb_link) {
		struct vm_area_struct *vma_tmp;

		__rb_parent = *__rb_link;
		vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);

		if (vma_tmp->vm_end > addr) {
			vma = vma_tmp;
			if (vma_tmp->vm_start <= addr)
377
				break;
Linus Torvalds's avatar
Linus Torvalds committed
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
			__rb_link = &__rb_parent->rb_left;
		} else {
			rb_prev = __rb_parent;
			__rb_link = &__rb_parent->rb_right;
		}
	}

	*pprev = NULL;
	if (rb_prev)
		*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
	*rb_link = __rb_link;
	*rb_parent = __rb_parent;
	return vma;
}

void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
		struct rb_node **rb_link, struct rb_node *rb_parent)
{
	rb_link_node(&vma->vm_rb, rb_parent, rb_link);
	rb_insert_color(&vma->vm_rb, &mm->mm_rb);
}

400
static void __vma_link_file(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
401
{
ZhenwenXu's avatar
ZhenwenXu committed
402
	struct file *file;
Linus Torvalds's avatar
Linus Torvalds committed
403
404
405
406
407
408

	file = vma->vm_file;
	if (file) {
		struct address_space *mapping = file->f_mapping;

		if (vma->vm_flags & VM_DENYWRITE)
409
			atomic_dec(&file->f_path.dentry->d_inode->i_writecount);
Linus Torvalds's avatar
Linus Torvalds committed
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
		if (vma->vm_flags & VM_SHARED)
			mapping->i_mmap_writable++;

		flush_dcache_mmap_lock(mapping);
		if (unlikely(vma->vm_flags & VM_NONLINEAR))
			vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
		else
			vma_prio_tree_insert(vma, &mapping->i_mmap);
		flush_dcache_mmap_unlock(mapping);
	}
}

static void
__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
	struct vm_area_struct *prev, struct rb_node **rb_link,
	struct rb_node *rb_parent)
{
	__vma_link_list(mm, vma, prev, rb_parent);
	__vma_link_rb(mm, vma, rb_link, rb_parent);
}

static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
			struct vm_area_struct *prev, struct rb_node **rb_link,
			struct rb_node *rb_parent)
{
	struct address_space *mapping = NULL;

	if (vma->vm_file)
		mapping = vma->vm_file->f_mapping;

440
	if (mapping)
441
		mutex_lock(&mapping->i_mmap_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
442
443
444
445
446

	__vma_link(mm, vma, prev, rb_link, rb_parent);
	__vma_link_file(vma);

	if (mapping)
447
		mutex_unlock(&mapping->i_mmap_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
448
449
450
451
452
453
454
455
456
457

	mm->map_count++;
	validate_mm(mm);
}

/*
 * Helper for vma_adjust in the split_vma insert case:
 * insert vm structure into list and rbtree and anon_vma,
 * but it has already been inserted into prio_tree earlier.
 */
ZhenwenXu's avatar
ZhenwenXu committed
458
static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
459
{
ZhenwenXu's avatar
ZhenwenXu committed
460
461
	struct vm_area_struct *__vma, *prev;
	struct rb_node **rb_link, *rb_parent;
Linus Torvalds's avatar
Linus Torvalds committed
462
463

	__vma = find_vma_prepare(mm, vma->vm_start,&prev, &rb_link, &rb_parent);
464
	BUG_ON(__vma && __vma->vm_start < vma->vm_end);
Linus Torvalds's avatar
Linus Torvalds committed
465
466
467
468
469
470
471
472
	__vma_link(mm, vma, prev, rb_link, rb_parent);
	mm->map_count++;
}

static inline void
__vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
		struct vm_area_struct *prev)
{
473
474
475
476
477
	struct vm_area_struct *next = vma->vm_next;

	prev->vm_next = next;
	if (next)
		next->vm_prev = prev;
Linus Torvalds's avatar
Linus Torvalds committed
478
479
480
481
482
483
484
485
486
487
488
489
	rb_erase(&vma->vm_rb, &mm->mm_rb);
	if (mm->mmap_cache == vma)
		mm->mmap_cache = prev;
}

/*
 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
 * is already present in an i_mmap tree without adjusting the tree.
 * The following helper function should be used when such adjustments
 * are necessary.  The "insert" vma (if any) is to be inserted
 * before we drop the necessary locks.
 */
490
int vma_adjust(struct vm_area_struct *vma, unsigned long start,
Linus Torvalds's avatar
Linus Torvalds committed
491
492
493
494
495
496
497
	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
{
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *next = vma->vm_next;
	struct vm_area_struct *importer = NULL;
	struct address_space *mapping = NULL;
	struct prio_tree_root *root = NULL;
498
	struct anon_vma *anon_vma = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
499
500
501
502
503
	struct file *file = vma->vm_file;
	long adjust_next = 0;
	int remove_next = 0;

	if (next && !insert) {
504
505
		struct vm_area_struct *exporter = NULL;

Linus Torvalds's avatar
Linus Torvalds committed
506
507
508
509
510
511
512
		if (end >= next->vm_end) {
			/*
			 * vma expands, overlapping all the next, and
			 * perhaps the one after too (mprotect case 6).
			 */
again:			remove_next = 1 + (end > next->vm_end);
			end = next->vm_end;
513
			exporter = next;
Linus Torvalds's avatar
Linus Torvalds committed
514
515
516
517
518
519
520
			importer = vma;
		} else if (end > next->vm_start) {
			/*
			 * vma expands, overlapping part of the next:
			 * mprotect case 5 shifting the boundary up.
			 */
			adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
521
			exporter = next;
Linus Torvalds's avatar
Linus Torvalds committed
522
523
524
525
526
527
528
529
			importer = vma;
		} else if (end < vma->vm_end) {
			/*
			 * vma shrinks, and !insert tells it's not
			 * split_vma inserting another: so it must be
			 * mprotect case 4 shifting the boundary down.
			 */
			adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
530
			exporter = vma;
Linus Torvalds's avatar
Linus Torvalds committed
531
532
533
			importer = next;
		}

534
535
536
537
538
		/*
		 * Easily overlooked: when mprotect shifts the boundary,
		 * make sure the expanding vma has anon_vma set if the
		 * shrinking vma had, to cover any anon pages imported.
		 */
539
540
		if (exporter && exporter->anon_vma && !importer->anon_vma) {
			if (anon_vma_clone(importer, exporter))
541
				return -ENOMEM;
542
			importer->anon_vma = exporter->anon_vma;
543
544
545
		}
	}

Linus Torvalds's avatar
Linus Torvalds committed
546
547
548
549
	if (file) {
		mapping = file->f_mapping;
		if (!(vma->vm_flags & VM_NONLINEAR))
			root = &mapping->i_mmap;
550
		mutex_lock(&mapping->i_mmap_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
551
552
553
554
555
556
557
558
559
560
561
		if (insert) {
			/*
			 * Put into prio_tree now, so instantiated pages
			 * are visible to arm/parisc __flush_dcache_page
			 * throughout; but we cannot insert into address
			 * space until vma start or end is updated.
			 */
			__vma_link_file(insert);
		}
	}

562
563
	vma_adjust_trans_huge(vma, start, end, adjust_next);

564
565
566
567
568
569
	/*
	 * When changing only vma->vm_end, we don't really need anon_vma
	 * lock. This is a fairly rare case by itself, but the anon_vma
	 * lock may be shared between many sibling processes.  Skipping
	 * the lock for brk adjustments makes a difference sometimes.
	 */
570
	if (vma->anon_vma && (importer || start != vma->vm_start)) {
571
572
573
574
		anon_vma = vma->anon_vma;
		anon_vma_lock(anon_vma);
	}

Linus Torvalds's avatar
Linus Torvalds committed
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
	if (root) {
		flush_dcache_mmap_lock(mapping);
		vma_prio_tree_remove(vma, root);
		if (adjust_next)
			vma_prio_tree_remove(next, root);
	}

	vma->vm_start = start;
	vma->vm_end = end;
	vma->vm_pgoff = pgoff;
	if (adjust_next) {
		next->vm_start += adjust_next << PAGE_SHIFT;
		next->vm_pgoff += adjust_next;
	}

	if (root) {
		if (adjust_next)
			vma_prio_tree_insert(next, root);
		vma_prio_tree_insert(vma, root);
		flush_dcache_mmap_unlock(mapping);
	}

	if (remove_next) {
		/*
		 * vma_merge has merged next into vma, and needs
		 * us to remove next before dropping the locks.
		 */
		__vma_unlink(mm, next, vma);
		if (file)
			__remove_shared_vm_struct(next, file, mapping);
	} else if (insert) {
		/*
		 * split_vma has split insert from vma, and needs
		 * us to insert it before dropping the locks
		 * (it may either follow vma or precede it).
		 */
		__insert_vm_struct(mm, insert);
	}

614
615
	if (anon_vma)
		anon_vma_unlock(anon_vma);
Linus Torvalds's avatar
Linus Torvalds committed
616
	if (mapping)
617
		mutex_unlock(&mapping->i_mmap_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
618
619

	if (remove_next) {
Matt Helsley's avatar
Matt Helsley committed
620
		if (file) {
Linus Torvalds's avatar
Linus Torvalds committed
621
			fput(file);
Matt Helsley's avatar
Matt Helsley committed
622
623
624
			if (next->vm_flags & VM_EXECUTABLE)
				removed_exe_file_vma(mm);
		}
625
626
		if (next->anon_vma)
			anon_vma_merge(vma, next);
Linus Torvalds's avatar
Linus Torvalds committed
627
		mm->map_count--;
628
		mpol_put(vma_policy(next));
Linus Torvalds's avatar
Linus Torvalds committed
629
630
631
632
633
634
635
636
637
638
639
640
641
		kmem_cache_free(vm_area_cachep, next);
		/*
		 * In mprotect's case 6 (see comments on vma_merge),
		 * we must remove another next too. It would clutter
		 * up the code too much to do both in one go.
		 */
		if (remove_next == 2) {
			next = vma->vm_next;
			goto again;
		}
	}

	validate_mm(mm);
642
643

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
644
645
646
647
648
649
650
651
652
}

/*
 * If the vma has a ->close operation then the driver probably needs to release
 * per-vma resources, so we don't attempt to merge those.
 */
static inline int is_mergeable_vma(struct vm_area_struct *vma,
			struct file *file, unsigned long vm_flags)
{
Hugh Dickins's avatar
Hugh Dickins committed
653
654
	/* VM_CAN_NONLINEAR may get set later by f_op->mmap() */
	if ((vma->vm_flags ^ vm_flags) & ~VM_CAN_NONLINEAR)
Linus Torvalds's avatar
Linus Torvalds committed
655
656
657
658
659
660
661
662
663
		return 0;
	if (vma->vm_file != file)
		return 0;
	if (vma->vm_ops && vma->vm_ops->close)
		return 0;
	return 1;
}

static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
Shaohua Li's avatar
Shaohua Li committed
664
665
					struct anon_vma *anon_vma2,
					struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
666
{
Shaohua Li's avatar
Shaohua Li committed
667
668
669
670
671
672
673
674
	/*
	 * The list_is_singular() test is to avoid merging VMA cloned from
	 * parents. This can improve scalability caused by anon_vma lock.
	 */
	if ((!anon_vma1 || !anon_vma2) && (!vma ||
		list_is_singular(&vma->anon_vma_chain)))
		return 1;
	return anon_vma1 == anon_vma2;
Linus Torvalds's avatar
Linus Torvalds committed
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
}

/*
 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
 * in front of (at a lower virtual address and file offset than) the vma.
 *
 * We cannot merge two vmas if they have differently assigned (non-NULL)
 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
 *
 * We don't check here for the merged mmap wrapping around the end of pagecache
 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
 * wrap, nor mmaps which cover the final page at index -1UL.
 */
static int
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
	struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
{
	if (is_mergeable_vma(vma, file, vm_flags) &&
Shaohua Li's avatar
Shaohua Li committed
693
	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
Linus Torvalds's avatar
Linus Torvalds committed
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
		if (vma->vm_pgoff == vm_pgoff)
			return 1;
	}
	return 0;
}

/*
 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
 * beyond (at a higher virtual address and file offset than) the vma.
 *
 * We cannot merge two vmas if they have differently assigned (non-NULL)
 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
 */
static int
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
	struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
{
	if (is_mergeable_vma(vma, file, vm_flags) &&
Shaohua Li's avatar
Shaohua Li committed
712
	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
Linus Torvalds's avatar
Linus Torvalds committed
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
		pgoff_t vm_pglen;
		vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
			return 1;
	}
	return 0;
}

/*
 * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
 * whether that can be merged with its predecessor or its successor.
 * Or both (it neatly fills a hole).
 *
 * In most cases - when called for mmap, brk or mremap - [addr,end) is
 * certain not to be mapped by the time vma_merge is called; but when
 * called for mprotect, it is certain to be already mapped (either at
 * an offset within prev, or at the start of next), and the flags of
 * this area are about to be changed to vm_flags - and the no-change
 * case has already been eliminated.
 *
 * The following mprotect cases have to be considered, where AAAA is
 * the area passed down from mprotect_fixup, never extending beyond one
 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
 *
 *     AAAA             AAAA                AAAA          AAAA
 *    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPNNNNXXXX
 *    cannot merge    might become    might become    might become
 *                    PPNNNNNNNNNN    PPPPPPPPPPNN    PPPPPPPPPPPP 6 or
 *    mmap, brk or    case 4 below    case 5 below    PPPPPPPPXXXX 7 or
 *    mremap move:                                    PPPPNNNNNNNN 8
 *        AAAA
 *    PPPP    NNNN    PPPPPPPPPPPP    PPPPPPPPNNNN    PPPPNNNNNNNN
 *    might become    case 1 below    case 2 below    case 3 below
 *
 * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
 * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
 */
struct vm_area_struct *vma_merge(struct mm_struct *mm,
			struct vm_area_struct *prev, unsigned long addr,
			unsigned long end, unsigned long vm_flags,
		     	struct anon_vma *anon_vma, struct file *file,
			pgoff_t pgoff, struct mempolicy *policy)
{
	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
	struct vm_area_struct *area, *next;
758
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789

	/*
	 * We later require that vma->vm_flags == vm_flags,
	 * so this tests vma->vm_flags & VM_SPECIAL, too.
	 */
	if (vm_flags & VM_SPECIAL)
		return NULL;

	if (prev)
		next = prev->vm_next;
	else
		next = mm->mmap;
	area = next;
	if (next && next->vm_end == end)		/* cases 6, 7, 8 */
		next = next->vm_next;

	/*
	 * Can it merge with the predecessor?
	 */
	if (prev && prev->vm_end == addr &&
  			mpol_equal(vma_policy(prev), policy) &&
			can_vma_merge_after(prev, vm_flags,
						anon_vma, file, pgoff)) {
		/*
		 * OK, it can.  Can we now merge in the successor as well?
		 */
		if (next && end == next->vm_start &&
				mpol_equal(policy, vma_policy(next)) &&
				can_vma_merge_before(next, vm_flags,
					anon_vma, file, pgoff+pglen) &&
				is_mergeable_anon_vma(prev->anon_vma,
Shaohua Li's avatar
Shaohua Li committed
790
						      next->anon_vma, NULL)) {
Linus Torvalds's avatar
Linus Torvalds committed
791
							/* cases 1, 6 */
792
			err = vma_adjust(prev, prev->vm_start,
Linus Torvalds's avatar
Linus Torvalds committed
793
794
				next->vm_end, prev->vm_pgoff, NULL);
		} else					/* cases 2, 5, 7 */
795
			err = vma_adjust(prev, prev->vm_start,
Linus Torvalds's avatar
Linus Torvalds committed
796
				end, prev->vm_pgoff, NULL);
797
798
		if (err)
			return NULL;
Andrea Arcangeli's avatar
Andrea Arcangeli committed
799
		khugepaged_enter_vma_merge(prev);
Linus Torvalds's avatar
Linus Torvalds committed
800
801
802
803
804
805
806
807
808
809
810
		return prev;
	}

	/*
	 * Can this new request be merged in front of next?
	 */
	if (next && end == next->vm_start &&
 			mpol_equal(policy, vma_policy(next)) &&
			can_vma_merge_before(next, vm_flags,
					anon_vma, file, pgoff+pglen)) {
		if (prev && addr < prev->vm_end)	/* case 4 */
811
			err = vma_adjust(prev, prev->vm_start,
Linus Torvalds's avatar
Linus Torvalds committed
812
813
				addr, prev->vm_pgoff, NULL);
		else					/* cases 3, 8 */
814
			err = vma_adjust(area, addr, next->vm_end,
Linus Torvalds's avatar
Linus Torvalds committed
815
				next->vm_pgoff - pglen, NULL);
816
817
		if (err)
			return NULL;
Andrea Arcangeli's avatar
Andrea Arcangeli committed
818
		khugepaged_enter_vma_merge(area);
Linus Torvalds's avatar
Linus Torvalds committed
819
820
821
822
823
824
		return area;
	}

	return NULL;
}

825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
/*
 * Rough compatbility check to quickly see if it's even worth looking
 * at sharing an anon_vma.
 *
 * They need to have the same vm_file, and the flags can only differ
 * in things that mprotect may change.
 *
 * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
 * we can merge the two vma's. For example, we refuse to merge a vma if
 * there is a vm_ops->close() function, because that indicates that the
 * driver is doing some kind of reference counting. But that doesn't
 * really matter for the anon_vma sharing case.
 */
static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
{
	return a->vm_end == b->vm_start &&
		mpol_equal(vma_policy(a), vma_policy(b)) &&
		a->vm_file == b->vm_file &&
		!((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC)) &&
		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
}

/*
 * Do some basic sanity checking to see if we can re-use the anon_vma
 * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
 * the same as 'old', the other will be the new one that is trying
 * to share the anon_vma.
 *
 * NOTE! This runs with mm_sem held for reading, so it is possible that
 * the anon_vma of 'old' is concurrently in the process of being set up
 * by another page fault trying to merge _that_. But that's ok: if it
 * is being set up, that automatically means that it will be a singleton
 * acceptable for merging, so we can do all of this optimistically. But
 * we do that ACCESS_ONCE() to make sure that we never re-load the pointer.
 *
 * IOW: that the "list_is_singular()" test on the anon_vma_chain only
 * matters for the 'stable anon_vma' case (ie the thing we want to avoid
 * is to return an anon_vma that is "complex" due to having gone through
 * a fork).
 *
 * We also make sure that the two vma's are compatible (adjacent,
 * and with the same memory policies). That's all stable, even with just
 * a read lock on the mm_sem.
 */
static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
{
	if (anon_vma_compatible(a, b)) {
		struct anon_vma *anon_vma = ACCESS_ONCE(old->anon_vma);

		if (anon_vma && list_is_singular(&old->anon_vma_chain))
			return anon_vma;
	}
	return NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
880
881
882
883
884
885
886
887
888
889
/*
 * find_mergeable_anon_vma is used by anon_vma_prepare, to check
 * neighbouring vmas for a suitable anon_vma, before it goes off
 * to allocate a new anon_vma.  It checks because a repetitive
 * sequence of mprotects and faults may otherwise lead to distinct
 * anon_vmas being allocated, preventing vma merge in subsequent
 * mprotect.
 */
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
{
890
	struct anon_vma *anon_vma;
Linus Torvalds's avatar
Linus Torvalds committed
891
892
893
894
895
896
	struct vm_area_struct *near;

	near = vma->vm_next;
	if (!near)
		goto try_prev;

897
898
899
	anon_vma = reusable_anon_vma(near, vma, near);
	if (anon_vma)
		return anon_vma;
Linus Torvalds's avatar
Linus Torvalds committed
900
try_prev:
901
	near = vma->vm_prev;
Linus Torvalds's avatar
Linus Torvalds committed
902
903
904
	if (!near)
		goto none;

905
906
907
	anon_vma = reusable_anon_vma(near, near, vma);
	if (anon_vma)
		return anon_vma;
Linus Torvalds's avatar
Linus Torvalds committed
908
909
910
911
912
913
914
915
916
917
918
919
920
none:
	/*
	 * There's no absolute need to look only at touching neighbours:
	 * we could search further afield for "compatible" anon_vmas.
	 * But it would probably just be a waste of time searching,
	 * or lead to too many vmas hanging off the same anon_vma.
	 * We're trying to allow mprotect remerging later on,
	 * not trying to minimize memory used for anon_vmas.
	 */
	return NULL;
}

#ifdef CONFIG_PROC_FS
921
void vm_stat_account(struct mm_struct *mm, unsigned long flags,
Linus Torvalds's avatar
Linus Torvalds committed
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
						struct file *file, long pages)
{
	const unsigned long stack_flags
		= VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);

	if (file) {
		mm->shared_vm += pages;
		if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
			mm->exec_vm += pages;
	} else if (flags & stack_flags)
		mm->stack_vm += pages;
	if (flags & (VM_RESERVED|VM_IO))
		mm->reserved_vm += pages;
}
#endif /* CONFIG_PROC_FS */

/*
939
 * The caller must hold down_write(&current->mm->mmap_sem).
Linus Torvalds's avatar
Linus Torvalds committed
940
941
 */

ZhenwenXu's avatar
ZhenwenXu committed
942
unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
Linus Torvalds's avatar
Linus Torvalds committed
943
944
945
946
947
			unsigned long len, unsigned long prot,
			unsigned long flags, unsigned long pgoff)
{
	struct mm_struct * mm = current->mm;
	struct inode *inode;
948
	vm_flags_t vm_flags;
Linus Torvalds's avatar
Linus Torvalds committed
949
	int error;
Miklos Szeredi's avatar
Miklos Szeredi committed
950
	unsigned long reqprot = prot;
Linus Torvalds's avatar
Linus Torvalds committed
951
952
953
954
955
956
957
958

	/*
	 * Does the application expect PROT_READ to imply PROT_EXEC?
	 *
	 * (the exception is when the underlying filesystem is noexec
	 *  mounted, in which case we dont add PROT_EXEC.)
	 */
	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
959
		if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
Linus Torvalds's avatar
Linus Torvalds committed
960
961
962
963
964
			prot |= PROT_EXEC;

	if (!len)
		return -EINVAL;

965
966
967
	if (!(flags & MAP_FIXED))
		addr = round_hint_to_min(addr);

Linus Torvalds's avatar
Linus Torvalds committed
968
969
	/* Careful about overflows.. */
	len = PAGE_ALIGN(len);
970
	if (!len)
Linus Torvalds's avatar
Linus Torvalds committed
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
		return -ENOMEM;

	/* offset overflow? */
	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
               return -EOVERFLOW;

	/* Too many mappings? */
	if (mm->map_count > sysctl_max_map_count)
		return -ENOMEM;

	/* Obtain the address to map to. we verify (or select) it and ensure
	 * that it represents a valid section of the address space.
	 */
	addr = get_unmapped_area(file, addr, len, pgoff, flags);
	if (addr & ~PAGE_MASK)
		return addr;

	/* Do simple checking here so the lower-level routines won't have
	 * to. we assume access permissions have been handled by the open
	 * of the memory object, so we don't do any here.
	 */
	vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;

Huang Shijie's avatar
Huang Shijie committed
995
	if (flags & MAP_LOCKED)
Linus Torvalds's avatar
Linus Torvalds committed
996
997
		if (!can_do_mlock())
			return -EPERM;
998

Linus Torvalds's avatar
Linus Torvalds committed
999
1000
	/* mlock MCL_FUTURE? */
	if (vm_flags & VM_LOCKED) {