mmap.c 89.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
/*
 * mm/mmap.c
 *
 * Written by obz.
 *
Alan Cox's avatar
Alan Cox committed
6
 * Address space accounting code	<alan@lxorguk.ukuu.org.uk>
Linus Torvalds's avatar
Linus Torvalds committed
7
8
 */

9
10
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

11
#include <linux/kernel.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
#include <linux/slab.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
13
#include <linux/backing-dev.h>
Linus Torvalds's avatar
Linus Torvalds committed
14
#include <linux/mm.h>
Davidlohr Bueso's avatar
Davidlohr Bueso committed
15
#include <linux/vmacache.h>
Linus Torvalds's avatar
Linus Torvalds committed
16
17
18
19
20
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/syscalls.h>
21
#include <linux/capability.h>
Linus Torvalds's avatar
Linus Torvalds committed
22
23
24
25
26
27
28
#include <linux/init.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/personality.h>
#include <linux/security.h>
#include <linux/hugetlb.h>
#include <linux/profile.h>
29
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
31
32
#include <linux/mount.h>
#include <linux/mempolicy.h>
#include <linux/rmap.h>
Andrea Arcangeli's avatar
Andrea Arcangeli committed
33
#include <linux/mmu_notifier.h>
34
#include <linux/mmdebug.h>
35
#include <linux/perf_event.h>
Al Viro's avatar
Al Viro committed
36
#include <linux/audit.h>
Andrea Arcangeli's avatar
Andrea Arcangeli committed
37
#include <linux/khugepaged.h>
38
#include <linux/uprobes.h>
39
#include <linux/rbtree_augmented.h>
40
#include <linux/sched/sysctl.h>
41
42
#include <linux/notifier.h>
#include <linux/memory.h>
43
#include <linux/printk.h>
Linus Torvalds's avatar
Linus Torvalds committed
44
45
46
47

#include <asm/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/tlb.h>
48
#include <asm/mmu_context.h>
Linus Torvalds's avatar
Linus Torvalds committed
49

50
51
#include "internal.h"

52
53
54
55
#ifndef arch_mmap_check
#define arch_mmap_check(addr, len, flags)	(0)
#endif

56
57
58
59
#ifndef arch_rebalance_pgtables
#define arch_rebalance_pgtables(addr, len)		(addr)
#endif

60
61
62
63
static void unmap_region(struct mm_struct *mm,
		struct vm_area_struct *vma, struct vm_area_struct *prev,
		unsigned long start, unsigned long end);

Linus Torvalds's avatar
Linus Torvalds committed
64
65
66
67
68
69
70
71
72
/* description of effects of mapping type and prot in current implementation.
 * this is due to the limited x86 page protection hardware.  The expected
 * behavior is in parens:
 *
 * map_type	prot
 *		PROT_NONE	PROT_READ	PROT_WRITE	PROT_EXEC
 * MAP_SHARED	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (yes) yes	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
vishnu.ps's avatar
vishnu.ps committed
73
 *
Linus Torvalds's avatar
Linus Torvalds committed
74
75
76
77
78
79
80
81
82
83
 * MAP_PRIVATE	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (copy) copy	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
 *
 */
pgprot_t protection_map[16] = {
	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
};

84
85
pgprot_t vm_get_page_prot(unsigned long vm_flags)
{
86
87
88
	return __pgprot(pgprot_val(protection_map[vm_flags &
				(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
			pgprot_val(arch_vm_get_page_prot(vm_flags)));
89
90
91
}
EXPORT_SYMBOL(vm_get_page_prot);

92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
{
	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
}

/* Update vma->vm_page_prot to reflect vma->vm_flags. */
void vma_set_page_prot(struct vm_area_struct *vma)
{
	unsigned long vm_flags = vma->vm_flags;

	vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
	if (vma_wants_writenotify(vma)) {
		vm_flags &= ~VM_SHARED;
		vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot,
						     vm_flags);
	}
}


111
112
int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;  /* heuristic overcommit */
int sysctl_overcommit_ratio __read_mostly = 50;	/* default is 50% */
113
unsigned long sysctl_overcommit_kbytes __read_mostly;
114
int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
115
unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
116
unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
117
118
119
120
121
/*
 * Make sure vm_committed_as in one cacheline and not cacheline shared with
 * other variables. It can be updated by several CPUs frequently.
 */
struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
Linus Torvalds's avatar
Linus Torvalds committed
122

123
124
125
126
127
128
129
130
131
132
133
134
135
136
/*
 * The global memory commitment made in the system can be a metric
 * that can be used to drive ballooning decisions when Linux is hosted
 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
 * balancing memory across competing virtual machines that are hosted.
 * Several metrics drive this policy engine including the guest reported
 * memory commitment.
 */
unsigned long vm_memory_committed(void)
{
	return percpu_counter_read_positive(&vm_committed_as);
}
EXPORT_SYMBOL_GPL(vm_memory_committed);

Linus Torvalds's avatar
Linus Torvalds committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
/*
 * Check that a process has enough memory to allocate a new virtual
 * mapping. 0 means there is enough memory for the allocation to
 * succeed and -ENOMEM implies there is not.
 *
 * We currently support three overcommit policies, which are set via the
 * vm.overcommit_memory sysctl.  See Documentation/vm/overcommit-accounting
 *
 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
 * Additional code 2002 Jul 20 by Robert Love.
 *
 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
 *
 * Note this is a helper function intended to be used by LSMs which
 * wish to use this logic.
 */
153
int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
Linus Torvalds's avatar
Linus Torvalds committed
154
{
155
	long free, allowed, reserve;
Linus Torvalds's avatar
Linus Torvalds committed
156

157
158
159
160
	VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) <
			-(s64)vm_committed_as_batch * num_online_cpus(),
			"memory commitment underflow");

Linus Torvalds's avatar
Linus Torvalds committed
161
162
163
164
165
166
167
168
169
	vm_acct_memory(pages);

	/*
	 * Sometimes we want to use more memory than we have
	 */
	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
		return 0;

	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
170
171
172
173
174
175
176
177
178
179
		free = global_page_state(NR_FREE_PAGES);
		free += global_page_state(NR_FILE_PAGES);

		/*
		 * shmem pages shouldn't be counted as free in this
		 * case, they can't be purged, only swapped out, and
		 * that won't affect the overall amount of available
		 * memory in the system.
		 */
		free -= global_page_state(NR_SHMEM);
Linus Torvalds's avatar
Linus Torvalds committed
180

181
		free += get_nr_swap_pages();
Linus Torvalds's avatar
Linus Torvalds committed
182
183
184
185
186
187
188

		/*
		 * Any slabs which are created with the
		 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
		 * which are reclaimable, under pressure.  The dentry
		 * cache and most inode caches should fall into this
		 */
189
		free += global_page_state(NR_SLAB_RECLAIMABLE);
Linus Torvalds's avatar
Linus Torvalds committed
190

191
192
193
		/*
		 * Leave reserved pages. The pages are not for anonymous pages.
		 */
194
		if (free <= totalreserve_pages)
195
196
			goto error;
		else
197
			free -= totalreserve_pages;
198
199

		/*
200
		 * Reserve some for root
201
		 */
Linus Torvalds's avatar
Linus Torvalds committed
202
		if (!cap_sys_admin)
203
			free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
Linus Torvalds's avatar
Linus Torvalds committed
204
205
206

		if (free > pages)
			return 0;
207
208

		goto error;
Linus Torvalds's avatar
Linus Torvalds committed
209
210
	}

211
	allowed = vm_commit_limit();
Linus Torvalds's avatar
Linus Torvalds committed
212
	/*
213
	 * Reserve some for root
Linus Torvalds's avatar
Linus Torvalds committed
214
215
	 */
	if (!cap_sys_admin)
216
		allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
Linus Torvalds's avatar
Linus Torvalds committed
217

218
219
220
221
222
	/*
	 * Don't let a single process grow so big a user can't recover
	 */
	if (mm) {
		reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
223
		allowed -= min_t(long, mm->total_vm / 32, reserve);
224
	}
Linus Torvalds's avatar
Linus Torvalds committed
225

226
	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
Linus Torvalds's avatar
Linus Torvalds committed
227
		return 0;
228
error:
Linus Torvalds's avatar
Linus Torvalds committed
229
230
231
232
233
234
	vm_unacct_memory(pages);

	return -ENOMEM;
}

/*
235
 * Requires inode->i_mapping->i_mmap_rwsem
Linus Torvalds's avatar
Linus Torvalds committed
236
237
238
239
240
 */
static void __remove_shared_vm_struct(struct vm_area_struct *vma,
		struct file *file, struct address_space *mapping)
{
	if (vma->vm_flags & VM_DENYWRITE)
Al Viro's avatar
Al Viro committed
241
		atomic_inc(&file_inode(file)->i_writecount);
Linus Torvalds's avatar
Linus Torvalds committed
242
	if (vma->vm_flags & VM_SHARED)
243
		mapping_unmap_writable(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
244
245

	flush_dcache_mmap_lock(mapping);
246
	vma_interval_tree_remove(vma, &mapping->i_mmap);
Linus Torvalds's avatar
Linus Torvalds committed
247
248
249
250
	flush_dcache_mmap_unlock(mapping);
}

/*
251
 * Unlink a file-based vm structure from its interval tree, to hide
252
 * vma from rmap and vmtruncate before freeing its page tables.
Linus Torvalds's avatar
Linus Torvalds committed
253
 */
254
void unlink_file_vma(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
255
256
257
258
259
{
	struct file *file = vma->vm_file;

	if (file) {
		struct address_space *mapping = file->f_mapping;
260
		i_mmap_lock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
261
		__remove_shared_vm_struct(vma, file, mapping);
262
		i_mmap_unlock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
263
	}
264
265
266
267
268
269
270
271
272
273
}

/*
 * Close a vm structure and free it, returning the next.
 */
static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
{
	struct vm_area_struct *next = vma->vm_next;

	might_sleep();
Linus Torvalds's avatar
Linus Torvalds committed
274
275
	if (vma->vm_ops && vma->vm_ops->close)
		vma->vm_ops->close(vma);
276
	if (vma->vm_file)
277
		fput(vma->vm_file);
278
	mpol_put(vma_policy(vma));
Linus Torvalds's avatar
Linus Torvalds committed
279
	kmem_cache_free(vm_area_cachep, vma);
280
	return next;
Linus Torvalds's avatar
Linus Torvalds committed
281
282
}

283
284
static unsigned long do_brk(unsigned long addr, unsigned long len);

285
SYSCALL_DEFINE1(brk, unsigned long, brk)
Linus Torvalds's avatar
Linus Torvalds committed
286
{
287
	unsigned long retval;
Linus Torvalds's avatar
Linus Torvalds committed
288
289
	unsigned long newbrk, oldbrk;
	struct mm_struct *mm = current->mm;
290
	unsigned long min_brk;
291
	bool populate;
Linus Torvalds's avatar
Linus Torvalds committed
292
293
294

	down_write(&mm->mmap_sem);

295
#ifdef CONFIG_COMPAT_BRK
296
297
298
299
300
	/*
	 * CONFIG_COMPAT_BRK can still be overridden by setting
	 * randomize_va_space to 2, which will still cause mm->start_brk
	 * to be arbitrarily shifted
	 */
301
	if (current->brk_randomized)
302
303
304
		min_brk = mm->start_brk;
	else
		min_brk = mm->end_data;
305
306
307
308
#else
	min_brk = mm->start_brk;
#endif
	if (brk < min_brk)
Linus Torvalds's avatar
Linus Torvalds committed
309
		goto out;
Ram Gupta's avatar
Ram Gupta committed
310
311
312
313
314
315
316

	/*
	 * Check against rlimit here. If this check is done later after the test
	 * of oldbrk with newbrk then it can escape the test and let the data
	 * segment grow beyond its set limit the in case where the limit is
	 * not page aligned -Ram Gupta
	 */
317
318
	if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
			      mm->end_data, mm->start_data))
Ram Gupta's avatar
Ram Gupta committed
319
320
		goto out;

Linus Torvalds's avatar
Linus Torvalds committed
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
	newbrk = PAGE_ALIGN(brk);
	oldbrk = PAGE_ALIGN(mm->brk);
	if (oldbrk == newbrk)
		goto set_brk;

	/* Always allow shrinking brk. */
	if (brk <= mm->brk) {
		if (!do_munmap(mm, newbrk, oldbrk-newbrk))
			goto set_brk;
		goto out;
	}

	/* Check against existing mmap mappings. */
	if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
		goto out;

	/* Ok, looks good - let it rip. */
	if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
		goto out;
340

Linus Torvalds's avatar
Linus Torvalds committed
341
342
set_brk:
	mm->brk = brk;
343
344
345
346
347
348
	populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
	up_write(&mm->mmap_sem);
	if (populate)
		mm_populate(oldbrk, newbrk - oldbrk);
	return brk;

Linus Torvalds's avatar
Linus Torvalds committed
349
350
351
352
353
354
out:
	retval = mm->brk;
	up_write(&mm->mmap_sem);
	return retval;
}

355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
static long vma_compute_subtree_gap(struct vm_area_struct *vma)
{
	unsigned long max, subtree_gap;
	max = vma->vm_start;
	if (vma->vm_prev)
		max -= vma->vm_prev->vm_end;
	if (vma->vm_rb.rb_left) {
		subtree_gap = rb_entry(vma->vm_rb.rb_left,
				struct vm_area_struct, vm_rb)->rb_subtree_gap;
		if (subtree_gap > max)
			max = subtree_gap;
	}
	if (vma->vm_rb.rb_right) {
		subtree_gap = rb_entry(vma->vm_rb.rb_right,
				struct vm_area_struct, vm_rb)->rb_subtree_gap;
		if (subtree_gap > max)
			max = subtree_gap;
	}
	return max;
}

376
#ifdef CONFIG_DEBUG_VM_RB
Linus Torvalds's avatar
Linus Torvalds committed
377
378
static int browse_rb(struct rb_root *root)
{
379
	int i = 0, j, bug = 0;
Linus Torvalds's avatar
Linus Torvalds committed
380
381
382
383
384
385
	struct rb_node *nd, *pn = NULL;
	unsigned long prev = 0, pend = 0;

	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
		struct vm_area_struct *vma;
		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
386
		if (vma->vm_start < prev) {
387
388
			pr_emerg("vm_start %lx < prev %lx\n",
				  vma->vm_start, prev);
389
390
391
			bug = 1;
		}
		if (vma->vm_start < pend) {
392
393
			pr_emerg("vm_start %lx < pend %lx\n",
				  vma->vm_start, pend);
394
395
396
			bug = 1;
		}
		if (vma->vm_start > vma->vm_end) {
397
398
			pr_emerg("vm_start %lx > vm_end %lx\n",
				  vma->vm_start, vma->vm_end);
399
400
401
			bug = 1;
		}
		if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
402
			pr_emerg("free gap %lx, correct %lx\n",
403
404
405
406
			       vma->rb_subtree_gap,
			       vma_compute_subtree_gap(vma));
			bug = 1;
		}
Linus Torvalds's avatar
Linus Torvalds committed
407
408
		i++;
		pn = nd;
David Miller's avatar
David Miller committed
409
410
		prev = vma->vm_start;
		pend = vma->vm_end;
Linus Torvalds's avatar
Linus Torvalds committed
411
412
	}
	j = 0;
413
	for (nd = pn; nd; nd = rb_prev(nd))
Linus Torvalds's avatar
Linus Torvalds committed
414
		j++;
415
	if (i != j) {
416
		pr_emerg("backwards %d, forwards %d\n", j, i);
417
		bug = 1;
Linus Torvalds's avatar
Linus Torvalds committed
418
	}
419
	return bug ? -1 : i;
Linus Torvalds's avatar
Linus Torvalds committed
420
421
}

422
423
424
425
426
427
428
static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
{
	struct rb_node *nd;

	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
		struct vm_area_struct *vma;
		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
429
430
431
		VM_BUG_ON_VMA(vma != ignore &&
			vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
			vma);
Linus Torvalds's avatar
Linus Torvalds committed
432
433
434
	}
}

435
static void validate_mm(struct mm_struct *mm)
Linus Torvalds's avatar
Linus Torvalds committed
436
437
438
{
	int bug = 0;
	int i = 0;
439
	unsigned long highest_address = 0;
440
	struct vm_area_struct *vma = mm->mmap;
441

442
443
	while (vma) {
		struct anon_vma_chain *avc;
444

445
		vma_lock_anon_vma(vma);
446
447
		list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
			anon_vma_interval_tree_verify(avc);
448
		vma_unlock_anon_vma(vma);
449
		highest_address = vma->vm_end;
450
		vma = vma->vm_next;
Linus Torvalds's avatar
Linus Torvalds committed
451
452
		i++;
	}
453
	if (i != mm->map_count) {
454
		pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
455
456
457
		bug = 1;
	}
	if (highest_address != mm->highest_vm_end) {
458
		pr_emerg("mm->highest_vm_end %lx, found %lx\n",
459
			  mm->highest_vm_end, highest_address);
460
461
		bug = 1;
	}
Linus Torvalds's avatar
Linus Torvalds committed
462
	i = browse_rb(&mm->mm_rb);
463
	if (i != mm->map_count) {
464
465
		if (i != -1)
			pr_emerg("map_count %d rb %d\n", mm->map_count, i);
466
467
		bug = 1;
	}
468
	VM_BUG_ON_MM(bug, mm);
Linus Torvalds's avatar
Linus Torvalds committed
469
470
}
#else
471
#define validate_mm_rb(root, ignore) do { } while (0)
Linus Torvalds's avatar
Linus Torvalds committed
472
473
474
#define validate_mm(mm) do { } while (0)
#endif

475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
		     unsigned long, rb_subtree_gap, vma_compute_subtree_gap)

/*
 * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
 * vma->vm_prev->vm_end values changed, without modifying the vma's position
 * in the rbtree.
 */
static void vma_gap_update(struct vm_area_struct *vma)
{
	/*
	 * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
	 * function that does exacltly what we want.
	 */
	vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
}

static inline void vma_rb_insert(struct vm_area_struct *vma,
				 struct rb_root *root)
{
	/* All rb_subtree_gap values must be consistent prior to insertion */
	validate_mm_rb(root, NULL);

	rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
}

static void vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
{
	/*
	 * All rb_subtree_gap values must be consistent prior to erase,
	 * with the possible exception of the vma being erased.
	 */
	validate_mm_rb(root, vma);

	/*
	 * Note rb_erase_augmented is a fairly large inline function,
	 * so make sure we instantiate it only once with our desired
	 * augmented rbtree callbacks.
	 */
	rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
}

517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
/*
 * vma has some anon_vma assigned, and is already inserted on that
 * anon_vma's interval trees.
 *
 * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
 * vma must be removed from the anon_vma's interval trees using
 * anon_vma_interval_tree_pre_update_vma().
 *
 * After the update, the vma will be reinserted using
 * anon_vma_interval_tree_post_update_vma().
 *
 * The entire update must be protected by exclusive mmap_sem and by
 * the root anon_vma's mutex.
 */
static inline void
anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
{
	struct anon_vma_chain *avc;

	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
}

static inline void
anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
{
	struct anon_vma_chain *avc;

	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
}

549
550
551
static int find_vma_links(struct mm_struct *mm, unsigned long addr,
		unsigned long end, struct vm_area_struct **pprev,
		struct rb_node ***rb_link, struct rb_node **rb_parent)
Linus Torvalds's avatar
Linus Torvalds committed
552
{
553
	struct rb_node **__rb_link, *__rb_parent, *rb_prev;
Linus Torvalds's avatar
Linus Torvalds committed
554
555
556
557
558
559
560
561
562
563
564

	__rb_link = &mm->mm_rb.rb_node;
	rb_prev = __rb_parent = NULL;

	while (*__rb_link) {
		struct vm_area_struct *vma_tmp;

		__rb_parent = *__rb_link;
		vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);

		if (vma_tmp->vm_end > addr) {
565
566
567
			/* Fail if an existing vma overlaps the area */
			if (vma_tmp->vm_start < end)
				return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
568
569
570
571
572
573
574
575
576
577
578
579
			__rb_link = &__rb_parent->rb_left;
		} else {
			rb_prev = __rb_parent;
			__rb_link = &__rb_parent->rb_right;
		}
	}

	*pprev = NULL;
	if (rb_prev)
		*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
	*rb_link = __rb_link;
	*rb_parent = __rb_parent;
580
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
581
582
}

583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
static unsigned long count_vma_pages_range(struct mm_struct *mm,
		unsigned long addr, unsigned long end)
{
	unsigned long nr_pages = 0;
	struct vm_area_struct *vma;

	/* Find first overlaping mapping */
	vma = find_vma_intersection(mm, addr, end);
	if (!vma)
		return 0;

	nr_pages = (min(end, vma->vm_end) -
		max(addr, vma->vm_start)) >> PAGE_SHIFT;

	/* Iterate over the rest of the overlaps */
	for (vma = vma->vm_next; vma; vma = vma->vm_next) {
		unsigned long overlap_len;

		if (vma->vm_start > end)
			break;

		overlap_len = min(end, vma->vm_end) - vma->vm_start;
		nr_pages += overlap_len >> PAGE_SHIFT;
	}

	return nr_pages;
}

Linus Torvalds's avatar
Linus Torvalds committed
611
612
613
void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
		struct rb_node **rb_link, struct rb_node *rb_parent)
{
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
	/* Update tracking information for the gap following the new vma. */
	if (vma->vm_next)
		vma_gap_update(vma->vm_next);
	else
		mm->highest_vm_end = vma->vm_end;

	/*
	 * vma->vm_prev wasn't known when we followed the rbtree to find the
	 * correct insertion point for that vma. As a result, we could not
	 * update the vma vm_rb parents rb_subtree_gap values on the way down.
	 * So, we first insert the vma with a zero rb_subtree_gap value
	 * (to be consistent with what we did on the way down), and then
	 * immediately update the gap to the correct value. Finally we
	 * rebalance the rbtree after all augmented values have been set.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
629
	rb_link_node(&vma->vm_rb, rb_parent, rb_link);
630
631
632
	vma->rb_subtree_gap = 0;
	vma_gap_update(vma);
	vma_rb_insert(vma, &mm->mm_rb);
Linus Torvalds's avatar
Linus Torvalds committed
633
634
}

635
static void __vma_link_file(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
636
{
ZhenwenXu's avatar
ZhenwenXu committed
637
	struct file *file;
Linus Torvalds's avatar
Linus Torvalds committed
638
639
640
641
642
643

	file = vma->vm_file;
	if (file) {
		struct address_space *mapping = file->f_mapping;

		if (vma->vm_flags & VM_DENYWRITE)
Al Viro's avatar
Al Viro committed
644
			atomic_dec(&file_inode(file)->i_writecount);
Linus Torvalds's avatar
Linus Torvalds committed
645
		if (vma->vm_flags & VM_SHARED)
646
			atomic_inc(&mapping->i_mmap_writable);
Linus Torvalds's avatar
Linus Torvalds committed
647
648

		flush_dcache_mmap_lock(mapping);
649
		vma_interval_tree_insert(vma, &mapping->i_mmap);
Linus Torvalds's avatar
Linus Torvalds committed
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
		flush_dcache_mmap_unlock(mapping);
	}
}

static void
__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
	struct vm_area_struct *prev, struct rb_node **rb_link,
	struct rb_node *rb_parent)
{
	__vma_link_list(mm, vma, prev, rb_parent);
	__vma_link_rb(mm, vma, rb_link, rb_parent);
}

static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
			struct vm_area_struct *prev, struct rb_node **rb_link,
			struct rb_node *rb_parent)
{
	struct address_space *mapping = NULL;

669
	if (vma->vm_file) {
Linus Torvalds's avatar
Linus Torvalds committed
670
		mapping = vma->vm_file->f_mapping;
671
		i_mmap_lock_write(mapping);
672
	}
Linus Torvalds's avatar
Linus Torvalds committed
673
674
675
676
677

	__vma_link(mm, vma, prev, rb_link, rb_parent);
	__vma_link_file(vma);

	if (mapping)
678
		i_mmap_unlock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
679
680
681
682
683
684

	mm->map_count++;
	validate_mm(mm);
}

/*
685
 * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
686
 * mm's list and rbtree.  It has already been inserted into the interval tree.
Linus Torvalds's avatar
Linus Torvalds committed
687
 */
ZhenwenXu's avatar
ZhenwenXu committed
688
static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
689
{
690
	struct vm_area_struct *prev;
ZhenwenXu's avatar
ZhenwenXu committed
691
	struct rb_node **rb_link, *rb_parent;
Linus Torvalds's avatar
Linus Torvalds committed
692

693
694
695
	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
			   &prev, &rb_link, &rb_parent))
		BUG();
Linus Torvalds's avatar
Linus Torvalds committed
696
697
698
699
700
701
702
703
	__vma_link(mm, vma, prev, rb_link, rb_parent);
	mm->map_count++;
}

static inline void
__vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
		struct vm_area_struct *prev)
{
704
	struct vm_area_struct *next;
705

706
707
	vma_rb_erase(vma, &mm->mm_rb);
	prev->vm_next = next = vma->vm_next;
708
709
	if (next)
		next->vm_prev = prev;
Davidlohr Bueso's avatar
Davidlohr Bueso committed
710
711
712

	/* Kill the cache */
	vmacache_invalidate(mm);
Linus Torvalds's avatar
Linus Torvalds committed
713
714
715
716
717
718
719
720
721
}

/*
 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
 * is already present in an i_mmap tree without adjusting the tree.
 * The following helper function should be used when such adjustments
 * are necessary.  The "insert" vma (if any) is to be inserted
 * before we drop the necessary locks.
 */
722
int vma_adjust(struct vm_area_struct *vma, unsigned long start,
Linus Torvalds's avatar
Linus Torvalds committed
723
724
725
726
727
728
	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
{
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *next = vma->vm_next;
	struct vm_area_struct *importer = NULL;
	struct address_space *mapping = NULL;
729
	struct rb_root *root = NULL;
730
	struct anon_vma *anon_vma = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
731
	struct file *file = vma->vm_file;
732
	bool start_changed = false, end_changed = false;
Linus Torvalds's avatar
Linus Torvalds committed
733
734
735
736
	long adjust_next = 0;
	int remove_next = 0;

	if (next && !insert) {
737
738
		struct vm_area_struct *exporter = NULL;

Linus Torvalds's avatar
Linus Torvalds committed
739
740
741
742
743
744
745
		if (end >= next->vm_end) {
			/*
			 * vma expands, overlapping all the next, and
			 * perhaps the one after too (mprotect case 6).
			 */
again:			remove_next = 1 + (end > next->vm_end);
			end = next->vm_end;
746
			exporter = next;
Linus Torvalds's avatar
Linus Torvalds committed
747
748
749
750
751
752
753
			importer = vma;
		} else if (end > next->vm_start) {
			/*
			 * vma expands, overlapping part of the next:
			 * mprotect case 5 shifting the boundary up.
			 */
			adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
754
			exporter = next;
Linus Torvalds's avatar
Linus Torvalds committed
755
756
757
758
759
760
761
			importer = vma;
		} else if (end < vma->vm_end) {
			/*
			 * vma shrinks, and !insert tells it's not
			 * split_vma inserting another: so it must be
			 * mprotect case 4 shifting the boundary down.
			 */
vishnu.ps's avatar
vishnu.ps committed
762
			adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT);
763
			exporter = vma;
Linus Torvalds's avatar
Linus Torvalds committed
764
765
766
			importer = next;
		}

767
768
769
770
771
		/*
		 * Easily overlooked: when mprotect shifts the boundary,
		 * make sure the expanding vma has anon_vma set if the
		 * shrinking vma had, to cover any anon pages imported.
		 */
772
		if (exporter && exporter->anon_vma && !importer->anon_vma) {
773
774
			int error;

775
			importer->anon_vma = exporter->anon_vma;
776
			error = anon_vma_clone(importer, exporter);
777
778
			if (error) {
				importer->anon_vma = NULL;
779
				return error;
780
			}
781
782
783
		}
	}

Linus Torvalds's avatar
Linus Torvalds committed
784
785
	if (file) {
		mapping = file->f_mapping;
786
787
		root = &mapping->i_mmap;
		uprobe_munmap(vma, vma->vm_start, vma->vm_end);
788

789
790
		if (adjust_next)
			uprobe_munmap(next, next->vm_start, next->vm_end);
791

792
		i_mmap_lock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
793
794
		if (insert) {
			/*
795
			 * Put into interval tree now, so instantiated pages
Linus Torvalds's avatar
Linus Torvalds committed
796
797
798
799
800
801
802
803
			 * are visible to arm/parisc __flush_dcache_page
			 * throughout; but we cannot insert into address
			 * space until vma start or end is updated.
			 */
			__vma_link_file(insert);
		}
	}

804
805
	vma_adjust_trans_huge(vma, start, end, adjust_next);

806
807
808
809
	anon_vma = vma->anon_vma;
	if (!anon_vma && adjust_next)
		anon_vma = next->anon_vma;
	if (anon_vma) {
810
811
		VM_BUG_ON_VMA(adjust_next && next->anon_vma &&
			  anon_vma != next->anon_vma, next);
812
		anon_vma_lock_write(anon_vma);
813
814
815
816
		anon_vma_interval_tree_pre_update_vma(vma);
		if (adjust_next)
			anon_vma_interval_tree_pre_update_vma(next);
	}
817

Linus Torvalds's avatar
Linus Torvalds committed
818
819
	if (root) {
		flush_dcache_mmap_lock(mapping);
820
		vma_interval_tree_remove(vma, root);
Linus Torvalds's avatar
Linus Torvalds committed
821
		if (adjust_next)
822
			vma_interval_tree_remove(next, root);
Linus Torvalds's avatar
Linus Torvalds committed
823
824
	}

825
826
827
828
829
830
831
832
	if (start != vma->vm_start) {
		vma->vm_start = start;
		start_changed = true;
	}
	if (end != vma->vm_end) {
		vma->vm_end = end;
		end_changed = true;
	}
Linus Torvalds's avatar
Linus Torvalds committed
833
834
835
836
837
838
839
840
	vma->vm_pgoff = pgoff;
	if (adjust_next) {
		next->vm_start += adjust_next << PAGE_SHIFT;
		next->vm_pgoff += adjust_next;
	}

	if (root) {
		if (adjust_next)
841
842
			vma_interval_tree_insert(next, root);
		vma_interval_tree_insert(vma, root);
Linus Torvalds's avatar
Linus Torvalds committed
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
		flush_dcache_mmap_unlock(mapping);
	}

	if (remove_next) {
		/*
		 * vma_merge has merged next into vma, and needs
		 * us to remove next before dropping the locks.
		 */
		__vma_unlink(mm, next, vma);
		if (file)
			__remove_shared_vm_struct(next, file, mapping);
	} else if (insert) {
		/*
		 * split_vma has split insert from vma, and needs
		 * us to insert it before dropping the locks
		 * (it may either follow vma or precede it).
		 */
		__insert_vm_struct(mm, insert);
861
862
863
864
865
866
867
868
869
	} else {
		if (start_changed)
			vma_gap_update(vma);
		if (end_changed) {
			if (!next)
				mm->highest_vm_end = end;
			else if (!adjust_next)
				vma_gap_update(next);
		}
Linus Torvalds's avatar
Linus Torvalds committed
870
871
	}

872
873
874
875
	if (anon_vma) {
		anon_vma_interval_tree_post_update_vma(vma);
		if (adjust_next)
			anon_vma_interval_tree_post_update_vma(next);
876
		anon_vma_unlock_write(anon_vma);
877
	}
Linus Torvalds's avatar
Linus Torvalds committed
878
	if (mapping)
879
		i_mmap_unlock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
880

881
	if (root) {
882
		uprobe_mmap(vma);
883
884

		if (adjust_next)
885
			uprobe_mmap(next);
886
887
	}

Linus Torvalds's avatar
Linus Torvalds committed
888
	if (remove_next) {
Matt Helsley's avatar
Matt Helsley committed
889
		if (file) {
890
			uprobe_munmap(next, next->vm_start, next->vm_end);
Linus Torvalds's avatar
Linus Torvalds committed
891
			fput(file);
Matt Helsley's avatar
Matt Helsley committed
892
		}
893
894
		if (next->anon_vma)
			anon_vma_merge(vma, next);
Linus Torvalds's avatar
Linus Torvalds committed
895
		mm->map_count--;
896
		mpol_put(vma_policy(next));
Linus Torvalds's avatar
Linus Torvalds committed
897
898
899
900
901
902
		kmem_cache_free(vm_area_cachep, next);
		/*
		 * In mprotect's case 6 (see comments on vma_merge),
		 * we must remove another next too. It would clutter
		 * up the code too much to do both in one go.
		 */
903
904
		next = vma->vm_next;
		if (remove_next == 2)
Linus Torvalds's avatar
Linus Torvalds committed
905
			goto again;
906
907
908
909
		else if (next)
			vma_gap_update(next);
		else
			mm->highest_vm_end = end;
Linus Torvalds's avatar
Linus Torvalds committed
910
	}
911
	if (insert && file)
912
		uprobe_mmap(insert);
Linus Torvalds's avatar
Linus Torvalds committed
913
914

	validate_mm(mm);
915
916

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
917
918
919
920
921
922
923
924
925
}

/*
 * If the vma has a ->close operation then the driver probably needs to release
 * per-vma resources, so we don't attempt to merge those.
 */
static inline int is_mergeable_vma(struct vm_area_struct *vma,
			struct file *file, unsigned long vm_flags)
{
926
927
928
929
930
931
932
933
934
	/*
	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
	 * match the flags but dirty bit -- the caller should mark
	 * merged VMA as dirty. If dirty bit won't be excluded from
	 * comparison, we increase pressue on the memory system forcing
	 * the kernel to generate new VMAs when old one could be
	 * extended instead.
	 */
	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
Linus Torvalds's avatar
Linus Torvalds committed
935
936
937
938
939
940
941
942
943
		return 0;
	if (vma->vm_file != file)
		return 0;
	if (vma->vm_ops && vma->vm_ops->close)
		return 0;
	return 1;
}

static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
Shaohua Li's avatar
Shaohua Li committed
944
945
					struct anon_vma *anon_vma2,
					struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
946
{
Shaohua Li's avatar
Shaohua Li committed
947
948
949
950
951
952
953
954
	/*
	 * The list_is_singular() test is to avoid merging VMA cloned from
	 * parents. This can improve scalability caused by anon_vma lock.
	 */
	if ((!anon_vma1 || !anon_vma2) && (!vma ||
		list_is_singular(&vma->anon_vma_chain)))
		return 1;
	return anon_vma1 == anon_vma2;
Linus Torvalds's avatar
Linus Torvalds committed
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
}

/*
 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
 * in front of (at a lower virtual address and file offset than) the vma.
 *
 * We cannot merge two vmas if they have differently assigned (non-NULL)
 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
 *
 * We don't check here for the merged mmap wrapping around the end of pagecache
 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
 * wrap, nor mmaps which cover the final page at index -1UL.
 */
static int
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
	struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
{
	if (is_mergeable_vma(vma, file, vm_flags) &&
Shaohua Li's avatar
Shaohua Li committed
973
	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
Linus Torvalds's avatar
Linus Torvalds committed
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
		if (vma->vm_pgoff == vm_pgoff)
			return 1;
	}
	return 0;
}

/*
 * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
 * beyond (at a higher virtual address and file offset than) the vma.
 *
 * We cannot merge two vmas if they have differently assigned (non-NULL)
 * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
 */
static int
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
	struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
{
	if (is_mergeable_vma(vma, file, vm_flags) &&
Shaohua Li's avatar
Shaohua Li committed
992
	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
Linus Torvalds's avatar
Linus Torvalds committed
993
		pgoff_t vm_pglen;
994
		vm_pglen = vma_pages(vma);
Linus Torvalds's avatar
Linus Torvalds committed
995
996
997
998
999
1000
		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
			return 1;
	}
	return 0;
}