migrate.c 44 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/*
 * Memory Migration functionality - linux/mm/migration.c
 *
 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
 *
 * Page migration was first developed in the context of the memory hotplug
 * project. The main authors of the migration code are:
 *
 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
 * Hirokazu Takahashi <taka@valinux.co.jp>
 * Dave Hansen <haveblue@us.ibm.com>
Christoph Lameter's avatar
Christoph Lameter committed
12
 * Christoph Lameter
13
14
15
 */

#include <linux/migrate.h>
16
#include <linux/export.h>
17
#include <linux/swap.h>
18
#include <linux/swapops.h>
19
#include <linux/pagemap.h>
20
#include <linux/buffer_head.h>
21
#include <linux/mm_inline.h>
22
#include <linux/nsproxy.h>
23
#include <linux/pagevec.h>
24
#include <linux/ksm.h>
25
26
27
28
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
29
#include <linux/writeback.h>
30
31
#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
32
#include <linux/security.h>
33
#include <linux/memcontrol.h>
34
#include <linux/syscalls.h>
35
#include <linux/hugetlb.h>
36
#include <linux/hugetlb_cgroup.h>
37
#include <linux/gfp.h>
38
#include <linux/balloon_compaction.h>
39

40
41
#include <asm/tlbflush.h>

42
43
44
#define CREATE_TRACE_POINTS
#include <trace/events/migrate.h>

45
46
47
#include "internal.h"

/*
48
 * migrate_prep() needs to be called before we start compiling a list of pages
49
50
 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
 * undesirable, use migrate_prep_local()
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 */
int migrate_prep(void)
{
	/*
	 * Clear the LRU lists so pages can be isolated.
	 * Note that pages may be moved off the LRU after we have
	 * drained them. Those pages will fail to migrate like other
	 * pages that may be busy.
	 */
	lru_add_drain_all();

	return 0;
}

65
66
67
68
69
70
71
72
/* Do the necessary work of migrate_prep but not if it involves other CPUs */
int migrate_prep_local(void)
{
	lru_add_drain();

	return 0;
}

73
/*
74
75
 * Add isolated pages on the list back to the LRU under page lock
 * to avoid leaking evictable pages back onto unevictable list.
76
 */
77
void putback_lru_pages(struct list_head *l)
78
79
80
81
{
	struct page *page;
	struct page *page2;

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
	list_for_each_entry_safe(page, page2, l, lru) {
		list_del(&page->lru);
		dec_zone_page_state(page, NR_ISOLATED_ANON +
				page_is_file_cache(page));
			putback_lru_page(page);
	}
}

/*
 * Put previously isolated pages back onto the appropriate lists
 * from where they were once taken off for compaction/migration.
 *
 * This function shall be used instead of putback_lru_pages(),
 * whenever the isolated pageset has been built by isolate_migratepages_range()
 */
void putback_movable_pages(struct list_head *l)
{
	struct page *page;
	struct page *page2;

102
	list_for_each_entry_safe(page, page2, l, lru) {
103
		list_del(&page->lru);
104
		dec_zone_page_state(page, NR_ISOLATED_ANON +
105
				page_is_file_cache(page));
106
107
108
109
		if (unlikely(balloon_page_movable(page)))
			balloon_page_putback(page);
		else
			putback_lru_page(page);
110
111
112
	}
}

113
114
115
/*
 * Restore a potential migration pte to a working pte entry
 */
116
117
static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
				 unsigned long addr, void *old)
118
119
120
121
122
123
124
{
	struct mm_struct *mm = vma->vm_mm;
	swp_entry_t entry;
 	pmd_t *pmd;
	pte_t *ptep, pte;
 	spinlock_t *ptl;

125
126
127
128
129
130
	if (unlikely(PageHuge(new))) {
		ptep = huge_pte_offset(mm, addr);
		if (!ptep)
			goto out;
		ptl = &mm->page_table_lock;
	} else {
Bob Liu's avatar
Bob Liu committed
131
132
		pmd = mm_find_pmd(mm, addr);
		if (!pmd)
133
			goto out;
134
135
		if (pmd_trans_huge(*pmd))
			goto out;
136

137
		ptep = pte_offset_map(pmd, addr);
138

139
140
141
142
		/*
		 * Peek to check is_swap_pte() before taking ptlock?  No, we
		 * can race mremap's move_ptes(), which skips anon_vma lock.
		 */
143
144
145

		ptl = pte_lockptr(mm, pmd);
	}
146
147
148
149

 	spin_lock(ptl);
	pte = *ptep;
	if (!is_swap_pte(pte))
150
		goto unlock;
151
152
153

	entry = pte_to_swp_entry(pte);

154
155
156
	if (!is_migration_entry(entry) ||
	    migration_entry_to_page(entry) != old)
		goto unlock;
157
158
159
160
161

	get_page(new);
	pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
	if (is_write_migration_entry(entry))
		pte = pte_mkwrite(pte);
162
#ifdef CONFIG_HUGETLB_PAGE
163
	if (PageHuge(new)) {
164
		pte = pte_mkhuge(pte);
165
166
		pte = arch_make_huge_pte(pte, vma, new, 0);
	}
167
#endif
168
	flush_dcache_page(new);
169
	set_pte_at(mm, addr, ptep, pte);
170

171
172
173
174
175
176
	if (PageHuge(new)) {
		if (PageAnon(new))
			hugepage_add_anon_rmap(new, vma, addr);
		else
			page_dup_rmap(new);
	} else if (PageAnon(new))
177
178
179
180
181
		page_add_anon_rmap(new, vma, addr);
	else
		page_add_file_rmap(new);

	/* No need to invalidate - it was non-present before */
182
	update_mmu_cache(vma, addr, ptep);
183
unlock:
184
	pte_unmap_unlock(ptep, ptl);
185
186
out:
	return SWAP_AGAIN;
187
188
}

189
190
191
192
193
194
/*
 * Get rid of all migration entries and replace them by
 * references to the indicated page.
 */
static void remove_migration_ptes(struct page *old, struct page *new)
{
195
	rmap_walk(new, remove_migration_pte, old);
196
197
}

198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
/*
 * Something used the pte of a page under migration. We need to
 * get to the page and wait until migration is finished.
 * When we return from this function the fault will be retried.
 */
void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
				unsigned long address)
{
	pte_t *ptep, pte;
	spinlock_t *ptl;
	swp_entry_t entry;
	struct page *page;

	ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
	pte = *ptep;
	if (!is_swap_pte(pte))
		goto out;

	entry = pte_to_swp_entry(pte);
	if (!is_migration_entry(entry))
		goto out;

	page = migration_entry_to_page(entry);

Nick Piggin's avatar
Nick Piggin committed
222
223
224
225
226
227
228
229
230
	/*
	 * Once radix-tree replacement of page migration started, page_count
	 * *must* be zero. And, we don't want to call wait_on_page_locked()
	 * against a page without get_page().
	 * So, we use get_page_unless_zero(), here. Even failed, page fault
	 * will occur again.
	 */
	if (!get_page_unless_zero(page))
		goto out;
231
232
233
234
235
236
237
238
	pte_unmap_unlock(ptep, ptl);
	wait_on_page_locked(page);
	put_page(page);
	return;
out:
	pte_unmap_unlock(ptep, ptl);
}

239
240
#ifdef CONFIG_BLOCK
/* Returns true if all buffers are successfully locked */
241
242
static bool buffer_migrate_lock_buffers(struct buffer_head *head,
							enum migrate_mode mode)
243
244
245
246
{
	struct buffer_head *bh = head;

	/* Simple case, sync compaction */
247
	if (mode != MIGRATE_ASYNC) {
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
		do {
			get_bh(bh);
			lock_buffer(bh);
			bh = bh->b_this_page;

		} while (bh != head);

		return true;
	}

	/* async case, we cannot block on lock_buffer so use trylock_buffer */
	do {
		get_bh(bh);
		if (!trylock_buffer(bh)) {
			/*
			 * We failed to lock the buffer and cannot stall in
			 * async migration. Release the taken locks
			 */
			struct buffer_head *failed_bh = bh;
			put_bh(failed_bh);
			bh = head;
			while (bh != failed_bh) {
				unlock_buffer(bh);
				put_bh(bh);
				bh = bh->b_this_page;
			}
			return false;
		}

		bh = bh->b_this_page;
	} while (bh != head);
	return true;
}
#else
static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
283
							enum migrate_mode mode)
284
285
286
287
288
{
	return true;
}
#endif /* CONFIG_BLOCK */

289
/*
290
 * Replace the page in the mapping.
291
292
293
294
 *
 * The number of remaining references must be:
 * 1 for anonymous pages without a mapping
 * 2 for pages with a mapping
295
 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
296
 */
297
static int migrate_page_move_mapping(struct address_space *mapping,
298
		struct page *newpage, struct page *page,
299
		struct buffer_head *head, enum migrate_mode mode)
300
{
301
	int expected_count = 0;
302
	void **pslot;
303

304
	if (!mapping) {
305
		/* Anonymous page without mapping */
306
307
		if (page_count(page) != 1)
			return -EAGAIN;
308
		return MIGRATEPAGE_SUCCESS;
309
310
	}

Nick Piggin's avatar
Nick Piggin committed
311
	spin_lock_irq(&mapping->tree_lock);
312

313
314
	pslot = radix_tree_lookup_slot(&mapping->page_tree,
 					page_index(page));
315

316
	expected_count = 2 + page_has_private(page);
Nick Piggin's avatar
Nick Piggin committed
317
	if (page_count(page) != expected_count ||
318
		radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
Nick Piggin's avatar
Nick Piggin committed
319
		spin_unlock_irq(&mapping->tree_lock);
320
		return -EAGAIN;
321
322
	}

Nick Piggin's avatar
Nick Piggin committed
323
	if (!page_freeze_refs(page, expected_count)) {
Nick Piggin's avatar
Nick Piggin committed
324
		spin_unlock_irq(&mapping->tree_lock);
Nick Piggin's avatar
Nick Piggin committed
325
326
327
		return -EAGAIN;
	}

328
329
330
331
332
333
334
	/*
	 * In the async migration case of moving a page with buffers, lock the
	 * buffers using trylock before the mapping is moved. If the mapping
	 * was moved, we later failed to lock the buffers and could not move
	 * the mapping back due to an elevated page count, we would have to
	 * block waiting on other references to be dropped.
	 */
335
336
	if (mode == MIGRATE_ASYNC && head &&
			!buffer_migrate_lock_buffers(head, mode)) {
337
338
339
340
341
		page_unfreeze_refs(page, expected_count);
		spin_unlock_irq(&mapping->tree_lock);
		return -EAGAIN;
	}

342
343
344
	/*
	 * Now we know that no one else is looking at the page.
	 */
345
	get_page(newpage);	/* add cache reference */
346
347
348
349
350
	if (PageSwapCache(page)) {
		SetPageSwapCache(newpage);
		set_page_private(newpage, page_private(page));
	}

351
352
353
	radix_tree_replace_slot(pslot, newpage);

	/*
354
355
	 * Drop cache reference from old page by unfreezing
	 * to one less reference.
356
357
	 * We know this isn't the last reference.
	 */
358
	page_unfreeze_refs(page, expected_count - 1);
359

360
361
362
363
364
365
366
367
368
369
370
371
	/*
	 * If moved to a different zone then also account
	 * the page for that zone. Other VM counters will be
	 * taken care of when we establish references to the
	 * new page and drop references to the old page.
	 *
	 * Note that anonymous pages are accounted for
	 * via NR_FILE_PAGES and NR_ANON_PAGES if they
	 * are mapped to swap space.
	 */
	__dec_zone_page_state(page, NR_FILE_PAGES);
	__inc_zone_page_state(newpage, NR_FILE_PAGES);
372
	if (!PageSwapCache(page) && PageSwapBacked(page)) {
373
374
375
		__dec_zone_page_state(page, NR_SHMEM);
		__inc_zone_page_state(newpage, NR_SHMEM);
	}
Nick Piggin's avatar
Nick Piggin committed
376
	spin_unlock_irq(&mapping->tree_lock);
377

378
	return MIGRATEPAGE_SUCCESS;
379
380
}

381
382
383
384
385
386
387
388
389
390
391
392
393
/*
 * The expected number of remaining references is the same as that
 * of migrate_page_move_mapping().
 */
int migrate_huge_page_move_mapping(struct address_space *mapping,
				   struct page *newpage, struct page *page)
{
	int expected_count;
	void **pslot;

	if (!mapping) {
		if (page_count(page) != 1)
			return -EAGAIN;
394
		return MIGRATEPAGE_SUCCESS;
395
396
397
398
399
400
401
402
403
	}

	spin_lock_irq(&mapping->tree_lock);

	pslot = radix_tree_lookup_slot(&mapping->page_tree,
					page_index(page));

	expected_count = 2 + page_has_private(page);
	if (page_count(page) != expected_count ||
404
		radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
405
406
407
408
409
410
411
412
413
414
415
416
417
		spin_unlock_irq(&mapping->tree_lock);
		return -EAGAIN;
	}

	if (!page_freeze_refs(page, expected_count)) {
		spin_unlock_irq(&mapping->tree_lock);
		return -EAGAIN;
	}

	get_page(newpage);

	radix_tree_replace_slot(pslot, newpage);

418
	page_unfreeze_refs(page, expected_count - 1);
419
420

	spin_unlock_irq(&mapping->tree_lock);
421
	return MIGRATEPAGE_SUCCESS;
422
423
}

424
425
426
/*
 * Copy the page to its new location
 */
427
void migrate_page_copy(struct page *newpage, struct page *page)
428
{
429
	if (PageHuge(page) || PageTransHuge(page))
430
431
432
		copy_huge_page(newpage, page);
	else
		copy_highpage(newpage, page);
433
434
435
436
437
438
439

	if (PageError(page))
		SetPageError(newpage);
	if (PageReferenced(page))
		SetPageReferenced(newpage);
	if (PageUptodate(page))
		SetPageUptodate(newpage);
440
441
	if (TestClearPageActive(page)) {
		VM_BUG_ON(PageUnevictable(page));
442
		SetPageActive(newpage);
443
444
	} else if (TestClearPageUnevictable(page))
		SetPageUnevictable(newpage);
445
446
447
448
449
450
451
	if (PageChecked(page))
		SetPageChecked(newpage);
	if (PageMappedToDisk(page))
		SetPageMappedToDisk(newpage);

	if (PageDirty(page)) {
		clear_page_dirty_for_io(page);
452
453
454
455
456
		/*
		 * Want to mark the page and the radix tree as dirty, and
		 * redo the accounting that clear_page_dirty_for_io undid,
		 * but we can't use set_page_dirty because that function
		 * is actually a signal that all of the page has become dirty.
Lucas De Marchi's avatar
Lucas De Marchi committed
457
		 * Whereas only part of our page may be dirty.
458
		 */
459
460
461
462
		if (PageSwapBacked(page))
			SetPageDirty(newpage);
		else
			__set_page_dirty_nobuffers(newpage);
463
464
 	}

465
	mlock_migrate_page(newpage, page);
466
	ksm_migrate_page(newpage, page);
467
468
469
470
	/*
	 * Please do not reorder this without considering how mm/ksm.c's
	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
	 */
471
472
473
474
475
476
477
478
479
480
481
482
	ClearPageSwapCache(page);
	ClearPagePrivate(page);
	set_page_private(page, 0);

	/*
	 * If any waiters have accumulated on the new page then
	 * wake them up.
	 */
	if (PageWriteback(newpage))
		end_page_writeback(newpage);
}

483
484
485
486
487
/************************************************************
 *                    Migration functions
 ***********************************************************/

/* Always fail migration. Used for mappings that are not movable */
488
489
int fail_migrate_page(struct address_space *mapping,
			struct page *newpage, struct page *page)
490
491
492
493
494
{
	return -EIO;
}
EXPORT_SYMBOL(fail_migrate_page);

495
496
/*
 * Common logic to directly migrate a single page suitable for
497
 * pages that do not use PagePrivate/PagePrivate2.
498
499
500
 *
 * Pages are locked upon entry and exit.
 */
501
int migrate_page(struct address_space *mapping,
502
503
		struct page *newpage, struct page *page,
		enum migrate_mode mode)
504
505
506
507
508
{
	int rc;

	BUG_ON(PageWriteback(page));	/* Writeback must be complete */

509
	rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode);
510

511
	if (rc != MIGRATEPAGE_SUCCESS)
512
513
514
		return rc;

	migrate_page_copy(newpage, page);
515
	return MIGRATEPAGE_SUCCESS;
516
517
518
}
EXPORT_SYMBOL(migrate_page);

519
#ifdef CONFIG_BLOCK
520
521
522
523
524
/*
 * Migration function for pages with buffers. This function can only be used
 * if the underlying filesystem guarantees that no other references to "page"
 * exist.
 */
525
int buffer_migrate_page(struct address_space *mapping,
526
		struct page *newpage, struct page *page, enum migrate_mode mode)
527
528
529
530
531
{
	struct buffer_head *bh, *head;
	int rc;

	if (!page_has_buffers(page))
532
		return migrate_page(mapping, newpage, page, mode);
533
534
535

	head = page_buffers(page);

536
	rc = migrate_page_move_mapping(mapping, newpage, page, head, mode);
537

538
	if (rc != MIGRATEPAGE_SUCCESS)
539
540
		return rc;

541
542
543
544
545
	/*
	 * In the async case, migrate_page_move_mapping locked the buffers
	 * with an IRQ-safe spinlock held. In the sync case, the buffers
	 * need to be locked now
	 */
546
547
	if (mode != MIGRATE_ASYNC)
		BUG_ON(!buffer_migrate_lock_buffers(head, mode));
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573

	ClearPagePrivate(page);
	set_page_private(newpage, page_private(page));
	set_page_private(page, 0);
	put_page(page);
	get_page(newpage);

	bh = head;
	do {
		set_bh_page(bh, newpage, bh_offset(bh));
		bh = bh->b_this_page;

	} while (bh != head);

	SetPagePrivate(newpage);

	migrate_page_copy(newpage, page);

	bh = head;
	do {
		unlock_buffer(bh);
 		put_bh(bh);
		bh = bh->b_this_page;

	} while (bh != head);

574
	return MIGRATEPAGE_SUCCESS;
575
576
}
EXPORT_SYMBOL(buffer_migrate_page);
577
#endif
578

579
580
581
582
/*
 * Writeback a page to clean the dirty state
 */
static int writeout(struct address_space *mapping, struct page *page)
583
{
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_NONE,
		.nr_to_write = 1,
		.range_start = 0,
		.range_end = LLONG_MAX,
		.for_reclaim = 1
	};
	int rc;

	if (!mapping->a_ops->writepage)
		/* No write method for the address space */
		return -EINVAL;

	if (!clear_page_dirty_for_io(page))
		/* Someone else already triggered a write */
		return -EAGAIN;

601
	/*
602
603
604
605
606
607
	 * A dirty page may imply that the underlying filesystem has
	 * the page on some queue. So the page must be clean for
	 * migration. Writeout may mean we loose the lock and the
	 * page state is no longer what we checked for earlier.
	 * At this point we know that the migration attempt cannot
	 * be successful.
608
	 */
609
	remove_migration_ptes(page, page);
610

611
	rc = mapping->a_ops->writepage(page, &wbc);
612

613
614
615
616
	if (rc != AOP_WRITEPAGE_ACTIVATE)
		/* unlocked. Relock */
		lock_page(page);

Hugh Dickins's avatar
Hugh Dickins committed
617
	return (rc < 0) ? -EIO : -EAGAIN;
618
619
620
621
622
623
}

/*
 * Default handling if a filesystem does not provide a migration function.
 */
static int fallback_migrate_page(struct address_space *mapping,
624
	struct page *newpage, struct page *page, enum migrate_mode mode)
625
{
626
	if (PageDirty(page)) {
627
628
		/* Only writeback pages in full synchronous migration */
		if (mode != MIGRATE_SYNC)
629
			return -EBUSY;
630
		return writeout(mapping, page);
631
	}
632
633
634
635
636

	/*
	 * Buffers may be managed in a filesystem specific way.
	 * We must have no buffers or drop them.
	 */
637
	if (page_has_private(page) &&
638
639
640
	    !try_to_release_page(page, GFP_KERNEL))
		return -EAGAIN;

641
	return migrate_page(mapping, newpage, page, mode);
642
643
}

644
645
646
647
648
649
/*
 * Move a page to a newly allocated page
 * The page is locked and all ptes have been successfully removed.
 *
 * The new page will have replaced the old page if this function
 * is successful.
650
651
652
 *
 * Return value:
 *   < 0 - error code
653
 *  MIGRATEPAGE_SUCCESS - success
654
 */
655
static int move_to_new_page(struct page *newpage, struct page *page,
656
				int remap_swapcache, enum migrate_mode mode)
657
658
659
660
661
662
663
664
665
{
	struct address_space *mapping;
	int rc;

	/*
	 * Block others from accessing the page when we get around to
	 * establishing additional references. We are the only one
	 * holding a reference to the new page at this point.
	 */
Nick Piggin's avatar
Nick Piggin committed
666
	if (!trylock_page(newpage))
667
668
669
670
671
		BUG();

	/* Prepare mapping for the new page.*/
	newpage->index = page->index;
	newpage->mapping = page->mapping;
672
673
	if (PageSwapBacked(page))
		SetPageSwapBacked(newpage);
674
675
676

	mapping = page_mapping(page);
	if (!mapping)
677
		rc = migrate_page(mapping, newpage, page, mode);
678
	else if (mapping->a_ops->migratepage)
679
		/*
680
681
682
683
		 * Most pages have a mapping and most filesystems provide a
		 * migratepage callback. Anonymous pages are part of swap
		 * space which also has its own migratepage callback. This
		 * is the most common path for page migration.
684
		 */
685
		rc = mapping->a_ops->migratepage(mapping,
686
						newpage, page, mode);
687
	else
688
		rc = fallback_migrate_page(mapping, newpage, page, mode);
689

690
	if (rc != MIGRATEPAGE_SUCCESS) {
691
		newpage->mapping = NULL;
692
693
694
	} else {
		if (remap_swapcache)
			remove_migration_ptes(page, newpage);
695
		page->mapping = NULL;
696
	}
697
698
699
700
701
702

	unlock_page(newpage);

	return rc;
}

703
static int __unmap_and_move(struct page *page, struct page *newpage,
704
				int force, enum migrate_mode mode)
705
{
706
	int rc = -EAGAIN;
707
	int remap_swapcache = 1;
708
	struct mem_cgroup *mem;
709
	struct anon_vma *anon_vma = NULL;
710

Nick Piggin's avatar
Nick Piggin committed
711
	if (!trylock_page(page)) {
712
		if (!force || mode == MIGRATE_ASYNC)
713
			goto out;
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728

		/*
		 * It's not safe for direct compaction to call lock_page.
		 * For example, during page readahead pages are added locked
		 * to the LRU. Later, when the IO completes the pages are
		 * marked uptodate and unlocked. However, the queueing
		 * could be merging multiple pages for one bio (e.g.
		 * mpage_readpages). If an allocation happens for the
		 * second or third page, the process can end up locking
		 * the same page twice and deadlocking. Rather than
		 * trying to be clever about what pages can be locked,
		 * avoid the use of lock_page for direct compaction
		 * altogether.
		 */
		if (current->flags & PF_MEMALLOC)
729
			goto out;
730

731
732
733
		lock_page(page);
	}

734
	/* charge against new page */
735
	mem_cgroup_prepare_migration(page, newpage, &mem);
736

737
	if (PageWriteback(page)) {
738
		/*
739
		 * Only in the case of a full synchronous migration is it
740
741
742
		 * necessary to wait for PageWriteback. In the async case,
		 * the retry loop is too short and in the sync-light case,
		 * the overhead of stalling is too much
743
		 */
744
		if (mode != MIGRATE_SYNC) {
745
746
747
748
			rc = -EBUSY;
			goto uncharge;
		}
		if (!force)
749
			goto uncharge;
750
751
752
		wait_on_page_writeback(page);
	}
	/*
753
754
	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
	 * we cannot notice that anon_vma is freed while we migrates a page.
755
	 * This get_anon_vma() delays freeing anon_vma pointer until the end
756
	 * of migration. File cache pages are no problem because of page_lock()
757
758
	 * File Caches may use write_page() or lock_page() in migration, then,
	 * just care Anon page here.
759
	 */
Hugh Dickins's avatar
Hugh Dickins committed
760
	if (PageAnon(page) && !PageKsm(page)) {
761
		/*
762
		 * Only page_lock_anon_vma_read() understands the subtleties of
763
764
		 * getting a hold on an anon_vma from outside one of its mms.
		 */
765
		anon_vma = page_get_anon_vma(page);
766
767
		if (anon_vma) {
			/*
768
			 * Anon page
769
770
			 */
		} else if (PageSwapCache(page)) {
771
772
773
774
775
776
777
778
779
780
781
782
783
784
			/*
			 * We cannot be sure that the anon_vma of an unmapped
			 * swapcache page is safe to use because we don't
			 * know in advance if the VMA that this page belonged
			 * to still exists. If the VMA and others sharing the
			 * data have been freed, then the anon_vma could
			 * already be invalid.
			 *
			 * To avoid this possibility, swapcache pages get
			 * migrated but are not remapped when migration
			 * completes
			 */
			remap_swapcache = 0;
		} else {
785
			goto uncharge;
786
		}
787
	}
788

789
790
791
792
793
794
795
796
797
798
799
800
	if (unlikely(balloon_page_movable(page))) {
		/*
		 * A ballooned page does not need any special attention from
		 * physical to virtual reverse mapping procedures.
		 * Skip any attempt to unmap PTEs or to remap swap cache,
		 * in order to avoid burning cycles at rmap level, and perform
		 * the page migration right away (proteced by page lock).
		 */
		rc = balloon_page_migrate(newpage, page, mode);
		goto uncharge;
	}

801
	/*
802
803
804
805
806
807
808
809
810
811
	 * Corner case handling:
	 * 1. When a new swap-cache page is read into, it is added to the LRU
	 * and treated as swapcache but it has no rmap yet.
	 * Calling try_to_unmap() against a page->mapping==NULL page will
	 * trigger a BUG.  So handle it here.
	 * 2. An orphaned page (see truncate_complete_page) might have
	 * fs-private metadata. The page can be picked up due to memory
	 * offlining.  Everywhere else except page reclaim, the page is
	 * invisible to the vm, so the page can not be migrated.  So try to
	 * free the metadata, so the page can be freed.
812
	 */
813
	if (!page->mapping) {
814
815
		VM_BUG_ON(PageAnon(page));
		if (page_has_private(page)) {
816
			try_to_free_buffers(page);
817
			goto uncharge;
818
		}
819
		goto skip_unmap;
820
821
	}

822
	/* Establish migration ptes or remove ptes */
823
	try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
824

825
skip_unmap:
826
	if (!page_mapped(page))
827
		rc = move_to_new_page(newpage, page, remap_swapcache, mode);
828

829
	if (rc && remap_swapcache)
830
		remove_migration_ptes(page, page);
831
832

	/* Drop an anon_vma reference if we took one */
833
	if (anon_vma)
834
		put_anon_vma(anon_vma);
835

836
uncharge:
837
838
839
	mem_cgroup_end_migration(mem, page, newpage,
				 (rc == MIGRATEPAGE_SUCCESS ||
				  rc == MIGRATEPAGE_BALLOON_SUCCESS));
840
	unlock_page(page);
841
842
843
out:
	return rc;
}
844

845
846
847
848
849
/*
 * Obtain the lock on page, remove all ptes and migrate the page
 * to the newly allocated page in newpage.
 */
static int unmap_and_move(new_page_t get_new_page, unsigned long private,
850
			struct page *page, int force, enum migrate_mode mode)
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
{
	int rc = 0;
	int *result = NULL;
	struct page *newpage = get_new_page(page, private, &result);

	if (!newpage)
		return -ENOMEM;

	if (page_count(page) == 1) {
		/* page was freed from under us. So we are done. */
		goto out;
	}

	if (unlikely(PageTransHuge(page)))
		if (unlikely(split_huge_page(page)))
			goto out;

868
	rc = __unmap_and_move(page, newpage, force, mode);
869
870
871
872
873
874
875
876
877
878
879
880

	if (unlikely(rc == MIGRATEPAGE_BALLOON_SUCCESS)) {
		/*
		 * A ballooned page has been migrated already.
		 * Now, it's the time to wrap-up counters,
		 * handle the page back to Buddy and return.
		 */
		dec_zone_page_state(page, NR_ISOLATED_ANON +
				    page_is_file_cache(page));
		balloon_page_free(page);
		return MIGRATEPAGE_SUCCESS;
	}
881
out:
882
	if (rc != -EAGAIN) {
883
884
885
886
887
888
889
		/*
		 * A page that has been migrated has all references
		 * removed and will be freed. A page that has not been
		 * migrated will have kepts its references and be
		 * restored.
		 */
		list_del(&page->lru);
890
		dec_zone_page_state(page, NR_ISOLATED_ANON +
891
				page_is_file_cache(page));
892
		putback_lru_page(page);
893
	}
894
895
896
897
	/*
	 * Move the new page to the LRU. If migration was not successful
	 * then this will free the page.
	 */
898
	putback_lru_page(newpage);
899
900
901
902
903
904
	if (result) {
		if (rc)
			*result = rc;
		else
			*result = page_to_nid(newpage);
	}
905
906
907
	return rc;
}

908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
/*
 * Counterpart of unmap_and_move_page() for hugepage migration.
 *
 * This function doesn't wait the completion of hugepage I/O
 * because there is no race between I/O and migration for hugepage.
 * Note that currently hugepage I/O occurs only in direct I/O
 * where no lock is held and PG_writeback is irrelevant,
 * and writeback status of all subpages are counted in the reference
 * count of the head page (i.e. if all subpages of a 2MB hugepage are
 * under direct I/O, the reference of the head page is 512 and a bit more.)
 * This means that when we try to migrate hugepage whose subpages are
 * doing direct I/O, some references remain after try_to_unmap() and
 * hugepage migration fails without data corruption.
 *
 * There is also no race when direct I/O is issued on the page under migration,
 * because then pte is replaced with migration swap entry and direct I/O code
 * will wait in the page fault for migration to complete.
 */
static int unmap_and_move_huge_page(new_page_t get_new_page,
				unsigned long private, struct page *hpage,
928
				int force, enum migrate_mode mode)
929
930
931
932
933
934
935
936
937
938
939
940
{
	int rc = 0;
	int *result = NULL;
	struct page *new_hpage = get_new_page(hpage, private, &result);
	struct anon_vma *anon_vma = NULL;

	if (!new_hpage)
		return -ENOMEM;

	rc = -EAGAIN;

	if (!trylock_page(hpage)) {
941
		if (!force || mode != MIGRATE_SYNC)
942
943
944
945
			goto out;
		lock_page(hpage);
	}

946
947
	if (PageAnon(hpage))
		anon_vma = page_get_anon_vma(hpage);
948
949
950
951

	try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);

	if (!page_mapped(hpage))
952
		rc = move_to_new_page(new_hpage, hpage, 1, mode);
953
954
955
956

	if (rc)
		remove_migration_ptes(hpage, hpage);

Hugh Dickins's avatar
Hugh Dickins committed
957
	if (anon_vma)
958
		put_anon_vma(anon_vma);
959
960
961
962

	if (!rc)
		hugetlb_cgroup_migrate(hpage, new_hpage);

963
	unlock_page(hpage);
964
out:
965
966
967
968
969
970
971
972
973
974
	put_page(new_hpage);
	if (result) {
		if (rc)
			*result = rc;
		else
			*result = page_to_nid(new_hpage);
	}
	return rc;
}

975
/*
976
977
 * migrate_pages - migrate the pages specified in a list, to the free pages
 *		   supplied as the target for the page migration
978
 *
979
980
981
982
983
984
985
 * @from:		The list of pages to be migrated.
 * @get_new_page:	The function used to allocate free pages to be used
 *			as the target of the page migration.
 * @private:		Private data to be passed on to get_new_page()
 * @mode:		The migration mode that specifies the constraints for
 *			page migration, if any.
 * @reason:		The reason for page migration.
986
 *
987
988
989
 * The function returns after 10 attempts or if no pages are movable any more
 * because the list has become empty or no retryable pages exist any more.
 * The caller should call putback_lru_pages() to return pages to the LRU
990
 * or free list only if ret != 0.
991
 *
992
 * Returns the number of pages that were not migrated, or an error code.
993
 */
994
995
int migrate_pages(struct list_head *from, new_page_t get_new_page,
		unsigned long private, enum migrate_mode mode, int reason)
996
{
997
	int retry = 1;
998
	int nr_failed = 0;
999
	int nr_succeeded = 0;
1000
1001
1002
1003
1004
1005
1006
1007
1008
	int pass = 0;
	struct page *page;
	struct page *page2;
	int swapwrite = current->flags & PF_SWAPWRITE;
	int rc;

	if (!swapwrite)
		current->flags |= PF_SWAPWRITE;

1009
1010
	for(pass = 0; pass < 10 && retry; pass++) {
		retry = 0;
1011

1012
1013
		list_for_each_entry_safe(page, page2, from, lru) {
			cond_resched();
1014

1015
			rc = unmap_and_move(get_new_page, private,
1016
						page, pass > 2, mode);
1017

1018
			switch(rc) {
1019
1020
			case -ENOMEM:
				goto out;
1021
			case -EAGAIN:
1022
				retry++;
1023
				break;
1024
			case MIGRATEPAGE_SUCCESS:
1025
				nr_succeeded++;
1026
1027
				break;
			default:
1028
1029
				/* Permanent failure */
				nr_failed++;
1030
				break;
1031
			}
1032
1033
		}
	}
1034
	rc = nr_failed + retry;
1035
out:
1036
1037
1038
1039
	if (nr_succeeded)
		count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
	if (nr_failed)
		count_vm_events(PGMIGRATE_FAIL, nr_failed);
1040
1041
	trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);

1042
1043
1044
	if (!swapwrite)
		current->flags &= ~PF_SWAPWRITE;

1045
	return rc;
1046
}
1047

1048
int migrate_huge_page(struct page *hpage, new_page_t get_new_page,
1049
		      unsigned long private, enum migrate_mode mode)
1050
{
1051
1052
1053
	int pass, rc;

	for (pass = 0; pass < 10; pass++) {
1054
1055
		rc = unmap_and_move_huge_page(get_new_page, private,
						hpage, pass > 2, mode);
1056
1057
1058
1059
1060
		switch (rc) {
		case -ENOMEM:
			goto out;
		case -EAGAIN:
			/* try again */
1061
			cond_resched();
1062
			break;
1063
		case MIGRATEPAGE_SUCCESS:
1064
1065
1066
1067
			goto out;
		default:
			rc = -EIO;
			goto out;
1068
1069
1070
		}
	}
out:
1071
	return rc;
1072
1073
}

1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
#ifdef CONFIG_NUMA
/*
 * Move a list of individual pages
 */
struct page_to_node {
	unsigned long addr;
	struct page *page;
	int node;
	int status;
};

static struct page *new_page_node(struct page *p, unsigned long private,
		int **result)
{
	struct page_to_node *pm = (struct page_to_node *)private;

	while (pm->node != MAX_NUMNODES && pm->page != p)
		pm++;

	if (pm->node == MAX_NUMNODES)
		return NULL;

	*result = &pm->status;

1098
	return alloc_pages_exact_node(pm->node,
1099
				GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
1100
1101
1102
1103
1104
1105
}

/*
 * Move a set of pages as indicated in the pm array. The addr
 * field must be set to the virtual address of the page to be moved
 * and the node number must contain a valid target node.
1106
 * The pm array ends with node = MAX_NUMNODES.
1107
 */
1108
1109
1110
static int do_move_page_to_node_array(struct mm_struct *mm,
				      struct page_to_node *pm,
				      int migrate_all)
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
{
	int err;
	struct page_to_node *pp;
	LIST_HEAD(pagelist);

	down_read(&mm->mmap_sem);

	/*
	 * Build a list of pages to migrate
	 */
	for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
		struct vm_area_struct *vma;
		struct page *page;

		err = -EFAULT;
		vma = find_vma(mm, pp->addr);
1127
		if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
1128
1129
			goto set_status;

1130
		page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT);
1131
1132
1133
1134
1135

		err = PTR_ERR(page);
		if (IS_ERR(page))
			goto set_status;

1136
1137
1138
1139
		err = -ENOENT;
		if (!page)
			goto set_status;

1140
		/* Use PageReserved to check for zero page */
Hugh Dickins's avatar
Hugh Dickins committed
1141
		if (PageReserved(page))
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
			goto put_and_set;

		pp->page = page;
		err = page_to_nid(page);

		if (err == pp->node)
			/*
			 * Node already in the right place
			 */
			goto put_and_set;

		err = -EACCES;
		if (page_mapcount(page) > 1 &&
				!migrate_all)
			goto put_and_set;

1158
		err = isolate_lru_page(page);
1159
		if (!err) {
1160
			list_add_tail(&page->lru, &pagelist);
1161
1162
1163
			inc_zone_page_state(page, NR_ISOLATED_ANON +
					    page_is_file_cache(page));
		}
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
put_and_set:
		/*
		 * Either remove the duplicate refcount from
		 * isolate_lru_page() or drop the page ref if it was
		 * not isolated.
		 */
		put_page(page);
set_status:
		pp->status = err;
	}

1175
	err = 0;
1176
	if (!list_empty(&pagelist)) {
1177
		err = migrate_pages(&pagelist, new_page_node,
1178
				(unsigned long)pm, MIGRATE_SYNC, MR_SYSCALL);
1179
1180
1181
		if (err)
			putback_lru_pages(&pagelist);
	}
1182
1183
1184
1185
1186

	up_read(&mm->mmap_sem);
	return err;
}

1187
1188
1189
1190
/*
 * Migrate an array of page address onto an array of nodes and fill
 * the corresponding array of status.
 */