migrate.c 44.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/*
 * Memory Migration functionality - linux/mm/migration.c
 *
 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
 *
 * Page migration was first developed in the context of the memory hotplug
 * project. The main authors of the migration code are:
 *
 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
 * Hirokazu Takahashi <taka@valinux.co.jp>
 * Dave Hansen <haveblue@us.ibm.com>
Christoph Lameter's avatar
Christoph Lameter committed
12
 * Christoph Lameter
13
14
15
 */

#include <linux/migrate.h>
16
#include <linux/export.h>
17
#include <linux/swap.h>
18
#include <linux/swapops.h>
19
#include <linux/pagemap.h>
20
#include <linux/buffer_head.h>
21
#include <linux/mm_inline.h>
22
#include <linux/nsproxy.h>
23
#include <linux/pagevec.h>
24
#include <linux/ksm.h>
25
26
27
28
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
29
#include <linux/writeback.h>
30
31
#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
32
#include <linux/security.h>
33
#include <linux/memcontrol.h>
34
#include <linux/syscalls.h>
35
#include <linux/hugetlb.h>
36
#include <linux/hugetlb_cgroup.h>
37
#include <linux/gfp.h>
38
#include <linux/balloon_compaction.h>
39

40
41
#include <asm/tlbflush.h>

42
43
44
#define CREATE_TRACE_POINTS
#include <trace/events/migrate.h>

45
46
47
#include "internal.h"

/*
48
 * migrate_prep() needs to be called before we start compiling a list of pages
49
50
 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
 * undesirable, use migrate_prep_local()
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 */
int migrate_prep(void)
{
	/*
	 * Clear the LRU lists so pages can be isolated.
	 * Note that pages may be moved off the LRU after we have
	 * drained them. Those pages will fail to migrate like other
	 * pages that may be busy.
	 */
	lru_add_drain_all();

	return 0;
}

65
66
67
68
69
70
71
72
/* Do the necessary work of migrate_prep but not if it involves other CPUs */
int migrate_prep_local(void)
{
	lru_add_drain();

	return 0;
}

73
/*
74
75
 * Add isolated pages on the list back to the LRU under page lock
 * to avoid leaking evictable pages back onto unevictable list.
76
 */
77
void putback_lru_pages(struct list_head *l)
78
79
80
81
{
	struct page *page;
	struct page *page2;

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
	list_for_each_entry_safe(page, page2, l, lru) {
		list_del(&page->lru);
		dec_zone_page_state(page, NR_ISOLATED_ANON +
				page_is_file_cache(page));
			putback_lru_page(page);
	}
}

/*
 * Put previously isolated pages back onto the appropriate lists
 * from where they were once taken off for compaction/migration.
 *
 * This function shall be used instead of putback_lru_pages(),
 * whenever the isolated pageset has been built by isolate_migratepages_range()
 */
void putback_movable_pages(struct list_head *l)
{
	struct page *page;
	struct page *page2;

102
	list_for_each_entry_safe(page, page2, l, lru) {
103
104
105
106
		if (unlikely(PageHuge(page))) {
			putback_active_hugepage(page);
			continue;
		}
107
		list_del(&page->lru);
108
		dec_zone_page_state(page, NR_ISOLATED_ANON +
109
				page_is_file_cache(page));
110
111
112
113
		if (unlikely(balloon_page_movable(page)))
			balloon_page_putback(page);
		else
			putback_lru_page(page);
114
115
116
	}
}

117
118
119
/*
 * Restore a potential migration pte to a working pte entry
 */
120
121
static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
				 unsigned long addr, void *old)
122
123
124
125
126
127
128
{
	struct mm_struct *mm = vma->vm_mm;
	swp_entry_t entry;
 	pmd_t *pmd;
	pte_t *ptep, pte;
 	spinlock_t *ptl;

129
130
131
132
133
134
	if (unlikely(PageHuge(new))) {
		ptep = huge_pte_offset(mm, addr);
		if (!ptep)
			goto out;
		ptl = &mm->page_table_lock;
	} else {
Bob Liu's avatar
Bob Liu committed
135
136
		pmd = mm_find_pmd(mm, addr);
		if (!pmd)
137
			goto out;
138
139
		if (pmd_trans_huge(*pmd))
			goto out;
140

141
		ptep = pte_offset_map(pmd, addr);
142

143
144
145
146
		/*
		 * Peek to check is_swap_pte() before taking ptlock?  No, we
		 * can race mremap's move_ptes(), which skips anon_vma lock.
		 */
147
148
149

		ptl = pte_lockptr(mm, pmd);
	}
150
151
152
153

 	spin_lock(ptl);
	pte = *ptep;
	if (!is_swap_pte(pte))
154
		goto unlock;
155
156
157

	entry = pte_to_swp_entry(pte);

158
159
160
	if (!is_migration_entry(entry) ||
	    migration_entry_to_page(entry) != old)
		goto unlock;
161
162
163
164
165

	get_page(new);
	pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
	if (is_write_migration_entry(entry))
		pte = pte_mkwrite(pte);
166
#ifdef CONFIG_HUGETLB_PAGE
167
	if (PageHuge(new)) {
168
		pte = pte_mkhuge(pte);
169
170
		pte = arch_make_huge_pte(pte, vma, new, 0);
	}
171
#endif
172
	flush_dcache_page(new);
173
	set_pte_at(mm, addr, ptep, pte);
174

175
176
177
178
179
180
	if (PageHuge(new)) {
		if (PageAnon(new))
			hugepage_add_anon_rmap(new, vma, addr);
		else
			page_dup_rmap(new);
	} else if (PageAnon(new))
181
182
183
184
185
		page_add_anon_rmap(new, vma, addr);
	else
		page_add_file_rmap(new);

	/* No need to invalidate - it was non-present before */
186
	update_mmu_cache(vma, addr, ptep);
187
unlock:
188
	pte_unmap_unlock(ptep, ptl);
189
190
out:
	return SWAP_AGAIN;
191
192
}

193
194
195
196
197
198
/*
 * Get rid of all migration entries and replace them by
 * references to the indicated page.
 */
static void remove_migration_ptes(struct page *old, struct page *new)
{
199
	rmap_walk(new, remove_migration_pte, old);
200
201
}

202
203
204
205
206
/*
 * Something used the pte of a page under migration. We need to
 * get to the page and wait until migration is finished.
 * When we return from this function the fault will be retried.
 */
207
208
static void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
				spinlock_t *ptl)
209
{
210
	pte_t pte;
211
212
213
	swp_entry_t entry;
	struct page *page;

214
	spin_lock(ptl);
215
216
217
218
219
220
221
222
223
224
	pte = *ptep;
	if (!is_swap_pte(pte))
		goto out;

	entry = pte_to_swp_entry(pte);
	if (!is_migration_entry(entry))
		goto out;

	page = migration_entry_to_page(entry);

Nick Piggin's avatar
Nick Piggin committed
225
226
227
228
229
230
231
232
233
	/*
	 * Once radix-tree replacement of page migration started, page_count
	 * *must* be zero. And, we don't want to call wait_on_page_locked()
	 * against a page without get_page().
	 * So, we use get_page_unless_zero(), here. Even failed, page fault
	 * will occur again.
	 */
	if (!get_page_unless_zero(page))
		goto out;
234
235
236
237
238
239
240
241
	pte_unmap_unlock(ptep, ptl);
	wait_on_page_locked(page);
	put_page(page);
	return;
out:
	pte_unmap_unlock(ptep, ptl);
}

242
243
244
245
246
247
248
249
250
251
252
253
254
255
void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
				unsigned long address)
{
	spinlock_t *ptl = pte_lockptr(mm, pmd);
	pte_t *ptep = pte_offset_map(pmd, address);
	__migration_entry_wait(mm, ptep, ptl);
}

void migration_entry_wait_huge(struct mm_struct *mm, pte_t *pte)
{
	spinlock_t *ptl = &(mm)->page_table_lock;
	__migration_entry_wait(mm, pte, ptl);
}

256
257
#ifdef CONFIG_BLOCK
/* Returns true if all buffers are successfully locked */
258
259
static bool buffer_migrate_lock_buffers(struct buffer_head *head,
							enum migrate_mode mode)
260
261
262
263
{
	struct buffer_head *bh = head;

	/* Simple case, sync compaction */
264
	if (mode != MIGRATE_ASYNC) {
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
		do {
			get_bh(bh);
			lock_buffer(bh);
			bh = bh->b_this_page;

		} while (bh != head);

		return true;
	}

	/* async case, we cannot block on lock_buffer so use trylock_buffer */
	do {
		get_bh(bh);
		if (!trylock_buffer(bh)) {
			/*
			 * We failed to lock the buffer and cannot stall in
			 * async migration. Release the taken locks
			 */
			struct buffer_head *failed_bh = bh;
			put_bh(failed_bh);
			bh = head;
			while (bh != failed_bh) {
				unlock_buffer(bh);
				put_bh(bh);
				bh = bh->b_this_page;
			}
			return false;
		}

		bh = bh->b_this_page;
	} while (bh != head);
	return true;
}
#else
static inline bool buffer_migrate_lock_buffers(struct buffer_head *head,
300
							enum migrate_mode mode)
301
302
303
304
305
{
	return true;
}
#endif /* CONFIG_BLOCK */

306
/*
307
 * Replace the page in the mapping.
308
309
310
311
 *
 * The number of remaining references must be:
 * 1 for anonymous pages without a mapping
 * 2 for pages with a mapping
312
 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
313
 */
314
static int migrate_page_move_mapping(struct address_space *mapping,
315
		struct page *newpage, struct page *page,
316
		struct buffer_head *head, enum migrate_mode mode)
317
{
318
	int expected_count = 0;
319
	void **pslot;
320

321
	if (!mapping) {
322
		/* Anonymous page without mapping */
323
324
		if (page_count(page) != 1)
			return -EAGAIN;
325
		return MIGRATEPAGE_SUCCESS;
326
327
	}

Nick Piggin's avatar
Nick Piggin committed
328
	spin_lock_irq(&mapping->tree_lock);
329

330
331
	pslot = radix_tree_lookup_slot(&mapping->page_tree,
 					page_index(page));
332

333
	expected_count = 2 + page_has_private(page);
Nick Piggin's avatar
Nick Piggin committed
334
	if (page_count(page) != expected_count ||
335
		radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
Nick Piggin's avatar
Nick Piggin committed
336
		spin_unlock_irq(&mapping->tree_lock);
337
		return -EAGAIN;
338
339
	}

Nick Piggin's avatar
Nick Piggin committed
340
	if (!page_freeze_refs(page, expected_count)) {
Nick Piggin's avatar
Nick Piggin committed
341
		spin_unlock_irq(&mapping->tree_lock);
Nick Piggin's avatar
Nick Piggin committed
342
343
344
		return -EAGAIN;
	}

345
346
347
348
349
350
351
	/*
	 * In the async migration case of moving a page with buffers, lock the
	 * buffers using trylock before the mapping is moved. If the mapping
	 * was moved, we later failed to lock the buffers and could not move
	 * the mapping back due to an elevated page count, we would have to
	 * block waiting on other references to be dropped.
	 */
352
353
	if (mode == MIGRATE_ASYNC && head &&
			!buffer_migrate_lock_buffers(head, mode)) {
354
355
356
357
358
		page_unfreeze_refs(page, expected_count);
		spin_unlock_irq(&mapping->tree_lock);
		return -EAGAIN;
	}

359
360
361
	/*
	 * Now we know that no one else is looking at the page.
	 */
362
	get_page(newpage);	/* add cache reference */
363
364
365
366
367
	if (PageSwapCache(page)) {
		SetPageSwapCache(newpage);
		set_page_private(newpage, page_private(page));
	}

368
369
370
	radix_tree_replace_slot(pslot, newpage);

	/*
371
372
	 * Drop cache reference from old page by unfreezing
	 * to one less reference.
373
374
	 * We know this isn't the last reference.
	 */
375
	page_unfreeze_refs(page, expected_count - 1);
376

377
378
379
380
381
382
383
384
385
386
387
388
	/*
	 * If moved to a different zone then also account
	 * the page for that zone. Other VM counters will be
	 * taken care of when we establish references to the
	 * new page and drop references to the old page.
	 *
	 * Note that anonymous pages are accounted for
	 * via NR_FILE_PAGES and NR_ANON_PAGES if they
	 * are mapped to swap space.
	 */
	__dec_zone_page_state(page, NR_FILE_PAGES);
	__inc_zone_page_state(newpage, NR_FILE_PAGES);
389
	if (!PageSwapCache(page) && PageSwapBacked(page)) {
390
391
392
		__dec_zone_page_state(page, NR_SHMEM);
		__inc_zone_page_state(newpage, NR_SHMEM);
	}
Nick Piggin's avatar
Nick Piggin committed
393
	spin_unlock_irq(&mapping->tree_lock);
394

395
	return MIGRATEPAGE_SUCCESS;
396
397
}

398
399
400
401
402
403
404
405
406
407
408
409
410
/*
 * The expected number of remaining references is the same as that
 * of migrate_page_move_mapping().
 */
int migrate_huge_page_move_mapping(struct address_space *mapping,
				   struct page *newpage, struct page *page)
{
	int expected_count;
	void **pslot;

	if (!mapping) {
		if (page_count(page) != 1)
			return -EAGAIN;
411
		return MIGRATEPAGE_SUCCESS;
412
413
414
415
416
417
418
419
420
	}

	spin_lock_irq(&mapping->tree_lock);

	pslot = radix_tree_lookup_slot(&mapping->page_tree,
					page_index(page));

	expected_count = 2 + page_has_private(page);
	if (page_count(page) != expected_count ||
421
		radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) {
422
423
424
425
426
427
428
429
430
431
432
433
434
		spin_unlock_irq(&mapping->tree_lock);
		return -EAGAIN;
	}

	if (!page_freeze_refs(page, expected_count)) {
		spin_unlock_irq(&mapping->tree_lock);
		return -EAGAIN;
	}

	get_page(newpage);

	radix_tree_replace_slot(pslot, newpage);

435
	page_unfreeze_refs(page, expected_count - 1);
436
437

	spin_unlock_irq(&mapping->tree_lock);
438
	return MIGRATEPAGE_SUCCESS;
439
440
}

441
442
443
/*
 * Copy the page to its new location
 */
444
void migrate_page_copy(struct page *newpage, struct page *page)
445
{
446
	if (PageHuge(page) || PageTransHuge(page))
447
448
449
		copy_huge_page(newpage, page);
	else
		copy_highpage(newpage, page);
450
451
452
453
454
455
456

	if (PageError(page))
		SetPageError(newpage);
	if (PageReferenced(page))
		SetPageReferenced(newpage);
	if (PageUptodate(page))
		SetPageUptodate(newpage);
457
458
	if (TestClearPageActive(page)) {
		VM_BUG_ON(PageUnevictable(page));
459
		SetPageActive(newpage);
460
461
	} else if (TestClearPageUnevictable(page))
		SetPageUnevictable(newpage);
462
463
464
465
466
467
468
	if (PageChecked(page))
		SetPageChecked(newpage);
	if (PageMappedToDisk(page))
		SetPageMappedToDisk(newpage);

	if (PageDirty(page)) {
		clear_page_dirty_for_io(page);
469
470
471
472
473
		/*
		 * Want to mark the page and the radix tree as dirty, and
		 * redo the accounting that clear_page_dirty_for_io undid,
		 * but we can't use set_page_dirty because that function
		 * is actually a signal that all of the page has become dirty.
Lucas De Marchi's avatar
Lucas De Marchi committed
474
		 * Whereas only part of our page may be dirty.
475
		 */
476
477
478
479
		if (PageSwapBacked(page))
			SetPageDirty(newpage);
		else
			__set_page_dirty_nobuffers(newpage);
480
481
 	}

482
	mlock_migrate_page(newpage, page);
483
	ksm_migrate_page(newpage, page);
484
485
486
487
	/*
	 * Please do not reorder this without considering how mm/ksm.c's
	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
	 */
488
489
490
491
492
493
494
495
496
497
498
499
	ClearPageSwapCache(page);
	ClearPagePrivate(page);
	set_page_private(page, 0);

	/*
	 * If any waiters have accumulated on the new page then
	 * wake them up.
	 */
	if (PageWriteback(newpage))
		end_page_writeback(newpage);
}

500
501
502
503
504
/************************************************************
 *                    Migration functions
 ***********************************************************/

/* Always fail migration. Used for mappings that are not movable */
505
506
int fail_migrate_page(struct address_space *mapping,
			struct page *newpage, struct page *page)
507
508
509
510
511
{
	return -EIO;
}
EXPORT_SYMBOL(fail_migrate_page);

512
513
/*
 * Common logic to directly migrate a single page suitable for
514
 * pages that do not use PagePrivate/PagePrivate2.
515
516
517
 *
 * Pages are locked upon entry and exit.
 */
518
int migrate_page(struct address_space *mapping,
519
520
		struct page *newpage, struct page *page,
		enum migrate_mode mode)
521
522
523
524
525
{
	int rc;

	BUG_ON(PageWriteback(page));	/* Writeback must be complete */

526
	rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode);
527

528
	if (rc != MIGRATEPAGE_SUCCESS)
529
530
531
		return rc;

	migrate_page_copy(newpage, page);
532
	return MIGRATEPAGE_SUCCESS;
533
534
535
}
EXPORT_SYMBOL(migrate_page);

536
#ifdef CONFIG_BLOCK
537
538
539
540
541
/*
 * Migration function for pages with buffers. This function can only be used
 * if the underlying filesystem guarantees that no other references to "page"
 * exist.
 */
542
int buffer_migrate_page(struct address_space *mapping,
543
		struct page *newpage, struct page *page, enum migrate_mode mode)
544
545
546
547
548
{
	struct buffer_head *bh, *head;
	int rc;

	if (!page_has_buffers(page))
549
		return migrate_page(mapping, newpage, page, mode);
550
551
552

	head = page_buffers(page);

553
	rc = migrate_page_move_mapping(mapping, newpage, page, head, mode);
554

555
	if (rc != MIGRATEPAGE_SUCCESS)
556
557
		return rc;

558
559
560
561
562
	/*
	 * In the async case, migrate_page_move_mapping locked the buffers
	 * with an IRQ-safe spinlock held. In the sync case, the buffers
	 * need to be locked now
	 */
563
564
	if (mode != MIGRATE_ASYNC)
		BUG_ON(!buffer_migrate_lock_buffers(head, mode));
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590

	ClearPagePrivate(page);
	set_page_private(newpage, page_private(page));
	set_page_private(page, 0);
	put_page(page);
	get_page(newpage);

	bh = head;
	do {
		set_bh_page(bh, newpage, bh_offset(bh));
		bh = bh->b_this_page;

	} while (bh != head);

	SetPagePrivate(newpage);

	migrate_page_copy(newpage, page);

	bh = head;
	do {
		unlock_buffer(bh);
 		put_bh(bh);
		bh = bh->b_this_page;

	} while (bh != head);

591
	return MIGRATEPAGE_SUCCESS;
592
593
}
EXPORT_SYMBOL(buffer_migrate_page);
594
#endif
595

596
597
598
599
/*
 * Writeback a page to clean the dirty state
 */
static int writeout(struct address_space *mapping, struct page *page)
600
{
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_NONE,
		.nr_to_write = 1,
		.range_start = 0,
		.range_end = LLONG_MAX,
		.for_reclaim = 1
	};
	int rc;

	if (!mapping->a_ops->writepage)
		/* No write method for the address space */
		return -EINVAL;

	if (!clear_page_dirty_for_io(page))
		/* Someone else already triggered a write */
		return -EAGAIN;

618
	/*
619
620
621
622
623
624
	 * A dirty page may imply that the underlying filesystem has
	 * the page on some queue. So the page must be clean for
	 * migration. Writeout may mean we loose the lock and the
	 * page state is no longer what we checked for earlier.
	 * At this point we know that the migration attempt cannot
	 * be successful.
625
	 */
626
	remove_migration_ptes(page, page);
627

628
	rc = mapping->a_ops->writepage(page, &wbc);
629

630
631
632
633
	if (rc != AOP_WRITEPAGE_ACTIVATE)
		/* unlocked. Relock */
		lock_page(page);

Hugh Dickins's avatar
Hugh Dickins committed
634
	return (rc < 0) ? -EIO : -EAGAIN;
635
636
637
638
639
640
}

/*
 * Default handling if a filesystem does not provide a migration function.
 */
static int fallback_migrate_page(struct address_space *mapping,
641
	struct page *newpage, struct page *page, enum migrate_mode mode)
642
{
643
	if (PageDirty(page)) {
644
645
		/* Only writeback pages in full synchronous migration */
		if (mode != MIGRATE_SYNC)
646
			return -EBUSY;
647
		return writeout(mapping, page);
648
	}
649
650
651
652
653

	/*
	 * Buffers may be managed in a filesystem specific way.
	 * We must have no buffers or drop them.
	 */
654
	if (page_has_private(page) &&
655
656
657
	    !try_to_release_page(page, GFP_KERNEL))
		return -EAGAIN;

658
	return migrate_page(mapping, newpage, page, mode);
659
660
}

661
662
663
664
665
666
/*
 * Move a page to a newly allocated page
 * The page is locked and all ptes have been successfully removed.
 *
 * The new page will have replaced the old page if this function
 * is successful.
667
668
669
 *
 * Return value:
 *   < 0 - error code
670
 *  MIGRATEPAGE_SUCCESS - success
671
 */
672
static int move_to_new_page(struct page *newpage, struct page *page,
673
				int remap_swapcache, enum migrate_mode mode)
674
675
676
677
678
679
680
681
682
{
	struct address_space *mapping;
	int rc;

	/*
	 * Block others from accessing the page when we get around to
	 * establishing additional references. We are the only one
	 * holding a reference to the new page at this point.
	 */
Nick Piggin's avatar
Nick Piggin committed
683
	if (!trylock_page(newpage))
684
685
686
687
688
		BUG();

	/* Prepare mapping for the new page.*/
	newpage->index = page->index;
	newpage->mapping = page->mapping;
689
690
	if (PageSwapBacked(page))
		SetPageSwapBacked(newpage);
691
692
693

	mapping = page_mapping(page);
	if (!mapping)
694
		rc = migrate_page(mapping, newpage, page, mode);
695
	else if (mapping->a_ops->migratepage)
696
		/*
697
698
699
700
		 * Most pages have a mapping and most filesystems provide a
		 * migratepage callback. Anonymous pages are part of swap
		 * space which also has its own migratepage callback. This
		 * is the most common path for page migration.
701
		 */
702
		rc = mapping->a_ops->migratepage(mapping,
703
						newpage, page, mode);
704
	else
705
		rc = fallback_migrate_page(mapping, newpage, page, mode);
706

707
	if (rc != MIGRATEPAGE_SUCCESS) {
708
		newpage->mapping = NULL;
709
710
711
	} else {
		if (remap_swapcache)
			remove_migration_ptes(page, newpage);
712
		page->mapping = NULL;
713
	}
714
715
716
717
718
719

	unlock_page(newpage);

	return rc;
}

720
static int __unmap_and_move(struct page *page, struct page *newpage,
721
				int force, enum migrate_mode mode)
722
{
723
	int rc = -EAGAIN;
724
	int remap_swapcache = 1;
725
	struct mem_cgroup *mem;
726
	struct anon_vma *anon_vma = NULL;
727

Nick Piggin's avatar
Nick Piggin committed
728
	if (!trylock_page(page)) {
729
		if (!force || mode == MIGRATE_ASYNC)
730
			goto out;
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745

		/*
		 * It's not safe for direct compaction to call lock_page.
		 * For example, during page readahead pages are added locked
		 * to the LRU. Later, when the IO completes the pages are
		 * marked uptodate and unlocked. However, the queueing
		 * could be merging multiple pages for one bio (e.g.
		 * mpage_readpages). If an allocation happens for the
		 * second or third page, the process can end up locking
		 * the same page twice and deadlocking. Rather than
		 * trying to be clever about what pages can be locked,
		 * avoid the use of lock_page for direct compaction
		 * altogether.
		 */
		if (current->flags & PF_MEMALLOC)
746
			goto out;
747

748
749
750
		lock_page(page);
	}

751
	/* charge against new page */
752
	mem_cgroup_prepare_migration(page, newpage, &mem);
753

754
	if (PageWriteback(page)) {
755
		/*
756
		 * Only in the case of a full synchronous migration is it
757
758
759
		 * necessary to wait for PageWriteback. In the async case,
		 * the retry loop is too short and in the sync-light case,
		 * the overhead of stalling is too much
760
		 */
761
		if (mode != MIGRATE_SYNC) {
762
763
764
765
			rc = -EBUSY;
			goto uncharge;
		}
		if (!force)
766
			goto uncharge;
767
768
769
		wait_on_page_writeback(page);
	}
	/*
770
771
	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
	 * we cannot notice that anon_vma is freed while we migrates a page.
772
	 * This get_anon_vma() delays freeing anon_vma pointer until the end
773
	 * of migration. File cache pages are no problem because of page_lock()
774
775
	 * File Caches may use write_page() or lock_page() in migration, then,
	 * just care Anon page here.
776
	 */
Hugh Dickins's avatar
Hugh Dickins committed
777
	if (PageAnon(page) && !PageKsm(page)) {
778
		/*
779
		 * Only page_lock_anon_vma_read() understands the subtleties of
780
781
		 * getting a hold on an anon_vma from outside one of its mms.
		 */
782
		anon_vma = page_get_anon_vma(page);
783
784
		if (anon_vma) {
			/*
785
			 * Anon page
786
787
			 */
		} else if (PageSwapCache(page)) {
788
789
790
791
792
793
794
795
796
797
798
799
800
801
			/*
			 * We cannot be sure that the anon_vma of an unmapped
			 * swapcache page is safe to use because we don't
			 * know in advance if the VMA that this page belonged
			 * to still exists. If the VMA and others sharing the
			 * data have been freed, then the anon_vma could
			 * already be invalid.
			 *
			 * To avoid this possibility, swapcache pages get
			 * migrated but are not remapped when migration
			 * completes
			 */
			remap_swapcache = 0;
		} else {
802
			goto uncharge;
803
		}
804
	}
805

806
807
808
809
810
811
812
813
814
815
816
817
	if (unlikely(balloon_page_movable(page))) {
		/*
		 * A ballooned page does not need any special attention from
		 * physical to virtual reverse mapping procedures.
		 * Skip any attempt to unmap PTEs or to remap swap cache,
		 * in order to avoid burning cycles at rmap level, and perform
		 * the page migration right away (proteced by page lock).
		 */
		rc = balloon_page_migrate(newpage, page, mode);
		goto uncharge;
	}

818
	/*
819
820
821
822
823
824
825
826
827
828
	 * Corner case handling:
	 * 1. When a new swap-cache page is read into, it is added to the LRU
	 * and treated as swapcache but it has no rmap yet.
	 * Calling try_to_unmap() against a page->mapping==NULL page will
	 * trigger a BUG.  So handle it here.
	 * 2. An orphaned page (see truncate_complete_page) might have
	 * fs-private metadata. The page can be picked up due to memory
	 * offlining.  Everywhere else except page reclaim, the page is
	 * invisible to the vm, so the page can not be migrated.  So try to
	 * free the metadata, so the page can be freed.
829
	 */
830
	if (!page->mapping) {
831
832
		VM_BUG_ON(PageAnon(page));
		if (page_has_private(page)) {
833
			try_to_free_buffers(page);
834
			goto uncharge;
835
		}
836
		goto skip_unmap;
837
838
	}

839
	/* Establish migration ptes or remove ptes */
840
	try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
841

842
skip_unmap:
843
	if (!page_mapped(page))
844
		rc = move_to_new_page(newpage, page, remap_swapcache, mode);
845

846
	if (rc && remap_swapcache)
847
		remove_migration_ptes(page, page);
848
849

	/* Drop an anon_vma reference if we took one */
850
	if (anon_vma)
851
		put_anon_vma(anon_vma);
852

853
uncharge:
854
855
856
	mem_cgroup_end_migration(mem, page, newpage,
				 (rc == MIGRATEPAGE_SUCCESS ||
				  rc == MIGRATEPAGE_BALLOON_SUCCESS));
857
	unlock_page(page);
858
859
860
out:
	return rc;
}
861

862
863
864
865
866
/*
 * Obtain the lock on page, remove all ptes and migrate the page
 * to the newly allocated page in newpage.
 */
static int unmap_and_move(new_page_t get_new_page, unsigned long private,
867
			struct page *page, int force, enum migrate_mode mode)
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
{
	int rc = 0;
	int *result = NULL;
	struct page *newpage = get_new_page(page, private, &result);

	if (!newpage)
		return -ENOMEM;

	if (page_count(page) == 1) {
		/* page was freed from under us. So we are done. */
		goto out;
	}

	if (unlikely(PageTransHuge(page)))
		if (unlikely(split_huge_page(page)))
			goto out;

885
	rc = __unmap_and_move(page, newpage, force, mode);
886
887
888
889
890
891
892
893
894
895
896
897

	if (unlikely(rc == MIGRATEPAGE_BALLOON_SUCCESS)) {
		/*
		 * A ballooned page has been migrated already.
		 * Now, it's the time to wrap-up counters,
		 * handle the page back to Buddy and return.
		 */
		dec_zone_page_state(page, NR_ISOLATED_ANON +
				    page_is_file_cache(page));
		balloon_page_free(page);
		return MIGRATEPAGE_SUCCESS;
	}
898
out:
899
	if (rc != -EAGAIN) {
900
901
902
903
904
905
906
		/*
		 * A page that has been migrated has all references
		 * removed and will be freed. A page that has not been
		 * migrated will have kepts its references and be
		 * restored.
		 */
		list_del(&page->lru);
907
		dec_zone_page_state(page, NR_ISOLATED_ANON +
908
				page_is_file_cache(page));
909
		putback_lru_page(page);
910
	}
911
912
913
914
	/*
	 * Move the new page to the LRU. If migration was not successful
	 * then this will free the page.
	 */
915
	putback_lru_page(newpage);
916
917
918
919
920
921
	if (result) {
		if (rc)
			*result = rc;
		else
			*result = page_to_nid(newpage);
	}
922
923
924
	return rc;
}

925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
/*
 * Counterpart of unmap_and_move_page() for hugepage migration.
 *
 * This function doesn't wait the completion of hugepage I/O
 * because there is no race between I/O and migration for hugepage.
 * Note that currently hugepage I/O occurs only in direct I/O
 * where no lock is held and PG_writeback is irrelevant,
 * and writeback status of all subpages are counted in the reference
 * count of the head page (i.e. if all subpages of a 2MB hugepage are
 * under direct I/O, the reference of the head page is 512 and a bit more.)
 * This means that when we try to migrate hugepage whose subpages are
 * doing direct I/O, some references remain after try_to_unmap() and
 * hugepage migration fails without data corruption.
 *
 * There is also no race when direct I/O is issued on the page under migration,
 * because then pte is replaced with migration swap entry and direct I/O code
 * will wait in the page fault for migration to complete.
 */
static int unmap_and_move_huge_page(new_page_t get_new_page,
				unsigned long private, struct page *hpage,
945
				int force, enum migrate_mode mode)
946
947
948
949
950
951
{
	int rc = 0;
	int *result = NULL;
	struct page *new_hpage = get_new_page(hpage, private, &result);
	struct anon_vma *anon_vma = NULL;

952
953
954
955
956
957
958
959
960
961
	/*
	 * Movability of hugepages depends on architectures and hugepage size.
	 * This check is necessary because some callers of hugepage migration
	 * like soft offline and memory hotremove don't walk through page
	 * tables or check whether the hugepage is pmd-based or not before
	 * kicking migration.
	 */
	if (!hugepage_migration_support(page_hstate(hpage)))
		return -ENOSYS;

962
963
964
965
966
967
	if (!new_hpage)
		return -ENOMEM;

	rc = -EAGAIN;

	if (!trylock_page(hpage)) {
968
		if (!force || mode != MIGRATE_SYNC)
969
970
971
972
			goto out;
		lock_page(hpage);
	}

973
974
	if (PageAnon(hpage))
		anon_vma = page_get_anon_vma(hpage);
975
976
977
978

	try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);

	if (!page_mapped(hpage))
979
		rc = move_to_new_page(new_hpage, hpage, 1, mode);
980
981
982
983

	if (rc)
		remove_migration_ptes(hpage, hpage);

Hugh Dickins's avatar
Hugh Dickins committed
984
	if (anon_vma)
985
		put_anon_vma(anon_vma);
986
987
988
989

	if (!rc)
		hugetlb_cgroup_migrate(hpage, new_hpage);

990
	unlock_page(hpage);
991
out:
992
993
	if (rc != -EAGAIN)
		putback_active_hugepage(hpage);
994
995
996
997
998
999
1000
1001
1002
1003
	put_page(new_hpage);
	if (result) {
		if (rc)
			*result = rc;
		else
			*result = page_to_nid(new_hpage);
	}
	return rc;
}

1004
/*
1005
1006
 * migrate_pages - migrate the pages specified in a list, to the free pages
 *		   supplied as the target for the page migration
1007
 *
1008
1009
1010
1011
1012
1013
1014
 * @from:		The list of pages to be migrated.
 * @get_new_page:	The function used to allocate free pages to be used
 *			as the target of the page migration.
 * @private:		Private data to be passed on to get_new_page()
 * @mode:		The migration mode that specifies the constraints for
 *			page migration, if any.
 * @reason:		The reason for page migration.
1015
 *
1016
1017
1018
 * The function returns after 10 attempts or if no pages are movable any more
 * because the list has become empty or no retryable pages exist any more.
 * The caller should call putback_lru_pages() to return pages to the LRU
1019
 * or free list only if ret != 0.
1020
 *
1021
 * Returns the number of pages that were not migrated, or an error code.
1022
 */
1023
1024
int migrate_pages(struct list_head *from, new_page_t get_new_page,
		unsigned long private, enum migrate_mode mode, int reason)
1025
{
1026
	int retry = 1;
1027
	int nr_failed = 0;
1028
	int nr_succeeded = 0;
1029
1030
1031
1032
1033
1034
1035
1036
1037
	int pass = 0;
	struct page *page;
	struct page *page2;
	int swapwrite = current->flags & PF_SWAPWRITE;
	int rc;

	if (!swapwrite)
		current->flags |= PF_SWAPWRITE;

1038
1039
	for(pass = 0; pass < 10 && retry; pass++) {
		retry = 0;
1040

1041
1042
		list_for_each_entry_safe(page, page2, from, lru) {
			cond_resched();
1043

1044
1045
1046
1047
1048
			if (PageHuge(page))
				rc = unmap_and_move_huge_page(get_new_page,
						private, page, pass > 2, mode);
			else
				rc = unmap_and_move(get_new_page, private,
1049
						page, pass > 2, mode);
1050

1051
			switch(rc) {
1052
1053
			case -ENOMEM:
				goto out;
1054
			case -EAGAIN:
1055
				retry++;
1056
				break;
1057
			case MIGRATEPAGE_SUCCESS:
1058
				nr_succeeded++;
1059
1060
				break;
			default:
1061
1062
				/* Permanent failure */
				nr_failed++;
1063
				break;
1064
			}
1065
1066
		}
	}
1067
	rc = nr_failed + retry;
1068
out:
1069
1070
1071
1072
	if (nr_succeeded)
		count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
	if (nr_failed)
		count_vm_events(PGMIGRATE_FAIL, nr_failed);
1073
1074
	trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);

1075
1076
1077
	if (!swapwrite)
		current->flags &= ~PF_SWAPWRITE;

1078
	return rc;
1079
}
1080

1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
#ifdef CONFIG_NUMA
/*
 * Move a list of individual pages
 */
struct page_to_node {
	unsigned long addr;
	struct page *page;
	int node;
	int status;
};

static struct page *new_page_node(struct page *p, unsigned long private,
		int **result)
{
	struct page_to_node *pm = (struct page_to_node *)private;

	while (pm->node != MAX_NUMNODES && pm->page != p)
		pm++;

	if (pm->node == MAX_NUMNODES)
		return NULL;

	*result = &pm->status;

1105
1106
1107
1108
1109
	if (PageHuge(p))
		return alloc_huge_page_node(page_hstate(compound_head(p)),
					pm->node);
	else
		return alloc_pages_exact_node(pm->node,
1110
				GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
1111
1112
1113
1114
1115
1116
}

/*
 * Move a set of pages as indicated in the pm array. The addr
 * field must be set to the virtual address of the page to be moved
 * and the node number must contain a valid target node.
1117
 * The pm array ends with node = MAX_NUMNODES.
1118
 */
1119
1120
1121
static int do_move_page_to_node_array(struct mm_struct *mm,
				      struct page_to_node *pm,
				      int migrate_all)
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
{
	int err;
	struct page_to_node *pp;
	LIST_HEAD(pagelist);

	down_read(&mm->mmap_sem);

	/*
	 * Build a list of pages to migrate
	 */
	for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
		struct vm_area_struct *vma;
		struct page *page;

		err = -EFAULT;
		vma = find_vma(mm, pp->addr);
1138
		if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
1139
1140
			goto set_status;

1141
		page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT);
1142
1143
1144
1145
1146

		err = PTR_ERR(page);
		if (IS_ERR(page))
			goto set_status;

1147
1148
1149
1150
		err = -ENOENT;
		if (!page)
			goto set_status;

1151
		/* Use PageReserved to check for zero page */
Hugh Dickins's avatar
Hugh Dickins committed
1152
		if (PageReserved(page))
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
			goto put_and_set;

		pp->page = page;
		err = page_to_nid(page);

		if (err == pp->node)
			/*
			 * Node already in the right place
			 */
			goto put_and_set;

		err = -EACCES;
		if (page_mapcount(page) > 1 &&
				!migrate_all)
			goto put_and_set;

1169
1170
1171
1172
1173
		if (PageHuge(page)) {
			isolate_huge_page(page, &pagelist);
			goto put_and_set;
		}

1174
		err = isolate_lru_page(page);
1175
		if (!err) {
1176
			list_add_tail(&page->lru, &pagelist);
1177
1178
1179
			inc_zone_page_state(page, NR_ISOLATED_ANON +
					    page_is_file_cache(page));
		}
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
put_and_set:
		/*
		 * Either remove the duplicate refcount from
		 * isolate_lru_page() or drop the page ref if it was
		 * not isolated.
		 */
		put_page(page);
set_status:
		pp->status = err;
	}

1191
	err = 0;
1192
	if (!list_empty(&pagelist)) {
1193
		err = migrate_pages(&pagelist, new_page_node,
1194
				(unsigned long)pm, MIGRATE_SYNC, MR_SYSCALL);
1195
		if (err)
1196
			putback_movable_pages(&pagelist);
1197
	}