Commit 0edaecfa authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Linus Torvalds
Browse files

hugetlb: add a list for tracking in-use HugeTLB pages



hugepage_activelist will be used to track currently used HugeTLB pages.
We need to find the in-use HugeTLB pages to support HugeTLB cgroup removal.
On cgroup removal we update the page's HugeTLB cgroup to point to parent
cgroup.
Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Reviewed-by: default avatarKAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Hillf Danton <dhillf@gmail.com>
Reviewed-by: default avatarMichal Hocko <mhocko@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 189ebff2
...@@ -211,6 +211,7 @@ struct hstate { ...@@ -211,6 +211,7 @@ struct hstate {
unsigned long resv_huge_pages; unsigned long resv_huge_pages;
unsigned long surplus_huge_pages; unsigned long surplus_huge_pages;
unsigned long nr_overcommit_huge_pages; unsigned long nr_overcommit_huge_pages;
struct list_head hugepage_activelist;
struct list_head hugepage_freelists[MAX_NUMNODES]; struct list_head hugepage_freelists[MAX_NUMNODES];
unsigned int nr_huge_pages_node[MAX_NUMNODES]; unsigned int nr_huge_pages_node[MAX_NUMNODES];
unsigned int free_huge_pages_node[MAX_NUMNODES]; unsigned int free_huge_pages_node[MAX_NUMNODES];
......
...@@ -510,7 +510,7 @@ void copy_huge_page(struct page *dst, struct page *src) ...@@ -510,7 +510,7 @@ void copy_huge_page(struct page *dst, struct page *src)
static void enqueue_huge_page(struct hstate *h, struct page *page) static void enqueue_huge_page(struct hstate *h, struct page *page)
{ {
int nid = page_to_nid(page); int nid = page_to_nid(page);
list_add(&page->lru, &h->hugepage_freelists[nid]); list_move(&page->lru, &h->hugepage_freelists[nid]);
h->free_huge_pages++; h->free_huge_pages++;
h->free_huge_pages_node[nid]++; h->free_huge_pages_node[nid]++;
} }
...@@ -522,7 +522,7 @@ static struct page *dequeue_huge_page_node(struct hstate *h, int nid) ...@@ -522,7 +522,7 @@ static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
if (list_empty(&h->hugepage_freelists[nid])) if (list_empty(&h->hugepage_freelists[nid]))
return NULL; return NULL;
page = list_entry(h->hugepage_freelists[nid].next, struct page, lru); page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
list_del(&page->lru); list_move(&page->lru, &h->hugepage_activelist);
set_page_refcounted(page); set_page_refcounted(page);
h->free_huge_pages--; h->free_huge_pages--;
h->free_huge_pages_node[nid]--; h->free_huge_pages_node[nid]--;
...@@ -626,10 +626,11 @@ static void free_huge_page(struct page *page) ...@@ -626,10 +626,11 @@ static void free_huge_page(struct page *page)
page->mapping = NULL; page->mapping = NULL;
BUG_ON(page_count(page)); BUG_ON(page_count(page));
BUG_ON(page_mapcount(page)); BUG_ON(page_mapcount(page));
INIT_LIST_HEAD(&page->lru);
spin_lock(&hugetlb_lock); spin_lock(&hugetlb_lock);
if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) { if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
/* remove the page from active list */
list_del(&page->lru);
update_and_free_page(h, page); update_and_free_page(h, page);
h->surplus_huge_pages--; h->surplus_huge_pages--;
h->surplus_huge_pages_node[nid]--; h->surplus_huge_pages_node[nid]--;
...@@ -642,6 +643,7 @@ static void free_huge_page(struct page *page) ...@@ -642,6 +643,7 @@ static void free_huge_page(struct page *page)
static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
{ {
INIT_LIST_HEAD(&page->lru);
set_compound_page_dtor(page, free_huge_page); set_compound_page_dtor(page, free_huge_page);
spin_lock(&hugetlb_lock); spin_lock(&hugetlb_lock);
h->nr_huge_pages++; h->nr_huge_pages++;
...@@ -890,6 +892,7 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, int nid) ...@@ -890,6 +892,7 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
spin_lock(&hugetlb_lock); spin_lock(&hugetlb_lock);
if (page) { if (page) {
INIT_LIST_HEAD(&page->lru);
r_nid = page_to_nid(page); r_nid = page_to_nid(page);
set_compound_page_dtor(page, free_huge_page); set_compound_page_dtor(page, free_huge_page);
/* /*
...@@ -994,7 +997,6 @@ static int gather_surplus_pages(struct hstate *h, int delta) ...@@ -994,7 +997,6 @@ static int gather_surplus_pages(struct hstate *h, int delta)
list_for_each_entry_safe(page, tmp, &surplus_list, lru) { list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
if ((--needed) < 0) if ((--needed) < 0)
break; break;
list_del(&page->lru);
/* /*
* This page is now managed by the hugetlb allocator and has * This page is now managed by the hugetlb allocator and has
* no users -- drop the buddy allocator's reference. * no users -- drop the buddy allocator's reference.
...@@ -1009,7 +1011,6 @@ static int gather_surplus_pages(struct hstate *h, int delta) ...@@ -1009,7 +1011,6 @@ static int gather_surplus_pages(struct hstate *h, int delta)
/* Free unnecessary surplus pages to the buddy allocator */ /* Free unnecessary surplus pages to the buddy allocator */
if (!list_empty(&surplus_list)) { if (!list_empty(&surplus_list)) {
list_for_each_entry_safe(page, tmp, &surplus_list, lru) { list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
list_del(&page->lru);
put_page(page); put_page(page);
} }
} }
...@@ -1909,6 +1910,7 @@ void __init hugetlb_add_hstate(unsigned order) ...@@ -1909,6 +1910,7 @@ void __init hugetlb_add_hstate(unsigned order)
h->free_huge_pages = 0; h->free_huge_pages = 0;
for (i = 0; i < MAX_NUMNODES; ++i) for (i = 0; i < MAX_NUMNODES; ++i)
INIT_LIST_HEAD(&h->hugepage_freelists[i]); INIT_LIST_HEAD(&h->hugepage_freelists[i]);
INIT_LIST_HEAD(&h->hugepage_activelist);
h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]); h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]); h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment