Commit dab4ead1 authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds

mm, page_owner: make init_pages_in_zone() faster

In init_pages_in_zone() we currently use the generic set_page_owner()
function to initialize page_owner info for early allocated pages.  This
means we needlessly do lookup_page_ext() twice for each page, and more
importantly save_stack(), which has to unwind the stack and find the
corresponding stack depot handle.  Because the stack is always the same
for the initialization, unwind it once in init_pages_in_zone() and reuse
the handle.  Also avoid the repeated lookup_page_ext().

This can significantly reduce boot times with page_owner=on on large
machines, especially for kernels built without frame pointer, where the
stack unwinding is noticeably slower.

[ don't duplicate code of __set_page_owner(), per Michal Hocko]
[ coding-style fixes]
[ create statically allocated fake stack trace for early allocated pages, per Michal]
Link: default avatarVlastimil Babka <>
Acked-by: default avatarMichal Hocko <>
Cc: Joonsoo Kim <>
Cc: Mel Gorman <>
Cc: Yang Shi <>
Cc: Laura Abbott <>
Cc: Vinayak Menon <>
Cc: zhong jiang <>
Signed-off-by: default avatarAndrew Morton <>
Signed-off-by: default avatarLinus Torvalds <>
parent b95046b0
......@@ -30,6 +30,7 @@ DEFINE_STATIC_KEY_FALSE(page_owner_inited);
static depot_stack_handle_t dummy_handle;
static depot_stack_handle_t failure_handle;
static depot_stack_handle_t early_handle;
static void init_early_allocated_pages(void);
......@@ -53,7 +54,7 @@ static bool need_page_owner(void)
return true;
static noinline void register_dummy_stack(void)
static __always_inline depot_stack_handle_t create_dummy_stack(void)
unsigned long entries[4];
struct stack_trace dummy;
......@@ -64,21 +65,22 @@ static noinline void register_dummy_stack(void)
dummy.skip = 0;
dummy_handle = depot_save_stack(&dummy, GFP_KERNEL);
return depot_save_stack(&dummy, GFP_KERNEL);
static noinline void register_failure_stack(void)
static noinline void register_dummy_stack(void)
unsigned long entries[4];
struct stack_trace failure;
dummy_handle = create_dummy_stack();
failure.nr_entries = 0;
failure.max_entries = ARRAY_SIZE(entries);
failure.entries = &entries[0];
failure.skip = 0;
static noinline void register_failure_stack(void)
failure_handle = create_dummy_stack();
failure_handle = depot_save_stack(&failure, GFP_KERNEL);
static noinline void register_early_stack(void)
early_handle = create_dummy_stack();
static void init_page_owner(void)
......@@ -88,6 +90,7 @@ static void init_page_owner(void)
......@@ -165,17 +168,13 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags)
return handle;
noinline void __set_page_owner(struct page *page, unsigned int order,
gfp_t gfp_mask)
static inline void __set_page_owner_handle(struct page_ext *page_ext,
depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask)
struct page_ext *page_ext = lookup_page_ext(page);
struct page_owner *page_owner;
if (unlikely(!page_ext))
page_owner = get_page_owner(page_ext);
page_owner->handle = save_stack(gfp_mask);
page_owner->handle = handle;
page_owner->order = order;
page_owner->gfp_mask = gfp_mask;
page_owner->last_migrate_reason = -1;
......@@ -183,6 +182,19 @@ noinline void __set_page_owner(struct page *page, unsigned int order,
__set_bit(PAGE_EXT_OWNER, &page_ext->flags);
noinline void __set_page_owner(struct page *page, unsigned int order,
gfp_t gfp_mask)
struct page_ext *page_ext = lookup_page_ext(page);
depot_stack_handle_t handle;
if (unlikely(!page_ext))
handle = save_stack(gfp_mask);
__set_page_owner_handle(page_ext, handle, order, gfp_mask);
void __set_page_owner_migrate_reason(struct page *page, int reason)
struct page_ext *page_ext = lookup_page_ext(page);
......@@ -565,12 +577,12 @@ static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
if (unlikely(!page_ext))
/* Maybe overraping zone */
/* Maybe overlapping zone */
if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
/* Found early allocated page */
set_page_owner(page, 0, 0);
__set_page_owner_handle(page_ext, early_handle, 0, 0);
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment