hugetlbpage.c 5.75 KB
Newer Older
James Hogan's avatar
James Hogan committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
/*
 * arch/metag/mm/hugetlbpage.c
 *
 * METAG HugeTLB page support.
 *
 * Cloned from SuperH
 *
 * Cloned from sparc64 by Paul Mundt.
 *
 * Copyright (C) 2002, 2003 David S. Miller (davem@redhat.com)
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/sysctl.h>

#include <asm/mman.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>

/*
 * If the arch doesn't supply something else, assume that hugepage
 * size aligned regions are ok without further preparation.
 */
int prepare_hugepage_range(struct file *file, unsigned long addr,
						unsigned long len)
{
	struct mm_struct *mm = current->mm;
	struct hstate *h = hstate_file(file);
	struct vm_area_struct *vma;

	if (len & ~huge_page_mask(h))
		return -EINVAL;
	if (addr & ~huge_page_mask(h))
		return -EINVAL;
	if (TASK_SIZE - len < addr)
		return -EINVAL;

	vma = find_vma(mm, ALIGN_HUGEPT(addr));
	if (vma && !(vma->vm_flags & MAP_HUGETLB))
		return -EINVAL;

	vma = find_vma(mm, addr);
	if (vma) {
		if (addr + len > vma->vm_start)
			return -EINVAL;
		if (!(vma->vm_flags & MAP_HUGETLB) &&
		    (ALIGN_HUGEPT(addr + len) > vma->vm_start))
			return -EINVAL;
	}
	return 0;
}

pte_t *huge_pte_alloc(struct mm_struct *mm,
			unsigned long addr, unsigned long sz)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	pgd = pgd_offset(mm, addr);
	pud = pud_offset(pgd, addr);
	pmd = pmd_offset(pud, addr);
	pte = pte_alloc_map(mm, NULL, pmd, addr);
	pgd->pgd &= ~_PAGE_SZ_MASK;
	pgd->pgd |= _PAGE_SZHUGE;

	return pte;
}

pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte = NULL;

	pgd = pgd_offset(mm, addr);
	pud = pud_offset(pgd, addr);
	pmd = pmd_offset(pud, addr);
	pte = pte_offset_kernel(pmd, addr);

	return pte;
}

int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
{
	return 0;
}

struct page *follow_huge_addr(struct mm_struct *mm,
			      unsigned long address, int write)
{
	return ERR_PTR(-EINVAL);
}

int pmd_huge(pmd_t pmd)
{
	return pmd_page_shift(pmd) > PAGE_SHIFT;
}

int pud_huge(pud_t pud)
{
	return 0;
}

113
114
115
116
117
int pmd_huge_support(void)
{
	return 1;
}

James Hogan's avatar
James Hogan committed
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
			     pmd_t *pmd, int write)
{
	return NULL;
}

#ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA

/*
 * Look for an unmapped area starting after another hugetlb vma.
 * There are guaranteed to be no huge pte's spare if all the huge pages are
 * full size (4MB), so in that case compile out this search.
 */
#if HPAGE_SHIFT == HUGEPT_SHIFT
static inline unsigned long
hugetlb_get_unmapped_area_existing(unsigned long len)
{
	return 0;
}
#else
static unsigned long
hugetlb_get_unmapped_area_existing(unsigned long len)
{
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;
	unsigned long start_addr, addr;
	int after_huge;

	if (mm->context.part_huge) {
		start_addr = mm->context.part_huge;
		after_huge = 1;
	} else {
		start_addr = TASK_UNMAPPED_BASE;
		after_huge = 0;
	}
new_search:
	addr = start_addr;

	for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
		if ((!vma && !after_huge) || TASK_SIZE - len < addr) {
			/*
			 * Start a new search - just in case we missed
			 * some holes.
			 */
			if (start_addr != TASK_UNMAPPED_BASE) {
				start_addr = TASK_UNMAPPED_BASE;
				goto new_search;
			}
			return 0;
		}
		/* skip ahead if we've aligned right over some vmas */
		if (vma && vma->vm_end <= addr)
			continue;
		/* space before the next vma? */
		if (after_huge && (!vma || ALIGN_HUGEPT(addr + len)
			    <= vma->vm_start)) {
			unsigned long end = addr + len;
			if (end & HUGEPT_MASK)
				mm->context.part_huge = end;
			else if (addr == mm->context.part_huge)
				mm->context.part_huge = 0;
			return addr;
		}
		if (vma && (vma->vm_flags & MAP_HUGETLB)) {
			/* space after a huge vma in 2nd level page table? */
			if (vma->vm_end & HUGEPT_MASK) {
				after_huge = 1;
				/* no need to align to the next PT block */
				addr = vma->vm_end;
				continue;
			}
		}
		after_huge = 0;
		addr = ALIGN_HUGEPT(vma->vm_end);
	}
}
#endif

/* Do a full search to find an area without any nearby normal pages. */
static unsigned long
hugetlb_get_unmapped_area_new_pmd(unsigned long len)
{
200
201
202
203
204
205
206
207
208
	struct vm_unmapped_area_info info;

	info.flags = 0;
	info.length = len;
	info.low_limit = TASK_UNMAPPED_BASE;
	info.high_limit = TASK_SIZE;
	info.align_mask = PAGE_MASK & HUGEPT_MASK;
	info.align_offset = 0;
	return vm_unmapped_area(&info);
James Hogan's avatar
James Hogan committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
}

unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);

	if (len & ~huge_page_mask(h))
		return -EINVAL;
	if (len > TASK_SIZE)
		return -ENOMEM;

	if (flags & MAP_FIXED) {
		if (prepare_hugepage_range(file, addr, len))
			return -EINVAL;
		return addr;
	}

	if (addr) {
		addr = ALIGN(addr, huge_page_size(h));
		if (!prepare_hugepage_range(file, addr, len))
			return addr;
	}

	/*
	 * Look for an existing hugetlb vma with space after it (this is to to
	 * minimise fragmentation caused by huge pages.
	 */
	addr = hugetlb_get_unmapped_area_existing(len);
	if (addr)
		return addr;

	/*
	 * Find an unmapped naturally aligned set of 4MB blocks that we can use
	 * for huge pages.
	 */
246
	return hugetlb_get_unmapped_area_new_pmd(len);
James Hogan's avatar
James Hogan committed
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
}

#endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/

/* necessary for boot time 4MB huge page allocation */
static __init int setup_hugepagesz(char *opt)
{
	unsigned long ps = memparse(opt, &opt);
	if (ps == (1 << HPAGE_SHIFT)) {
		hugetlb_add_hstate(HPAGE_SHIFT - PAGE_SHIFT);
	} else {
		pr_err("hugepagesz: Unsupported page size %lu M\n",
		       ps >> 20);
		return 0;
	}
	return 1;
}
__setup("hugepagesz=", setup_hugepagesz);