zsmalloc.c 59.4 KB
Newer Older
1 2 3 4
/*
 * zsmalloc memory allocator
 *
 * Copyright (C) 2011  Nitin Gupta
Minchan Kim's avatar
Minchan Kim committed
5
 * Copyright (C) 2012, 2013 Minchan Kim
6 7 8 9 10 11 12 13
 *
 * This code is released using a dual license strategy: BSD/GPL
 * You can choose the license that better fits your requirements.
 *
 * Released under the terms of 3-clause BSD License
 * Released under the terms of GNU General Public License Version 2.0
 */

Nitin Gupta's avatar
Nitin Gupta committed
14 15 16 17 18
/*
 * Following is how we use various fields and flags of underlying
 * struct page(s) to form a zspage.
 *
 * Usage of struct page fields:
19
 *	page->private: points to zspage
Minchan Kim's avatar
Minchan Kim committed
20 21 22
 *	page->freelist(index): links together all component pages of a zspage
 *		For the huge page, this is always 0, so we use this field
 *		to store handle.
23
 *	page->units: first object offset in a subpage of zspage
Nitin Gupta's avatar
Nitin Gupta committed
24 25 26
 *
 * Usage of struct page flags:
 *	PG_private: identifies the first component page
27
 *	PG_owner_priv_1: identifies the huge component page
Nitin Gupta's avatar
Nitin Gupta committed
28 29 30
 *
 */

31 32
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

33 34
#include <linux/module.h>
#include <linux/kernel.h>
Minchan Kim's avatar
Minchan Kim committed
35
#include <linux/sched.h>
36
#include <linux/magic.h>
37 38 39 40 41 42 43 44 45
#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/highmem.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <asm/tlbflush.h>
#include <asm/pgtable.h>
#include <linux/cpumask.h>
#include <linux/cpu.h>
46
#include <linux/vmalloc.h>
47
#include <linux/preempt.h>
48 49
#include <linux/spinlock.h>
#include <linux/types.h>
50
#include <linux/debugfs.h>
Minchan Kim's avatar
Minchan Kim committed
51
#include <linux/zsmalloc.h>
52
#include <linux/zpool.h>
Minchan Kim's avatar
Minchan Kim committed
53
#include <linux/mount.h>
54
#include <linux/migrate.h>
Minchan Kim's avatar
Minchan Kim committed
55 56 57
#include <linux/pagemap.h>

#define ZSPAGE_MAGIC	0x58
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73

/*
 * This must be power of 2 and greater than of equal to sizeof(link_free).
 * These two conditions ensure that any 'struct link_free' itself doesn't
 * span more than 1 page which avoids complex case of mapping 2 pages simply
 * to restore link_free pointer values.
 */
#define ZS_ALIGN		8

/*
 * A single 'zspage' is composed of up to 2^N discontiguous 0-order (single)
 * pages. ZS_MAX_ZSPAGE_ORDER defines upper limit on N.
 */
#define ZS_MAX_ZSPAGE_ORDER 2
#define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)

74 75
#define ZS_HANDLE_SIZE (sizeof(unsigned long))

76 77
/*
 * Object location (<PFN>, <obj_idx>) is encoded as
Nitin Cupta's avatar
Nitin Cupta committed
78
 * as single (unsigned long) handle value.
79
 *
Minchan Kim's avatar
Minchan Kim committed
80
 * Note that object index <obj_idx> starts from 0.
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96
 *
 * This is made more complicated by various memory models and PAE.
 */

#ifndef MAX_PHYSMEM_BITS
#ifdef CONFIG_HIGHMEM64G
#define MAX_PHYSMEM_BITS 36
#else /* !CONFIG_HIGHMEM64G */
/*
 * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just
 * be PAGE_SHIFT
 */
#define MAX_PHYSMEM_BITS BITS_PER_LONG
#endif
#endif
#define _PFN_BITS		(MAX_PHYSMEM_BITS - PAGE_SHIFT)
Minchan Kim's avatar
Minchan Kim committed
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116

/*
 * Memory for allocating for handle keeps object position by
 * encoding <page, obj_idx> and the encoded value has a room
 * in least bit(ie, look at obj_to_location).
 * We use the bit to synchronize between object access by
 * user and migration.
 */
#define HANDLE_PIN_BIT	0

/*
 * Head in allocated object should have OBJ_ALLOCATED_TAG
 * to identify the object was allocated or not.
 * It's okay to add the status bit in the least bit because
 * header keeps handle which is 4byte-aligned address so we
 * have room for two bit at least.
 */
#define OBJ_ALLOCATED_TAG 1
#define OBJ_TAG_BITS 1
#define OBJ_INDEX_BITS	(BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS)
117 118
#define OBJ_INDEX_MASK	((_AC(1, UL) << OBJ_INDEX_BITS) - 1)

119 120 121 122 123
#define FULLNESS_BITS	2
#define CLASS_BITS	8
#define ISOLATED_BITS	3
#define MAGIC_VAL_BITS	8

124 125 126 127
#define MAX(a, b) ((a) >= (b) ? (a) : (b))
/* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */
#define ZS_MIN_ALLOC_SIZE \
	MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
128
/* each chunk includes extra space to keep handle */
129
#define ZS_MAX_ALLOC_SIZE	PAGE_SIZE
130 131

/*
132
 * On systems with 4K page size, this gives 255 size classes! There is a
133 134 135 136 137 138 139 140 141 142 143
 * trader-off here:
 *  - Large number of size classes is potentially wasteful as free page are
 *    spread across these classes
 *  - Small number of size classes causes large internal fragmentation
 *  - Probably its better to use specific size classes (empirically
 *    determined). NOTE: all those class sizes must be set as multiple of
 *    ZS_ALIGN to make sure link_free itself never has to span 2 pages.
 *
 *  ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN
 *  (reason above)
 */
144
#define ZS_SIZE_CLASS_DELTA	(PAGE_SIZE >> CLASS_BITS)
145 146
#define ZS_SIZE_CLASSES	(DIV_ROUND_UP(ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE, \
				      ZS_SIZE_CLASS_DELTA) + 1)
147 148 149

enum fullness_group {
	ZS_EMPTY,
Minchan Kim's avatar
Minchan Kim committed
150 151 152 153
	ZS_ALMOST_EMPTY,
	ZS_ALMOST_FULL,
	ZS_FULL,
	NR_ZS_FULLNESS,
154 155
};

156
enum zs_stat_type {
Minchan Kim's avatar
Minchan Kim committed
157 158 159 160
	CLASS_EMPTY,
	CLASS_ALMOST_EMPTY,
	CLASS_ALMOST_FULL,
	CLASS_FULL,
161 162
	OBJ_ALLOCATED,
	OBJ_USED,
Minchan Kim's avatar
Minchan Kim committed
163
	NR_ZS_STAT_TYPE,
164 165 166 167 168 169
};

struct zs_size_stat {
	unsigned long objs[NR_ZS_STAT_TYPE];
};

170 171
#ifdef CONFIG_ZSMALLOC_STAT
static struct dentry *zs_stat_root;
172 173
#endif

Minchan Kim's avatar
Minchan Kim committed
174 175 176 177
#ifdef CONFIG_COMPACTION
static struct vfsmount *zsmalloc_mnt;
#endif

178 179 180 181 182
/*
 * We assign a page to ZS_ALMOST_EMPTY fullness group when:
 *	n <= N / f, where
 * n = number of allocated objects
 * N = total number of objects zspage can store
183
 * f = fullness_threshold_frac
184 185 186 187 188 189 190 191 192 193 194
 *
 * Similarly, we assign zspage to:
 *	ZS_ALMOST_FULL	when n > N / f
 *	ZS_EMPTY	when n == 0
 *	ZS_FULL		when n == N
 *
 * (see: fix_fullness_group())
 */
static const int fullness_threshold_frac = 4;

struct size_class {
195
	spinlock_t lock;
Minchan Kim's avatar
Minchan Kim committed
196
	struct list_head fullness_list[NR_ZS_FULLNESS];
197 198 199 200 201
	/*
	 * Size of objects stored in this class. Must be multiple
	 * of ZS_ALIGN.
	 */
	int size;
202
	int objs_per_zspage;
203 204
	/* Number of PAGE_SIZE sized pages to combine to form a 'zspage' */
	int pages_per_zspage;
Minchan Kim's avatar
Minchan Kim committed
205 206 207

	unsigned int index;
	struct zs_size_stat stats;
208 209
};

Minchan Kim's avatar
Minchan Kim committed
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
/* huge object: pages_per_zspage == 1 && maxobj_per_zspage == 1 */
static void SetPageHugeObject(struct page *page)
{
	SetPageOwnerPriv1(page);
}

static void ClearPageHugeObject(struct page *page)
{
	ClearPageOwnerPriv1(page);
}

static int PageHugeObject(struct page *page)
{
	return PageOwnerPriv1(page);
}

226 227
/*
 * Placed within free objects to form a singly linked list.
228
 * For every zspage, zspage->freeobj gives head of this list.
229 230 231 232
 *
 * This must be power of 2 and less than or equal to ZS_ALIGN
 */
struct link_free {
233 234
	union {
		/*
Minchan Kim's avatar
Minchan Kim committed
235
		 * Free object index;
236 237
		 * It's valid for non-allocated object
		 */
Minchan Kim's avatar
Minchan Kim committed
238
		unsigned long next;
239 240 241 242 243
		/*
		 * Handle of allocated object.
		 */
		unsigned long handle;
	};
244 245 246
};

struct zs_pool {
247
	const char *name;
248

249
	struct size_class *size_class[ZS_SIZE_CLASSES];
250
	struct kmem_cache *handle_cachep;
251
	struct kmem_cache *zspage_cachep;
252

253
	atomic_long_t pages_allocated;
254

255
	struct zs_pool_stats stats;
256 257 258 259 260 261 262 263

	/* Compact classes */
	struct shrinker shrinker;
	/*
	 * To signify that register_shrinker() was successful
	 * and unregister_shrinker() will not Oops.
	 */
	bool shrinker_enabled;
264 265 266
#ifdef CONFIG_ZSMALLOC_STAT
	struct dentry *stat_dentry;
#endif
Minchan Kim's avatar
Minchan Kim committed
267 268 269 270
#ifdef CONFIG_COMPACTION
	struct inode *inode;
	struct work_struct free_work;
#endif
271
};
272

273 274 275
struct zspage {
	struct {
		unsigned int fullness:FULLNESS_BITS;
Minchan Kim's avatar
Minchan Kim committed
276
		unsigned int class:CLASS_BITS + 1;
Minchan Kim's avatar
Minchan Kim committed
277 278
		unsigned int isolated:ISOLATED_BITS;
		unsigned int magic:MAGIC_VAL_BITS;
279 280
	};
	unsigned int inuse;
Minchan Kim's avatar
Minchan Kim committed
281
	unsigned int freeobj;
282 283
	struct page *first_page;
	struct list_head list; /* fullness list */
Minchan Kim's avatar
Minchan Kim committed
284 285 286
#ifdef CONFIG_COMPACTION
	rwlock_t lock;
#endif
287
};
288

289
struct mapping_area {
290
#ifdef CONFIG_PGTABLE_MAPPING
291 292 293 294 295 296 297 298
	struct vm_struct *vm; /* vm area for mapping object that span pages */
#else
	char *vm_buf; /* copy buffer for objects that span pages */
#endif
	char *vm_addr; /* address of kmap_atomic()'ed pages */
	enum zs_mapmode vm_mm; /* mapping mode */
};

Minchan Kim's avatar
Minchan Kim committed
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
#ifdef CONFIG_COMPACTION
static int zs_register_migration(struct zs_pool *pool);
static void zs_unregister_migration(struct zs_pool *pool);
static void migrate_lock_init(struct zspage *zspage);
static void migrate_read_lock(struct zspage *zspage);
static void migrate_read_unlock(struct zspage *zspage);
static void kick_deferred_free(struct zs_pool *pool);
static void init_deferred_free(struct zs_pool *pool);
static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
#else
static int zsmalloc_mount(void) { return 0; }
static void zsmalloc_unmount(void) {}
static int zs_register_migration(struct zs_pool *pool) { return 0; }
static void zs_unregister_migration(struct zs_pool *pool) {}
static void migrate_lock_init(struct zspage *zspage) {}
static void migrate_read_lock(struct zspage *zspage) {}
static void migrate_read_unlock(struct zspage *zspage) {}
static void kick_deferred_free(struct zs_pool *pool) {}
static void init_deferred_free(struct zs_pool *pool) {}
static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}
#endif

321
static int create_cache(struct zs_pool *pool)
322 323 324
{
	pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,
					0, 0, NULL);
325 326 327 328 329 330 331 332 333 334 335 336
	if (!pool->handle_cachep)
		return 1;

	pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),
					0, 0, NULL);
	if (!pool->zspage_cachep) {
		kmem_cache_destroy(pool->handle_cachep);
		pool->handle_cachep = NULL;
		return 1;
	}

	return 0;
337 338
}

339
static void destroy_cache(struct zs_pool *pool)
340
{
341
	kmem_cache_destroy(pool->handle_cachep);
342
	kmem_cache_destroy(pool->zspage_cachep);
343 344
}

345
static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp)
346 347
{
	return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
Minchan Kim's avatar
Minchan Kim committed
348
			gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
349 350
}

351
static void cache_free_handle(struct zs_pool *pool, unsigned long handle)
352 353 354 355
{
	kmem_cache_free(pool->handle_cachep, (void *)handle);
}

356 357
static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)
{
Minchan Kim's avatar
Minchan Kim committed
358 359
	return kmem_cache_alloc(pool->zspage_cachep,
			flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE));
360
}
361 362 363 364 365 366

static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
{
	kmem_cache_free(pool->zspage_cachep, zspage);
}

367 368
static void record_obj(unsigned long handle, unsigned long obj)
{
369 370 371 372 373 374
	/*
	 * lsb of @obj represents handle lock while other bits
	 * represent object value the handle is pointing so
	 * updating shouldn't do store tearing.
	 */
	WRITE_ONCE(*(unsigned long *)handle, obj);
375 376
}

377 378 379 380
/* zpool driver */

#ifdef CONFIG_ZPOOL

381
static void *zs_zpool_create(const char *name, gfp_t gfp,
382
			     const struct zpool_ops *zpool_ops,
Dan Streetman's avatar
Dan Streetman committed
383
			     struct zpool *zpool)
384
{
385 386 387 388 389 390
	/*
	 * Ignore global gfp flags: zs_malloc() may be invoked from
	 * different contexts and its caller must provide a valid
	 * gfp mask.
	 */
	return zs_create_pool(name);
391 392 393 394 395 396 397 398 399 400
}

static void zs_zpool_destroy(void *pool)
{
	zs_destroy_pool(pool);
}

static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp,
			unsigned long *handle)
{
401
	*handle = zs_malloc(pool, size, gfp);
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441
	return *handle ? 0 : -1;
}
static void zs_zpool_free(void *pool, unsigned long handle)
{
	zs_free(pool, handle);
}

static int zs_zpool_shrink(void *pool, unsigned int pages,
			unsigned int *reclaimed)
{
	return -EINVAL;
}

static void *zs_zpool_map(void *pool, unsigned long handle,
			enum zpool_mapmode mm)
{
	enum zs_mapmode zs_mm;

	switch (mm) {
	case ZPOOL_MM_RO:
		zs_mm = ZS_MM_RO;
		break;
	case ZPOOL_MM_WO:
		zs_mm = ZS_MM_WO;
		break;
	case ZPOOL_MM_RW: /* fallthru */
	default:
		zs_mm = ZS_MM_RW;
		break;
	}

	return zs_map_object(pool, handle, zs_mm);
}
static void zs_zpool_unmap(void *pool, unsigned long handle)
{
	zs_unmap_object(pool, handle);
}

static u64 zs_zpool_total_size(void *pool)
{
442
	return zs_get_total_pages(pool) << PAGE_SHIFT;
443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
}

static struct zpool_driver zs_zpool_driver = {
	.type =		"zsmalloc",
	.owner =	THIS_MODULE,
	.create =	zs_zpool_create,
	.destroy =	zs_zpool_destroy,
	.malloc =	zs_zpool_malloc,
	.free =		zs_zpool_free,
	.shrink =	zs_zpool_shrink,
	.map =		zs_zpool_map,
	.unmap =	zs_zpool_unmap,
	.total_size =	zs_zpool_total_size,
};

458
MODULE_ALIAS("zpool-zsmalloc");
459 460
#endif /* CONFIG_ZPOOL */

461 462 463
/* per-cpu VM mapping areas for zspage accesses that cross page boundaries */
static DEFINE_PER_CPU(struct mapping_area, zs_map_area);

Minchan Kim's avatar
Minchan Kim committed
464 465 466 467 468
static bool is_zspage_isolated(struct zspage *zspage)
{
	return zspage->isolated;
}

469
static __maybe_unused int is_first_page(struct page *page)
470
{
471
	return PagePrivate(page);
472 473
}

Minchan Kim's avatar
Minchan Kim committed
474
/* Protected by class->lock */
475
static inline int get_zspage_inuse(struct zspage *zspage)
Minchan Kim's avatar
Minchan Kim committed
476
{
477
	return zspage->inuse;
Minchan Kim's avatar
Minchan Kim committed
478 479
}

480
static inline void set_zspage_inuse(struct zspage *zspage, int val)
Minchan Kim's avatar
Minchan Kim committed
481
{
482
	zspage->inuse = val;
Minchan Kim's avatar
Minchan Kim committed
483 484
}

485
static inline void mod_zspage_inuse(struct zspage *zspage, int val)
Minchan Kim's avatar
Minchan Kim committed
486
{
487
	zspage->inuse += val;
Minchan Kim's avatar
Minchan Kim committed
488 489
}

Minchan Kim's avatar
Minchan Kim committed
490
static inline struct page *get_first_page(struct zspage *zspage)
Minchan Kim's avatar
Minchan Kim committed
491
{
Minchan Kim's avatar
Minchan Kim committed
492
	struct page *first_page = zspage->first_page;
493

Minchan Kim's avatar
Minchan Kim committed
494 495
	VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
	return first_page;
Minchan Kim's avatar
Minchan Kim committed
496 497
}

Minchan Kim's avatar
Minchan Kim committed
498
static inline int get_first_obj_offset(struct page *page)
Minchan Kim's avatar
Minchan Kim committed
499
{
Minchan Kim's avatar
Minchan Kim committed
500 501
	return page->units;
}
502

Minchan Kim's avatar
Minchan Kim committed
503 504 505
static inline void set_first_obj_offset(struct page *page, int offset)
{
	page->units = offset;
Minchan Kim's avatar
Minchan Kim committed
506 507
}

Minchan Kim's avatar
Minchan Kim committed
508
static inline unsigned int get_freeobj(struct zspage *zspage)
Minchan Kim's avatar
Minchan Kim committed
509
{
Minchan Kim's avatar
Minchan Kim committed
510
	return zspage->freeobj;
Minchan Kim's avatar
Minchan Kim committed
511 512
}

Minchan Kim's avatar
Minchan Kim committed
513
static inline void set_freeobj(struct zspage *zspage, unsigned int obj)
Minchan Kim's avatar
Minchan Kim committed
514
{
Minchan Kim's avatar
Minchan Kim committed
515
	zspage->freeobj = obj;
Minchan Kim's avatar
Minchan Kim committed
516 517
}

518
static void get_zspage_mapping(struct zspage *zspage,
519
				unsigned int *class_idx,
520 521
				enum fullness_group *fullness)
{
Minchan Kim's avatar
Minchan Kim committed
522 523
	BUG_ON(zspage->magic != ZSPAGE_MAGIC);

524 525
	*fullness = zspage->fullness;
	*class_idx = zspage->class;
526 527
}

528
static void set_zspage_mapping(struct zspage *zspage,
529
				unsigned int class_idx,
530 531
				enum fullness_group fullness)
{
532 533
	zspage->class = class_idx;
	zspage->fullness = fullness;
534 535
}

Nitin Cupta's avatar
Nitin Cupta committed
536 537 538 539 540 541 542
/*
 * zsmalloc divides the pool into various size classes where each
 * class maintains a list of zspages where each zspage is divided
 * into equal sized chunks. Each allocation falls into one of these
 * classes depending on its size. This function returns index of the
 * size class which has chunk size big enough to hold the give size.
 */
543 544 545 546 547 548 549 550
static int get_size_class_index(int size)
{
	int idx = 0;

	if (likely(size > ZS_MIN_ALLOC_SIZE))
		idx = DIV_ROUND_UP(size - ZS_MIN_ALLOC_SIZE,
				ZS_SIZE_CLASS_DELTA);

551
	return min_t(int, ZS_SIZE_CLASSES - 1, idx);
552 553
}

554
/* type can be of enum type zs_stat_type or fullness_group */
Minchan Kim's avatar
Minchan Kim committed
555
static inline void zs_stat_inc(struct size_class *class,
556
				int type, unsigned long cnt)
Minchan Kim's avatar
Minchan Kim committed
557
{
Minchan Kim's avatar
Minchan Kim committed
558
	class->stats.objs[type] += cnt;
Minchan Kim's avatar
Minchan Kim committed
559 560
}

561
/* type can be of enum type zs_stat_type or fullness_group */
Minchan Kim's avatar
Minchan Kim committed
562
static inline void zs_stat_dec(struct size_class *class,
563
				int type, unsigned long cnt)
Minchan Kim's avatar
Minchan Kim committed
564
{
Minchan Kim's avatar
Minchan Kim committed
565
	class->stats.objs[type] -= cnt;
Minchan Kim's avatar
Minchan Kim committed
566 567
}

568
/* type can be of enum type zs_stat_type or fullness_group */
Minchan Kim's avatar
Minchan Kim committed
569
static inline unsigned long zs_stat_get(struct size_class *class,
570
				int type)
Minchan Kim's avatar
Minchan Kim committed
571
{
Minchan Kim's avatar
Minchan Kim committed
572
	return class->stats.objs[type];
Minchan Kim's avatar
Minchan Kim committed
573 574
}

575 576
#ifdef CONFIG_ZSMALLOC_STAT

577
static void __init zs_stat_init(void)
Minchan Kim's avatar
Minchan Kim committed
578
{
579 580 581 582
	if (!debugfs_initialized()) {
		pr_warn("debugfs not available, stat dir not created\n");
		return;
	}
Minchan Kim's avatar
Minchan Kim committed
583 584 585

	zs_stat_root = debugfs_create_dir("zsmalloc", NULL);
	if (!zs_stat_root)
586
		pr_warn("debugfs 'zsmalloc' stat dir creation failed\n");
Minchan Kim's avatar
Minchan Kim committed
587 588 589 590 591 592 593
}

static void __exit zs_stat_exit(void)
{
	debugfs_remove_recursive(zs_stat_root);
}

594 595
static unsigned long zs_can_compact(struct size_class *class);

Minchan Kim's avatar
Minchan Kim committed
596 597 598 599 600 601 602
static int zs_stats_size_show(struct seq_file *s, void *v)
{
	int i;
	struct zs_pool *pool = s->private;
	struct size_class *class;
	int objs_per_zspage;
	unsigned long class_almost_full, class_almost_empty;
603
	unsigned long obj_allocated, obj_used, pages_used, freeable;
Minchan Kim's avatar
Minchan Kim committed
604 605
	unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
	unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
606
	unsigned long total_freeable = 0;
Minchan Kim's avatar
Minchan Kim committed
607

608
	seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
Minchan Kim's avatar
Minchan Kim committed
609 610
			"class", "size", "almost_full", "almost_empty",
			"obj_allocated", "obj_used", "pages_used",
611
			"pages_per_zspage", "freeable");
Minchan Kim's avatar
Minchan Kim committed
612

613
	for (i = 0; i < ZS_SIZE_CLASSES; i++) {
Minchan Kim's avatar
Minchan Kim committed
614 615 616 617 618 619 620 621 622 623
		class = pool->size_class[i];

		if (class->index != i)
			continue;

		spin_lock(&class->lock);
		class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL);
		class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
		obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
		obj_used = zs_stat_get(class, OBJ_USED);
624
		freeable = zs_can_compact(class);
Minchan Kim's avatar
Minchan Kim committed
625 626
		spin_unlock(&class->lock);

627
		objs_per_zspage = class->objs_per_zspage;
Minchan Kim's avatar
Minchan Kim committed
628 629 630
		pages_used = obj_allocated / objs_per_zspage *
				class->pages_per_zspage;

631 632
		seq_printf(s, " %5u %5u %11lu %12lu %13lu"
				" %10lu %10lu %16d %8lu\n",
Minchan Kim's avatar
Minchan Kim committed
633 634
			i, class->size, class_almost_full, class_almost_empty,
			obj_allocated, obj_used, pages_used,
635
			class->pages_per_zspage, freeable);
Minchan Kim's avatar
Minchan Kim committed
636 637 638 639 640 641

		total_class_almost_full += class_almost_full;
		total_class_almost_empty += class_almost_empty;
		total_objs += obj_allocated;
		total_used_objs += obj_used;
		total_pages += pages_used;
642
		total_freeable += freeable;
Minchan Kim's avatar
Minchan Kim committed
643 644 645
	}

	seq_puts(s, "\n");
646
	seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
Minchan Kim's avatar
Minchan Kim committed
647 648
			"Total", "", total_class_almost_full,
			total_class_almost_empty, total_objs,
649
			total_used_objs, total_pages, "", total_freeable);
Minchan Kim's avatar
Minchan Kim committed
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665

	return 0;
}

static int zs_stats_size_open(struct inode *inode, struct file *file)
{
	return single_open(file, zs_stats_size_show, inode->i_private);
}

static const struct file_operations zs_stat_size_ops = {
	.open           = zs_stats_size_open,
	.read           = seq_read,
	.llseek         = seq_lseek,
	.release        = single_release,
};

666
static void zs_pool_stat_create(struct zs_pool *pool, const char *name)
Minchan Kim's avatar
Minchan Kim committed
667 668 669
{
	struct dentry *entry;

670 671
	if (!zs_stat_root) {
		pr_warn("no root stat dir, not creating <%s> stat dir\n", name);
672
		return;
673
	}
Minchan Kim's avatar
Minchan Kim committed
674 675 676 677

	entry = debugfs_create_dir(name, zs_stat_root);
	if (!entry) {
		pr_warn("debugfs dir <%s> creation failed\n", name);
678
		return;
Minchan Kim's avatar
Minchan Kim committed
679 680 681 682 683 684 685 686
	}
	pool->stat_dentry = entry;

	entry = debugfs_create_file("classes", S_IFREG | S_IRUGO,
			pool->stat_dentry, pool, &zs_stat_size_ops);
	if (!entry) {
		pr_warn("%s: debugfs file entry <%s> creation failed\n",
				name, "classes");
687 688
		debugfs_remove_recursive(pool->stat_dentry);
		pool->stat_dentry = NULL;
Minchan Kim's avatar
Minchan Kim committed
689 690 691 692 693 694 695 696 697
	}
}

static void zs_pool_stat_destroy(struct zs_pool *pool)
{
	debugfs_remove_recursive(pool->stat_dentry);
}

#else /* CONFIG_ZSMALLOC_STAT */
698
static void __init zs_stat_init(void)
Minchan Kim's avatar
Minchan Kim committed
699 700 701 702 703 704 705
{
}

static void __exit zs_stat_exit(void)
{
}

706
static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name)
Minchan Kim's avatar
Minchan Kim committed
707 708 709 710 711 712 713 714
{
}

static inline void zs_pool_stat_destroy(struct zs_pool *pool)
{
}
#endif

Minchan Kim's avatar
Minchan Kim committed
715

Nitin Cupta's avatar
Nitin Cupta committed
716 717 718 719 720 721 722
/*
 * For each size class, zspages are divided into different groups
 * depending on how "full" they are. This was done so that we could
 * easily find empty or nearly empty zspages when we try to shrink
 * the pool (not yet implemented). This function returns fullness
 * status of the given page.
 */
723
static enum fullness_group get_fullness_group(struct size_class *class,
724
						struct zspage *zspage)
725
{
726
	int inuse, objs_per_zspage;
727
	enum fullness_group fg;
Minchan Kim's avatar
Minchan Kim committed
728

729
	inuse = get_zspage_inuse(zspage);
730
	objs_per_zspage = class->objs_per_zspage;
731 732 733

	if (inuse == 0)
		fg = ZS_EMPTY;
734
	else if (inuse == objs_per_zspage)
735
		fg = ZS_FULL;
736
	else if (inuse <= 3 * objs_per_zspage / fullness_threshold_frac)
737 738 739 740 741 742 743
		fg = ZS_ALMOST_EMPTY;
	else
		fg = ZS_ALMOST_FULL;

	return fg;
}

Nitin Cupta's avatar
Nitin Cupta committed
744 745 746 747 748 749
/*
 * Each size class maintains various freelists and zspages are assigned
 * to one of these freelists based on the number of live objects they
 * have. This functions inserts the given zspage into the freelist
 * identified by <class, fullness_group>.
 */
750
static void insert_zspage(struct size_class *class,
751 752
				struct zspage *zspage,
				enum fullness_group fullness)
753
{
754
	struct zspage *head;
755

Minchan Kim's avatar
Minchan Kim committed
756
	zs_stat_inc(class, fullness, 1);
757 758
	head = list_first_entry_or_null(&class->fullness_list[fullness],
					struct zspage, list);
759
	/*
760 761
	 * We want to see more ZS_FULL pages and less almost empty/full.
	 * Put pages with higher ->inuse first.
762
	 */
763 764 765 766 767 768 769
	if (head) {
		if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {
			list_add(&zspage->list, &head->list);
			return;
		}
	}
	list_add(&zspage->list, &class->fullness_list[fullness]);
770 771
}

Nitin Cupta's avatar
Nitin Cupta committed
772 773 774 775
/*
 * This function removes the given zspage from the freelist identified
 * by <class, fullness_group>.
 */
776
static void remove_zspage(struct size_class *class,
777 778
				struct zspage *zspage,
				enum fullness_group fullness)
779
{
780
	VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
Minchan Kim's avatar
Minchan Kim committed
781
	VM_BUG_ON(is_zspage_isolated(zspage));
782

783
	list_del_init(&zspage->list);
Minchan Kim's avatar
Minchan Kim committed
784
	zs_stat_dec(class, fullness, 1);
785 786
}

Nitin Cupta's avatar
Nitin Cupta committed
787 788 789 790 791 792 793 794 795
/*
 * Each size class maintains zspages in different fullness groups depending
 * on the number of live objects they contain. When allocating or freeing
 * objects, the fullness status of the page can change, say, from ALMOST_FULL
 * to ALMOST_EMPTY when freeing an object. This function checks if such
 * a status change has occurred for the given page and accordingly moves the
 * page from the freelist of the old fullness group to that of the new
 * fullness group.
 */
796
static enum fullness_group fix_fullness_group(struct size_class *class,
797
						struct zspage *zspage)
798 799 800 801
{
	int class_idx;
	enum fullness_group currfg, newfg;

802 803
	get_zspage_mapping(zspage, &class_idx, &currfg);
	newfg = get_fullness_group(class, zspage);
804 805 806
	if (newfg == currfg)
		goto out;

Minchan Kim's avatar
Minchan Kim committed
807 808 809 810 811
	if (!is_zspage_isolated(zspage)) {
		remove_zspage(class, zspage, currfg);
		insert_zspage(class, zspage, newfg);
	}

812
	set_zspage_mapping(zspage, class_idx, newfg);
813 814 815 816 817 818 819 820 821 822

out:
	return newfg;
}

/*
 * We have to decide on how many pages to link together
 * to form a zspage for each size class. This is important
 * to reduce wastage due to unusable space left at end of
 * each zspage which is given as:
823 824
 *     wastage = Zp % class_size
 *     usage = Zp - wastage
825 826 827 828 829 830
 * where Zp = zspage size = k * PAGE_SIZE where k = 1, 2, ...
 *
 * For example, for size class of 3/8 * PAGE_SIZE, we should
 * link together 3 PAGE_SIZE sized pages to form a zspage
 * since then we can perfectly fit in 8 such objects.
 */
831
static int get_pages_per_zspage(int class_size)
832 833 834 835 836
{
	int i, max_usedpc = 0;
	/* zspage order which gives maximum used size per KB */
	int max_usedpc_order = 1;

837
	for (i = 1; i <= ZS_MAX_PAGES_PER_ZSPAGE; i++) {
838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
		int zspage_size;
		int waste, usedpc;

		zspage_size = i * PAGE_SIZE;
		waste = zspage_size % class_size;
		usedpc = (zspage_size - waste) * 100 / zspage_size;

		if (usedpc > max_usedpc) {
			max_usedpc = usedpc;
			max_usedpc_order = i;
		}
	}

	return max_usedpc_order;
}

854
static struct zspage *get_zspage(struct page *page)
855
{
Minchan Kim's avatar
Minchan Kim committed
856 857 858 859
	struct zspage *zspage = (struct zspage *)page->private;

	BUG_ON(zspage->magic != ZSPAGE_MAGIC);
	return zspage;
860 861 862 863
}

static struct page *get_next_page(struct page *page)
{
Minchan Kim's avatar
Minchan Kim committed
864 865 866 867
	if (unlikely(PageHugeObject(page)))
		return NULL;

	return page->freelist;
868 869
}

Minchan Kim's avatar
Minchan Kim committed
870 871 872 873
/**
 * obj_to_location - get (<page>, <obj_idx>) from encoded object value
 * @page: page object resides in zspage
 * @obj_idx: object index
874
 */
Minchan Kim's avatar
Minchan Kim committed
875 876
static void obj_to_location(unsigned long obj, struct page **page,
				unsigned int *obj_idx)
877
{
Minchan Kim's avatar
Minchan Kim committed
878 879 880 881
	obj >>= OBJ_TAG_BITS;
	*page = pfn_to_page(obj >> OBJ_INDEX_BITS);
	*obj_idx = (obj & OBJ_INDEX_MASK);
}
882

Minchan Kim's avatar
Minchan Kim committed
883 884 885 886 887 888 889 890
/**
 * location_to_obj - get obj value encoded from (<page>, <obj_idx>)
 * @page: page object resides in zspage
 * @obj_idx: object index
 */
static unsigned long location_to_obj(struct page *page, unsigned int obj_idx)
{
	unsigned long obj;
891

Minchan Kim's avatar
Minchan Kim committed
892
	obj = page_to_pfn(page) << OBJ_INDEX_BITS;
Minchan Kim's avatar
Minchan Kim committed
893
	obj |= obj_idx & OBJ_INDEX_MASK;
Minchan Kim's avatar
Minchan Kim committed
894
	obj <<= OBJ_TAG_BITS;
895

Minchan Kim's avatar
Minchan Kim committed
896
	return obj;
897 898
}

899 900 901 902 903
static unsigned long handle_to_obj(unsigned long handle)
{
	return *(unsigned long *)handle;
}

Minchan Kim's avatar
Minchan Kim committed
904
static unsigned long obj_to_head(struct page *page, void *obj)
Minchan Kim's avatar
Minchan Kim committed
905
{
Minchan Kim's avatar
Minchan Kim committed
906
	if (unlikely(PageHugeObject(page))) {
Minchan Kim's avatar
Minchan Kim committed
907
		VM_BUG_ON_PAGE(!is_first_page(page), page);
908
		return page->index;
909 910
	} else
		return *(unsigned long *)obj;
Minchan Kim's avatar
Minchan Kim committed
911 912
}

Minchan Kim's avatar
Minchan Kim committed
913 914 915 916 917
static inline int testpin_tag(unsigned long handle)
{
	return bit_spin_is_locked(HANDLE_PIN_BIT, (unsigned long *)handle);
}

Minchan Kim's avatar
Minchan Kim committed
918 919
static inline int trypin_tag(unsigned long handle)
{
Minchan Kim's avatar
Minchan Kim committed
920
	return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle);
Minchan Kim's avatar
Minchan Kim committed
921 922 923 924
}

static void pin_tag(unsigned long handle)
{
Minchan Kim's avatar
Minchan Kim committed
925
	bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle);
Minchan Kim's avatar
Minchan Kim committed
926 927 928 929
}

static void unpin_tag(unsigned long handle)
{
Minchan Kim's avatar
Minchan Kim committed
930
	bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle);
Minchan Kim's avatar
Minchan Kim committed
931 932
}

933 934
static void reset_page(struct page *page)
{
Minchan Kim's avatar
Minchan Kim committed
935
	__ClearPageMovable(page);
936
	ClearPagePrivate(page);
937
	set_page_private(page, 0);
Minchan Kim's avatar
Minchan Kim committed
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
	page_mapcount_reset(page);
	ClearPageHugeObject(page);
	page->freelist = NULL;
}

/*
 * To prevent zspage destroy during migration, zspage freeing should
 * hold locks of all pages in the zspage.
 */
void lock_zspage(struct zspage *zspage)
{
	struct page *page = get_first_page(zspage);

	do {
		lock_page(page);
	} while ((page = get_next_page(page)) != NULL);
}

int trylock_zspage(struct zspage *zspage)
{
	struct page *cursor, *fail;

	for (cursor = get_first_page(zspage); cursor != NULL; cursor =
					get_next_page(cursor)) {
		if (!trylock_page(cursor)) {
			fail = cursor;
			goto unlock;
		}
	}

	return 1;
unlock:
	for (cursor = get_first_page(zspage); cursor != fail; cursor =
					get_next_page(cursor))
		unlock_page(cursor);

	return 0;
975 976
}

Minchan Kim's avatar
Minchan Kim committed
977 978
static void __free_zspage(struct zs_pool *pool, struct size_class *class,
				struct zspage *zspage)
979
{
980
	struct page *page, *next;
Minchan Kim's avatar
Minchan Kim committed
981 982 983 984 985 986
	enum fullness_group fg;
	unsigned int class_idx;

	get_zspage_mapping(zspage, &class_idx, &fg);

	assert_spin_locked(&class->lock);
987

988
	VM_BUG_ON(get_zspage_inuse(zspage));
Minchan Kim's avatar
Minchan Kim committed
989
	VM_BUG_ON(fg != ZS_EMPTY);
990

Minchan Kim's avatar
Minchan Kim committed
991
	next = page = get_first_page(zspage);
992
	do {
Minchan Kim's avatar
Minchan Kim committed
993 994
		VM_BUG_ON_PAGE(!PageLocked(page), page);
		next = get_next_page(page);
995
		reset_page(page);
Minchan Kim's avatar
Minchan Kim committed
996
		unlock_page(page);
Minchan Kim's avatar
Minchan Kim committed
997
		dec_zone_page_state(page, NR_ZSPAGES);
998 999 1000
		put_page(page);
		page = next;
	} while (page != NULL);
1001

1002
	cache_free_zspage(pool, zspage);
Minchan Kim's avatar
Minchan Kim committed
1003

1004
	zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage);
Minchan Kim's avatar
Minchan Kim committed
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	atomic_long_sub(class->pages_per_zspage,
					&pool->pages_allocated);
}

static void free_zspage(struct zs_pool *pool, struct size_class *class,
				struct zspage *zspage)
{
	VM_BUG_ON(get_zspage_inuse(zspage));
	VM_BUG_ON(list_empty(&zspage->list));

	if (!trylock_zspage(zspage)) {
		kick_deferred_free(pool);
		return;
	}

	remove_zspage(class, zspage, ZS_EMPTY);
	__free_zspage(pool, class, zspage);
1022 1023 1024
}

/* Initialize a newly allocated zspage */
1025
static void init_zspage(struct size_class *class, struct zspage *zspage)
1026
{
Minchan Kim's avatar
Minchan Kim committed
1027
	unsigned int freeobj = 1;
1028
	unsigned long off = 0;
Minchan Kim's avatar
Minchan Kim committed
1029
	struct page *page = get_first_page(zspage);
Minchan Kim's avatar
Minchan Kim committed
1030

1031 1032 1033
	while (page) {
		struct page *next_page;
		struct link_free *link;
1034
		void *vaddr;
1035

1036
		set_first_obj_offset(page, off);
1037

1038 1039
		vaddr = kmap_atomic(page);
		link = (struct link_free *)vaddr + off / sizeof(*link);
1040 1041

		while ((off += class->size) < PAGE_SIZE) {
1042
			link->next = freeobj++ << OBJ_TAG_BITS;
1043
			link += class->size / sizeof(*link);
1044 1045 1046 1047 1048 1049 1050 1051
		}

		/*
		 * We now come to the last (full or partial) object on this
		 * page, which must point to the first object on the next
		 * page (if present)
		 */
		next_page = get_next_page(page);
Minchan Kim's avatar
Minchan Kim committed
1052
		if (next_page) {
1053
			link->next = freeobj++ << OBJ_TAG_BITS;
Minchan Kim's avatar
Minchan Kim committed
1054 1055
		} else {
			/*
1056
			 * Reset OBJ_TAG_BITS bit to last link to tell
Minchan Kim's avatar
Minchan Kim committed
1057 1058
			 * whether it's allocated object or not.
			 */
1059
			link->next = -1 << OBJ_TAG_BITS;
Minchan Kim's avatar
Minchan Kim committed
1060
		}
1061
		kunmap_atomic(vaddr);
1062
		page = next_page;
1063
		off %= PAGE_SIZE;
1064
	}
1065

Minchan Kim's avatar
Minchan Kim committed
1066
	set_freeobj(zspage, 0);
1067 1068
}

Minchan Kim's avatar
Minchan Kim committed
1069 1070
static void create_page_chain(struct size_class *class, struct zspage *zspage,
				struct page *pages[])
1071
{
1072 1073 1074
	int i;
	struct page *page;
	struct page *prev_page = NULL;
Minchan Kim's avatar
Minchan Kim committed
1075
	int nr_pages = class->pages_per_zspage;
1076 1077 1078

	/*
	 * Allocate individual pages and link them together as:
Minchan Kim's avatar
Minchan Kim committed
1079
	 * 1. all pages are linked together using page->freelist
1080
	 * 2. each sub-page point to zspage using page->private
1081
	 *
1082
	 * we set PG_private to identify the first page (i.e. no other sub-page
1083
	 * has this flag set).
1084
	 */
1085 1086
	for (i = 0; i < nr_pages; i++) {
		page = pages[i];
1087
		set_page_private(page, (unsigned long)zspage);
Minchan Kim's avatar
Minchan Kim committed
1088
		page->freelist = NULL;
1089
		if (i == 0) {
1090
			zspage->first_page = page;
1091
			SetPagePrivate(page);
Minchan Kim's avatar
Minchan Kim committed
1092 1093 1094
			if (unlikely(class->objs_per_zspage == 1 &&
					class->pages_per_zspage == 1))
				SetPageHugeObject(page);
1095
		} else {
Minchan Kim's avatar
Minchan Kim committed
1096
			prev_page->freelist = page;
1097 1098 1099
		}
		prev_page = page;
	}
1100
}
1101

1102 1103 1104
/*
 * Allocate a zspage for the given size class
 */
1105 1106 1107
static struct zspage *alloc_zspage(struct zs_pool *pool,
					struct size_class *class,
					gfp_t gfp)
1108 1109 1110
{
	int i;
	struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE];
1111 1112 1113 1114 1115 1116
	struct zspage *zspage = cache_alloc_zspage(pool, gfp);

	if (!zspage)
		return NULL;

	memset(zspage, 0, sizeof(struct zspage));
Minchan Kim's avatar
Minchan Kim committed
1117 1118
	zspage->magic = ZSPAGE_MAGIC;
	migrate_lock_init(zspage);
1119

1120 1121
	for (i = 0; i < class->pages_per_zspage; i++) {
		struct page *page;
1122

1123
		page = alloc_page(gfp);
1124
		if (!page) {
Minchan Kim's avatar
Minchan Kim committed
1125 1126
			while (--i >= 0) {
				dec_zone_page_state(pages[i], NR_ZSPAGES);
1127
				__free_page(pages[i]);
Minchan Kim's avatar
Minchan Kim committed
1128
			}
1129
			cache_free_zspage(pool, zspage);
1130 1131
			return NULL;
		}
Minchan Kim's avatar
Minchan Kim committed
1132 1133

		inc_zone_page_state(page, NR_ZSPAGES);
1134
		pages[i] = page;
1135 1136
	}

Minchan Kim's avatar
Minchan Kim committed
1137
	create_page_chain(class, zspage, pages);
1138
	init_zspage(class, zspage);
1139

1140
	return zspage;
1141 1142
}

1143
static struct zspage *find_get_zspage(struct size_class *class)
1144 1145
{
	int i;
1146
	struct zspage *zspage;
1147

Minchan Kim's avatar
Minchan Kim committed
1148
	for (i = ZS_ALMOST_FULL; i >= ZS_EMPTY; i--) {
1149 1150 1151
		zspage = list_first_entry_or_null(&class->fullness_list[i],
				struct zspage, list);
		if (zspage)
1152 1153 1154
			break;
	}

1155
	return zspage;
1156 1157
}

1158
#ifdef CONFIG_PGTABLE_MAPPING
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
static inline int __zs_cpu_up(struct mapping_area *area)
{
	/*
	 * Make sure we don't leak memory if a cpu UP notification
	 * and zs_init() race and both call zs_cpu_up() on the same cpu
	 */
	if (area->vm)
		return 0;
	area->vm = alloc_vm_area(PAGE_SIZE * 2, NULL);
	if (!area->vm)
		return -ENOMEM;
	return 0;
}

static inline void __zs_cpu_down(struct mapping_area *area)
{
	if (area->vm)
		free_vm_area(area->vm);
	area->vm = NULL;
}

static inline void *__zs_map_object(struct mapping_area *area,
				struct page *pages[2], int off, int size)
{
1183
	BUG_ON(map_vm_area(area->vm, PAGE_KERNEL, pages));
1184 1185 1186 1187 1188 1189 1190 1191 1192
	area->vm_addr = area->vm->addr;
	return area->vm_addr + off;
}

static inline void __zs_unmap_object(struct mapping_area *area,
				struct page *pages[2], int off, int size)
{
	unsigned long addr = (unsigned long)area->vm_addr;

1193
	unmap_kernel_range(addr, PAGE_SIZE * 2);
1194 1195
}

1196
#else /* CONFIG_PGTABLE_MAPPING */