memcontrol.h 32.5 KB
Newer Older
1
2
3
4
5
/* memcontrol.h - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
6
7
8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
 *
9
10
11
12
13
14
15
16
17
18
19
20
21
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#ifndef _LINUX_MEMCONTROL_H
#define _LINUX_MEMCONTROL_H
22
#include <linux/cgroup.h>
23
#include <linux/vm_event_item.h>
24
#include <linux/hardirq.h>
25
#include <linux/jump_label.h>
Michal Hocko's avatar
Michal Hocko committed
26
27
28
#include <linux/page_counter.h>
#include <linux/vmpressure.h>
#include <linux/eventfd.h>
29
30
#include <linux/mm.h>
#include <linux/vmstat.h>
Michal Hocko's avatar
Michal Hocko committed
31
#include <linux/writeback.h>
32
#include <linux/page-flags.h>
33

34
struct mem_cgroup;
35
36
struct page;
struct mm_struct;
37
struct kmem_cache;
38

39
40
41
42
43
44
45
46
47
/* Cgroup-specific page state, on top of universal node page state */
enum memcg_stat_item {
	MEMCG_CACHE = NR_VM_NODE_STAT_ITEMS,
	MEMCG_RSS,
	MEMCG_RSS_HUGE,
	MEMCG_SWAP,
	MEMCG_SOCK,
	/* XXX: why are these zone and not node counters? */
	MEMCG_KERNEL_STACK_KB,
48
	MEMCG_NR_STAT,
49
50
};

51
52
enum memcg_memory_event {
	MEMCG_LOW,
53
54
55
	MEMCG_HIGH,
	MEMCG_MAX,
	MEMCG_OOM,
56
	MEMCG_OOM_KILL,
57
58
	MEMCG_SWAP_MAX,
	MEMCG_SWAP_FAIL,
59
	MEMCG_NR_MEMORY_EVENTS,
60
61
};

Roman Gushchin's avatar
Roman Gushchin committed
62
63
64
65
66
67
enum mem_cgroup_protection {
	MEMCG_PROT_NONE,
	MEMCG_PROT_LOW,
	MEMCG_PROT_MIN,
};

68
struct mem_cgroup_reclaim_cookie {
69
	pg_data_t *pgdat;
70
71
72
73
	int priority;
	unsigned int generation;
};

74
75
76
77
78
79
80
#ifdef CONFIG_MEMCG

#define MEM_CGROUP_ID_SHIFT	16
#define MEM_CGROUP_ID_MAX	USHRT_MAX

struct mem_cgroup_id {
	int id;
81
	refcount_t ref;
82
83
};

Michal Hocko's avatar
Michal Hocko committed
84
85
86
87
88
89
90
91
92
93
94
95
96
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
	MEM_CGROUP_TARGET_NUMAINFO,
	MEM_CGROUP_NTARGETS,
};

97
98
struct memcg_vmstats_percpu {
	long stat[MEMCG_NR_STAT];
99
	unsigned long events[NR_VM_EVENT_ITEMS];
Michal Hocko's avatar
Michal Hocko committed
100
101
102
103
104
105
106
107
108
109
	unsigned long nr_page_events;
	unsigned long targets[MEM_CGROUP_NTARGETS];
};

struct mem_cgroup_reclaim_iter {
	struct mem_cgroup *position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

110
111
112
113
struct lruvec_stat {
	long count[NR_VM_NODE_STAT_ITEMS];
};

114
115
116
117
118
119
120
121
122
/*
 * Bitmap of shrinker::id corresponding to memcg-aware shrinkers,
 * which have elements charged to this memcg.
 */
struct memcg_shrinker_map {
	struct rcu_head rcu;
	unsigned long map[0];
};

Michal Hocko's avatar
Michal Hocko committed
123
124
125
/*
 * per-zone information in memory controller.
 */
126
struct mem_cgroup_per_node {
Michal Hocko's avatar
Michal Hocko committed
127
	struct lruvec		lruvec;
128
129
130

	struct lruvec_stat __percpu *lruvec_stat_cpu;
	atomic_long_t		lruvec_stat[NR_VM_NODE_STAT_ITEMS];
131
	atomic_long_t		lruvec_stat_local[NR_VM_NODE_STAT_ITEMS];
132

133
	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
Michal Hocko's avatar
Michal Hocko committed
134
135
136

	struct mem_cgroup_reclaim_iter	iter[DEF_PRIORITY + 1];

137
138
139
#ifdef CONFIG_MEMCG_KMEM
	struct memcg_shrinker_map __rcu	*shrinker_map;
#endif
Michal Hocko's avatar
Michal Hocko committed
140
141
142
143
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long		usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
144
145
146
	bool			congested;	/* memcg has many dirty pages */
						/* backed by a congested BDI */

Michal Hocko's avatar
Michal Hocko committed
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
						/* use container_of	   */
};

struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	unsigned long threshold;
};

/* For threshold */
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below or equal to usage. */
	int current_threshold;
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

177
178
179
180
181
182
enum memcg_kmem_state {
	KMEM_NONE,
	KMEM_ALLOCATED,
	KMEM_ONLINE,
};

183
184
185
186
187
188
189
190
191
#if defined(CONFIG_SMP)
struct memcg_padding {
	char x[0];
} ____cacheline_internodealigned_in_smp;
#define MEMCG_PADDING(name)      struct memcg_padding name;
#else
#define MEMCG_PADDING(name)
#endif

Michal Hocko's avatar
Michal Hocko committed
192
193
194
195
196
197
198
199
200
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;

201
202
203
	/* Private memcg ID. Used to ID objects that outlive the cgroup */
	struct mem_cgroup_id id;

Michal Hocko's avatar
Michal Hocko committed
204
205
	/* Accounted resources */
	struct page_counter memory;
206
	struct page_counter swap;
207
208

	/* Legacy consumer-oriented counters */
Michal Hocko's avatar
Michal Hocko committed
209
210
	struct page_counter memsw;
	struct page_counter kmem;
211
	struct page_counter tcpmem;
Michal Hocko's avatar
Michal Hocko committed
212

213
	/* Upper bound of normal memory consumption range */
Michal Hocko's avatar
Michal Hocko committed
214
215
	unsigned long high;

216
217
218
	/* Range enforcement for interrupt charges */
	struct work_struct high_work;

Michal Hocko's avatar
Michal Hocko committed
219
220
221
222
223
224
225
226
227
228
	unsigned long soft_limit;

	/* vmpressure notifications */
	struct vmpressure vmpressure;

	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;

229
230
231
232
233
	/*
	 * Should the OOM killer kill all belonging tasks, had it kill one?
	 */
	bool oom_group;

Michal Hocko's avatar
Michal Hocko committed
234
235
236
237
238
239
240
241
	/* protected by memcg_oom_lock */
	bool		oom_lock;
	int		under_oom;

	int	swappiness;
	/* OOM-Killer disable */
	int		oom_kill_disable;

242
	/* memory.events */
243
244
	struct cgroup_file events_file;

245
246
247
	/* handle for "memory.swap.events" */
	struct cgroup_file swap_events_file;

Michal Hocko's avatar
Michal Hocko committed
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
	struct mem_cgroup_thresholds thresholds;

	/* thresholds for mem+swap usage. RCU-protected */
	struct mem_cgroup_thresholds memsw_thresholds;

	/* For oom notifier event fd */
	struct list_head oom_notify;

	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long move_charge_at_immigrate;
265
266
267
268
269
270
	/* taken only while moving_account > 0 */
	spinlock_t		move_lock;
	unsigned long		move_lock_flags;

	MEMCG_PADDING(_pad1_);

Michal Hocko's avatar
Michal Hocko committed
271
272
273
274
275
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t		moving_account;
	struct task_struct	*move_lock_task;
276

277
	/* memory.stat */
278
	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
279
280
281

	MEMCG_PADDING(_pad2_);

282
	atomic_long_t		vmstats[MEMCG_NR_STAT];
283
284
	atomic_long_t		vmstats_local[MEMCG_NR_STAT];

285
	atomic_long_t		vmevents[NR_VM_EVENT_ITEMS];
286
287
288
	atomic_long_t		vmevents_local[NR_VM_EVENT_ITEMS];

	atomic_long_t		memory_events[MEMCG_NR_MEMORY_EVENTS];
Michal Hocko's avatar
Michal Hocko committed
289

290
291
292
	unsigned long		socket_pressure;

	/* Legacy tcp memory accounting */
293
294
	bool			tcpmem_active;
	int			tcpmem_pressure;
295

296
#ifdef CONFIG_MEMCG_KMEM
Michal Hocko's avatar
Michal Hocko committed
297
298
        /* Index in the kmem_cache->memcg_params.memcg_caches array */
	int kmemcg_id;
299
	enum memcg_kmem_state kmem_state;
300
	struct list_head kmem_caches;
Michal Hocko's avatar
Michal Hocko committed
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
#endif

	int last_scanned_node;
#if MAX_NUMNODES > 1
	nodemask_t	scan_nodes;
	atomic_t	numainfo_events;
	atomic_t	numainfo_updating;
#endif

#ifdef CONFIG_CGROUP_WRITEBACK
	struct list_head cgwb_list;
	struct wb_domain cgwb_domain;
#endif

	/* List of events which userspace want to receive */
	struct list_head event_list;
	spinlock_t event_list_lock;

	struct mem_cgroup_per_node *nodeinfo[0];
	/* WARNING: nodeinfo must be the last member here */
};
322

323
324
325
326
327
328
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
#define MEMCG_CHARGE_BATCH 32U

329
extern struct mem_cgroup *root_mem_cgroup;
Tejun Heo's avatar
Tejun Heo committed
330

331
332
333
334
335
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

336
337
338
339
340
static inline bool mem_cgroup_disabled(void)
{
	return !cgroup_subsys_enabled(memory_cgrp_subsys);
}

Roman Gushchin's avatar
Roman Gushchin committed
341
342
enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
						struct mem_cgroup *memcg);
343

344
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
345
346
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
347
348
349
int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
350
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
351
352
353
			      bool lrucare, bool compound);
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound);
354
void mem_cgroup_uncharge(struct page *page);
355
void mem_cgroup_uncharge_list(struct list_head *page_list);
356

357
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
358

359
360
static struct mem_cgroup_per_node *
mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
361
{
362
	return memcg->nodeinfo[nid];
363
364
365
}

/**
366
367
 * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
 * @node: node of the wanted lruvec
368
369
 * @memcg: memcg of the wanted lruvec
 *
370
371
 * Returns the lru list vector holding pages for a given @node or a given
 * @memcg and @zone. This can be the node lruvec, if the memory controller
372
373
 * is disabled.
 */
374
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
375
				struct mem_cgroup *memcg)
376
{
377
	struct mem_cgroup_per_node *mz;
378
379
380
	struct lruvec *lruvec;

	if (mem_cgroup_disabled()) {
381
		lruvec = node_lruvec(pgdat);
382
383
384
		goto out;
	}

385
	mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
386
387
388
389
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
390
	 * we have to be prepared to initialize lruvec->pgdat here;
391
392
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
393
394
	if (unlikely(lruvec->pgdat != pgdat))
		lruvec->pgdat = pgdat;
395
396
397
	return lruvec;
}

398
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
399

400
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
401
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
402

403
404
struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);

405
406
struct mem_cgroup *get_mem_cgroup_from_page(struct page *page);

Michal Hocko's avatar
Michal Hocko committed
407
408
409
410
411
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
	return css ? container_of(css, struct mem_cgroup, css) : NULL;
}

412
413
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
414
415
	if (memcg)
		css_put(&memcg->css);
416
417
}

418
419
420
#define mem_cgroup_from_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

Michal Hocko's avatar
Michal Hocko committed
421
422
423
424
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
				   struct mem_cgroup *,
				   struct mem_cgroup_reclaim_cookie *);
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
425
426
int mem_cgroup_scan_tasks(struct mem_cgroup *,
			  int (*)(struct task_struct *, void *), void *);
Michal Hocko's avatar
Michal Hocko committed
427

428
429
430
431
432
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return 0;

433
	return memcg->id.id;
434
}
435
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
436

437
438
439
440
441
static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
{
	return mem_cgroup_from_css(seq_css(m));
}

442
443
444
445
446
447
448
449
450
451
452
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
	struct mem_cgroup_per_node *mz;

	if (mem_cgroup_disabled())
		return NULL;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->memcg;
}

453
454
455
456
457
458
459
460
461
462
463
464
465
466
/**
 * parent_mem_cgroup - find the accounting parent of a memcg
 * @memcg: memcg whose parent to find
 *
 * Returns the parent memcg, or NULL if this is the root or the memory
 * controller is in legacy no-hierarchy mode.
 */
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg->memory.parent)
		return NULL;
	return mem_cgroup_from_counter(memcg->memory.parent, memory);
}

Michal Hocko's avatar
Michal Hocko committed
467
468
469
470
471
472
473
474
475
static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
			      struct mem_cgroup *root)
{
	if (root == memcg)
		return true;
	if (!root->use_hierarchy)
		return false;
	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
}
Glauber Costa's avatar
Glauber Costa committed
476

477
478
static inline bool mm_match_cgroup(struct mm_struct *mm,
				   struct mem_cgroup *memcg)
479
{
480
	struct mem_cgroup *task_memcg;
481
	bool match = false;
482

483
	rcu_read_lock();
484
	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
485
	if (task_memcg)
486
		match = mem_cgroup_is_descendant(task_memcg, memcg);
487
	rcu_read_unlock();
488
	return match;
489
}
490

491
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
492
ino_t page_cgroup_ino(struct page *page);
493

494
495
496
497
498
499
500
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return true;
	return !!(memcg->css.flags & CSS_ONLINE);
}

501
502
503
/*
 * For memory reclaim.
 */
504
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
Michal Hocko's avatar
Michal Hocko committed
505
506

void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
507
		int zid, int nr_pages);
Michal Hocko's avatar
Michal Hocko committed
508

509
510
511
512
513
514
515
516
static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
		enum lru_list lru, int zone_idx)
{
	struct mem_cgroup_per_node *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->lru_zone_size[zone_idx][lru];
Michal Hocko's avatar
Michal Hocko committed
517
518
}

519
520
void mem_cgroup_handle_over_high(void);

521
unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
522

523
void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
524
				struct task_struct *p);
525

526
527
void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);

528
static inline void mem_cgroup_enter_user_fault(void)
529
{
530
531
	WARN_ON(current->in_user_fault);
	current->in_user_fault = 1;
532
533
}

534
static inline void mem_cgroup_exit_user_fault(void)
535
{
536
537
	WARN_ON(!current->in_user_fault);
	current->in_user_fault = 0;
538
539
}

540
541
static inline bool task_in_memcg_oom(struct task_struct *p)
{
542
	return p->memcg_in_oom;
543
544
}

545
bool mem_cgroup_oom_synchronize(bool wait);
546
547
548
struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
					    struct mem_cgroup *oom_domain);
void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
549

Andrew Morton's avatar
Andrew Morton committed
550
#ifdef CONFIG_MEMCG_SWAP
551
552
extern int do_swap_account;
#endif
553

554
555
struct mem_cgroup *lock_page_memcg(struct page *page);
void __unlock_page_memcg(struct mem_cgroup *memcg);
556
void unlock_page_memcg(struct page *page);
557

558
559
560
561
562
563
564
565
566
567
568
569
570
571
/*
 * idx can be of type enum memcg_stat_item or node_stat_item.
 * Keep in sync with memcg_exact_page_state().
 */
static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
{
	long x = atomic_long_read(&memcg->vmstats[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

572
573
574
575
/*
 * idx can be of type enum memcg_stat_item or node_stat_item.
 * Keep in sync with memcg_exact_page_state().
 */
576
577
static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
						   int idx)
578
{
579
	long x = atomic_long_read(&memcg->vmstats_local[idx]);
580
581
582
583
584
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
585
586
}

587
void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
588

589
/* idx can be of type enum memcg_stat_item or node_stat_item */
590
static inline void mod_memcg_state(struct mem_cgroup *memcg,
591
				   int idx, int val)
592
{
593
594
595
	unsigned long flags;

	local_irq_save(flags);
596
	__mod_memcg_state(memcg, idx, val);
597
	local_irq_restore(flags);
598
599
}

Michal Hocko's avatar
Michal Hocko committed
600
/**
601
 * mod_memcg_page_state - update page state statistics
602
 * @page: the page
Michal Hocko's avatar
Michal Hocko committed
603
604
605
 * @idx: page state item to account
 * @val: number of pages (positive or negative)
 *
606
607
608
 * The @page must be locked or the caller must use lock_page_memcg()
 * to prevent double accounting when the page is concurrently being
 * moved to another memcg:
609
 *
610
 *   lock_page(page) or lock_page_memcg(page)
611
 *   if (TestClearPageState(page))
612
 *     mod_memcg_page_state(page, state, -1);
613
 *   unlock_page(page) or unlock_page_memcg(page)
614
615
 *
 * Kernel pages are an exception to this, since they'll never move.
Michal Hocko's avatar
Michal Hocko committed
616
 */
617
static inline void __mod_memcg_page_state(struct page *page,
618
					  int idx, int val)
619
620
621
622
623
{
	if (page->mem_cgroup)
		__mod_memcg_state(page->mem_cgroup, idx, val);
}

624
static inline void mod_memcg_page_state(struct page *page,
625
					int idx, int val)
Michal Hocko's avatar
Michal Hocko committed
626
{
627
	if (page->mem_cgroup)
628
		mod_memcg_state(page->mem_cgroup, idx, val);
Michal Hocko's avatar
Michal Hocko committed
629
630
}

631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
					      enum node_stat_item idx)
{
	struct mem_cgroup_per_node *pn;
	long x;

	if (mem_cgroup_disabled())
		return node_page_state(lruvec_pgdat(lruvec), idx);

	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	x = atomic_long_read(&pn->lruvec_stat[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

649
650
static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
						    enum node_stat_item idx)
651
{
652
	struct mem_cgroup_per_node *pn;
653
	long x;
654
655
656
657
658

	if (mem_cgroup_disabled())
		return node_page_state(lruvec_pgdat(lruvec), idx);

	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
659
	x = atomic_long_read(&pn->lruvec_stat_local[idx]);
660
661
662
663
664
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
665
666
}

667
668
void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
			int val);
669
670
671
672

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
673
674
675
	unsigned long flags;

	local_irq_save(flags);
676
	__mod_lruvec_state(lruvec, idx, val);
677
	local_irq_restore(flags);
678
679
680
681
682
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
683
684
	pg_data_t *pgdat = page_pgdat(page);
	struct lruvec *lruvec;
685

686
687
688
	/* Untracked pages have no memcg, no lruvec. Update only the node */
	if (!page->mem_cgroup) {
		__mod_node_page_state(pgdat, idx, val);
689
		return;
690
691
692
693
	}

	lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
	__mod_lruvec_state(lruvec, idx, val);
694
695
696
697
698
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
699
700
701
	unsigned long flags;

	local_irq_save(flags);
702
	__mod_lruvec_page_state(page, idx, val);
703
	local_irq_restore(flags);
704
705
}

706
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
707
708
						gfp_t gfp_mask,
						unsigned long *total_scanned);
709

710
711
void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
			  unsigned long count);
712

713
static inline void count_memcg_events(struct mem_cgroup *memcg,
714
715
				      enum vm_event_item idx,
				      unsigned long count)
716
{
717
718
719
	unsigned long flags;

	local_irq_save(flags);
720
	__count_memcg_events(memcg, idx, count);
721
	local_irq_restore(flags);
722
723
724
}

static inline void count_memcg_page_event(struct page *page,
725
					  enum vm_event_item idx)
726
727
728
729
730
731
732
{
	if (page->mem_cgroup)
		count_memcg_events(page->mem_cgroup, idx, 1);
}

static inline void count_memcg_event_mm(struct mm_struct *mm,
					enum vm_event_item idx)
733
{
Michal Hocko's avatar
Michal Hocko committed
734
735
	struct mem_cgroup *memcg;

736
737
	if (mem_cgroup_disabled())
		return;
Michal Hocko's avatar
Michal Hocko committed
738
739
740

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
741
	if (likely(memcg))
742
		count_memcg_events(memcg, idx, 1);
Michal Hocko's avatar
Michal Hocko committed
743
	rcu_read_unlock();
744
}
745

746
747
static inline void memcg_memory_event(struct mem_cgroup *memcg,
				      enum memcg_memory_event event)
748
{
749
	atomic_long_inc(&memcg->memory_events[event]);
750
751
752
	cgroup_file_notify(&memcg->events_file);
}

753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
static inline void memcg_memory_event_mm(struct mm_struct *mm,
					 enum memcg_memory_event event)
{
	struct mem_cgroup *memcg;

	if (mem_cgroup_disabled())
		return;

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (likely(memcg))
		memcg_memory_event(memcg, event);
	rcu_read_unlock();
}

768
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
769
void mem_cgroup_split_huge_fixup(struct page *head);
770
771
#endif

Andrew Morton's avatar
Andrew Morton committed
772
#else /* CONFIG_MEMCG */
773
774
775
776

#define MEM_CGROUP_ID_SHIFT	0
#define MEM_CGROUP_ID_MAX	0

777
778
struct mem_cgroup;

779
780
781
782
783
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return true;
}

784
785
786
787
788
static inline bool mem_cgroup_disabled(void)
{
	return true;
}

789
790
static inline void memcg_memory_event(struct mem_cgroup *memcg,
				      enum memcg_memory_event event)
791
792
793
{
}

794
795
796
797
798
static inline void memcg_memory_event_mm(struct mm_struct *mm,
					 enum memcg_memory_event event)
{
}

Roman Gushchin's avatar
Roman Gushchin committed
799
800
static inline enum mem_cgroup_protection mem_cgroup_protected(
	struct mem_cgroup *root, struct mem_cgroup *memcg)
801
{
Roman Gushchin's avatar
Roman Gushchin committed
802
	return MEMCG_PROT_NONE;
803
804
}

805
806
static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
					gfp_t gfp_mask,
807
808
					struct mem_cgroup **memcgp,
					bool compound)
809
{
810
	*memcgp = NULL;
811
812
813
	return 0;
}

814
815
816
817
818
819
820
821
822
823
static inline int mem_cgroup_try_charge_delay(struct page *page,
					      struct mm_struct *mm,
					      gfp_t gfp_mask,
					      struct mem_cgroup **memcgp,
					      bool compound)
{
	*memcgp = NULL;
	return 0;
}

824
825
static inline void mem_cgroup_commit_charge(struct page *page,
					    struct mem_cgroup *memcg,
826
					    bool lrucare, bool compound)
827
828
829
{
}

830
static inline void mem_cgroup_cancel_charge(struct page *page,
831
832
					    struct mem_cgroup *memcg,
					    bool compound)
833
834
835
{
}

836
static inline void mem_cgroup_uncharge(struct page *page)
837
838
839
{
}

840
static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
841
842
843
{
}

844
static inline void mem_cgroup_migrate(struct page *old, struct page *new)
845
846
847
{
}

848
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
849
				struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki's avatar
KAMEZAWA Hiroyuki committed
850
{
851
	return node_lruvec(pgdat);
KAMEZAWA Hiroyuki's avatar
KAMEZAWA Hiroyuki committed
852
853
}

854
static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
855
						    struct pglist_data *pgdat)
856
{
857
	return &pgdat->lruvec;
858
859
}

860
static inline bool mm_match_cgroup(struct mm_struct *mm,
861
		struct mem_cgroup *memcg)
862
{
863
	return true;
864
865
}

866
867
static inline bool task_in_mem_cgroup(struct task_struct *task,
				      const struct mem_cgroup *memcg)
868
{
869
	return true;
870
871
}

872
873
874
875
876
static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
{
	return NULL;
}

877
878
879
880
881
static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
{
	return NULL;
}

882
883
884
885
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}

886
887
888
889
890
891
892
893
894
895
896
897
898
static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
		struct mem_cgroup *prev,
		struct mem_cgroup_reclaim_cookie *reclaim)
{
	return NULL;
}

static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
					 struct mem_cgroup *prev)
{
}

899
900
901
902
903
904
static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
		int (*fn)(struct task_struct *, void *), void *arg)
{
	return 0;
}

905
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
906
{
907
908
909
910
911
912
913
914
	return 0;
}

static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	WARN_ON_ONCE(id);
	/* XXX: This should always return root_mem_cgroup */
	return NULL;
915
}
916

917
918
919
920
921
static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
{
	return NULL;
}

922
923
924
925
926
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
	return NULL;
}

927
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
928
{
929
	return true;
930
931
}

932
933
934
935
936
937
static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
		enum lru_list lru, int zone_idx)
{
	return 0;
}
938

939
static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
940
941
942
943
{
	return 0;
}

944
static inline void
945
946
947
948
949
950
mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
{
}

static inline void
mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
951
952
953
{
}

954
955
956
957
958
959
static inline struct mem_cgroup *lock_page_memcg(struct page *page)
{
	return NULL;
}

static inline void __unlock_page_memcg(struct mem_cgroup *memcg)
960
961
962
{
}

963
static inline void unlock_page_memcg(struct page *page)
964
965
966
{
}

967
968
969
970
static inline void mem_cgroup_handle_over_high(void)
{
}

971
static inline void mem_cgroup_enter_user_fault(void)
972
973
974
{
}

975
static inline void mem_cgroup_exit_user_fault(void)
976
977
978
{
}

979
980
981
982
983
static inline bool task_in_memcg_oom(struct task_struct *p)
{
	return false;
}

984
static inline bool mem_cgroup_oom_synchronize(bool wait)
985
986
987
988
{
	return false;
}

989
990
991
992
993
994
995
996
997
998
static inline struct mem_cgroup *mem_cgroup_get_oom_group(
	struct task_struct *victim, struct mem_cgroup *oom_domain)
{
	return NULL;
}

static inline void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
{
}

999
1000
1001
1002
1003
static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
{
	return 0;
}

1004
1005
static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
						   int idx)
1006
1007
1008
1009
{
	return 0;
}

1010
static inline void __mod_memcg_state(struct mem_cgroup *memcg,
1011
				     int idx,
1012
				     int nr)
1013
1014
1015
{
}

1016
static inline void mod_memcg_state(struct mem_cgroup *memcg,
1017
				   int idx,
1018
				   int nr)
1019
1020
1021
{
}

1022
static inline void __mod_memcg_page_state(struct page *page,
1023
					  int idx,
1024
					  int nr)
1025
1026
1027
{
}

1028
static inline void mod_memcg_page_state(struct page *page,
1029
					int idx,
1030
					int nr)
1031
1032
1033
{
}

1034
1035
1036
1037
1038
1039
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
					      enum node_stat_item idx)
{
	return node_page_state(lruvec_pgdat(lruvec), idx);
}

1040
1041
static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
						    enum node_stat_item idx)
1042
{
1043
	return node_page_state(lruvec_pgdat(lruvec), idx);
1044
1045
}

1046
1047
static inline void __mod_lruvec_state(struct lruvec *lruvec,
				      enum node_stat_item idx, int val)
1048
{
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
	__mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
	mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
	__mod_node_page_state(page_pgdat(page), idx, val);
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
	mod_node_page_state(page_pgdat(page), idx, val);
1068
1069
}

1070
static inline
1071
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
1072
1073
					    gfp_t gfp_mask,
					    unsigned long *total_scanned)
1074
{
1075
	return 0;
1076
1077
}

1078
static inline void mem_cgroup_split_huge_fixup(struct page *head)
1079
1080
1081
{
}

1082
1083
1084
1085
1086
1087
static inline void count_memcg_events(struct mem_cgroup *memcg,
				      enum vm_event_item idx,
				      unsigned long count)
{
}

1088
1089
1090
1091
1092
1093
static inline void __count_memcg_events(struct mem_cgroup *memcg,
					enum vm_event_item idx,
					unsigned long count)
{
}

1094
static inline void count_memcg_page_event(struct page *page,
1095
					  int idx)
1096
1097
1098
{
}

1099
static inline
1100
void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
1101
1102
{
}
Andrew Morton's avatar
Andrew Morton committed
1103
#endif /* CONFIG_MEMCG */
1104

1105
/* idx can be of type enum memcg_stat_item or node_stat_item */
1106
static inline void __inc_memcg_state(struct mem_cgroup *memcg,
1107
				     int idx)
1108
1109
1110
1111
{
	__mod_memcg_state(memcg, idx, 1);
}

1112
/* idx can be of type enum memcg_stat_item or node_stat_item */
1113
static inline void __dec_memcg_state(struct mem_cgroup *memcg,
1114
				     int idx)
1115
1116
1117
1118
{
	__mod_memcg_state(memcg, idx, -1);
}

1119
/* idx can be of type enum memcg_stat_item or node_stat_item */
1120
static inline void __inc_memcg_page_state(struct page *page,
1121
					  int idx)
1122
1123
1124
1125
{
	__mod_memcg_page_state(page, idx, 1);
}

1126
/* idx can be of type enum memcg_stat_item or node_stat_item */
1127
static inline void __dec_memcg_page_state(struct page *page,
1128
					  int idx)
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152