memcontrol.h 35.2 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0-or-later */
2 3 4 5 6
/* memcontrol.h - Memory Controller
 *
 * Copyright IBM Corporation, 2007
 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
 *
7 8
 * Copyright 2007 OpenVZ SWsoft Inc
 * Author: Pavel Emelianov <xemul@openvz.org>
9 10 11 12
 */

#ifndef _LINUX_MEMCONTROL_H
#define _LINUX_MEMCONTROL_H
13
#include <linux/cgroup.h>
14
#include <linux/vm_event_item.h>
15
#include <linux/hardirq.h>
16
#include <linux/jump_label.h>
Michal Hocko's avatar
Michal Hocko committed
17 18 19
#include <linux/page_counter.h>
#include <linux/vmpressure.h>
#include <linux/eventfd.h>
20 21
#include <linux/mm.h>
#include <linux/vmstat.h>
Michal Hocko's avatar
Michal Hocko committed
22
#include <linux/writeback.h>
23
#include <linux/page-flags.h>
24

25
struct mem_cgroup;
26 27
struct page;
struct mm_struct;
28
struct kmem_cache;
29

30 31 32 33 34 35 36 37 38
/* Cgroup-specific page state, on top of universal node page state */
enum memcg_stat_item {
	MEMCG_CACHE = NR_VM_NODE_STAT_ITEMS,
	MEMCG_RSS,
	MEMCG_RSS_HUGE,
	MEMCG_SWAP,
	MEMCG_SOCK,
	/* XXX: why are these zone and not node counters? */
	MEMCG_KERNEL_STACK_KB,
39
	MEMCG_NR_STAT,
40 41
};

42 43
enum memcg_memory_event {
	MEMCG_LOW,
44 45 46
	MEMCG_HIGH,
	MEMCG_MAX,
	MEMCG_OOM,
47
	MEMCG_OOM_KILL,
48 49
	MEMCG_SWAP_MAX,
	MEMCG_SWAP_FAIL,
50
	MEMCG_NR_MEMORY_EVENTS,
51 52
};

Roman Gushchin's avatar
Roman Gushchin committed
53 54 55 56 57 58
enum mem_cgroup_protection {
	MEMCG_PROT_NONE,
	MEMCG_PROT_LOW,
	MEMCG_PROT_MIN,
};

59
struct mem_cgroup_reclaim_cookie {
60
	pg_data_t *pgdat;
61 62 63
	unsigned int generation;
};

64 65 66 67 68 69 70
#ifdef CONFIG_MEMCG

#define MEM_CGROUP_ID_SHIFT	16
#define MEM_CGROUP_ID_MAX	USHRT_MAX

struct mem_cgroup_id {
	int id;
71
	refcount_t ref;
72 73
};

Michal Hocko's avatar
Michal Hocko committed
74 75 76 77 78 79 80 81 82 83 84 85
/*
 * Per memcg event counter is incremented at every pagein/pageout. With THP,
 * it will be incremated by the number of pages. This counter is used for
 * for trigger some periodic events. This is straightforward and better
 * than using jiffies etc. to handle periodic memcg event.
 */
enum mem_cgroup_events_target {
	MEM_CGROUP_TARGET_THRESH,
	MEM_CGROUP_TARGET_SOFTLIMIT,
	MEM_CGROUP_NTARGETS,
};

86 87
struct memcg_vmstats_percpu {
	long stat[MEMCG_NR_STAT];
88
	unsigned long events[NR_VM_EVENT_ITEMS];
Michal Hocko's avatar
Michal Hocko committed
89 90 91 92 93 94 95 96 97 98
	unsigned long nr_page_events;
	unsigned long targets[MEM_CGROUP_NTARGETS];
};

struct mem_cgroup_reclaim_iter {
	struct mem_cgroup *position;
	/* scan generation, increased every round-trip */
	unsigned int generation;
};

99 100 101 102
struct lruvec_stat {
	long count[NR_VM_NODE_STAT_ITEMS];
};

103 104 105 106 107 108 109 110 111
/*
 * Bitmap of shrinker::id corresponding to memcg-aware shrinkers,
 * which have elements charged to this memcg.
 */
struct memcg_shrinker_map {
	struct rcu_head rcu;
	unsigned long map[0];
};

Michal Hocko's avatar
Michal Hocko committed
112
/*
113
 * per-node information in memory controller.
Michal Hocko's avatar
Michal Hocko committed
114
 */
115
struct mem_cgroup_per_node {
Michal Hocko's avatar
Michal Hocko committed
116
	struct lruvec		lruvec;
117

118 119 120 121
	/* Legacy local VM stats */
	struct lruvec_stat __percpu *lruvec_stat_local;

	/* Subtree VM stats (batched updates) */
122 123 124
	struct lruvec_stat __percpu *lruvec_stat_cpu;
	atomic_long_t		lruvec_stat[NR_VM_NODE_STAT_ITEMS];

125
	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
Michal Hocko's avatar
Michal Hocko committed
126

127
	struct mem_cgroup_reclaim_iter	iter;
Michal Hocko's avatar
Michal Hocko committed
128

129
	struct memcg_shrinker_map __rcu	*shrinker_map;
130

Michal Hocko's avatar
Michal Hocko committed
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
	struct rb_node		tree_node;	/* RB tree node */
	unsigned long		usage_in_excess;/* Set to the value by which */
						/* the soft limit is exceeded*/
	bool			on_tree;
	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
						/* use container_of	   */
};

struct mem_cgroup_threshold {
	struct eventfd_ctx *eventfd;
	unsigned long threshold;
};

/* For threshold */
struct mem_cgroup_threshold_ary {
	/* An array index points to threshold just below or equal to usage. */
	int current_threshold;
	/* Size of entries[] */
	unsigned int size;
	/* Array of thresholds */
	struct mem_cgroup_threshold entries[0];
};

struct mem_cgroup_thresholds {
	/* Primary thresholds array */
	struct mem_cgroup_threshold_ary *primary;
	/*
	 * Spare threshold array.
	 * This is needed to make mem_cgroup_unregister_event() "never fail".
	 * It must be able to store at least primary->size - 1 entries.
	 */
	struct mem_cgroup_threshold_ary *spare;
};

165 166 167 168 169 170
enum memcg_kmem_state {
	KMEM_NONE,
	KMEM_ALLOCATED,
	KMEM_ONLINE,
};

171 172 173 174 175 176 177 178 179
#if defined(CONFIG_SMP)
struct memcg_padding {
	char x[0];
} ____cacheline_internodealigned_in_smp;
#define MEMCG_PADDING(name)      struct memcg_padding name;
#else
#define MEMCG_PADDING(name)
#endif

180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
/*
 * Remember four most recent foreign writebacks with dirty pages in this
 * cgroup.  Inode sharing is expected to be uncommon and, even if we miss
 * one in a given round, we're likely to catch it later if it keeps
 * foreign-dirtying, so a fairly low count should be enough.
 *
 * See mem_cgroup_track_foreign_dirty_slowpath() for details.
 */
#define MEMCG_CGWB_FRN_CNT	4

struct memcg_cgwb_frn {
	u64 bdi_id;			/* bdi->id of the foreign inode */
	int memcg_id;			/* memcg->css.id of foreign inode */
	u64 at;				/* jiffies_64 at the time of dirtying */
	struct wb_completion done;	/* tracks in-flight foreign writebacks */
};

Michal Hocko's avatar
Michal Hocko committed
197 198 199 200 201 202 203 204 205
/*
 * The memory controller data structure. The memory controller controls both
 * page cache and RSS per cgroup. We would eventually like to provide
 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
 * to help the administrator determine what knobs to tune.
 */
struct mem_cgroup {
	struct cgroup_subsys_state css;

206 207 208
	/* Private memcg ID. Used to ID objects that outlive the cgroup */
	struct mem_cgroup_id id;

Michal Hocko's avatar
Michal Hocko committed
209 210
	/* Accounted resources */
	struct page_counter memory;
211
	struct page_counter swap;
212 213

	/* Legacy consumer-oriented counters */
Michal Hocko's avatar
Michal Hocko committed
214 215
	struct page_counter memsw;
	struct page_counter kmem;
216
	struct page_counter tcpmem;
Michal Hocko's avatar
Michal Hocko committed
217

218
	/* Upper bound of normal memory consumption range */
Michal Hocko's avatar
Michal Hocko committed
219 220
	unsigned long high;

221 222 223
	/* Range enforcement for interrupt charges */
	struct work_struct high_work;

Michal Hocko's avatar
Michal Hocko committed
224 225 226 227 228 229 230 231 232 233
	unsigned long soft_limit;

	/* vmpressure notifications */
	struct vmpressure vmpressure;

	/*
	 * Should the accounting and control be hierarchical, per subtree?
	 */
	bool use_hierarchy;

234 235 236 237 238
	/*
	 * Should the OOM killer kill all belonging tasks, had it kill one?
	 */
	bool oom_group;

Michal Hocko's avatar
Michal Hocko committed
239 240 241 242 243 244 245 246
	/* protected by memcg_oom_lock */
	bool		oom_lock;
	int		under_oom;

	int	swappiness;
	/* OOM-Killer disable */
	int		oom_kill_disable;

247
	/* memory.events and memory.events.local */
248
	struct cgroup_file events_file;
249
	struct cgroup_file events_local_file;
250

251 252 253
	/* handle for "memory.swap.events" */
	struct cgroup_file swap_events_file;

Michal Hocko's avatar
Michal Hocko committed
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
	/* protect arrays of thresholds */
	struct mutex thresholds_lock;

	/* thresholds for memory usage. RCU-protected */
	struct mem_cgroup_thresholds thresholds;

	/* thresholds for mem+swap usage. RCU-protected */
	struct mem_cgroup_thresholds memsw_thresholds;

	/* For oom notifier event fd */
	struct list_head oom_notify;

	/*
	 * Should we move charges of a task when a task is moved into this
	 * mem_cgroup ? And what type of charges should we move ?
	 */
	unsigned long move_charge_at_immigrate;
271 272 273 274 275 276
	/* taken only while moving_account > 0 */
	spinlock_t		move_lock;
	unsigned long		move_lock_flags;

	MEMCG_PADDING(_pad1_);

Michal Hocko's avatar
Michal Hocko committed
277 278 279 280 281
	/*
	 * set > 0 if pages under this cgroup are moving to other cgroup.
	 */
	atomic_t		moving_account;
	struct task_struct	*move_lock_task;
282

283 284 285 286
	/* Legacy local VM stats and events */
	struct memcg_vmstats_percpu __percpu *vmstats_local;

	/* Subtree VM stats and events (batched updates) */
287
	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
288 289 290

	MEMCG_PADDING(_pad2_);

291 292
	atomic_long_t		vmstats[MEMCG_NR_STAT];
	atomic_long_t		vmevents[NR_VM_EVENT_ITEMS];
293

294
	/* memory.events */
295
	atomic_long_t		memory_events[MEMCG_NR_MEMORY_EVENTS];
296
	atomic_long_t		memory_events_local[MEMCG_NR_MEMORY_EVENTS];
Michal Hocko's avatar
Michal Hocko committed
297

298 299 300
	unsigned long		socket_pressure;

	/* Legacy tcp memory accounting */
301 302
	bool			tcpmem_active;
	int			tcpmem_pressure;
303

304
#ifdef CONFIG_MEMCG_KMEM
Michal Hocko's avatar
Michal Hocko committed
305 306
        /* Index in the kmem_cache->memcg_params.memcg_caches array */
	int kmemcg_id;
307
	enum memcg_kmem_state kmem_state;
308
	struct list_head kmem_caches;
Michal Hocko's avatar
Michal Hocko committed
309 310 311 312 313
#endif

#ifdef CONFIG_CGROUP_WRITEBACK
	struct list_head cgwb_list;
	struct wb_domain cgwb_domain;
314
	struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
Michal Hocko's avatar
Michal Hocko committed
315 316 317 318 319 320
#endif

	/* List of events which userspace want to receive */
	struct list_head event_list;
	spinlock_t event_list_lock;

321 322 323 324
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	struct deferred_split deferred_split_queue;
#endif

Michal Hocko's avatar
Michal Hocko committed
325 326 327
	struct mem_cgroup_per_node *nodeinfo[0];
	/* WARNING: nodeinfo must be the last member here */
};
328

329 330 331 332 333 334
/*
 * size of first charge trial. "32" comes from vmscan.c's magic value.
 * TODO: maybe necessary to use big numbers in big irons.
 */
#define MEMCG_CHARGE_BATCH 32U

335
extern struct mem_cgroup *root_mem_cgroup;
Tejun Heo's avatar
Tejun Heo committed
336

337 338 339 340 341
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return (memcg == root_mem_cgroup);
}

342 343 344 345 346
static inline bool mem_cgroup_disabled(void)
{
	return !cgroup_subsys_enabled(memory_cgrp_subsys);
}

347 348
static inline unsigned long mem_cgroup_protection(struct mem_cgroup *memcg,
						  bool in_low_reclaim)
349
{
350 351 352 353 354
	if (mem_cgroup_disabled())
		return 0;

	if (in_low_reclaim)
		return READ_ONCE(memcg->memory.emin);
355

356 357
	return max(READ_ONCE(memcg->memory.emin),
		   READ_ONCE(memcg->memory.elow));
358 359
}

Roman Gushchin's avatar
Roman Gushchin committed
360 361
enum mem_cgroup_protection mem_cgroup_protected(struct mem_cgroup *root,
						struct mem_cgroup *memcg);
362

363
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
364 365
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
366 367 368
int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm,
			  gfp_t gfp_mask, struct mem_cgroup **memcgp,
			  bool compound);
369
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
370 371 372
			      bool lrucare, bool compound);
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
		bool compound);
373
void mem_cgroup_uncharge(struct page *page);
374
void mem_cgroup_uncharge_list(struct list_head *page_list);
375

376
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
377

378 379
static struct mem_cgroup_per_node *
mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
380
{
381
	return memcg->nodeinfo[nid];
382 383 384
}

/**
385
 * mem_cgroup_lruvec - get the lru list vector for a memcg & node
386 387
 * @memcg: memcg of the wanted lruvec
 *
388 389 390
 * Returns the lru list vector holding pages for a given @memcg &
 * @node combination. This can be the node lruvec, if the memory
 * controller is disabled.
391
 */
392 393
static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
					       struct pglist_data *pgdat)
394
{
395
	struct mem_cgroup_per_node *mz;
396 397 398
	struct lruvec *lruvec;

	if (mem_cgroup_disabled()) {
399
		lruvec = &pgdat->__lruvec;
400 401 402
		goto out;
	}

403 404 405
	if (!memcg)
		memcg = root_mem_cgroup;

406
	mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
407 408 409 410
	lruvec = &mz->lruvec;
out:
	/*
	 * Since a node can be onlined after the mem_cgroup was created,
411
	 * we have to be prepared to initialize lruvec->pgdat here;
412 413
	 * and if offlined then reonlined, we need to reinitialize it.
	 */
414 415
	if (unlikely(lruvec->pgdat != pgdat))
		lruvec->pgdat = pgdat;
416 417 418
	return lruvec;
}

419
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
420

421
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
422

423 424
struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);

425 426
struct mem_cgroup *get_mem_cgroup_from_page(struct page *page);

Michal Hocko's avatar
Michal Hocko committed
427 428 429 430 431
static inline
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
	return css ? container_of(css, struct mem_cgroup, css) : NULL;
}

432 433
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
434 435
	if (memcg)
		css_put(&memcg->css);
436 437
}

438 439 440
#define mem_cgroup_from_counter(counter, member)	\
	container_of(counter, struct mem_cgroup, member)

Michal Hocko's avatar
Michal Hocko committed
441 442 443 444
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
				   struct mem_cgroup *,
				   struct mem_cgroup_reclaim_cookie *);
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
445 446
int mem_cgroup_scan_tasks(struct mem_cgroup *,
			  int (*)(struct task_struct *, void *), void *);
Michal Hocko's avatar
Michal Hocko committed
447

448 449 450 451 452
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return 0;

453
	return memcg->id.id;
454
}
455
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
456

457 458 459 460 461
static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
{
	return mem_cgroup_from_css(seq_css(m));
}

462 463 464 465 466 467 468 469 470 471 472
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
{
	struct mem_cgroup_per_node *mz;

	if (mem_cgroup_disabled())
		return NULL;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->memcg;
}

473 474 475 476 477 478 479 480 481 482 483 484 485 486
/**
 * parent_mem_cgroup - find the accounting parent of a memcg
 * @memcg: memcg whose parent to find
 *
 * Returns the parent memcg, or NULL if this is the root or the memory
 * controller is in legacy no-hierarchy mode.
 */
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
	if (!memcg->memory.parent)
		return NULL;
	return mem_cgroup_from_counter(memcg->memory.parent, memory);
}

Michal Hocko's avatar
Michal Hocko committed
487 488 489 490 491 492 493 494 495
static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
			      struct mem_cgroup *root)
{
	if (root == memcg)
		return true;
	if (!root->use_hierarchy)
		return false;
	return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
}
Glauber Costa's avatar
Glauber Costa committed
496

497 498
static inline bool mm_match_cgroup(struct mm_struct *mm,
				   struct mem_cgroup *memcg)
499
{
500
	struct mem_cgroup *task_memcg;
501
	bool match = false;
502

503
	rcu_read_lock();
504
	task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
505
	if (task_memcg)
506
		match = mem_cgroup_is_descendant(task_memcg, memcg);
507
	rcu_read_unlock();
508
	return match;
509
}
510

511
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
512
ino_t page_cgroup_ino(struct page *page);
513

514 515 516 517 518 519 520
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
{
	if (mem_cgroup_disabled())
		return true;
	return !!(memcg->css.flags & CSS_ONLINE);
}

521 522 523
/*
 * For memory reclaim.
 */
524
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
Michal Hocko's avatar
Michal Hocko committed
525 526

void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
527
		int zid, int nr_pages);
Michal Hocko's avatar
Michal Hocko committed
528

529 530 531 532 533 534 535 536
static inline
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
		enum lru_list lru, int zone_idx)
{
	struct mem_cgroup_per_node *mz;

	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	return mz->lru_zone_size[zone_idx][lru];
Michal Hocko's avatar
Michal Hocko committed
537 538
}

539 540
void mem_cgroup_handle_over_high(void);

541
unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
542

543 544
unsigned long mem_cgroup_size(struct mem_cgroup *memcg);

545
void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
546
				struct task_struct *p);
547

548 549
void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);

550
static inline void mem_cgroup_enter_user_fault(void)
551
{
552 553
	WARN_ON(current->in_user_fault);
	current->in_user_fault = 1;
554 555
}

556
static inline void mem_cgroup_exit_user_fault(void)
557
{
558 559
	WARN_ON(!current->in_user_fault);
	current->in_user_fault = 0;
560 561
}

562 563
static inline bool task_in_memcg_oom(struct task_struct *p)
{
564
	return p->memcg_in_oom;
565 566
}

567
bool mem_cgroup_oom_synchronize(bool wait);
568 569 570
struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
					    struct mem_cgroup *oom_domain);
void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
571

Andrew Morton's avatar
Andrew Morton committed
572
#ifdef CONFIG_MEMCG_SWAP
573 574
extern int do_swap_account;
#endif
575

576 577
struct mem_cgroup *lock_page_memcg(struct page *page);
void __unlock_page_memcg(struct mem_cgroup *memcg);
578
void unlock_page_memcg(struct page *page);
579

580 581 582 583 584 585 586 587 588 589 590 591 592 593
/*
 * idx can be of type enum memcg_stat_item or node_stat_item.
 * Keep in sync with memcg_exact_page_state().
 */
static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
{
	long x = atomic_long_read(&memcg->vmstats[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

594 595 596 597
/*
 * idx can be of type enum memcg_stat_item or node_stat_item.
 * Keep in sync with memcg_exact_page_state().
 */
598 599
static inline unsigned long memcg_page_state_local(struct mem_cgroup *memcg,
						   int idx)
600
{
601 602 603 604 605
	long x = 0;
	int cpu;

	for_each_possible_cpu(cpu)
		x += per_cpu(memcg->vmstats_local->stat[idx], cpu);
606 607 608 609 610
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
611 612
}

613
void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val);
614

615
/* idx can be of type enum memcg_stat_item or node_stat_item */
616
static inline void mod_memcg_state(struct mem_cgroup *memcg,
617
				   int idx, int val)
618
{
619 620 621
	unsigned long flags;

	local_irq_save(flags);
622
	__mod_memcg_state(memcg, idx, val);
623
	local_irq_restore(flags);
624 625
}

Michal Hocko's avatar
Michal Hocko committed
626
/**
627
 * mod_memcg_page_state - update page state statistics
628
 * @page: the page
Michal Hocko's avatar
Michal Hocko committed
629 630 631
 * @idx: page state item to account
 * @val: number of pages (positive or negative)
 *
632 633 634
 * The @page must be locked or the caller must use lock_page_memcg()
 * to prevent double accounting when the page is concurrently being
 * moved to another memcg:
635
 *
636
 *   lock_page(page) or lock_page_memcg(page)
637
 *   if (TestClearPageState(page))
638
 *     mod_memcg_page_state(page, state, -1);
639
 *   unlock_page(page) or unlock_page_memcg(page)
640 641
 *
 * Kernel pages are an exception to this, since they'll never move.
Michal Hocko's avatar
Michal Hocko committed
642
 */
643
static inline void __mod_memcg_page_state(struct page *page,
644
					  int idx, int val)
645 646 647 648 649
{
	if (page->mem_cgroup)
		__mod_memcg_state(page->mem_cgroup, idx, val);
}

650
static inline void mod_memcg_page_state(struct page *page,
651
					int idx, int val)
Michal Hocko's avatar
Michal Hocko committed
652
{
653
	if (page->mem_cgroup)
654
		mod_memcg_state(page->mem_cgroup, idx, val);
Michal Hocko's avatar
Michal Hocko committed
655 656
}

657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
					      enum node_stat_item idx)
{
	struct mem_cgroup_per_node *pn;
	long x;

	if (mem_cgroup_disabled())
		return node_page_state(lruvec_pgdat(lruvec), idx);

	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
	x = atomic_long_read(&pn->lruvec_stat[idx]);
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
}

675 676
static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
						    enum node_stat_item idx)
677
{
678
	struct mem_cgroup_per_node *pn;
679 680
	long x = 0;
	int cpu;
681 682 683 684 685

	if (mem_cgroup_disabled())
		return node_page_state(lruvec_pgdat(lruvec), idx);

	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
686 687
	for_each_possible_cpu(cpu)
		x += per_cpu(pn->lruvec_stat_local->count[idx], cpu);
688 689 690 691 692
#ifdef CONFIG_SMP
	if (x < 0)
		x = 0;
#endif
	return x;
693 694
}

695 696
void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
			int val);
697
void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val);
698
void mod_memcg_obj_state(void *p, int idx, int val);
699 700 701 702

static inline void mod_lruvec_state(struct lruvec *lruvec,
				    enum node_stat_item idx, int val)
{
703 704 705
	unsigned long flags;

	local_irq_save(flags);
706
	__mod_lruvec_state(lruvec, idx, val);
707
	local_irq_restore(flags);
708 709 710 711 712
}

static inline void __mod_lruvec_page_state(struct page *page,
					   enum node_stat_item idx, int val)
{
713 714
	pg_data_t *pgdat = page_pgdat(page);
	struct lruvec *lruvec;
715

716 717 718
	/* Untracked pages have no memcg, no lruvec. Update only the node */
	if (!page->mem_cgroup) {
		__mod_node_page_state(pgdat, idx, val);
719
		return;
720 721
	}

722
	lruvec = mem_cgroup_lruvec(page->mem_cgroup, pgdat);
723
	__mod_lruvec_state(lruvec, idx, val);
724 725 726 727 728
}

static inline void mod_lruvec_page_state(struct page *page,
					 enum node_stat_item idx, int val)
{
729 730 731
	unsigned long flags;

	local_irq_save(flags);
732
	__mod_lruvec_page_state(page, idx, val);
733
	local_irq_restore(flags);
734 735
}

736
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
737 738
						gfp_t gfp_mask,
						unsigned long *total_scanned);
739

740 741
void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
			  unsigned long count);
742

743
static inline void count_memcg_events(struct mem_cgroup *memcg,
744 745
				      enum vm_event_item idx,
				      unsigned long count)
746
{
747 748 749
	unsigned long flags;

	local_irq_save(flags);
750
	__count_memcg_events(memcg, idx, count);
751
	local_irq_restore(flags);
752 753 754
}

static inline void count_memcg_page_event(struct page *page,
755
					  enum vm_event_item idx)
756 757 758 759 760 761 762
{
	if (page->mem_cgroup)
		count_memcg_events(page->mem_cgroup, idx, 1);
}

static inline void count_memcg_event_mm(struct mm_struct *mm,
					enum vm_event_item idx)
763
{
Michal Hocko's avatar
Michal Hocko committed
764 765
	struct mem_cgroup *memcg;

766 767
	if (mem_cgroup_disabled())
		return;
Michal Hocko's avatar
Michal Hocko committed
768 769 770

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
771
	if (likely(memcg))
772
		count_memcg_events(memcg, idx, 1);
Michal Hocko's avatar
Michal Hocko committed
773
	rcu_read_unlock();
774
}
775

776 777
static inline void memcg_memory_event(struct mem_cgroup *memcg,
				      enum memcg_memory_event event)
778
{
779 780 781
	atomic_long_inc(&memcg->memory_events_local[event]);
	cgroup_file_notify(&memcg->events_local_file);

782 783 784 785 786 787 788 789
	do {
		atomic_long_inc(&memcg->memory_events[event]);
		cgroup_file_notify(&memcg->events_file);

		if (cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_LOCAL_EVENTS)
			break;
	} while ((memcg = parent_mem_cgroup(memcg)) &&
		 !mem_cgroup_is_root(memcg));
790 791
}

792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
static inline void memcg_memory_event_mm(struct mm_struct *mm,
					 enum memcg_memory_event event)
{
	struct mem_cgroup *memcg;

	if (mem_cgroup_disabled())
		return;

	rcu_read_lock();
	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
	if (likely(memcg))
		memcg_memory_event(memcg, event);
	rcu_read_unlock();
}

807
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
808
void mem_cgroup_split_huge_fixup(struct page *head);
809 810
#endif

Andrew Morton's avatar
Andrew Morton committed
811
#else /* CONFIG_MEMCG */
812 813 814 815

#define MEM_CGROUP_ID_SHIFT	0
#define MEM_CGROUP_ID_MAX	0

816 817
struct mem_cgroup;

818 819 820 821 822
static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
{
	return true;
}

823 824 825 826 827
static inline bool mem_cgroup_disabled(void)
{
	return true;
}

828 829
static inline void memcg_memory_event(struct mem_cgroup *memcg,
				      enum memcg_memory_event event)
830 831 832
{
}

833 834 835 836 837
static inline void memcg_memory_event_mm(struct mm_struct *mm,
					 enum memcg_memory_event event)
{
}

838 839
static inline unsigned long mem_cgroup_protection(struct mem_cgroup *memcg,
						  bool in_low_reclaim)
840
{
841
	return 0;
842 843
}

Roman Gushchin's avatar
Roman Gushchin committed
844 845
static inline enum mem_cgroup_protection mem_cgroup_protected(
	struct mem_cgroup *root, struct mem_cgroup *memcg)
846
{
Roman Gushchin's avatar
Roman Gushchin committed
847
	return MEMCG_PROT_NONE;
848 849
}

850 851
static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
					gfp_t gfp_mask,
852 853
					struct mem_cgroup **memcgp,
					bool compound)
854
{
855
	*memcgp = NULL;
856 857 858
	return 0;
}

859 860 861 862 863 864 865 866 867 868
static inline int mem_cgroup_try_charge_delay(struct page *page,
					      struct mm_struct *mm,
					      gfp_t gfp_mask,
					      struct mem_cgroup **memcgp,
					      bool compound)
{
	*memcgp = NULL;
	return 0;
}

869 870
static inline void mem_cgroup_commit_charge(struct page *page,
					    struct mem_cgroup *memcg,
871
					    bool lrucare, bool compound)
872 873 874
{
}

875
static inline void mem_cgroup_cancel_charge(struct page *page,
876 877
					    struct mem_cgroup *memcg,
					    bool compound)
878 879 880
{
}

881
static inline void mem_cgroup_uncharge(struct page *page)
882 883 884
{
}

885
static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
886 887 888
{
}

889
static inline void mem_cgroup_migrate(struct page *old, struct page *new)
890 891 892
{
}

893 894
static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
					       struct pglist_data *pgdat)
KAMEZAWA Hiroyuki's avatar
KAMEZAWA Hiroyuki committed
895
{
896
	return &pgdat->__lruvec;
KAMEZAWA Hiroyuki's avatar
KAMEZAWA Hiroyuki committed
897 898
}

899
static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
900
						    struct pglist_data *pgdat)
901
{
902
	return &pgdat->__lruvec;
903 904
}

905 906 907 908 909
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
{
	return NULL;
}

910
static inline bool mm_match_cgroup(struct mm_struct *mm,
911
		struct mem_cgroup *memcg)
912
{
913
	return true;
914 915
}

916 917 918 919 920
static inline struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
{
	return NULL;
}

921 922 923 924 925
static inline struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
{
	return NULL;
}

926 927 928 929
static inline void mem_cgroup_put(struct mem_cgroup *memcg)
{
}

930 931 932 933 934 935 936 937 938 939 940 941 942
static inline struct mem_cgroup *
mem_cgroup_iter(struct mem_cgroup *root,
		struct mem_cgroup *prev,
		struct mem_cgroup_reclaim_cookie *reclaim)
{
	return NULL;
}

static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
					 struct mem_cgroup *prev)
{
}

943 944 945 946 947 948
static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
		int (*fn)(struct task_struct *, void *), void *arg)
{
	return 0;
}

949
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
950
{
951 952 953 954 955 956 957 958
	return 0;
}

static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
{
	WARN_ON_ONCE(id);
	/* XXX: This should always return root_mem_cgroup */
	return NULL;
959
}
960

Chris Down's avatar