memory.c 17.3 KB
Newer Older
1
/*
2
 * Memory subsystem support
3 4 5 6 7 8 9 10 11 12 13 14 15
 *
 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
 *            Dave Hansen <haveblue@us.ibm.com>
 *
 * This file provides the necessary infrastructure to represent
 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
 * All arch-independent code that assumes MEMORY_HOTPLUG requires
 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/topology.h>
16
#include <linux/capability.h>
17 18 19 20
#include <linux/device.h>
#include <linux/memory.h>
#include <linux/memory_hotplug.h>
#include <linux/mm.h>
21
#include <linux/mutex.h>
22
#include <linux/stat.h>
23
#include <linux/slab.h>
24

Arun Sharma's avatar
Arun Sharma committed
25
#include <linux/atomic.h>
26 27
#include <asm/uaccess.h>

28 29
static DEFINE_MUTEX(mem_sysfs_mutex);

30
#define MEMORY_CLASS_NAME	"memory"
31

32 33
#define to_memory_block(dev) container_of(dev, struct memory_block, dev)

34 35 36 37 38 39
static int sections_per_block;

static inline int base_memory_block_id(int section_nr)
{
	return section_nr / sections_per_block;
}
40

41 42 43
static int memory_subsys_online(struct device *dev);
static int memory_subsys_offline(struct device *dev);

44
static struct bus_type memory_subsys = {
45
	.name = MEMORY_CLASS_NAME,
46
	.dev_name = MEMORY_CLASS_NAME,
47 48
	.online = memory_subsys_online,
	.offline = memory_subsys_offline,
49 50
};

51
static BLOCKING_NOTIFIER_HEAD(memory_chain);
52

53
int register_memory_notifier(struct notifier_block *nb)
54
{
55
        return blocking_notifier_chain_register(&memory_chain, nb);
56
}
57
EXPORT_SYMBOL(register_memory_notifier);
58

59
void unregister_memory_notifier(struct notifier_block *nb)
60
{
61
        blocking_notifier_chain_unregister(&memory_chain, nb);
62
}
63
EXPORT_SYMBOL(unregister_memory_notifier);
64

65 66 67 68 69 70 71 72 73 74 75 76 77 78
static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain);

int register_memory_isolate_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_register(&memory_isolate_chain, nb);
}
EXPORT_SYMBOL(register_memory_isolate_notifier);

void unregister_memory_isolate_notifier(struct notifier_block *nb)
{
	atomic_notifier_chain_unregister(&memory_isolate_chain, nb);
}
EXPORT_SYMBOL(unregister_memory_isolate_notifier);

79 80
static void memory_block_release(struct device *dev)
{
81
	struct memory_block *mem = to_memory_block(dev);
82 83 84 85

	kfree(mem);
}

86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
unsigned long __weak memory_block_size_bytes(void)
{
	return MIN_MEMORY_BLOCK_SIZE;
}

static unsigned long get_memory_block_size(void)
{
	unsigned long block_sz;

	block_sz = memory_block_size_bytes();

	/* Validate blk_sz is a power of 2 and not less than section size */
	if ((block_sz & (block_sz - 1)) || (block_sz < MIN_MEMORY_BLOCK_SIZE)) {
		WARN_ON(1);
		block_sz = MIN_MEMORY_BLOCK_SIZE;
	}

	return block_sz;
}

106 107 108 109 110
/*
 * use this as the physical section index that this memsection
 * uses.
 */

111 112
static ssize_t show_mem_start_phys_index(struct device *dev,
			struct device_attribute *attr, char *buf)
113
{
114
	struct memory_block *mem = to_memory_block(dev);
115 116 117 118 119 120
	unsigned long phys_index;

	phys_index = mem->start_section_nr / sections_per_block;
	return sprintf(buf, "%08lx\n", phys_index);
}

121 122 123
/*
 * Show whether the section of memory is likely to be hot-removable
 */
124 125
static ssize_t show_mem_removable(struct device *dev,
			struct device_attribute *attr, char *buf)
126
{
127 128
	unsigned long i, pfn;
	int ret = 1;
129
	struct memory_block *mem = to_memory_block(dev);
130

131
	for (i = 0; i < sections_per_block; i++) {
132 133
		if (!present_section_nr(mem->start_section_nr + i))
			continue;
134
		pfn = section_nr_to_pfn(mem->start_section_nr + i);
135 136 137
		ret &= is_mem_section_removable(pfn, PAGES_PER_SECTION);
	}

138 139 140
	return sprintf(buf, "%d\n", ret);
}

141 142 143
/*
 * online, offline, going offline, etc.
 */
144 145
static ssize_t show_mem_state(struct device *dev,
			struct device_attribute *attr, char *buf)
146
{
147
	struct memory_block *mem = to_memory_block(dev);
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
	ssize_t len = 0;

	/*
	 * We can probably put these states in a nice little array
	 * so that they're not open-coded
	 */
	switch (mem->state) {
		case MEM_ONLINE:
			len = sprintf(buf, "online\n");
			break;
		case MEM_OFFLINE:
			len = sprintf(buf, "offline\n");
			break;
		case MEM_GOING_OFFLINE:
			len = sprintf(buf, "going-offline\n");
			break;
		default:
			len = sprintf(buf, "ERROR-UNKNOWN-%ld\n",
					mem->state);
			WARN_ON(1);
			break;
	}

	return len;
}

174
int memory_notify(unsigned long val, void *v)
175
{
176
	return blocking_notifier_call_chain(&memory_chain, val, v);
177 178
}

179 180 181 182 183
int memory_isolate_notify(unsigned long val, void *v)
{
	return atomic_notifier_call_chain(&memory_isolate_chain, val, v);
}

184 185 186 187
/*
 * The probe routines leave the pages reserved, just as the bootmem code does.
 * Make sure they're still that way.
 */
188
static bool pages_correctly_reserved(unsigned long start_pfn)
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
{
	int i, j;
	struct page *page;
	unsigned long pfn = start_pfn;

	/*
	 * memmap between sections is not contiguous except with
	 * SPARSEMEM_VMEMMAP. We lookup the page once per section
	 * and assume memmap is contiguous within each section
	 */
	for (i = 0; i < sections_per_block; i++, pfn += PAGES_PER_SECTION) {
		if (WARN_ON_ONCE(!pfn_valid(pfn)))
			return false;
		page = pfn_to_page(pfn);

		for (j = 0; j < PAGES_PER_SECTION; j++) {
			if (PageReserved(page + j))
				continue;

			printk(KERN_WARNING "section number %ld page number %d "
				"not reserved, was it already online?\n",
				pfn_to_section_nr(pfn), j);

			return false;
		}
	}

	return true;
}

219 220 221 222 223
/*
 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
 * OK to have direct references to sparsemem variables in here.
 */
static int
224
memory_block_action(unsigned long phys_index, unsigned long action, int online_type)
225
{
226
	unsigned long start_pfn;
227
	unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
228
	struct page *first_page;
229 230
	int ret;

231
	first_page = pfn_to_page(phys_index << PFN_SECTION_SHIFT);
232
	start_pfn = page_to_pfn(first_page);
233

234 235
	switch (action) {
		case MEM_ONLINE:
236
			if (!pages_correctly_reserved(start_pfn))
237 238
				return -EBUSY;

239
			ret = online_pages(start_pfn, nr_pages, online_type);
240 241
			break;
		case MEM_OFFLINE:
242
			ret = offline_pages(start_pfn, nr_pages);
243 244
			break;
		default:
245 246
			WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "
			     "%ld\n", __func__, phys_index, action, action);
247 248 249 250 251 252
			ret = -EINVAL;
	}

	return ret;
}

253 254
static int memory_block_change_state(struct memory_block *mem,
		unsigned long to_state, unsigned long from_state_req)
255
{
256
	int ret = 0;
257

258 259
	if (mem->state != from_state_req)
		return -EINVAL;
260

261 262 263
	if (to_state == MEM_OFFLINE)
		mem->state = MEM_GOING_OFFLINE;

264 265 266
	ret = memory_block_action(mem->start_section_nr, to_state,
				mem->online_type);

267
	mem->state = ret ? from_state_req : to_state;
268

269 270
	return ret;
}
271

272
/* The device lock serializes operations on memory_subsys_[online|offline] */
273 274
static int memory_subsys_online(struct device *dev)
{
275
	struct memory_block *mem = to_memory_block(dev);
276
	int ret;
277

278 279
	if (mem->state == MEM_ONLINE)
		return 0;
280

281 282 283 284 285 286 287
	/*
	 * If we are called from store_mem_state(), online_type will be
	 * set >= 0 Otherwise we were called from the device online
	 * attribute and need to set the online_type.
	 */
	if (mem->online_type < 0)
		mem->online_type = ONLINE_KEEP;
288

289
	ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE);
290

291 292
	/* clear online_type */
	mem->online_type = -1;
293 294 295 296 297

	return ret;
}

static int memory_subsys_offline(struct device *dev)
298
{
299
	struct memory_block *mem = to_memory_block(dev);
300

301 302
	if (mem->state == MEM_OFFLINE)
		return 0;
303

304
	return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE);
305
}
306

307
static ssize_t
308 309
store_mem_state(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t count)
310
{
311
	struct memory_block *mem = to_memory_block(dev);
312
	int ret, online_type;
313

314 315 316
	ret = lock_device_hotplug_sysfs();
	if (ret)
		return ret;
317

318 319 320 321 322 323 324 325
	if (!strncmp(buf, "online_kernel", min_t(int, count, 13)))
		online_type = ONLINE_KERNEL;
	else if (!strncmp(buf, "online_movable", min_t(int, count, 14)))
		online_type = ONLINE_MOVABLE;
	else if (!strncmp(buf, "online", min_t(int, count, 6)))
		online_type = ONLINE_KEEP;
	else if (!strncmp(buf, "offline", min_t(int, count, 7)))
		online_type = -1;
326 327 328 329
	else {
		ret = -EINVAL;
		goto err;
	}
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349

	switch (online_type) {
	case ONLINE_KERNEL:
	case ONLINE_MOVABLE:
	case ONLINE_KEEP:
		/*
		 * mem->online_type is not protected so there can be a
		 * race here.  However, when racing online, the first
		 * will succeed and the second will just return as the
		 * block will already be online.  The online type
		 * could be either one, but that is expected.
		 */
		mem->online_type = online_type;
		ret = device_online(&mem->dev);
		break;
	case -1:
		ret = device_offline(&mem->dev);
		break;
	default:
		ret = -EINVAL; /* should never happen */
350 351
	}

352
err:
353
	unlock_device_hotplug();
354

355 356 357 358 359 360 361 362 363 364 365 366 367 368
	if (ret)
		return ret;
	return count;
}

/*
 * phys_device is a bad name for this.  What I really want
 * is a way to differentiate between memory ranges that
 * are part of physical devices that constitute
 * a complete removable unit or fru.
 * i.e. do these ranges belong to the same physical device,
 * s.t. if I offline all of these sections I can then
 * remove the physical device?
 */
369 370
static ssize_t show_phys_device(struct device *dev,
				struct device_attribute *attr, char *buf)
371
{
372
	struct memory_block *mem = to_memory_block(dev);
373 374 375
	return sprintf(buf, "%d\n", mem->phys_device);
}

376 377 378 379
static DEVICE_ATTR(phys_index, 0444, show_mem_start_phys_index, NULL);
static DEVICE_ATTR(state, 0644, show_mem_state, store_mem_state);
static DEVICE_ATTR(phys_device, 0444, show_phys_device, NULL);
static DEVICE_ATTR(removable, 0444, show_mem_removable, NULL);
380 381 382 383 384

/*
 * Block size attribute stuff
 */
static ssize_t
385
print_block_size(struct device *dev, struct device_attribute *attr,
386
		 char *buf)
387
{
388
	return sprintf(buf, "%lx\n", get_memory_block_size());
389 390
}

391
static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL);
392 393 394 395 396 397 398 399 400

/*
 * Some architectures will have custom drivers to do this, and
 * will not need to do it from userspace.  The fake hot-add code
 * as well as ppc64 will do all of their discovery in userspace
 * and will require this interface.
 */
#ifdef CONFIG_ARCH_MEMORY_PROBE
static ssize_t
401
memory_probe_store(struct device *dev, struct device_attribute *attr,
402
		   const char *buf, size_t count)
403 404
{
	u64 phys_addr;
405
	int nid;
406
	int i, ret;
407
	unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
408 409 410

	phys_addr = simple_strtoull(buf, NULL, 0);

411 412 413
	if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
		return -EINVAL;

414 415 416 417 418
	for (i = 0; i < sections_per_block; i++) {
		nid = memory_add_physaddr_to_nid(phys_addr);
		ret = add_memory(nid, phys_addr,
				 PAGES_PER_SECTION << PAGE_SHIFT);
		if (ret)
419
			goto out;
420 421 422

		phys_addr += MIN_MEMORY_BLOCK_SIZE;
	}
423

424 425 426
	ret = count;
out:
	return ret;
427 428
}

429
static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store);
430 431
#endif

432 433 434 435 436 437 438
#ifdef CONFIG_MEMORY_FAILURE
/*
 * Support for offlining pages of memory
 */

/* Soft offline a page */
static ssize_t
439 440
store_soft_offline_page(struct device *dev,
			struct device_attribute *attr,
441
			const char *buf, size_t count)
442 443 444 445 446
{
	int ret;
	u64 pfn;
	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
447
	if (kstrtoull(buf, 0, &pfn) < 0)
448 449 450 451 452 453 454 455 456 457
		return -EINVAL;
	pfn >>= PAGE_SHIFT;
	if (!pfn_valid(pfn))
		return -ENXIO;
	ret = soft_offline_page(pfn_to_page(pfn), 0);
	return ret == 0 ? count : ret;
}

/* Forcibly offline a page, including killing processes. */
static ssize_t
458 459
store_hard_offline_page(struct device *dev,
			struct device_attribute *attr,
460
			const char *buf, size_t count)
461 462 463 464 465
{
	int ret;
	u64 pfn;
	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
466
	if (kstrtoull(buf, 0, &pfn) < 0)
467 468
		return -EINVAL;
	pfn >>= PAGE_SHIFT;
469
	ret = memory_failure(pfn, 0, 0);
470 471 472
	return ret ? ret : count;
}

473 474
static DEVICE_ATTR(soft_offline_page, S_IWUSR, NULL, store_soft_offline_page);
static DEVICE_ATTR(hard_offline_page, S_IWUSR, NULL, store_hard_offline_page);
475 476
#endif

477 478 479 480 481
/*
 * Note that phys_device is optional.  It is here to allow for
 * differentiation between which *physical* devices each
 * section belongs to...
 */
482 483 484 485
int __weak arch_get_memory_phys_device(unsigned long start_pfn)
{
	return 0;
}
486

487 488 489 490
/*
 * A reference for the returned object is held and the reference for the
 * hinted object is released.
 */
491 492
struct memory_block *find_memory_block_hinted(struct mem_section *section,
					      struct memory_block *hint)
493
{
494
	int block_id = base_memory_block_id(__section_nr(section));
495 496
	struct device *hintdev = hint ? &hint->dev : NULL;
	struct device *dev;
497

498 499 500 501
	dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev);
	if (hint)
		put_device(&hint->dev);
	if (!dev)
502
		return NULL;
503
	return to_memory_block(dev);
504 505
}

506 507 508 509 510 511
/*
 * For now, we have a linear search to go find the appropriate
 * memory_block corresponding to a particular phys_index. If
 * this gets to be a real problem, we can always use a radix
 * tree or something here.
 *
512
 * This could be made generic for all device subsystems.
513 514 515 516 517 518
 */
struct memory_block *find_memory_block(struct mem_section *section)
{
	return find_memory_block_hinted(section, NULL);
}

519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
static struct attribute *memory_memblk_attrs[] = {
	&dev_attr_phys_index.attr,
	&dev_attr_state.attr,
	&dev_attr_phys_device.attr,
	&dev_attr_removable.attr,
	NULL
};

static struct attribute_group memory_memblk_attr_group = {
	.attrs = memory_memblk_attrs,
};

static const struct attribute_group *memory_memblk_attr_groups[] = {
	&memory_memblk_attr_group,
	NULL,
};

/*
 * register_memory - Setup a sysfs device for a memory block
 */
static
int register_memory(struct memory_block *memory)
{
	memory->dev.bus = &memory_subsys;
	memory->dev.id = memory->start_section_nr / sections_per_block;
	memory->dev.release = memory_block_release;
	memory->dev.groups = memory_memblk_attr_groups;
546
	memory->dev.offline = memory->state == MEM_OFFLINE;
547

548
	return device_register(&memory->dev);
549 550
}

551 552
static int init_memory_block(struct memory_block **memory,
			     struct mem_section *section, unsigned long state)
553
{
554
	struct memory_block *mem;
555
	unsigned long start_pfn;
556
	int scn_nr;
557 558
	int ret = 0;

559
	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
560 561 562
	if (!mem)
		return -ENOMEM;

563
	scn_nr = __section_nr(section);
564 565 566
	mem->start_section_nr =
			base_memory_block_id(scn_nr) * sections_per_block;
	mem->end_section_nr = mem->start_section_nr + sections_per_block - 1;
567
	mem->state = state;
568
	mem->section_count++;
569
	start_pfn = section_nr_to_pfn(mem->start_section_nr);
570 571
	mem->phys_device = arch_get_memory_phys_device(start_pfn);

572 573 574 575 576 577
	ret = register_memory(mem);

	*memory = mem;
	return ret;
}

578
static int add_memory_block(int base_section_nr)
579
{
580 581
	struct memory_block *mem;
	int i, ret, section_count = 0, section_nr;
582

583 584 585 586 587 588 589 590
	for (i = base_section_nr;
	     (i < base_section_nr + sections_per_block) && i < NR_MEM_SECTIONS;
	     i++) {
		if (!present_section_nr(i))
			continue;
		if (section_count == 0)
			section_nr = i;
		section_count++;
591 592
	}

593 594 595 596 597 598 599
	if (section_count == 0)
		return 0;
	ret = init_memory_block(&mem, __nr_to_section(section_nr), MEM_ONLINE);
	if (ret)
		return ret;
	mem->section_count = section_count;
	return 0;
600 601
}

602

603 604 605 606 607 608
/*
 * need an interface for the VM to add new memory regions,
 * but without onlining it.
 */
int register_new_memory(int nid, struct mem_section *section)
{
609 610
	int ret = 0;
	struct memory_block *mem;
611 612 613

	mutex_lock(&mem_sysfs_mutex);

614 615 616 617 618 619 620 621 622 623 624 625 626 627
	mem = find_memory_block(section);
	if (mem) {
		mem->section_count++;
		put_device(&mem->dev);
	} else {
		ret = init_memory_block(&mem, section, MEM_OFFLINE);
		if (ret)
			goto out;
	}

	if (mem->section_count == sections_per_block)
		ret = register_mem_sect_under_node(mem, nid);
out:
	mutex_unlock(&mem_sysfs_mutex);
628
	return ret;
629 630 631 632 633 634 635 636 637
}

#ifdef CONFIG_MEMORY_HOTREMOVE
static void
unregister_memory(struct memory_block *memory)
{
	BUG_ON(memory->dev.bus != &memory_subsys);

	/* drop the ref. we got in remove_memory_block() */
638
	put_device(&memory->dev);
639 640 641 642 643
	device_unregister(&memory->dev);
}

static int remove_memory_block(unsigned long node_id,
			       struct mem_section *section, int phys_device)
644 645 646
{
	struct memory_block *mem;

647
	mutex_lock(&mem_sysfs_mutex);
648
	mem = find_memory_block(section);
649
	unregister_mem_sect_under_nodes(mem, __section_nr(section));
650 651

	mem->section_count--;
652
	if (mem->section_count == 0)
653
		unregister_memory(mem);
654
	else
655
		put_device(&mem->dev);
656

657
	mutex_unlock(&mem_sysfs_mutex);
658 659 660 661 662
	return 0;
}

int unregister_memory_section(struct mem_section *section)
{
663
	if (!present_section(section))
664 665 666 667
		return -EINVAL;

	return remove_memory_block(0, section, 0);
}
668
#endif /* CONFIG_MEMORY_HOTREMOVE */
669

670 671 672 673 674 675
/* return true if the memory block is offlined, otherwise, return false */
bool is_memblock_offlined(struct memory_block *mem)
{
	return mem->state == MEM_OFFLINE;
}

676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698
static struct attribute *memory_root_attrs[] = {
#ifdef CONFIG_ARCH_MEMORY_PROBE
	&dev_attr_probe.attr,
#endif

#ifdef CONFIG_MEMORY_FAILURE
	&dev_attr_soft_offline_page.attr,
	&dev_attr_hard_offline_page.attr,
#endif

	&dev_attr_block_size_bytes.attr,
	NULL
};

static struct attribute_group memory_root_attr_group = {
	.attrs = memory_root_attrs,
};

static const struct attribute_group *memory_root_attr_groups[] = {
	&memory_root_attr_group,
	NULL,
};

699 700 701 702 703 704 705
/*
 * Initialize the sysfs support for memory devices...
 */
int __init memory_dev_init(void)
{
	unsigned int i;
	int ret;
706
	int err;
707
	unsigned long block_sz;
708

709
	ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);
710 711
	if (ret)
		goto out;
712

713 714 715
	block_sz = get_memory_block_size();
	sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;

716 717 718 719
	/*
	 * Create entries for memory sections that were found
	 * during boot and have been initialized
	 */
720
	mutex_lock(&mem_sysfs_mutex);
721 722
	for (i = 0; i < NR_MEM_SECTIONS; i += sections_per_block) {
		err = add_memory_block(i);
723 724
		if (!ret)
			ret = err;
725
	}
726
	mutex_unlock(&mem_sysfs_mutex);
727

728 729
out:
	if (ret)
730
		printk(KERN_ERR "%s() failed: %d\n", __func__, ret);
731 732
	return ret;
}