memory.c 18.1 KB
Newer Older
1
/*
2
 * Memory subsystem support
3 4 5 6 7 8 9 10 11 12 13 14 15
 *
 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
 *            Dave Hansen <haveblue@us.ibm.com>
 *
 * This file provides the necessary infrastructure to represent
 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
 * All arch-independent code that assumes MEMORY_HOTPLUG requires
 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/topology.h>
16
#include <linux/capability.h>
17 18 19 20 21
#include <linux/device.h>
#include <linux/memory.h>
#include <linux/kobject.h>
#include <linux/memory_hotplug.h>
#include <linux/mm.h>
22
#include <linux/mutex.h>
23
#include <linux/stat.h>
24
#include <linux/slab.h>
25

Arun Sharma's avatar
Arun Sharma committed
26
#include <linux/atomic.h>
27 28
#include <asm/uaccess.h>

29 30
static DEFINE_MUTEX(mem_sysfs_mutex);

31
#define MEMORY_CLASS_NAME	"memory"
32 33 34 35 36 37 38

static int sections_per_block;

static inline int base_memory_block_id(int section_nr)
{
	return section_nr / sections_per_block;
}
39

40
static struct bus_type memory_subsys = {
41
	.name = MEMORY_CLASS_NAME,
42
	.dev_name = MEMORY_CLASS_NAME,
43 44
};

45
static BLOCKING_NOTIFIER_HEAD(memory_chain);
46

47
int register_memory_notifier(struct notifier_block *nb)
48
{
49
        return blocking_notifier_chain_register(&memory_chain, nb);
50
}
51
EXPORT_SYMBOL(register_memory_notifier);
52

53
void unregister_memory_notifier(struct notifier_block *nb)
54
{
55
        blocking_notifier_chain_unregister(&memory_chain, nb);
56
}
57
EXPORT_SYMBOL(unregister_memory_notifier);
58

59 60 61 62 63 64 65 66 67 68 69 70 71 72
static ATOMIC_NOTIFIER_HEAD(memory_isolate_chain);

int register_memory_isolate_notifier(struct notifier_block *nb)
{
	return atomic_notifier_chain_register(&memory_isolate_chain, nb);
}
EXPORT_SYMBOL(register_memory_isolate_notifier);

void unregister_memory_isolate_notifier(struct notifier_block *nb)
{
	atomic_notifier_chain_unregister(&memory_isolate_chain, nb);
}
EXPORT_SYMBOL(unregister_memory_isolate_notifier);

73 74 75 76 77 78 79
static void memory_block_release(struct device *dev)
{
	struct memory_block *mem = container_of(dev, struct memory_block, dev);

	kfree(mem);
}

80 81 82
/*
 * register_memory - Setup a sysfs device for a memory block
 */
83
static
84
int register_memory(struct memory_block *memory)
85 86 87
{
	int error;

88 89
	memory->dev.bus = &memory_subsys;
	memory->dev.id = memory->start_section_nr / sections_per_block;
90
	memory->dev.release = memory_block_release;
91

92
	error = device_register(&memory->dev);
93 94 95
	return error;
}

96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
unsigned long __weak memory_block_size_bytes(void)
{
	return MIN_MEMORY_BLOCK_SIZE;
}

static unsigned long get_memory_block_size(void)
{
	unsigned long block_sz;

	block_sz = memory_block_size_bytes();

	/* Validate blk_sz is a power of 2 and not less than section size */
	if ((block_sz & (block_sz - 1)) || (block_sz < MIN_MEMORY_BLOCK_SIZE)) {
		WARN_ON(1);
		block_sz = MIN_MEMORY_BLOCK_SIZE;
	}

	return block_sz;
}

116 117 118 119 120
/*
 * use this as the physical section index that this memsection
 * uses.
 */

121 122
static ssize_t show_mem_start_phys_index(struct device *dev,
			struct device_attribute *attr, char *buf)
123 124
{
	struct memory_block *mem =
125
		container_of(dev, struct memory_block, dev);
126 127 128 129 130 131
	unsigned long phys_index;

	phys_index = mem->start_section_nr / sections_per_block;
	return sprintf(buf, "%08lx\n", phys_index);
}

132 133
static ssize_t show_mem_end_phys_index(struct device *dev,
			struct device_attribute *attr, char *buf)
134 135
{
	struct memory_block *mem =
136
		container_of(dev, struct memory_block, dev);
137 138 139 140
	unsigned long phys_index;

	phys_index = mem->end_section_nr / sections_per_block;
	return sprintf(buf, "%08lx\n", phys_index);
141 142
}

143 144 145
/*
 * Show whether the section of memory is likely to be hot-removable
 */
146 147
static ssize_t show_mem_removable(struct device *dev,
			struct device_attribute *attr, char *buf)
148
{
149 150
	unsigned long i, pfn;
	int ret = 1;
151
	struct memory_block *mem =
152
		container_of(dev, struct memory_block, dev);
153

154
	for (i = 0; i < sections_per_block; i++) {
155
		pfn = section_nr_to_pfn(mem->start_section_nr + i);
156 157 158
		ret &= is_mem_section_removable(pfn, PAGES_PER_SECTION);
	}

159 160 161
	return sprintf(buf, "%d\n", ret);
}

162 163 164
/*
 * online, offline, going offline, etc.
 */
165 166
static ssize_t show_mem_state(struct device *dev,
			struct device_attribute *attr, char *buf)
167 168
{
	struct memory_block *mem =
169
		container_of(dev, struct memory_block, dev);
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
	ssize_t len = 0;

	/*
	 * We can probably put these states in a nice little array
	 * so that they're not open-coded
	 */
	switch (mem->state) {
		case MEM_ONLINE:
			len = sprintf(buf, "online\n");
			break;
		case MEM_OFFLINE:
			len = sprintf(buf, "offline\n");
			break;
		case MEM_GOING_OFFLINE:
			len = sprintf(buf, "going-offline\n");
			break;
		default:
			len = sprintf(buf, "ERROR-UNKNOWN-%ld\n",
					mem->state);
			WARN_ON(1);
			break;
	}

	return len;
}

196
int memory_notify(unsigned long val, void *v)
197
{
198
	return blocking_notifier_call_chain(&memory_chain, val, v);
199 200
}

201 202 203 204 205
int memory_isolate_notify(unsigned long val, void *v)
{
	return atomic_notifier_call_chain(&memory_isolate_chain, val, v);
}

206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
/*
 * The probe routines leave the pages reserved, just as the bootmem code does.
 * Make sure they're still that way.
 */
static bool pages_correctly_reserved(unsigned long start_pfn,
					unsigned long nr_pages)
{
	int i, j;
	struct page *page;
	unsigned long pfn = start_pfn;

	/*
	 * memmap between sections is not contiguous except with
	 * SPARSEMEM_VMEMMAP. We lookup the page once per section
	 * and assume memmap is contiguous within each section
	 */
	for (i = 0; i < sections_per_block; i++, pfn += PAGES_PER_SECTION) {
		if (WARN_ON_ONCE(!pfn_valid(pfn)))
			return false;
		page = pfn_to_page(pfn);

		for (j = 0; j < PAGES_PER_SECTION; j++) {
			if (PageReserved(page + j))
				continue;

			printk(KERN_WARNING "section number %ld page number %d "
				"not reserved, was it already online?\n",
				pfn_to_section_nr(pfn), j);

			return false;
		}
	}

	return true;
}

242 243 244 245 246
/*
 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
 * OK to have direct references to sparsemem variables in here.
 */
static int
247
memory_block_action(unsigned long phys_index, unsigned long action, int online_type)
248
{
249
	unsigned long start_pfn;
250
	unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
251
	struct page *first_page;
252 253
	int ret;

254
	first_page = pfn_to_page(phys_index << PFN_SECTION_SHIFT);
255
	start_pfn = page_to_pfn(first_page);
256

257 258
	switch (action) {
		case MEM_ONLINE:
259 260 261
			if (!pages_correctly_reserved(start_pfn, nr_pages))
				return -EBUSY;

262
			ret = online_pages(start_pfn, nr_pages, online_type);
263 264
			break;
		case MEM_OFFLINE:
265
			ret = offline_pages(start_pfn, nr_pages);
266 267
			break;
		default:
268 269
			WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "
			     "%ld\n", __func__, phys_index, action, action);
270 271 272 273 274 275
			ret = -EINVAL;
	}

	return ret;
}

276
static int __memory_block_change_state(struct memory_block *mem,
277 278
		unsigned long to_state, unsigned long from_state_req,
		int online_type)
279
{
280
	int ret = 0;
281

282 283 284 285 286
	if (mem->state != from_state_req) {
		ret = -EINVAL;
		goto out;
	}

287 288 289
	if (to_state == MEM_OFFLINE)
		mem->state = MEM_GOING_OFFLINE;

290
	ret = memory_block_action(mem->start_section_nr, to_state, online_type);
291

292
	if (ret) {
293
		mem->state = from_state_req;
294 295
		goto out;
	}
296

297 298 299 300 301 302 303 304 305 306 307
	mem->state = to_state;
	switch (mem->state) {
	case MEM_OFFLINE:
		kobject_uevent(&mem->dev.kobj, KOBJ_OFFLINE);
		break;
	case MEM_ONLINE:
		kobject_uevent(&mem->dev.kobj, KOBJ_ONLINE);
		break;
	default:
		break;
	}
308 309 310 311
out:
	return ret;
}

312
static int memory_block_change_state(struct memory_block *mem,
313 314
		unsigned long to_state, unsigned long from_state_req,
		int online_type)
315 316 317 318
{
	int ret;

	mutex_lock(&mem->state_mutex);
319 320
	ret = __memory_block_change_state(mem, to_state, from_state_req,
					  online_type);
321 322 323 324
	mutex_unlock(&mem->state_mutex);

	return ret;
}
325
static ssize_t
326 327
store_mem_state(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t count)
328 329 330 331
{
	struct memory_block *mem;
	int ret = -EINVAL;

332
	mem = container_of(dev, struct memory_block, dev);
333

334 335 336 337 338 339 340 341 342 343 344 345
	if (!strncmp(buf, "online_kernel", min_t(int, count, 13)))
		ret = memory_block_change_state(mem, MEM_ONLINE,
						MEM_OFFLINE, ONLINE_KERNEL);
	else if (!strncmp(buf, "online_movable", min_t(int, count, 14)))
		ret = memory_block_change_state(mem, MEM_ONLINE,
						MEM_OFFLINE, ONLINE_MOVABLE);
	else if (!strncmp(buf, "online", min_t(int, count, 6)))
		ret = memory_block_change_state(mem, MEM_ONLINE,
						MEM_OFFLINE, ONLINE_KEEP);
	else if(!strncmp(buf, "offline", min_t(int, count, 7)))
		ret = memory_block_change_state(mem, MEM_OFFLINE,
						MEM_ONLINE, -1);
346

347 348 349 350 351 352 353 354 355 356 357 358 359 360
	if (ret)
		return ret;
	return count;
}

/*
 * phys_device is a bad name for this.  What I really want
 * is a way to differentiate between memory ranges that
 * are part of physical devices that constitute
 * a complete removable unit or fru.
 * i.e. do these ranges belong to the same physical device,
 * s.t. if I offline all of these sections I can then
 * remove the physical device?
 */
361 362
static ssize_t show_phys_device(struct device *dev,
				struct device_attribute *attr, char *buf)
363 364
{
	struct memory_block *mem =
365
		container_of(dev, struct memory_block, dev);
366 367 368
	return sprintf(buf, "%d\n", mem->phys_device);
}

369 370 371 372 373
static DEVICE_ATTR(phys_index, 0444, show_mem_start_phys_index, NULL);
static DEVICE_ATTR(end_phys_index, 0444, show_mem_end_phys_index, NULL);
static DEVICE_ATTR(state, 0644, show_mem_state, store_mem_state);
static DEVICE_ATTR(phys_device, 0444, show_phys_device, NULL);
static DEVICE_ATTR(removable, 0444, show_mem_removable, NULL);
374 375

#define mem_create_simple_file(mem, attr_name)	\
376
	device_create_file(&mem->dev, &dev_attr_##attr_name)
377
#define mem_remove_simple_file(mem, attr_name)	\
378
	device_remove_file(&mem->dev, &dev_attr_##attr_name)
379 380 381 382 383

/*
 * Block size attribute stuff
 */
static ssize_t
384
print_block_size(struct device *dev, struct device_attribute *attr,
385
		 char *buf)
386
{
387
	return sprintf(buf, "%lx\n", get_memory_block_size());
388 389
}

390
static DEVICE_ATTR(block_size_bytes, 0444, print_block_size, NULL);
391 392 393

static int block_size_init(void)
{
394 395
	return device_create_file(memory_subsys.dev_root,
				  &dev_attr_block_size_bytes);
396 397 398 399 400 401 402 403 404 405
}

/*
 * Some architectures will have custom drivers to do this, and
 * will not need to do it from userspace.  The fake hot-add code
 * as well as ppc64 will do all of their discovery in userspace
 * and will require this interface.
 */
#ifdef CONFIG_ARCH_MEMORY_PROBE
static ssize_t
406
memory_probe_store(struct device *dev, struct device_attribute *attr,
407
		   const char *buf, size_t count)
408 409
{
	u64 phys_addr;
410
	int nid;
411
	int i, ret;
412
	unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
413 414 415

	phys_addr = simple_strtoull(buf, NULL, 0);

416 417 418
	if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
		return -EINVAL;

419 420 421 422 423
	for (i = 0; i < sections_per_block; i++) {
		nid = memory_add_physaddr_to_nid(phys_addr);
		ret = add_memory(nid, phys_addr,
				 PAGES_PER_SECTION << PAGE_SHIFT);
		if (ret)
424
			goto out;
425 426 427

		phys_addr += MIN_MEMORY_BLOCK_SIZE;
	}
428

429 430 431
	ret = count;
out:
	return ret;
432
}
433
static DEVICE_ATTR(probe, S_IWUSR, NULL, memory_probe_store);
434 435 436

static int memory_probe_init(void)
{
437
	return device_create_file(memory_subsys.dev_root, &dev_attr_probe);
438 439
}
#else
440 441 442 443
static inline int memory_probe_init(void)
{
	return 0;
}
444 445
#endif

446 447 448 449 450 451 452
#ifdef CONFIG_MEMORY_FAILURE
/*
 * Support for offlining pages of memory
 */

/* Soft offline a page */
static ssize_t
453 454
store_soft_offline_page(struct device *dev,
			struct device_attribute *attr,
455
			const char *buf, size_t count)
456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
{
	int ret;
	u64 pfn;
	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
	if (strict_strtoull(buf, 0, &pfn) < 0)
		return -EINVAL;
	pfn >>= PAGE_SHIFT;
	if (!pfn_valid(pfn))
		return -ENXIO;
	ret = soft_offline_page(pfn_to_page(pfn), 0);
	return ret == 0 ? count : ret;
}

/* Forcibly offline a page, including killing processes. */
static ssize_t
472 473
store_hard_offline_page(struct device *dev,
			struct device_attribute *attr,
474
			const char *buf, size_t count)
475 476 477 478 479 480 481 482
{
	int ret;
	u64 pfn;
	if (!capable(CAP_SYS_ADMIN))
		return -EPERM;
	if (strict_strtoull(buf, 0, &pfn) < 0)
		return -EINVAL;
	pfn >>= PAGE_SHIFT;
483
	ret = memory_failure(pfn, 0, 0);
484 485 486
	return ret ? ret : count;
}

487 488
static DEVICE_ATTR(soft_offline_page, S_IWUSR, NULL, store_soft_offline_page);
static DEVICE_ATTR(hard_offline_page, S_IWUSR, NULL, store_hard_offline_page);
489 490 491 492 493

static __init int memory_fail_init(void)
{
	int err;

494 495
	err = device_create_file(memory_subsys.dev_root,
				&dev_attr_soft_offline_page);
496
	if (!err)
497 498
		err = device_create_file(memory_subsys.dev_root,
				&dev_attr_hard_offline_page);
499 500 501 502 503 504 505 506 507
	return err;
}
#else
static inline int memory_fail_init(void)
{
	return 0;
}
#endif

508 509 510 511 512
/*
 * Note that phys_device is optional.  It is here to allow for
 * differentiation between which *physical* devices each
 * section belongs to...
 */
513 514 515 516
int __weak arch_get_memory_phys_device(unsigned long start_pfn)
{
	return 0;
}
517

518 519 520 521
/*
 * A reference for the returned object is held and the reference for the
 * hinted object is released.
 */
522 523
struct memory_block *find_memory_block_hinted(struct mem_section *section,
					      struct memory_block *hint)
524
{
525
	int block_id = base_memory_block_id(__section_nr(section));
526 527
	struct device *hintdev = hint ? &hint->dev : NULL;
	struct device *dev;
528

529 530 531 532
	dev = subsys_find_device_by_id(&memory_subsys, block_id, hintdev);
	if (hint)
		put_device(&hint->dev);
	if (!dev)
533
		return NULL;
534
	return container_of(dev, struct memory_block, dev);
535 536
}

537 538 539 540 541 542
/*
 * For now, we have a linear search to go find the appropriate
 * memory_block corresponding to a particular phys_index. If
 * this gets to be a real problem, we can always use a radix
 * tree or something here.
 *
543
 * This could be made generic for all device subsystems.
544 545 546 547 548 549
 */
struct memory_block *find_memory_block(struct mem_section *section)
{
	return find_memory_block_hinted(section, NULL);
}

550 551
static int init_memory_block(struct memory_block **memory,
			     struct mem_section *section, unsigned long state)
552
{
553
	struct memory_block *mem;
554
	unsigned long start_pfn;
555
	int scn_nr;
556 557
	int ret = 0;

558
	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
559 560 561
	if (!mem)
		return -ENOMEM;

562
	scn_nr = __section_nr(section);
563 564 565
	mem->start_section_nr =
			base_memory_block_id(scn_nr) * sections_per_block;
	mem->end_section_nr = mem->start_section_nr + sections_per_block - 1;
566
	mem->state = state;
567
	mem->section_count++;
568
	mutex_init(&mem->state_mutex);
569
	start_pfn = section_nr_to_pfn(mem->start_section_nr);
570 571
	mem->phys_device = arch_get_memory_phys_device(start_pfn);

572
	ret = register_memory(mem);
573 574
	if (!ret)
		ret = mem_create_simple_file(mem, phys_index);
575 576
	if (!ret)
		ret = mem_create_simple_file(mem, end_phys_index);
577 578 579 580 581 582
	if (!ret)
		ret = mem_create_simple_file(mem, state);
	if (!ret)
		ret = mem_create_simple_file(mem, phys_device);
	if (!ret)
		ret = mem_create_simple_file(mem, removable);
583 584 585 586 587 588

	*memory = mem;
	return ret;
}

static int add_memory_section(int nid, struct mem_section *section,
589
			struct memory_block **mem_p,
590 591
			unsigned long state, enum mem_add_context context)
{
592 593
	struct memory_block *mem = NULL;
	int scn_nr = __section_nr(section);
594 595 596 597
	int ret = 0;

	mutex_lock(&mem_sysfs_mutex);

598 599 600 601 602 603 604 605 606 607 608
	if (context == BOOT) {
		/* same memory block ? */
		if (mem_p && *mem_p)
			if (scn_nr >= (*mem_p)->start_section_nr &&
			    scn_nr <= (*mem_p)->end_section_nr) {
				mem = *mem_p;
				kobject_get(&mem->dev.kobj);
			}
	} else
		mem = find_memory_block(section);

609 610
	if (mem) {
		mem->section_count++;
611
		kobject_put(&mem->dev.kobj);
612
	} else {
613
		ret = init_memory_block(&mem, section, state);
614 615 616 617 618
		/* store memory_block pointer for next loop */
		if (!ret && context == BOOT)
			if (mem_p)
				*mem_p = mem;
	}
619

620
	if (!ret) {
621 622
		if (context == HOTPLUG &&
		    mem->section_count == sections_per_block)
623 624 625
			ret = register_mem_sect_under_node(mem, nid);
	}

626
	mutex_unlock(&mem_sysfs_mutex);
627 628 629
	return ret;
}

630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
/*
 * need an interface for the VM to add new memory regions,
 * but without onlining it.
 */
int register_new_memory(int nid, struct mem_section *section)
{
	return add_memory_section(nid, section, NULL, MEM_OFFLINE, HOTPLUG);
}

#ifdef CONFIG_MEMORY_HOTREMOVE
static void
unregister_memory(struct memory_block *memory)
{
	BUG_ON(memory->dev.bus != &memory_subsys);

	/* drop the ref. we got in remove_memory_block() */
	kobject_put(&memory->dev.kobj);
	device_unregister(&memory->dev);
}

static int remove_memory_block(unsigned long node_id,
			       struct mem_section *section, int phys_device)
652 653 654
{
	struct memory_block *mem;

655
	mutex_lock(&mem_sysfs_mutex);
656
	mem = find_memory_block(section);
657
	unregister_mem_sect_under_nodes(mem, __section_nr(section));
658 659 660 661

	mem->section_count--;
	if (mem->section_count == 0) {
		mem_remove_simple_file(mem, phys_index);
662
		mem_remove_simple_file(mem, end_phys_index);
663 664 665
		mem_remove_simple_file(mem, state);
		mem_remove_simple_file(mem, phys_device);
		mem_remove_simple_file(mem, removable);
666 667
		unregister_memory(mem);
	} else
668
		kobject_put(&mem->dev.kobj);
669

670
	mutex_unlock(&mem_sysfs_mutex);
671 672 673 674 675
	return 0;
}

int unregister_memory_section(struct mem_section *section)
{
676
	if (!present_section(section))
677 678 679 680
		return -EINVAL;

	return remove_memory_block(0, section, 0);
}
681
#endif /* CONFIG_MEMORY_HOTREMOVE */
682

683 684 685 686 687 688 689 690 691
/*
 * offline one memory block. If the memory block has been offlined, do nothing.
 */
int offline_memory_block(struct memory_block *mem)
{
	int ret = 0;

	mutex_lock(&mem->state_mutex);
	if (mem->state != MEM_OFFLINE)
692
		ret = __memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE, -1);
693 694 695 696 697
	mutex_unlock(&mem->state_mutex);

	return ret;
}

698 699 700 701 702 703
/* return true if the memory block is offlined, otherwise, return false */
bool is_memblock_offlined(struct memory_block *mem)
{
	return mem->state == MEM_OFFLINE;
}

704 705 706 707 708 709 710
/*
 * Initialize the sysfs support for memory devices...
 */
int __init memory_dev_init(void)
{
	unsigned int i;
	int ret;
711
	int err;
712
	unsigned long block_sz;
713
	struct memory_block *mem = NULL;
714

715
	ret = subsys_system_register(&memory_subsys, NULL);
716 717
	if (ret)
		goto out;
718

719 720 721
	block_sz = get_memory_block_size();
	sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;

722 723 724 725 726
	/*
	 * Create entries for memory sections that were found
	 * during boot and have been initialized
	 */
	for (i = 0; i < NR_MEM_SECTIONS; i++) {
727
		if (!present_section_nr(i))
728
			continue;
729 730 731 732
		/* don't need to reuse memory_block if only one per block */
		err = add_memory_section(0, __nr_to_section(i),
				 (sections_per_block == 1) ? NULL : &mem,
					 MEM_ONLINE,
733
					 BOOT);
734 735
		if (!ret)
			ret = err;
736 737
	}

738
	err = memory_probe_init();
739 740 741
	if (!ret)
		ret = err;
	err = memory_fail_init();
742 743 744 745 746 747 748
	if (!ret)
		ret = err;
	err = block_size_init();
	if (!ret)
		ret = err;
out:
	if (ret)
749
		printk(KERN_ERR "%s() failed: %d\n", __func__, ret);
750 751
	return ret;
}