FILE
memory_hotplug.c
File preview with syntax highlighting.
linux-6.19.14/mm/memory_hotplug.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * linux/mm/memory_hotplug.c 4 * 5 * Copyright (C) 6 */ 7 8#include <linux/stddef.h> 9#include <linux/mm.h> 10#include <linux/sched/signal.h> 11#include <linux/swap.h> 12#include <linux/interrupt.h> 13#include <linux/pagemap.h> 14#include <linux/compiler.h> 15#include <linux/export.h> 16#include <linux/writeback.h> 17#include <linux/slab.h> 18#include <linux/sysctl.h> 19#include <linux/cpu.h> 20#include <linux/memory.h> 21#include <linux/memremap.h> 22#include <linux/memory_hotplug.h> 23#include <linux/vmalloc.h> 24#include <linux/ioport.h> 25#include <linux/delay.h> 26#include <linux/migrate.h> 27#include <linux/page-isolation.h> 28#include <linux/pfn.h> 29#include <linux/suspend.h> 30#include <linux/mm_inline.h> 31#include <linux/firmware-map.h> 32#include <linux/stop_machine.h> 33#include <linux/hugetlb.h> 34#include <linux/memblock.h> 35#include <linux/compaction.h> 36#include <linux/rmap.h> 37#include <linux/module.h> 38#include <linux/node.h> 39 40#include <asm/tlbflush.h> 41 42#include "internal.h" 43#include "shuffle.h" 44 45enum { 46 MEMMAP_ON_MEMORY_DISABLE = 0, 47 MEMMAP_ON_MEMORY_ENABLE, 48 MEMMAP_ON_MEMORY_FORCE, 49}; 50 51static int memmap_mode __read_mostly = MEMMAP_ON_MEMORY_DISABLE; 52 53static inline unsigned long memory_block_memmap_size(void) 54{ 55 return PHYS_PFN(memory_block_size_bytes()) * sizeof(struct page); 56} 57 58static inline unsigned long memory_block_memmap_on_memory_pages(void) 59{ 60 unsigned long nr_pages = PFN_UP(memory_block_memmap_size()); 61 62 /* 63 * In "forced" memmap_on_memory mode, we add extra pages to align the 64 * vmemmap size to cover full pageblocks. That way, we can add memory 65 * even if the vmemmap size is not properly aligned, however, we might waste 66 * memory. 67 */ 68 if (memmap_mode == MEMMAP_ON_MEMORY_FORCE) 69 return pageblock_align(nr_pages); 70 return nr_pages; 71} 72 73#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY 74/* 75 * memory_hotplug.memmap_on_memory parameter 76 */ 77static int set_memmap_mode(const char *val, const struct kernel_param *kp) 78{ 79 int ret, mode; 80 bool enabled; 81 82 if (sysfs_streq(val, "force") || sysfs_streq(val, "FORCE")) { 83 mode = MEMMAP_ON_MEMORY_FORCE; 84 } else { 85 ret = kstrtobool(val, &enabled); 86 if (ret < 0) 87 return ret; 88 if (enabled) 89 mode = MEMMAP_ON_MEMORY_ENABLE; 90 else 91 mode = MEMMAP_ON_MEMORY_DISABLE; 92 } 93 *((int *)kp->arg) = mode; 94 if (mode == MEMMAP_ON_MEMORY_FORCE) { 95 unsigned long memmap_pages = memory_block_memmap_on_memory_pages(); 96 97 pr_info_once("Memory hotplug will waste %ld pages in each memory block\n", 98 memmap_pages - PFN_UP(memory_block_memmap_size())); 99 } 100 return 0; 101} 102 103static int get_memmap_mode(char *buffer, const struct kernel_param *kp) 104{ 105 int mode = *((int *)kp->arg); 106 107 if (mode == MEMMAP_ON_MEMORY_FORCE) 108 return sprintf(buffer, "force\n"); 109 return sprintf(buffer, "%c\n", mode ? 'Y' : 'N'); 110} 111 112static const struct kernel_param_ops memmap_mode_ops = { 113 .set = set_memmap_mode, 114 .get = get_memmap_mode, 115}; 116module_param_cb(memmap_on_memory, &memmap_mode_ops, &memmap_mode, 0444); 117MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug\n" 118 "With value \"force\" it could result in memory wastage due " 119 "to memmap size limitations (Y/N/force)"); 120 121static inline bool mhp_memmap_on_memory(void) 122{ 123 return memmap_mode != MEMMAP_ON_MEMORY_DISABLE; 124} 125#else 126static inline bool mhp_memmap_on_memory(void) 127{ 128 return false; 129} 130#endif 131 132enum { 133 ONLINE_POLICY_CONTIG_ZONES = 0, 134 ONLINE_POLICY_AUTO_MOVABLE, 135}; 136 137static const char * const online_policy_to_str[] = { 138 [ONLINE_POLICY_CONTIG_ZONES] = "contig-zones", 139 [ONLINE_POLICY_AUTO_MOVABLE] = "auto-movable", 140}; 141 142static int set_online_policy(const char *val, const struct kernel_param *kp) 143{ 144 int ret = sysfs_match_string(online_policy_to_str, val); 145 146 if (ret < 0) 147 return ret; 148 *((int *)kp->arg) = ret; 149 return 0; 150} 151 152static int get_online_policy(char *buffer, const struct kernel_param *kp) 153{ 154 return sprintf(buffer, "%s\n", online_policy_to_str[*((int *)kp->arg)]); 155} 156 157/* 158 * memory_hotplug.online_policy: configure online behavior when onlining without 159 * specifying a zone (MMOP_ONLINE) 160 * 161 * "contig-zones": keep zone contiguous 162 * "auto-movable": online memory to ZONE_MOVABLE if the configuration 163 * (auto_movable_ratio, auto_movable_numa_aware) allows for it 164 */ 165static int online_policy __read_mostly = ONLINE_POLICY_CONTIG_ZONES; 166static const struct kernel_param_ops online_policy_ops = { 167 .set = set_online_policy, 168 .get = get_online_policy, 169}; 170module_param_cb(online_policy, &online_policy_ops, &online_policy, 0644); 171MODULE_PARM_DESC(online_policy, 172 "Set the online policy (\"contig-zones\", \"auto-movable\") " 173 "Default: \"contig-zones\""); 174 175/* 176 * memory_hotplug.auto_movable_ratio: specify maximum MOVABLE:KERNEL ratio 177 * 178 * The ratio represent an upper limit and the kernel might decide to not 179 * online some memory to ZONE_MOVABLE -- e.g., because hotplugged KERNEL memory 180 * doesn't allow for more MOVABLE memory. 181 */ 182static unsigned int auto_movable_ratio __read_mostly = 301; 183module_param(auto_movable_ratio, uint, 0644); 184MODULE_PARM_DESC(auto_movable_ratio, 185 "Set the maximum ratio of MOVABLE:KERNEL memory in the system " 186 "in percent for \"auto-movable\" online policy. Default: 301"); 187 188/* 189 * memory_hotplug.auto_movable_numa_aware: consider numa node stats 190 */ 191#ifdef CONFIG_NUMA 192static bool auto_movable_numa_aware __read_mostly = true; 193module_param(auto_movable_numa_aware, bool, 0644); 194MODULE_PARM_DESC(auto_movable_numa_aware, 195 "Consider numa node stats in addition to global stats in " 196 "\"auto-movable\" online policy. Default: true"); 197#endif /* CONFIG_NUMA */ 198 199/* 200 * online_page_callback contains pointer to current page onlining function. 201 * Initially it is generic_online_page(). If it is required it could be 202 * changed by calling set_online_page_callback() for callback registration 203 * and restore_online_page_callback() for generic callback restore. 204 */ 205 206static online_page_callback_t online_page_callback = generic_online_page; 207static DEFINE_MUTEX(online_page_callback_lock); 208 209DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock); 210 211void get_online_mems(void) 212{ 213 percpu_down_read(&mem_hotplug_lock); 214} 215 216void put_online_mems(void) 217{ 218 percpu_up_read(&mem_hotplug_lock); 219} 220 221bool movable_node_enabled = false; 222 223static int mhp_default_online_type = -1; 224int mhp_get_default_online_type(void) 225{ 226 if (mhp_default_online_type >= 0) 227 return mhp_default_online_type; 228 229 if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_OFFLINE)) 230 mhp_default_online_type = MMOP_OFFLINE; 231 else if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_ONLINE_AUTO)) 232 mhp_default_online_type = MMOP_ONLINE; 233 else if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_ONLINE_KERNEL)) 234 mhp_default_online_type = MMOP_ONLINE_KERNEL; 235 else if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_ONLINE_MOVABLE)) 236 mhp_default_online_type = MMOP_ONLINE_MOVABLE; 237 else 238 mhp_default_online_type = MMOP_OFFLINE; 239 240 return mhp_default_online_type; 241} 242 243void mhp_set_default_online_type(int online_type) 244{ 245 mhp_default_online_type = online_type; 246} 247 248static int __init setup_memhp_default_state(char *str) 249{ 250 const int online_type = mhp_online_type_from_str(str); 251 252 if (online_type >= 0) 253 mhp_default_online_type = online_type; 254 255 return 1; 256} 257__setup("memhp_default_state=", setup_memhp_default_state); 258 259void mem_hotplug_begin(void) 260{ 261 cpus_read_lock(); 262 percpu_down_write(&mem_hotplug_lock); 263} 264 265void mem_hotplug_done(void) 266{ 267 percpu_up_write(&mem_hotplug_lock); 268 cpus_read_unlock(); 269} 270 271u64 max_mem_size = U64_MAX; 272 273/* add this memory to iomem resource */ 274static struct resource *register_memory_resource(u64 start, u64 size, 275 const char *resource_name) 276{ 277 struct resource *res; 278 unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; 279 280 if (strcmp(resource_name, "System RAM")) 281 flags |= IORESOURCE_SYSRAM_DRIVER_MANAGED; 282 283 if (!mhp_range_allowed(start, size, true)) 284 return ERR_PTR(-E2BIG); 285 286 /* 287 * Make sure value parsed from 'mem=' only restricts memory adding 288 * while booting, so that memory hotplug won't be impacted. Please 289 * refer to document of 'mem=' in kernel-parameters.txt for more 290 * details. 291 */ 292 if (start + size > max_mem_size && system_state < SYSTEM_RUNNING) 293 return ERR_PTR(-E2BIG); 294 295 /* 296 * Request ownership of the new memory range. This might be 297 * a child of an existing resource that was present but 298 * not marked as busy. 299 */ 300 res = __request_region(&iomem_resource, start, size, 301 resource_name, flags); 302 303 if (!res) { 304 pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n", 305 start, start + size); 306 return ERR_PTR(-EEXIST); 307 } 308 return res; 309} 310 311static void release_memory_resource(struct resource *res) 312{ 313 if (!res) 314 return; 315 release_resource(res); 316 kfree(res); 317} 318 319static int check_pfn_span(unsigned long pfn, unsigned long nr_pages) 320{ 321 /* 322 * Disallow all operations smaller than a sub-section and only 323 * allow operations smaller than a section for 324 * SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range() 325 * enforces a larger memory_block_size_bytes() granularity for 326 * memory that will be marked online, so this check should only 327 * fire for direct arch_{add,remove}_memory() users outside of 328 * add_memory_resource(). 329 */ 330 unsigned long min_align; 331 332 if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 333 min_align = PAGES_PER_SUBSECTION; 334 else 335 min_align = PAGES_PER_SECTION; 336 if (!IS_ALIGNED(pfn | nr_pages, min_align)) 337 return -EINVAL; 338 return 0; 339} 340 341/* 342 * Return page for the valid pfn only if the page is online. All pfn 343 * walkers which rely on the fully initialized page->flags and others 344 * should use this rather than pfn_valid && pfn_to_page 345 */ 346struct page *pfn_to_online_page(unsigned long pfn) 347{ 348 unsigned long nr = pfn_to_section_nr(pfn); 349 struct dev_pagemap *pgmap; 350 struct mem_section *ms; 351 352 if (nr >= NR_MEM_SECTIONS) 353 return NULL; 354 355 ms = __nr_to_section(nr); 356 if (!online_section(ms)) 357 return NULL; 358 359 /* 360 * Save some code text when online_section() + 361 * pfn_section_valid() are sufficient. 362 */ 363 if (IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) && !pfn_valid(pfn)) 364 return NULL; 365 366 if (!pfn_section_valid(ms, pfn)) 367 return NULL; 368 369 if (!online_device_section(ms)) 370 return pfn_to_page(pfn); 371 372 /* 373 * Slowpath: when ZONE_DEVICE collides with 374 * ZONE_{NORMAL,MOVABLE} within the same section some pfns in 375 * the section may be 'offline' but 'valid'. Only 376 * get_dev_pagemap() can determine sub-section online status. 377 */ 378 pgmap = get_dev_pagemap(pfn); 379 put_dev_pagemap(pgmap); 380 381 /* The presence of a pgmap indicates ZONE_DEVICE offline pfn */ 382 if (pgmap) 383 return NULL; 384 385 return pfn_to_page(pfn); 386} 387EXPORT_SYMBOL_GPL(pfn_to_online_page); 388 389int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, 390 struct mhp_params *params) 391{ 392 const unsigned long end_pfn = pfn + nr_pages; 393 unsigned long cur_nr_pages; 394 int err; 395 struct vmem_altmap *altmap = params->altmap; 396 397 if (WARN_ON_ONCE(!pgprot_val(params->pgprot))) 398 return -EINVAL; 399 400 VM_BUG_ON(!mhp_range_allowed(PFN_PHYS(pfn), nr_pages * PAGE_SIZE, false)); 401 402 if (altmap) { 403 /* 404 * Validate altmap is within bounds of the total request 405 */ 406 if (altmap->base_pfn != pfn 407 || vmem_altmap_offset(altmap) > nr_pages) { 408 pr_warn_once("memory add fail, invalid altmap\n"); 409 return -EINVAL; 410 } 411 altmap->alloc = 0; 412 } 413 414 if (check_pfn_span(pfn, nr_pages)) { 415 WARN(1, "Misaligned %s start: %#lx end: %#lx\n", __func__, pfn, pfn + nr_pages - 1); 416 return -EINVAL; 417 } 418 419 for (; pfn < end_pfn; pfn += cur_nr_pages) { 420 /* Select all remaining pages up to the next section boundary */ 421 cur_nr_pages = min(end_pfn - pfn, 422 SECTION_ALIGN_UP(pfn + 1) - pfn); 423 err = sparse_add_section(nid, pfn, cur_nr_pages, altmap, 424 params->pgmap); 425 if (err) 426 break; 427 cond_resched(); 428 } 429 vmemmap_populate_print_last(); 430 return err; 431} 432 433/* find the smallest valid pfn in the range [start_pfn, end_pfn) */ 434static unsigned long find_smallest_section_pfn(int nid, struct zone *zone, 435 unsigned long start_pfn, 436 unsigned long end_pfn) 437{ 438 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) { 439 if (unlikely(!pfn_to_online_page(start_pfn))) 440 continue; 441 442 if (unlikely(pfn_to_nid(start_pfn) != nid)) 443 continue; 444 445 if (zone != page_zone(pfn_to_page(start_pfn))) 446 continue; 447 448 return start_pfn; 449 } 450 451 return 0; 452} 453 454/* find the biggest valid pfn in the range [start_pfn, end_pfn). */ 455static unsigned long find_biggest_section_pfn(int nid, struct zone *zone, 456 unsigned long start_pfn, 457 unsigned long end_pfn) 458{ 459 unsigned long pfn; 460 461 /* pfn is the end pfn of a memory section. */ 462 pfn = end_pfn - 1; 463 for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) { 464 if (unlikely(!pfn_to_online_page(pfn))) 465 continue; 466 467 if (unlikely(pfn_to_nid(pfn) != nid)) 468 continue; 469 470 if (zone != page_zone(pfn_to_page(pfn))) 471 continue; 472 473 return pfn; 474 } 475 476 return 0; 477} 478 479static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, 480 unsigned long end_pfn) 481{ 482 unsigned long pfn; 483 int nid = zone_to_nid(zone); 484 485 if (zone->zone_start_pfn == start_pfn) { 486 /* 487 * If the section is smallest section in the zone, it need 488 * shrink zone->zone_start_pfn and zone->zone_spanned_pages. 489 * In this case, we find second smallest valid mem_section 490 * for shrinking zone. 491 */ 492 pfn = find_smallest_section_pfn(nid, zone, end_pfn, 493 zone_end_pfn(zone)); 494 if (pfn) { 495 zone->spanned_pages = zone_end_pfn(zone) - pfn; 496 zone->zone_start_pfn = pfn; 497 } else { 498 zone->zone_start_pfn = 0; 499 zone->spanned_pages = 0; 500 } 501 } else if (zone_end_pfn(zone) == end_pfn) { 502 /* 503 * If the section is biggest section in the zone, it need 504 * shrink zone->spanned_pages. 505 * In this case, we find second biggest valid mem_section for 506 * shrinking zone. 507 */ 508 pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn, 509 start_pfn); 510 if (pfn) 511 zone->spanned_pages = pfn - zone->zone_start_pfn + 1; 512 else { 513 zone->zone_start_pfn = 0; 514 zone->spanned_pages = 0; 515 } 516 } 517} 518 519static void update_pgdat_span(struct pglist_data *pgdat) 520{ 521 unsigned long node_start_pfn = 0, node_end_pfn = 0; 522 struct zone *zone; 523 524 for (zone = pgdat->node_zones; 525 zone < pgdat->node_zones + MAX_NR_ZONES; zone++) { 526 unsigned long end_pfn = zone_end_pfn(zone); 527 528 /* No need to lock the zones, they can't change. */ 529 if (!zone->spanned_pages) 530 continue; 531 if (!node_end_pfn) { 532 node_start_pfn = zone->zone_start_pfn; 533 node_end_pfn = end_pfn; 534 continue; 535 } 536 537 if (end_pfn > node_end_pfn) 538 node_end_pfn = end_pfn; 539 if (zone->zone_start_pfn < node_start_pfn) 540 node_start_pfn = zone->zone_start_pfn; 541 } 542 543 pgdat->node_start_pfn = node_start_pfn; 544 pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; 545} 546 547void remove_pfn_range_from_zone(struct zone *zone, 548 unsigned long start_pfn, 549 unsigned long nr_pages) 550{ 551 const unsigned long end_pfn = start_pfn + nr_pages; 552 struct pglist_data *pgdat = zone->zone_pgdat; 553 unsigned long pfn, cur_nr_pages; 554 555 /* Poison struct pages because they are now uninitialized again. */ 556 for (pfn = start_pfn; pfn < end_pfn; pfn += cur_nr_pages) { 557 cond_resched(); 558 559 /* Select all remaining pages up to the next section boundary */ 560 cur_nr_pages = 561 min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn); 562 page_init_poison(pfn_to_page(pfn), 563 sizeof(struct page) * cur_nr_pages); 564 } 565 566 /* 567 * Zone shrinking code cannot properly deal with ZONE_DEVICE. So 568 * we will not try to shrink the zones - which is okay as 569 * set_zone_contiguous() cannot deal with ZONE_DEVICE either way. 570 */ 571 if (zone_is_zone_device(zone)) 572 return; 573 574 clear_zone_contiguous(zone); 575 576 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); 577 update_pgdat_span(pgdat); 578 579 set_zone_contiguous(zone); 580} 581 582/** 583 * __remove_pages() - remove sections of pages 584 * @pfn: starting pageframe (must be aligned to start of a section) 585 * @nr_pages: number of pages to remove (must be multiple of section size) 586 * @altmap: alternative device page map or %NULL if default memmap is used 587 * 588 * Generic helper function to remove section mappings and sysfs entries 589 * for the section of the memory we are removing. Caller needs to make 590 * sure that pages are marked reserved and zones are adjust properly by 591 * calling offline_pages(). 592 */ 593void __remove_pages(unsigned long pfn, unsigned long nr_pages, 594 struct vmem_altmap *altmap) 595{ 596 const unsigned long end_pfn = pfn + nr_pages; 597 unsigned long cur_nr_pages; 598 599 if (check_pfn_span(pfn, nr_pages)) { 600 WARN(1, "Misaligned %s start: %#lx end: %#lx\n", __func__, pfn, pfn + nr_pages - 1); 601 return; 602 } 603 604 for (; pfn < end_pfn; pfn += cur_nr_pages) { 605 cond_resched(); 606 /* Select all remaining pages up to the next section boundary */ 607 cur_nr_pages = min(end_pfn - pfn, 608 SECTION_ALIGN_UP(pfn + 1) - pfn); 609 sparse_remove_section(pfn, cur_nr_pages, altmap); 610 } 611} 612 613int set_online_page_callback(online_page_callback_t callback) 614{ 615 int rc = -EINVAL; 616 617 get_online_mems(); 618 mutex_lock(&online_page_callback_lock); 619 620 if (online_page_callback == generic_online_page) { 621 online_page_callback = callback; 622 rc = 0; 623 } 624 625 mutex_unlock(&online_page_callback_lock); 626 put_online_mems(); 627 628 return rc; 629} 630EXPORT_SYMBOL_GPL(set_online_page_callback); 631 632int restore_online_page_callback(online_page_callback_t callback) 633{ 634 int rc = -EINVAL; 635 636 get_online_mems(); 637 mutex_lock(&online_page_callback_lock); 638 639 if (online_page_callback == callback) { 640 online_page_callback = generic_online_page; 641 rc = 0; 642 } 643 644 mutex_unlock(&online_page_callback_lock); 645 put_online_mems(); 646 647 return rc; 648} 649EXPORT_SYMBOL_GPL(restore_online_page_callback); 650 651/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 652void generic_online_page(struct page *page, unsigned int order) 653{ 654 __free_pages_core(page, order, MEMINIT_HOTPLUG); 655} 656EXPORT_SYMBOL_GPL(generic_online_page); 657 658static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages) 659{ 660 const unsigned long end_pfn = start_pfn + nr_pages; 661 unsigned long pfn; 662 663 /* 664 * Online the pages in MAX_PAGE_ORDER aligned chunks. The callback might 665 * decide to not expose all pages to the buddy (e.g., expose them 666 * later). We account all pages as being online and belonging to this 667 * zone ("present"). 668 * When using memmap_on_memory, the range might not be aligned to 669 * MAX_ORDER_NR_PAGES - 1, but pageblock aligned. __ffs() will detect 670 * this and the first chunk to online will be pageblock_nr_pages. 671 */ 672 for (pfn = start_pfn; pfn < end_pfn;) { 673 struct page *page = pfn_to_page(pfn); 674 int order; 675 676 /* 677 * Free to online pages in the largest chunks alignment allows. 678 * 679 * __ffs() behaviour is undefined for 0. start == 0 is 680 * MAX_PAGE_ORDER-aligned, Set order to MAX_PAGE_ORDER for 681 * the case. 682 */ 683 if (pfn) 684 order = min_t(int, MAX_PAGE_ORDER, __ffs(pfn)); 685 else 686 order = MAX_PAGE_ORDER; 687 688 /* 689 * Exposing the page to the buddy by freeing can cause 690 * issues with debug_pagealloc enabled: some archs don't 691 * like double-unmappings. So treat them like any pages that 692 * were allocated from the buddy. 693 */ 694 debug_pagealloc_map_pages(page, 1 << order); 695 (*online_page_callback)(page, order); 696 pfn += (1UL << order); 697 } 698 699 /* mark all involved sections as online */ 700 online_mem_sections(start_pfn, end_pfn); 701} 702 703static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn, 704 unsigned long nr_pages) 705{ 706 unsigned long old_end_pfn = zone_end_pfn(zone); 707 708 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) 709 zone->zone_start_pfn = start_pfn; 710 711 zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn; 712} 713 714static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn, 715 unsigned long nr_pages) 716{ 717 unsigned long old_end_pfn = pgdat_end_pfn(pgdat); 718 719 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) 720 pgdat->node_start_pfn = start_pfn; 721 722 pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn; 723 724} 725 726#ifdef CONFIG_ZONE_DEVICE 727static void section_taint_zone_device(unsigned long pfn) 728{ 729 struct mem_section *ms = __pfn_to_section(pfn); 730 731 ms->section_mem_map |= SECTION_TAINT_ZONE_DEVICE; 732} 733#else 734static inline void section_taint_zone_device(unsigned long pfn) 735{ 736} 737#endif 738 739/* 740 * Associate the pfn range with the given zone, initializing the memmaps 741 * and resizing the pgdat/zone data to span the added pages. After this 742 * call, all affected pages are PageOffline(). 743 * 744 * All aligned pageblocks are initialized to the specified migratetype 745 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 746 * zone stats (e.g., nr_isolate_pageblock) are touched. 747 */ 748void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, 749 unsigned long nr_pages, 750 struct vmem_altmap *altmap, int migratetype, 751 bool isolate_pageblock) 752{ 753 struct pglist_data *pgdat = zone->zone_pgdat; 754 int nid = pgdat->node_id; 755 756 clear_zone_contiguous(zone); 757 758 if (zone_is_empty(zone)) 759 init_currently_empty_zone(zone, start_pfn, nr_pages); 760 resize_zone_range(zone, start_pfn, nr_pages); 761 resize_pgdat_range(pgdat, start_pfn, nr_pages); 762 763 /* 764 * Subsection population requires care in pfn_to_online_page(). 765 * Set the taint to enable the slow path detection of 766 * ZONE_DEVICE pages in an otherwise ZONE_{NORMAL,MOVABLE} 767 * section. 768 */ 769 if (zone_is_zone_device(zone)) { 770 if (!IS_ALIGNED(start_pfn, PAGES_PER_SECTION)) 771 section_taint_zone_device(start_pfn); 772 if (!IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION)) 773 section_taint_zone_device(start_pfn + nr_pages); 774 } 775 776 /* 777 * TODO now we have a visible range of pages which are not associated 778 * with their zone properly. Not nice but set_pfnblock_migratetype() 779 * expects the zone spans the pfn range. All the pages in the range 780 * are reserved so nobody should be touching them so we should be safe 781 */ 782 memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0, 783 MEMINIT_HOTPLUG, altmap, migratetype, 784 isolate_pageblock); 785 786 set_zone_contiguous(zone); 787} 788 789struct auto_movable_stats { 790 unsigned long kernel_early_pages; 791 unsigned long movable_pages; 792}; 793 794static void auto_movable_stats_account_zone(struct auto_movable_stats *stats, 795 struct zone *zone) 796{ 797 if (zone_idx(zone) == ZONE_MOVABLE) { 798 stats->movable_pages += zone->present_pages; 799 } else { 800 stats->kernel_early_pages += zone->present_early_pages; 801#ifdef CONFIG_CMA 802 /* 803 * CMA pages (never on hotplugged memory) behave like 804 * ZONE_MOVABLE. 805 */ 806 stats->movable_pages += zone->cma_pages; 807 stats->kernel_early_pages -= zone->cma_pages; 808#endif /* CONFIG_CMA */ 809 } 810} 811struct auto_movable_group_stats { 812 unsigned long movable_pages; 813 unsigned long req_kernel_early_pages; 814}; 815 816static int auto_movable_stats_account_group(struct memory_group *group, 817 void *arg) 818{ 819 const int ratio = READ_ONCE(auto_movable_ratio); 820 struct auto_movable_group_stats *stats = arg; 821 long pages; 822 823 /* 824 * We don't support modifying the config while the auto-movable online 825 * policy is already enabled. Just avoid the division by zero below. 826 */ 827 if (!ratio) 828 return 0; 829 830 /* 831 * Calculate how many early kernel pages this group requires to 832 * satisfy the configured zone ratio. 833 */ 834 pages = group->present_movable_pages * 100 / ratio; 835 pages -= group->present_kernel_pages; 836 837 if (pages > 0) 838 stats->req_kernel_early_pages += pages; 839 stats->movable_pages += group->present_movable_pages; 840 return 0; 841} 842 843static bool auto_movable_can_online_movable(int nid, struct memory_group *group, 844 unsigned long nr_pages) 845{ 846 unsigned long kernel_early_pages, movable_pages; 847 struct auto_movable_group_stats group_stats = {}; 848 struct auto_movable_stats stats = {}; 849 struct zone *zone; 850 int i; 851 852 /* Walk all relevant zones and collect MOVABLE vs. KERNEL stats. */ 853 if (nid == NUMA_NO_NODE) { 854 /* TODO: cache values */ 855 for_each_populated_zone(zone) 856 auto_movable_stats_account_zone(&stats, zone); 857 } else { 858 for (i = 0; i < MAX_NR_ZONES; i++) { 859 pg_data_t *pgdat = NODE_DATA(nid); 860 861 zone = pgdat->node_zones + i; 862 if (populated_zone(zone)) 863 auto_movable_stats_account_zone(&stats, zone); 864 } 865 } 866 867 kernel_early_pages = stats.kernel_early_pages; 868 movable_pages = stats.movable_pages; 869 870 /* 871 * Kernel memory inside dynamic memory group allows for more MOVABLE 872 * memory within the same group. Remove the effect of all but the 873 * current group from the stats. 874 */ 875 walk_dynamic_memory_groups(nid, auto_movable_stats_account_group, 876 group, &group_stats); 877 if (kernel_early_pages <= group_stats.req_kernel_early_pages) 878 return false; 879 kernel_early_pages -= group_stats.req_kernel_early_pages; 880 movable_pages -= group_stats.movable_pages; 881 882 if (group && group->is_dynamic) 883 kernel_early_pages += group->present_kernel_pages; 884 885 /* 886 * Test if we could online the given number of pages to ZONE_MOVABLE 887 * and still stay in the configured ratio. 888 */ 889 movable_pages += nr_pages; 890 return movable_pages <= (auto_movable_ratio * kernel_early_pages) / 100; 891} 892 893/* 894 * Returns a default kernel memory zone for the given pfn range. 895 * If no kernel zone covers this pfn range it will automatically go 896 * to the ZONE_NORMAL. 897 */ 898static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn, 899 unsigned long nr_pages) 900{ 901 struct pglist_data *pgdat = NODE_DATA(nid); 902 int zid; 903 904 for (zid = 0; zid < ZONE_NORMAL; zid++) { 905 struct zone *zone = &pgdat->node_zones[zid]; 906 907 if (zone_intersects(zone, start_pfn, nr_pages)) 908 return zone; 909 } 910 911 return &pgdat->node_zones[ZONE_NORMAL]; 912} 913 914/* 915 * Determine to which zone to online memory dynamically based on user 916 * configuration and system stats. We care about the following ratio: 917 * 918 * MOVABLE : KERNEL 919 * 920 * Whereby MOVABLE is memory in ZONE_MOVABLE and KERNEL is memory in 921 * one of the kernel zones. CMA pages inside one of the kernel zones really 922 * behaves like ZONE_MOVABLE, so we treat them accordingly. 923 * 924 * We don't allow for hotplugged memory in a KERNEL zone to increase the 925 * amount of MOVABLE memory we can have, so we end up with: 926 * 927 * MOVABLE : KERNEL_EARLY 928 * 929 * Whereby KERNEL_EARLY is memory in one of the kernel zones, available sinze 930 * boot. We base our calculation on KERNEL_EARLY internally, because: 931 * 932 * a) Hotplugged memory in one of the kernel zones can sometimes still get 933 * hotunplugged, especially when hot(un)plugging individual memory blocks. 934 * There is no coordination across memory devices, therefore "automatic" 935 * hotunplugging, as implemented in hypervisors, could result in zone 936 * imbalances. 937 * b) Early/boot memory in one of the kernel zones can usually not get 938 * hotunplugged again (e.g., no firmware interface to unplug, fragmented 939 * with unmovable allocations). While there are corner cases where it might 940 * still work, it is barely relevant in practice. 941 * 942 * Exceptions are dynamic memory groups, which allow for more MOVABLE 943 * memory within the same memory group -- because in that case, there is 944 * coordination within the single memory device managed by a single driver. 945 * 946 * We rely on "present pages" instead of "managed pages", as the latter is 947 * highly unreliable and dynamic in virtualized environments, and does not 948 * consider boot time allocations. For example, memory ballooning adjusts the 949 * managed pages when inflating/deflating the balloon, and balloon compaction 950 * can even migrate inflated pages between zones. 951 * 952 * Using "present pages" is better but some things to keep in mind are: 953 * 954 * a) Some memblock allocations, such as for the crashkernel area, are 955 * effectively unused by the kernel, yet they account to "present pages". 956 * Fortunately, these allocations are comparatively small in relevant setups 957 * (e.g., fraction of system memory). 958 * b) Some hotplugged memory blocks in virtualized environments, especially 959 * hotplugged by virtio-mem, look like they are completely present, however, 960 * only parts of the memory block are actually currently usable. 961 * "present pages" is an upper limit that can get reached at runtime. As 962 * we base our calculations on KERNEL_EARLY, this is not an issue. 963 */ 964static struct zone *auto_movable_zone_for_pfn(int nid, 965 struct memory_group *group, 966 unsigned long pfn, 967 unsigned long nr_pages) 968{ 969 unsigned long online_pages = 0, max_pages, end_pfn; 970 struct page *page; 971 972 if (!auto_movable_ratio) 973 goto kernel_zone; 974 975 if (group && !group->is_dynamic) { 976 max_pages = group->s.max_pages; 977 online_pages = group->present_movable_pages; 978 979 /* If anything is !MOVABLE online the rest !MOVABLE. */ 980 if (group->present_kernel_pages) 981 goto kernel_zone; 982 } else if (!group || group->d.unit_pages == nr_pages) { 983 max_pages = nr_pages; 984 } else { 985 max_pages = group->d.unit_pages; 986 /* 987 * Take a look at all online sections in the current unit. 988 * We can safely assume that all pages within a section belong 989 * to the same zone, because dynamic memory groups only deal 990 * with hotplugged memory. 991 */ 992 pfn = ALIGN_DOWN(pfn, group->d.unit_pages); 993 end_pfn = pfn + group->d.unit_pages; 994 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 995 page = pfn_to_online_page(pfn); 996 if (!page) 997 continue; 998 /* If anything is !MOVABLE online the rest !MOVABLE. */ 999 if (!is_zone_movable_page(page)) 1000 goto kernel_zone; 1001 online_pages += PAGES_PER_SECTION; 1002 } 1003 } 1004 1005 /* 1006 * Online MOVABLE if we could *currently* online all remaining parts 1007 * MOVABLE. We expect to (add+) online them immediately next, so if 1008 * nobody interferes, all will be MOVABLE if possible. 1009 */ 1010 nr_pages = max_pages - online_pages; 1011 if (!auto_movable_can_online_movable(NUMA_NO_NODE, group, nr_pages)) 1012 goto kernel_zone; 1013 1014#ifdef CONFIG_NUMA 1015 if (auto_movable_numa_aware && 1016 !auto_movable_can_online_movable(nid, group, nr_pages)) 1017 goto kernel_zone; 1018#endif /* CONFIG_NUMA */ 1019 1020 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; 1021kernel_zone: 1022 return default_kernel_zone_for_pfn(nid, pfn, nr_pages); 1023} 1024 1025static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn, 1026 unsigned long nr_pages) 1027{ 1028 struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn, 1029 nr_pages); 1030 struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; 1031 bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages); 1032 bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages); 1033 1034 /* 1035 * We inherit the existing zone in a simple case where zones do not 1036 * overlap in the given range 1037 */ 1038 if (in_kernel ^ in_movable) 1039 return (in_kernel) ? kernel_zone : movable_zone; 1040 1041 /* 1042 * If the range doesn't belong to any zone or two zones overlap in the 1043 * given range then we use movable zone only if movable_node is 1044 * enabled because we always online to a kernel zone by default. 1045 */ 1046 return movable_node_enabled ? movable_zone : kernel_zone; 1047} 1048 1049struct zone *zone_for_pfn_range(int online_type, int nid, 1050 struct memory_group *group, unsigned long start_pfn, 1051 unsigned long nr_pages) 1052{ 1053 if (online_type == MMOP_ONLINE_KERNEL) 1054 return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages); 1055 1056 if (online_type == MMOP_ONLINE_MOVABLE) 1057 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; 1058 1059 if (online_policy == ONLINE_POLICY_AUTO_MOVABLE) 1060 return auto_movable_zone_for_pfn(nid, group, start_pfn, nr_pages); 1061 1062 return default_zone_for_pfn(nid, start_pfn, nr_pages); 1063} 1064 1065/* 1066 * This function should only be called by memory_block_{online,offline}, 1067 * and {online,offline}_pages. 1068 */ 1069void adjust_present_page_count(struct page *page, struct memory_group *group, 1070 long nr_pages) 1071{ 1072 struct zone *zone = page_zone(page); 1073 const bool movable = zone_idx(zone) == ZONE_MOVABLE; 1074 1075 /* 1076 * We only support onlining/offlining/adding/removing of complete 1077 * memory blocks; therefore, either all is either early or hotplugged. 1078 */ 1079 if (early_section(__pfn_to_section(page_to_pfn(page)))) 1080 zone->present_early_pages += nr_pages; 1081 zone->present_pages += nr_pages; 1082 zone->zone_pgdat->node_present_pages += nr_pages; 1083 1084 if (group && movable) 1085 group->present_movable_pages += nr_pages; 1086 else if (group && !movable) 1087 group->present_kernel_pages += nr_pages; 1088} 1089 1090int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages, 1091 struct zone *zone) 1092{ 1093 unsigned long end_pfn = pfn + nr_pages; 1094 int ret, i; 1095 1096 ret = kasan_add_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages)); 1097 if (ret) 1098 return ret; 1099 1100 move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE, 1101 false); 1102 1103 for (i = 0; i < nr_pages; i++) { 1104 struct page *page = pfn_to_page(pfn + i); 1105 1106 __ClearPageOffline(page); 1107 SetPageVmemmapSelfHosted(page); 1108 } 1109 1110 /* 1111 * It might be that the vmemmap_pages fully span sections. If that is 1112 * the case, mark those sections online here as otherwise they will be 1113 * left offline. 1114 */ 1115 if (nr_pages >= PAGES_PER_SECTION) 1116 online_mem_sections(pfn, ALIGN_DOWN(end_pfn, PAGES_PER_SECTION)); 1117 1118 return ret; 1119} 1120 1121void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages) 1122{ 1123 unsigned long end_pfn = pfn + nr_pages; 1124 1125 /* 1126 * It might be that the vmemmap_pages fully span sections. If that is 1127 * the case, mark those sections offline here as otherwise they will be 1128 * left online. 1129 */ 1130 if (nr_pages >= PAGES_PER_SECTION) 1131 offline_mem_sections(pfn, ALIGN_DOWN(end_pfn, PAGES_PER_SECTION)); 1132 1133 /* 1134 * The pages associated with this vmemmap have been offlined, so 1135 * we can reset its state here. 1136 */ 1137 remove_pfn_range_from_zone(page_zone(pfn_to_page(pfn)), pfn, nr_pages); 1138 kasan_remove_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages)); 1139} 1140 1141/* 1142 * Must be called with mem_hotplug_lock in write mode. 1143 */ 1144int online_pages(unsigned long pfn, unsigned long nr_pages, 1145 struct zone *zone, struct memory_group *group) 1146{ 1147 struct memory_notify mem_arg = { 1148 .start_pfn = pfn, 1149 .nr_pages = nr_pages, 1150 }; 1151 struct node_notify node_arg = { 1152 .nid = NUMA_NO_NODE, 1153 }; 1154 const int nid = zone_to_nid(zone); 1155 int need_zonelists_rebuild = 0; 1156 unsigned long flags; 1157 int ret; 1158 1159 /* 1160 * {on,off}lining is constrained to full memory sections (or more 1161 * precisely to memory blocks from the user space POV). 1162 * memmap_on_memory is an exception because it reserves initial part 1163 * of the physical memory space for vmemmaps. That space is pageblock 1164 * aligned. 1165 */ 1166 if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(pfn) || 1167 !IS_ALIGNED(pfn + nr_pages, PAGES_PER_SECTION))) 1168 return -EINVAL; 1169 1170 1171 /* associate pfn range with the zone */ 1172 move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_MOVABLE, 1173 true); 1174 1175 if (!node_state(nid, N_MEMORY)) { 1176 /* Adding memory to the node for the first time */ 1177 node_arg.nid = nid; 1178 ret = node_notify(NODE_ADDING_FIRST_MEMORY, &node_arg); 1179 ret = notifier_to_errno(ret); 1180 if (ret) 1181 goto failed_addition; 1182 } 1183 1184 ret = memory_notify(MEM_GOING_ONLINE, &mem_arg); 1185 ret = notifier_to_errno(ret); 1186 if (ret) 1187 goto failed_addition; 1188 1189 /* 1190 * Fixup the number of isolated pageblocks before marking the sections 1191 * onlining, such that undo_isolate_page_range() works correctly. 1192 */ 1193 spin_lock_irqsave(&zone->lock, flags); 1194 zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages; 1195 spin_unlock_irqrestore(&zone->lock, flags); 1196 1197 /* 1198 * If this zone is not populated, then it is not in zonelist. 1199 * This means the page allocator ignores this zone. 1200 * So, zonelist must be updated after online. 1201 */ 1202 if (!populated_zone(zone)) { 1203 need_zonelists_rebuild = 1; 1204 setup_zone_pageset(zone); 1205 } 1206 1207 online_pages_range(pfn, nr_pages); 1208 adjust_present_page_count(pfn_to_page(pfn), group, nr_pages); 1209 1210 if (node_arg.nid >= 0) 1211 node_set_state(nid, N_MEMORY); 1212 /* 1213 * Check whether we are adding normal memory to the node for the first 1214 * time. 1215 */ 1216 if (!node_state(nid, N_NORMAL_MEMORY) && zone_idx(zone) <= ZONE_NORMAL) 1217 node_set_state(nid, N_NORMAL_MEMORY); 1218 1219 if (need_zonelists_rebuild) 1220 build_all_zonelists(NULL); 1221 1222 /* Basic onlining is complete, allow allocation of onlined pages. */ 1223 undo_isolate_page_range(pfn, pfn + nr_pages); 1224 1225 /* 1226 * Freshly onlined pages aren't shuffled (e.g., all pages are placed to 1227 * the tail of the freelist when undoing isolation). Shuffle the whole 1228 * zone to make sure the just onlined pages are properly distributed 1229 * across the whole freelist - to create an initial shuffle. 1230 */ 1231 shuffle_zone(zone); 1232 1233 /* reinitialise watermarks and update pcp limits */ 1234 init_per_zone_wmark_min(); 1235 1236 kswapd_run(nid); 1237 kcompactd_run(nid); 1238 1239 if (node_arg.nid >= 0) 1240 /* First memory added successfully. Notify consumers. */ 1241 node_notify(NODE_ADDED_FIRST_MEMORY, &node_arg); 1242 1243 writeback_set_ratelimit(); 1244 1245 memory_notify(MEM_ONLINE, &mem_arg); 1246 return 0; 1247 1248failed_addition: 1249 pr_debug("online_pages [mem %#010llx-%#010llx] failed\n", 1250 (unsigned long long) pfn << PAGE_SHIFT, 1251 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1); 1252 memory_notify(MEM_CANCEL_ONLINE, &mem_arg); 1253 if (node_arg.nid != NUMA_NO_NODE) 1254 node_notify(NODE_CANCEL_ADDING_FIRST_MEMORY, &node_arg); 1255 remove_pfn_range_from_zone(zone, pfn, nr_pages); 1256 return ret; 1257} 1258 1259/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1260static pg_data_t *hotadd_init_pgdat(int nid) 1261{ 1262 struct pglist_data *pgdat; 1263 1264 /* 1265 * NODE_DATA is preallocated (free_area_init) but its internal 1266 * state is not allocated completely. Add missing pieces. 1267 * Completely offline nodes stay around and they just need 1268 * reintialization. 1269 */ 1270 pgdat = NODE_DATA(nid); 1271 1272 /* init node's zones as empty zones, we don't have any present pages.*/ 1273 free_area_init_core_hotplug(pgdat); 1274 1275 /* 1276 * The node we allocated has no zone fallback lists. For avoiding 1277 * to access not-initialized zonelist, build here. 1278 */ 1279 build_all_zonelists(pgdat); 1280 1281 return pgdat; 1282} 1283 1284/* 1285 * __try_online_node - online a node if offlined 1286 * @nid: the node ID 1287 * @set_node_online: Whether we want to online the node 1288 * called by cpu_up() to online a node without onlined memory. 1289 * 1290 * Returns: 1291 * 1 -> a new node has been allocated 1292 * 0 -> the node is already online 1293 * -ENOMEM -> the node could not be allocated 1294 */ 1295static int __try_online_node(int nid, bool set_node_online) 1296{ 1297 pg_data_t *pgdat; 1298 int ret = 1; 1299 1300 if (node_online(nid)) 1301 return 0; 1302 1303 pgdat = hotadd_init_pgdat(nid); 1304 if (!pgdat) { 1305 pr_err("Cannot online node %d due to NULL pgdat\n", nid); 1306 ret = -ENOMEM; 1307 goto out; 1308 } 1309 1310 if (set_node_online) { 1311 node_set_online(nid); 1312 ret = register_node(nid); 1313 BUG_ON(ret); 1314 } 1315out: 1316 return ret; 1317} 1318 1319/* 1320 * Users of this function always want to online/register the node 1321 */ 1322int try_online_node(int nid) 1323{ 1324 int ret; 1325 1326 mem_hotplug_begin(); 1327 ret = __try_online_node(nid, true); 1328 mem_hotplug_done(); 1329 return ret; 1330} 1331 1332static int check_hotplug_memory_range(u64 start, u64 size) 1333{ 1334 /* memory range must be block size aligned */ 1335 if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) || 1336 !IS_ALIGNED(size, memory_block_size_bytes())) { 1337 pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx", 1338 memory_block_size_bytes(), start, size); 1339 return -EINVAL; 1340 } 1341 1342 return 0; 1343} 1344 1345static int online_memory_block(struct memory_block *mem, void *arg) 1346{ 1347 mem->online_type = mhp_get_default_online_type(); 1348 return device_online(&mem->dev); 1349} 1350 1351#ifndef arch_supports_memmap_on_memory 1352static inline bool arch_supports_memmap_on_memory(unsigned long vmemmap_size) 1353{ 1354 /* 1355 * As default, we want the vmemmap to span a complete PMD such that we 1356 * can map the vmemmap using a single PMD if supported by the 1357 * architecture. 1358 */ 1359 return IS_ALIGNED(vmemmap_size, PMD_SIZE); 1360} 1361#endif 1362 1363bool mhp_supports_memmap_on_memory(void) 1364{ 1365 unsigned long vmemmap_size = memory_block_memmap_size(); 1366 unsigned long memmap_pages = memory_block_memmap_on_memory_pages(); 1367 1368 /* 1369 * Besides having arch support and the feature enabled at runtime, we 1370 * need a few more assumptions to hold true: 1371 * 1372 * a) The vmemmap pages span complete PMDs: We don't want vmemmap code 1373 * to populate memory from the altmap for unrelated parts (i.e., 1374 * other memory blocks) 1375 * 1376 * b) The vmemmap pages (and thereby the pages that will be exposed to 1377 * the buddy) have to cover full pageblocks: memory onlining/offlining 1378 * code requires applicable ranges to be page-aligned, for example, to 1379 * set the migratetypes properly. 1380 * 1381 * TODO: Although we have a check here to make sure that vmemmap pages 1382 * fully populate a PMD, it is not the right place to check for 1383 * this. A much better solution involves improving vmemmap code 1384 * to fallback to base pages when trying to populate vmemmap using 1385 * altmap as an alternative source of memory, and we do not exactly 1386 * populate a single PMD. 1387 */ 1388 if (!mhp_memmap_on_memory()) 1389 return false; 1390 1391 /* 1392 * Make sure the vmemmap allocation is fully contained 1393 * so that we always allocate vmemmap memory from altmap area. 1394 */ 1395 if (!IS_ALIGNED(vmemmap_size, PAGE_SIZE)) 1396 return false; 1397 1398 /* 1399 * start pfn should be pageblock_nr_pages aligned for correctly 1400 * setting migrate types 1401 */ 1402 if (!pageblock_aligned(memmap_pages)) 1403 return false; 1404 1405 if (memmap_pages == PHYS_PFN(memory_block_size_bytes())) 1406 /* No effective hotplugged memory doesn't make sense. */ 1407 return false; 1408 1409 return arch_supports_memmap_on_memory(vmemmap_size); 1410} 1411EXPORT_SYMBOL_GPL(mhp_supports_memmap_on_memory); 1412 1413static void remove_memory_blocks_and_altmaps(u64 start, u64 size) 1414{ 1415 unsigned long memblock_size = memory_block_size_bytes(); 1416 u64 cur_start; 1417 1418 /* 1419 * For memmap_on_memory, the altmaps were added on a per-memblock 1420 * basis; we have to process each individual memory block. 1421 */ 1422 for (cur_start = start; cur_start < start + size; 1423 cur_start += memblock_size) { 1424 struct vmem_altmap *altmap = NULL; 1425 struct memory_block *mem; 1426 1427 mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(cur_start))); 1428 if (WARN_ON_ONCE(!mem)) 1429 continue; 1430 1431 altmap = mem->altmap; 1432 mem->altmap = NULL; 1433 1434 remove_memory_block_devices(cur_start, memblock_size); 1435 1436 arch_remove_memory(cur_start, memblock_size, altmap); 1437 1438 /* Verify that all vmemmap pages have actually been freed. */ 1439 WARN(altmap->alloc, "Altmap not fully unmapped"); 1440 kfree(altmap); 1441 } 1442} 1443 1444static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group, 1445 u64 start, u64 size) 1446{ 1447 unsigned long memblock_size = memory_block_size_bytes(); 1448 u64 cur_start; 1449 int ret; 1450 1451 for (cur_start = start; cur_start < start + size; 1452 cur_start += memblock_size) { 1453 struct mhp_params params = { .pgprot = 1454 pgprot_mhp(PAGE_KERNEL) }; 1455 struct vmem_altmap mhp_altmap = { 1456 .base_pfn = PHYS_PFN(cur_start), 1457 .end_pfn = PHYS_PFN(cur_start + memblock_size - 1), 1458 }; 1459 1460 mhp_altmap.free = memory_block_memmap_on_memory_pages(); 1461 params.altmap = kmemdup(&mhp_altmap, sizeof(struct vmem_altmap), 1462 GFP_KERNEL); 1463 if (!params.altmap) { 1464 ret = -ENOMEM; 1465 goto out; 1466 } 1467 1468 /* call arch's memory hotadd */ 1469 ret = arch_add_memory(nid, cur_start, memblock_size, ¶ms); 1470 if (ret < 0) { 1471 kfree(params.altmap); 1472 goto out; 1473 } 1474 1475 /* create memory block devices after memory was added */ 1476 ret = create_memory_block_devices(cur_start, memblock_size, nid, 1477 params.altmap, group); 1478 if (ret) { 1479 arch_remove_memory(cur_start, memblock_size, NULL); 1480 kfree(params.altmap); 1481 goto out; 1482 } 1483 } 1484 1485 return 0; 1486out: 1487 if (ret && cur_start != start) 1488 remove_memory_blocks_and_altmaps(start, cur_start - start); 1489 return ret; 1490} 1491 1492/* 1493 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1494 * and online/offline operations (triggered e.g. by sysfs). 1495 * 1496 * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG 1497 */ 1498int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) 1499{ 1500 struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) }; 1501 enum memblock_flags memblock_flags = MEMBLOCK_NONE; 1502 struct memory_group *group = NULL; 1503 u64 start, size; 1504 bool new_node = false; 1505 int ret; 1506 1507 start = res->start; 1508 size = resource_size(res); 1509 1510 ret = check_hotplug_memory_range(start, size); 1511 if (ret) 1512 return ret; 1513 1514 if (mhp_flags & MHP_NID_IS_MGID) { 1515 group = memory_group_find_by_id(nid); 1516 if (!group) 1517 return -EINVAL; 1518 nid = group->nid; 1519 } 1520 1521 if (!node_possible(nid)) { 1522 WARN(1, "node %d was absent from the node_possible_map\n", nid); 1523 return -EINVAL; 1524 } 1525 1526 mem_hotplug_begin(); 1527 1528 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) { 1529 if (res->flags & IORESOURCE_SYSRAM_DRIVER_MANAGED) 1530 memblock_flags = MEMBLOCK_DRIVER_MANAGED; 1531 ret = memblock_add_node(start, size, nid, memblock_flags); 1532 if (ret) 1533 goto error_mem_hotplug_end; 1534 } 1535 1536 ret = __try_online_node(nid, false); 1537 if (ret < 0) 1538 goto error_memblock_remove; 1539 if (ret) { 1540 node_set_online(nid); 1541 ret = register_node(nid); 1542 if (WARN_ON(ret)) { 1543 node_set_offline(nid); 1544 goto error_memblock_remove; 1545 } 1546 new_node = true; 1547 } 1548 1549 /* 1550 * Self hosted memmap array 1551 */ 1552 if ((mhp_flags & MHP_MEMMAP_ON_MEMORY) && 1553 mhp_supports_memmap_on_memory()) { 1554 ret = create_altmaps_and_memory_blocks(nid, group, start, size); 1555 if (ret) 1556 goto error; 1557 } else { 1558 ret = arch_add_memory(nid, start, size, ¶ms); 1559 if (ret < 0) 1560 goto error; 1561 1562 /* create memory block devices after memory was added */ 1563 ret = create_memory_block_devices(start, size, nid, NULL, group); 1564 if (ret) { 1565 arch_remove_memory(start, size, params.altmap); 1566 goto error; 1567 } 1568 } 1569 1570 register_memory_blocks_under_node_hotplug(nid, PFN_DOWN(start), 1571 PFN_UP(start + size - 1)); 1572 1573 /* create new memmap entry */ 1574 if (!strcmp(res->name, "System RAM")) 1575 firmware_map_add_hotplug(start, start + size, "System RAM"); 1576 1577 /* device_online() will take the lock when calling online_pages() */ 1578 mem_hotplug_done(); 1579 1580 /* 1581 * In case we're allowed to merge the resource, flag it and trigger 1582 * merging now that adding succeeded. 1583 */ 1584 if (mhp_flags & MHP_MERGE_RESOURCE) 1585 merge_system_ram_resource(res); 1586 1587 /* online pages if requested */ 1588 if (mhp_get_default_online_type() != MMOP_OFFLINE) 1589 walk_memory_blocks(start, size, NULL, online_memory_block); 1590 1591 return ret; 1592error: 1593 if (new_node) { 1594 node_set_offline(nid); 1595 unregister_node(nid); 1596 } 1597error_memblock_remove: 1598 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) 1599 memblock_remove(start, size); 1600error_mem_hotplug_end: 1601 mem_hotplug_done(); 1602 return ret; 1603} 1604 1605/* requires device_hotplug_lock, see add_memory_resource() */ 1606int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) 1607{ 1608 struct resource *res; 1609 int ret; 1610 1611 res = register_memory_resource(start, size, "System RAM"); 1612 if (IS_ERR(res)) 1613 return PTR_ERR(res); 1614 1615 ret = add_memory_resource(nid, res, mhp_flags); 1616 if (ret < 0) 1617 release_memory_resource(res); 1618 return ret; 1619} 1620 1621int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) 1622{ 1623 int rc; 1624 1625 lock_device_hotplug(); 1626 rc = __add_memory(nid, start, size, mhp_flags); 1627 unlock_device_hotplug(); 1628 1629 return rc; 1630} 1631EXPORT_SYMBOL_GPL(add_memory); 1632 1633/* 1634 * Add special, driver-managed memory to the system as system RAM. Such 1635 * memory is not exposed via the raw firmware-provided memmap as system 1636 * RAM, instead, it is detected and added by a driver - during cold boot, 1637 * after a reboot, and after kexec. 1638 * 1639 * Reasons why this memory should not be used for the initial memmap of a 1640 * kexec kernel or for placing kexec images: 1641 * - The booting kernel is in charge of determining how this memory will be 1642 * used (e.g., use persistent memory as system RAM) 1643 * - Coordination with a hypervisor is required before this memory 1644 * can be used (e.g., inaccessible parts). 1645 * 1646 * For this memory, no entries in /sys/firmware/memmap ("raw firmware-provided 1647 * memory map") are created. Also, the created memory resource is flagged 1648 * with IORESOURCE_SYSRAM_DRIVER_MANAGED, so in-kernel users can special-case 1649 * this memory as well (esp., not place kexec images onto it). 1650 * 1651 * The resource_name (visible via /proc/iomem) has to have the format 1652 * "System RAM ($DRIVER)". 1653 */ 1654int add_memory_driver_managed(int nid, u64 start, u64 size, 1655 const char *resource_name, mhp_t mhp_flags) 1656{ 1657 struct resource *res; 1658 int rc; 1659 1660 if (!resource_name || 1661 strstr(resource_name, "System RAM (") != resource_name || 1662 resource_name[strlen(resource_name) - 1] != ')') 1663 return -EINVAL; 1664 1665 lock_device_hotplug(); 1666 1667 res = register_memory_resource(start, size, resource_name); 1668 if (IS_ERR(res)) { 1669 rc = PTR_ERR(res); 1670 goto out_unlock; 1671 } 1672 1673 rc = add_memory_resource(nid, res, mhp_flags); 1674 if (rc < 0) 1675 release_memory_resource(res); 1676 1677out_unlock: 1678 unlock_device_hotplug(); 1679 return rc; 1680} 1681EXPORT_SYMBOL_GPL(add_memory_driver_managed); 1682 1683/* 1684 * Platforms should define arch_get_mappable_range() that provides 1685 * maximum possible addressable physical memory range for which the 1686 * linear mapping could be created. The platform returned address 1687 * range must adhere to these following semantics. 1688 * 1689 * - range.start <= range.end 1690 * - Range includes both end points [range.start..range.end] 1691 * 1692 * There is also a fallback definition provided here, allowing the 1693 * entire possible physical address range in case any platform does 1694 * not define arch_get_mappable_range(). 1695 */ 1696struct range __weak arch_get_mappable_range(void) 1697{ 1698 struct range mhp_range = { 1699 .start = 0UL, 1700 .end = -1ULL, 1701 }; 1702 return mhp_range; 1703} 1704 1705struct range mhp_get_pluggable_range(bool need_mapping) 1706{ 1707 const u64 max_phys = DIRECT_MAP_PHYSMEM_END; 1708 struct range mhp_range; 1709 1710 if (need_mapping) { 1711 mhp_range = arch_get_mappable_range(); 1712 if (mhp_range.start > max_phys) { 1713 mhp_range.start = 0; 1714 mhp_range.end = 0; 1715 } 1716 mhp_range.end = min_t(u64, mhp_range.end, max_phys); 1717 } else { 1718 mhp_range.start = 0; 1719 mhp_range.end = max_phys; 1720 } 1721 return mhp_range; 1722} 1723EXPORT_SYMBOL_GPL(mhp_get_pluggable_range); 1724 1725bool mhp_range_allowed(u64 start, u64 size, bool need_mapping) 1726{ 1727 struct range mhp_range = mhp_get_pluggable_range(need_mapping); 1728 u64 end = start + size; 1729 1730 if (start < end && start >= mhp_range.start && (end - 1) <= mhp_range.end) 1731 return true; 1732 1733 pr_warn("Hotplug memory [%#llx-%#llx] exceeds maximum addressable range [%#llx-%#llx]\n", 1734 start, end, mhp_range.start, mhp_range.end); 1735 return false; 1736} 1737 1738#ifdef CONFIG_MEMORY_HOTREMOVE 1739/* 1740 * Scan pfn range [start,end) to find movable/migratable pages (LRU and 1741 * hugetlb folio, movable_ops pages). Will skip over most unmovable 1742 * pages (esp., pages that can be skipped when offlining), but bail out on 1743 * definitely unmovable pages. 1744 * 1745 * Returns: 1746 * 0 in case a movable page is found and movable_pfn was updated. 1747 * -ENOENT in case no movable page was found. 1748 * -EBUSY in case a definitely unmovable page was found. 1749 */ 1750static int scan_movable_pages(unsigned long start, unsigned long end, 1751 unsigned long *movable_pfn) 1752{ 1753 unsigned long pfn; 1754 1755 for_each_valid_pfn(pfn, start, end) { 1756 struct page *page; 1757 struct folio *folio; 1758 1759 page = pfn_to_page(pfn); 1760 if (PageLRU(page) || page_has_movable_ops(page)) 1761 goto found; 1762 1763 /* 1764 * PageOffline() pages that do not have movable_ops and 1765 * have a reference count > 0 (after MEM_GOING_OFFLINE) are 1766 * definitely unmovable. If their reference count would be 0, 1767 * they could at least be skipped when offlining memory. 1768 */ 1769 if (PageOffline(page) && page_count(page)) 1770 return -EBUSY; 1771 1772 if (!PageHuge(page)) 1773 continue; 1774 folio = page_folio(page); 1775 /* 1776 * This test is racy as we hold no reference or lock. The 1777 * hugetlb page could have been free'ed and head is no longer 1778 * a hugetlb page before the following check. In such unlikely 1779 * cases false positives and negatives are possible. Calling 1780 * code must deal with these scenarios. 1781 */ 1782 if (folio_test_hugetlb_migratable(folio)) 1783 goto found; 1784 pfn |= folio_nr_pages(folio) - 1; 1785 } 1786 return -ENOENT; 1787found: 1788 *movable_pfn = pfn; 1789 return 0; 1790} 1791 1792static void do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) 1793{ 1794 struct folio *folio; 1795 unsigned long pfn; 1796 LIST_HEAD(source); 1797 static DEFINE_RATELIMIT_STATE(migrate_rs, DEFAULT_RATELIMIT_INTERVAL, 1798 DEFAULT_RATELIMIT_BURST); 1799 1800 for_each_valid_pfn(pfn, start_pfn, end_pfn) { 1801 struct page *page; 1802 1803 page = pfn_to_page(pfn); 1804 folio = page_folio(page); 1805 1806 if (!folio_try_get(folio)) 1807 continue; 1808 1809 if (unlikely(page_folio(page) != folio)) 1810 goto put_folio; 1811 1812 if (folio_test_large(folio)) 1813 pfn = folio_pfn(folio) + folio_nr_pages(folio) - 1; 1814 1815 if (folio_contain_hwpoisoned_page(folio)) { 1816 /* 1817 * unmap_poisoned_folio() cannot handle large folios 1818 * in all cases yet. 1819 */ 1820 if (folio_test_large(folio) && !folio_test_hugetlb(folio)) 1821 goto put_folio; 1822 if (folio_test_lru(folio) && !folio_isolate_lru(folio)) 1823 goto put_folio; 1824 if (folio_mapped(folio)) { 1825 folio_lock(folio); 1826 unmap_poisoned_folio(folio, pfn, false); 1827 folio_unlock(folio); 1828 } 1829 1830 goto put_folio; 1831 } 1832 1833 if (!isolate_folio_to_list(folio, &source)) { 1834 if (__ratelimit(&migrate_rs)) { 1835 pr_warn("failed to isolate pfn %lx\n", 1836 page_to_pfn(page)); 1837 dump_page(page, "isolation failed"); 1838 } 1839 } 1840put_folio: 1841 folio_put(folio); 1842 } 1843 if (!list_empty(&source)) { 1844 nodemask_t nmask = node_states[N_MEMORY]; 1845 struct migration_target_control mtc = { 1846 .nmask = &nmask, 1847 .gfp_mask = GFP_KERNEL | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, 1848 .reason = MR_MEMORY_HOTPLUG, 1849 }; 1850 int ret; 1851 1852 /* 1853 * We have checked that migration range is on a single zone so 1854 * we can use the nid of the first page to all the others. 1855 */ 1856 mtc.nid = folio_nid(list_first_entry(&source, struct folio, lru)); 1857 1858 /* 1859 * try to allocate from a different node but reuse this node 1860 * if there are no other online nodes to be used (e.g. we are 1861 * offlining a part of the only existing node) 1862 */ 1863 node_clear(mtc.nid, nmask); 1864 if (nodes_empty(nmask)) 1865 node_set(mtc.nid, nmask); 1866 ret = migrate_pages(&source, alloc_migration_target, NULL, 1867 (unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_HOTPLUG, NULL); 1868 if (ret) { 1869 list_for_each_entry(folio, &source, lru) { 1870 if (__ratelimit(&migrate_rs)) { 1871 pr_warn("migrating pfn %lx failed ret:%d\n", 1872 folio_pfn(folio), ret); 1873 dump_page(&folio->page, 1874 "migration failure"); 1875 } 1876 } 1877 putback_movable_pages(&source); 1878 } 1879 } 1880} 1881 1882static int __init cmdline_parse_movable_node(char *p) 1883{ 1884 movable_node_enabled = true; 1885 return 0; 1886} 1887early_param("movable_node", cmdline_parse_movable_node); 1888 1889static int count_system_ram_pages_cb(unsigned long start_pfn, 1890 unsigned long nr_pages, void *data) 1891{ 1892 unsigned long *nr_system_ram_pages = data; 1893 1894 *nr_system_ram_pages += nr_pages; 1895 return 0; 1896} 1897 1898/* 1899 * Must be called with mem_hotplug_lock in write mode. 1900 */ 1901int offline_pages(unsigned long start_pfn, unsigned long nr_pages, 1902 struct zone *zone, struct memory_group *group) 1903{ 1904 unsigned long pfn, managed_pages, system_ram_pages = 0; 1905 const unsigned long end_pfn = start_pfn + nr_pages; 1906 struct pglist_data *pgdat = zone->zone_pgdat; 1907 const int node = zone_to_nid(zone); 1908 struct memory_notify mem_arg = { 1909 .start_pfn = start_pfn, 1910 .nr_pages = nr_pages, 1911 }; 1912 struct node_notify node_arg = { 1913 .nid = NUMA_NO_NODE, 1914 }; 1915 unsigned long flags; 1916 char *reason; 1917 int ret; 1918 unsigned long normal_pages = 0; 1919 enum zone_type zt; 1920 1921 /* 1922 * {on,off}lining is constrained to full memory sections (or more 1923 * precisely to memory blocks from the user space POV). 1924 * memmap_on_memory is an exception because it reserves initial part 1925 * of the physical memory space for vmemmaps. That space is pageblock 1926 * aligned. 1927 */ 1928 if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(start_pfn) || 1929 !IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION))) 1930 return -EINVAL; 1931 1932 /* 1933 * Don't allow to offline memory blocks that contain holes. 1934 * Consequently, memory blocks with holes can never get onlined 1935 * via the hotplug path - online_pages() - as hotplugged memory has 1936 * no holes. This way, we don't have to worry about memory holes, 1937 * don't need pfn_valid() checks, and can avoid using 1938 * walk_system_ram_range() later. 1939 */ 1940 walk_system_ram_range(start_pfn, nr_pages, &system_ram_pages, 1941 count_system_ram_pages_cb); 1942 if (system_ram_pages != nr_pages) { 1943 ret = -EINVAL; 1944 reason = "memory holes"; 1945 goto failed_removal; 1946 } 1947 1948 /* 1949 * We only support offlining of memory blocks managed by a single zone, 1950 * checked by calling code. This is just a sanity check that we might 1951 * want to remove in the future. 1952 */ 1953 if (WARN_ON_ONCE(page_zone(pfn_to_page(start_pfn)) != zone || 1954 page_zone(pfn_to_page(end_pfn - 1)) != zone)) { 1955 ret = -EINVAL; 1956 reason = "multizone range"; 1957 goto failed_removal; 1958 } 1959 1960 /* 1961 * Disable pcplists so that page isolation cannot race with freeing 1962 * in a way that pages from isolated pageblock are left on pcplists. 1963 */ 1964 zone_pcp_disable(zone); 1965 lru_cache_disable(); 1966 1967 /* set above range as isolated */ 1968 ret = start_isolate_page_range(start_pfn, end_pfn, 1969 PB_ISOLATE_MODE_MEM_OFFLINE); 1970 if (ret) { 1971 reason = "failure to isolate range"; 1972 goto failed_removal_pcplists_disabled; 1973 } 1974 1975 /* 1976 * Check whether the node will have no present pages after we offline 1977 * 'nr_pages' more. If so, we know that the node will become empty, and 1978 * so we will clear N_MEMORY for it. 1979 */ 1980 if (nr_pages >= pgdat->node_present_pages) { 1981 node_arg.nid = node; 1982 ret = node_notify(NODE_REMOVING_LAST_MEMORY, &node_arg); 1983 ret = notifier_to_errno(ret); 1984 if (ret) { 1985 reason = "node notifier failure"; 1986 goto failed_removal_isolated; 1987 } 1988 } 1989 1990 ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg); 1991 ret = notifier_to_errno(ret); 1992 if (ret) { 1993 reason = "notifier failure"; 1994 goto failed_removal_isolated; 1995 } 1996 1997 do { 1998 pfn = start_pfn; 1999 do { 2000 /* 2001 * Historically we always checked for any signal and 2002 * can't limit it to fatal signals without eventually 2003 * breaking user space. 2004 */ 2005 if (signal_pending(current)) { 2006 ret = -EINTR; 2007 reason = "signal backoff"; 2008 goto failed_removal_isolated; 2009 } 2010 2011 cond_resched(); 2012 2013 ret = scan_movable_pages(pfn, end_pfn, &pfn); 2014 if (!ret) { 2015 /* 2016 * TODO: fatal migration failures should bail 2017 * out 2018 */ 2019 do_migrate_range(pfn, end_pfn); 2020 } 2021 } while (!ret); 2022 2023 if (ret != -ENOENT) { 2024 reason = "unmovable page"; 2025 goto failed_removal_isolated; 2026 } 2027 2028 /* 2029 * Dissolve free hugetlb folios in the memory block before doing 2030 * offlining actually in order to make hugetlbfs's object 2031 * counting consistent. 2032 */ 2033 ret = dissolve_free_hugetlb_folios(start_pfn, end_pfn); 2034 if (ret) { 2035 reason = "failure to dissolve huge pages"; 2036 goto failed_removal_isolated; 2037 } 2038 2039 ret = test_pages_isolated(start_pfn, end_pfn, 2040 PB_ISOLATE_MODE_MEM_OFFLINE); 2041 2042 } while (ret); 2043 2044 /* Mark all sections offline and remove free pages from the buddy. */ 2045 managed_pages = __offline_isolated_pages(start_pfn, end_pfn); 2046 pr_debug("Offlined Pages %ld\n", nr_pages); 2047 2048 /* 2049 * The memory sections are marked offline, and the pageblock flags 2050 * effectively stale; nobody should be touching them. Fixup the number 2051 * of isolated pageblocks, memory onlining will properly revert this. 2052 */ 2053 spin_lock_irqsave(&zone->lock, flags); 2054 zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages; 2055 spin_unlock_irqrestore(&zone->lock, flags); 2056 2057 lru_cache_enable(); 2058 zone_pcp_enable(zone); 2059 2060 /* removal success */ 2061 adjust_managed_page_count(pfn_to_page(start_pfn), -managed_pages); 2062 adjust_present_page_count(pfn_to_page(start_pfn), group, -nr_pages); 2063 2064 /* reinitialise watermarks and update pcp limits */ 2065 init_per_zone_wmark_min(); 2066 2067 /* 2068 * Check whether this operation removes the last normal memory from 2069 * the node. We do this before clearing N_MEMORY to avoid the possible 2070 * transient "!N_MEMORY && N_NORMAL_MEMORY" state. 2071 */ 2072 if (zone_idx(zone) <= ZONE_NORMAL) { 2073 for (zt = 0; zt <= ZONE_NORMAL; zt++) 2074 normal_pages += pgdat->node_zones[zt].present_pages; 2075 if (!normal_pages) 2076 node_clear_state(node, N_NORMAL_MEMORY); 2077 } 2078 /* 2079 * Make sure to mark the node as memory-less before rebuilding the zone 2080 * list. Otherwise this node would still appear in the fallback lists. 2081 */ 2082 if (node_arg.nid >= 0) 2083 node_clear_state(node, N_MEMORY); 2084 if (!populated_zone(zone)) { 2085 zone_pcp_reset(zone); 2086 build_all_zonelists(NULL); 2087 } 2088 2089 if (node_arg.nid >= 0) { 2090 kcompactd_stop(node); 2091 kswapd_stop(node); 2092 /* Node went memoryless. Notify consumers */ 2093 node_notify(NODE_REMOVED_LAST_MEMORY, &node_arg); 2094 } 2095 2096 writeback_set_ratelimit(); 2097 2098 memory_notify(MEM_OFFLINE, &mem_arg); 2099 remove_pfn_range_from_zone(zone, start_pfn, nr_pages); 2100 return 0; 2101 2102failed_removal_isolated: 2103 /* pushback to free area */ 2104 undo_isolate_page_range(start_pfn, end_pfn); 2105 memory_notify(MEM_CANCEL_OFFLINE, &mem_arg); 2106 if (node_arg.nid != NUMA_NO_NODE) 2107 node_notify(NODE_CANCEL_REMOVING_LAST_MEMORY, &node_arg); 2108failed_removal_pcplists_disabled: 2109 lru_cache_enable(); 2110 zone_pcp_enable(zone); 2111failed_removal: 2112 pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n", 2113 (unsigned long long) start_pfn << PAGE_SHIFT, 2114 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1, 2115 reason); 2116 return ret; 2117} 2118 2119static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) 2120{ 2121 int *nid = arg; 2122 2123 *nid = mem->nid; 2124 if (unlikely(mem->state != MEM_OFFLINE)) { 2125 phys_addr_t beginpa, endpa; 2126 2127 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); 2128 endpa = beginpa + memory_block_size_bytes() - 1; 2129 pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n", 2130 &beginpa, &endpa); 2131 2132 return -EBUSY; 2133 } 2134 return 0; 2135} 2136 2137static int count_memory_range_altmaps_cb(struct memory_block *mem, void *arg) 2138{ 2139 u64 *num_altmaps = (u64 *)arg; 2140 2141 if (mem->altmap) 2142 *num_altmaps += 1; 2143 2144 return 0; 2145} 2146 2147static int check_cpu_on_node(int nid) 2148{ 2149 int cpu; 2150 2151 for_each_present_cpu(cpu) { 2152 if (cpu_to_node(cpu) == nid) 2153 /* 2154 * the cpu on this node isn't removed, and we can't 2155 * offline this node. 2156 */ 2157 return -EBUSY; 2158 } 2159 2160 return 0; 2161} 2162 2163static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) 2164{ 2165 int nid = *(int *)arg; 2166 2167 /* 2168 * If a memory block belongs to multiple nodes, the stored nid is not 2169 * reliable. However, such blocks are always online (e.g., cannot get 2170 * offlined) and, therefore, are still spanned by the node. 2171 */ 2172 return mem->nid == nid ? -EEXIST : 0; 2173} 2174 2175/** 2176 * try_offline_node 2177 * @nid: the node ID 2178 * 2179 * Offline a node if all memory sections and cpus of the node are removed. 2180 * 2181 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 2182 * and online/offline operations before this call. 2183 */ 2184void try_offline_node(int nid) 2185{ 2186 int rc; 2187 2188 /* 2189 * If the node still spans pages (especially ZONE_DEVICE), don't 2190 * offline it. A node spans memory after move_pfn_range_to_zone(), 2191 * e.g., after the memory block was onlined. 2192 */ 2193 if (node_spanned_pages(nid)) 2194 return; 2195 2196 /* 2197 * Especially offline memory blocks might not be spanned by the 2198 * node. They will get spanned by the node once they get onlined. 2199 * However, they link to the node in sysfs and can get onlined later. 2200 */ 2201 rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb); 2202 if (rc) 2203 return; 2204 2205 if (check_cpu_on_node(nid)) 2206 return; 2207 2208 /* 2209 * all memory/cpu of this node are removed, we can offline this 2210 * node now. 2211 */ 2212 node_set_offline(nid); 2213 unregister_node(nid); 2214} 2215EXPORT_SYMBOL(try_offline_node); 2216 2217static int memory_blocks_have_altmaps(u64 start, u64 size) 2218{ 2219 u64 num_memblocks = size / memory_block_size_bytes(); 2220 u64 num_altmaps = 0; 2221 2222 if (!mhp_memmap_on_memory()) 2223 return 0; 2224 2225 walk_memory_blocks(start, size, &num_altmaps, 2226 count_memory_range_altmaps_cb); 2227 2228 if (num_altmaps == 0) 2229 return 0; 2230 2231 if (WARN_ON_ONCE(num_memblocks != num_altmaps)) 2232 return -EINVAL; 2233 2234 return 1; 2235} 2236 2237static int try_remove_memory(u64 start, u64 size) 2238{ 2239 int rc, nid = NUMA_NO_NODE; 2240 2241 BUG_ON(check_hotplug_memory_range(start, size)); 2242 2243 /* 2244 * All memory blocks must be offlined before removing memory. Check 2245 * whether all memory blocks in question are offline and return error 2246 * if this is not the case. 2247 * 2248 * While at it, determine the nid. Note that if we'd have mixed nodes, 2249 * we'd only try to offline the last determined one -- which is good 2250 * enough for the cases we care about. 2251 */ 2252 rc = walk_memory_blocks(start, size, &nid, check_memblock_offlined_cb); 2253 if (rc) 2254 return rc; 2255 2256 /* remove memmap entry */ 2257 firmware_map_remove(start, start + size, "System RAM"); 2258 2259 mem_hotplug_begin(); 2260 2261 rc = memory_blocks_have_altmaps(start, size); 2262 if (rc < 0) { 2263 mem_hotplug_done(); 2264 return rc; 2265 } else if (!rc) { 2266 /* 2267 * Memory block device removal under the device_hotplug_lock is 2268 * a barrier against racing online attempts. 2269 * No altmaps present, do the removal directly 2270 */ 2271 remove_memory_block_devices(start, size); 2272 arch_remove_memory(start, size, NULL); 2273 } else { 2274 /* all memblocks in the range have altmaps */ 2275 remove_memory_blocks_and_altmaps(start, size); 2276 } 2277 2278 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) 2279 memblock_remove(start, size); 2280 2281 release_mem_region_adjustable(start, size); 2282 2283 if (nid != NUMA_NO_NODE) 2284 try_offline_node(nid); 2285 2286 mem_hotplug_done(); 2287 return 0; 2288} 2289 2290/** 2291 * __remove_memory - Remove memory if every memory block is offline 2292 * @start: physical address of the region to remove 2293 * @size: size of the region to remove 2294 * 2295 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 2296 * and online/offline operations before this call, as required by 2297 * try_offline_node(). 2298 */ 2299void __remove_memory(u64 start, u64 size) 2300{ 2301 2302 /* 2303 * trigger BUG() if some memory is not offlined prior to calling this 2304 * function 2305 */ 2306 if (try_remove_memory(start, size)) 2307 BUG(); 2308} 2309 2310/* 2311 * Remove memory if every memory block is offline, otherwise return -EBUSY is 2312 * some memory is not offline 2313 */ 2314int remove_memory(u64 start, u64 size) 2315{ 2316 int rc; 2317 2318 lock_device_hotplug(); 2319 rc = try_remove_memory(start, size); 2320 unlock_device_hotplug(); 2321 2322 return rc; 2323} 2324EXPORT_SYMBOL_GPL(remove_memory); 2325 2326static int try_offline_memory_block(struct memory_block *mem, void *arg) 2327{ 2328 uint8_t online_type = MMOP_ONLINE_KERNEL; 2329 uint8_t **online_types = arg; 2330 struct page *page; 2331 int rc; 2332 2333 /* 2334 * Sense the online_type via the zone of the memory block. Offlining 2335 * with multiple zones within one memory block will be rejected 2336 * by offlining code ... so we don't care about that. 2337 */ 2338 page = pfn_to_online_page(section_nr_to_pfn(mem->start_section_nr)); 2339 if (page && page_zonenum(page) == ZONE_MOVABLE) 2340 online_type = MMOP_ONLINE_MOVABLE; 2341 2342 rc = device_offline(&mem->dev); 2343 /* 2344 * Default is MMOP_OFFLINE - change it only if offlining succeeded, 2345 * so try_reonline_memory_block() can do the right thing. 2346 */ 2347 if (!rc) 2348 **online_types = online_type; 2349 2350 (*online_types)++; 2351 /* Ignore if already offline. */ 2352 return rc < 0 ? rc : 0; 2353} 2354 2355static int try_reonline_memory_block(struct memory_block *mem, void *arg) 2356{ 2357 uint8_t **online_types = arg; 2358 int rc; 2359 2360 if (**online_types != MMOP_OFFLINE) { 2361 mem->online_type = **online_types; 2362 rc = device_online(&mem->dev); 2363 if (rc < 0) 2364 pr_warn("%s: Failed to re-online memory: %d", 2365 __func__, rc); 2366 } 2367 2368 /* Continue processing all remaining memory blocks. */ 2369 (*online_types)++; 2370 return 0; 2371} 2372 2373/* 2374 * Try to offline and remove memory. Might take a long time to finish in case 2375 * memory is still in use. Primarily useful for memory devices that logically 2376 * unplugged all memory (so it's no longer in use) and want to offline + remove 2377 * that memory. 2378 */ 2379int offline_and_remove_memory(u64 start, u64 size) 2380{ 2381 const unsigned long mb_count = size / memory_block_size_bytes(); 2382 uint8_t *online_types, *tmp; 2383 int rc; 2384 2385 if (!IS_ALIGNED(start, memory_block_size_bytes()) || 2386 !IS_ALIGNED(size, memory_block_size_bytes()) || !size) 2387 return -EINVAL; 2388 2389 /* 2390 * We'll remember the old online type of each memory block, so we can 2391 * try to revert whatever we did when offlining one memory block fails 2392 * after offlining some others succeeded. 2393 */ 2394 online_types = kmalloc_array(mb_count, sizeof(*online_types), 2395 GFP_KERNEL); 2396 if (!online_types) 2397 return -ENOMEM; 2398 /* 2399 * Initialize all states to MMOP_OFFLINE, so when we abort processing in 2400 * try_offline_memory_block(), we'll skip all unprocessed blocks in 2401 * try_reonline_memory_block(). 2402 */ 2403 memset(online_types, MMOP_OFFLINE, mb_count); 2404 2405 lock_device_hotplug(); 2406 2407 tmp = online_types; 2408 rc = walk_memory_blocks(start, size, &tmp, try_offline_memory_block); 2409 2410 /* 2411 * In case we succeeded to offline all memory, remove it. 2412 * This cannot fail as it cannot get onlined in the meantime. 2413 */ 2414 if (!rc) { 2415 rc = try_remove_memory(start, size); 2416 if (rc) 2417 pr_err("%s: Failed to remove memory: %d", __func__, rc); 2418 } 2419 2420 /* 2421 * Rollback what we did. While memory onlining might theoretically fail 2422 * (nacked by a notifier), it barely ever happens. 2423 */ 2424 if (rc) { 2425 tmp = online_types; 2426 walk_memory_blocks(start, size, &tmp, 2427 try_reonline_memory_block); 2428 } 2429 unlock_device_hotplug(); 2430 2431 kfree(online_types); 2432 return rc; 2433} 2434EXPORT_SYMBOL_GPL(offline_and_remove_memory); 2435#endif /* CONFIG_MEMORY_HOTREMOVE */
~/
Read-only file preview.