FILE
sparse.c
File preview with syntax highlighting.
linux-6.19.14/mm/sparse.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * sparse memory mappings. 4 */ 5#include <linux/mm.h> 6#include <linux/slab.h> 7#include <linux/mmzone.h> 8#include <linux/memblock.h> 9#include <linux/compiler.h> 10#include <linux/highmem.h> 11#include <linux/export.h> 12#include <linux/spinlock.h> 13#include <linux/vmalloc.h> 14#include <linux/swap.h> 15#include <linux/swapops.h> 16#include <linux/bootmem_info.h> 17#include <linux/vmstat.h> 18#include "internal.h" 19#include <asm/dma.h> 20 21/* 22 * Permanent SPARSEMEM data: 23 * 24 * 1) mem_section - memory sections, mem_map's for valid memory 25 */ 26#ifdef CONFIG_SPARSEMEM_EXTREME 27struct mem_section **mem_section; 28#else 29struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 30 ____cacheline_internodealigned_in_smp; 31#endif 32EXPORT_SYMBOL(mem_section); 33 34#ifdef NODE_NOT_IN_PAGE_FLAGS 35/* 36 * If we did not store the node number in the page then we have to 37 * do a lookup in the section_to_node_table in order to find which 38 * node the page belongs to. 39 */ 40#if MAX_NUMNODES <= 256 41static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 42#else 43static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 44#endif 45 46int memdesc_nid(memdesc_flags_t mdf) 47{ 48 return section_to_node_table[memdesc_section(mdf)]; 49} 50EXPORT_SYMBOL(memdesc_nid); 51 52static void set_section_nid(unsigned long section_nr, int nid) 53{ 54 section_to_node_table[section_nr] = nid; 55} 56#else /* !NODE_NOT_IN_PAGE_FLAGS */ 57static inline void set_section_nid(unsigned long section_nr, int nid) 58{ 59} 60#endif 61 62#ifdef CONFIG_SPARSEMEM_EXTREME 63static noinline struct mem_section __ref *sparse_index_alloc(int nid) 64{ 65 struct mem_section *section = NULL; 66 unsigned long array_size = SECTIONS_PER_ROOT * 67 sizeof(struct mem_section); 68 69 if (slab_is_available()) { 70 section = kzalloc_node(array_size, GFP_KERNEL, nid); 71 } else { 72 section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, 73 nid); 74 if (!section) 75 panic("%s: Failed to allocate %lu bytes nid=%d\n", 76 __func__, array_size, nid); 77 } 78 79 return section; 80} 81 82static int __meminit sparse_index_init(unsigned long section_nr, int nid) 83{ 84 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 85 struct mem_section *section; 86 87 /* 88 * An existing section is possible in the sub-section hotplug 89 * case. First hot-add instantiates, follow-on hot-add reuses 90 * the existing section. 91 * 92 * The mem_hotplug_lock resolves the apparent race below. 93 */ 94 if (mem_section[root]) 95 return 0; 96 97 section = sparse_index_alloc(nid); 98 if (!section) 99 return -ENOMEM; 100 101 mem_section[root] = section; 102 103 return 0; 104} 105#else /* !SPARSEMEM_EXTREME */ 106static inline int sparse_index_init(unsigned long section_nr, int nid) 107{ 108 return 0; 109} 110#endif 111 112/* 113 * During early boot, before section_mem_map is used for an actual 114 * mem_map, we use section_mem_map to store the section's NUMA 115 * node. This keeps us from having to use another data structure. The 116 * node information is cleared just before we store the real mem_map. 117 */ 118static inline unsigned long sparse_encode_early_nid(int nid) 119{ 120 return ((unsigned long)nid << SECTION_NID_SHIFT); 121} 122 123static inline int sparse_early_nid(struct mem_section *section) 124{ 125 return (section->section_mem_map >> SECTION_NID_SHIFT); 126} 127 128/* Validate the physical addressing limitations of the model */ 129static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 130 unsigned long *end_pfn) 131{ 132 unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT; 133 134 /* 135 * Sanity checks - do not allow an architecture to pass 136 * in larger pfns than the maximum scope of sparsemem: 137 */ 138 if (*start_pfn > max_sparsemem_pfn) { 139 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 140 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 141 *start_pfn, *end_pfn, max_sparsemem_pfn); 142 WARN_ON_ONCE(1); 143 *start_pfn = max_sparsemem_pfn; 144 *end_pfn = max_sparsemem_pfn; 145 } else if (*end_pfn > max_sparsemem_pfn) { 146 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 147 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 148 *start_pfn, *end_pfn, max_sparsemem_pfn); 149 WARN_ON_ONCE(1); 150 *end_pfn = max_sparsemem_pfn; 151 } 152} 153 154/* 155 * There are a number of times that we loop over NR_MEM_SECTIONS, 156 * looking for section_present() on each. But, when we have very 157 * large physical address spaces, NR_MEM_SECTIONS can also be 158 * very large which makes the loops quite long. 159 * 160 * Keeping track of this gives us an easy way to break out of 161 * those loops early. 162 */ 163unsigned long __highest_present_section_nr; 164static void __section_mark_present(struct mem_section *ms, 165 unsigned long section_nr) 166{ 167 if (section_nr > __highest_present_section_nr) 168 __highest_present_section_nr = section_nr; 169 170 ms->section_mem_map |= SECTION_MARKED_PRESENT; 171} 172 173static inline unsigned long first_present_section_nr(void) 174{ 175 return next_present_section_nr(-1); 176} 177 178#ifdef CONFIG_SPARSEMEM_VMEMMAP 179static void subsection_mask_set(unsigned long *map, unsigned long pfn, 180 unsigned long nr_pages) 181{ 182 int idx = subsection_map_index(pfn); 183 int end = subsection_map_index(pfn + nr_pages - 1); 184 185 bitmap_set(map, idx, end - idx + 1); 186} 187 188void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) 189{ 190 int end_sec_nr = pfn_to_section_nr(pfn + nr_pages - 1); 191 unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn); 192 193 for (nr = start_sec_nr; nr <= end_sec_nr; nr++) { 194 struct mem_section *ms; 195 unsigned long pfns; 196 197 pfns = min(nr_pages, PAGES_PER_SECTION 198 - (pfn & ~PAGE_SECTION_MASK)); 199 ms = __nr_to_section(nr); 200 subsection_mask_set(ms->usage->subsection_map, pfn, pfns); 201 202 pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr, 203 pfns, subsection_map_index(pfn), 204 subsection_map_index(pfn + pfns - 1)); 205 206 pfn += pfns; 207 nr_pages -= pfns; 208 } 209} 210#else 211void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) 212{ 213} 214#endif 215 216/* Record a memory area against a node. */ 217static void __init memory_present(int nid, unsigned long start, unsigned long end) 218{ 219 unsigned long pfn; 220 221 start &= PAGE_SECTION_MASK; 222 mminit_validate_memmodel_limits(&start, &end); 223 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 224 unsigned long section_nr = pfn_to_section_nr(pfn); 225 struct mem_section *ms; 226 227 sparse_index_init(section_nr, nid); 228 set_section_nid(section_nr, nid); 229 230 ms = __nr_to_section(section_nr); 231 if (!ms->section_mem_map) { 232 ms->section_mem_map = sparse_encode_early_nid(nid) | 233 SECTION_IS_ONLINE; 234 __section_mark_present(ms, section_nr); 235 } 236 } 237} 238 239/* 240 * Mark all memblocks as present using memory_present(). 241 * This is a convenience function that is useful to mark all of the systems 242 * memory as present during initialization. 243 */ 244static void __init memblocks_present(void) 245{ 246 unsigned long start, end; 247 int i, nid; 248 249#ifdef CONFIG_SPARSEMEM_EXTREME 250 if (unlikely(!mem_section)) { 251 unsigned long size, align; 252 253 size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; 254 align = 1 << (INTERNODE_CACHE_SHIFT); 255 mem_section = memblock_alloc_or_panic(size, align); 256 } 257#endif 258 259 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) 260 memory_present(nid, start, end); 261} 262 263/* 264 * Subtle, we encode the real pfn into the mem_map such that 265 * the identity pfn - section_mem_map will return the actual 266 * physical page frame number. 267 */ 268static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 269{ 270 unsigned long coded_mem_map = 271 (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 272 BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); 273 BUG_ON(coded_mem_map & ~SECTION_MAP_MASK); 274 return coded_mem_map; 275} 276 277#ifdef CONFIG_MEMORY_HOTPLUG 278/* 279 * Decode mem_map from the coded memmap 280 */ 281struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 282{ 283 /* mask off the extra low bits of information */ 284 coded_mem_map &= SECTION_MAP_MASK; 285 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 286} 287#endif /* CONFIG_MEMORY_HOTPLUG */ 288 289static void __meminit sparse_init_one_section(struct mem_section *ms, 290 unsigned long pnum, struct page *mem_map, 291 struct mem_section_usage *usage, unsigned long flags) 292{ 293 ms->section_mem_map &= ~SECTION_MAP_MASK; 294 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) 295 | SECTION_HAS_MEM_MAP | flags; 296 ms->usage = usage; 297} 298 299static unsigned long usemap_size(void) 300{ 301 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long); 302} 303 304size_t mem_section_usage_size(void) 305{ 306 return sizeof(struct mem_section_usage) + usemap_size(); 307} 308 309#ifdef CONFIG_MEMORY_HOTREMOVE 310static inline phys_addr_t pgdat_to_phys(struct pglist_data *pgdat) 311{ 312#ifndef CONFIG_NUMA 313 VM_BUG_ON(pgdat != &contig_page_data); 314 return __pa_symbol(&contig_page_data); 315#else 316 return __pa(pgdat); 317#endif 318} 319 320static struct mem_section_usage * __init 321sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 322 unsigned long size) 323{ 324 struct mem_section_usage *usage; 325 unsigned long goal, limit; 326 int nid; 327 /* 328 * A page may contain usemaps for other sections preventing the 329 * page being freed and making a section unremovable while 330 * other sections referencing the usemap remain active. Similarly, 331 * a pgdat can prevent a section being removed. If section A 332 * contains a pgdat and section B contains the usemap, both 333 * sections become inter-dependent. This allocates usemaps 334 * from the same section as the pgdat where possible to avoid 335 * this problem. 336 */ 337 goal = pgdat_to_phys(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT); 338 limit = goal + (1UL << PA_SECTION_SHIFT); 339 nid = early_pfn_to_nid(goal >> PAGE_SHIFT); 340again: 341 usage = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, goal, limit, nid); 342 if (!usage && limit) { 343 limit = MEMBLOCK_ALLOC_ACCESSIBLE; 344 goto again; 345 } 346 return usage; 347} 348 349static void __init check_usemap_section_nr(int nid, 350 struct mem_section_usage *usage) 351{ 352 unsigned long usemap_snr, pgdat_snr; 353 static unsigned long old_usemap_snr; 354 static unsigned long old_pgdat_snr; 355 struct pglist_data *pgdat = NODE_DATA(nid); 356 int usemap_nid; 357 358 /* First call */ 359 if (!old_usemap_snr) { 360 old_usemap_snr = NR_MEM_SECTIONS; 361 old_pgdat_snr = NR_MEM_SECTIONS; 362 } 363 364 usemap_snr = pfn_to_section_nr(__pa(usage) >> PAGE_SHIFT); 365 pgdat_snr = pfn_to_section_nr(pgdat_to_phys(pgdat) >> PAGE_SHIFT); 366 if (usemap_snr == pgdat_snr) 367 return; 368 369 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 370 /* skip redundant message */ 371 return; 372 373 old_usemap_snr = usemap_snr; 374 old_pgdat_snr = pgdat_snr; 375 376 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 377 if (usemap_nid != nid) { 378 pr_info("node %d must be removed before remove section %ld\n", 379 nid, usemap_snr); 380 return; 381 } 382 /* 383 * There is a circular dependency. 384 * Some platforms allow un-removable section because they will just 385 * gather other removable sections for dynamic partitioning. 386 * Just notify un-removable section's number here. 387 */ 388 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n", 389 usemap_snr, pgdat_snr, nid); 390} 391#else 392static struct mem_section_usage * __init 393sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 394 unsigned long size) 395{ 396 return memblock_alloc_node(size, SMP_CACHE_BYTES, pgdat->node_id); 397} 398 399static void __init check_usemap_section_nr(int nid, 400 struct mem_section_usage *usage) 401{ 402} 403#endif /* CONFIG_MEMORY_HOTREMOVE */ 404 405#ifdef CONFIG_SPARSEMEM_VMEMMAP 406unsigned long __init section_map_size(void) 407{ 408 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); 409} 410 411#else 412unsigned long __init section_map_size(void) 413{ 414 return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); 415} 416 417struct page __init *__populate_section_memmap(unsigned long pfn, 418 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 419 struct dev_pagemap *pgmap) 420{ 421 unsigned long size = section_map_size(); 422 struct page *map = sparse_buffer_alloc(size); 423 phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 424 425 if (map) 426 return map; 427 428 map = memmap_alloc(size, size, addr, nid, false); 429 if (!map) 430 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", 431 __func__, size, PAGE_SIZE, nid, &addr); 432 433 return map; 434} 435#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 436 437static void *sparsemap_buf __meminitdata; 438static void *sparsemap_buf_end __meminitdata; 439 440static inline void __meminit sparse_buffer_free(unsigned long size) 441{ 442 WARN_ON(!sparsemap_buf || size == 0); 443 memblock_free(sparsemap_buf, size); 444} 445 446static void __init sparse_buffer_init(unsigned long size, int nid) 447{ 448 phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 449 WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */ 450 /* 451 * Pre-allocated buffer is mainly used by __populate_section_memmap 452 * and we want it to be properly aligned to the section size - this is 453 * especially the case for VMEMMAP which maps memmap to PMDs 454 */ 455 sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true); 456 sparsemap_buf_end = sparsemap_buf + size; 457} 458 459static void __init sparse_buffer_fini(void) 460{ 461 unsigned long size = sparsemap_buf_end - sparsemap_buf; 462 463 if (sparsemap_buf && size > 0) 464 sparse_buffer_free(size); 465 sparsemap_buf = NULL; 466} 467 468void * __meminit sparse_buffer_alloc(unsigned long size) 469{ 470 void *ptr = NULL; 471 472 if (sparsemap_buf) { 473 ptr = (void *) roundup((unsigned long)sparsemap_buf, size); 474 if (ptr + size > sparsemap_buf_end) 475 ptr = NULL; 476 else { 477 /* Free redundant aligned space */ 478 if ((unsigned long)(ptr - sparsemap_buf) > 0) 479 sparse_buffer_free((unsigned long)(ptr - sparsemap_buf)); 480 sparsemap_buf = ptr + size; 481 } 482 } 483 return ptr; 484} 485 486void __weak __meminit vmemmap_populate_print_last(void) 487{ 488} 489 490static void *sparse_usagebuf __meminitdata; 491static void *sparse_usagebuf_end __meminitdata; 492 493/* 494 * Helper function that is used for generic section initialization, and 495 * can also be used by any hooks added above. 496 */ 497void __init sparse_init_early_section(int nid, struct page *map, 498 unsigned long pnum, unsigned long flags) 499{ 500 BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end); 501 check_usemap_section_nr(nid, sparse_usagebuf); 502 sparse_init_one_section(__nr_to_section(pnum), pnum, map, 503 sparse_usagebuf, SECTION_IS_EARLY | flags); 504 sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size(); 505} 506 507static int __init sparse_usage_init(int nid, unsigned long map_count) 508{ 509 unsigned long size; 510 511 size = mem_section_usage_size() * map_count; 512 sparse_usagebuf = sparse_early_usemaps_alloc_pgdat_section( 513 NODE_DATA(nid), size); 514 if (!sparse_usagebuf) { 515 sparse_usagebuf_end = NULL; 516 return -ENOMEM; 517 } 518 519 sparse_usagebuf_end = sparse_usagebuf + size; 520 return 0; 521} 522 523static void __init sparse_usage_fini(void) 524{ 525 sparse_usagebuf = sparse_usagebuf_end = NULL; 526} 527 528/* 529 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) 530 * And number of present sections in this node is map_count. 531 */ 532static void __init sparse_init_nid(int nid, unsigned long pnum_begin, 533 unsigned long pnum_end, 534 unsigned long map_count) 535{ 536 unsigned long pnum; 537 struct page *map; 538 struct mem_section *ms; 539 540 if (sparse_usage_init(nid, map_count)) { 541 pr_err("%s: node[%d] usemap allocation failed", __func__, nid); 542 goto failed; 543 } 544 545 sparse_buffer_init(map_count * section_map_size(), nid); 546 547 sparse_vmemmap_init_nid_early(nid); 548 549 for_each_present_section_nr(pnum_begin, pnum) { 550 unsigned long pfn = section_nr_to_pfn(pnum); 551 552 if (pnum >= pnum_end) 553 break; 554 555 ms = __nr_to_section(pnum); 556 if (!preinited_vmemmap_section(ms)) { 557 map = __populate_section_memmap(pfn, PAGES_PER_SECTION, 558 nid, NULL, NULL); 559 if (!map) { 560 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", 561 __func__, nid); 562 pnum_begin = pnum; 563 sparse_usage_fini(); 564 sparse_buffer_fini(); 565 goto failed; 566 } 567 memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), 568 PAGE_SIZE)); 569 sparse_init_early_section(nid, map, pnum, 0); 570 } 571 } 572 sparse_usage_fini(); 573 sparse_buffer_fini(); 574 return; 575failed: 576 /* 577 * We failed to allocate, mark all the following pnums as not present, 578 * except the ones already initialized earlier. 579 */ 580 for_each_present_section_nr(pnum_begin, pnum) { 581 if (pnum >= pnum_end) 582 break; 583 ms = __nr_to_section(pnum); 584 if (!preinited_vmemmap_section(ms)) 585 ms->section_mem_map = 0; 586 ms->section_mem_map = 0; 587 } 588} 589 590/* 591 * Allocate the accumulated non-linear sections, allocate a mem_map 592 * for each and record the physical to section mapping. 593 */ 594void __init sparse_init(void) 595{ 596 unsigned long pnum_end, pnum_begin, map_count = 1; 597 int nid_begin; 598 599 /* see include/linux/mmzone.h 'struct mem_section' definition */ 600 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); 601 memblocks_present(); 602 603 pnum_begin = first_present_section_nr(); 604 nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); 605 606 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ 607 set_pageblock_order(); 608 609 for_each_present_section_nr(pnum_begin + 1, pnum_end) { 610 int nid = sparse_early_nid(__nr_to_section(pnum_end)); 611 612 if (nid == nid_begin) { 613 map_count++; 614 continue; 615 } 616 /* Init node with sections in range [pnum_begin, pnum_end) */ 617 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 618 nid_begin = nid; 619 pnum_begin = pnum_end; 620 map_count = 1; 621 } 622 /* cover the last node */ 623 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 624 vmemmap_populate_print_last(); 625} 626 627#ifdef CONFIG_MEMORY_HOTPLUG 628 629/* Mark all memory sections within the pfn range as online */ 630void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn) 631{ 632 unsigned long pfn; 633 634 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 635 unsigned long section_nr = pfn_to_section_nr(pfn); 636 struct mem_section *ms; 637 638 /* onlining code should never touch invalid ranges */ 639 if (WARN_ON(!valid_section_nr(section_nr))) 640 continue; 641 642 ms = __nr_to_section(section_nr); 643 ms->section_mem_map |= SECTION_IS_ONLINE; 644 } 645} 646 647/* Mark all memory sections within the pfn range as offline */ 648void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn) 649{ 650 unsigned long pfn; 651 652 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 653 unsigned long section_nr = pfn_to_section_nr(pfn); 654 struct mem_section *ms; 655 656 /* 657 * TODO this needs some double checking. Offlining code makes 658 * sure to check pfn_valid but those checks might be just bogus 659 */ 660 if (WARN_ON(!valid_section_nr(section_nr))) 661 continue; 662 663 ms = __nr_to_section(section_nr); 664 ms->section_mem_map &= ~SECTION_IS_ONLINE; 665 } 666} 667 668#ifdef CONFIG_SPARSEMEM_VMEMMAP 669static struct page * __meminit populate_section_memmap(unsigned long pfn, 670 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 671 struct dev_pagemap *pgmap) 672{ 673 return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); 674} 675 676static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, 677 struct vmem_altmap *altmap) 678{ 679 unsigned long start = (unsigned long) pfn_to_page(pfn); 680 unsigned long end = start + nr_pages * sizeof(struct page); 681 682 vmemmap_free(start, end, altmap); 683} 684static void free_map_bootmem(struct page *memmap) 685{ 686 unsigned long start = (unsigned long)memmap; 687 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); 688 689 vmemmap_free(start, end, NULL); 690} 691 692static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) 693{ 694 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; 695 DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; 696 struct mem_section *ms = __pfn_to_section(pfn); 697 unsigned long *subsection_map = ms->usage 698 ? &ms->usage->subsection_map[0] : NULL; 699 700 subsection_mask_set(map, pfn, nr_pages); 701 if (subsection_map) 702 bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION); 703 704 if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), 705 "section already deactivated (%#lx + %ld)\n", 706 pfn, nr_pages)) 707 return -EINVAL; 708 709 bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); 710 return 0; 711} 712 713static bool is_subsection_map_empty(struct mem_section *ms) 714{ 715 return bitmap_empty(&ms->usage->subsection_map[0], 716 SUBSECTIONS_PER_SECTION); 717} 718 719static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) 720{ 721 struct mem_section *ms = __pfn_to_section(pfn); 722 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; 723 unsigned long *subsection_map; 724 int rc = 0; 725 726 subsection_mask_set(map, pfn, nr_pages); 727 728 subsection_map = &ms->usage->subsection_map[0]; 729 730 if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) 731 rc = -EINVAL; 732 else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION)) 733 rc = -EEXIST; 734 else 735 bitmap_or(subsection_map, map, subsection_map, 736 SUBSECTIONS_PER_SECTION); 737 738 return rc; 739} 740#else 741static struct page * __meminit populate_section_memmap(unsigned long pfn, 742 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 743 struct dev_pagemap *pgmap) 744{ 745 return kvmalloc_node(array_size(sizeof(struct page), 746 PAGES_PER_SECTION), GFP_KERNEL, nid); 747} 748 749static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, 750 struct vmem_altmap *altmap) 751{ 752 kvfree(pfn_to_page(pfn)); 753} 754 755static void free_map_bootmem(struct page *memmap) 756{ 757 unsigned long maps_section_nr, removing_section_nr, i; 758 unsigned long type, nr_pages; 759 struct page *page = virt_to_page(memmap); 760 761 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 762 >> PAGE_SHIFT; 763 764 for (i = 0; i < nr_pages; i++, page++) { 765 type = bootmem_type(page); 766 767 BUG_ON(type == NODE_INFO); 768 769 maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 770 removing_section_nr = bootmem_info(page); 771 772 /* 773 * When this function is called, the removing section is 774 * logical offlined state. This means all pages are isolated 775 * from page allocator. If removing section's memmap is placed 776 * on the same section, it must not be freed. 777 * If it is freed, page allocator may allocate it which will 778 * be removed physically soon. 779 */ 780 if (maps_section_nr != removing_section_nr) 781 put_page_bootmem(page); 782 } 783} 784 785static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) 786{ 787 return 0; 788} 789 790static bool is_subsection_map_empty(struct mem_section *ms) 791{ 792 return true; 793} 794 795static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) 796{ 797 return 0; 798} 799#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 800 801/* 802 * To deactivate a memory region, there are 3 cases to handle across 803 * two configurations (SPARSEMEM_VMEMMAP={y,n}): 804 * 805 * 1. deactivation of a partial hot-added section (only possible in 806 * the SPARSEMEM_VMEMMAP=y case). 807 * a) section was present at memory init. 808 * b) section was hot-added post memory init. 809 * 2. deactivation of a complete hot-added section. 810 * 3. deactivation of a complete section from memory init. 811 * 812 * For 1, when subsection_map does not empty we will not be freeing the 813 * usage map, but still need to free the vmemmap range. 814 * 815 * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified 816 */ 817static void section_deactivate(unsigned long pfn, unsigned long nr_pages, 818 struct vmem_altmap *altmap) 819{ 820 struct mem_section *ms = __pfn_to_section(pfn); 821 bool section_is_early = early_section(ms); 822 struct page *memmap = NULL; 823 bool empty; 824 825 if (clear_subsection_map(pfn, nr_pages)) 826 return; 827 828 empty = is_subsection_map_empty(ms); 829 if (empty) { 830 unsigned long section_nr = pfn_to_section_nr(pfn); 831 832 /* 833 * Mark the section invalid so that valid_section() 834 * return false. This prevents code from dereferencing 835 * ms->usage array. 836 */ 837 ms->section_mem_map &= ~SECTION_HAS_MEM_MAP; 838 839 /* 840 * When removing an early section, the usage map is kept (as the 841 * usage maps of other sections fall into the same page). It 842 * will be re-used when re-adding the section - which is then no 843 * longer an early section. If the usage map is PageReserved, it 844 * was allocated during boot. 845 */ 846 if (!PageReserved(virt_to_page(ms->usage))) { 847 kfree_rcu(ms->usage, rcu); 848 WRITE_ONCE(ms->usage, NULL); 849 } 850 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 851 } 852 853 /* 854 * The memmap of early sections is always fully populated. See 855 * section_activate() and pfn_valid() . 856 */ 857 if (!section_is_early) { 858 memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE))); 859 depopulate_section_memmap(pfn, nr_pages, altmap); 860 } else if (memmap) { 861 memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), 862 PAGE_SIZE))); 863 free_map_bootmem(memmap); 864 } 865 866 if (empty) 867 ms->section_mem_map = (unsigned long)NULL; 868} 869 870static struct page * __meminit section_activate(int nid, unsigned long pfn, 871 unsigned long nr_pages, struct vmem_altmap *altmap, 872 struct dev_pagemap *pgmap) 873{ 874 struct mem_section *ms = __pfn_to_section(pfn); 875 struct mem_section_usage *usage = NULL; 876 struct page *memmap; 877 int rc; 878 879 if (!ms->usage) { 880 usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); 881 if (!usage) 882 return ERR_PTR(-ENOMEM); 883 ms->usage = usage; 884 } 885 886 rc = fill_subsection_map(pfn, nr_pages); 887 if (rc) { 888 if (usage) 889 ms->usage = NULL; 890 kfree(usage); 891 return ERR_PTR(rc); 892 } 893 894 /* 895 * The early init code does not consider partially populated 896 * initial sections, it simply assumes that memory will never be 897 * referenced. If we hot-add memory into such a section then we 898 * do not need to populate the memmap and can simply reuse what 899 * is already there. 900 */ 901 if (nr_pages < PAGES_PER_SECTION && early_section(ms)) 902 return pfn_to_page(pfn); 903 904 memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); 905 if (!memmap) { 906 section_deactivate(pfn, nr_pages, altmap); 907 return ERR_PTR(-ENOMEM); 908 } 909 memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)); 910 911 return memmap; 912} 913 914/** 915 * sparse_add_section - add a memory section, or populate an existing one 916 * @nid: The node to add section on 917 * @start_pfn: start pfn of the memory range 918 * @nr_pages: number of pfns to add in the section 919 * @altmap: alternate pfns to allocate the memmap backing store 920 * @pgmap: alternate compound page geometry for devmap mappings 921 * 922 * This is only intended for hotplug. 923 * 924 * Note that only VMEMMAP supports sub-section aligned hotplug, 925 * the proper alignment and size are gated by check_pfn_span(). 926 * 927 * 928 * Return: 929 * * 0 - On success. 930 * * -EEXIST - Section has been present. 931 * * -ENOMEM - Out of memory. 932 */ 933int __meminit sparse_add_section(int nid, unsigned long start_pfn, 934 unsigned long nr_pages, struct vmem_altmap *altmap, 935 struct dev_pagemap *pgmap) 936{ 937 unsigned long section_nr = pfn_to_section_nr(start_pfn); 938 struct mem_section *ms; 939 struct page *memmap; 940 int ret; 941 942 ret = sparse_index_init(section_nr, nid); 943 if (ret < 0) 944 return ret; 945 946 memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap); 947 if (IS_ERR(memmap)) 948 return PTR_ERR(memmap); 949 950 /* 951 * Poison uninitialized struct pages in order to catch invalid flags 952 * combinations. 953 */ 954 page_init_poison(memmap, sizeof(struct page) * nr_pages); 955 956 ms = __nr_to_section(section_nr); 957 set_section_nid(section_nr, nid); 958 __section_mark_present(ms, section_nr); 959 960 /* Align memmap to section boundary in the subsection case */ 961 if (section_nr_to_pfn(section_nr) != start_pfn) 962 memmap = pfn_to_page(section_nr_to_pfn(section_nr)); 963 sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0); 964 965 return 0; 966} 967 968void sparse_remove_section(unsigned long pfn, unsigned long nr_pages, 969 struct vmem_altmap *altmap) 970{ 971 struct mem_section *ms = __pfn_to_section(pfn); 972 973 if (WARN_ON_ONCE(!valid_section(ms))) 974 return; 975 976 section_deactivate(pfn, nr_pages, altmap); 977} 978#endif /* CONFIG_MEMORY_HOTPLUG */
~/
Read-only file preview.