FILE
mm_init.c
File preview with syntax highlighting.
linux-6.19.14/mm/mm_init.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * mm_init.c - Memory initialisation verification and debugging 4 * 5 * Copyright 2008 IBM Corporation, 2008 6 * Author Mel Gorman <mel@csn.ul.ie> 7 * 8 */ 9#include <linux/kernel.h> 10#include <linux/init.h> 11#include <linux/kobject.h> 12#include <linux/export.h> 13#include <linux/memory.h> 14#include <linux/notifier.h> 15#include <linux/sched.h> 16#include <linux/mman.h> 17#include <linux/memblock.h> 18#include <linux/page-isolation.h> 19#include <linux/padata.h> 20#include <linux/nmi.h> 21#include <linux/buffer_head.h> 22#include <linux/kmemleak.h> 23#include <linux/kfence.h> 24#include <linux/page_ext.h> 25#include <linux/pti.h> 26#include <linux/pgtable.h> 27#include <linux/stackdepot.h> 28#include <linux/swap.h> 29#include <linux/cma.h> 30#include <linux/crash_dump.h> 31#include <linux/execmem.h> 32#include <linux/vmstat.h> 33#include <linux/kexec_handover.h> 34#include <linux/hugetlb.h> 35#include "internal.h" 36#include "slab.h" 37#include "shuffle.h" 38 39#include <asm/setup.h> 40 41#ifndef CONFIG_NUMA 42unsigned long max_mapnr; 43EXPORT_SYMBOL(max_mapnr); 44 45struct page *mem_map; 46EXPORT_SYMBOL(mem_map); 47#endif 48 49/* 50 * high_memory defines the upper bound on direct map memory, then end 51 * of ZONE_NORMAL. 52 */ 53void *high_memory; 54EXPORT_SYMBOL(high_memory); 55 56#ifdef CONFIG_DEBUG_MEMORY_INIT 57int __meminitdata mminit_loglevel; 58 59/* The zonelists are simply reported, validation is manual. */ 60void __init mminit_verify_zonelist(void) 61{ 62 int nid; 63 64 if (mminit_loglevel < MMINIT_VERIFY) 65 return; 66 67 for_each_online_node(nid) { 68 pg_data_t *pgdat = NODE_DATA(nid); 69 struct zone *zone; 70 struct zoneref *z; 71 struct zonelist *zonelist; 72 int i, listid, zoneid; 73 74 for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { 75 76 /* Identify the zone and nodelist */ 77 zoneid = i % MAX_NR_ZONES; 78 listid = i / MAX_NR_ZONES; 79 zonelist = &pgdat->node_zonelists[listid]; 80 zone = &pgdat->node_zones[zoneid]; 81 if (!populated_zone(zone)) 82 continue; 83 84 /* Print information about the zonelist */ 85 printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ", 86 listid > 0 ? "thisnode" : "general", nid, 87 zone->name); 88 89 /* Iterate the zonelist */ 90 for_each_zone_zonelist(zone, z, zonelist, zoneid) 91 pr_cont("%d:%s ", zone_to_nid(zone), zone->name); 92 pr_cont("\n"); 93 } 94 } 95} 96 97void __init mminit_verify_pageflags_layout(void) 98{ 99 int shift, width; 100 unsigned long or_mask, add_mask; 101 102 shift = BITS_PER_LONG; 103 width = shift - NR_NON_PAGEFLAG_BITS; 104 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", 105 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", 106 SECTIONS_WIDTH, 107 NODES_WIDTH, 108 ZONES_WIDTH, 109 LAST_CPUPID_WIDTH, 110 KASAN_TAG_WIDTH, 111 LRU_GEN_WIDTH, 112 LRU_REFS_WIDTH, 113 NR_PAGEFLAGS); 114 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", 115 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", 116 SECTIONS_SHIFT, 117 NODES_SHIFT, 118 ZONES_SHIFT, 119 LAST_CPUPID_SHIFT, 120 KASAN_TAG_WIDTH); 121 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts", 122 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n", 123 (unsigned long)SECTIONS_PGSHIFT, 124 (unsigned long)NODES_PGSHIFT, 125 (unsigned long)ZONES_PGSHIFT, 126 (unsigned long)LAST_CPUPID_PGSHIFT, 127 (unsigned long)KASAN_TAG_PGSHIFT); 128 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid", 129 "Node/Zone ID: %lu -> %lu\n", 130 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT), 131 (unsigned long)ZONEID_PGOFF); 132 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage", 133 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n", 134 shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0); 135#ifdef NODE_NOT_IN_PAGE_FLAGS 136 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 137 "Node not in page flags"); 138#endif 139#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS 140 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", 141 "Last cpupid not in page flags"); 142#endif 143 144 if (SECTIONS_WIDTH) { 145 shift -= SECTIONS_WIDTH; 146 BUG_ON(shift != SECTIONS_PGSHIFT); 147 } 148 if (NODES_WIDTH) { 149 shift -= NODES_WIDTH; 150 BUG_ON(shift != NODES_PGSHIFT); 151 } 152 if (ZONES_WIDTH) { 153 shift -= ZONES_WIDTH; 154 BUG_ON(shift != ZONES_PGSHIFT); 155 } 156 157 /* Check for bitmask overlaps */ 158 or_mask = (ZONES_MASK << ZONES_PGSHIFT) | 159 (NODES_MASK << NODES_PGSHIFT) | 160 (SECTIONS_MASK << SECTIONS_PGSHIFT); 161 add_mask = (ZONES_MASK << ZONES_PGSHIFT) + 162 (NODES_MASK << NODES_PGSHIFT) + 163 (SECTIONS_MASK << SECTIONS_PGSHIFT); 164 BUG_ON(or_mask != add_mask); 165} 166 167static __init int set_mminit_loglevel(char *str) 168{ 169 get_option(&str, &mminit_loglevel); 170 return 0; 171} 172early_param("mminit_loglevel", set_mminit_loglevel); 173#endif /* CONFIG_DEBUG_MEMORY_INIT */ 174 175struct kobject *mm_kobj; 176 177#ifdef CONFIG_SMP 178s32 vm_committed_as_batch = 32; 179 180void mm_compute_batch(int overcommit_policy) 181{ 182 u64 memsized_batch; 183 s32 nr = num_present_cpus(); 184 s32 batch = max_t(s32, nr*2, 32); 185 unsigned long ram_pages = totalram_pages(); 186 187 /* 188 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of 189 * (total memory/#cpus), and lift it to 25% for other policies 190 * to easy the possible lock contention for percpu_counter 191 * vm_committed_as, while the max limit is INT_MAX 192 */ 193 if (overcommit_policy == OVERCOMMIT_NEVER) 194 memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX); 195 else 196 memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); 197 198 vm_committed_as_batch = max_t(s32, memsized_batch, batch); 199} 200 201static int __meminit mm_compute_batch_notifier(struct notifier_block *self, 202 unsigned long action, void *arg) 203{ 204 switch (action) { 205 case MEM_ONLINE: 206 case MEM_OFFLINE: 207 mm_compute_batch(sysctl_overcommit_memory); 208 break; 209 default: 210 break; 211 } 212 return NOTIFY_OK; 213} 214 215static int __init mm_compute_batch_init(void) 216{ 217 mm_compute_batch(sysctl_overcommit_memory); 218 hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI); 219 return 0; 220} 221 222__initcall(mm_compute_batch_init); 223 224#endif 225 226static int __init mm_sysfs_init(void) 227{ 228 mm_kobj = kobject_create_and_add("mm", kernel_kobj); 229 if (!mm_kobj) 230 return -ENOMEM; 231 232 return 0; 233} 234postcore_initcall(mm_sysfs_init); 235 236static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; 237static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; 238static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; 239 240static unsigned long required_kernelcore __initdata; 241static unsigned long required_kernelcore_percent __initdata; 242static unsigned long required_movablecore __initdata; 243static unsigned long required_movablecore_percent __initdata; 244 245static unsigned long nr_kernel_pages __initdata; 246static unsigned long nr_all_pages __initdata; 247 248static bool deferred_struct_pages __meminitdata; 249 250static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); 251 252static int __init cmdline_parse_core(char *p, unsigned long *core, 253 unsigned long *percent) 254{ 255 unsigned long long coremem; 256 char *endptr; 257 258 if (!p) 259 return -EINVAL; 260 261 /* Value may be a percentage of total memory, otherwise bytes */ 262 coremem = simple_strtoull(p, &endptr, 0); 263 if (*endptr == '%') { 264 /* Paranoid check for percent values greater than 100 */ 265 WARN_ON(coremem > 100); 266 267 *percent = coremem; 268 } else { 269 coremem = memparse(p, &p); 270 /* Paranoid check that UL is enough for the coremem value */ 271 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); 272 273 *core = coremem >> PAGE_SHIFT; 274 *percent = 0UL; 275 } 276 return 0; 277} 278 279bool mirrored_kernelcore __initdata_memblock; 280 281/* 282 * kernelcore=size sets the amount of memory for use for allocations that 283 * cannot be reclaimed or migrated. 284 */ 285static int __init cmdline_parse_kernelcore(char *p) 286{ 287 /* parse kernelcore=mirror */ 288 if (parse_option_str(p, "mirror")) { 289 mirrored_kernelcore = true; 290 return 0; 291 } 292 293 return cmdline_parse_core(p, &required_kernelcore, 294 &required_kernelcore_percent); 295} 296early_param("kernelcore", cmdline_parse_kernelcore); 297 298/* 299 * movablecore=size sets the amount of memory for use for allocations that 300 * can be reclaimed or migrated. 301 */ 302static int __init cmdline_parse_movablecore(char *p) 303{ 304 return cmdline_parse_core(p, &required_movablecore, 305 &required_movablecore_percent); 306} 307early_param("movablecore", cmdline_parse_movablecore); 308 309/* 310 * early_calculate_totalpages() 311 * Sum pages in active regions for movable zone. 312 * Populate N_MEMORY for calculating usable_nodes. 313 */ 314static unsigned long __init early_calculate_totalpages(void) 315{ 316 unsigned long totalpages = 0; 317 unsigned long start_pfn, end_pfn; 318 int i, nid; 319 320 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 321 unsigned long pages = end_pfn - start_pfn; 322 323 totalpages += pages; 324 if (pages) 325 node_set_state(nid, N_MEMORY); 326 } 327 return totalpages; 328} 329 330/* 331 * This finds a zone that can be used for ZONE_MOVABLE pages. The 332 * assumption is made that zones within a node are ordered in monotonic 333 * increasing memory addresses so that the "highest" populated zone is used 334 */ 335static void __init find_usable_zone_for_movable(void) 336{ 337 int zone_index; 338 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { 339 if (zone_index == ZONE_MOVABLE) 340 continue; 341 342 if (arch_zone_highest_possible_pfn[zone_index] > 343 arch_zone_lowest_possible_pfn[zone_index]) 344 break; 345 } 346 347 VM_BUG_ON(zone_index == -1); 348 movable_zone = zone_index; 349} 350 351/* 352 * Find the PFN the Movable zone begins in each node. Kernel memory 353 * is spread evenly between nodes as long as the nodes have enough 354 * memory. When they don't, some nodes will have more kernelcore than 355 * others 356 */ 357static void __init find_zone_movable_pfns_for_nodes(void) 358{ 359 int i, nid; 360 unsigned long usable_startpfn; 361 unsigned long kernelcore_node, kernelcore_remaining; 362 /* save the state before borrow the nodemask */ 363 nodemask_t saved_node_state = node_states[N_MEMORY]; 364 unsigned long totalpages = early_calculate_totalpages(); 365 int usable_nodes = nodes_weight(node_states[N_MEMORY]); 366 struct memblock_region *r; 367 368 /* Need to find movable_zone earlier when movable_node is specified. */ 369 find_usable_zone_for_movable(); 370 371 /* 372 * If movable_node is specified, ignore kernelcore and movablecore 373 * options. 374 */ 375 if (movable_node_is_enabled()) { 376 for_each_mem_region(r) { 377 if (!memblock_is_hotpluggable(r)) 378 continue; 379 380 nid = memblock_get_region_node(r); 381 382 usable_startpfn = memblock_region_memory_base_pfn(r); 383 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 384 min(usable_startpfn, zone_movable_pfn[nid]) : 385 usable_startpfn; 386 } 387 388 goto out2; 389 } 390 391 /* 392 * If kernelcore=mirror is specified, ignore movablecore option 393 */ 394 if (mirrored_kernelcore) { 395 bool mem_below_4gb_not_mirrored = false; 396 397 if (!memblock_has_mirror()) { 398 pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n"); 399 goto out; 400 } 401 402 if (is_kdump_kernel()) { 403 pr_warn("The system is under kdump, ignore kernelcore=mirror.\n"); 404 goto out; 405 } 406 407 for_each_mem_region(r) { 408 if (memblock_is_mirror(r)) 409 continue; 410 411 nid = memblock_get_region_node(r); 412 413 usable_startpfn = memblock_region_memory_base_pfn(r); 414 415 if (usable_startpfn < PHYS_PFN(SZ_4G)) { 416 mem_below_4gb_not_mirrored = true; 417 continue; 418 } 419 420 zone_movable_pfn[nid] = zone_movable_pfn[nid] ? 421 min(usable_startpfn, zone_movable_pfn[nid]) : 422 usable_startpfn; 423 } 424 425 if (mem_below_4gb_not_mirrored) 426 pr_warn("This configuration results in unmirrored kernel memory.\n"); 427 428 goto out2; 429 } 430 431 /* 432 * If kernelcore=nn% or movablecore=nn% was specified, calculate the 433 * amount of necessary memory. 434 */ 435 if (required_kernelcore_percent) 436 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / 437 10000UL; 438 if (required_movablecore_percent) 439 required_movablecore = (totalpages * 100 * required_movablecore_percent) / 440 10000UL; 441 442 /* 443 * If movablecore= was specified, calculate what size of 444 * kernelcore that corresponds so that memory usable for 445 * any allocation type is evenly spread. If both kernelcore 446 * and movablecore are specified, then the value of kernelcore 447 * will be used for required_kernelcore if it's greater than 448 * what movablecore would have allowed. 449 */ 450 if (required_movablecore) { 451 unsigned long corepages; 452 453 /* 454 * Round-up so that ZONE_MOVABLE is at least as large as what 455 * was requested by the user 456 */ 457 required_movablecore = 458 round_up(required_movablecore, MAX_ORDER_NR_PAGES); 459 required_movablecore = min(totalpages, required_movablecore); 460 corepages = totalpages - required_movablecore; 461 462 required_kernelcore = max(required_kernelcore, corepages); 463 } 464 465 /* 466 * If kernelcore was not specified or kernelcore size is larger 467 * than totalpages, there is no ZONE_MOVABLE. 468 */ 469 if (!required_kernelcore || required_kernelcore >= totalpages) 470 goto out; 471 472 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ 473 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; 474 475restart: 476 /* Spread kernelcore memory as evenly as possible throughout nodes */ 477 kernelcore_node = required_kernelcore / usable_nodes; 478 for_each_node_state(nid, N_MEMORY) { 479 unsigned long start_pfn, end_pfn; 480 481 /* 482 * Recalculate kernelcore_node if the division per node 483 * now exceeds what is necessary to satisfy the requested 484 * amount of memory for the kernel 485 */ 486 if (required_kernelcore < kernelcore_node) 487 kernelcore_node = required_kernelcore / usable_nodes; 488 489 /* 490 * As the map is walked, we track how much memory is usable 491 * by the kernel using kernelcore_remaining. When it is 492 * 0, the rest of the node is usable by ZONE_MOVABLE 493 */ 494 kernelcore_remaining = kernelcore_node; 495 496 /* Go through each range of PFNs within this node */ 497 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 498 unsigned long size_pages; 499 500 start_pfn = max(start_pfn, zone_movable_pfn[nid]); 501 if (start_pfn >= end_pfn) 502 continue; 503 504 /* Account for what is only usable for kernelcore */ 505 if (start_pfn < usable_startpfn) { 506 unsigned long kernel_pages; 507 kernel_pages = min(end_pfn, usable_startpfn) 508 - start_pfn; 509 510 kernelcore_remaining -= min(kernel_pages, 511 kernelcore_remaining); 512 required_kernelcore -= min(kernel_pages, 513 required_kernelcore); 514 515 /* Continue if range is now fully accounted */ 516 if (end_pfn <= usable_startpfn) { 517 518 /* 519 * Push zone_movable_pfn to the end so 520 * that if we have to rebalance 521 * kernelcore across nodes, we will 522 * not double account here 523 */ 524 zone_movable_pfn[nid] = end_pfn; 525 continue; 526 } 527 start_pfn = usable_startpfn; 528 } 529 530 /* 531 * The usable PFN range for ZONE_MOVABLE is from 532 * start_pfn->end_pfn. Calculate size_pages as the 533 * number of pages used as kernelcore 534 */ 535 size_pages = end_pfn - start_pfn; 536 if (size_pages > kernelcore_remaining) 537 size_pages = kernelcore_remaining; 538 zone_movable_pfn[nid] = start_pfn + size_pages; 539 540 /* 541 * Some kernelcore has been met, update counts and 542 * break if the kernelcore for this node has been 543 * satisfied 544 */ 545 required_kernelcore -= min(required_kernelcore, 546 size_pages); 547 kernelcore_remaining -= size_pages; 548 if (!kernelcore_remaining) 549 break; 550 } 551 } 552 553 /* 554 * If there is still required_kernelcore, we do another pass with one 555 * less node in the count. This will push zone_movable_pfn[nid] further 556 * along on the nodes that still have memory until kernelcore is 557 * satisfied 558 */ 559 usable_nodes--; 560 if (usable_nodes && required_kernelcore > usable_nodes) 561 goto restart; 562 563out2: 564 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ 565 for_each_node_state(nid, N_MEMORY) { 566 unsigned long start_pfn, end_pfn; 567 568 zone_movable_pfn[nid] = 569 round_up(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); 570 571 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 572 if (zone_movable_pfn[nid] >= end_pfn) 573 zone_movable_pfn[nid] = 0; 574 } 575 576out: 577 /* restore the node_state */ 578 node_states[N_MEMORY] = saved_node_state; 579} 580 581void __meminit __init_single_page(struct page *page, unsigned long pfn, 582 unsigned long zone, int nid) 583{ 584 mm_zero_struct_page(page); 585 set_page_links(page, zone, nid, pfn); 586 init_page_count(page); 587 atomic_set(&page->_mapcount, -1); 588 page_cpupid_reset_last(page); 589 page_kasan_tag_reset(page); 590 591 INIT_LIST_HEAD(&page->lru); 592#ifdef WANT_PAGE_VIRTUAL 593 /* The shift won't overflow because ZONE_NORMAL is below 4G. */ 594 if (!is_highmem_idx(zone)) 595 set_page_address(page, __va(pfn << PAGE_SHIFT)); 596#endif 597} 598 599#ifdef CONFIG_NUMA 600/* 601 * During memory init memblocks map pfns to nids. The search is expensive and 602 * this caches recent lookups. The implementation of __early_pfn_to_nid 603 * treats start/end as pfns. 604 */ 605struct mminit_pfnnid_cache { 606 unsigned long last_start; 607 unsigned long last_end; 608 int last_nid; 609}; 610 611static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; 612 613/* 614 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. 615 */ 616static int __meminit __early_pfn_to_nid(unsigned long pfn, 617 struct mminit_pfnnid_cache *state) 618{ 619 unsigned long start_pfn, end_pfn; 620 int nid; 621 622 if (state->last_start <= pfn && pfn < state->last_end) 623 return state->last_nid; 624 625 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); 626 if (nid != NUMA_NO_NODE) { 627 state->last_start = start_pfn; 628 state->last_end = end_pfn; 629 state->last_nid = nid; 630 } 631 632 return nid; 633} 634 635int __meminit early_pfn_to_nid(unsigned long pfn) 636{ 637 static DEFINE_SPINLOCK(early_pfn_lock); 638 int nid; 639 640 spin_lock(&early_pfn_lock); 641 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); 642 if (nid < 0) 643 nid = first_online_node; 644 spin_unlock(&early_pfn_lock); 645 646 return nid; 647} 648 649int hashdist = HASHDIST_DEFAULT; 650 651static int __init set_hashdist(char *str) 652{ 653 if (!str) 654 return 0; 655 hashdist = simple_strtoul(str, &str, 0); 656 return 1; 657} 658__setup("hashdist=", set_hashdist); 659 660static inline void fixup_hashdist(void) 661{ 662 if (num_node_state(N_MEMORY) == 1) 663 hashdist = 0; 664} 665#else 666static inline void fixup_hashdist(void) {} 667#endif /* CONFIG_NUMA */ 668 669/* 670 * Initialize a reserved page unconditionally, finding its zone first. 671 */ 672void __meminit __init_page_from_nid(unsigned long pfn, int nid) 673{ 674 pg_data_t *pgdat; 675 int zid; 676 677 pgdat = NODE_DATA(nid); 678 679 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 680 struct zone *zone = &pgdat->node_zones[zid]; 681 682 if (zone_spans_pfn(zone, pfn)) 683 break; 684 } 685 __init_single_page(pfn_to_page(pfn), pfn, zid, nid); 686 687 if (pageblock_aligned(pfn)) 688 init_pageblock_migratetype(pfn_to_page(pfn), MIGRATE_MOVABLE, 689 false); 690} 691 692#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 693static inline void pgdat_set_deferred_range(pg_data_t *pgdat) 694{ 695 pgdat->first_deferred_pfn = ULONG_MAX; 696} 697 698/* Returns true if the struct page for the pfn is initialised */ 699static inline bool __meminit early_page_initialised(unsigned long pfn, int nid) 700{ 701 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) 702 return false; 703 704 return true; 705} 706 707/* 708 * Returns true when the remaining initialisation should be deferred until 709 * later in the boot cycle when it can be parallelised. 710 */ 711static bool __meminit 712defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 713{ 714 static unsigned long prev_end_pfn, nr_initialised; 715 716 if (early_page_ext_enabled()) 717 return false; 718 719 /* Always populate low zones for address-constrained allocations */ 720 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) 721 return false; 722 723 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) 724 return true; 725 726 /* 727 * prev_end_pfn static that contains the end of previous zone 728 * No need to protect because called very early in boot before smp_init. 729 */ 730 if (prev_end_pfn != end_pfn) { 731 prev_end_pfn = end_pfn; 732 nr_initialised = 0; 733 } 734 735 /* 736 * We start only with one section of pages, more pages are added as 737 * needed until the rest of deferred pages are initialized. 738 */ 739 nr_initialised++; 740 if ((nr_initialised > PAGES_PER_SECTION) && 741 (pfn & (PAGES_PER_SECTION - 1)) == 0) { 742 NODE_DATA(nid)->first_deferred_pfn = pfn; 743 return true; 744 } 745 return false; 746} 747 748static void __meminit __init_deferred_page(unsigned long pfn, int nid) 749{ 750 if (early_page_initialised(pfn, nid)) 751 return; 752 753 __init_page_from_nid(pfn, nid); 754} 755#else 756static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} 757 758static inline bool early_page_initialised(unsigned long pfn, int nid) 759{ 760 return true; 761} 762 763static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) 764{ 765 return false; 766} 767 768static inline void __init_deferred_page(unsigned long pfn, int nid) 769{ 770} 771#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 772 773void __meminit init_deferred_page(unsigned long pfn, int nid) 774{ 775 __init_deferred_page(pfn, nid); 776} 777 778/* 779 * Initialised pages do not have PageReserved set. This function is 780 * called for each range allocated by the bootmem allocator and 781 * marks the pages PageReserved. The remaining valid pages are later 782 * sent to the buddy page allocator. 783 */ 784void __meminit reserve_bootmem_region(phys_addr_t start, 785 phys_addr_t end, int nid) 786{ 787 unsigned long pfn; 788 789 for_each_valid_pfn(pfn, PFN_DOWN(start), PFN_UP(end)) { 790 struct page *page = pfn_to_page(pfn); 791 792 __init_deferred_page(pfn, nid); 793 794 /* 795 * no need for atomic set_bit because the struct 796 * page is not visible yet so nobody should 797 * access it yet. 798 */ 799 __SetPageReserved(page); 800 } 801} 802 803/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ 804static bool __meminit 805overlap_memmap_init(unsigned long zone, unsigned long *pfn) 806{ 807 static struct memblock_region *r; 808 809 if (mirrored_kernelcore && zone == ZONE_MOVABLE) { 810 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { 811 for_each_mem_region(r) { 812 if (*pfn < memblock_region_memory_end_pfn(r)) 813 break; 814 } 815 } 816 if (*pfn >= memblock_region_memory_base_pfn(r) && 817 memblock_is_mirror(r)) { 818 *pfn = memblock_region_memory_end_pfn(r); 819 return true; 820 } 821 } 822 return false; 823} 824 825/* 826 * Only struct pages that correspond to ranges defined by memblock.memory 827 * are zeroed and initialized by going through __init_single_page() during 828 * memmap_init_zone_range(). 829 * 830 * But, there could be struct pages that correspond to holes in 831 * memblock.memory. This can happen because of the following reasons: 832 * - physical memory bank size is not necessarily the exact multiple of the 833 * arbitrary section size 834 * - early reserved memory may not be listed in memblock.memory 835 * - non-memory regions covered by the contiguous flatmem mapping 836 * - memory layouts defined with memmap= kernel parameter may not align 837 * nicely with memmap sections 838 * 839 * Explicitly initialize those struct pages so that: 840 * - PG_Reserved is set 841 * - zone and node links point to zone and node that span the page if the 842 * hole is in the middle of a zone 843 * - zone and node links point to adjacent zone/node if the hole falls on 844 * the zone boundary; the pages in such holes will be prepended to the 845 * zone/node above the hole except for the trailing pages in the last 846 * section that will be appended to the zone/node below. 847 */ 848static void __init init_unavailable_range(unsigned long spfn, 849 unsigned long epfn, 850 int zone, int node) 851{ 852 unsigned long pfn; 853 u64 pgcnt = 0; 854 855 for_each_valid_pfn(pfn, spfn, epfn) { 856 __init_single_page(pfn_to_page(pfn), pfn, zone, node); 857 __SetPageReserved(pfn_to_page(pfn)); 858 pgcnt++; 859 } 860 861 if (pgcnt) 862 pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n", 863 node, zone_names[zone], pgcnt); 864} 865 866/* 867 * Initially all pages are reserved - free ones are freed 868 * up by memblock_free_all() once the early boot process is 869 * done. Non-atomic initialization, single-pass. 870 * 871 * All aligned pageblocks are initialized to the specified migratetype 872 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related 873 * zone stats (e.g., nr_isolate_pageblock) are touched. 874 */ 875void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, 876 unsigned long start_pfn, unsigned long zone_end_pfn, 877 enum meminit_context context, 878 struct vmem_altmap *altmap, int migratetype, 879 bool isolate_pageblock) 880{ 881 unsigned long pfn, end_pfn = start_pfn + size; 882 struct page *page; 883 884 if (highest_memmap_pfn < end_pfn - 1) 885 highest_memmap_pfn = end_pfn - 1; 886 887#ifdef CONFIG_ZONE_DEVICE 888 /* 889 * Honor reservation requested by the driver for this ZONE_DEVICE 890 * memory. We limit the total number of pages to initialize to just 891 * those that might contain the memory mapping. We will defer the 892 * ZONE_DEVICE page initialization until after we have released 893 * the hotplug lock. 894 */ 895 if (zone == ZONE_DEVICE) { 896 if (!altmap) 897 return; 898 899 if (start_pfn == altmap->base_pfn) 900 start_pfn += altmap->reserve; 901 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 902 } 903#endif 904 905 for (pfn = start_pfn; pfn < end_pfn; ) { 906 /* 907 * There can be holes in boot-time mem_map[]s handed to this 908 * function. They do not exist on hotplugged memory. 909 */ 910 if (context == MEMINIT_EARLY) { 911 if (overlap_memmap_init(zone, &pfn)) 912 continue; 913 if (defer_init(nid, pfn, zone_end_pfn)) { 914 deferred_struct_pages = true; 915 break; 916 } 917 } 918 919 page = pfn_to_page(pfn); 920 __init_single_page(page, pfn, zone, nid); 921 if (context == MEMINIT_HOTPLUG) { 922#ifdef CONFIG_ZONE_DEVICE 923 if (zone == ZONE_DEVICE) 924 __SetPageReserved(page); 925 else 926#endif 927 __SetPageOffline(page); 928 } 929 930 /* 931 * Usually, we want to mark the pageblock MIGRATE_MOVABLE, 932 * such that unmovable allocations won't be scattered all 933 * over the place during system boot. 934 */ 935 if (pageblock_aligned(pfn)) { 936 init_pageblock_migratetype(page, migratetype, 937 isolate_pageblock); 938 cond_resched(); 939 } 940 pfn++; 941 } 942} 943 944static void __init memmap_init_zone_range(struct zone *zone, 945 unsigned long start_pfn, 946 unsigned long end_pfn, 947 unsigned long *hole_pfn) 948{ 949 unsigned long zone_start_pfn = zone->zone_start_pfn; 950 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; 951 int nid = zone_to_nid(zone), zone_id = zone_idx(zone); 952 953 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); 954 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); 955 956 if (start_pfn >= end_pfn) 957 return; 958 959 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, 960 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE, 961 false); 962 963 if (*hole_pfn < start_pfn) 964 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); 965 966 *hole_pfn = end_pfn; 967} 968 969static void __init memmap_init(void) 970{ 971 unsigned long start_pfn, end_pfn; 972 unsigned long hole_pfn = 0; 973 int i, j, zone_id = 0, nid; 974 975 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 976 struct pglist_data *node = NODE_DATA(nid); 977 978 for (j = 0; j < MAX_NR_ZONES; j++) { 979 struct zone *zone = node->node_zones + j; 980 981 if (!populated_zone(zone)) 982 continue; 983 984 memmap_init_zone_range(zone, start_pfn, end_pfn, 985 &hole_pfn); 986 zone_id = j; 987 } 988 } 989 990 /* 991 * Initialize the memory map for hole in the range [memory_end, 992 * section_end] for SPARSEMEM and in the range [memory_end, memmap_end] 993 * for FLATMEM. 994 * Append the pages in this hole to the highest zone in the last 995 * node. 996 */ 997#ifdef CONFIG_SPARSEMEM 998 end_pfn = round_up(end_pfn, PAGES_PER_SECTION); 999#else 1000 end_pfn = round_up(end_pfn, MAX_ORDER_NR_PAGES); 1001#endif 1002 if (hole_pfn < end_pfn) 1003 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); 1004} 1005 1006#ifdef CONFIG_ZONE_DEVICE 1007static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, 1008 unsigned long zone_idx, int nid, 1009 struct dev_pagemap *pgmap) 1010{ 1011 1012 __init_single_page(page, pfn, zone_idx, nid); 1013 1014 /* 1015 * Mark page reserved as it will need to wait for onlining 1016 * phase for it to be fully associated with a zone. 1017 * 1018 * We can use the non-atomic __set_bit operation for setting 1019 * the flag as we are still initializing the pages. 1020 */ 1021 __SetPageReserved(page); 1022 1023 /* 1024 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer 1025 * and zone_device_data. It is a bug if a ZONE_DEVICE page is 1026 * ever freed or placed on a driver-private list. 1027 */ 1028 page_folio(page)->pgmap = pgmap; 1029 page->zone_device_data = NULL; 1030 1031 /* 1032 * Mark the block movable so that blocks are reserved for 1033 * movable at startup. This will force kernel allocations 1034 * to reserve their blocks rather than leaking throughout 1035 * the address space during boot when many long-lived 1036 * kernel allocations are made. 1037 * 1038 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap 1039 * because this is done early in section_activate() 1040 */ 1041 if (pageblock_aligned(pfn)) { 1042 init_pageblock_migratetype(page, MIGRATE_MOVABLE, false); 1043 cond_resched(); 1044 } 1045 1046 /* 1047 * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released 1048 * directly to the driver page allocator which will set the page count 1049 * to 1 when allocating the page. 1050 * 1051 * MEMORY_TYPE_GENERIC and MEMORY_TYPE_FS_DAX pages automatically have 1052 * their refcount reset to one whenever they are freed (ie. after 1053 * their refcount drops to 0). 1054 */ 1055 switch (pgmap->type) { 1056 case MEMORY_DEVICE_FS_DAX: 1057 case MEMORY_DEVICE_PRIVATE: 1058 case MEMORY_DEVICE_COHERENT: 1059 case MEMORY_DEVICE_PCI_P2PDMA: 1060 set_page_count(page, 0); 1061 break; 1062 1063 case MEMORY_DEVICE_GENERIC: 1064 break; 1065 } 1066} 1067 1068/* 1069 * With compound page geometry and when struct pages are stored in ram most 1070 * tail pages are reused. Consequently, the amount of unique struct pages to 1071 * initialize is a lot smaller that the total amount of struct pages being 1072 * mapped. This is a paired / mild layering violation with explicit knowledge 1073 * of how the sparse_vmemmap internals handle compound pages in the lack 1074 * of an altmap. See vmemmap_populate_compound_pages(). 1075 */ 1076static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, 1077 struct dev_pagemap *pgmap) 1078{ 1079 if (!vmemmap_can_optimize(altmap, pgmap)) 1080 return pgmap_vmemmap_nr(pgmap); 1081 1082 return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); 1083} 1084 1085static void __ref memmap_init_compound(struct page *head, 1086 unsigned long head_pfn, 1087 unsigned long zone_idx, int nid, 1088 struct dev_pagemap *pgmap, 1089 unsigned long nr_pages) 1090{ 1091 unsigned long pfn, end_pfn = head_pfn + nr_pages; 1092 unsigned int order = pgmap->vmemmap_shift; 1093 1094 /* 1095 * We have to initialize the pages, including setting up page links. 1096 * prep_compound_page() does not take care of that, so instead we 1097 * open-code prep_compound_page() so we can take care of initializing 1098 * the pages in the same go. 1099 */ 1100 __SetPageHead(head); 1101 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { 1102 struct page *page = pfn_to_page(pfn); 1103 1104 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1105 prep_compound_tail(head, pfn - head_pfn); 1106 set_page_count(page, 0); 1107 } 1108 prep_compound_head(head, order); 1109} 1110 1111void __ref memmap_init_zone_device(struct zone *zone, 1112 unsigned long start_pfn, 1113 unsigned long nr_pages, 1114 struct dev_pagemap *pgmap) 1115{ 1116 unsigned long pfn, end_pfn = start_pfn + nr_pages; 1117 struct pglist_data *pgdat = zone->zone_pgdat; 1118 struct vmem_altmap *altmap = pgmap_altmap(pgmap); 1119 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); 1120 unsigned long zone_idx = zone_idx(zone); 1121 unsigned long start = jiffies; 1122 int nid = pgdat->node_id; 1123 1124 if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) 1125 return; 1126 1127 /* 1128 * The call to memmap_init should have already taken care 1129 * of the pages reserved for the memmap, so we can just jump to 1130 * the end of that region and start processing the device pages. 1131 */ 1132 if (altmap) { 1133 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); 1134 nr_pages = end_pfn - start_pfn; 1135 } 1136 1137 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { 1138 struct page *page = pfn_to_page(pfn); 1139 1140 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); 1141 1142 if (pfns_per_compound == 1) 1143 continue; 1144 1145 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, 1146 compound_nr_pages(altmap, pgmap)); 1147 } 1148 1149 pr_debug("%s initialised %lu pages in %ums\n", __func__, 1150 nr_pages, jiffies_to_msecs(jiffies - start)); 1151} 1152#endif 1153 1154/* 1155 * The zone ranges provided by the architecture do not include ZONE_MOVABLE 1156 * because it is sized independent of architecture. Unlike the other zones, 1157 * the starting point for ZONE_MOVABLE is not fixed. It may be different 1158 * in each node depending on the size of each node and how evenly kernelcore 1159 * is distributed. This helper function adjusts the zone ranges 1160 * provided by the architecture for a given node by using the end of the 1161 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that 1162 * zones within a node are in order of monotonic increases memory addresses 1163 */ 1164static void __init adjust_zone_range_for_zone_movable(int nid, 1165 unsigned long zone_type, 1166 unsigned long node_end_pfn, 1167 unsigned long *zone_start_pfn, 1168 unsigned long *zone_end_pfn) 1169{ 1170 /* Only adjust if ZONE_MOVABLE is on this node */ 1171 if (zone_movable_pfn[nid]) { 1172 /* Size ZONE_MOVABLE */ 1173 if (zone_type == ZONE_MOVABLE) { 1174 *zone_start_pfn = zone_movable_pfn[nid]; 1175 *zone_end_pfn = min(node_end_pfn, 1176 arch_zone_highest_possible_pfn[movable_zone]); 1177 1178 /* Adjust for ZONE_MOVABLE starting within this range */ 1179 } else if (!mirrored_kernelcore && 1180 *zone_start_pfn < zone_movable_pfn[nid] && 1181 *zone_end_pfn > zone_movable_pfn[nid]) { 1182 *zone_end_pfn = zone_movable_pfn[nid]; 1183 1184 /* Check if this whole range is within ZONE_MOVABLE */ 1185 } else if (*zone_start_pfn >= zone_movable_pfn[nid]) 1186 *zone_start_pfn = *zone_end_pfn; 1187 } 1188} 1189 1190/* 1191 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, 1192 * then all holes in the requested range will be accounted for. 1193 */ 1194static unsigned long __init __absent_pages_in_range(int nid, 1195 unsigned long range_start_pfn, 1196 unsigned long range_end_pfn) 1197{ 1198 unsigned long nr_absent = range_end_pfn - range_start_pfn; 1199 unsigned long start_pfn, end_pfn; 1200 int i; 1201 1202 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { 1203 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); 1204 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); 1205 nr_absent -= end_pfn - start_pfn; 1206 } 1207 return nr_absent; 1208} 1209 1210/** 1211 * absent_pages_in_range - Return number of page frames in holes within a range 1212 * @start_pfn: The start PFN to start searching for holes 1213 * @end_pfn: The end PFN to stop searching for holes 1214 * 1215 * Return: the number of pages frames in memory holes within a range. 1216 */ 1217unsigned long __init absent_pages_in_range(unsigned long start_pfn, 1218 unsigned long end_pfn) 1219{ 1220 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); 1221} 1222 1223/* Return the number of page frames in holes in a zone on a node */ 1224static unsigned long __init zone_absent_pages_in_node(int nid, 1225 unsigned long zone_type, 1226 unsigned long zone_start_pfn, 1227 unsigned long zone_end_pfn) 1228{ 1229 unsigned long nr_absent; 1230 1231 /* zone is empty, we don't have any absent pages */ 1232 if (zone_start_pfn == zone_end_pfn) 1233 return 0; 1234 1235 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); 1236 1237 /* 1238 * ZONE_MOVABLE handling. 1239 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages 1240 * and vice versa. 1241 */ 1242 if (mirrored_kernelcore && zone_movable_pfn[nid]) { 1243 unsigned long start_pfn, end_pfn; 1244 struct memblock_region *r; 1245 1246 for_each_mem_region(r) { 1247 start_pfn = clamp(memblock_region_memory_base_pfn(r), 1248 zone_start_pfn, zone_end_pfn); 1249 end_pfn = clamp(memblock_region_memory_end_pfn(r), 1250 zone_start_pfn, zone_end_pfn); 1251 1252 if (zone_type == ZONE_MOVABLE && 1253 memblock_is_mirror(r)) 1254 nr_absent += end_pfn - start_pfn; 1255 1256 if (zone_type == ZONE_NORMAL && 1257 !memblock_is_mirror(r)) 1258 nr_absent += end_pfn - start_pfn; 1259 } 1260 } 1261 1262 return nr_absent; 1263} 1264 1265/* 1266 * Return the number of pages a zone spans in a node, including holes 1267 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() 1268 */ 1269static unsigned long __init zone_spanned_pages_in_node(int nid, 1270 unsigned long zone_type, 1271 unsigned long node_start_pfn, 1272 unsigned long node_end_pfn, 1273 unsigned long *zone_start_pfn, 1274 unsigned long *zone_end_pfn) 1275{ 1276 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; 1277 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; 1278 1279 /* Get the start and end of the zone */ 1280 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); 1281 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); 1282 adjust_zone_range_for_zone_movable(nid, zone_type, node_end_pfn, 1283 zone_start_pfn, zone_end_pfn); 1284 1285 /* Check that this node has pages within the zone's required range */ 1286 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) 1287 return 0; 1288 1289 /* Move the zone boundaries inside the node if necessary */ 1290 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); 1291 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); 1292 1293 /* Return the spanned pages */ 1294 return *zone_end_pfn - *zone_start_pfn; 1295} 1296 1297static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat) 1298{ 1299 struct zone *z; 1300 1301 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) { 1302 z->zone_start_pfn = 0; 1303 z->spanned_pages = 0; 1304 z->present_pages = 0; 1305#if defined(CONFIG_MEMORY_HOTPLUG) 1306 z->present_early_pages = 0; 1307#endif 1308 } 1309 1310 pgdat->node_spanned_pages = 0; 1311 pgdat->node_present_pages = 0; 1312 pr_debug("On node %d totalpages: 0\n", pgdat->node_id); 1313} 1314 1315static void __init calc_nr_kernel_pages(void) 1316{ 1317 unsigned long start_pfn, end_pfn; 1318 phys_addr_t start_addr, end_addr; 1319 u64 u; 1320#ifdef CONFIG_HIGHMEM 1321 unsigned long high_zone_low = arch_zone_lowest_possible_pfn[ZONE_HIGHMEM]; 1322#endif 1323 1324 for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) { 1325 start_pfn = PFN_UP(start_addr); 1326 end_pfn = PFN_DOWN(end_addr); 1327 1328 if (start_pfn < end_pfn) { 1329 nr_all_pages += end_pfn - start_pfn; 1330#ifdef CONFIG_HIGHMEM 1331 start_pfn = clamp(start_pfn, 0, high_zone_low); 1332 end_pfn = clamp(end_pfn, 0, high_zone_low); 1333#endif 1334 nr_kernel_pages += end_pfn - start_pfn; 1335 } 1336 } 1337} 1338 1339static void __init calculate_node_totalpages(struct pglist_data *pgdat, 1340 unsigned long node_start_pfn, 1341 unsigned long node_end_pfn) 1342{ 1343 unsigned long realtotalpages = 0, totalpages = 0; 1344 enum zone_type i; 1345 1346 for (i = 0; i < MAX_NR_ZONES; i++) { 1347 struct zone *zone = pgdat->node_zones + i; 1348 unsigned long zone_start_pfn, zone_end_pfn; 1349 unsigned long spanned, absent; 1350 unsigned long real_size; 1351 1352 spanned = zone_spanned_pages_in_node(pgdat->node_id, i, 1353 node_start_pfn, 1354 node_end_pfn, 1355 &zone_start_pfn, 1356 &zone_end_pfn); 1357 absent = zone_absent_pages_in_node(pgdat->node_id, i, 1358 zone_start_pfn, 1359 zone_end_pfn); 1360 1361 real_size = spanned - absent; 1362 1363 if (spanned) 1364 zone->zone_start_pfn = zone_start_pfn; 1365 else 1366 zone->zone_start_pfn = 0; 1367 zone->spanned_pages = spanned; 1368 zone->present_pages = real_size; 1369#if defined(CONFIG_MEMORY_HOTPLUG) 1370 zone->present_early_pages = real_size; 1371#endif 1372 1373 totalpages += spanned; 1374 realtotalpages += real_size; 1375 } 1376 1377 pgdat->node_spanned_pages = totalpages; 1378 pgdat->node_present_pages = realtotalpages; 1379 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); 1380} 1381 1382#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1383static void pgdat_init_split_queue(struct pglist_data *pgdat) 1384{ 1385 struct deferred_split *ds_queue = &pgdat->deferred_split_queue; 1386 1387 spin_lock_init(&ds_queue->split_queue_lock); 1388 INIT_LIST_HEAD(&ds_queue->split_queue); 1389 ds_queue->split_queue_len = 0; 1390} 1391#else 1392static void pgdat_init_split_queue(struct pglist_data *pgdat) {} 1393#endif 1394 1395#ifdef CONFIG_COMPACTION 1396static void pgdat_init_kcompactd(struct pglist_data *pgdat) 1397{ 1398 init_waitqueue_head(&pgdat->kcompactd_wait); 1399} 1400#else 1401static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} 1402#endif 1403 1404static void __meminit pgdat_init_internals(struct pglist_data *pgdat) 1405{ 1406 int i; 1407 1408 pgdat_resize_init(pgdat); 1409 pgdat_kswapd_lock_init(pgdat); 1410 1411 pgdat_init_split_queue(pgdat); 1412 pgdat_init_kcompactd(pgdat); 1413 1414 init_waitqueue_head(&pgdat->kswapd_wait); 1415 init_waitqueue_head(&pgdat->pfmemalloc_wait); 1416 1417 for (i = 0; i < NR_VMSCAN_THROTTLE; i++) 1418 init_waitqueue_head(&pgdat->reclaim_wait[i]); 1419 1420 pgdat_page_ext_init(pgdat); 1421 lruvec_init(&pgdat->__lruvec); 1422} 1423 1424static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, 1425 unsigned long remaining_pages) 1426{ 1427 atomic_long_set(&zone->managed_pages, remaining_pages); 1428 zone_set_nid(zone, nid); 1429 zone->name = zone_names[idx]; 1430 zone->zone_pgdat = NODE_DATA(nid); 1431 spin_lock_init(&zone->lock); 1432 zone_seqlock_init(zone); 1433 zone_pcp_init(zone); 1434} 1435 1436static void __meminit zone_init_free_lists(struct zone *zone) 1437{ 1438 unsigned int order, t; 1439 for_each_migratetype_order(order, t) { 1440 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); 1441 zone->free_area[order].nr_free = 0; 1442 } 1443 1444#ifdef CONFIG_UNACCEPTED_MEMORY 1445 INIT_LIST_HEAD(&zone->unaccepted_pages); 1446#endif 1447} 1448 1449void __meminit init_currently_empty_zone(struct zone *zone, 1450 unsigned long zone_start_pfn, 1451 unsigned long size) 1452{ 1453 struct pglist_data *pgdat = zone->zone_pgdat; 1454 int zone_idx = zone_idx(zone) + 1; 1455 1456 if (zone_idx > pgdat->nr_zones) 1457 pgdat->nr_zones = zone_idx; 1458 1459 zone->zone_start_pfn = zone_start_pfn; 1460 1461 mminit_dprintk(MMINIT_TRACE, "memmap_init", 1462 "Initialising map node %d zone %lu pfns %lu -> %lu\n", 1463 pgdat->node_id, 1464 (unsigned long)zone_idx(zone), 1465 zone_start_pfn, (zone_start_pfn + size)); 1466 1467 zone_init_free_lists(zone); 1468 zone->initialized = 1; 1469} 1470 1471#ifndef CONFIG_SPARSEMEM 1472/* 1473 * Calculate the size of the zone->pageblock_flags rounded to an unsigned long 1474 * Start by making sure zonesize is a multiple of pageblock_order by rounding 1475 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally 1476 * round what is now in bits to nearest long in bits, then return it in 1477 * bytes. 1478 */ 1479static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) 1480{ 1481 unsigned long usemapsize; 1482 1483 zonesize += zone_start_pfn & (pageblock_nr_pages-1); 1484 usemapsize = round_up(zonesize, pageblock_nr_pages); 1485 usemapsize = usemapsize >> pageblock_order; 1486 usemapsize *= NR_PAGEBLOCK_BITS; 1487 usemapsize = round_up(usemapsize, BITS_PER_LONG); 1488 1489 return usemapsize / BITS_PER_BYTE; 1490} 1491 1492static void __ref setup_usemap(struct zone *zone) 1493{ 1494 unsigned long usemapsize = usemap_size(zone->zone_start_pfn, 1495 zone->spanned_pages); 1496 zone->pageblock_flags = NULL; 1497 if (usemapsize) { 1498 zone->pageblock_flags = 1499 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, 1500 zone_to_nid(zone)); 1501 if (!zone->pageblock_flags) 1502 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", 1503 usemapsize, zone->name, zone_to_nid(zone)); 1504 } 1505} 1506#else 1507static inline void setup_usemap(struct zone *zone) {} 1508#endif /* CONFIG_SPARSEMEM */ 1509 1510#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 1511 1512/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ 1513void __init set_pageblock_order(void) 1514{ 1515 unsigned int order = PAGE_BLOCK_MAX_ORDER; 1516 1517 /* Check that pageblock_nr_pages has not already been setup */ 1518 if (pageblock_order) 1519 return; 1520 1521 /* Don't let pageblocks exceed the maximum allocation granularity. */ 1522 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) 1523 order = HUGETLB_PAGE_ORDER; 1524 1525 /* 1526 * Assume the largest contiguous order of interest is a huge page. 1527 * This value may be variable depending on boot parameters on powerpc. 1528 */ 1529 pageblock_order = order; 1530} 1531#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1532 1533/* 1534 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() 1535 * is unused as pageblock_order is set at compile-time. See 1536 * include/linux/pageblock-flags.h for the values of pageblock_order based on 1537 * the kernel config 1538 */ 1539void __init set_pageblock_order(void) 1540{ 1541} 1542 1543#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ 1544 1545/* 1546 * Set up the zone data structures 1547 * - init pgdat internals 1548 * - init all zones belonging to this node 1549 * 1550 * NOTE: this function is only called during memory hotplug 1551 */ 1552#ifdef CONFIG_MEMORY_HOTPLUG 1553void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) 1554{ 1555 int nid = pgdat->node_id; 1556 enum zone_type z; 1557 int cpu; 1558 1559 pgdat_init_internals(pgdat); 1560 1561 if (pgdat->per_cpu_nodestats == &boot_nodestats) 1562 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 1563 1564 /* 1565 * Reset the nr_zones, order and highest_zoneidx before reuse. 1566 * Note that kswapd will init kswapd_highest_zoneidx properly 1567 * when it starts in the near future. 1568 */ 1569 pgdat->nr_zones = 0; 1570 pgdat->kswapd_order = 0; 1571 pgdat->kswapd_highest_zoneidx = 0; 1572 pgdat->node_start_pfn = 0; 1573 pgdat->node_present_pages = 0; 1574 1575 for_each_online_cpu(cpu) { 1576 struct per_cpu_nodestat *p; 1577 1578 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); 1579 memset(p, 0, sizeof(*p)); 1580 } 1581 1582 /* 1583 * When memory is hot-added, all the memory is in offline state. So 1584 * clear all zones' present_pages and managed_pages because they will 1585 * be updated in online_pages() and offline_pages(). 1586 */ 1587 for (z = 0; z < MAX_NR_ZONES; z++) { 1588 struct zone *zone = pgdat->node_zones + z; 1589 1590 zone->present_pages = 0; 1591 zone_init_internals(zone, z, nid, 0); 1592 } 1593} 1594#endif 1595 1596static void __init free_area_init_core(struct pglist_data *pgdat) 1597{ 1598 enum zone_type j; 1599 int nid = pgdat->node_id; 1600 1601 pgdat_init_internals(pgdat); 1602 pgdat->per_cpu_nodestats = &boot_nodestats; 1603 1604 for (j = 0; j < MAX_NR_ZONES; j++) { 1605 struct zone *zone = pgdat->node_zones + j; 1606 unsigned long size = zone->spanned_pages; 1607 1608 /* 1609 * Initialize zone->managed_pages as 0 , it will be reset 1610 * when memblock allocator frees pages into buddy system. 1611 */ 1612 zone_init_internals(zone, j, nid, zone->present_pages); 1613 1614 if (!size) 1615 continue; 1616 1617 setup_usemap(zone); 1618 init_currently_empty_zone(zone, zone->zone_start_pfn, size); 1619 } 1620} 1621 1622void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, 1623 phys_addr_t min_addr, int nid, bool exact_nid) 1624{ 1625 void *ptr; 1626 1627 /* 1628 * Kmemleak will explicitly scan mem_map by traversing all valid 1629 * `struct *page`,so memblock does not need to be added to the scan list. 1630 */ 1631 if (exact_nid) 1632 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, 1633 MEMBLOCK_ALLOC_NOLEAKTRACE, 1634 nid); 1635 else 1636 ptr = memblock_alloc_try_nid_raw(size, align, min_addr, 1637 MEMBLOCK_ALLOC_NOLEAKTRACE, 1638 nid); 1639 1640 if (ptr && size > 0) 1641 page_init_poison(ptr, size); 1642 1643 return ptr; 1644} 1645 1646#ifdef CONFIG_FLATMEM 1647static void __init alloc_node_mem_map(struct pglist_data *pgdat) 1648{ 1649 unsigned long start, offset, size, end; 1650 struct page *map; 1651 1652 /* Skip empty nodes */ 1653 if (!pgdat->node_spanned_pages) 1654 return; 1655 1656 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); 1657 offset = pgdat->node_start_pfn - start; 1658 /* 1659 * The zone's endpoints aren't required to be MAX_PAGE_ORDER 1660 * aligned but the node_mem_map endpoints must be in order 1661 * for the buddy allocator to function correctly. 1662 */ 1663 end = ALIGN(pgdat_end_pfn(pgdat), MAX_ORDER_NR_PAGES); 1664 size = (end - start) * sizeof(struct page); 1665 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, 1666 pgdat->node_id, false); 1667 if (!map) 1668 panic("Failed to allocate %ld bytes for node %d memory map\n", 1669 size, pgdat->node_id); 1670 pgdat->node_mem_map = map + offset; 1671 memmap_boot_pages_add(DIV_ROUND_UP(size, PAGE_SIZE)); 1672 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", 1673 __func__, pgdat->node_id, (unsigned long)pgdat, 1674 (unsigned long)pgdat->node_mem_map); 1675 1676 /* the global mem_map is just set as node 0's */ 1677 WARN_ON(pgdat != NODE_DATA(0)); 1678 1679 mem_map = pgdat->node_mem_map; 1680 if (page_to_pfn(mem_map) != pgdat->node_start_pfn) 1681 mem_map -= offset; 1682 1683 max_mapnr = end - start; 1684} 1685#else 1686static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } 1687#endif /* CONFIG_FLATMEM */ 1688 1689/** 1690 * get_pfn_range_for_nid - Return the start and end page frames for a node 1691 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. 1692 * @start_pfn: Passed by reference. On return, it will have the node start_pfn. 1693 * @end_pfn: Passed by reference. On return, it will have the node end_pfn. 1694 * 1695 * It returns the start and end page frame of a node based on information 1696 * provided by memblock_set_node(). If called for a node 1697 * with no available memory, the start and end PFNs will be 0. 1698 */ 1699void __init get_pfn_range_for_nid(unsigned int nid, 1700 unsigned long *start_pfn, unsigned long *end_pfn) 1701{ 1702 unsigned long this_start_pfn, this_end_pfn; 1703 int i; 1704 1705 *start_pfn = -1UL; 1706 *end_pfn = 0; 1707 1708 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { 1709 *start_pfn = min(*start_pfn, this_start_pfn); 1710 *end_pfn = max(*end_pfn, this_end_pfn); 1711 } 1712 1713 if (*start_pfn == -1UL) 1714 *start_pfn = 0; 1715} 1716 1717static void __init free_area_init_node(int nid) 1718{ 1719 pg_data_t *pgdat = NODE_DATA(nid); 1720 unsigned long start_pfn = 0; 1721 unsigned long end_pfn = 0; 1722 1723 /* pg_data_t should be reset to zero when it's allocated */ 1724 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); 1725 1726 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 1727 1728 pgdat->node_id = nid; 1729 pgdat->node_start_pfn = start_pfn; 1730 pgdat->per_cpu_nodestats = NULL; 1731 1732 if (start_pfn != end_pfn) { 1733 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, 1734 (u64)start_pfn << PAGE_SHIFT, 1735 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); 1736 1737 calculate_node_totalpages(pgdat, start_pfn, end_pfn); 1738 } else { 1739 pr_info("Initmem setup node %d as memoryless\n", nid); 1740 1741 reset_memoryless_node_totalpages(pgdat); 1742 } 1743 1744 alloc_node_mem_map(pgdat); 1745 pgdat_set_deferred_range(pgdat); 1746 1747 free_area_init_core(pgdat); 1748 lru_gen_init_pgdat(pgdat); 1749} 1750 1751/* Any regular or high memory on that node ? */ 1752static void __init check_for_memory(pg_data_t *pgdat) 1753{ 1754 enum zone_type zone_type; 1755 1756 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { 1757 struct zone *zone = &pgdat->node_zones[zone_type]; 1758 if (populated_zone(zone)) { 1759 if (IS_ENABLED(CONFIG_HIGHMEM)) 1760 node_set_state(pgdat->node_id, N_HIGH_MEMORY); 1761 if (zone_type <= ZONE_NORMAL) 1762 node_set_state(pgdat->node_id, N_NORMAL_MEMORY); 1763 break; 1764 } 1765 } 1766} 1767 1768#if MAX_NUMNODES > 1 1769/* 1770 * Figure out the number of possible node ids. 1771 */ 1772void __init setup_nr_node_ids(void) 1773{ 1774 unsigned int highest; 1775 1776 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); 1777 nr_node_ids = highest + 1; 1778} 1779#endif 1780 1781/* 1782 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For 1783 * such cases we allow max_zone_pfn sorted in the descending order 1784 */ 1785static bool arch_has_descending_max_zone_pfns(void) 1786{ 1787 return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40); 1788} 1789 1790static void __init set_high_memory(void) 1791{ 1792 phys_addr_t highmem = memblock_end_of_DRAM(); 1793 1794 /* 1795 * Some architectures (e.g. ARM) set high_memory very early and 1796 * use it in arch setup code. 1797 * If an architecture already set high_memory don't overwrite it 1798 */ 1799 if (high_memory) 1800 return; 1801 1802#ifdef CONFIG_HIGHMEM 1803 if (arch_has_descending_max_zone_pfns() || 1804 highmem > PFN_PHYS(arch_zone_lowest_possible_pfn[ZONE_HIGHMEM])) 1805 highmem = PFN_PHYS(arch_zone_lowest_possible_pfn[ZONE_HIGHMEM]); 1806#endif 1807 1808 high_memory = phys_to_virt(highmem - 1) + 1; 1809} 1810 1811/** 1812 * free_area_init - Initialise all pg_data_t and zone data 1813 * @max_zone_pfn: an array of max PFNs for each zone 1814 * 1815 * This will call free_area_init_node() for each active node in the system. 1816 * Using the page ranges provided by memblock_set_node(), the size of each 1817 * zone in each node and their holes is calculated. If the maximum PFN 1818 * between two adjacent zones match, it is assumed that the zone is empty. 1819 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed 1820 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone 1821 * starts where the previous one ended. For example, ZONE_DMA32 starts 1822 * at arch_max_dma_pfn. 1823 */ 1824void __init free_area_init(unsigned long *max_zone_pfn) 1825{ 1826 unsigned long start_pfn, end_pfn; 1827 int i, nid, zone; 1828 bool descending; 1829 1830 /* Record where the zone boundaries are */ 1831 memset(arch_zone_lowest_possible_pfn, 0, 1832 sizeof(arch_zone_lowest_possible_pfn)); 1833 memset(arch_zone_highest_possible_pfn, 0, 1834 sizeof(arch_zone_highest_possible_pfn)); 1835 1836 start_pfn = PHYS_PFN(memblock_start_of_DRAM()); 1837 descending = arch_has_descending_max_zone_pfns(); 1838 1839 for (i = 0; i < MAX_NR_ZONES; i++) { 1840 if (descending) 1841 zone = MAX_NR_ZONES - i - 1; 1842 else 1843 zone = i; 1844 1845 if (zone == ZONE_MOVABLE) 1846 continue; 1847 1848 end_pfn = max(max_zone_pfn[zone], start_pfn); 1849 arch_zone_lowest_possible_pfn[zone] = start_pfn; 1850 arch_zone_highest_possible_pfn[zone] = end_pfn; 1851 1852 start_pfn = end_pfn; 1853 } 1854 1855 /* Find the PFNs that ZONE_MOVABLE begins at in each node */ 1856 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); 1857 find_zone_movable_pfns_for_nodes(); 1858 1859 /* Print out the zone ranges */ 1860 pr_info("Zone ranges:\n"); 1861 for (i = 0; i < MAX_NR_ZONES; i++) { 1862 if (i == ZONE_MOVABLE) 1863 continue; 1864 pr_info(" %-8s ", zone_names[i]); 1865 if (arch_zone_lowest_possible_pfn[i] == 1866 arch_zone_highest_possible_pfn[i]) 1867 pr_cont("empty\n"); 1868 else 1869 pr_cont("[mem %#018Lx-%#018Lx]\n", 1870 (u64)arch_zone_lowest_possible_pfn[i] 1871 << PAGE_SHIFT, 1872 ((u64)arch_zone_highest_possible_pfn[i] 1873 << PAGE_SHIFT) - 1); 1874 } 1875 1876 /* Print out the PFNs ZONE_MOVABLE begins at in each node */ 1877 pr_info("Movable zone start for each node\n"); 1878 for (i = 0; i < MAX_NUMNODES; i++) { 1879 if (zone_movable_pfn[i]) 1880 pr_info(" Node %d: %#018Lx\n", i, 1881 (u64)zone_movable_pfn[i] << PAGE_SHIFT); 1882 } 1883 1884 /* 1885 * Print out the early node map, and initialize the 1886 * subsection-map relative to active online memory ranges to 1887 * enable future "sub-section" extensions of the memory map. 1888 */ 1889 pr_info("Early memory node ranges\n"); 1890 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 1891 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, 1892 (u64)start_pfn << PAGE_SHIFT, 1893 ((u64)end_pfn << PAGE_SHIFT) - 1); 1894 subsection_map_init(start_pfn, end_pfn - start_pfn); 1895 } 1896 1897 /* Initialise every node */ 1898 mminit_verify_pageflags_layout(); 1899 setup_nr_node_ids(); 1900 set_pageblock_order(); 1901 1902 for_each_node(nid) { 1903 pg_data_t *pgdat; 1904 1905 if (!node_online(nid)) 1906 alloc_offline_node_data(nid); 1907 1908 pgdat = NODE_DATA(nid); 1909 free_area_init_node(nid); 1910 1911 /* 1912 * No sysfs hierarchy will be created via register_node() 1913 *for memory-less node because here it's not marked as N_MEMORY 1914 *and won't be set online later. The benefit is userspace 1915 *program won't be confused by sysfs files/directories of 1916 *memory-less node. The pgdat will get fully initialized by 1917 *hotadd_init_pgdat() when memory is hotplugged into this node. 1918 */ 1919 if (pgdat->node_present_pages) { 1920 node_set_state(nid, N_MEMORY); 1921 check_for_memory(pgdat); 1922 } 1923 } 1924 1925 for_each_node_state(nid, N_MEMORY) 1926 sparse_vmemmap_init_nid_late(nid); 1927 1928 calc_nr_kernel_pages(); 1929 memmap_init(); 1930 1931 /* disable hash distribution for systems with a single node */ 1932 fixup_hashdist(); 1933 1934 set_high_memory(); 1935} 1936 1937/** 1938 * node_map_pfn_alignment - determine the maximum internode alignment 1939 * 1940 * This function should be called after node map is populated and sorted. 1941 * It calculates the maximum power of two alignment which can distinguish 1942 * all the nodes. 1943 * 1944 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value 1945 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the 1946 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is 1947 * shifted, 1GiB is enough and this function will indicate so. 1948 * 1949 * This is used to test whether pfn -> nid mapping of the chosen memory 1950 * model has fine enough granularity to avoid incorrect mapping for the 1951 * populated node map. 1952 * 1953 * Return: the determined alignment in pfn's. 0 if there is no alignment 1954 * requirement (single node). 1955 */ 1956unsigned long __init node_map_pfn_alignment(void) 1957{ 1958 unsigned long accl_mask = 0, last_end = 0; 1959 unsigned long start, end, mask; 1960 int last_nid = NUMA_NO_NODE; 1961 int i, nid; 1962 1963 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { 1964 if (!start || last_nid < 0 || last_nid == nid) { 1965 last_nid = nid; 1966 last_end = end; 1967 continue; 1968 } 1969 1970 /* 1971 * Start with a mask granular enough to pin-point to the 1972 * start pfn and tick off bits one-by-one until it becomes 1973 * too coarse to separate the current node from the last. 1974 */ 1975 mask = ~((1 << __ffs(start)) - 1); 1976 while (mask && last_end <= (start & (mask << 1))) 1977 mask <<= 1; 1978 1979 /* accumulate all internode masks */ 1980 accl_mask |= mask; 1981 } 1982 1983 /* convert mask to number of pages */ 1984 return ~accl_mask + 1; 1985} 1986 1987#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1988static void __init deferred_free_pages(unsigned long pfn, 1989 unsigned long nr_pages) 1990{ 1991 struct page *page; 1992 unsigned long i; 1993 1994 if (!nr_pages) 1995 return; 1996 1997 page = pfn_to_page(pfn); 1998 1999 /* Free a large naturally-aligned chunk if possible */ 2000 if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { 2001 for (i = 0; i < nr_pages; i += pageblock_nr_pages) 2002 init_pageblock_migratetype(page + i, MIGRATE_MOVABLE, 2003 false); 2004 __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY); 2005 return; 2006 } 2007 2008 /* Accept chunks smaller than MAX_PAGE_ORDER upfront */ 2009 accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE); 2010 2011 for (i = 0; i < nr_pages; i++, page++, pfn++) { 2012 if (pageblock_aligned(pfn)) 2013 init_pageblock_migratetype(page, MIGRATE_MOVABLE, 2014 false); 2015 __free_pages_core(page, 0, MEMINIT_EARLY); 2016 } 2017} 2018 2019/* Completion tracking for deferred_init_memmap() threads */ 2020static atomic_t pgdat_init_n_undone __initdata; 2021static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); 2022 2023static inline void __init pgdat_init_report_one_done(void) 2024{ 2025 if (atomic_dec_and_test(&pgdat_init_n_undone)) 2026 complete(&pgdat_init_all_done_comp); 2027} 2028 2029/* 2030 * Initialize struct pages. We minimize pfn page lookups and scheduler checks 2031 * by performing it only once every MAX_ORDER_NR_PAGES. 2032 * Return number of pages initialized. 2033 */ 2034static unsigned long __init deferred_init_pages(struct zone *zone, 2035 unsigned long pfn, unsigned long end_pfn) 2036{ 2037 int nid = zone_to_nid(zone); 2038 unsigned long nr_pages = end_pfn - pfn; 2039 int zid = zone_idx(zone); 2040 struct page *page = pfn_to_page(pfn); 2041 2042 for (; pfn < end_pfn; pfn++, page++) 2043 __init_single_page(page, pfn, zid, nid); 2044 return nr_pages; 2045} 2046 2047/* 2048 * Initialize and free pages. 2049 * 2050 * At this point reserved pages and struct pages that correspond to holes in 2051 * memblock.memory are already intialized so every free range has a valid 2052 * memory map around it. 2053 * This ensures that access of pages that are ahead of the range being 2054 * initialized (computing buddy page in __free_one_page()) always reads a valid 2055 * struct page. 2056 * 2057 * In order to try and improve CPU cache locality we have the loop broken along 2058 * max page order boundaries. 2059 */ 2060static unsigned long __init 2061deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, 2062 struct zone *zone, bool can_resched) 2063{ 2064 int nid = zone_to_nid(zone); 2065 unsigned long nr_pages = 0; 2066 phys_addr_t start, end; 2067 u64 i = 0; 2068 2069 for_each_free_mem_range(i, nid, 0, &start, &end, NULL) { 2070 unsigned long spfn = PFN_UP(start); 2071 unsigned long epfn = PFN_DOWN(end); 2072 2073 if (spfn >= end_pfn) 2074 break; 2075 2076 spfn = max(spfn, start_pfn); 2077 epfn = min(epfn, end_pfn); 2078 2079 while (spfn < epfn) { 2080 unsigned long mo_pfn = ALIGN(spfn + 1, MAX_ORDER_NR_PAGES); 2081 unsigned long chunk_end = min(mo_pfn, epfn); 2082 2083 nr_pages += deferred_init_pages(zone, spfn, chunk_end); 2084 deferred_free_pages(spfn, chunk_end - spfn); 2085 2086 spfn = chunk_end; 2087 2088 if (can_resched) 2089 cond_resched(); 2090 else 2091 touch_nmi_watchdog(); 2092 } 2093 } 2094 2095 return nr_pages; 2096} 2097 2098static void __init 2099deferred_init_memmap_job(unsigned long start_pfn, unsigned long end_pfn, 2100 void *arg) 2101{ 2102 struct zone *zone = arg; 2103 2104 deferred_init_memmap_chunk(start_pfn, end_pfn, zone, true); 2105} 2106 2107static unsigned int __init 2108deferred_page_init_max_threads(const struct cpumask *node_cpumask) 2109{ 2110 return max(cpumask_weight(node_cpumask), 1U); 2111} 2112 2113/* Initialise remaining memory on a node */ 2114static int __init deferred_init_memmap(void *data) 2115{ 2116 pg_data_t *pgdat = data; 2117 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 2118 int max_threads = deferred_page_init_max_threads(cpumask); 2119 unsigned long first_init_pfn, last_pfn, flags; 2120 unsigned long start = jiffies; 2121 struct zone *zone; 2122 2123 /* Bind memory initialisation thread to a local node if possible */ 2124 if (!cpumask_empty(cpumask)) 2125 set_cpus_allowed_ptr(current, cpumask); 2126 2127 pgdat_resize_lock(pgdat, &flags); 2128 first_init_pfn = pgdat->first_deferred_pfn; 2129 if (first_init_pfn == ULONG_MAX) { 2130 pgdat_resize_unlock(pgdat, &flags); 2131 pgdat_init_report_one_done(); 2132 return 0; 2133 } 2134 2135 /* Sanity check boundaries */ 2136 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); 2137 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); 2138 pgdat->first_deferred_pfn = ULONG_MAX; 2139 2140 /* 2141 * Once we unlock here, the zone cannot be grown anymore, thus if an 2142 * interrupt thread must allocate this early in boot, zone must be 2143 * pre-grown prior to start of deferred page initialization. 2144 */ 2145 pgdat_resize_unlock(pgdat, &flags); 2146 2147 /* Only the highest zone is deferred */ 2148 zone = pgdat->node_zones + pgdat->nr_zones - 1; 2149 last_pfn = SECTION_ALIGN_UP(zone_end_pfn(zone)); 2150 2151 struct padata_mt_job job = { 2152 .thread_fn = deferred_init_memmap_job, 2153 .fn_arg = zone, 2154 .start = first_init_pfn, 2155 .size = last_pfn - first_init_pfn, 2156 .align = PAGES_PER_SECTION, 2157 .min_chunk = PAGES_PER_SECTION, 2158 .max_threads = max_threads, 2159 .numa_aware = false, 2160 }; 2161 2162 padata_do_multithreaded(&job); 2163 2164 /* Sanity check that the next zone really is unpopulated */ 2165 WARN_ON(pgdat->nr_zones < MAX_NR_ZONES && populated_zone(++zone)); 2166 2167 pr_info("node %d deferred pages initialised in %ums\n", 2168 pgdat->node_id, jiffies_to_msecs(jiffies - start)); 2169 2170 pgdat_init_report_one_done(); 2171 return 0; 2172} 2173 2174/* 2175 * If this zone has deferred pages, try to grow it by initializing enough 2176 * deferred pages to satisfy the allocation specified by order, rounded up to 2177 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments 2178 * of SECTION_SIZE bytes by initializing struct pages in increments of 2179 * PAGES_PER_SECTION * sizeof(struct page) bytes. 2180 * 2181 * Return true when zone was grown, otherwise return false. We return true even 2182 * when we grow less than requested, to let the caller decide if there are 2183 * enough pages to satisfy the allocation. 2184 */ 2185bool __init deferred_grow_zone(struct zone *zone, unsigned int order) 2186{ 2187 unsigned long nr_pages_needed = SECTION_ALIGN_UP(1 << order); 2188 pg_data_t *pgdat = zone->zone_pgdat; 2189 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; 2190 unsigned long spfn, epfn, flags; 2191 unsigned long nr_pages = 0; 2192 2193 /* Only the last zone may have deferred pages */ 2194 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) 2195 return false; 2196 2197 pgdat_resize_lock(pgdat, &flags); 2198 2199 /* 2200 * If someone grew this zone while we were waiting for spinlock, return 2201 * true, as there might be enough pages already. 2202 */ 2203 if (first_deferred_pfn != pgdat->first_deferred_pfn) { 2204 pgdat_resize_unlock(pgdat, &flags); 2205 return true; 2206 } 2207 2208 /* 2209 * Initialize at least nr_pages_needed in section chunks. 2210 * If a section has less free memory than nr_pages_needed, the next 2211 * section will be also initialized. 2212 * Note, that it still does not guarantee that allocation of order can 2213 * be satisfied if the sections are fragmented because of memblock 2214 * allocations. 2215 */ 2216 for (spfn = first_deferred_pfn, epfn = SECTION_ALIGN_UP(spfn + 1); 2217 nr_pages < nr_pages_needed && spfn < zone_end_pfn(zone); 2218 spfn = epfn, epfn += PAGES_PER_SECTION) { 2219 nr_pages += deferred_init_memmap_chunk(spfn, epfn, zone, false); 2220 } 2221 2222 /* 2223 * There were no pages to initialize and free which means the zone's 2224 * memory map is completely initialized. 2225 */ 2226 pgdat->first_deferred_pfn = nr_pages ? spfn : ULONG_MAX; 2227 2228 pgdat_resize_unlock(pgdat, &flags); 2229 2230 return nr_pages > 0; 2231} 2232 2233#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 2234 2235#ifdef CONFIG_CMA 2236void __init init_cma_reserved_pageblock(struct page *page) 2237{ 2238 unsigned i = pageblock_nr_pages; 2239 struct page *p = page; 2240 2241 do { 2242 __ClearPageReserved(p); 2243 set_page_count(p, 0); 2244 } while (++p, --i); 2245 2246 init_pageblock_migratetype(page, MIGRATE_CMA, false); 2247 set_page_refcounted(page); 2248 /* pages were reserved and not allocated */ 2249 clear_page_tag_ref(page); 2250 __free_pages(page, pageblock_order); 2251 2252 adjust_managed_page_count(page, pageblock_nr_pages); 2253 page_zone(page)->cma_pages += pageblock_nr_pages; 2254} 2255/* 2256 * Similar to above, but only set the migrate type and stats. 2257 */ 2258void __init init_cma_pageblock(struct page *page) 2259{ 2260 init_pageblock_migratetype(page, MIGRATE_CMA, false); 2261 adjust_managed_page_count(page, pageblock_nr_pages); 2262 page_zone(page)->cma_pages += pageblock_nr_pages; 2263} 2264#endif 2265 2266void set_zone_contiguous(struct zone *zone) 2267{ 2268 unsigned long block_start_pfn = zone->zone_start_pfn; 2269 unsigned long block_end_pfn; 2270 2271 block_end_pfn = pageblock_end_pfn(block_start_pfn); 2272 for (; block_start_pfn < zone_end_pfn(zone); 2273 block_start_pfn = block_end_pfn, 2274 block_end_pfn += pageblock_nr_pages) { 2275 2276 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); 2277 2278 if (!__pageblock_pfn_to_page(block_start_pfn, 2279 block_end_pfn, zone)) 2280 return; 2281 cond_resched(); 2282 } 2283 2284 /* We confirm that there is no hole */ 2285 zone->contiguous = true; 2286} 2287 2288/* 2289 * Check if a PFN range intersects multiple zones on one or more 2290 * NUMA nodes. Specify the @nid argument if it is known that this 2291 * PFN range is on one node, NUMA_NO_NODE otherwise. 2292 */ 2293bool pfn_range_intersects_zones(int nid, unsigned long start_pfn, 2294 unsigned long nr_pages) 2295{ 2296 struct zone *zone, *izone = NULL; 2297 2298 for_each_zone(zone) { 2299 if (nid != NUMA_NO_NODE && zone_to_nid(zone) != nid) 2300 continue; 2301 2302 if (zone_intersects(zone, start_pfn, nr_pages)) { 2303 if (izone != NULL) 2304 return true; 2305 izone = zone; 2306 } 2307 2308 } 2309 2310 return false; 2311} 2312 2313static void __init mem_init_print_info(void); 2314void __init page_alloc_init_late(void) 2315{ 2316 struct zone *zone; 2317 int nid; 2318 2319#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 2320 2321 /* There will be num_node_state(N_MEMORY) threads */ 2322 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); 2323 for_each_node_state(nid, N_MEMORY) { 2324 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); 2325 } 2326 2327 /* Block until all are initialised */ 2328 wait_for_completion(&pgdat_init_all_done_comp); 2329 2330 /* 2331 * We initialized the rest of the deferred pages. Permanently disable 2332 * on-demand struct page initialization. 2333 */ 2334 static_branch_disable(&deferred_pages); 2335 2336 /* Reinit limits that are based on free pages after the kernel is up */ 2337 files_maxfiles_init(); 2338#endif 2339 2340 /* Accounting of total+free memory is stable at this point. */ 2341 mem_init_print_info(); 2342 buffer_init(); 2343 2344 /* Discard memblock private memory */ 2345 memblock_discard(); 2346 2347 for_each_node_state(nid, N_MEMORY) 2348 shuffle_free_memory(NODE_DATA(nid)); 2349 2350 for_each_populated_zone(zone) 2351 set_zone_contiguous(zone); 2352 2353 /* Initialize page ext after all struct pages are initialized. */ 2354 if (deferred_struct_pages) 2355 page_ext_init(); 2356 2357 page_alloc_sysctl_init(); 2358} 2359 2360/* 2361 * Adaptive scale is meant to reduce sizes of hash tables on large memory 2362 * machines. As memory size is increased the scale is also increased but at 2363 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory 2364 * quadruples the scale is increased by one, which means the size of hash table 2365 * only doubles, instead of quadrupling as well. 2366 * Because 32-bit systems cannot have large physical memory, where this scaling 2367 * makes sense, it is disabled on such platforms. 2368 */ 2369#if __BITS_PER_LONG > 32 2370#define ADAPT_SCALE_BASE (64ul << 30) 2371#define ADAPT_SCALE_SHIFT 2 2372#define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) 2373#endif 2374 2375/* 2376 * allocate a large system hash table from bootmem 2377 * - it is assumed that the hash table must contain an exact power-of-2 2378 * quantity of entries 2379 * - limit is the number of hash buckets, not the total allocation size 2380 */ 2381void *__init alloc_large_system_hash(const char *tablename, 2382 unsigned long bucketsize, 2383 unsigned long numentries, 2384 int scale, 2385 int flags, 2386 unsigned int *_hash_shift, 2387 unsigned int *_hash_mask, 2388 unsigned long low_limit, 2389 unsigned long high_limit) 2390{ 2391 unsigned long long max = high_limit; 2392 unsigned long log2qty, size; 2393 void *table; 2394 gfp_t gfp_flags; 2395 bool virt; 2396 bool huge; 2397 2398 /* allow the kernel cmdline to have a say */ 2399 if (!numentries) { 2400 /* round applicable memory size up to nearest megabyte */ 2401 numentries = nr_kernel_pages; 2402 2403 /* It isn't necessary when PAGE_SIZE >= 1MB */ 2404 if (PAGE_SIZE < SZ_1M) 2405 numentries = round_up(numentries, SZ_1M / PAGE_SIZE); 2406 2407#if __BITS_PER_LONG > 32 2408 if (!high_limit) { 2409 unsigned long adapt; 2410 2411 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; 2412 adapt <<= ADAPT_SCALE_SHIFT) 2413 scale++; 2414 } 2415#endif 2416 2417 /* limit to 1 bucket per 2^scale bytes of low memory */ 2418 if (scale > PAGE_SHIFT) 2419 numentries >>= (scale - PAGE_SHIFT); 2420 else 2421 numentries <<= (PAGE_SHIFT - scale); 2422 2423 if (unlikely((numentries * bucketsize) < PAGE_SIZE)) 2424 numentries = PAGE_SIZE / bucketsize; 2425 } 2426 numentries = roundup_pow_of_two(numentries); 2427 2428 /* limit allocation size to 1/16 total memory by default */ 2429 if (max == 0) { 2430 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; 2431 do_div(max, bucketsize); 2432 } 2433 max = min(max, 0x80000000ULL); 2434 2435 if (numentries < low_limit) 2436 numentries = low_limit; 2437 if (numentries > max) 2438 numentries = max; 2439 2440 log2qty = ilog2(numentries); 2441 2442 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; 2443 do { 2444 virt = false; 2445 size = bucketsize << log2qty; 2446 if (flags & HASH_EARLY) { 2447 if (flags & HASH_ZERO) 2448 table = memblock_alloc(size, SMP_CACHE_BYTES); 2449 else 2450 table = memblock_alloc_raw(size, 2451 SMP_CACHE_BYTES); 2452 } else if (get_order(size) > MAX_PAGE_ORDER || hashdist) { 2453 table = vmalloc_huge(size, gfp_flags); 2454 virt = true; 2455 if (table) 2456 huge = is_vm_area_hugepages(table); 2457 } else { 2458 /* 2459 * If bucketsize is not a power-of-two, we may free 2460 * some pages at the end of hash table which 2461 * alloc_pages_exact() automatically does 2462 */ 2463 table = alloc_pages_exact(size, gfp_flags); 2464 kmemleak_alloc(table, size, 1, gfp_flags); 2465 } 2466 } while (!table && size > PAGE_SIZE && --log2qty); 2467 2468 if (!table) 2469 panic("Failed to allocate %s hash table\n", tablename); 2470 2471 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", 2472 tablename, 1UL << log2qty, get_order(size), size, 2473 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); 2474 2475 if (_hash_shift) 2476 *_hash_shift = log2qty; 2477 if (_hash_mask) 2478 *_hash_mask = (1 << log2qty) - 1; 2479 2480 return table; 2481} 2482 2483void __init memblock_free_pages(struct page *page, unsigned long pfn, 2484 unsigned int order) 2485{ 2486 if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) { 2487 int nid = early_pfn_to_nid(pfn); 2488 2489 if (!early_page_initialised(pfn, nid)) 2490 return; 2491 } 2492 2493 if (!kmsan_memblock_free_pages(page, order)) { 2494 /* KMSAN will take care of these pages. */ 2495 return; 2496 } 2497 2498 /* pages were reserved and not allocated */ 2499 clear_page_tag_ref(page); 2500 __free_pages_core(page, order, MEMINIT_EARLY); 2501} 2502 2503DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); 2504EXPORT_SYMBOL(init_on_alloc); 2505 2506DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); 2507EXPORT_SYMBOL(init_on_free); 2508 2509static bool _init_on_alloc_enabled_early __read_mostly 2510 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); 2511static int __init early_init_on_alloc(char *buf) 2512{ 2513 2514 return kstrtobool(buf, &_init_on_alloc_enabled_early); 2515} 2516early_param("init_on_alloc", early_init_on_alloc); 2517 2518static bool _init_on_free_enabled_early __read_mostly 2519 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); 2520static int __init early_init_on_free(char *buf) 2521{ 2522 return kstrtobool(buf, &_init_on_free_enabled_early); 2523} 2524early_param("init_on_free", early_init_on_free); 2525 2526DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); 2527 2528static bool check_pages_enabled_early __initdata; 2529 2530static int __init early_check_pages(char *buf) 2531{ 2532 return kstrtobool(buf, &check_pages_enabled_early); 2533} 2534early_param("check_pages", early_check_pages); 2535 2536/* 2537 * Enable static keys related to various memory debugging and hardening options. 2538 * Some override others, and depend on early params that are evaluated in the 2539 * order of appearance. So we need to first gather the full picture of what was 2540 * enabled, and then make decisions. 2541 */ 2542static void __init mem_debugging_and_hardening_init(void) 2543{ 2544 bool page_poisoning_requested = false; 2545 bool want_check_pages = check_pages_enabled_early; 2546 2547#ifdef CONFIG_PAGE_POISONING 2548 /* 2549 * Page poisoning is debug page alloc for some arches. If 2550 * either of those options are enabled, enable poisoning. 2551 */ 2552 if (page_poisoning_enabled() || 2553 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && 2554 debug_pagealloc_enabled())) { 2555 static_branch_enable(&_page_poisoning_enabled); 2556 page_poisoning_requested = true; 2557 want_check_pages = true; 2558 } 2559#endif 2560 2561 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && 2562 page_poisoning_requested) { 2563 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " 2564 "will take precedence over init_on_alloc and init_on_free\n"); 2565 _init_on_alloc_enabled_early = false; 2566 _init_on_free_enabled_early = false; 2567 } 2568 2569 if (_init_on_alloc_enabled_early) { 2570 want_check_pages = true; 2571 static_branch_enable(&init_on_alloc); 2572 } else { 2573 static_branch_disable(&init_on_alloc); 2574 } 2575 2576 if (_init_on_free_enabled_early) { 2577 want_check_pages = true; 2578 static_branch_enable(&init_on_free); 2579 } else { 2580 static_branch_disable(&init_on_free); 2581 } 2582 2583 if (IS_ENABLED(CONFIG_KMSAN) && 2584 (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) 2585 pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); 2586 2587#ifdef CONFIG_DEBUG_PAGEALLOC 2588 if (debug_pagealloc_enabled()) { 2589 want_check_pages = true; 2590 static_branch_enable(&_debug_pagealloc_enabled); 2591 2592 if (debug_guardpage_minorder()) 2593 static_branch_enable(&_debug_guardpage_enabled); 2594 } 2595#endif 2596 2597 /* 2598 * Any page debugging or hardening option also enables sanity checking 2599 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's 2600 * enabled already. 2601 */ 2602 if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages) 2603 static_branch_enable(&check_pages_enabled); 2604} 2605 2606/* Report memory auto-initialization states for this boot. */ 2607static void __init report_meminit(void) 2608{ 2609 const char *stack; 2610 2611 if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN)) 2612 stack = "all(pattern)"; 2613 else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO)) 2614 stack = "all(zero)"; 2615 else 2616 stack = "off"; 2617 2618 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", 2619 stack, str_on_off(want_init_on_alloc(GFP_KERNEL)), 2620 str_on_off(want_init_on_free())); 2621 if (want_init_on_free()) 2622 pr_info("mem auto-init: clearing system memory may take some time...\n"); 2623} 2624 2625static void __init mem_init_print_info(void) 2626{ 2627 unsigned long physpages, codesize, datasize, rosize, bss_size; 2628 unsigned long init_code_size, init_data_size; 2629 2630 physpages = get_num_physpages(); 2631 codesize = _etext - _stext; 2632 datasize = _edata - _sdata; 2633 rosize = __end_rodata - __start_rodata; 2634 bss_size = __bss_stop - __bss_start; 2635 init_data_size = __init_end - __init_begin; 2636 init_code_size = _einittext - _sinittext; 2637 2638 /* 2639 * Detect special cases and adjust section sizes accordingly: 2640 * 1) .init.* may be embedded into .data sections 2641 * 2) .init.text.* may be out of [__init_begin, __init_end], 2642 * please refer to arch/tile/kernel/vmlinux.lds.S. 2643 * 3) .rodata.* may be embedded into .text or .data sections. 2644 */ 2645#define adj_init_size(start, end, size, pos, adj) \ 2646 do { \ 2647 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ 2648 size -= adj; \ 2649 } while (0) 2650 2651 adj_init_size(__init_begin, __init_end, init_data_size, 2652 _sinittext, init_code_size); 2653 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); 2654 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); 2655 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); 2656 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); 2657 2658#undef adj_init_size 2659 2660 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" 2661#ifdef CONFIG_HIGHMEM 2662 ", %luK highmem" 2663#endif 2664 ")\n", 2665 K(nr_free_pages()), K(physpages), 2666 codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, 2667 (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, 2668 K(physpages - totalram_pages() - totalcma_pages), 2669 K(totalcma_pages) 2670#ifdef CONFIG_HIGHMEM 2671 , K(totalhigh_pages()) 2672#endif 2673 ); 2674} 2675 2676void __init __weak arch_mm_preinit(void) 2677{ 2678} 2679 2680void __init __weak mem_init(void) 2681{ 2682} 2683 2684/* 2685 * Set up kernel memory allocators 2686 */ 2687void __init mm_core_init(void) 2688{ 2689 arch_mm_preinit(); 2690 hugetlb_bootmem_alloc(); 2691 2692 /* Initializations relying on SMP setup */ 2693 BUILD_BUG_ON(MAX_ZONELISTS > 2); 2694 build_all_zonelists(NULL); 2695 page_alloc_init_cpuhp(); 2696 alloc_tag_sec_init(); 2697 /* 2698 * page_ext requires contiguous pages, 2699 * bigger than MAX_PAGE_ORDER unless SPARSEMEM. 2700 */ 2701 page_ext_init_flatmem(); 2702 mem_debugging_and_hardening_init(); 2703 kfence_alloc_pool_and_metadata(); 2704 report_meminit(); 2705 kmsan_init_shadow(); 2706 stack_depot_early_init(); 2707 2708 /* 2709 * KHO memory setup must happen while memblock is still active, but 2710 * as close as possible to buddy initialization 2711 */ 2712 kho_memory_init(); 2713 2714 memblock_free_all(); 2715 mem_init(); 2716 kmem_cache_init(); 2717 /* 2718 * page_owner must be initialized after buddy is ready, and also after 2719 * slab is ready so that stack_depot_init() works properly 2720 */ 2721 page_ext_init_flatmem_late(); 2722 kmemleak_init(); 2723 ptlock_cache_init(); 2724 pgtable_cache_init(); 2725 debug_objects_mem_init(); 2726 vmalloc_init(); 2727 /* If no deferred init page_ext now, as vmap is fully initialized */ 2728 if (!deferred_struct_pages) 2729 page_ext_init(); 2730 /* Should be run before the first non-init thread is created */ 2731 init_espfix_bsp(); 2732 /* Should be run after espfix64 is set up. */ 2733 pti_init(); 2734 kmsan_init_runtime(); 2735 mm_cache_init(); 2736 execmem_init(); 2737}
~/
Read-only file preview.