FILE
memblock.c
File preview with syntax highlighting.
linux-6.19.14/mm/memblock.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Procedures for maintaining information about logical memory blocks. 4 * 5 * Peter Bergner, IBM Corp. June 2001. 6 * Copyright (C) 2001 Peter Bergner. 7 */ 8 9#include <linux/kernel.h> 10#include <linux/slab.h> 11#include <linux/init.h> 12#include <linux/bitops.h> 13#include <linux/poison.h> 14#include <linux/pfn.h> 15#include <linux/debugfs.h> 16#include <linux/kmemleak.h> 17#include <linux/seq_file.h> 18#include <linux/memblock.h> 19#include <linux/mutex.h> 20 21#ifdef CONFIG_KEXEC_HANDOVER 22#include <linux/libfdt.h> 23#include <linux/kexec_handover.h> 24#endif /* CONFIG_KEXEC_HANDOVER */ 25 26#include <asm/sections.h> 27#include <linux/io.h> 28 29#include "internal.h" 30 31#define INIT_MEMBLOCK_REGIONS 128 32#define INIT_PHYSMEM_REGIONS 4 33 34#ifndef INIT_MEMBLOCK_RESERVED_REGIONS 35# define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS 36#endif 37 38#ifndef INIT_MEMBLOCK_MEMORY_REGIONS 39#define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS 40#endif 41 42/** 43 * DOC: memblock overview 44 * 45 * Memblock is a method of managing memory regions during the early 46 * boot period when the usual kernel memory allocators are not up and 47 * running. 48 * 49 * Memblock views the system memory as collections of contiguous 50 * regions. There are several types of these collections: 51 * 52 * * ``memory`` - describes the physical memory available to the 53 * kernel; this may differ from the actual physical memory installed 54 * in the system, for instance when the memory is restricted with 55 * ``mem=`` command line parameter 56 * * ``reserved`` - describes the regions that were allocated 57 * * ``physmem`` - describes the actual physical memory available during 58 * boot regardless of the possible restrictions and memory hot(un)plug; 59 * the ``physmem`` type is only available on some architectures. 60 * 61 * Each region is represented by struct memblock_region that 62 * defines the region extents, its attributes and NUMA node id on NUMA 63 * systems. Every memory type is described by the struct memblock_type 64 * which contains an array of memory regions along with 65 * the allocator metadata. The "memory" and "reserved" types are nicely 66 * wrapped with struct memblock. This structure is statically 67 * initialized at build time. The region arrays are initially sized to 68 * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and 69 * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array 70 * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS. 71 * The memblock_allow_resize() enables automatic resizing of the region 72 * arrays during addition of new regions. This feature should be used 73 * with care so that memory allocated for the region array will not 74 * overlap with areas that should be reserved, for example initrd. 75 * 76 * The early architecture setup should tell memblock what the physical 77 * memory layout is by using memblock_add() or memblock_add_node() 78 * functions. The first function does not assign the region to a NUMA 79 * node and it is appropriate for UMA systems. Yet, it is possible to 80 * use it on NUMA systems as well and assign the region to a NUMA node 81 * later in the setup process using memblock_set_node(). The 82 * memblock_add_node() performs such an assignment directly. 83 * 84 * Once memblock is setup the memory can be allocated using one of the 85 * API variants: 86 * 87 * * memblock_phys_alloc*() - these functions return the **physical** 88 * address of the allocated memory 89 * * memblock_alloc*() - these functions return the **virtual** address 90 * of the allocated memory. 91 * 92 * Note, that both API variants use implicit assumptions about allowed 93 * memory ranges and the fallback methods. Consult the documentation 94 * of memblock_alloc_internal() and memblock_alloc_range_nid() 95 * functions for more elaborate description. 96 * 97 * As the system boot progresses, the architecture specific mem_init() 98 * function frees all the memory to the buddy page allocator. 99 * 100 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the 101 * memblock data structures (except "physmem") will be discarded after the 102 * system initialization completes. 103 */ 104 105#ifndef CONFIG_NUMA 106struct pglist_data __refdata contig_page_data; 107EXPORT_SYMBOL(contig_page_data); 108#endif 109 110unsigned long max_low_pfn; 111unsigned long min_low_pfn; 112unsigned long max_pfn; 113unsigned long long max_possible_pfn; 114 115#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 116/* When set to true, only allocate from MEMBLOCK_KHO_SCRATCH ranges */ 117static bool kho_scratch_only; 118#else 119#define kho_scratch_only false 120#endif 121 122static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock; 123static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock; 124#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 125static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS]; 126#endif 127 128struct memblock memblock __initdata_memblock = { 129 .memory.regions = memblock_memory_init_regions, 130 .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS, 131 .memory.name = "memory", 132 133 .reserved.regions = memblock_reserved_init_regions, 134 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS, 135 .reserved.name = "reserved", 136 137 .bottom_up = false, 138 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 139}; 140 141#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 142struct memblock_type physmem = { 143 .regions = memblock_physmem_init_regions, 144 .max = INIT_PHYSMEM_REGIONS, 145 .name = "physmem", 146}; 147#endif 148 149/* 150 * keep a pointer to &memblock.memory in the text section to use it in 151 * __next_mem_range() and its helpers. 152 * For architectures that do not keep memblock data after init, this 153 * pointer will be reset to NULL at memblock_discard() 154 */ 155static __refdata struct memblock_type *memblock_memory = &memblock.memory; 156 157#define for_each_memblock_type(i, memblock_type, rgn) \ 158 for (i = 0, rgn = &memblock_type->regions[0]; \ 159 i < memblock_type->cnt; \ 160 i++, rgn = &memblock_type->regions[i]) 161 162#define memblock_dbg(fmt, ...) \ 163 do { \ 164 if (memblock_debug) \ 165 pr_info(fmt, ##__VA_ARGS__); \ 166 } while (0) 167 168static int memblock_debug __initdata_memblock; 169static bool system_has_some_mirror __initdata_memblock; 170static int memblock_can_resize __initdata_memblock; 171static int memblock_memory_in_slab __initdata_memblock; 172static int memblock_reserved_in_slab __initdata_memblock; 173 174bool __init_memblock memblock_has_mirror(void) 175{ 176 return system_has_some_mirror; 177} 178 179static enum memblock_flags __init_memblock choose_memblock_flags(void) 180{ 181 /* skip non-scratch memory for kho early boot allocations */ 182 if (kho_scratch_only) 183 return MEMBLOCK_KHO_SCRATCH; 184 185 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; 186} 187 188/* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 189static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 190{ 191 return *size = min(*size, PHYS_ADDR_MAX - base); 192} 193 194/* 195 * Address comparison utilities 196 */ 197unsigned long __init_memblock 198memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, phys_addr_t base2, 199 phys_addr_t size2) 200{ 201 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 202} 203 204bool __init_memblock memblock_overlaps_region(struct memblock_type *type, 205 phys_addr_t base, phys_addr_t size) 206{ 207 unsigned long i; 208 209 memblock_cap_size(base, &size); 210 211 for (i = 0; i < type->cnt; i++) 212 if (memblock_addrs_overlap(base, size, type->regions[i].base, 213 type->regions[i].size)) 214 return true; 215 return false; 216} 217 218/** 219 * __memblock_find_range_bottom_up - find free area utility in bottom-up 220 * @start: start of candidate range 221 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 222 * %MEMBLOCK_ALLOC_ACCESSIBLE 223 * @size: size of free area to find 224 * @align: alignment of free area to find 225 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 226 * @flags: pick from blocks based on memory attributes 227 * 228 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 229 * 230 * Return: 231 * Found address on success, 0 on failure. 232 */ 233static phys_addr_t __init_memblock 234__memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 235 phys_addr_t size, phys_addr_t align, int nid, 236 enum memblock_flags flags) 237{ 238 phys_addr_t this_start, this_end, cand; 239 u64 i; 240 241 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) { 242 this_start = clamp(this_start, start, end); 243 this_end = clamp(this_end, start, end); 244 245 cand = round_up(this_start, align); 246 if (cand < this_end && this_end - cand >= size) 247 return cand; 248 } 249 250 return 0; 251} 252 253/** 254 * __memblock_find_range_top_down - find free area utility, in top-down 255 * @start: start of candidate range 256 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 257 * %MEMBLOCK_ALLOC_ACCESSIBLE 258 * @size: size of free area to find 259 * @align: alignment of free area to find 260 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 261 * @flags: pick from blocks based on memory attributes 262 * 263 * Utility called from memblock_find_in_range_node(), find free area top-down. 264 * 265 * Return: 266 * Found address on success, 0 on failure. 267 */ 268static phys_addr_t __init_memblock 269__memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 270 phys_addr_t size, phys_addr_t align, int nid, 271 enum memblock_flags flags) 272{ 273 phys_addr_t this_start, this_end, cand; 274 u64 i; 275 276 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end, 277 NULL) { 278 this_start = clamp(this_start, start, end); 279 this_end = clamp(this_end, start, end); 280 281 if (this_end < size) 282 continue; 283 284 cand = round_down(this_end - size, align); 285 if (cand >= this_start) 286 return cand; 287 } 288 289 return 0; 290} 291 292/** 293 * memblock_find_in_range_node - find free area in given range and node 294 * @size: size of free area to find 295 * @align: alignment of free area to find 296 * @start: start of candidate range 297 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 298 * %MEMBLOCK_ALLOC_ACCESSIBLE 299 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 300 * @flags: pick from blocks based on memory attributes 301 * 302 * Find @size free area aligned to @align in the specified range and node. 303 * 304 * Return: 305 * Found address on success, 0 on failure. 306 */ 307static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 308 phys_addr_t align, phys_addr_t start, 309 phys_addr_t end, int nid, 310 enum memblock_flags flags) 311{ 312 /* pump up @end */ 313 if (end == MEMBLOCK_ALLOC_ACCESSIBLE || 314 end == MEMBLOCK_ALLOC_NOLEAKTRACE) 315 end = memblock.current_limit; 316 317 /* avoid allocating the first page */ 318 start = max_t(phys_addr_t, start, PAGE_SIZE); 319 end = max(start, end); 320 321 if (memblock_bottom_up()) 322 return __memblock_find_range_bottom_up(start, end, size, align, 323 nid, flags); 324 else 325 return __memblock_find_range_top_down(start, end, size, align, 326 nid, flags); 327} 328 329/** 330 * memblock_find_in_range - find free area in given range 331 * @start: start of candidate range 332 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 333 * %MEMBLOCK_ALLOC_ACCESSIBLE 334 * @size: size of free area to find 335 * @align: alignment of free area to find 336 * 337 * Find @size free area aligned to @align in the specified range. 338 * 339 * Return: 340 * Found address on success, 0 on failure. 341 */ 342static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 343 phys_addr_t end, phys_addr_t size, 344 phys_addr_t align) 345{ 346 phys_addr_t ret; 347 enum memblock_flags flags = choose_memblock_flags(); 348 349again: 350 ret = memblock_find_in_range_node(size, align, start, end, 351 NUMA_NO_NODE, flags); 352 353 if (!ret && (flags & MEMBLOCK_MIRROR)) { 354 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 355 &size); 356 flags &= ~MEMBLOCK_MIRROR; 357 goto again; 358 } 359 360 return ret; 361} 362 363static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 364{ 365 type->total_size -= type->regions[r].size; 366 memmove(&type->regions[r], &type->regions[r + 1], 367 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 368 type->cnt--; 369 370 /* Special case for empty arrays */ 371 if (type->cnt == 0) { 372 WARN_ON(type->total_size != 0); 373 type->regions[0].base = 0; 374 type->regions[0].size = 0; 375 type->regions[0].flags = 0; 376 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 377 } 378} 379 380#ifndef CONFIG_ARCH_KEEP_MEMBLOCK 381/** 382 * memblock_discard - discard memory and reserved arrays if they were allocated 383 */ 384void __init memblock_discard(void) 385{ 386 phys_addr_t addr, size; 387 388 if (memblock.reserved.regions != memblock_reserved_init_regions) { 389 addr = __pa(memblock.reserved.regions); 390 size = PAGE_ALIGN(sizeof(struct memblock_region) * 391 memblock.reserved.max); 392 if (memblock_reserved_in_slab) 393 kfree(memblock.reserved.regions); 394 else 395 memblock_free_late(addr, size); 396 } 397 398 if (memblock.memory.regions != memblock_memory_init_regions) { 399 addr = __pa(memblock.memory.regions); 400 size = PAGE_ALIGN(sizeof(struct memblock_region) * 401 memblock.memory.max); 402 if (memblock_memory_in_slab) 403 kfree(memblock.memory.regions); 404 else 405 memblock_free_late(addr, size); 406 } 407 408 memblock_memory = NULL; 409} 410#endif 411 412/** 413 * memblock_double_array - double the size of the memblock regions array 414 * @type: memblock type of the regions array being doubled 415 * @new_area_start: starting address of memory range to avoid overlap with 416 * @new_area_size: size of memory range to avoid overlap with 417 * 418 * Double the size of the @type regions array. If memblock is being used to 419 * allocate memory for a new reserved regions array and there is a previously 420 * allocated memory range [@new_area_start, @new_area_start + @new_area_size] 421 * waiting to be reserved, ensure the memory used by the new array does 422 * not overlap. 423 * 424 * Return: 425 * 0 on success, -1 on failure. 426 */ 427static int __init_memblock memblock_double_array(struct memblock_type *type, 428 phys_addr_t new_area_start, 429 phys_addr_t new_area_size) 430{ 431 struct memblock_region *new_array, *old_array; 432 phys_addr_t old_alloc_size, new_alloc_size; 433 phys_addr_t old_size, new_size, addr, new_end; 434 int use_slab = slab_is_available(); 435 int *in_slab; 436 437 /* We don't allow resizing until we know about the reserved regions 438 * of memory that aren't suitable for allocation 439 */ 440 if (!memblock_can_resize) 441 panic("memblock: cannot resize %s array\n", type->name); 442 443 /* Calculate new doubled size */ 444 old_size = type->max * sizeof(struct memblock_region); 445 new_size = old_size << 1; 446 /* 447 * We need to allocated new one align to PAGE_SIZE, 448 * so we can free them completely later. 449 */ 450 old_alloc_size = PAGE_ALIGN(old_size); 451 new_alloc_size = PAGE_ALIGN(new_size); 452 453 /* Retrieve the slab flag */ 454 if (type == &memblock.memory) 455 in_slab = &memblock_memory_in_slab; 456 else 457 in_slab = &memblock_reserved_in_slab; 458 459 /* Try to find some space for it */ 460 if (use_slab) { 461 new_array = kmalloc(new_size, GFP_KERNEL); 462 addr = new_array ? __pa(new_array) : 0; 463 } else { 464 /* only exclude range when trying to double reserved.regions */ 465 if (type != &memblock.reserved) 466 new_area_start = new_area_size = 0; 467 468 addr = memblock_find_in_range(new_area_start + new_area_size, 469 memblock.current_limit, 470 new_alloc_size, PAGE_SIZE); 471 if (!addr && new_area_size) 472 addr = memblock_find_in_range(0, 473 min(new_area_start, memblock.current_limit), 474 new_alloc_size, PAGE_SIZE); 475 476 if (addr) { 477 /* The memory may not have been accepted, yet. */ 478 accept_memory(addr, new_alloc_size); 479 480 new_array = __va(addr); 481 } else { 482 new_array = NULL; 483 } 484 } 485 if (!addr) { 486 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 487 type->name, type->max, type->max * 2); 488 return -1; 489 } 490 491 new_end = addr + new_size - 1; 492 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]", 493 type->name, type->max * 2, &addr, &new_end); 494 495 /* 496 * Found space, we now need to move the array over before we add the 497 * reserved region since it may be our reserved array itself that is 498 * full. 499 */ 500 memcpy(new_array, type->regions, old_size); 501 memset(new_array + type->max, 0, old_size); 502 old_array = type->regions; 503 type->regions = new_array; 504 type->max <<= 1; 505 506 /* Free old array. We needn't free it if the array is the static one */ 507 if (*in_slab) 508 kfree(old_array); 509 else if (old_array != memblock_memory_init_regions && 510 old_array != memblock_reserved_init_regions) 511 memblock_free(old_array, old_alloc_size); 512 513 /* 514 * Reserve the new array if that comes from the memblock. Otherwise, we 515 * needn't do it 516 */ 517 if (!use_slab) 518 BUG_ON(memblock_reserve_kern(addr, new_alloc_size)); 519 520 /* Update slab flag */ 521 *in_slab = use_slab; 522 523 return 0; 524} 525 526/** 527 * memblock_merge_regions - merge neighboring compatible regions 528 * @type: memblock type to scan 529 * @start_rgn: start scanning from (@start_rgn - 1) 530 * @end_rgn: end scanning at (@end_rgn - 1) 531 * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn) 532 */ 533static void __init_memblock memblock_merge_regions(struct memblock_type *type, 534 unsigned long start_rgn, 535 unsigned long end_rgn) 536{ 537 int i = 0; 538 if (start_rgn) 539 i = start_rgn - 1; 540 end_rgn = min(end_rgn, type->cnt - 1); 541 while (i < end_rgn) { 542 struct memblock_region *this = &type->regions[i]; 543 struct memblock_region *next = &type->regions[i + 1]; 544 545 if (this->base + this->size != next->base || 546 memblock_get_region_node(this) != 547 memblock_get_region_node(next) || 548 this->flags != next->flags) { 549 BUG_ON(this->base + this->size > next->base); 550 i++; 551 continue; 552 } 553 554 this->size += next->size; 555 /* move forward from next + 1, index of which is i + 2 */ 556 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 557 type->cnt--; 558 end_rgn--; 559 } 560} 561 562/** 563 * memblock_insert_region - insert new memblock region 564 * @type: memblock type to insert into 565 * @idx: index for the insertion point 566 * @base: base address of the new region 567 * @size: size of the new region 568 * @nid: node id of the new region 569 * @flags: flags of the new region 570 * 571 * Insert new memblock region [@base, @base + @size) into @type at @idx. 572 * @type must already have extra room to accommodate the new region. 573 */ 574static void __init_memblock memblock_insert_region(struct memblock_type *type, 575 int idx, phys_addr_t base, 576 phys_addr_t size, 577 int nid, 578 enum memblock_flags flags) 579{ 580 struct memblock_region *rgn = &type->regions[idx]; 581 582 BUG_ON(type->cnt >= type->max); 583 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 584 rgn->base = base; 585 rgn->size = size; 586 rgn->flags = flags; 587 memblock_set_region_node(rgn, nid); 588 type->cnt++; 589 type->total_size += size; 590} 591 592/** 593 * memblock_add_range - add new memblock region 594 * @type: memblock type to add new region into 595 * @base: base address of the new region 596 * @size: size of the new region 597 * @nid: nid of the new region 598 * @flags: flags of the new region 599 * 600 * Add new memblock region [@base, @base + @size) into @type. The new region 601 * is allowed to overlap with existing ones - overlaps don't affect already 602 * existing regions. @type is guaranteed to be minimal (all neighbouring 603 * compatible regions are merged) after the addition. 604 * 605 * Return: 606 * 0 on success, -errno on failure. 607 */ 608static int __init_memblock memblock_add_range(struct memblock_type *type, 609 phys_addr_t base, phys_addr_t size, 610 int nid, enum memblock_flags flags) 611{ 612 bool insert = false; 613 phys_addr_t obase = base; 614 phys_addr_t end = base + memblock_cap_size(base, &size); 615 int idx, nr_new, start_rgn = -1, end_rgn; 616 struct memblock_region *rgn; 617 618 if (!size) 619 return 0; 620 621 /* special case for empty array */ 622 if (type->regions[0].size == 0) { 623 WARN_ON(type->cnt != 0 || type->total_size); 624 type->regions[0].base = base; 625 type->regions[0].size = size; 626 type->regions[0].flags = flags; 627 memblock_set_region_node(&type->regions[0], nid); 628 type->total_size = size; 629 type->cnt = 1; 630 return 0; 631 } 632 633 /* 634 * The worst case is when new range overlaps all existing regions, 635 * then we'll need type->cnt + 1 empty regions in @type. So if 636 * type->cnt * 2 + 1 is less than or equal to type->max, we know 637 * that there is enough empty regions in @type, and we can insert 638 * regions directly. 639 */ 640 if (type->cnt * 2 + 1 <= type->max) 641 insert = true; 642 643repeat: 644 /* 645 * The following is executed twice. Once with %false @insert and 646 * then with %true. The first counts the number of regions needed 647 * to accommodate the new area. The second actually inserts them. 648 */ 649 base = obase; 650 nr_new = 0; 651 652 for_each_memblock_type(idx, type, rgn) { 653 phys_addr_t rbase = rgn->base; 654 phys_addr_t rend = rbase + rgn->size; 655 656 if (rbase >= end) 657 break; 658 if (rend <= base) 659 continue; 660 /* 661 * @rgn overlaps. If it separates the lower part of new 662 * area, insert that portion. 663 */ 664 if (rbase > base) { 665#ifdef CONFIG_NUMA 666 WARN_ON(nid != memblock_get_region_node(rgn)); 667#endif 668 WARN_ON(flags != MEMBLOCK_NONE && flags != rgn->flags); 669 nr_new++; 670 if (insert) { 671 if (start_rgn == -1) 672 start_rgn = idx; 673 end_rgn = idx + 1; 674 memblock_insert_region(type, idx++, base, 675 rbase - base, nid, 676 flags); 677 } 678 } 679 /* area below @rend is dealt with, forget about it */ 680 base = min(rend, end); 681 } 682 683 /* insert the remaining portion */ 684 if (base < end) { 685 nr_new++; 686 if (insert) { 687 if (start_rgn == -1) 688 start_rgn = idx; 689 end_rgn = idx + 1; 690 memblock_insert_region(type, idx, base, end - base, 691 nid, flags); 692 } 693 } 694 695 if (!nr_new) 696 return 0; 697 698 /* 699 * If this was the first round, resize array and repeat for actual 700 * insertions; otherwise, merge and return. 701 */ 702 if (!insert) { 703 while (type->cnt + nr_new > type->max) 704 if (memblock_double_array(type, obase, size) < 0) 705 return -ENOMEM; 706 insert = true; 707 goto repeat; 708 } else { 709 memblock_merge_regions(type, start_rgn, end_rgn); 710 return 0; 711 } 712} 713 714/** 715 * memblock_add_node - add new memblock region within a NUMA node 716 * @base: base address of the new region 717 * @size: size of the new region 718 * @nid: nid of the new region 719 * @flags: flags of the new region 720 * 721 * Add new memblock region [@base, @base + @size) to the "memory" 722 * type. See memblock_add_range() description for mode details 723 * 724 * Return: 725 * 0 on success, -errno on failure. 726 */ 727int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 728 int nid, enum memblock_flags flags) 729{ 730 phys_addr_t end = base + size - 1; 731 732 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 733 &base, &end, nid, flags, (void *)_RET_IP_); 734 735 return memblock_add_range(&memblock.memory, base, size, nid, flags); 736} 737 738/** 739 * memblock_add - add new memblock region 740 * @base: base address of the new region 741 * @size: size of the new region 742 * 743 * Add new memblock region [@base, @base + @size) to the "memory" 744 * type. See memblock_add_range() description for mode details 745 * 746 * Return: 747 * 0 on success, -errno on failure. 748 */ 749int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 750{ 751 phys_addr_t end = base + size - 1; 752 753 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 754 &base, &end, (void *)_RET_IP_); 755 756 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0); 757} 758 759/** 760 * memblock_validate_numa_coverage - check if amount of memory with 761 * no node ID assigned is less than a threshold 762 * @threshold_bytes: maximal memory size that can have unassigned node 763 * ID (in bytes). 764 * 765 * A buggy firmware may report memory that does not belong to any node. 766 * Check if amount of such memory is below @threshold_bytes. 767 * 768 * Return: true on success, false on failure. 769 */ 770bool __init_memblock memblock_validate_numa_coverage(unsigned long threshold_bytes) 771{ 772 unsigned long nr_pages = 0; 773 unsigned long start_pfn, end_pfn, mem_size_mb; 774 int nid, i; 775 776 /* calculate lose page */ 777 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 778 if (!numa_valid_node(nid)) 779 nr_pages += end_pfn - start_pfn; 780 } 781 782 if ((nr_pages << PAGE_SHIFT) > threshold_bytes) { 783 mem_size_mb = memblock_phys_mem_size() / SZ_1M; 784 pr_err("NUMA: no nodes coverage for %luMB of %luMB RAM\n", 785 (nr_pages << PAGE_SHIFT) / SZ_1M, mem_size_mb); 786 return false; 787 } 788 789 return true; 790} 791 792 793/** 794 * memblock_isolate_range - isolate given range into disjoint memblocks 795 * @type: memblock type to isolate range for 796 * @base: base of range to isolate 797 * @size: size of range to isolate 798 * @start_rgn: out parameter for the start of isolated region 799 * @end_rgn: out parameter for the end of isolated region 800 * 801 * Walk @type and ensure that regions don't cross the boundaries defined by 802 * [@base, @base + @size). Crossing regions are split at the boundaries, 803 * which may create at most two more regions. The index of the first 804 * region inside the range is returned in *@start_rgn and the index of the 805 * first region after the range is returned in *@end_rgn. 806 * 807 * Return: 808 * 0 on success, -errno on failure. 809 */ 810static int __init_memblock memblock_isolate_range(struct memblock_type *type, 811 phys_addr_t base, phys_addr_t size, 812 int *start_rgn, int *end_rgn) 813{ 814 phys_addr_t end = base + memblock_cap_size(base, &size); 815 int idx; 816 struct memblock_region *rgn; 817 818 *start_rgn = *end_rgn = 0; 819 820 if (!size) 821 return 0; 822 823 /* we'll create at most two more regions */ 824 while (type->cnt + 2 > type->max) 825 if (memblock_double_array(type, base, size) < 0) 826 return -ENOMEM; 827 828 for_each_memblock_type(idx, type, rgn) { 829 phys_addr_t rbase = rgn->base; 830 phys_addr_t rend = rbase + rgn->size; 831 832 if (rbase >= end) 833 break; 834 if (rend <= base) 835 continue; 836 837 if (rbase < base) { 838 /* 839 * @rgn intersects from below. Split and continue 840 * to process the next region - the new top half. 841 */ 842 rgn->base = base; 843 rgn->size -= base - rbase; 844 type->total_size -= base - rbase; 845 memblock_insert_region(type, idx, rbase, base - rbase, 846 memblock_get_region_node(rgn), 847 rgn->flags); 848 } else if (rend > end) { 849 /* 850 * @rgn intersects from above. Split and redo the 851 * current region - the new bottom half. 852 */ 853 rgn->base = end; 854 rgn->size -= end - rbase; 855 type->total_size -= end - rbase; 856 memblock_insert_region(type, idx--, rbase, end - rbase, 857 memblock_get_region_node(rgn), 858 rgn->flags); 859 } else { 860 /* @rgn is fully contained, record it */ 861 if (!*end_rgn) 862 *start_rgn = idx; 863 *end_rgn = idx + 1; 864 } 865 } 866 867 return 0; 868} 869 870static int __init_memblock memblock_remove_range(struct memblock_type *type, 871 phys_addr_t base, phys_addr_t size) 872{ 873 int start_rgn, end_rgn; 874 int i, ret; 875 876 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 877 if (ret) 878 return ret; 879 880 for (i = end_rgn - 1; i >= start_rgn; i--) 881 memblock_remove_region(type, i); 882 return 0; 883} 884 885int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 886{ 887 phys_addr_t end = base + size - 1; 888 889 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 890 &base, &end, (void *)_RET_IP_); 891 892 return memblock_remove_range(&memblock.memory, base, size); 893} 894 895/** 896 * memblock_free - free boot memory allocation 897 * @ptr: starting address of the boot memory allocation 898 * @size: size of the boot memory block in bytes 899 * 900 * Free boot memory block previously allocated by memblock_alloc_xx() API. 901 * The freeing memory will not be released to the buddy allocator. 902 */ 903void __init_memblock memblock_free(void *ptr, size_t size) 904{ 905 if (ptr) 906 memblock_phys_free(__pa(ptr), size); 907} 908 909/** 910 * memblock_phys_free - free boot memory block 911 * @base: phys starting address of the boot memory block 912 * @size: size of the boot memory block in bytes 913 * 914 * Free boot memory block previously allocated by memblock_phys_alloc_xx() API. 915 * The freeing memory will not be released to the buddy allocator. 916 */ 917int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size) 918{ 919 phys_addr_t end = base + size - 1; 920 921 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 922 &base, &end, (void *)_RET_IP_); 923 924 kmemleak_free_part_phys(base, size); 925 return memblock_remove_range(&memblock.reserved, base, size); 926} 927 928int __init_memblock __memblock_reserve(phys_addr_t base, phys_addr_t size, 929 int nid, enum memblock_flags flags) 930{ 931 phys_addr_t end = base + size - 1; 932 933 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 934 &base, &end, nid, flags, (void *)_RET_IP_); 935 936 return memblock_add_range(&memblock.reserved, base, size, nid, flags); 937} 938 939#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 940int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) 941{ 942 phys_addr_t end = base + size - 1; 943 944 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 945 &base, &end, (void *)_RET_IP_); 946 947 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0); 948} 949#endif 950 951#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH 952__init void memblock_set_kho_scratch_only(void) 953{ 954 kho_scratch_only = true; 955} 956 957__init void memblock_clear_kho_scratch_only(void) 958{ 959 kho_scratch_only = false; 960} 961 962__init void memmap_init_kho_scratch_pages(void) 963{ 964 phys_addr_t start, end; 965 unsigned long pfn; 966 int nid; 967 u64 i; 968 969 if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) 970 return; 971 972 /* 973 * Initialize struct pages for free scratch memory. 974 * The struct pages for reserved scratch memory will be set up in 975 * reserve_bootmem_region() 976 */ 977 __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE, 978 MEMBLOCK_KHO_SCRATCH, &start, &end, &nid) { 979 for (pfn = PFN_UP(start); pfn < PFN_DOWN(end); pfn++) 980 init_deferred_page(pfn, nid); 981 } 982} 983#endif 984 985/** 986 * memblock_setclr_flag - set or clear flag for a memory region 987 * @type: memblock type to set/clear flag for 988 * @base: base address of the region 989 * @size: size of the region 990 * @set: set or clear the flag 991 * @flag: the flag to update 992 * 993 * This function isolates region [@base, @base + @size), and sets/clears flag 994 * 995 * Return: 0 on success, -errno on failure. 996 */ 997static int __init_memblock memblock_setclr_flag(struct memblock_type *type, 998 phys_addr_t base, phys_addr_t size, int set, int flag) 999{ 1000 int i, ret, start_rgn, end_rgn; 1001 1002 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1003 if (ret) 1004 return ret; 1005 1006 for (i = start_rgn; i < end_rgn; i++) { 1007 struct memblock_region *r = &type->regions[i]; 1008 1009 if (set) 1010 r->flags |= flag; 1011 else 1012 r->flags &= ~flag; 1013 } 1014 1015 memblock_merge_regions(type, start_rgn, end_rgn); 1016 return 0; 1017} 1018 1019/** 1020 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 1021 * @base: the base phys addr of the region 1022 * @size: the size of the region 1023 * 1024 * Return: 0 on success, -errno on failure. 1025 */ 1026int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 1027{ 1028 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_HOTPLUG); 1029} 1030 1031/** 1032 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 1033 * @base: the base phys addr of the region 1034 * @size: the size of the region 1035 * 1036 * Return: 0 on success, -errno on failure. 1037 */ 1038int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 1039{ 1040 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_HOTPLUG); 1041} 1042 1043/** 1044 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR. 1045 * @base: the base phys addr of the region 1046 * @size: the size of the region 1047 * 1048 * Return: 0 on success, -errno on failure. 1049 */ 1050int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size) 1051{ 1052 if (!mirrored_kernelcore) 1053 return 0; 1054 1055 system_has_some_mirror = true; 1056 1057 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_MIRROR); 1058} 1059 1060/** 1061 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP. 1062 * @base: the base phys addr of the region 1063 * @size: the size of the region 1064 * 1065 * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the 1066 * direct mapping of the physical memory. These regions will still be 1067 * covered by the memory map. The struct page representing NOMAP memory 1068 * frames in the memory map will be PageReserved() 1069 * 1070 * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from 1071 * memblock, the caller must inform kmemleak to ignore that memory 1072 * 1073 * Return: 0 on success, -errno on failure. 1074 */ 1075int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size) 1076{ 1077 return memblock_setclr_flag(&memblock.memory, base, size, 1, MEMBLOCK_NOMAP); 1078} 1079 1080/** 1081 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region. 1082 * @base: the base phys addr of the region 1083 * @size: the size of the region 1084 * 1085 * Return: 0 on success, -errno on failure. 1086 */ 1087int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size) 1088{ 1089 return memblock_setclr_flag(&memblock.memory, base, size, 0, MEMBLOCK_NOMAP); 1090} 1091 1092/** 1093 * memblock_reserved_mark_noinit - Mark a reserved memory region with flag 1094 * MEMBLOCK_RSRV_NOINIT 1095 * 1096 * @base: the base phys addr of the region 1097 * @size: the size of the region 1098 * 1099 * The struct pages for the reserved regions marked %MEMBLOCK_RSRV_NOINIT will 1100 * not be fully initialized to allow the caller optimize their initialization. 1101 * 1102 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled, setting this flag 1103 * completely bypasses the initialization of struct pages for such region. 1104 * 1105 * When %CONFIG_DEFERRED_STRUCT_PAGE_INIT is disabled, struct pages in this 1106 * region will be initialized with default values but won't be marked as 1107 * reserved. 1108 * 1109 * Return: 0 on success, -errno on failure. 1110 */ 1111int __init_memblock memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size) 1112{ 1113 return memblock_setclr_flag(&memblock.reserved, base, size, 1, 1114 MEMBLOCK_RSRV_NOINIT); 1115} 1116 1117/** 1118 * memblock_mark_kho_scratch - Mark a memory region as MEMBLOCK_KHO_SCRATCH. 1119 * @base: the base phys addr of the region 1120 * @size: the size of the region 1121 * 1122 * Only memory regions marked with %MEMBLOCK_KHO_SCRATCH will be considered 1123 * for allocations during early boot with kexec handover. 1124 * 1125 * Return: 0 on success, -errno on failure. 1126 */ 1127__init int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size) 1128{ 1129 return memblock_setclr_flag(&memblock.memory, base, size, 1, 1130 MEMBLOCK_KHO_SCRATCH); 1131} 1132 1133/** 1134 * memblock_clear_kho_scratch - Clear MEMBLOCK_KHO_SCRATCH flag for a 1135 * specified region. 1136 * @base: the base phys addr of the region 1137 * @size: the size of the region 1138 * 1139 * Return: 0 on success, -errno on failure. 1140 */ 1141__init int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size) 1142{ 1143 return memblock_setclr_flag(&memblock.memory, base, size, 0, 1144 MEMBLOCK_KHO_SCRATCH); 1145} 1146 1147static bool should_skip_region(struct memblock_type *type, 1148 struct memblock_region *m, 1149 int nid, int flags) 1150{ 1151 int m_nid = memblock_get_region_node(m); 1152 1153 /* we never skip regions when iterating memblock.reserved or physmem */ 1154 if (type != memblock_memory) 1155 return false; 1156 1157 /* only memory regions are associated with nodes, check it */ 1158 if (numa_valid_node(nid) && nid != m_nid) 1159 return true; 1160 1161 /* skip hotpluggable memory regions if needed */ 1162 if (movable_node_is_enabled() && memblock_is_hotpluggable(m) && 1163 !(flags & MEMBLOCK_HOTPLUG)) 1164 return true; 1165 1166 /* if we want mirror memory skip non-mirror memory regions */ 1167 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 1168 return true; 1169 1170 /* skip nomap memory unless we were asked for it explicitly */ 1171 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 1172 return true; 1173 1174 /* skip driver-managed memory unless we were asked for it explicitly */ 1175 if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m)) 1176 return true; 1177 1178 /* 1179 * In early alloc during kexec handover, we can only consider 1180 * MEMBLOCK_KHO_SCRATCH regions for the allocations 1181 */ 1182 if ((flags & MEMBLOCK_KHO_SCRATCH) && !memblock_is_kho_scratch(m)) 1183 return true; 1184 1185 return false; 1186} 1187 1188/** 1189 * __next_mem_range - next function for for_each_free_mem_range() etc. 1190 * @idx: pointer to u64 loop variable 1191 * @nid: node selector, %NUMA_NO_NODE for all nodes 1192 * @flags: pick from blocks based on memory attributes 1193 * @type_a: pointer to memblock_type from where the range is taken 1194 * @type_b: pointer to memblock_type which excludes memory from being taken 1195 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1196 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1197 * @out_nid: ptr to int for nid of the range, can be %NULL 1198 * 1199 * Find the first area from *@idx which matches @nid, fill the out 1200 * parameters, and update *@idx for the next iteration. The lower 32bit of 1201 * *@idx contains index into type_a and the upper 32bit indexes the 1202 * areas before each region in type_b. For example, if type_b regions 1203 * look like the following, 1204 * 1205 * 0:[0-16), 1:[32-48), 2:[128-130) 1206 * 1207 * The upper 32bit indexes the following regions. 1208 * 1209 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 1210 * 1211 * As both region arrays are sorted, the function advances the two indices 1212 * in lockstep and returns each intersection. 1213 */ 1214void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags, 1215 struct memblock_type *type_a, 1216 struct memblock_type *type_b, phys_addr_t *out_start, 1217 phys_addr_t *out_end, int *out_nid) 1218{ 1219 int idx_a = *idx & 0xffffffff; 1220 int idx_b = *idx >> 32; 1221 1222 for (; idx_a < type_a->cnt; idx_a++) { 1223 struct memblock_region *m = &type_a->regions[idx_a]; 1224 1225 phys_addr_t m_start = m->base; 1226 phys_addr_t m_end = m->base + m->size; 1227 int m_nid = memblock_get_region_node(m); 1228 1229 if (should_skip_region(type_a, m, nid, flags)) 1230 continue; 1231 1232 if (!type_b) { 1233 if (out_start) 1234 *out_start = m_start; 1235 if (out_end) 1236 *out_end = m_end; 1237 if (out_nid) 1238 *out_nid = m_nid; 1239 idx_a++; 1240 *idx = (u32)idx_a | (u64)idx_b << 32; 1241 return; 1242 } 1243 1244 /* scan areas before each reservation */ 1245 for (; idx_b < type_b->cnt + 1; idx_b++) { 1246 struct memblock_region *r; 1247 phys_addr_t r_start; 1248 phys_addr_t r_end; 1249 1250 r = &type_b->regions[idx_b]; 1251 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1252 r_end = idx_b < type_b->cnt ? 1253 r->base : PHYS_ADDR_MAX; 1254 1255 /* 1256 * if idx_b advanced past idx_a, 1257 * break out to advance idx_a 1258 */ 1259 if (r_start >= m_end) 1260 break; 1261 /* if the two regions intersect, we're done */ 1262 if (m_start < r_end) { 1263 if (out_start) 1264 *out_start = 1265 max(m_start, r_start); 1266 if (out_end) 1267 *out_end = min(m_end, r_end); 1268 if (out_nid) 1269 *out_nid = m_nid; 1270 /* 1271 * The region which ends first is 1272 * advanced for the next iteration. 1273 */ 1274 if (m_end <= r_end) 1275 idx_a++; 1276 else 1277 idx_b++; 1278 *idx = (u32)idx_a | (u64)idx_b << 32; 1279 return; 1280 } 1281 } 1282 } 1283 1284 /* signal end of iteration */ 1285 *idx = ULLONG_MAX; 1286} 1287 1288/** 1289 * __next_mem_range_rev - generic next function for for_each_*_range_rev() 1290 * 1291 * @idx: pointer to u64 loop variable 1292 * @nid: node selector, %NUMA_NO_NODE for all nodes 1293 * @flags: pick from blocks based on memory attributes 1294 * @type_a: pointer to memblock_type from where the range is taken 1295 * @type_b: pointer to memblock_type which excludes memory from being taken 1296 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1297 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1298 * @out_nid: ptr to int for nid of the range, can be %NULL 1299 * 1300 * Finds the next range from type_a which is not marked as unsuitable 1301 * in type_b. 1302 * 1303 * Reverse of __next_mem_range(). 1304 */ 1305void __init_memblock __next_mem_range_rev(u64 *idx, int nid, 1306 enum memblock_flags flags, 1307 struct memblock_type *type_a, 1308 struct memblock_type *type_b, 1309 phys_addr_t *out_start, 1310 phys_addr_t *out_end, int *out_nid) 1311{ 1312 int idx_a = *idx & 0xffffffff; 1313 int idx_b = *idx >> 32; 1314 1315 if (*idx == (u64)ULLONG_MAX) { 1316 idx_a = type_a->cnt - 1; 1317 if (type_b != NULL) 1318 idx_b = type_b->cnt; 1319 else 1320 idx_b = 0; 1321 } 1322 1323 for (; idx_a >= 0; idx_a--) { 1324 struct memblock_region *m = &type_a->regions[idx_a]; 1325 1326 phys_addr_t m_start = m->base; 1327 phys_addr_t m_end = m->base + m->size; 1328 int m_nid = memblock_get_region_node(m); 1329 1330 if (should_skip_region(type_a, m, nid, flags)) 1331 continue; 1332 1333 if (!type_b) { 1334 if (out_start) 1335 *out_start = m_start; 1336 if (out_end) 1337 *out_end = m_end; 1338 if (out_nid) 1339 *out_nid = m_nid; 1340 idx_a--; 1341 *idx = (u32)idx_a | (u64)idx_b << 32; 1342 return; 1343 } 1344 1345 /* scan areas before each reservation */ 1346 for (; idx_b >= 0; idx_b--) { 1347 struct memblock_region *r; 1348 phys_addr_t r_start; 1349 phys_addr_t r_end; 1350 1351 r = &type_b->regions[idx_b]; 1352 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1353 r_end = idx_b < type_b->cnt ? 1354 r->base : PHYS_ADDR_MAX; 1355 /* 1356 * if idx_b advanced past idx_a, 1357 * break out to advance idx_a 1358 */ 1359 1360 if (r_end <= m_start) 1361 break; 1362 /* if the two regions intersect, we're done */ 1363 if (m_end > r_start) { 1364 if (out_start) 1365 *out_start = max(m_start, r_start); 1366 if (out_end) 1367 *out_end = min(m_end, r_end); 1368 if (out_nid) 1369 *out_nid = m_nid; 1370 if (m_start >= r_start) 1371 idx_a--; 1372 else 1373 idx_b--; 1374 *idx = (u32)idx_a | (u64)idx_b << 32; 1375 return; 1376 } 1377 } 1378 } 1379 /* signal end of iteration */ 1380 *idx = ULLONG_MAX; 1381} 1382 1383/* 1384 * Common iterator interface used to define for_each_mem_pfn_range(). 1385 */ 1386void __init_memblock __next_mem_pfn_range(int *idx, int nid, 1387 unsigned long *out_start_pfn, 1388 unsigned long *out_end_pfn, int *out_nid) 1389{ 1390 struct memblock_type *type = &memblock.memory; 1391 struct memblock_region *r; 1392 int r_nid; 1393 1394 while (++*idx < type->cnt) { 1395 r = &type->regions[*idx]; 1396 r_nid = memblock_get_region_node(r); 1397 1398 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 1399 continue; 1400 if (!numa_valid_node(nid) || nid == r_nid) 1401 break; 1402 } 1403 if (*idx >= type->cnt) { 1404 *idx = -1; 1405 return; 1406 } 1407 1408 if (out_start_pfn) 1409 *out_start_pfn = PFN_UP(r->base); 1410 if (out_end_pfn) 1411 *out_end_pfn = PFN_DOWN(r->base + r->size); 1412 if (out_nid) 1413 *out_nid = r_nid; 1414} 1415 1416/** 1417 * memblock_set_node - set node ID on memblock regions 1418 * @base: base of area to set node ID for 1419 * @size: size of area to set node ID for 1420 * @type: memblock type to set node ID for 1421 * @nid: node ID to set 1422 * 1423 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid. 1424 * Regions which cross the area boundaries are split as necessary. 1425 * 1426 * Return: 1427 * 0 on success, -errno on failure. 1428 */ 1429int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 1430 struct memblock_type *type, int nid) 1431{ 1432#ifdef CONFIG_NUMA 1433 int start_rgn, end_rgn; 1434 int i, ret; 1435 1436 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1437 if (ret) 1438 return ret; 1439 1440 for (i = start_rgn; i < end_rgn; i++) 1441 memblock_set_region_node(&type->regions[i], nid); 1442 1443 memblock_merge_regions(type, start_rgn, end_rgn); 1444#endif 1445 return 0; 1446} 1447 1448/** 1449 * memblock_alloc_range_nid - allocate boot memory block 1450 * @size: size of memory block to be allocated in bytes 1451 * @align: alignment of the region and block's size 1452 * @start: the lower bound of the memory region to allocate (phys address) 1453 * @end: the upper bound of the memory region to allocate (phys address) 1454 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1455 * @exact_nid: control the allocation fall back to other nodes 1456 * 1457 * The allocation is performed from memory region limited by 1458 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE. 1459 * 1460 * If the specified node can not hold the requested memory and @exact_nid 1461 * is false, the allocation falls back to any node in the system. 1462 * 1463 * For systems with memory mirroring, the allocation is attempted first 1464 * from the regions with mirroring enabled and then retried from any 1465 * memory region. 1466 * 1467 * In addition, function using kmemleak_alloc_phys for allocated boot 1468 * memory block, it is never reported as leaks. 1469 * 1470 * Return: 1471 * Physical address of allocated memory block on success, %0 on failure. 1472 */ 1473phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, 1474 phys_addr_t align, phys_addr_t start, 1475 phys_addr_t end, int nid, 1476 bool exact_nid) 1477{ 1478 enum memblock_flags flags = choose_memblock_flags(); 1479 phys_addr_t found; 1480 1481 /* 1482 * Detect any accidental use of these APIs after slab is ready, as at 1483 * this moment memblock may be deinitialized already and its 1484 * internal data may be destroyed (after execution of memblock_free_all) 1485 */ 1486 if (WARN_ON_ONCE(slab_is_available())) { 1487 void *vaddr = kzalloc_node(size, GFP_NOWAIT, nid); 1488 1489 return vaddr ? virt_to_phys(vaddr) : 0; 1490 } 1491 1492 if (!align) { 1493 /* Can't use WARNs this early in boot on powerpc */ 1494 dump_stack(); 1495 align = SMP_CACHE_BYTES; 1496 } 1497 1498again: 1499 found = memblock_find_in_range_node(size, align, start, end, nid, 1500 flags); 1501 if (found && !__memblock_reserve(found, size, nid, MEMBLOCK_RSRV_KERN)) 1502 goto done; 1503 1504 if (numa_valid_node(nid) && !exact_nid) { 1505 found = memblock_find_in_range_node(size, align, start, 1506 end, NUMA_NO_NODE, 1507 flags); 1508 if (found && !memblock_reserve_kern(found, size)) 1509 goto done; 1510 } 1511 1512 if (flags & MEMBLOCK_MIRROR) { 1513 flags &= ~MEMBLOCK_MIRROR; 1514 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 1515 &size); 1516 goto again; 1517 } 1518 1519 return 0; 1520 1521done: 1522 /* 1523 * Skip kmemleak for those places like kasan_init() and 1524 * early_pgtable_alloc() due to high volume. 1525 */ 1526 if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) 1527 /* 1528 * Memblock allocated blocks are never reported as 1529 * leaks. This is because many of these blocks are 1530 * only referred via the physical address which is 1531 * not looked up by kmemleak. 1532 */ 1533 kmemleak_alloc_phys(found, size, 0); 1534 1535 /* 1536 * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, 1537 * require memory to be accepted before it can be used by the 1538 * guest. 1539 * 1540 * Accept the memory of the allocated buffer. 1541 */ 1542 accept_memory(found, size); 1543 1544 return found; 1545} 1546 1547/** 1548 * memblock_phys_alloc_range - allocate a memory block inside specified range 1549 * @size: size of memory block to be allocated in bytes 1550 * @align: alignment of the region and block's size 1551 * @start: the lower bound of the memory region to allocate (physical address) 1552 * @end: the upper bound of the memory region to allocate (physical address) 1553 * 1554 * Allocate @size bytes in the between @start and @end. 1555 * 1556 * Return: physical address of the allocated memory block on success, 1557 * %0 on failure. 1558 */ 1559phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, 1560 phys_addr_t align, 1561 phys_addr_t start, 1562 phys_addr_t end) 1563{ 1564 memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n", 1565 __func__, (u64)size, (u64)align, &start, &end, 1566 (void *)_RET_IP_); 1567 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, 1568 false); 1569} 1570 1571/** 1572 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node 1573 * @size: size of memory block to be allocated in bytes 1574 * @align: alignment of the region and block's size 1575 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1576 * 1577 * Allocates memory block from the specified NUMA node. If the node 1578 * has no available memory, attempts to allocated from any node in the 1579 * system. 1580 * 1581 * Return: physical address of the allocated memory block on success, 1582 * %0 on failure. 1583 */ 1584phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1585{ 1586 return memblock_alloc_range_nid(size, align, 0, 1587 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false); 1588} 1589 1590/** 1591 * memblock_alloc_internal - allocate boot memory block 1592 * @size: size of memory block to be allocated in bytes 1593 * @align: alignment of the region and block's size 1594 * @min_addr: the lower bound of the memory region to allocate (phys address) 1595 * @max_addr: the upper bound of the memory region to allocate (phys address) 1596 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1597 * @exact_nid: control the allocation fall back to other nodes 1598 * 1599 * Allocates memory block using memblock_alloc_range_nid() and 1600 * converts the returned physical address to virtual. 1601 * 1602 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1603 * will fall back to memory below @min_addr. Other constraints, such 1604 * as node and mirrored memory will be handled again in 1605 * memblock_alloc_range_nid(). 1606 * 1607 * Return: 1608 * Virtual address of allocated memory block on success, NULL on failure. 1609 */ 1610static void * __init memblock_alloc_internal( 1611 phys_addr_t size, phys_addr_t align, 1612 phys_addr_t min_addr, phys_addr_t max_addr, 1613 int nid, bool exact_nid) 1614{ 1615 phys_addr_t alloc; 1616 1617 1618 if (max_addr > memblock.current_limit) 1619 max_addr = memblock.current_limit; 1620 1621 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid, 1622 exact_nid); 1623 1624 /* retry allocation without lower limit */ 1625 if (!alloc && min_addr) 1626 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid, 1627 exact_nid); 1628 1629 if (!alloc) 1630 return NULL; 1631 1632 return phys_to_virt(alloc); 1633} 1634 1635/** 1636 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node 1637 * without zeroing memory 1638 * @size: size of memory block to be allocated in bytes 1639 * @align: alignment of the region and block's size 1640 * @min_addr: the lower bound of the memory region from where the allocation 1641 * is preferred (phys address) 1642 * @max_addr: the upper bound of the memory region from where the allocation 1643 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1644 * allocate only from memory limited by memblock.current_limit value 1645 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1646 * 1647 * Public function, provides additional debug information (including caller 1648 * info), if enabled. Does not zero allocated memory. 1649 * 1650 * Return: 1651 * Virtual address of allocated memory block on success, NULL on failure. 1652 */ 1653void * __init memblock_alloc_exact_nid_raw( 1654 phys_addr_t size, phys_addr_t align, 1655 phys_addr_t min_addr, phys_addr_t max_addr, 1656 int nid) 1657{ 1658 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1659 __func__, (u64)size, (u64)align, nid, &min_addr, 1660 &max_addr, (void *)_RET_IP_); 1661 1662 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1663 true); 1664} 1665 1666/** 1667 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing 1668 * memory and without panicking 1669 * @size: size of memory block to be allocated in bytes 1670 * @align: alignment of the region and block's size 1671 * @min_addr: the lower bound of the memory region from where the allocation 1672 * is preferred (phys address) 1673 * @max_addr: the upper bound of the memory region from where the allocation 1674 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1675 * allocate only from memory limited by memblock.current_limit value 1676 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1677 * 1678 * Public function, provides additional debug information (including caller 1679 * info), if enabled. Does not zero allocated memory, does not panic if request 1680 * cannot be satisfied. 1681 * 1682 * Return: 1683 * Virtual address of allocated memory block on success, NULL on failure. 1684 */ 1685void * __init memblock_alloc_try_nid_raw( 1686 phys_addr_t size, phys_addr_t align, 1687 phys_addr_t min_addr, phys_addr_t max_addr, 1688 int nid) 1689{ 1690 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1691 __func__, (u64)size, (u64)align, nid, &min_addr, 1692 &max_addr, (void *)_RET_IP_); 1693 1694 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1695 false); 1696} 1697 1698/** 1699 * memblock_alloc_try_nid - allocate boot memory block 1700 * @size: size of memory block to be allocated in bytes 1701 * @align: alignment of the region and block's size 1702 * @min_addr: the lower bound of the memory region from where the allocation 1703 * is preferred (phys address) 1704 * @max_addr: the upper bound of the memory region from where the allocation 1705 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1706 * allocate only from memory limited by memblock.current_limit value 1707 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1708 * 1709 * Public function, provides additional debug information (including caller 1710 * info), if enabled. This function zeroes the allocated memory. 1711 * 1712 * Return: 1713 * Virtual address of allocated memory block on success, NULL on failure. 1714 */ 1715void * __init memblock_alloc_try_nid( 1716 phys_addr_t size, phys_addr_t align, 1717 phys_addr_t min_addr, phys_addr_t max_addr, 1718 int nid) 1719{ 1720 void *ptr; 1721 1722 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1723 __func__, (u64)size, (u64)align, nid, &min_addr, 1724 &max_addr, (void *)_RET_IP_); 1725 ptr = memblock_alloc_internal(size, align, 1726 min_addr, max_addr, nid, false); 1727 if (ptr) 1728 memset(ptr, 0, size); 1729 1730 return ptr; 1731} 1732 1733/** 1734 * __memblock_alloc_or_panic - Try to allocate memory and panic on failure 1735 * @size: size of memory block to be allocated in bytes 1736 * @align: alignment of the region and block's size 1737 * @func: caller func name 1738 * 1739 * This function attempts to allocate memory using memblock_alloc, 1740 * and in case of failure, it calls panic with the formatted message. 1741 * This function should not be used directly, please use the macro memblock_alloc_or_panic. 1742 */ 1743void *__init __memblock_alloc_or_panic(phys_addr_t size, phys_addr_t align, 1744 const char *func) 1745{ 1746 void *addr = memblock_alloc(size, align); 1747 1748 if (unlikely(!addr)) 1749 panic("%s: Failed to allocate %pap bytes\n", func, &size); 1750 return addr; 1751} 1752 1753/** 1754 * memblock_free_late - free pages directly to buddy allocator 1755 * @base: phys starting address of the boot memory block 1756 * @size: size of the boot memory block in bytes 1757 * 1758 * This is only useful when the memblock allocator has already been torn 1759 * down, but we are still initializing the system. Pages are released directly 1760 * to the buddy allocator. 1761 */ 1762void __init memblock_free_late(phys_addr_t base, phys_addr_t size) 1763{ 1764 phys_addr_t cursor, end; 1765 1766 end = base + size - 1; 1767 memblock_dbg("%s: [%pa-%pa] %pS\n", 1768 __func__, &base, &end, (void *)_RET_IP_); 1769 kmemleak_free_part_phys(base, size); 1770 cursor = PFN_UP(base); 1771 end = PFN_DOWN(base + size); 1772 1773 for (; cursor < end; cursor++) { 1774 memblock_free_pages(pfn_to_page(cursor), cursor, 0); 1775 totalram_pages_inc(); 1776 } 1777} 1778 1779/* 1780 * Remaining API functions 1781 */ 1782 1783phys_addr_t __init_memblock memblock_phys_mem_size(void) 1784{ 1785 return memblock.memory.total_size; 1786} 1787 1788phys_addr_t __init_memblock memblock_reserved_size(void) 1789{ 1790 return memblock.reserved.total_size; 1791} 1792 1793phys_addr_t __init_memblock memblock_reserved_kern_size(phys_addr_t limit, int nid) 1794{ 1795 struct memblock_region *r; 1796 phys_addr_t total = 0; 1797 1798 for_each_reserved_mem_region(r) { 1799 phys_addr_t size = r->size; 1800 1801 if (r->base > limit) 1802 break; 1803 1804 if (r->base + r->size > limit) 1805 size = limit - r->base; 1806 1807 if (nid == memblock_get_region_node(r) || !numa_valid_node(nid)) 1808 if (r->flags & MEMBLOCK_RSRV_KERN) 1809 total += size; 1810 } 1811 1812 return total; 1813} 1814 1815/** 1816 * memblock_estimated_nr_free_pages - return estimated number of free pages 1817 * from memblock point of view 1818 * 1819 * During bootup, subsystems might need a rough estimate of the number of free 1820 * pages in the whole system, before precise numbers are available from the 1821 * buddy. Especially with CONFIG_DEFERRED_STRUCT_PAGE_INIT, the numbers 1822 * obtained from the buddy might be very imprecise during bootup. 1823 * 1824 * Return: 1825 * An estimated number of free pages from memblock point of view. 1826 */ 1827unsigned long __init memblock_estimated_nr_free_pages(void) 1828{ 1829 return PHYS_PFN(memblock_phys_mem_size() - 1830 memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, NUMA_NO_NODE)); 1831} 1832 1833/* lowest address */ 1834phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1835{ 1836 return memblock.memory.regions[0].base; 1837} 1838 1839phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1840{ 1841 int idx = memblock.memory.cnt - 1; 1842 1843 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1844} 1845 1846static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) 1847{ 1848 phys_addr_t max_addr = PHYS_ADDR_MAX; 1849 struct memblock_region *r; 1850 1851 /* 1852 * translate the memory @limit size into the max address within one of 1853 * the memory memblock regions, if the @limit exceeds the total size 1854 * of those regions, max_addr will keep original value PHYS_ADDR_MAX 1855 */ 1856 for_each_mem_region(r) { 1857 if (limit <= r->size) { 1858 max_addr = r->base + limit; 1859 break; 1860 } 1861 limit -= r->size; 1862 } 1863 1864 return max_addr; 1865} 1866 1867void __init memblock_enforce_memory_limit(phys_addr_t limit) 1868{ 1869 phys_addr_t max_addr; 1870 1871 if (!limit) 1872 return; 1873 1874 max_addr = __find_max_addr(limit); 1875 1876 /* @limit exceeds the total size of the memory, do nothing */ 1877 if (max_addr == PHYS_ADDR_MAX) 1878 return; 1879 1880 /* truncate both memory and reserved regions */ 1881 memblock_remove_range(&memblock.memory, max_addr, 1882 PHYS_ADDR_MAX); 1883 memblock_remove_range(&memblock.reserved, max_addr, 1884 PHYS_ADDR_MAX); 1885} 1886 1887void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size) 1888{ 1889 int start_rgn, end_rgn; 1890 int i, ret; 1891 1892 if (!size) 1893 return; 1894 1895 if (!memblock_memory->total_size) { 1896 pr_warn("%s: No memory registered yet\n", __func__); 1897 return; 1898 } 1899 1900 ret = memblock_isolate_range(&memblock.memory, base, size, 1901 &start_rgn, &end_rgn); 1902 if (ret) 1903 return; 1904 1905 /* remove all the MAP regions */ 1906 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--) 1907 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1908 memblock_remove_region(&memblock.memory, i); 1909 1910 for (i = start_rgn - 1; i >= 0; i--) 1911 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1912 memblock_remove_region(&memblock.memory, i); 1913 1914 /* truncate the reserved regions */ 1915 memblock_remove_range(&memblock.reserved, 0, base); 1916 memblock_remove_range(&memblock.reserved, 1917 base + size, PHYS_ADDR_MAX); 1918} 1919 1920void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1921{ 1922 phys_addr_t max_addr; 1923 1924 if (!limit) 1925 return; 1926 1927 max_addr = __find_max_addr(limit); 1928 1929 /* @limit exceeds the total size of the memory, do nothing */ 1930 if (max_addr == PHYS_ADDR_MAX) 1931 return; 1932 1933 memblock_cap_memory_range(0, max_addr); 1934} 1935 1936static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1937{ 1938 unsigned int left = 0, right = type->cnt; 1939 1940 do { 1941 unsigned int mid = (right + left) / 2; 1942 1943 if (addr < type->regions[mid].base) 1944 right = mid; 1945 else if (addr >= (type->regions[mid].base + 1946 type->regions[mid].size)) 1947 left = mid + 1; 1948 else 1949 return mid; 1950 } while (left < right); 1951 return -1; 1952} 1953 1954bool __init_memblock memblock_is_reserved(phys_addr_t addr) 1955{ 1956 return memblock_search(&memblock.reserved, addr) != -1; 1957} 1958 1959bool __init_memblock memblock_is_memory(phys_addr_t addr) 1960{ 1961 return memblock_search(&memblock.memory, addr) != -1; 1962} 1963 1964bool __init_memblock memblock_is_map_memory(phys_addr_t addr) 1965{ 1966 int i = memblock_search(&memblock.memory, addr); 1967 1968 if (i == -1) 1969 return false; 1970 return !memblock_is_nomap(&memblock.memory.regions[i]); 1971} 1972 1973int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 1974 unsigned long *start_pfn, unsigned long *end_pfn) 1975{ 1976 struct memblock_type *type = &memblock.memory; 1977 int mid = memblock_search(type, PFN_PHYS(pfn)); 1978 1979 if (mid == -1) 1980 return NUMA_NO_NODE; 1981 1982 *start_pfn = PFN_DOWN(type->regions[mid].base); 1983 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size); 1984 1985 return memblock_get_region_node(&type->regions[mid]); 1986} 1987 1988/** 1989 * memblock_is_region_memory - check if a region is a subset of memory 1990 * @base: base of region to check 1991 * @size: size of region to check 1992 * 1993 * Check if the region [@base, @base + @size) is a subset of a memory block. 1994 * 1995 * Return: 1996 * 0 if false, non-zero if true 1997 */ 1998bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 1999{ 2000 int idx = memblock_search(&memblock.memory, base); 2001 phys_addr_t end = base + memblock_cap_size(base, &size); 2002 2003 if (idx == -1) 2004 return false; 2005 return (memblock.memory.regions[idx].base + 2006 memblock.memory.regions[idx].size) >= end; 2007} 2008 2009/** 2010 * memblock_is_region_reserved - check if a region intersects reserved memory 2011 * @base: base of region to check 2012 * @size: size of region to check 2013 * 2014 * Check if the region [@base, @base + @size) intersects a reserved 2015 * memory block. 2016 * 2017 * Return: 2018 * True if they intersect, false if not. 2019 */ 2020bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 2021{ 2022 return memblock_overlaps_region(&memblock.reserved, base, size); 2023} 2024 2025void __init_memblock memblock_trim_memory(phys_addr_t align) 2026{ 2027 phys_addr_t start, end, orig_start, orig_end; 2028 struct memblock_region *r; 2029 2030 for_each_mem_region(r) { 2031 orig_start = r->base; 2032 orig_end = r->base + r->size; 2033 start = round_up(orig_start, align); 2034 end = round_down(orig_end, align); 2035 2036 if (start == orig_start && end == orig_end) 2037 continue; 2038 2039 if (start < end) { 2040 r->base = start; 2041 r->size = end - start; 2042 } else { 2043 memblock_remove_region(&memblock.memory, 2044 r - memblock.memory.regions); 2045 r--; 2046 } 2047 } 2048} 2049 2050void __init_memblock memblock_set_current_limit(phys_addr_t limit) 2051{ 2052 memblock.current_limit = limit; 2053} 2054 2055phys_addr_t __init_memblock memblock_get_current_limit(void) 2056{ 2057 return memblock.current_limit; 2058} 2059 2060static void __init_memblock memblock_dump(struct memblock_type *type) 2061{ 2062 phys_addr_t base, end, size; 2063 enum memblock_flags flags; 2064 int idx; 2065 struct memblock_region *rgn; 2066 2067 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt); 2068 2069 for_each_memblock_type(idx, type, rgn) { 2070 char nid_buf[32] = ""; 2071 2072 base = rgn->base; 2073 size = rgn->size; 2074 end = base + size - 1; 2075 flags = rgn->flags; 2076#ifdef CONFIG_NUMA 2077 if (numa_valid_node(memblock_get_region_node(rgn))) 2078 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 2079 memblock_get_region_node(rgn)); 2080#endif 2081 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n", 2082 type->name, idx, &base, &end, &size, nid_buf, flags); 2083 } 2084} 2085 2086static void __init_memblock __memblock_dump_all(void) 2087{ 2088 pr_info("MEMBLOCK configuration:\n"); 2089 pr_info(" memory size = %pa reserved size = %pa\n", 2090 &memblock.memory.total_size, 2091 &memblock.reserved.total_size); 2092 2093 memblock_dump(&memblock.memory); 2094 memblock_dump(&memblock.reserved); 2095#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2096 memblock_dump(&physmem); 2097#endif 2098} 2099 2100void __init_memblock memblock_dump_all(void) 2101{ 2102 if (memblock_debug) 2103 __memblock_dump_all(); 2104} 2105 2106void __init memblock_allow_resize(void) 2107{ 2108 memblock_can_resize = 1; 2109} 2110 2111static int __init early_memblock(char *p) 2112{ 2113 if (p && strstr(p, "debug")) 2114 memblock_debug = 1; 2115 return 0; 2116} 2117early_param("memblock", early_memblock); 2118 2119static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn) 2120{ 2121 struct page *start_pg, *end_pg; 2122 phys_addr_t pg, pgend; 2123 2124 /* 2125 * Convert start_pfn/end_pfn to a struct page pointer. 2126 */ 2127 start_pg = pfn_to_page(start_pfn - 1) + 1; 2128 end_pg = pfn_to_page(end_pfn - 1) + 1; 2129 2130 /* 2131 * Convert to physical addresses, and round start upwards and end 2132 * downwards. 2133 */ 2134 pg = PAGE_ALIGN(__pa(start_pg)); 2135 pgend = PAGE_ALIGN_DOWN(__pa(end_pg)); 2136 2137 /* 2138 * If there are free pages between these, free the section of the 2139 * memmap array. 2140 */ 2141 if (pg < pgend) 2142 memblock_phys_free(pg, pgend - pg); 2143} 2144 2145/* 2146 * The mem_map array can get very big. Free the unused area of the memory map. 2147 */ 2148static void __init free_unused_memmap(void) 2149{ 2150 unsigned long start, end, prev_end = 0; 2151 int i; 2152 2153 if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) || 2154 IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 2155 return; 2156 2157 /* 2158 * This relies on each bank being in address order. 2159 * The banks are sorted previously in bootmem_init(). 2160 */ 2161 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { 2162#ifdef CONFIG_SPARSEMEM 2163 /* 2164 * Take care not to free memmap entries that don't exist 2165 * due to SPARSEMEM sections which aren't present. 2166 */ 2167 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION)); 2168#endif 2169 /* 2170 * Align down here since many operations in VM subsystem 2171 * presume that there are no holes in the memory map inside 2172 * a pageblock 2173 */ 2174 start = pageblock_start_pfn(start); 2175 2176 /* 2177 * If we had a previous bank, and there is a space 2178 * between the current bank and the previous, free it. 2179 */ 2180 if (prev_end && prev_end < start) 2181 free_memmap(prev_end, start); 2182 2183 /* 2184 * Align up here since many operations in VM subsystem 2185 * presume that there are no holes in the memory map inside 2186 * a pageblock 2187 */ 2188 prev_end = pageblock_align(end); 2189 } 2190 2191#ifdef CONFIG_SPARSEMEM 2192 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) { 2193 prev_end = pageblock_align(end); 2194 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); 2195 } 2196#endif 2197} 2198 2199static void __init __free_pages_memory(unsigned long start, unsigned long end) 2200{ 2201 int order; 2202 2203 while (start < end) { 2204 /* 2205 * Free the pages in the largest chunks alignment allows. 2206 * 2207 * __ffs() behaviour is undefined for 0. start == 0 is 2208 * MAX_PAGE_ORDER-aligned, set order to MAX_PAGE_ORDER for 2209 * the case. 2210 */ 2211 if (start) 2212 order = min_t(int, MAX_PAGE_ORDER, __ffs(start)); 2213 else 2214 order = MAX_PAGE_ORDER; 2215 2216 while (start + (1UL << order) > end) 2217 order--; 2218 2219 memblock_free_pages(pfn_to_page(start), start, order); 2220 2221 start += (1UL << order); 2222 } 2223} 2224 2225static unsigned long __init __free_memory_core(phys_addr_t start, 2226 phys_addr_t end) 2227{ 2228 unsigned long start_pfn = PFN_UP(start); 2229 unsigned long end_pfn = PFN_DOWN(end); 2230 2231 if (!IS_ENABLED(CONFIG_HIGHMEM) && end_pfn > max_low_pfn) 2232 end_pfn = max_low_pfn; 2233 2234 if (start_pfn >= end_pfn) 2235 return 0; 2236 2237 __free_pages_memory(start_pfn, end_pfn); 2238 2239 return end_pfn - start_pfn; 2240} 2241 2242static void __init memmap_init_reserved_pages(void) 2243{ 2244 struct memblock_region *region; 2245 phys_addr_t start, end; 2246 int nid; 2247 unsigned long max_reserved; 2248 2249 /* 2250 * set nid on all reserved pages and also treat struct 2251 * pages for the NOMAP regions as PageReserved 2252 */ 2253repeat: 2254 max_reserved = memblock.reserved.max; 2255 for_each_mem_region(region) { 2256 nid = memblock_get_region_node(region); 2257 start = region->base; 2258 end = start + region->size; 2259 2260 if (memblock_is_nomap(region)) 2261 reserve_bootmem_region(start, end, nid); 2262 2263 memblock_set_node(start, region->size, &memblock.reserved, nid); 2264 } 2265 /* 2266 * 'max' is changed means memblock.reserved has been doubled its 2267 * array, which may result a new reserved region before current 2268 * 'start'. Now we should repeat the procedure to set its node id. 2269 */ 2270 if (max_reserved != memblock.reserved.max) 2271 goto repeat; 2272 2273 /* 2274 * initialize struct pages for reserved regions that don't have 2275 * the MEMBLOCK_RSRV_NOINIT flag set 2276 */ 2277 for_each_reserved_mem_region(region) { 2278 if (!memblock_is_reserved_noinit(region)) { 2279 nid = memblock_get_region_node(region); 2280 start = region->base; 2281 end = start + region->size; 2282 2283 if (!numa_valid_node(nid)) 2284 nid = early_pfn_to_nid(PFN_DOWN(start)); 2285 2286 reserve_bootmem_region(start, end, nid); 2287 } 2288 } 2289} 2290 2291static unsigned long __init free_low_memory_core_early(void) 2292{ 2293 unsigned long count = 0; 2294 phys_addr_t start, end; 2295 u64 i; 2296 2297 memblock_clear_hotplug(0, -1); 2298 2299 memmap_init_reserved_pages(); 2300 2301 /* 2302 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id 2303 * because in some case like Node0 doesn't have RAM installed 2304 * low ram will be on Node1 2305 */ 2306 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, 2307 NULL) 2308 count += __free_memory_core(start, end); 2309 2310 return count; 2311} 2312 2313static int reset_managed_pages_done __initdata; 2314 2315static void __init reset_node_managed_pages(pg_data_t *pgdat) 2316{ 2317 struct zone *z; 2318 2319 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 2320 atomic_long_set(&z->managed_pages, 0); 2321} 2322 2323void __init reset_all_zones_managed_pages(void) 2324{ 2325 struct pglist_data *pgdat; 2326 2327 if (reset_managed_pages_done) 2328 return; 2329 2330 for_each_online_pgdat(pgdat) 2331 reset_node_managed_pages(pgdat); 2332 2333 reset_managed_pages_done = 1; 2334} 2335 2336/** 2337 * memblock_free_all - release free pages to the buddy allocator 2338 */ 2339void __init memblock_free_all(void) 2340{ 2341 unsigned long pages; 2342 2343 free_unused_memmap(); 2344 reset_all_zones_managed_pages(); 2345 2346 memblock_clear_kho_scratch_only(); 2347 pages = free_low_memory_core_early(); 2348 totalram_pages_add(pages); 2349} 2350 2351/* Keep a table to reserve named memory */ 2352#define RESERVE_MEM_MAX_ENTRIES 8 2353#define RESERVE_MEM_NAME_SIZE 16 2354struct reserve_mem_table { 2355 char name[RESERVE_MEM_NAME_SIZE]; 2356 phys_addr_t start; 2357 phys_addr_t size; 2358}; 2359static struct reserve_mem_table reserved_mem_table[RESERVE_MEM_MAX_ENTRIES]; 2360static int reserved_mem_count; 2361static DEFINE_MUTEX(reserve_mem_lock); 2362 2363/* Add wildcard region with a lookup name */ 2364static void __init reserved_mem_add(phys_addr_t start, phys_addr_t size, 2365 const char *name) 2366{ 2367 struct reserve_mem_table *map; 2368 2369 map = &reserved_mem_table[reserved_mem_count++]; 2370 map->start = start; 2371 map->size = size; 2372 strscpy(map->name, name); 2373} 2374 2375static struct reserve_mem_table *reserve_mem_find_by_name_nolock(const char *name) 2376{ 2377 struct reserve_mem_table *map; 2378 int i; 2379 2380 for (i = 0; i < reserved_mem_count; i++) { 2381 map = &reserved_mem_table[i]; 2382 if (!map->size) 2383 continue; 2384 if (strcmp(name, map->name) == 0) 2385 return map; 2386 } 2387 return NULL; 2388} 2389 2390/** 2391 * reserve_mem_find_by_name - Find reserved memory region with a given name 2392 * @name: The name that is attached to a reserved memory region 2393 * @start: If found, holds the start address 2394 * @size: If found, holds the size of the address. 2395 * 2396 * @start and @size are only updated if @name is found. 2397 * 2398 * Returns: 1 if found or 0 if not found. 2399 */ 2400int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t *size) 2401{ 2402 struct reserve_mem_table *map; 2403 2404 guard(mutex)(&reserve_mem_lock); 2405 map = reserve_mem_find_by_name_nolock(name); 2406 if (!map) 2407 return 0; 2408 2409 *start = map->start; 2410 *size = map->size; 2411 return 1; 2412} 2413EXPORT_SYMBOL_GPL(reserve_mem_find_by_name); 2414 2415/** 2416 * reserve_mem_release_by_name - Release reserved memory region with a given name 2417 * @name: The name that is attatched to a reserved memory region 2418 * 2419 * Forcibly release the pages in the reserved memory region so that those memory 2420 * can be used as free memory. After released the reserved region size becomes 0. 2421 * 2422 * Returns: 1 if released or 0 if not found. 2423 */ 2424int reserve_mem_release_by_name(const char *name) 2425{ 2426 char buf[RESERVE_MEM_NAME_SIZE + 12]; 2427 struct reserve_mem_table *map; 2428 void *start, *end; 2429 2430 guard(mutex)(&reserve_mem_lock); 2431 map = reserve_mem_find_by_name_nolock(name); 2432 if (!map) 2433 return 0; 2434 2435 start = phys_to_virt(map->start); 2436 end = start + map->size - 1; 2437 snprintf(buf, sizeof(buf), "reserve_mem:%s", name); 2438 free_reserved_area(start, end, 0, buf); 2439 map->size = 0; 2440 2441 return 1; 2442} 2443 2444#ifdef CONFIG_KEXEC_HANDOVER 2445#define MEMBLOCK_KHO_FDT "memblock" 2446#define MEMBLOCK_KHO_NODE_COMPATIBLE "memblock-v1" 2447#define RESERVE_MEM_KHO_NODE_COMPATIBLE "reserve-mem-v1" 2448 2449static int __init reserved_mem_preserve(void) 2450{ 2451 unsigned int nr_preserved = 0; 2452 int err; 2453 2454 for (unsigned int i = 0; i < reserved_mem_count; i++, nr_preserved++) { 2455 struct reserve_mem_table *map = &reserved_mem_table[i]; 2456 struct page *page = phys_to_page(map->start); 2457 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2458 2459 err = kho_preserve_pages(page, nr_pages); 2460 if (err) 2461 goto err_unpreserve; 2462 } 2463 2464 return 0; 2465 2466err_unpreserve: 2467 for (unsigned int i = 0; i < nr_preserved; i++) { 2468 struct reserve_mem_table *map = &reserved_mem_table[i]; 2469 struct page *page = phys_to_page(map->start); 2470 unsigned int nr_pages = map->size >> PAGE_SHIFT; 2471 2472 kho_unpreserve_pages(page, nr_pages); 2473 } 2474 2475 return err; 2476} 2477 2478static int __init prepare_kho_fdt(void) 2479{ 2480 struct page *fdt_page; 2481 void *fdt; 2482 int err; 2483 2484 fdt_page = alloc_page(GFP_KERNEL); 2485 if (!fdt_page) { 2486 err = -ENOMEM; 2487 goto err_report; 2488 } 2489 2490 fdt = page_to_virt(fdt_page); 2491 err = kho_preserve_pages(fdt_page, 1); 2492 if (err) 2493 goto err_free_fdt; 2494 2495 err |= fdt_create(fdt, PAGE_SIZE); 2496 err |= fdt_finish_reservemap(fdt); 2497 err |= fdt_begin_node(fdt, ""); 2498 err |= fdt_property_string(fdt, "compatible", MEMBLOCK_KHO_NODE_COMPATIBLE); 2499 2500 for (unsigned int i = 0; !err && i < reserved_mem_count; i++) { 2501 struct reserve_mem_table *map = &reserved_mem_table[i]; 2502 2503 err |= fdt_begin_node(fdt, map->name); 2504 err |= fdt_property_string(fdt, "compatible", RESERVE_MEM_KHO_NODE_COMPATIBLE); 2505 err |= fdt_property(fdt, "start", &map->start, sizeof(map->start)); 2506 err |= fdt_property(fdt, "size", &map->size, sizeof(map->size)); 2507 err |= fdt_end_node(fdt); 2508 } 2509 err |= fdt_end_node(fdt); 2510 err |= fdt_finish(fdt); 2511 2512 if (err) 2513 goto err_unpreserve_fdt; 2514 2515 err = kho_add_subtree(MEMBLOCK_KHO_FDT, fdt); 2516 if (err) 2517 goto err_unpreserve_fdt; 2518 2519 err = reserved_mem_preserve(); 2520 if (err) 2521 goto err_remove_subtree; 2522 2523 return 0; 2524 2525err_remove_subtree: 2526 kho_remove_subtree(fdt); 2527err_unpreserve_fdt: 2528 kho_unpreserve_pages(fdt_page, 1); 2529err_free_fdt: 2530 put_page(fdt_page); 2531err_report: 2532 pr_err("failed to prepare memblock FDT for KHO: %d\n", err); 2533 2534 return err; 2535} 2536 2537static int __init reserve_mem_init(void) 2538{ 2539 int err; 2540 2541 if (!kho_is_enabled() || !reserved_mem_count) 2542 return 0; 2543 2544 err = prepare_kho_fdt(); 2545 if (err) 2546 return err; 2547 return err; 2548} 2549late_initcall(reserve_mem_init); 2550 2551static void *__init reserve_mem_kho_retrieve_fdt(void) 2552{ 2553 phys_addr_t fdt_phys; 2554 static void *fdt; 2555 int err; 2556 2557 if (fdt) 2558 return fdt; 2559 2560 err = kho_retrieve_subtree(MEMBLOCK_KHO_FDT, &fdt_phys); 2561 if (err) { 2562 if (err != -ENOENT) 2563 pr_warn("failed to retrieve FDT '%s' from KHO: %d\n", 2564 MEMBLOCK_KHO_FDT, err); 2565 return NULL; 2566 } 2567 2568 fdt = phys_to_virt(fdt_phys); 2569 2570 err = fdt_node_check_compatible(fdt, 0, MEMBLOCK_KHO_NODE_COMPATIBLE); 2571 if (err) { 2572 pr_warn("FDT '%s' is incompatible with '%s': %d\n", 2573 MEMBLOCK_KHO_FDT, MEMBLOCK_KHO_NODE_COMPATIBLE, err); 2574 fdt = NULL; 2575 } 2576 2577 return fdt; 2578} 2579 2580static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2581 phys_addr_t align) 2582{ 2583 int err, len_start, len_size, offset; 2584 const phys_addr_t *p_start, *p_size; 2585 const void *fdt; 2586 2587 fdt = reserve_mem_kho_retrieve_fdt(); 2588 if (!fdt) 2589 return false; 2590 2591 offset = fdt_subnode_offset(fdt, 0, name); 2592 if (offset < 0) { 2593 pr_warn("FDT '%s' has no child '%s': %d\n", 2594 MEMBLOCK_KHO_FDT, name, offset); 2595 return false; 2596 } 2597 err = fdt_node_check_compatible(fdt, offset, RESERVE_MEM_KHO_NODE_COMPATIBLE); 2598 if (err) { 2599 pr_warn("Node '%s' is incompatible with '%s': %d\n", 2600 name, RESERVE_MEM_KHO_NODE_COMPATIBLE, err); 2601 return false; 2602 } 2603 2604 p_start = fdt_getprop(fdt, offset, "start", &len_start); 2605 p_size = fdt_getprop(fdt, offset, "size", &len_size); 2606 if (!p_start || len_start != sizeof(*p_start) || !p_size || 2607 len_size != sizeof(*p_size)) { 2608 return false; 2609 } 2610 2611 if (*p_start & (align - 1)) { 2612 pr_warn("KHO reserve-mem '%s' has wrong alignment (0x%lx, 0x%lx)\n", 2613 name, (long)align, (long)*p_start); 2614 return false; 2615 } 2616 2617 if (*p_size != size) { 2618 pr_warn("KHO reserve-mem '%s' has wrong size (0x%lx != 0x%lx)\n", 2619 name, (long)*p_size, (long)size); 2620 return false; 2621 } 2622 2623 reserved_mem_add(*p_start, size, name); 2624 pr_info("Revived memory reservation '%s' from KHO\n", name); 2625 2626 return true; 2627} 2628#else 2629static bool __init reserve_mem_kho_revive(const char *name, phys_addr_t size, 2630 phys_addr_t align) 2631{ 2632 return false; 2633} 2634#endif /* CONFIG_KEXEC_HANDOVER */ 2635 2636/* 2637 * Parse reserve_mem=nn:align:name 2638 */ 2639static int __init reserve_mem(char *p) 2640{ 2641 phys_addr_t start, size, align, tmp; 2642 char *name; 2643 char *oldp; 2644 int len; 2645 2646 if (!p) 2647 return -EINVAL; 2648 2649 /* Check if there's room for more reserved memory */ 2650 if (reserved_mem_count >= RESERVE_MEM_MAX_ENTRIES) 2651 return -EBUSY; 2652 2653 oldp = p; 2654 size = memparse(p, &p); 2655 if (!size || p == oldp) 2656 return -EINVAL; 2657 2658 if (*p != ':') 2659 return -EINVAL; 2660 2661 align = memparse(p+1, &p); 2662 if (*p != ':') 2663 return -EINVAL; 2664 2665 /* 2666 * memblock_phys_alloc() doesn't like a zero size align, 2667 * but it is OK for this command to have it. 2668 */ 2669 if (align < SMP_CACHE_BYTES) 2670 align = SMP_CACHE_BYTES; 2671 2672 name = p + 1; 2673 len = strlen(name); 2674 2675 /* name needs to have length but not too big */ 2676 if (!len || len >= RESERVE_MEM_NAME_SIZE) 2677 return -EINVAL; 2678 2679 /* Make sure that name has text */ 2680 for (p = name; *p; p++) { 2681 if (!isspace(*p)) 2682 break; 2683 } 2684 if (!*p) 2685 return -EINVAL; 2686 2687 /* Make sure the name is not already used */ 2688 if (reserve_mem_find_by_name(name, &start, &tmp)) 2689 return -EBUSY; 2690 2691 /* Pick previous allocations up from KHO if available */ 2692 if (reserve_mem_kho_revive(name, size, align)) 2693 return 1; 2694 2695 /* TODO: Allocation must be outside of scratch region */ 2696 start = memblock_phys_alloc(size, align); 2697 if (!start) 2698 return -ENOMEM; 2699 2700 reserved_mem_add(start, size, name); 2701 2702 return 1; 2703} 2704__setup("reserve_mem=", reserve_mem); 2705 2706#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK) 2707static const char * const flagname[] = { 2708 [ilog2(MEMBLOCK_HOTPLUG)] = "HOTPLUG", 2709 [ilog2(MEMBLOCK_MIRROR)] = "MIRROR", 2710 [ilog2(MEMBLOCK_NOMAP)] = "NOMAP", 2711 [ilog2(MEMBLOCK_DRIVER_MANAGED)] = "DRV_MNG", 2712 [ilog2(MEMBLOCK_RSRV_NOINIT)] = "RSV_NIT", 2713 [ilog2(MEMBLOCK_RSRV_KERN)] = "RSV_KERN", 2714 [ilog2(MEMBLOCK_KHO_SCRATCH)] = "KHO_SCRATCH", 2715}; 2716 2717static int memblock_debug_show(struct seq_file *m, void *private) 2718{ 2719 struct memblock_type *type = m->private; 2720 struct memblock_region *reg; 2721 int i, j, nid; 2722 unsigned int count = ARRAY_SIZE(flagname); 2723 phys_addr_t end; 2724 2725 for (i = 0; i < type->cnt; i++) { 2726 reg = &type->regions[i]; 2727 end = reg->base + reg->size - 1; 2728 nid = memblock_get_region_node(reg); 2729 2730 seq_printf(m, "%4d: ", i); 2731 seq_printf(m, "%pa..%pa ", ®->base, &end); 2732 if (numa_valid_node(nid)) 2733 seq_printf(m, "%4d ", nid); 2734 else 2735 seq_printf(m, "%4c ", 'x'); 2736 if (reg->flags) { 2737 for (j = 0; j < count; j++) { 2738 if (reg->flags & (1U << j)) { 2739 seq_printf(m, "%s\n", flagname[j]); 2740 break; 2741 } 2742 } 2743 if (j == count) 2744 seq_printf(m, "%s\n", "UNKNOWN"); 2745 } else { 2746 seq_printf(m, "%s\n", "NONE"); 2747 } 2748 } 2749 return 0; 2750} 2751DEFINE_SHOW_ATTRIBUTE(memblock_debug); 2752 2753static int __init memblock_init_debugfs(void) 2754{ 2755 struct dentry *root = debugfs_create_dir("memblock", NULL); 2756 2757 debugfs_create_file("memory", 0444, root, 2758 &memblock.memory, &memblock_debug_fops); 2759 debugfs_create_file("reserved", 0444, root, 2760 &memblock.reserved, &memblock_debug_fops); 2761#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2762 debugfs_create_file("physmem", 0444, root, &physmem, 2763 &memblock_debug_fops); 2764#endif 2765 2766 return 0; 2767} 2768__initcall(memblock_init_debugfs); 2769 2770#endif /* CONFIG_DEBUG_FS */
~/
Read-only file preview.