FILE
vmalloc.c
File preview with syntax highlighting.
linux-6.19.14/mm/vmalloc.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 1993 Linus Torvalds 4 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 5 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000 6 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002 7 * Numa awareness, Christoph Lameter, SGI, June 2005 8 * Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019 9 */ 10 11#include <linux/vmalloc.h> 12#include <linux/mm.h> 13#include <linux/module.h> 14#include <linux/highmem.h> 15#include <linux/sched/signal.h> 16#include <linux/slab.h> 17#include <linux/spinlock.h> 18#include <linux/interrupt.h> 19#include <linux/proc_fs.h> 20#include <linux/seq_file.h> 21#include <linux/set_memory.h> 22#include <linux/debugobjects.h> 23#include <linux/kallsyms.h> 24#include <linux/list.h> 25#include <linux/notifier.h> 26#include <linux/rbtree.h> 27#include <linux/xarray.h> 28#include <linux/io.h> 29#include <linux/rcupdate.h> 30#include <linux/pfn.h> 31#include <linux/kmemleak.h> 32#include <linux/atomic.h> 33#include <linux/compiler.h> 34#include <linux/memcontrol.h> 35#include <linux/llist.h> 36#include <linux/uio.h> 37#include <linux/bitops.h> 38#include <linux/rbtree_augmented.h> 39#include <linux/overflow.h> 40#include <linux/pgtable.h> 41#include <linux/hugetlb.h> 42#include <linux/sched/mm.h> 43#include <asm/tlbflush.h> 44#include <asm/shmparam.h> 45#include <linux/page_owner.h> 46 47#define CREATE_TRACE_POINTS 48#include <trace/events/vmalloc.h> 49 50#include "internal.h" 51#include "pgalloc-track.h" 52 53#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 54static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1; 55 56static int __init set_nohugeiomap(char *str) 57{ 58 ioremap_max_page_shift = PAGE_SHIFT; 59 return 0; 60} 61early_param("nohugeiomap", set_nohugeiomap); 62#else /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 63static const unsigned int ioremap_max_page_shift = PAGE_SHIFT; 64#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 65 66#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 67static bool __ro_after_init vmap_allow_huge = true; 68 69static int __init set_nohugevmalloc(char *str) 70{ 71 vmap_allow_huge = false; 72 return 0; 73} 74early_param("nohugevmalloc", set_nohugevmalloc); 75#else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 76static const bool vmap_allow_huge = false; 77#endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */ 78 79bool is_vmalloc_addr(const void *x) 80{ 81 unsigned long addr = (unsigned long)kasan_reset_tag(x); 82 83 return addr >= VMALLOC_START && addr < VMALLOC_END; 84} 85EXPORT_SYMBOL(is_vmalloc_addr); 86 87struct vfree_deferred { 88 struct llist_head list; 89 struct work_struct wq; 90}; 91static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred); 92 93/*** Page table manipulation functions ***/ 94static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 95 phys_addr_t phys_addr, pgprot_t prot, 96 unsigned int max_page_shift, pgtbl_mod_mask *mask) 97{ 98 pte_t *pte; 99 u64 pfn; 100 struct page *page; 101 unsigned long size = PAGE_SIZE; 102 103 if (WARN_ON_ONCE(!PAGE_ALIGNED(end - addr))) 104 return -EINVAL; 105 106 pfn = phys_addr >> PAGE_SHIFT; 107 pte = pte_alloc_kernel_track(pmd, addr, mask); 108 if (!pte) 109 return -ENOMEM; 110 111 arch_enter_lazy_mmu_mode(); 112 113 do { 114 if (unlikely(!pte_none(ptep_get(pte)))) { 115 if (pfn_valid(pfn)) { 116 page = pfn_to_page(pfn); 117 dump_page(page, "remapping already mapped page"); 118 } 119 BUG(); 120 } 121 122#ifdef CONFIG_HUGETLB_PAGE 123 size = arch_vmap_pte_range_map_size(addr, end, pfn, max_page_shift); 124 if (size != PAGE_SIZE) { 125 pte_t entry = pfn_pte(pfn, prot); 126 127 entry = arch_make_huge_pte(entry, ilog2(size), 0); 128 set_huge_pte_at(&init_mm, addr, pte, entry, size); 129 pfn += PFN_DOWN(size); 130 continue; 131 } 132#endif 133 set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 134 pfn++; 135 } while (pte += PFN_DOWN(size), addr += size, addr != end); 136 137 arch_leave_lazy_mmu_mode(); 138 *mask |= PGTBL_PTE_MODIFIED; 139 return 0; 140} 141 142static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, 143 phys_addr_t phys_addr, pgprot_t prot, 144 unsigned int max_page_shift) 145{ 146 if (max_page_shift < PMD_SHIFT) 147 return 0; 148 149 if (!arch_vmap_pmd_supported(prot)) 150 return 0; 151 152 if ((end - addr) != PMD_SIZE) 153 return 0; 154 155 if (!IS_ALIGNED(addr, PMD_SIZE)) 156 return 0; 157 158 if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 159 return 0; 160 161 if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 162 return 0; 163 164 return pmd_set_huge(pmd, phys_addr, prot); 165} 166 167static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 168 phys_addr_t phys_addr, pgprot_t prot, 169 unsigned int max_page_shift, pgtbl_mod_mask *mask) 170{ 171 pmd_t *pmd; 172 unsigned long next; 173 int err = 0; 174 175 pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 176 if (!pmd) 177 return -ENOMEM; 178 do { 179 next = pmd_addr_end(addr, end); 180 181 if (vmap_try_huge_pmd(pmd, addr, next, phys_addr, prot, 182 max_page_shift)) { 183 *mask |= PGTBL_PMD_MODIFIED; 184 continue; 185 } 186 187 err = vmap_pte_range(pmd, addr, next, phys_addr, prot, max_page_shift, mask); 188 if (err) 189 break; 190 } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 191 return err; 192} 193 194static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, 195 phys_addr_t phys_addr, pgprot_t prot, 196 unsigned int max_page_shift) 197{ 198 if (max_page_shift < PUD_SHIFT) 199 return 0; 200 201 if (!arch_vmap_pud_supported(prot)) 202 return 0; 203 204 if ((end - addr) != PUD_SIZE) 205 return 0; 206 207 if (!IS_ALIGNED(addr, PUD_SIZE)) 208 return 0; 209 210 if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 211 return 0; 212 213 if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 214 return 0; 215 216 return pud_set_huge(pud, phys_addr, prot); 217} 218 219static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 220 phys_addr_t phys_addr, pgprot_t prot, 221 unsigned int max_page_shift, pgtbl_mod_mask *mask) 222{ 223 pud_t *pud; 224 unsigned long next; 225 int err = 0; 226 227 pud = pud_alloc_track(&init_mm, p4d, addr, mask); 228 if (!pud) 229 return -ENOMEM; 230 do { 231 next = pud_addr_end(addr, end); 232 233 if (vmap_try_huge_pud(pud, addr, next, phys_addr, prot, 234 max_page_shift)) { 235 *mask |= PGTBL_PUD_MODIFIED; 236 continue; 237 } 238 239 err = vmap_pmd_range(pud, addr, next, phys_addr, prot, max_page_shift, mask); 240 if (err) 241 break; 242 } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 243 return err; 244} 245 246static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, 247 phys_addr_t phys_addr, pgprot_t prot, 248 unsigned int max_page_shift) 249{ 250 if (max_page_shift < P4D_SHIFT) 251 return 0; 252 253 if (!arch_vmap_p4d_supported(prot)) 254 return 0; 255 256 if ((end - addr) != P4D_SIZE) 257 return 0; 258 259 if (!IS_ALIGNED(addr, P4D_SIZE)) 260 return 0; 261 262 if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 263 return 0; 264 265 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 266 return 0; 267 268 return p4d_set_huge(p4d, phys_addr, prot); 269} 270 271static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 272 phys_addr_t phys_addr, pgprot_t prot, 273 unsigned int max_page_shift, pgtbl_mod_mask *mask) 274{ 275 p4d_t *p4d; 276 unsigned long next; 277 int err = 0; 278 279 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 280 if (!p4d) 281 return -ENOMEM; 282 do { 283 next = p4d_addr_end(addr, end); 284 285 if (vmap_try_huge_p4d(p4d, addr, next, phys_addr, prot, 286 max_page_shift)) { 287 *mask |= PGTBL_P4D_MODIFIED; 288 continue; 289 } 290 291 err = vmap_pud_range(p4d, addr, next, phys_addr, prot, max_page_shift, mask); 292 if (err) 293 break; 294 } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 295 return err; 296} 297 298static int vmap_range_noflush(unsigned long addr, unsigned long end, 299 phys_addr_t phys_addr, pgprot_t prot, 300 unsigned int max_page_shift) 301{ 302 pgd_t *pgd; 303 unsigned long start; 304 unsigned long next; 305 int err; 306 pgtbl_mod_mask mask = 0; 307 308 might_sleep(); 309 BUG_ON(addr >= end); 310 311 start = addr; 312 pgd = pgd_offset_k(addr); 313 do { 314 next = pgd_addr_end(addr, end); 315 err = vmap_p4d_range(pgd, addr, next, phys_addr, prot, 316 max_page_shift, &mask); 317 if (err) 318 break; 319 } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 320 321 if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 322 arch_sync_kernel_mappings(start, end); 323 324 return err; 325} 326 327int vmap_page_range(unsigned long addr, unsigned long end, 328 phys_addr_t phys_addr, pgprot_t prot) 329{ 330 int err; 331 332 err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), 333 ioremap_max_page_shift); 334 flush_cache_vmap(addr, end); 335 if (!err) 336 err = kmsan_ioremap_page_range(addr, end, phys_addr, prot, 337 ioremap_max_page_shift); 338 return err; 339} 340 341int ioremap_page_range(unsigned long addr, unsigned long end, 342 phys_addr_t phys_addr, pgprot_t prot) 343{ 344 struct vm_struct *area; 345 346 area = find_vm_area((void *)addr); 347 if (!area || !(area->flags & VM_IOREMAP)) { 348 WARN_ONCE(1, "vm_area at addr %lx is not marked as VM_IOREMAP\n", addr); 349 return -EINVAL; 350 } 351 if (addr != (unsigned long)area->addr || 352 (void *)end != area->addr + get_vm_area_size(area)) { 353 WARN_ONCE(1, "ioremap request [%lx,%lx) doesn't match vm_area [%lx, %lx)\n", 354 addr, end, (long)area->addr, 355 (long)area->addr + get_vm_area_size(area)); 356 return -ERANGE; 357 } 358 return vmap_page_range(addr, end, phys_addr, prot); 359} 360 361static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 362 pgtbl_mod_mask *mask) 363{ 364 pte_t *pte; 365 pte_t ptent; 366 unsigned long size = PAGE_SIZE; 367 368 pte = pte_offset_kernel(pmd, addr); 369 arch_enter_lazy_mmu_mode(); 370 371 do { 372#ifdef CONFIG_HUGETLB_PAGE 373 size = arch_vmap_pte_range_unmap_size(addr, pte); 374 if (size != PAGE_SIZE) { 375 if (WARN_ON(!IS_ALIGNED(addr, size))) { 376 addr = ALIGN_DOWN(addr, size); 377 pte = PTR_ALIGN_DOWN(pte, sizeof(*pte) * (size >> PAGE_SHIFT)); 378 } 379 ptent = huge_ptep_get_and_clear(&init_mm, addr, pte, size); 380 if (WARN_ON(end - addr < size)) 381 size = end - addr; 382 } else 383#endif 384 ptent = ptep_get_and_clear(&init_mm, addr, pte); 385 WARN_ON(!pte_none(ptent) && !pte_present(ptent)); 386 } while (pte += (size >> PAGE_SHIFT), addr += size, addr != end); 387 388 arch_leave_lazy_mmu_mode(); 389 *mask |= PGTBL_PTE_MODIFIED; 390} 391 392static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 393 pgtbl_mod_mask *mask) 394{ 395 pmd_t *pmd; 396 unsigned long next; 397 int cleared; 398 399 pmd = pmd_offset(pud, addr); 400 do { 401 next = pmd_addr_end(addr, end); 402 403 cleared = pmd_clear_huge(pmd); 404 if (cleared || pmd_bad(*pmd)) 405 *mask |= PGTBL_PMD_MODIFIED; 406 407 if (cleared) { 408 WARN_ON(next - addr < PMD_SIZE); 409 continue; 410 } 411 if (pmd_none_or_clear_bad(pmd)) 412 continue; 413 vunmap_pte_range(pmd, addr, next, mask); 414 415 cond_resched(); 416 } while (pmd++, addr = next, addr != end); 417} 418 419static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, 420 pgtbl_mod_mask *mask) 421{ 422 pud_t *pud; 423 unsigned long next; 424 int cleared; 425 426 pud = pud_offset(p4d, addr); 427 do { 428 next = pud_addr_end(addr, end); 429 430 cleared = pud_clear_huge(pud); 431 if (cleared || pud_bad(*pud)) 432 *mask |= PGTBL_PUD_MODIFIED; 433 434 if (cleared) { 435 WARN_ON(next - addr < PUD_SIZE); 436 continue; 437 } 438 if (pud_none_or_clear_bad(pud)) 439 continue; 440 vunmap_pmd_range(pud, addr, next, mask); 441 } while (pud++, addr = next, addr != end); 442} 443 444static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, 445 pgtbl_mod_mask *mask) 446{ 447 p4d_t *p4d; 448 unsigned long next; 449 450 p4d = p4d_offset(pgd, addr); 451 do { 452 next = p4d_addr_end(addr, end); 453 454 p4d_clear_huge(p4d); 455 if (p4d_bad(*p4d)) 456 *mask |= PGTBL_P4D_MODIFIED; 457 458 if (p4d_none_or_clear_bad(p4d)) 459 continue; 460 vunmap_pud_range(p4d, addr, next, mask); 461 } while (p4d++, addr = next, addr != end); 462} 463 464/* 465 * vunmap_range_noflush is similar to vunmap_range, but does not 466 * flush caches or TLBs. 467 * 468 * The caller is responsible for calling flush_cache_vmap() before calling 469 * this function, and flush_tlb_kernel_range after it has returned 470 * successfully (and before the addresses are expected to cause a page fault 471 * or be re-mapped for something else, if TLB flushes are being delayed or 472 * coalesced). 473 * 474 * This is an internal function only. Do not use outside mm/. 475 */ 476void __vunmap_range_noflush(unsigned long start, unsigned long end) 477{ 478 unsigned long next; 479 pgd_t *pgd; 480 unsigned long addr = start; 481 pgtbl_mod_mask mask = 0; 482 483 BUG_ON(addr >= end); 484 pgd = pgd_offset_k(addr); 485 do { 486 next = pgd_addr_end(addr, end); 487 if (pgd_bad(*pgd)) 488 mask |= PGTBL_PGD_MODIFIED; 489 if (pgd_none_or_clear_bad(pgd)) 490 continue; 491 vunmap_p4d_range(pgd, addr, next, &mask); 492 } while (pgd++, addr = next, addr != end); 493 494 if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 495 arch_sync_kernel_mappings(start, end); 496} 497 498void vunmap_range_noflush(unsigned long start, unsigned long end) 499{ 500 kmsan_vunmap_range_noflush(start, end); 501 __vunmap_range_noflush(start, end); 502} 503 504/** 505 * vunmap_range - unmap kernel virtual addresses 506 * @addr: start of the VM area to unmap 507 * @end: end of the VM area to unmap (non-inclusive) 508 * 509 * Clears any present PTEs in the virtual address range, flushes TLBs and 510 * caches. Any subsequent access to the address before it has been re-mapped 511 * is a kernel bug. 512 */ 513void vunmap_range(unsigned long addr, unsigned long end) 514{ 515 flush_cache_vunmap(addr, end); 516 vunmap_range_noflush(addr, end); 517 flush_tlb_kernel_range(addr, end); 518} 519 520static int vmap_pages_pte_range(pmd_t *pmd, unsigned long addr, 521 unsigned long end, pgprot_t prot, struct page **pages, int *nr, 522 pgtbl_mod_mask *mask) 523{ 524 int err = 0; 525 pte_t *pte; 526 527 /* 528 * nr is a running index into the array which helps higher level 529 * callers keep track of where we're up to. 530 */ 531 532 pte = pte_alloc_kernel_track(pmd, addr, mask); 533 if (!pte) 534 return -ENOMEM; 535 536 arch_enter_lazy_mmu_mode(); 537 538 do { 539 struct page *page = pages[*nr]; 540 541 if (WARN_ON(!pte_none(ptep_get(pte)))) { 542 err = -EBUSY; 543 break; 544 } 545 if (WARN_ON(!page)) { 546 err = -ENOMEM; 547 break; 548 } 549 if (WARN_ON(!pfn_valid(page_to_pfn(page)))) { 550 err = -EINVAL; 551 break; 552 } 553 554 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); 555 (*nr)++; 556 } while (pte++, addr += PAGE_SIZE, addr != end); 557 558 arch_leave_lazy_mmu_mode(); 559 *mask |= PGTBL_PTE_MODIFIED; 560 561 return err; 562} 563 564static int vmap_pages_pmd_range(pud_t *pud, unsigned long addr, 565 unsigned long end, pgprot_t prot, struct page **pages, int *nr, 566 pgtbl_mod_mask *mask) 567{ 568 pmd_t *pmd; 569 unsigned long next; 570 571 pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 572 if (!pmd) 573 return -ENOMEM; 574 do { 575 next = pmd_addr_end(addr, end); 576 if (vmap_pages_pte_range(pmd, addr, next, prot, pages, nr, mask)) 577 return -ENOMEM; 578 } while (pmd++, addr = next, addr != end); 579 return 0; 580} 581 582static int vmap_pages_pud_range(p4d_t *p4d, unsigned long addr, 583 unsigned long end, pgprot_t prot, struct page **pages, int *nr, 584 pgtbl_mod_mask *mask) 585{ 586 pud_t *pud; 587 unsigned long next; 588 589 pud = pud_alloc_track(&init_mm, p4d, addr, mask); 590 if (!pud) 591 return -ENOMEM; 592 do { 593 next = pud_addr_end(addr, end); 594 if (vmap_pages_pmd_range(pud, addr, next, prot, pages, nr, mask)) 595 return -ENOMEM; 596 } while (pud++, addr = next, addr != end); 597 return 0; 598} 599 600static int vmap_pages_p4d_range(pgd_t *pgd, unsigned long addr, 601 unsigned long end, pgprot_t prot, struct page **pages, int *nr, 602 pgtbl_mod_mask *mask) 603{ 604 p4d_t *p4d; 605 unsigned long next; 606 607 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 608 if (!p4d) 609 return -ENOMEM; 610 do { 611 next = p4d_addr_end(addr, end); 612 if (vmap_pages_pud_range(p4d, addr, next, prot, pages, nr, mask)) 613 return -ENOMEM; 614 } while (p4d++, addr = next, addr != end); 615 return 0; 616} 617 618static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, 619 pgprot_t prot, struct page **pages) 620{ 621 unsigned long start = addr; 622 pgd_t *pgd; 623 unsigned long next; 624 int err = 0; 625 int nr = 0; 626 pgtbl_mod_mask mask = 0; 627 628 BUG_ON(addr >= end); 629 pgd = pgd_offset_k(addr); 630 do { 631 next = pgd_addr_end(addr, end); 632 if (pgd_bad(*pgd)) 633 mask |= PGTBL_PGD_MODIFIED; 634 err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); 635 if (err) 636 break; 637 } while (pgd++, addr = next, addr != end); 638 639 if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 640 arch_sync_kernel_mappings(start, end); 641 642 return err; 643} 644 645/* 646 * vmap_pages_range_noflush is similar to vmap_pages_range, but does not 647 * flush caches. 648 * 649 * The caller is responsible for calling flush_cache_vmap() after this 650 * function returns successfully and before the addresses are accessed. 651 * 652 * This is an internal function only. Do not use outside mm/. 653 */ 654int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, 655 pgprot_t prot, struct page **pages, unsigned int page_shift) 656{ 657 unsigned int i, nr = (end - addr) >> PAGE_SHIFT; 658 659 WARN_ON(page_shift < PAGE_SHIFT); 660 661 if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC) || 662 page_shift == PAGE_SHIFT) 663 return vmap_small_pages_range_noflush(addr, end, prot, pages); 664 665 for (i = 0; i < nr; i += 1U << (page_shift - PAGE_SHIFT)) { 666 int err; 667 668 err = vmap_range_noflush(addr, addr + (1UL << page_shift), 669 page_to_phys(pages[i]), prot, 670 page_shift); 671 if (err) 672 return err; 673 674 addr += 1UL << page_shift; 675 } 676 677 return 0; 678} 679 680int vmap_pages_range_noflush(unsigned long addr, unsigned long end, 681 pgprot_t prot, struct page **pages, unsigned int page_shift, 682 gfp_t gfp_mask) 683{ 684 int ret = kmsan_vmap_pages_range_noflush(addr, end, prot, pages, 685 page_shift, gfp_mask); 686 687 if (ret) 688 return ret; 689 return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift); 690} 691 692static int __vmap_pages_range(unsigned long addr, unsigned long end, 693 pgprot_t prot, struct page **pages, unsigned int page_shift, 694 gfp_t gfp_mask) 695{ 696 int err; 697 698 err = vmap_pages_range_noflush(addr, end, prot, pages, page_shift, gfp_mask); 699 flush_cache_vmap(addr, end); 700 return err; 701} 702 703/** 704 * vmap_pages_range - map pages to a kernel virtual address 705 * @addr: start of the VM area to map 706 * @end: end of the VM area to map (non-inclusive) 707 * @prot: page protection flags to use 708 * @pages: pages to map (always PAGE_SIZE pages) 709 * @page_shift: maximum shift that the pages may be mapped with, @pages must 710 * be aligned and contiguous up to at least this shift. 711 * 712 * RETURNS: 713 * 0 on success, -errno on failure. 714 */ 715int vmap_pages_range(unsigned long addr, unsigned long end, 716 pgprot_t prot, struct page **pages, unsigned int page_shift) 717{ 718 return __vmap_pages_range(addr, end, prot, pages, page_shift, GFP_KERNEL); 719} 720 721static int check_sparse_vm_area(struct vm_struct *area, unsigned long start, 722 unsigned long end) 723{ 724 might_sleep(); 725 if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS)) 726 return -EINVAL; 727 if (WARN_ON_ONCE(area->flags & VM_NO_GUARD)) 728 return -EINVAL; 729 if (WARN_ON_ONCE(!(area->flags & VM_SPARSE))) 730 return -EINVAL; 731 if ((end - start) >> PAGE_SHIFT > totalram_pages()) 732 return -E2BIG; 733 if (start < (unsigned long)area->addr || 734 (void *)end > area->addr + get_vm_area_size(area)) 735 return -ERANGE; 736 return 0; 737} 738 739/** 740 * vm_area_map_pages - map pages inside given sparse vm_area 741 * @area: vm_area 742 * @start: start address inside vm_area 743 * @end: end address inside vm_area 744 * @pages: pages to map (always PAGE_SIZE pages) 745 */ 746int vm_area_map_pages(struct vm_struct *area, unsigned long start, 747 unsigned long end, struct page **pages) 748{ 749 int err; 750 751 err = check_sparse_vm_area(area, start, end); 752 if (err) 753 return err; 754 755 return vmap_pages_range(start, end, PAGE_KERNEL, pages, PAGE_SHIFT); 756} 757 758/** 759 * vm_area_unmap_pages - unmap pages inside given sparse vm_area 760 * @area: vm_area 761 * @start: start address inside vm_area 762 * @end: end address inside vm_area 763 */ 764void vm_area_unmap_pages(struct vm_struct *area, unsigned long start, 765 unsigned long end) 766{ 767 if (check_sparse_vm_area(area, start, end)) 768 return; 769 770 vunmap_range(start, end); 771} 772 773int is_vmalloc_or_module_addr(const void *x) 774{ 775 /* 776 * ARM, x86-64 and sparc64 put modules in a special place, 777 * and fall back on vmalloc() if that fails. Others 778 * just put it in the vmalloc space. 779 */ 780#if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR) 781 unsigned long addr = (unsigned long)kasan_reset_tag(x); 782 if (addr >= MODULES_VADDR && addr < MODULES_END) 783 return 1; 784#endif 785 return is_vmalloc_addr(x); 786} 787EXPORT_SYMBOL_GPL(is_vmalloc_or_module_addr); 788 789/* 790 * Walk a vmap address to the struct page it maps. Huge vmap mappings will 791 * return the tail page that corresponds to the base page address, which 792 * matches small vmap mappings. 793 */ 794struct page *vmalloc_to_page(const void *vmalloc_addr) 795{ 796 unsigned long addr = (unsigned long) vmalloc_addr; 797 struct page *page = NULL; 798 pgd_t *pgd = pgd_offset_k(addr); 799 p4d_t *p4d; 800 pud_t *pud; 801 pmd_t *pmd; 802 pte_t *ptep, pte; 803 804 /* 805 * XXX we might need to change this if we add VIRTUAL_BUG_ON for 806 * architectures that do not vmalloc module space 807 */ 808 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); 809 810 if (pgd_none(*pgd)) 811 return NULL; 812 if (WARN_ON_ONCE(pgd_leaf(*pgd))) 813 return NULL; /* XXX: no allowance for huge pgd */ 814 if (WARN_ON_ONCE(pgd_bad(*pgd))) 815 return NULL; 816 817 p4d = p4d_offset(pgd, addr); 818 if (p4d_none(*p4d)) 819 return NULL; 820 if (p4d_leaf(*p4d)) 821 return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); 822 if (WARN_ON_ONCE(p4d_bad(*p4d))) 823 return NULL; 824 825 pud = pud_offset(p4d, addr); 826 if (pud_none(*pud)) 827 return NULL; 828 if (pud_leaf(*pud)) 829 return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 830 if (WARN_ON_ONCE(pud_bad(*pud))) 831 return NULL; 832 833 pmd = pmd_offset(pud, addr); 834 if (pmd_none(*pmd)) 835 return NULL; 836 if (pmd_leaf(*pmd)) 837 return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 838 if (WARN_ON_ONCE(pmd_bad(*pmd))) 839 return NULL; 840 841 ptep = pte_offset_kernel(pmd, addr); 842 pte = ptep_get(ptep); 843 if (pte_present(pte)) 844 page = pte_page(pte); 845 846 return page; 847} 848EXPORT_SYMBOL(vmalloc_to_page); 849 850/* 851 * Map a vmalloc()-space virtual address to the physical page frame number. 852 */ 853unsigned long vmalloc_to_pfn(const void *vmalloc_addr) 854{ 855 return page_to_pfn(vmalloc_to_page(vmalloc_addr)); 856} 857EXPORT_SYMBOL(vmalloc_to_pfn); 858 859 860/*** Global kva allocator ***/ 861 862#define DEBUG_AUGMENT_PROPAGATE_CHECK 0 863#define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0 864 865 866static DEFINE_SPINLOCK(free_vmap_area_lock); 867static bool vmap_initialized __read_mostly; 868 869/* 870 * This kmem_cache is used for vmap_area objects. Instead of 871 * allocating from slab we reuse an object from this cache to 872 * make things faster. Especially in "no edge" splitting of 873 * free block. 874 */ 875static struct kmem_cache *vmap_area_cachep; 876 877/* 878 * This linked list is used in pair with free_vmap_area_root. 879 * It gives O(1) access to prev/next to perform fast coalescing. 880 */ 881static LIST_HEAD(free_vmap_area_list); 882 883/* 884 * This augment red-black tree represents the free vmap space. 885 * All vmap_area objects in this tree are sorted by va->va_start 886 * address. It is used for allocation and merging when a vmap 887 * object is released. 888 * 889 * Each vmap_area node contains a maximum available free block 890 * of its sub-tree, right or left. Therefore it is possible to 891 * find a lowest match of free area. 892 */ 893static struct rb_root free_vmap_area_root = RB_ROOT; 894 895/* 896 * Preload a CPU with one object for "no edge" split case. The 897 * aim is to get rid of allocations from the atomic context, thus 898 * to use more permissive allocation masks. 899 */ 900static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node); 901 902/* 903 * This structure defines a single, solid model where a list and 904 * rb-tree are part of one entity protected by the lock. Nodes are 905 * sorted in ascending order, thus for O(1) access to left/right 906 * neighbors a list is used as well as for sequential traversal. 907 */ 908struct rb_list { 909 struct rb_root root; 910 struct list_head head; 911 spinlock_t lock; 912}; 913 914/* 915 * A fast size storage contains VAs up to 1M size. A pool consists 916 * of linked between each other ready to go VAs of certain sizes. 917 * An index in the pool-array corresponds to number of pages + 1. 918 */ 919#define MAX_VA_SIZE_PAGES 256 920 921struct vmap_pool { 922 struct list_head head; 923 unsigned long len; 924}; 925 926/* 927 * An effective vmap-node logic. Users make use of nodes instead 928 * of a global heap. It allows to balance an access and mitigate 929 * contention. 930 */ 931static struct vmap_node { 932 /* Simple size segregated storage. */ 933 struct vmap_pool pool[MAX_VA_SIZE_PAGES]; 934 spinlock_t pool_lock; 935 bool skip_populate; 936 937 /* Bookkeeping data of this node. */ 938 struct rb_list busy; 939 struct rb_list lazy; 940 941 /* 942 * Ready-to-free areas. 943 */ 944 struct list_head purge_list; 945 struct work_struct purge_work; 946 unsigned long nr_purged; 947} single; 948 949/* 950 * Initial setup consists of one single node, i.e. a balancing 951 * is fully disabled. Later on, after vmap is initialized these 952 * parameters are updated based on a system capacity. 953 */ 954static struct vmap_node *vmap_nodes = &single; 955static __read_mostly unsigned int nr_vmap_nodes = 1; 956static __read_mostly unsigned int vmap_zone_size = 1; 957 958/* A simple iterator over all vmap-nodes. */ 959#define for_each_vmap_node(vn) \ 960 for ((vn) = &vmap_nodes[0]; \ 961 (vn) < &vmap_nodes[nr_vmap_nodes]; (vn)++) 962 963static inline unsigned int 964addr_to_node_id(unsigned long addr) 965{ 966 return (addr / vmap_zone_size) % nr_vmap_nodes; 967} 968 969static inline struct vmap_node * 970addr_to_node(unsigned long addr) 971{ 972 return &vmap_nodes[addr_to_node_id(addr)]; 973} 974 975static inline struct vmap_node * 976id_to_node(unsigned int id) 977{ 978 return &vmap_nodes[id % nr_vmap_nodes]; 979} 980 981static inline unsigned int 982node_to_id(struct vmap_node *node) 983{ 984 /* Pointer arithmetic. */ 985 unsigned int id = node - vmap_nodes; 986 987 if (likely(id < nr_vmap_nodes)) 988 return id; 989 990 WARN_ONCE(1, "An address 0x%p is out-of-bounds.\n", node); 991 return 0; 992} 993 994/* 995 * We use the value 0 to represent "no node", that is why 996 * an encoded value will be the node-id incremented by 1. 997 * It is always greater then 0. A valid node_id which can 998 * be encoded is [0:nr_vmap_nodes - 1]. If a passed node_id 999 * is not valid 0 is returned. 1000 */ 1001static unsigned int 1002encode_vn_id(unsigned int node_id) 1003{ 1004 /* Can store U8_MAX [0:254] nodes. */ 1005 if (node_id < nr_vmap_nodes) 1006 return (node_id + 1) << BITS_PER_BYTE; 1007 1008 /* Warn and no node encoded. */ 1009 WARN_ONCE(1, "Encode wrong node id (%u)\n", node_id); 1010 return 0; 1011} 1012 1013/* 1014 * Returns an encoded node-id, the valid range is within 1015 * [0:nr_vmap_nodes-1] values. Otherwise nr_vmap_nodes is 1016 * returned if extracted data is wrong. 1017 */ 1018static unsigned int 1019decode_vn_id(unsigned int val) 1020{ 1021 unsigned int node_id = (val >> BITS_PER_BYTE) - 1; 1022 1023 /* Can store U8_MAX [0:254] nodes. */ 1024 if (node_id < nr_vmap_nodes) 1025 return node_id; 1026 1027 /* If it was _not_ zero, warn. */ 1028 WARN_ONCE(node_id != UINT_MAX, 1029 "Decode wrong node id (%d)\n", node_id); 1030 1031 return nr_vmap_nodes; 1032} 1033 1034static bool 1035is_vn_id_valid(unsigned int node_id) 1036{ 1037 if (node_id < nr_vmap_nodes) 1038 return true; 1039 1040 return false; 1041} 1042 1043static __always_inline unsigned long 1044va_size(struct vmap_area *va) 1045{ 1046 return (va->va_end - va->va_start); 1047} 1048 1049static __always_inline unsigned long 1050get_subtree_max_size(struct rb_node *node) 1051{ 1052 struct vmap_area *va; 1053 1054 va = rb_entry_safe(node, struct vmap_area, rb_node); 1055 return va ? va->subtree_max_size : 0; 1056} 1057 1058RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb, 1059 struct vmap_area, rb_node, unsigned long, subtree_max_size, va_size) 1060 1061static void reclaim_and_purge_vmap_areas(void); 1062static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); 1063static void drain_vmap_area_work(struct work_struct *work); 1064static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work); 1065 1066static __cacheline_aligned_in_smp atomic_long_t nr_vmalloc_pages; 1067static __cacheline_aligned_in_smp atomic_long_t vmap_lazy_nr; 1068 1069unsigned long vmalloc_nr_pages(void) 1070{ 1071 return atomic_long_read(&nr_vmalloc_pages); 1072} 1073 1074static struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root) 1075{ 1076 struct rb_node *n = root->rb_node; 1077 1078 addr = (unsigned long)kasan_reset_tag((void *)addr); 1079 1080 while (n) { 1081 struct vmap_area *va; 1082 1083 va = rb_entry(n, struct vmap_area, rb_node); 1084 if (addr < va->va_start) 1085 n = n->rb_left; 1086 else if (addr >= va->va_end) 1087 n = n->rb_right; 1088 else 1089 return va; 1090 } 1091 1092 return NULL; 1093} 1094 1095/* Look up the first VA which satisfies addr < va_end, NULL if none. */ 1096static struct vmap_area * 1097__find_vmap_area_exceed_addr(unsigned long addr, struct rb_root *root) 1098{ 1099 struct vmap_area *va = NULL; 1100 struct rb_node *n = root->rb_node; 1101 1102 addr = (unsigned long)kasan_reset_tag((void *)addr); 1103 1104 while (n) { 1105 struct vmap_area *tmp; 1106 1107 tmp = rb_entry(n, struct vmap_area, rb_node); 1108 if (tmp->va_end > addr) { 1109 va = tmp; 1110 if (tmp->va_start <= addr) 1111 break; 1112 1113 n = n->rb_left; 1114 } else 1115 n = n->rb_right; 1116 } 1117 1118 return va; 1119} 1120 1121/* 1122 * Returns a node where a first VA, that satisfies addr < va_end, resides. 1123 * If success, a node is locked. A user is responsible to unlock it when a 1124 * VA is no longer needed to be accessed. 1125 * 1126 * Returns NULL if nothing found. 1127 */ 1128static struct vmap_node * 1129find_vmap_area_exceed_addr_lock(unsigned long addr, struct vmap_area **va) 1130{ 1131 unsigned long va_start_lowest; 1132 struct vmap_node *vn; 1133 1134repeat: 1135 va_start_lowest = 0; 1136 1137 for_each_vmap_node(vn) { 1138 spin_lock(&vn->busy.lock); 1139 *va = __find_vmap_area_exceed_addr(addr, &vn->busy.root); 1140 1141 if (*va) 1142 if (!va_start_lowest || (*va)->va_start < va_start_lowest) 1143 va_start_lowest = (*va)->va_start; 1144 spin_unlock(&vn->busy.lock); 1145 } 1146 1147 /* 1148 * Check if found VA exists, it might have gone away. In this case we 1149 * repeat the search because a VA has been removed concurrently and we 1150 * need to proceed to the next one, which is a rare case. 1151 */ 1152 if (va_start_lowest) { 1153 vn = addr_to_node(va_start_lowest); 1154 1155 spin_lock(&vn->busy.lock); 1156 *va = __find_vmap_area(va_start_lowest, &vn->busy.root); 1157 1158 if (*va) 1159 return vn; 1160 1161 spin_unlock(&vn->busy.lock); 1162 goto repeat; 1163 } 1164 1165 return NULL; 1166} 1167 1168/* 1169 * This function returns back addresses of parent node 1170 * and its left or right link for further processing. 1171 * 1172 * Otherwise NULL is returned. In that case all further 1173 * steps regarding inserting of conflicting overlap range 1174 * have to be declined and actually considered as a bug. 1175 */ 1176static __always_inline struct rb_node ** 1177find_va_links(struct vmap_area *va, 1178 struct rb_root *root, struct rb_node *from, 1179 struct rb_node **parent) 1180{ 1181 struct vmap_area *tmp_va; 1182 struct rb_node **link; 1183 1184 if (root) { 1185 link = &root->rb_node; 1186 if (unlikely(!*link)) { 1187 *parent = NULL; 1188 return link; 1189 } 1190 } else { 1191 link = &from; 1192 } 1193 1194 /* 1195 * Go to the bottom of the tree. When we hit the last point 1196 * we end up with parent rb_node and correct direction, i name 1197 * it link, where the new va->rb_node will be attached to. 1198 */ 1199 do { 1200 tmp_va = rb_entry(*link, struct vmap_area, rb_node); 1201 1202 /* 1203 * During the traversal we also do some sanity check. 1204 * Trigger the BUG() if there are sides(left/right) 1205 * or full overlaps. 1206 */ 1207 if (va->va_end <= tmp_va->va_start) 1208 link = &(*link)->rb_left; 1209 else if (va->va_start >= tmp_va->va_end) 1210 link = &(*link)->rb_right; 1211 else { 1212 WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n", 1213 va->va_start, va->va_end, tmp_va->va_start, tmp_va->va_end); 1214 1215 return NULL; 1216 } 1217 } while (*link); 1218 1219 *parent = &tmp_va->rb_node; 1220 return link; 1221} 1222 1223static __always_inline struct list_head * 1224get_va_next_sibling(struct rb_node *parent, struct rb_node **link) 1225{ 1226 struct list_head *list; 1227 1228 if (unlikely(!parent)) 1229 /* 1230 * The red-black tree where we try to find VA neighbors 1231 * before merging or inserting is empty, i.e. it means 1232 * there is no free vmap space. Normally it does not 1233 * happen but we handle this case anyway. 1234 */ 1235 return NULL; 1236 1237 list = &rb_entry(parent, struct vmap_area, rb_node)->list; 1238 return (&parent->rb_right == link ? list->next : list); 1239} 1240 1241static __always_inline void 1242__link_va(struct vmap_area *va, struct rb_root *root, 1243 struct rb_node *parent, struct rb_node **link, 1244 struct list_head *head, bool augment) 1245{ 1246 /* 1247 * VA is still not in the list, but we can 1248 * identify its future previous list_head node. 1249 */ 1250 if (likely(parent)) { 1251 head = &rb_entry(parent, struct vmap_area, rb_node)->list; 1252 if (&parent->rb_right != link) 1253 head = head->prev; 1254 } 1255 1256 /* Insert to the rb-tree */ 1257 rb_link_node(&va->rb_node, parent, link); 1258 if (augment) { 1259 /* 1260 * Some explanation here. Just perform simple insertion 1261 * to the tree. We do not set va->subtree_max_size to 1262 * its current size before calling rb_insert_augmented(). 1263 * It is because we populate the tree from the bottom 1264 * to parent levels when the node _is_ in the tree. 1265 * 1266 * Therefore we set subtree_max_size to zero after insertion, 1267 * to let __augment_tree_propagate_from() puts everything to 1268 * the correct order later on. 1269 */ 1270 rb_insert_augmented(&va->rb_node, 1271 root, &free_vmap_area_rb_augment_cb); 1272 va->subtree_max_size = 0; 1273 } else { 1274 rb_insert_color(&va->rb_node, root); 1275 } 1276 1277 /* Address-sort this list */ 1278 list_add(&va->list, head); 1279} 1280 1281static __always_inline void 1282link_va(struct vmap_area *va, struct rb_root *root, 1283 struct rb_node *parent, struct rb_node **link, 1284 struct list_head *head) 1285{ 1286 __link_va(va, root, parent, link, head, false); 1287} 1288 1289static __always_inline void 1290link_va_augment(struct vmap_area *va, struct rb_root *root, 1291 struct rb_node *parent, struct rb_node **link, 1292 struct list_head *head) 1293{ 1294 __link_va(va, root, parent, link, head, true); 1295} 1296 1297static __always_inline void 1298__unlink_va(struct vmap_area *va, struct rb_root *root, bool augment) 1299{ 1300 if (WARN_ON(RB_EMPTY_NODE(&va->rb_node))) 1301 return; 1302 1303 if (augment) 1304 rb_erase_augmented(&va->rb_node, 1305 root, &free_vmap_area_rb_augment_cb); 1306 else 1307 rb_erase(&va->rb_node, root); 1308 1309 list_del_init(&va->list); 1310 RB_CLEAR_NODE(&va->rb_node); 1311} 1312 1313static __always_inline void 1314unlink_va(struct vmap_area *va, struct rb_root *root) 1315{ 1316 __unlink_va(va, root, false); 1317} 1318 1319static __always_inline void 1320unlink_va_augment(struct vmap_area *va, struct rb_root *root) 1321{ 1322 __unlink_va(va, root, true); 1323} 1324 1325#if DEBUG_AUGMENT_PROPAGATE_CHECK 1326/* 1327 * Gets called when remove the node and rotate. 1328 */ 1329static __always_inline unsigned long 1330compute_subtree_max_size(struct vmap_area *va) 1331{ 1332 return max3(va_size(va), 1333 get_subtree_max_size(va->rb_node.rb_left), 1334 get_subtree_max_size(va->rb_node.rb_right)); 1335} 1336 1337static void 1338augment_tree_propagate_check(void) 1339{ 1340 struct vmap_area *va; 1341 unsigned long computed_size; 1342 1343 list_for_each_entry(va, &free_vmap_area_list, list) { 1344 computed_size = compute_subtree_max_size(va); 1345 if (computed_size != va->subtree_max_size) 1346 pr_emerg("tree is corrupted: %lu, %lu\n", 1347 va_size(va), va->subtree_max_size); 1348 } 1349} 1350#endif 1351 1352/* 1353 * This function populates subtree_max_size from bottom to upper 1354 * levels starting from VA point. The propagation must be done 1355 * when VA size is modified by changing its va_start/va_end. Or 1356 * in case of newly inserting of VA to the tree. 1357 * 1358 * It means that __augment_tree_propagate_from() must be called: 1359 * - After VA has been inserted to the tree(free path); 1360 * - After VA has been shrunk(allocation path); 1361 * - After VA has been increased(merging path). 1362 * 1363 * Please note that, it does not mean that upper parent nodes 1364 * and their subtree_max_size are recalculated all the time up 1365 * to the root node. 1366 * 1367 * 4--8 1368 * /\ 1369 * / \ 1370 * / \ 1371 * 2--2 8--8 1372 * 1373 * For example if we modify the node 4, shrinking it to 2, then 1374 * no any modification is required. If we shrink the node 2 to 1 1375 * its subtree_max_size is updated only, and set to 1. If we shrink 1376 * the node 8 to 6, then its subtree_max_size is set to 6 and parent 1377 * node becomes 4--6. 1378 */ 1379static __always_inline void 1380augment_tree_propagate_from(struct vmap_area *va) 1381{ 1382 /* 1383 * Populate the tree from bottom towards the root until 1384 * the calculated maximum available size of checked node 1385 * is equal to its current one. 1386 */ 1387 free_vmap_area_rb_augment_cb_propagate(&va->rb_node, NULL); 1388 1389#if DEBUG_AUGMENT_PROPAGATE_CHECK 1390 augment_tree_propagate_check(); 1391#endif 1392} 1393 1394static void 1395insert_vmap_area(struct vmap_area *va, 1396 struct rb_root *root, struct list_head *head) 1397{ 1398 struct rb_node **link; 1399 struct rb_node *parent; 1400 1401 link = find_va_links(va, root, NULL, &parent); 1402 if (link) 1403 link_va(va, root, parent, link, head); 1404} 1405 1406static void 1407insert_vmap_area_augment(struct vmap_area *va, 1408 struct rb_node *from, struct rb_root *root, 1409 struct list_head *head) 1410{ 1411 struct rb_node **link; 1412 struct rb_node *parent; 1413 1414 if (from) 1415 link = find_va_links(va, NULL, from, &parent); 1416 else 1417 link = find_va_links(va, root, NULL, &parent); 1418 1419 if (link) { 1420 link_va_augment(va, root, parent, link, head); 1421 augment_tree_propagate_from(va); 1422 } 1423} 1424 1425/* 1426 * Merge de-allocated chunk of VA memory with previous 1427 * and next free blocks. If coalesce is not done a new 1428 * free area is inserted. If VA has been merged, it is 1429 * freed. 1430 * 1431 * Please note, it can return NULL in case of overlap 1432 * ranges, followed by WARN() report. Despite it is a 1433 * buggy behaviour, a system can be alive and keep 1434 * ongoing. 1435 */ 1436static __always_inline struct vmap_area * 1437__merge_or_add_vmap_area(struct vmap_area *va, 1438 struct rb_root *root, struct list_head *head, bool augment) 1439{ 1440 struct vmap_area *sibling; 1441 struct list_head *next; 1442 struct rb_node **link; 1443 struct rb_node *parent; 1444 bool merged = false; 1445 1446 /* 1447 * Find a place in the tree where VA potentially will be 1448 * inserted, unless it is merged with its sibling/siblings. 1449 */ 1450 link = find_va_links(va, root, NULL, &parent); 1451 if (!link) 1452 return NULL; 1453 1454 /* 1455 * Get next node of VA to check if merging can be done. 1456 */ 1457 next = get_va_next_sibling(parent, link); 1458 if (unlikely(next == NULL)) 1459 goto insert; 1460 1461 /* 1462 * start end 1463 * | | 1464 * |<------VA------>|<-----Next----->| 1465 * | | 1466 * start end 1467 */ 1468 if (next != head) { 1469 sibling = list_entry(next, struct vmap_area, list); 1470 if (sibling->va_start == va->va_end) { 1471 sibling->va_start = va->va_start; 1472 1473 /* Free vmap_area object. */ 1474 kmem_cache_free(vmap_area_cachep, va); 1475 1476 /* Point to the new merged area. */ 1477 va = sibling; 1478 merged = true; 1479 } 1480 } 1481 1482 /* 1483 * start end 1484 * | | 1485 * |<-----Prev----->|<------VA------>| 1486 * | | 1487 * start end 1488 */ 1489 if (next->prev != head) { 1490 sibling = list_entry(next->prev, struct vmap_area, list); 1491 if (sibling->va_end == va->va_start) { 1492 /* 1493 * If both neighbors are coalesced, it is important 1494 * to unlink the "next" node first, followed by merging 1495 * with "previous" one. Otherwise the tree might not be 1496 * fully populated if a sibling's augmented value is 1497 * "normalized" because of rotation operations. 1498 */ 1499 if (merged) 1500 __unlink_va(va, root, augment); 1501 1502 sibling->va_end = va->va_end; 1503 1504 /* Free vmap_area object. */ 1505 kmem_cache_free(vmap_area_cachep, va); 1506 1507 /* Point to the new merged area. */ 1508 va = sibling; 1509 merged = true; 1510 } 1511 } 1512 1513insert: 1514 if (!merged) 1515 __link_va(va, root, parent, link, head, augment); 1516 1517 return va; 1518} 1519 1520static __always_inline struct vmap_area * 1521merge_or_add_vmap_area(struct vmap_area *va, 1522 struct rb_root *root, struct list_head *head) 1523{ 1524 return __merge_or_add_vmap_area(va, root, head, false); 1525} 1526 1527static __always_inline struct vmap_area * 1528merge_or_add_vmap_area_augment(struct vmap_area *va, 1529 struct rb_root *root, struct list_head *head) 1530{ 1531 va = __merge_or_add_vmap_area(va, root, head, true); 1532 if (va) 1533 augment_tree_propagate_from(va); 1534 1535 return va; 1536} 1537 1538static __always_inline bool 1539is_within_this_va(struct vmap_area *va, unsigned long size, 1540 unsigned long align, unsigned long vstart) 1541{ 1542 unsigned long nva_start_addr; 1543 1544 if (va->va_start > vstart) 1545 nva_start_addr = ALIGN(va->va_start, align); 1546 else 1547 nva_start_addr = ALIGN(vstart, align); 1548 1549 /* Can be overflowed due to big size or alignment. */ 1550 if (nva_start_addr + size < nva_start_addr || 1551 nva_start_addr < vstart) 1552 return false; 1553 1554 return (nva_start_addr + size <= va->va_end); 1555} 1556 1557/* 1558 * Find the first free block(lowest start address) in the tree, 1559 * that will accomplish the request corresponding to passing 1560 * parameters. Please note, with an alignment bigger than PAGE_SIZE, 1561 * a search length is adjusted to account for worst case alignment 1562 * overhead. 1563 */ 1564static __always_inline struct vmap_area * 1565find_vmap_lowest_match(struct rb_root *root, unsigned long size, 1566 unsigned long align, unsigned long vstart, bool adjust_search_size) 1567{ 1568 struct vmap_area *va; 1569 struct rb_node *node; 1570 unsigned long length; 1571 1572 /* Start from the root. */ 1573 node = root->rb_node; 1574 1575 /* Adjust the search size for alignment overhead. */ 1576 length = adjust_search_size ? size + align - 1 : size; 1577 1578 while (node) { 1579 va = rb_entry(node, struct vmap_area, rb_node); 1580 1581 if (get_subtree_max_size(node->rb_left) >= length && 1582 vstart < va->va_start) { 1583 node = node->rb_left; 1584 } else { 1585 if (is_within_this_va(va, size, align, vstart)) 1586 return va; 1587 1588 /* 1589 * Does not make sense to go deeper towards the right 1590 * sub-tree if it does not have a free block that is 1591 * equal or bigger to the requested search length. 1592 */ 1593 if (get_subtree_max_size(node->rb_right) >= length) { 1594 node = node->rb_right; 1595 continue; 1596 } 1597 1598 /* 1599 * OK. We roll back and find the first right sub-tree, 1600 * that will satisfy the search criteria. It can happen 1601 * due to "vstart" restriction or an alignment overhead 1602 * that is bigger then PAGE_SIZE. 1603 */ 1604 while ((node = rb_parent(node))) { 1605 va = rb_entry(node, struct vmap_area, rb_node); 1606 if (is_within_this_va(va, size, align, vstart)) 1607 return va; 1608 1609 if (get_subtree_max_size(node->rb_right) >= length && 1610 vstart <= va->va_start) { 1611 /* 1612 * Shift the vstart forward. Please note, we update it with 1613 * parent's start address adding "1" because we do not want 1614 * to enter same sub-tree after it has already been checked 1615 * and no suitable free block found there. 1616 */ 1617 vstart = va->va_start + 1; 1618 node = node->rb_right; 1619 break; 1620 } 1621 } 1622 } 1623 } 1624 1625 return NULL; 1626} 1627 1628#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1629#include <linux/random.h> 1630 1631static struct vmap_area * 1632find_vmap_lowest_linear_match(struct list_head *head, unsigned long size, 1633 unsigned long align, unsigned long vstart) 1634{ 1635 struct vmap_area *va; 1636 1637 list_for_each_entry(va, head, list) { 1638 if (!is_within_this_va(va, size, align, vstart)) 1639 continue; 1640 1641 return va; 1642 } 1643 1644 return NULL; 1645} 1646 1647static void 1648find_vmap_lowest_match_check(struct rb_root *root, struct list_head *head, 1649 unsigned long size, unsigned long align) 1650{ 1651 struct vmap_area *va_1, *va_2; 1652 unsigned long vstart; 1653 unsigned int rnd; 1654 1655 get_random_bytes(&rnd, sizeof(rnd)); 1656 vstart = VMALLOC_START + rnd; 1657 1658 va_1 = find_vmap_lowest_match(root, size, align, vstart, false); 1659 va_2 = find_vmap_lowest_linear_match(head, size, align, vstart); 1660 1661 if (va_1 != va_2) 1662 pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", 1663 va_1, va_2, vstart); 1664} 1665#endif 1666 1667enum fit_type { 1668 NOTHING_FIT = 0, 1669 FL_FIT_TYPE = 1, /* full fit */ 1670 LE_FIT_TYPE = 2, /* left edge fit */ 1671 RE_FIT_TYPE = 3, /* right edge fit */ 1672 NE_FIT_TYPE = 4 /* no edge fit */ 1673}; 1674 1675static __always_inline enum fit_type 1676classify_va_fit_type(struct vmap_area *va, 1677 unsigned long nva_start_addr, unsigned long size) 1678{ 1679 enum fit_type type; 1680 1681 /* Check if it is within VA. */ 1682 if (nva_start_addr < va->va_start || 1683 nva_start_addr + size > va->va_end) 1684 return NOTHING_FIT; 1685 1686 /* Now classify. */ 1687 if (va->va_start == nva_start_addr) { 1688 if (va->va_end == nva_start_addr + size) 1689 type = FL_FIT_TYPE; 1690 else 1691 type = LE_FIT_TYPE; 1692 } else if (va->va_end == nva_start_addr + size) { 1693 type = RE_FIT_TYPE; 1694 } else { 1695 type = NE_FIT_TYPE; 1696 } 1697 1698 return type; 1699} 1700 1701static __always_inline int 1702va_clip(struct rb_root *root, struct list_head *head, 1703 struct vmap_area *va, unsigned long nva_start_addr, 1704 unsigned long size) 1705{ 1706 struct vmap_area *lva = NULL; 1707 enum fit_type type = classify_va_fit_type(va, nva_start_addr, size); 1708 1709 if (type == FL_FIT_TYPE) { 1710 /* 1711 * No need to split VA, it fully fits. 1712 * 1713 * | | 1714 * V NVA V 1715 * |---------------| 1716 */ 1717 unlink_va_augment(va, root); 1718 kmem_cache_free(vmap_area_cachep, va); 1719 } else if (type == LE_FIT_TYPE) { 1720 /* 1721 * Split left edge of fit VA. 1722 * 1723 * | | 1724 * V NVA V R 1725 * |-------|-------| 1726 */ 1727 va->va_start += size; 1728 } else if (type == RE_FIT_TYPE) { 1729 /* 1730 * Split right edge of fit VA. 1731 * 1732 * | | 1733 * L V NVA V 1734 * |-------|-------| 1735 */ 1736 va->va_end = nva_start_addr; 1737 } else if (type == NE_FIT_TYPE) { 1738 /* 1739 * Split no edge of fit VA. 1740 * 1741 * | | 1742 * L V NVA V R 1743 * |---|-------|---| 1744 */ 1745 lva = __this_cpu_xchg(ne_fit_preload_node, NULL); 1746 if (unlikely(!lva)) { 1747 /* 1748 * For percpu allocator we do not do any pre-allocation 1749 * and leave it as it is. The reason is it most likely 1750 * never ends up with NE_FIT_TYPE splitting. In case of 1751 * percpu allocations offsets and sizes are aligned to 1752 * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE 1753 * are its main fitting cases. 1754 * 1755 * There are a few exceptions though, as an example it is 1756 * a first allocation (early boot up) when we have "one" 1757 * big free space that has to be split. 1758 * 1759 * Also we can hit this path in case of regular "vmap" 1760 * allocations, if "this" current CPU was not preloaded. 1761 * See the comment in alloc_vmap_area() why. If so, then 1762 * GFP_NOWAIT is used instead to get an extra object for 1763 * split purpose. That is rare and most time does not 1764 * occur. 1765 * 1766 * What happens if an allocation gets failed. Basically, 1767 * an "overflow" path is triggered to purge lazily freed 1768 * areas to free some memory, then, the "retry" path is 1769 * triggered to repeat one more time. See more details 1770 * in alloc_vmap_area() function. 1771 */ 1772 lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT); 1773 if (!lva) 1774 return -ENOMEM; 1775 } 1776 1777 /* 1778 * Build the remainder. 1779 */ 1780 lva->va_start = va->va_start; 1781 lva->va_end = nva_start_addr; 1782 1783 /* 1784 * Shrink this VA to remaining size. 1785 */ 1786 va->va_start = nva_start_addr + size; 1787 } else { 1788 return -EINVAL; 1789 } 1790 1791 if (type != FL_FIT_TYPE) { 1792 augment_tree_propagate_from(va); 1793 1794 if (lva) /* type == NE_FIT_TYPE */ 1795 insert_vmap_area_augment(lva, &va->rb_node, root, head); 1796 } 1797 1798 return 0; 1799} 1800 1801static unsigned long 1802va_alloc(struct vmap_area *va, 1803 struct rb_root *root, struct list_head *head, 1804 unsigned long size, unsigned long align, 1805 unsigned long vstart, unsigned long vend) 1806{ 1807 unsigned long nva_start_addr; 1808 int ret; 1809 1810 if (va->va_start > vstart) 1811 nva_start_addr = ALIGN(va->va_start, align); 1812 else 1813 nva_start_addr = ALIGN(vstart, align); 1814 1815 /* Check the "vend" restriction. */ 1816 if (nva_start_addr + size > vend) 1817 return -ERANGE; 1818 1819 /* Update the free vmap_area. */ 1820 ret = va_clip(root, head, va, nva_start_addr, size); 1821 if (WARN_ON_ONCE(ret)) 1822 return ret; 1823 1824 return nva_start_addr; 1825} 1826 1827/* 1828 * Returns a start address of the newly allocated area, if success. 1829 * Otherwise an error value is returned that indicates failure. 1830 */ 1831static __always_inline unsigned long 1832__alloc_vmap_area(struct rb_root *root, struct list_head *head, 1833 unsigned long size, unsigned long align, 1834 unsigned long vstart, unsigned long vend) 1835{ 1836 bool adjust_search_size = true; 1837 unsigned long nva_start_addr; 1838 struct vmap_area *va; 1839 1840 /* 1841 * Do not adjust when: 1842 * a) align <= PAGE_SIZE, because it does not make any sense. 1843 * All blocks(their start addresses) are at least PAGE_SIZE 1844 * aligned anyway; 1845 * b) a short range where a requested size corresponds to exactly 1846 * specified [vstart:vend] interval and an alignment > PAGE_SIZE. 1847 * With adjusted search length an allocation would not succeed. 1848 */ 1849 if (align <= PAGE_SIZE || (align > PAGE_SIZE && (vend - vstart) == size)) 1850 adjust_search_size = false; 1851 1852 va = find_vmap_lowest_match(root, size, align, vstart, adjust_search_size); 1853 if (unlikely(!va)) 1854 return -ENOENT; 1855 1856 nva_start_addr = va_alloc(va, root, head, size, align, vstart, vend); 1857 1858#if DEBUG_AUGMENT_LOWEST_MATCH_CHECK 1859 if (!IS_ERR_VALUE(nva_start_addr)) 1860 find_vmap_lowest_match_check(root, head, size, align); 1861#endif 1862 1863 return nva_start_addr; 1864} 1865 1866/* 1867 * Free a region of KVA allocated by alloc_vmap_area 1868 */ 1869static void free_vmap_area(struct vmap_area *va) 1870{ 1871 struct vmap_node *vn = addr_to_node(va->va_start); 1872 1873 /* 1874 * Remove from the busy tree/list. 1875 */ 1876 spin_lock(&vn->busy.lock); 1877 unlink_va(va, &vn->busy.root); 1878 spin_unlock(&vn->busy.lock); 1879 1880 /* 1881 * Insert/Merge it back to the free tree/list. 1882 */ 1883 spin_lock(&free_vmap_area_lock); 1884 merge_or_add_vmap_area_augment(va, &free_vmap_area_root, &free_vmap_area_list); 1885 spin_unlock(&free_vmap_area_lock); 1886} 1887 1888static inline void 1889preload_this_cpu_lock(spinlock_t *lock, gfp_t gfp_mask, int node) 1890{ 1891 struct vmap_area *va = NULL, *tmp; 1892 1893 /* 1894 * Preload this CPU with one extra vmap_area object. It is used 1895 * when fit type of free area is NE_FIT_TYPE. It guarantees that 1896 * a CPU that does an allocation is preloaded. 1897 * 1898 * We do it in non-atomic context, thus it allows us to use more 1899 * permissive allocation masks to be more stable under low memory 1900 * condition and high memory pressure. 1901 */ 1902 if (!this_cpu_read(ne_fit_preload_node)) 1903 va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 1904 1905 spin_lock(lock); 1906 1907 tmp = NULL; 1908 if (va && !__this_cpu_try_cmpxchg(ne_fit_preload_node, &tmp, va)) 1909 kmem_cache_free(vmap_area_cachep, va); 1910} 1911 1912static struct vmap_pool * 1913size_to_va_pool(struct vmap_node *vn, unsigned long size) 1914{ 1915 unsigned int idx = (size - 1) / PAGE_SIZE; 1916 1917 if (idx < MAX_VA_SIZE_PAGES) 1918 return &vn->pool[idx]; 1919 1920 return NULL; 1921} 1922 1923static bool 1924node_pool_add_va(struct vmap_node *n, struct vmap_area *va) 1925{ 1926 struct vmap_pool *vp; 1927 1928 vp = size_to_va_pool(n, va_size(va)); 1929 if (!vp) 1930 return false; 1931 1932 spin_lock(&n->pool_lock); 1933 list_add(&va->list, &vp->head); 1934 WRITE_ONCE(vp->len, vp->len + 1); 1935 spin_unlock(&n->pool_lock); 1936 1937 return true; 1938} 1939 1940static struct vmap_area * 1941node_pool_del_va(struct vmap_node *vn, unsigned long size, 1942 unsigned long align, unsigned long vstart, 1943 unsigned long vend) 1944{ 1945 struct vmap_area *va = NULL; 1946 struct vmap_pool *vp; 1947 int err = 0; 1948 1949 vp = size_to_va_pool(vn, size); 1950 if (!vp || list_empty(&vp->head)) 1951 return NULL; 1952 1953 spin_lock(&vn->pool_lock); 1954 if (!list_empty(&vp->head)) { 1955 va = list_first_entry(&vp->head, struct vmap_area, list); 1956 1957 if (IS_ALIGNED(va->va_start, align)) { 1958 /* 1959 * Do some sanity check and emit a warning 1960 * if one of below checks detects an error. 1961 */ 1962 err |= (va_size(va) != size); 1963 err |= (va->va_start < vstart); 1964 err |= (va->va_end > vend); 1965 1966 if (!WARN_ON_ONCE(err)) { 1967 list_del_init(&va->list); 1968 WRITE_ONCE(vp->len, vp->len - 1); 1969 } else { 1970 va = NULL; 1971 } 1972 } else { 1973 list_move_tail(&va->list, &vp->head); 1974 va = NULL; 1975 } 1976 } 1977 spin_unlock(&vn->pool_lock); 1978 1979 return va; 1980} 1981 1982static struct vmap_area * 1983node_alloc(unsigned long size, unsigned long align, 1984 unsigned long vstart, unsigned long vend, 1985 unsigned long *addr, unsigned int *vn_id) 1986{ 1987 struct vmap_area *va; 1988 1989 *vn_id = 0; 1990 *addr = -EINVAL; 1991 1992 /* 1993 * Fallback to a global heap if not vmalloc or there 1994 * is only one node. 1995 */ 1996 if (vstart != VMALLOC_START || vend != VMALLOC_END || 1997 nr_vmap_nodes == 1) 1998 return NULL; 1999 2000 *vn_id = raw_smp_processor_id() % nr_vmap_nodes; 2001 va = node_pool_del_va(id_to_node(*vn_id), size, align, vstart, vend); 2002 *vn_id = encode_vn_id(*vn_id); 2003 2004 if (va) 2005 *addr = va->va_start; 2006 2007 return va; 2008} 2009 2010static inline void setup_vmalloc_vm(struct vm_struct *vm, 2011 struct vmap_area *va, unsigned long flags, const void *caller) 2012{ 2013 vm->flags = flags; 2014 vm->addr = (void *)va->va_start; 2015 vm->size = vm->requested_size = va_size(va); 2016 vm->caller = caller; 2017 va->vm = vm; 2018} 2019 2020/* 2021 * Allocate a region of KVA of the specified size and alignment, within the 2022 * vstart and vend. If vm is passed in, the two will also be bound. 2023 */ 2024static struct vmap_area *alloc_vmap_area(unsigned long size, 2025 unsigned long align, 2026 unsigned long vstart, unsigned long vend, 2027 int node, gfp_t gfp_mask, 2028 unsigned long va_flags, struct vm_struct *vm) 2029{ 2030 struct vmap_node *vn; 2031 struct vmap_area *va; 2032 unsigned long freed; 2033 unsigned long addr; 2034 unsigned int vn_id; 2035 bool allow_block; 2036 int purged = 0; 2037 int ret; 2038 2039 if (unlikely(!size || offset_in_page(size) || !is_power_of_2(align))) 2040 return ERR_PTR(-EINVAL); 2041 2042 if (unlikely(!vmap_initialized)) 2043 return ERR_PTR(-EBUSY); 2044 2045 /* Only reclaim behaviour flags are relevant. */ 2046 gfp_mask = gfp_mask & GFP_RECLAIM_MASK; 2047 allow_block = gfpflags_allow_blocking(gfp_mask); 2048 might_sleep_if(allow_block); 2049 2050 /* 2051 * If a VA is obtained from a global heap(if it fails here) 2052 * it is anyway marked with this "vn_id" so it is returned 2053 * to this pool's node later. Such way gives a possibility 2054 * to populate pools based on users demand. 2055 * 2056 * On success a ready to go VA is returned. 2057 */ 2058 va = node_alloc(size, align, vstart, vend, &addr, &vn_id); 2059 if (!va) { 2060 va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node); 2061 if (unlikely(!va)) 2062 return ERR_PTR(-ENOMEM); 2063 2064 /* 2065 * Only scan the relevant parts containing pointers to other objects 2066 * to avoid false negatives. 2067 */ 2068 kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask); 2069 } 2070 2071retry: 2072 if (IS_ERR_VALUE(addr)) { 2073 preload_this_cpu_lock(&free_vmap_area_lock, gfp_mask, node); 2074 addr = __alloc_vmap_area(&free_vmap_area_root, &free_vmap_area_list, 2075 size, align, vstart, vend); 2076 spin_unlock(&free_vmap_area_lock); 2077 2078 /* 2079 * This is not a fast path. Check if yielding is needed. This 2080 * is the only reschedule point in the vmalloc() path. 2081 */ 2082 if (allow_block) 2083 cond_resched(); 2084 } 2085 2086 trace_alloc_vmap_area(addr, size, align, vstart, vend, IS_ERR_VALUE(addr)); 2087 2088 /* 2089 * If an allocation fails, the error value is 2090 * returned. Therefore trigger the overflow path. 2091 */ 2092 if (IS_ERR_VALUE(addr)) { 2093 if (allow_block) 2094 goto overflow; 2095 2096 /* 2097 * We can not trigger any reclaim logic because 2098 * sleeping is not allowed, thus fail an allocation. 2099 */ 2100 goto out_free_va; 2101 } 2102 2103 va->va_start = addr; 2104 va->va_end = addr + size; 2105 va->vm = NULL; 2106 va->flags = (va_flags | vn_id); 2107 2108 if (vm) { 2109 vm->addr = (void *)va->va_start; 2110 vm->size = va_size(va); 2111 va->vm = vm; 2112 } 2113 2114 vn = addr_to_node(va->va_start); 2115 2116 spin_lock(&vn->busy.lock); 2117 insert_vmap_area(va, &vn->busy.root, &vn->busy.head); 2118 spin_unlock(&vn->busy.lock); 2119 2120 BUG_ON(!IS_ALIGNED(va->va_start, align)); 2121 BUG_ON(va->va_start < vstart); 2122 BUG_ON(va->va_end > vend); 2123 2124 ret = kasan_populate_vmalloc(addr, size, gfp_mask); 2125 if (ret) { 2126 free_vmap_area(va); 2127 return ERR_PTR(ret); 2128 } 2129 2130 return va; 2131 2132overflow: 2133 if (!purged) { 2134 reclaim_and_purge_vmap_areas(); 2135 purged = 1; 2136 goto retry; 2137 } 2138 2139 freed = 0; 2140 blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); 2141 2142 if (freed > 0) { 2143 purged = 0; 2144 goto retry; 2145 } 2146 2147 if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) 2148 pr_warn("vmalloc_node_range for size %lu failed: Address range restricted to %#lx - %#lx\n", 2149 size, vstart, vend); 2150 2151out_free_va: 2152 kmem_cache_free(vmap_area_cachep, va); 2153 return ERR_PTR(-EBUSY); 2154} 2155 2156int register_vmap_purge_notifier(struct notifier_block *nb) 2157{ 2158 return blocking_notifier_chain_register(&vmap_notify_list, nb); 2159} 2160EXPORT_SYMBOL_GPL(register_vmap_purge_notifier); 2161 2162int unregister_vmap_purge_notifier(struct notifier_block *nb) 2163{ 2164 return blocking_notifier_chain_unregister(&vmap_notify_list, nb); 2165} 2166EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier); 2167 2168/* 2169 * lazy_max_pages is the maximum amount of virtual address space we gather up 2170 * before attempting to purge with a TLB flush. 2171 * 2172 * There is a tradeoff here: a larger number will cover more kernel page tables 2173 * and take slightly longer to purge, but it will linearly reduce the number of 2174 * global TLB flushes that must be performed. It would seem natural to scale 2175 * this number up linearly with the number of CPUs (because vmapping activity 2176 * could also scale linearly with the number of CPUs), however it is likely 2177 * that in practice, workloads might be constrained in other ways that mean 2178 * vmap activity will not scale linearly with CPUs. Also, I want to be 2179 * conservative and not introduce a big latency on huge systems, so go with 2180 * a less aggressive log scale. It will still be an improvement over the old 2181 * code, and it will be simple to change the scale factor if we find that it 2182 * becomes a problem on bigger systems. 2183 */ 2184static unsigned long lazy_max_pages(void) 2185{ 2186 unsigned int log; 2187 2188 log = fls(num_online_cpus()); 2189 2190 return log * (32UL * 1024 * 1024 / PAGE_SIZE); 2191} 2192 2193/* 2194 * Serialize vmap purging. There is no actual critical section protected 2195 * by this lock, but we want to avoid concurrent calls for performance 2196 * reasons and to make the pcpu_get_vm_areas more deterministic. 2197 */ 2198static DEFINE_MUTEX(vmap_purge_lock); 2199 2200/* for per-CPU blocks */ 2201static void purge_fragmented_blocks_allcpus(void); 2202 2203static void 2204reclaim_list_global(struct list_head *head) 2205{ 2206 struct vmap_area *va, *n; 2207 2208 if (list_empty(head)) 2209 return; 2210 2211 spin_lock(&free_vmap_area_lock); 2212 list_for_each_entry_safe(va, n, head, list) 2213 merge_or_add_vmap_area_augment(va, 2214 &free_vmap_area_root, &free_vmap_area_list); 2215 spin_unlock(&free_vmap_area_lock); 2216} 2217 2218static void 2219decay_va_pool_node(struct vmap_node *vn, bool full_decay) 2220{ 2221 LIST_HEAD(decay_list); 2222 struct rb_root decay_root = RB_ROOT; 2223 struct vmap_area *va, *nva; 2224 unsigned long n_decay, pool_len; 2225 int i; 2226 2227 for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { 2228 LIST_HEAD(tmp_list); 2229 2230 if (list_empty(&vn->pool[i].head)) 2231 continue; 2232 2233 /* Detach the pool, so no-one can access it. */ 2234 spin_lock(&vn->pool_lock); 2235 list_replace_init(&vn->pool[i].head, &tmp_list); 2236 spin_unlock(&vn->pool_lock); 2237 2238 pool_len = n_decay = vn->pool[i].len; 2239 WRITE_ONCE(vn->pool[i].len, 0); 2240 2241 /* Decay a pool by ~25% out of left objects. */ 2242 if (!full_decay) 2243 n_decay >>= 2; 2244 pool_len -= n_decay; 2245 2246 list_for_each_entry_safe(va, nva, &tmp_list, list) { 2247 if (!n_decay--) 2248 break; 2249 2250 list_del_init(&va->list); 2251 merge_or_add_vmap_area(va, &decay_root, &decay_list); 2252 } 2253 2254 /* 2255 * Attach the pool back if it has been partly decayed. 2256 * Please note, it is supposed that nobody(other contexts) 2257 * can populate the pool therefore a simple list replace 2258 * operation takes place here. 2259 */ 2260 if (!list_empty(&tmp_list)) { 2261 spin_lock(&vn->pool_lock); 2262 list_replace_init(&tmp_list, &vn->pool[i].head); 2263 WRITE_ONCE(vn->pool[i].len, pool_len); 2264 spin_unlock(&vn->pool_lock); 2265 } 2266 } 2267 2268 reclaim_list_global(&decay_list); 2269} 2270 2271#define KASAN_RELEASE_BATCH_SIZE 32 2272 2273static void 2274kasan_release_vmalloc_node(struct vmap_node *vn) 2275{ 2276 struct vmap_area *va; 2277 unsigned long start, end; 2278 unsigned int batch_count = 0; 2279 2280 start = list_first_entry(&vn->purge_list, struct vmap_area, list)->va_start; 2281 end = list_last_entry(&vn->purge_list, struct vmap_area, list)->va_end; 2282 2283 list_for_each_entry(va, &vn->purge_list, list) { 2284 if (is_vmalloc_or_module_addr((void *) va->va_start)) 2285 kasan_release_vmalloc(va->va_start, va->va_end, 2286 va->va_start, va->va_end, 2287 KASAN_VMALLOC_PAGE_RANGE); 2288 2289 if (need_resched() || (++batch_count >= KASAN_RELEASE_BATCH_SIZE)) { 2290 cond_resched(); 2291 batch_count = 0; 2292 } 2293 } 2294 2295 kasan_release_vmalloc(start, end, start, end, KASAN_VMALLOC_TLB_FLUSH); 2296} 2297 2298static void purge_vmap_node(struct work_struct *work) 2299{ 2300 struct vmap_node *vn = container_of(work, 2301 struct vmap_node, purge_work); 2302 unsigned long nr_purged_pages = 0; 2303 struct vmap_area *va, *n_va; 2304 LIST_HEAD(local_list); 2305 2306 if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) 2307 kasan_release_vmalloc_node(vn); 2308 2309 vn->nr_purged = 0; 2310 2311 list_for_each_entry_safe(va, n_va, &vn->purge_list, list) { 2312 unsigned long nr = va_size(va) >> PAGE_SHIFT; 2313 unsigned int vn_id = decode_vn_id(va->flags); 2314 2315 list_del_init(&va->list); 2316 2317 nr_purged_pages += nr; 2318 vn->nr_purged++; 2319 2320 if (is_vn_id_valid(vn_id) && !vn->skip_populate) 2321 if (node_pool_add_va(vn, va)) 2322 continue; 2323 2324 /* Go back to global. */ 2325 list_add(&va->list, &local_list); 2326 } 2327 2328 atomic_long_sub(nr_purged_pages, &vmap_lazy_nr); 2329 2330 reclaim_list_global(&local_list); 2331} 2332 2333/* 2334 * Purges all lazily-freed vmap areas. 2335 */ 2336static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end, 2337 bool full_pool_decay) 2338{ 2339 unsigned long nr_purged_areas = 0; 2340 unsigned int nr_purge_helpers; 2341 static cpumask_t purge_nodes; 2342 unsigned int nr_purge_nodes; 2343 struct vmap_node *vn; 2344 int i; 2345 2346 lockdep_assert_held(&vmap_purge_lock); 2347 2348 /* 2349 * Use cpumask to mark which node has to be processed. 2350 */ 2351 purge_nodes = CPU_MASK_NONE; 2352 2353 for_each_vmap_node(vn) { 2354 INIT_LIST_HEAD(&vn->purge_list); 2355 vn->skip_populate = full_pool_decay; 2356 decay_va_pool_node(vn, full_pool_decay); 2357 2358 if (RB_EMPTY_ROOT(&vn->lazy.root)) 2359 continue; 2360 2361 spin_lock(&vn->lazy.lock); 2362 WRITE_ONCE(vn->lazy.root.rb_node, NULL); 2363 list_replace_init(&vn->lazy.head, &vn->purge_list); 2364 spin_unlock(&vn->lazy.lock); 2365 2366 start = min(start, list_first_entry(&vn->purge_list, 2367 struct vmap_area, list)->va_start); 2368 2369 end = max(end, list_last_entry(&vn->purge_list, 2370 struct vmap_area, list)->va_end); 2371 2372 cpumask_set_cpu(node_to_id(vn), &purge_nodes); 2373 } 2374 2375 nr_purge_nodes = cpumask_weight(&purge_nodes); 2376 if (nr_purge_nodes > 0) { 2377 flush_tlb_kernel_range(start, end); 2378 2379 /* One extra worker is per a lazy_max_pages() full set minus one. */ 2380 nr_purge_helpers = atomic_long_read(&vmap_lazy_nr) / lazy_max_pages(); 2381 nr_purge_helpers = clamp(nr_purge_helpers, 1U, nr_purge_nodes) - 1; 2382 2383 for_each_cpu(i, &purge_nodes) { 2384 vn = &vmap_nodes[i]; 2385 2386 if (nr_purge_helpers > 0) { 2387 INIT_WORK(&vn->purge_work, purge_vmap_node); 2388 2389 if (cpumask_test_cpu(i, cpu_online_mask)) 2390 schedule_work_on(i, &vn->purge_work); 2391 else 2392 schedule_work(&vn->purge_work); 2393 2394 nr_purge_helpers--; 2395 } else { 2396 vn->purge_work.func = NULL; 2397 purge_vmap_node(&vn->purge_work); 2398 nr_purged_areas += vn->nr_purged; 2399 } 2400 } 2401 2402 for_each_cpu(i, &purge_nodes) { 2403 vn = &vmap_nodes[i]; 2404 2405 if (vn->purge_work.func) { 2406 flush_work(&vn->purge_work); 2407 nr_purged_areas += vn->nr_purged; 2408 } 2409 } 2410 } 2411 2412 trace_purge_vmap_area_lazy(start, end, nr_purged_areas); 2413 return nr_purged_areas > 0; 2414} 2415 2416/* 2417 * Reclaim vmap areas by purging fragmented blocks and purge_vmap_area_list. 2418 */ 2419static void reclaim_and_purge_vmap_areas(void) 2420 2421{ 2422 mutex_lock(&vmap_purge_lock); 2423 purge_fragmented_blocks_allcpus(); 2424 __purge_vmap_area_lazy(ULONG_MAX, 0, true); 2425 mutex_unlock(&vmap_purge_lock); 2426} 2427 2428static void drain_vmap_area_work(struct work_struct *work) 2429{ 2430 mutex_lock(&vmap_purge_lock); 2431 __purge_vmap_area_lazy(ULONG_MAX, 0, false); 2432 mutex_unlock(&vmap_purge_lock); 2433} 2434 2435/* 2436 * Free a vmap area, caller ensuring that the area has been unmapped, 2437 * unlinked and flush_cache_vunmap had been called for the correct 2438 * range previously. 2439 */ 2440static void free_vmap_area_noflush(struct vmap_area *va) 2441{ 2442 unsigned long nr_lazy_max = lazy_max_pages(); 2443 unsigned long va_start = va->va_start; 2444 unsigned int vn_id = decode_vn_id(va->flags); 2445 struct vmap_node *vn; 2446 unsigned long nr_lazy; 2447 2448 if (WARN_ON_ONCE(!list_empty(&va->list))) 2449 return; 2450 2451 nr_lazy = atomic_long_add_return_relaxed(va_size(va) >> PAGE_SHIFT, 2452 &vmap_lazy_nr); 2453 2454 /* 2455 * If it was request by a certain node we would like to 2456 * return it to that node, i.e. its pool for later reuse. 2457 */ 2458 vn = is_vn_id_valid(vn_id) ? 2459 id_to_node(vn_id):addr_to_node(va->va_start); 2460 2461 spin_lock(&vn->lazy.lock); 2462 insert_vmap_area(va, &vn->lazy.root, &vn->lazy.head); 2463 spin_unlock(&vn->lazy.lock); 2464 2465 trace_free_vmap_area_noflush(va_start, nr_lazy, nr_lazy_max); 2466 2467 /* After this point, we may free va at any time */ 2468 if (unlikely(nr_lazy > nr_lazy_max)) 2469 schedule_work(&drain_vmap_work); 2470} 2471 2472/* 2473 * Free and unmap a vmap area 2474 */ 2475static void free_unmap_vmap_area(struct vmap_area *va) 2476{ 2477 flush_cache_vunmap(va->va_start, va->va_end); 2478 vunmap_range_noflush(va->va_start, va->va_end); 2479 if (debug_pagealloc_enabled_static()) 2480 flush_tlb_kernel_range(va->va_start, va->va_end); 2481 2482 free_vmap_area_noflush(va); 2483} 2484 2485struct vmap_area *find_vmap_area(unsigned long addr) 2486{ 2487 struct vmap_node *vn; 2488 struct vmap_area *va; 2489 int i, j; 2490 2491 if (unlikely(!vmap_initialized)) 2492 return NULL; 2493 2494 /* 2495 * An addr_to_node_id(addr) converts an address to a node index 2496 * where a VA is located. If VA spans several zones and passed 2497 * addr is not the same as va->va_start, what is not common, we 2498 * may need to scan extra nodes. See an example: 2499 * 2500 * <----va----> 2501 * -|-----|-----|-----|-----|- 2502 * 1 2 0 1 2503 * 2504 * VA resides in node 1 whereas it spans 1, 2 an 0. If passed 2505 * addr is within 2 or 0 nodes we should do extra work. 2506 */ 2507 i = j = addr_to_node_id(addr); 2508 do { 2509 vn = &vmap_nodes[i]; 2510 2511 spin_lock(&vn->busy.lock); 2512 va = __find_vmap_area(addr, &vn->busy.root); 2513 spin_unlock(&vn->busy.lock); 2514 2515 if (va) 2516 return va; 2517 } while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j); 2518 2519 return NULL; 2520} 2521 2522static struct vmap_area *find_unlink_vmap_area(unsigned long addr) 2523{ 2524 struct vmap_node *vn; 2525 struct vmap_area *va; 2526 int i, j; 2527 2528 /* 2529 * Check the comment in the find_vmap_area() about the loop. 2530 */ 2531 i = j = addr_to_node_id(addr); 2532 do { 2533 vn = &vmap_nodes[i]; 2534 2535 spin_lock(&vn->busy.lock); 2536 va = __find_vmap_area(addr, &vn->busy.root); 2537 if (va) 2538 unlink_va(va, &vn->busy.root); 2539 spin_unlock(&vn->busy.lock); 2540 2541 if (va) 2542 return va; 2543 } while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j); 2544 2545 return NULL; 2546} 2547 2548/*** Per cpu kva allocator ***/ 2549 2550/* 2551 * vmap space is limited especially on 32 bit architectures. Ensure there is 2552 * room for at least 16 percpu vmap blocks per CPU. 2553 */ 2554/* 2555 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able 2556 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess 2557 * instead (we just need a rough idea) 2558 */ 2559#if BITS_PER_LONG == 32 2560#define VMALLOC_SPACE (128UL*1024*1024) 2561#else 2562#define VMALLOC_SPACE (128UL*1024*1024*1024) 2563#endif 2564 2565#define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) 2566#define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ 2567#define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ 2568#define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) 2569#define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ 2570#define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ 2571#define VMAP_BBMAP_BITS \ 2572 VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ 2573 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ 2574 VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) 2575 2576#define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) 2577 2578/* 2579 * Purge threshold to prevent overeager purging of fragmented blocks for 2580 * regular operations: Purge if vb->free is less than 1/4 of the capacity. 2581 */ 2582#define VMAP_PURGE_THRESHOLD (VMAP_BBMAP_BITS / 4) 2583 2584#define VMAP_RAM 0x1 /* indicates vm_map_ram area*/ 2585#define VMAP_BLOCK 0x2 /* mark out the vmap_block sub-type*/ 2586#define VMAP_FLAGS_MASK 0x3 2587 2588struct vmap_block_queue { 2589 spinlock_t lock; 2590 struct list_head free; 2591 2592 /* 2593 * An xarray requires an extra memory dynamically to 2594 * be allocated. If it is an issue, we can use rb-tree 2595 * instead. 2596 */ 2597 struct xarray vmap_blocks; 2598}; 2599 2600struct vmap_block { 2601 spinlock_t lock; 2602 struct vmap_area *va; 2603 unsigned long free, dirty; 2604 DECLARE_BITMAP(used_map, VMAP_BBMAP_BITS); 2605 unsigned long dirty_min, dirty_max; /*< dirty range */ 2606 struct list_head free_list; 2607 struct rcu_head rcu_head; 2608 struct list_head purge; 2609 unsigned int cpu; 2610}; 2611 2612/* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ 2613static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); 2614 2615/* 2616 * In order to fast access to any "vmap_block" associated with a 2617 * specific address, we use a hash. 2618 * 2619 * A per-cpu vmap_block_queue is used in both ways, to serialize 2620 * an access to free block chains among CPUs(alloc path) and it 2621 * also acts as a vmap_block hash(alloc/free paths). It means we 2622 * overload it, since we already have the per-cpu array which is 2623 * used as a hash table. When used as a hash a 'cpu' passed to 2624 * per_cpu() is not actually a CPU but rather a hash index. 2625 * 2626 * A hash function is addr_to_vb_xa() which hashes any address 2627 * to a specific index(in a hash) it belongs to. This then uses a 2628 * per_cpu() macro to access an array with generated index. 2629 * 2630 * An example: 2631 * 2632 * CPU_1 CPU_2 CPU_0 2633 * | | | 2634 * V V V 2635 * 0 10 20 30 40 50 60 2636 * |------|------|------|------|------|------|...<vmap address space> 2637 * CPU0 CPU1 CPU2 CPU0 CPU1 CPU2 2638 * 2639 * - CPU_1 invokes vm_unmap_ram(6), 6 belongs to CPU0 zone, thus 2640 * it access: CPU0/INDEX0 -> vmap_blocks -> xa_lock; 2641 * 2642 * - CPU_2 invokes vm_unmap_ram(11), 11 belongs to CPU1 zone, thus 2643 * it access: CPU1/INDEX1 -> vmap_blocks -> xa_lock; 2644 * 2645 * - CPU_0 invokes vm_unmap_ram(20), 20 belongs to CPU2 zone, thus 2646 * it access: CPU2/INDEX2 -> vmap_blocks -> xa_lock. 2647 * 2648 * This technique almost always avoids lock contention on insert/remove, 2649 * however xarray spinlocks protect against any contention that remains. 2650 */ 2651static struct xarray * 2652addr_to_vb_xa(unsigned long addr) 2653{ 2654 int index = (addr / VMAP_BLOCK_SIZE) % nr_cpu_ids; 2655 2656 /* 2657 * Please note, nr_cpu_ids points on a highest set 2658 * possible bit, i.e. we never invoke cpumask_next() 2659 * if an index points on it which is nr_cpu_ids - 1. 2660 */ 2661 if (!cpu_possible(index)) 2662 index = cpumask_next(index, cpu_possible_mask); 2663 2664 return &per_cpu(vmap_block_queue, index).vmap_blocks; 2665} 2666 2667/* 2668 * We should probably have a fallback mechanism to allocate virtual memory 2669 * out of partially filled vmap blocks. However vmap block sizing should be 2670 * fairly reasonable according to the vmalloc size, so it shouldn't be a 2671 * big problem. 2672 */ 2673 2674static unsigned long addr_to_vb_idx(unsigned long addr) 2675{ 2676 addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); 2677 addr /= VMAP_BLOCK_SIZE; 2678 return addr; 2679} 2680 2681static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) 2682{ 2683 unsigned long addr; 2684 2685 addr = va_start + (pages_off << PAGE_SHIFT); 2686 BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); 2687 return (void *)addr; 2688} 2689 2690/** 2691 * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this 2692 * block. Of course pages number can't exceed VMAP_BBMAP_BITS 2693 * @order: how many 2^order pages should be occupied in newly allocated block 2694 * @gfp_mask: flags for the page level allocator 2695 * 2696 * Return: virtual address in a newly allocated block or ERR_PTR(-errno) 2697 */ 2698static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) 2699{ 2700 struct vmap_block_queue *vbq; 2701 struct vmap_block *vb; 2702 struct vmap_area *va; 2703 struct xarray *xa; 2704 unsigned long vb_idx; 2705 int node, err; 2706 void *vaddr; 2707 2708 node = numa_node_id(); 2709 2710 vb = kmalloc_node(sizeof(struct vmap_block), gfp_mask, node); 2711 if (unlikely(!vb)) 2712 return ERR_PTR(-ENOMEM); 2713 2714 va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, 2715 VMALLOC_START, VMALLOC_END, 2716 node, gfp_mask, 2717 VMAP_RAM|VMAP_BLOCK, NULL); 2718 if (IS_ERR(va)) { 2719 kfree(vb); 2720 return ERR_CAST(va); 2721 } 2722 2723 vaddr = vmap_block_vaddr(va->va_start, 0); 2724 spin_lock_init(&vb->lock); 2725 vb->va = va; 2726 /* At least something should be left free */ 2727 BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); 2728 bitmap_zero(vb->used_map, VMAP_BBMAP_BITS); 2729 vb->free = VMAP_BBMAP_BITS - (1UL << order); 2730 vb->dirty = 0; 2731 vb->dirty_min = VMAP_BBMAP_BITS; 2732 vb->dirty_max = 0; 2733 bitmap_set(vb->used_map, 0, (1UL << order)); 2734 INIT_LIST_HEAD(&vb->free_list); 2735 vb->cpu = raw_smp_processor_id(); 2736 2737 xa = addr_to_vb_xa(va->va_start); 2738 vb_idx = addr_to_vb_idx(va->va_start); 2739 err = xa_insert(xa, vb_idx, vb, gfp_mask); 2740 if (err) { 2741 kfree(vb); 2742 free_vmap_area(va); 2743 return ERR_PTR(err); 2744 } 2745 /* 2746 * list_add_tail_rcu could happened in another core 2747 * rather than vb->cpu due to task migration, which 2748 * is safe as list_add_tail_rcu will ensure the list's 2749 * integrity together with list_for_each_rcu from read 2750 * side. 2751 */ 2752 vbq = per_cpu_ptr(&vmap_block_queue, vb->cpu); 2753 spin_lock(&vbq->lock); 2754 list_add_tail_rcu(&vb->free_list, &vbq->free); 2755 spin_unlock(&vbq->lock); 2756 2757 return vaddr; 2758} 2759 2760static void free_vmap_block(struct vmap_block *vb) 2761{ 2762 struct vmap_node *vn; 2763 struct vmap_block *tmp; 2764 struct xarray *xa; 2765 2766 xa = addr_to_vb_xa(vb->va->va_start); 2767 tmp = xa_erase(xa, addr_to_vb_idx(vb->va->va_start)); 2768 BUG_ON(tmp != vb); 2769 2770 vn = addr_to_node(vb->va->va_start); 2771 spin_lock(&vn->busy.lock); 2772 unlink_va(vb->va, &vn->busy.root); 2773 spin_unlock(&vn->busy.lock); 2774 2775 free_vmap_area_noflush(vb->va); 2776 kfree_rcu(vb, rcu_head); 2777} 2778 2779static bool purge_fragmented_block(struct vmap_block *vb, 2780 struct list_head *purge_list, bool force_purge) 2781{ 2782 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, vb->cpu); 2783 2784 if (vb->free + vb->dirty != VMAP_BBMAP_BITS || 2785 vb->dirty == VMAP_BBMAP_BITS) 2786 return false; 2787 2788 /* Don't overeagerly purge usable blocks unless requested */ 2789 if (!(force_purge || vb->free < VMAP_PURGE_THRESHOLD)) 2790 return false; 2791 2792 /* prevent further allocs after releasing lock */ 2793 WRITE_ONCE(vb->free, 0); 2794 /* prevent purging it again */ 2795 WRITE_ONCE(vb->dirty, VMAP_BBMAP_BITS); 2796 vb->dirty_min = 0; 2797 vb->dirty_max = VMAP_BBMAP_BITS; 2798 spin_lock(&vbq->lock); 2799 list_del_rcu(&vb->free_list); 2800 spin_unlock(&vbq->lock); 2801 list_add_tail(&vb->purge, purge_list); 2802 return true; 2803} 2804 2805static void free_purged_blocks(struct list_head *purge_list) 2806{ 2807 struct vmap_block *vb, *n_vb; 2808 2809 list_for_each_entry_safe(vb, n_vb, purge_list, purge) { 2810 list_del(&vb->purge); 2811 free_vmap_block(vb); 2812 } 2813} 2814 2815static void purge_fragmented_blocks(int cpu) 2816{ 2817 LIST_HEAD(purge); 2818 struct vmap_block *vb; 2819 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2820 2821 rcu_read_lock(); 2822 list_for_each_entry_rcu(vb, &vbq->free, free_list) { 2823 unsigned long free = READ_ONCE(vb->free); 2824 unsigned long dirty = READ_ONCE(vb->dirty); 2825 2826 if (free + dirty != VMAP_BBMAP_BITS || 2827 dirty == VMAP_BBMAP_BITS) 2828 continue; 2829 2830 spin_lock(&vb->lock); 2831 purge_fragmented_block(vb, &purge, true); 2832 spin_unlock(&vb->lock); 2833 } 2834 rcu_read_unlock(); 2835 free_purged_blocks(&purge); 2836} 2837 2838static void purge_fragmented_blocks_allcpus(void) 2839{ 2840 int cpu; 2841 2842 for_each_possible_cpu(cpu) 2843 purge_fragmented_blocks(cpu); 2844} 2845 2846static void *vb_alloc(unsigned long size, gfp_t gfp_mask) 2847{ 2848 struct vmap_block_queue *vbq; 2849 struct vmap_block *vb; 2850 void *vaddr = NULL; 2851 unsigned int order; 2852 2853 BUG_ON(offset_in_page(size)); 2854 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2855 if (WARN_ON(size == 0)) { 2856 /* 2857 * Allocating 0 bytes isn't what caller wants since 2858 * get_order(0) returns funny result. Just warn and terminate 2859 * early. 2860 */ 2861 return ERR_PTR(-EINVAL); 2862 } 2863 order = get_order(size); 2864 2865 rcu_read_lock(); 2866 vbq = raw_cpu_ptr(&vmap_block_queue); 2867 list_for_each_entry_rcu(vb, &vbq->free, free_list) { 2868 unsigned long pages_off; 2869 2870 if (READ_ONCE(vb->free) < (1UL << order)) 2871 continue; 2872 2873 spin_lock(&vb->lock); 2874 if (vb->free < (1UL << order)) { 2875 spin_unlock(&vb->lock); 2876 continue; 2877 } 2878 2879 pages_off = VMAP_BBMAP_BITS - vb->free; 2880 vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); 2881 WRITE_ONCE(vb->free, vb->free - (1UL << order)); 2882 bitmap_set(vb->used_map, pages_off, (1UL << order)); 2883 if (vb->free == 0) { 2884 spin_lock(&vbq->lock); 2885 list_del_rcu(&vb->free_list); 2886 spin_unlock(&vbq->lock); 2887 } 2888 2889 spin_unlock(&vb->lock); 2890 break; 2891 } 2892 2893 rcu_read_unlock(); 2894 2895 /* Allocate new block if nothing was found */ 2896 if (!vaddr) 2897 vaddr = new_vmap_block(order, gfp_mask); 2898 2899 return vaddr; 2900} 2901 2902static void vb_free(unsigned long addr, unsigned long size) 2903{ 2904 unsigned long offset; 2905 unsigned int order; 2906 struct vmap_block *vb; 2907 struct xarray *xa; 2908 2909 BUG_ON(offset_in_page(size)); 2910 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 2911 2912 flush_cache_vunmap(addr, addr + size); 2913 2914 order = get_order(size); 2915 offset = (addr & (VMAP_BLOCK_SIZE - 1)) >> PAGE_SHIFT; 2916 2917 xa = addr_to_vb_xa(addr); 2918 vb = xa_load(xa, addr_to_vb_idx(addr)); 2919 2920 spin_lock(&vb->lock); 2921 bitmap_clear(vb->used_map, offset, (1UL << order)); 2922 spin_unlock(&vb->lock); 2923 2924 vunmap_range_noflush(addr, addr + size); 2925 2926 if (debug_pagealloc_enabled_static()) 2927 flush_tlb_kernel_range(addr, addr + size); 2928 2929 spin_lock(&vb->lock); 2930 2931 /* Expand the not yet TLB flushed dirty range */ 2932 vb->dirty_min = min(vb->dirty_min, offset); 2933 vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); 2934 2935 WRITE_ONCE(vb->dirty, vb->dirty + (1UL << order)); 2936 if (vb->dirty == VMAP_BBMAP_BITS) { 2937 BUG_ON(vb->free); 2938 spin_unlock(&vb->lock); 2939 free_vmap_block(vb); 2940 } else 2941 spin_unlock(&vb->lock); 2942} 2943 2944static void _vm_unmap_aliases(unsigned long start, unsigned long end, int flush) 2945{ 2946 LIST_HEAD(purge_list); 2947 int cpu; 2948 2949 if (unlikely(!vmap_initialized)) 2950 return; 2951 2952 mutex_lock(&vmap_purge_lock); 2953 2954 for_each_possible_cpu(cpu) { 2955 struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); 2956 struct vmap_block *vb; 2957 unsigned long idx; 2958 2959 rcu_read_lock(); 2960 xa_for_each(&vbq->vmap_blocks, idx, vb) { 2961 spin_lock(&vb->lock); 2962 2963 /* 2964 * Try to purge a fragmented block first. If it's 2965 * not purgeable, check whether there is dirty 2966 * space to be flushed. 2967 */ 2968 if (!purge_fragmented_block(vb, &purge_list, false) && 2969 vb->dirty_max && vb->dirty != VMAP_BBMAP_BITS) { 2970 unsigned long va_start = vb->va->va_start; 2971 unsigned long s, e; 2972 2973 s = va_start + (vb->dirty_min << PAGE_SHIFT); 2974 e = va_start + (vb->dirty_max << PAGE_SHIFT); 2975 2976 start = min(s, start); 2977 end = max(e, end); 2978 2979 /* Prevent that this is flushed again */ 2980 vb->dirty_min = VMAP_BBMAP_BITS; 2981 vb->dirty_max = 0; 2982 2983 flush = 1; 2984 } 2985 spin_unlock(&vb->lock); 2986 } 2987 rcu_read_unlock(); 2988 } 2989 free_purged_blocks(&purge_list); 2990 2991 if (!__purge_vmap_area_lazy(start, end, false) && flush) 2992 flush_tlb_kernel_range(start, end); 2993 mutex_unlock(&vmap_purge_lock); 2994} 2995 2996/** 2997 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer 2998 * 2999 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily 3000 * to amortize TLB flushing overheads. What this means is that any page you 3001 * have now, may, in a former life, have been mapped into kernel virtual 3002 * address by the vmap layer and so there might be some CPUs with TLB entries 3003 * still referencing that page (additional to the regular 1:1 kernel mapping). 3004 * 3005 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can 3006 * be sure that none of the pages we have control over will have any aliases 3007 * from the vmap layer. 3008 */ 3009void vm_unmap_aliases(void) 3010{ 3011 _vm_unmap_aliases(ULONG_MAX, 0, 0); 3012} 3013EXPORT_SYMBOL_GPL(vm_unmap_aliases); 3014 3015/** 3016 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram 3017 * @mem: the pointer returned by vm_map_ram 3018 * @count: the count passed to that vm_map_ram call (cannot unmap partial) 3019 */ 3020void vm_unmap_ram(const void *mem, unsigned int count) 3021{ 3022 unsigned long size = (unsigned long)count << PAGE_SHIFT; 3023 unsigned long addr = (unsigned long)kasan_reset_tag(mem); 3024 struct vmap_area *va; 3025 3026 might_sleep(); 3027 BUG_ON(!addr); 3028 BUG_ON(addr < VMALLOC_START); 3029 BUG_ON(addr > VMALLOC_END); 3030 BUG_ON(!PAGE_ALIGNED(addr)); 3031 3032 kasan_poison_vmalloc(mem, size); 3033 3034 if (likely(count <= VMAP_MAX_ALLOC)) { 3035 debug_check_no_locks_freed(mem, size); 3036 vb_free(addr, size); 3037 return; 3038 } 3039 3040 va = find_unlink_vmap_area(addr); 3041 if (WARN_ON_ONCE(!va)) 3042 return; 3043 3044 debug_check_no_locks_freed((void *)va->va_start, va_size(va)); 3045 free_unmap_vmap_area(va); 3046} 3047EXPORT_SYMBOL(vm_unmap_ram); 3048 3049/** 3050 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) 3051 * @pages: an array of pointers to the pages to be mapped 3052 * @count: number of pages 3053 * @node: prefer to allocate data structures on this node 3054 * 3055 * If you use this function for less than VMAP_MAX_ALLOC pages, it could be 3056 * faster than vmap so it's good. But if you mix long-life and short-life 3057 * objects with vm_map_ram(), it could consume lots of address space through 3058 * fragmentation (especially on a 32bit machine). You could see failures in 3059 * the end. Please use this function for short-lived objects. 3060 * 3061 * Returns: a pointer to the address that has been mapped, or %NULL on failure 3062 */ 3063void *vm_map_ram(struct page **pages, unsigned int count, int node) 3064{ 3065 unsigned long size = (unsigned long)count << PAGE_SHIFT; 3066 unsigned long addr; 3067 void *mem; 3068 3069 if (likely(count <= VMAP_MAX_ALLOC)) { 3070 mem = vb_alloc(size, GFP_KERNEL); 3071 if (IS_ERR(mem)) 3072 return NULL; 3073 addr = (unsigned long)mem; 3074 } else { 3075 struct vmap_area *va; 3076 va = alloc_vmap_area(size, PAGE_SIZE, 3077 VMALLOC_START, VMALLOC_END, 3078 node, GFP_KERNEL, VMAP_RAM, 3079 NULL); 3080 if (IS_ERR(va)) 3081 return NULL; 3082 3083 addr = va->va_start; 3084 mem = (void *)addr; 3085 } 3086 3087 if (vmap_pages_range(addr, addr + size, PAGE_KERNEL, 3088 pages, PAGE_SHIFT) < 0) { 3089 vm_unmap_ram(mem, count); 3090 return NULL; 3091 } 3092 3093 /* 3094 * Mark the pages as accessible, now that they are mapped. 3095 * With hardware tag-based KASAN, marking is skipped for 3096 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 3097 */ 3098 mem = kasan_unpoison_vmalloc(mem, size, KASAN_VMALLOC_PROT_NORMAL); 3099 3100 return mem; 3101} 3102EXPORT_SYMBOL(vm_map_ram); 3103 3104static struct vm_struct *vmlist __initdata; 3105 3106static inline unsigned int vm_area_page_order(struct vm_struct *vm) 3107{ 3108#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 3109 return vm->page_order; 3110#else 3111 return 0; 3112#endif 3113} 3114 3115unsigned int get_vm_area_page_order(struct vm_struct *vm) 3116{ 3117 return vm_area_page_order(vm); 3118} 3119 3120static inline void set_vm_area_page_order(struct vm_struct *vm, unsigned int order) 3121{ 3122#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC 3123 vm->page_order = order; 3124#else 3125 BUG_ON(order != 0); 3126#endif 3127} 3128 3129/** 3130 * vm_area_add_early - add vmap area early during boot 3131 * @vm: vm_struct to add 3132 * 3133 * This function is used to add fixed kernel vm area to vmlist before 3134 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags 3135 * should contain proper values and the other fields should be zero. 3136 * 3137 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 3138 */ 3139void __init vm_area_add_early(struct vm_struct *vm) 3140{ 3141 struct vm_struct *tmp, **p; 3142 3143 BUG_ON(vmap_initialized); 3144 for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { 3145 if (tmp->addr >= vm->addr) { 3146 BUG_ON(tmp->addr < vm->addr + vm->size); 3147 break; 3148 } else 3149 BUG_ON(tmp->addr + tmp->size > vm->addr); 3150 } 3151 vm->next = *p; 3152 *p = vm; 3153} 3154 3155/** 3156 * vm_area_register_early - register vmap area early during boot 3157 * @vm: vm_struct to register 3158 * @align: requested alignment 3159 * 3160 * This function is used to register kernel vm area before 3161 * vmalloc_init() is called. @vm->size and @vm->flags should contain 3162 * proper values on entry and other fields should be zero. On return, 3163 * vm->addr contains the allocated address. 3164 * 3165 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. 3166 */ 3167void __init vm_area_register_early(struct vm_struct *vm, size_t align) 3168{ 3169 unsigned long addr = ALIGN(VMALLOC_START, align); 3170 struct vm_struct *cur, **p; 3171 3172 BUG_ON(vmap_initialized); 3173 3174 for (p = &vmlist; (cur = *p) != NULL; p = &cur->next) { 3175 if ((unsigned long)cur->addr - addr >= vm->size) 3176 break; 3177 addr = ALIGN((unsigned long)cur->addr + cur->size, align); 3178 } 3179 3180 BUG_ON(addr > VMALLOC_END - vm->size); 3181 vm->addr = (void *)addr; 3182 vm->next = *p; 3183 *p = vm; 3184 kasan_populate_early_vm_area_shadow(vm->addr, vm->size); 3185} 3186 3187static void clear_vm_uninitialized_flag(struct vm_struct *vm) 3188{ 3189 /* 3190 * Before removing VM_UNINITIALIZED, 3191 * we should make sure that vm has proper values. 3192 * Pair with smp_rmb() in vread_iter() and vmalloc_info_show(). 3193 */ 3194 smp_wmb(); 3195 vm->flags &= ~VM_UNINITIALIZED; 3196} 3197 3198struct vm_struct *__get_vm_area_node(unsigned long size, 3199 unsigned long align, unsigned long shift, unsigned long flags, 3200 unsigned long start, unsigned long end, int node, 3201 gfp_t gfp_mask, const void *caller) 3202{ 3203 struct vmap_area *va; 3204 struct vm_struct *area; 3205 unsigned long requested_size = size; 3206 3207 BUG_ON(in_interrupt()); 3208 size = ALIGN(size, 1ul << shift); 3209 if (unlikely(!size)) 3210 return NULL; 3211 3212 if (flags & VM_IOREMAP) 3213 align = 1ul << clamp_t(int, get_count_order_long(size), 3214 PAGE_SHIFT, IOREMAP_MAX_ORDER); 3215 3216 area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); 3217 if (unlikely(!area)) 3218 return NULL; 3219 3220 if (!(flags & VM_NO_GUARD)) 3221 size += PAGE_SIZE; 3222 3223 area->flags = flags; 3224 area->caller = caller; 3225 area->requested_size = requested_size; 3226 3227 va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area); 3228 if (IS_ERR(va)) { 3229 kfree(area); 3230 return NULL; 3231 } 3232 3233 /* 3234 * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a 3235 * best-effort approach, as they can be mapped outside of vmalloc code. 3236 * For VM_ALLOC mappings, the pages are marked as accessible after 3237 * getting mapped in __vmalloc_node_range(). 3238 * With hardware tag-based KASAN, marking is skipped for 3239 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 3240 */ 3241 if (!(flags & VM_ALLOC)) 3242 area->addr = kasan_unpoison_vmalloc(area->addr, requested_size, 3243 KASAN_VMALLOC_PROT_NORMAL); 3244 3245 return area; 3246} 3247 3248struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, 3249 unsigned long start, unsigned long end, 3250 const void *caller) 3251{ 3252 return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, start, end, 3253 NUMA_NO_NODE, GFP_KERNEL, caller); 3254} 3255 3256/** 3257 * get_vm_area - reserve a contiguous kernel virtual area 3258 * @size: size of the area 3259 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC 3260 * 3261 * Search an area of @size in the kernel virtual mapping area, 3262 * and reserved it for out purposes. Returns the area descriptor 3263 * on success or %NULL on failure. 3264 * 3265 * Return: the area descriptor on success or %NULL on failure. 3266 */ 3267struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) 3268{ 3269 return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 3270 VMALLOC_START, VMALLOC_END, 3271 NUMA_NO_NODE, GFP_KERNEL, 3272 __builtin_return_address(0)); 3273} 3274 3275struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, 3276 const void *caller) 3277{ 3278 return __get_vm_area_node(size, 1, PAGE_SHIFT, flags, 3279 VMALLOC_START, VMALLOC_END, 3280 NUMA_NO_NODE, GFP_KERNEL, caller); 3281} 3282 3283/** 3284 * find_vm_area - find a continuous kernel virtual area 3285 * @addr: base address 3286 * 3287 * Search for the kernel VM area starting at @addr, and return it. 3288 * It is up to the caller to do all required locking to keep the returned 3289 * pointer valid. 3290 * 3291 * Return: the area descriptor on success or %NULL on failure. 3292 */ 3293struct vm_struct *find_vm_area(const void *addr) 3294{ 3295 struct vmap_area *va; 3296 3297 va = find_vmap_area((unsigned long)addr); 3298 if (!va) 3299 return NULL; 3300 3301 return va->vm; 3302} 3303 3304/** 3305 * remove_vm_area - find and remove a continuous kernel virtual area 3306 * @addr: base address 3307 * 3308 * Search for the kernel VM area starting at @addr, and remove it. 3309 * This function returns the found VM area, but using it is NOT safe 3310 * on SMP machines, except for its size or flags. 3311 * 3312 * Return: the area descriptor on success or %NULL on failure. 3313 */ 3314struct vm_struct *remove_vm_area(const void *addr) 3315{ 3316 struct vmap_area *va; 3317 struct vm_struct *vm; 3318 3319 might_sleep(); 3320 3321 if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", 3322 addr)) 3323 return NULL; 3324 3325 va = find_unlink_vmap_area((unsigned long)addr); 3326 if (!va || !va->vm) 3327 return NULL; 3328 vm = va->vm; 3329 3330 debug_check_no_locks_freed(vm->addr, get_vm_area_size(vm)); 3331 debug_check_no_obj_freed(vm->addr, get_vm_area_size(vm)); 3332 kasan_free_module_shadow(vm); 3333 kasan_poison_vmalloc(vm->addr, get_vm_area_size(vm)); 3334 3335 free_unmap_vmap_area(va); 3336 return vm; 3337} 3338 3339static inline void set_area_direct_map(const struct vm_struct *area, 3340 int (*set_direct_map)(struct page *page)) 3341{ 3342 int i; 3343 3344 /* HUGE_VMALLOC passes small pages to set_direct_map */ 3345 for (i = 0; i < area->nr_pages; i++) 3346 if (page_address(area->pages[i])) 3347 set_direct_map(area->pages[i]); 3348} 3349 3350/* 3351 * Flush the vm mapping and reset the direct map. 3352 */ 3353static void vm_reset_perms(struct vm_struct *area) 3354{ 3355 unsigned long start = ULONG_MAX, end = 0; 3356 unsigned int page_order = vm_area_page_order(area); 3357 int flush_dmap = 0; 3358 int i; 3359 3360 /* 3361 * Find the start and end range of the direct mappings to make sure that 3362 * the vm_unmap_aliases() flush includes the direct map. 3363 */ 3364 for (i = 0; i < area->nr_pages; i += 1U << page_order) { 3365 unsigned long addr = (unsigned long)page_address(area->pages[i]); 3366 3367 if (addr) { 3368 unsigned long page_size; 3369 3370 page_size = PAGE_SIZE << page_order; 3371 start = min(addr, start); 3372 end = max(addr + page_size, end); 3373 flush_dmap = 1; 3374 } 3375 } 3376 3377 /* 3378 * Set direct map to something invalid so that it won't be cached if 3379 * there are any accesses after the TLB flush, then flush the TLB and 3380 * reset the direct map permissions to the default. 3381 */ 3382 set_area_direct_map(area, set_direct_map_invalid_noflush); 3383 _vm_unmap_aliases(start, end, flush_dmap); 3384 set_area_direct_map(area, set_direct_map_default_noflush); 3385} 3386 3387static void delayed_vfree_work(struct work_struct *w) 3388{ 3389 struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); 3390 struct llist_node *t, *llnode; 3391 3392 llist_for_each_safe(llnode, t, llist_del_all(&p->list)) 3393 vfree(llnode); 3394} 3395 3396/** 3397 * vfree_atomic - release memory allocated by vmalloc() 3398 * @addr: memory base address 3399 * 3400 * This one is just like vfree() but can be called in any atomic context 3401 * except NMIs. 3402 */ 3403void vfree_atomic(const void *addr) 3404{ 3405 struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); 3406 3407 BUG_ON(in_nmi()); 3408 kmemleak_free(addr); 3409 3410 /* 3411 * Use raw_cpu_ptr() because this can be called from preemptible 3412 * context. Preemption is absolutely fine here, because the llist_add() 3413 * implementation is lockless, so it works even if we are adding to 3414 * another cpu's list. schedule_work() should be fine with this too. 3415 */ 3416 if (addr && llist_add((struct llist_node *)addr, &p->list)) 3417 schedule_work(&p->wq); 3418} 3419 3420/** 3421 * vfree - Release memory allocated by vmalloc() 3422 * @addr: Memory base address 3423 * 3424 * Free the virtually continuous memory area starting at @addr, as obtained 3425 * from one of the vmalloc() family of APIs. This will usually also free the 3426 * physical memory underlying the virtual allocation, but that memory is 3427 * reference counted, so it will not be freed until the last user goes away. 3428 * 3429 * If @addr is NULL, no operation is performed. 3430 * 3431 * Context: 3432 * May sleep if called *not* from interrupt context. 3433 * Must not be called in NMI context (strictly speaking, it could be 3434 * if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling 3435 * conventions for vfree() arch-dependent would be a really bad idea). 3436 */ 3437void vfree(const void *addr) 3438{ 3439 struct vm_struct *vm; 3440 int i; 3441 3442 if (unlikely(in_interrupt())) { 3443 vfree_atomic(addr); 3444 return; 3445 } 3446 3447 BUG_ON(in_nmi()); 3448 kmemleak_free(addr); 3449 might_sleep(); 3450 3451 if (!addr) 3452 return; 3453 3454 vm = remove_vm_area(addr); 3455 if (unlikely(!vm)) { 3456 WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", 3457 addr); 3458 return; 3459 } 3460 3461 if (unlikely(vm->flags & VM_FLUSH_RESET_PERMS)) 3462 vm_reset_perms(vm); 3463 /* All pages of vm should be charged to same memcg, so use first one. */ 3464 if (vm->nr_pages && !(vm->flags & VM_MAP_PUT_PAGES)) 3465 mod_memcg_page_state(vm->pages[0], MEMCG_VMALLOC, -vm->nr_pages); 3466 for (i = 0; i < vm->nr_pages; i++) { 3467 struct page *page = vm->pages[i]; 3468 3469 BUG_ON(!page); 3470 /* 3471 * High-order allocs for huge vmallocs are split, so 3472 * can be freed as an array of order-0 allocations 3473 */ 3474 __free_page(page); 3475 cond_resched(); 3476 } 3477 if (!(vm->flags & VM_MAP_PUT_PAGES)) 3478 atomic_long_sub(vm->nr_pages, &nr_vmalloc_pages); 3479 kvfree(vm->pages); 3480 kfree(vm); 3481} 3482EXPORT_SYMBOL(vfree); 3483 3484/** 3485 * vunmap - release virtual mapping obtained by vmap() 3486 * @addr: memory base address 3487 * 3488 * Free the virtually contiguous memory area starting at @addr, 3489 * which was created from the page array passed to vmap(). 3490 * 3491 * Must not be called in interrupt context. 3492 */ 3493void vunmap(const void *addr) 3494{ 3495 struct vm_struct *vm; 3496 3497 BUG_ON(in_interrupt()); 3498 might_sleep(); 3499 3500 if (!addr) 3501 return; 3502 vm = remove_vm_area(addr); 3503 if (unlikely(!vm)) { 3504 WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n", 3505 addr); 3506 return; 3507 } 3508 kfree(vm); 3509} 3510EXPORT_SYMBOL(vunmap); 3511 3512/** 3513 * vmap - map an array of pages into virtually contiguous space 3514 * @pages: array of page pointers 3515 * @count: number of pages to map 3516 * @flags: vm_area->flags 3517 * @prot: page protection for the mapping 3518 * 3519 * Maps @count pages from @pages into contiguous kernel virtual space. 3520 * If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself 3521 * (which must be kmalloc or vmalloc memory) and one reference per pages in it 3522 * are transferred from the caller to vmap(), and will be freed / dropped when 3523 * vfree() is called on the return value. 3524 * 3525 * Return: the address of the area or %NULL on failure 3526 */ 3527void *vmap(struct page **pages, unsigned int count, 3528 unsigned long flags, pgprot_t prot) 3529{ 3530 struct vm_struct *area; 3531 unsigned long addr; 3532 unsigned long size; /* In bytes */ 3533 3534 might_sleep(); 3535 3536 if (WARN_ON_ONCE(flags & VM_FLUSH_RESET_PERMS)) 3537 return NULL; 3538 3539 /* 3540 * Your top guard is someone else's bottom guard. Not having a top 3541 * guard compromises someone else's mappings too. 3542 */ 3543 if (WARN_ON_ONCE(flags & VM_NO_GUARD)) 3544 flags &= ~VM_NO_GUARD; 3545 3546 if (count > totalram_pages()) 3547 return NULL; 3548 3549 size = (unsigned long)count << PAGE_SHIFT; 3550 area = get_vm_area_caller(size, flags, __builtin_return_address(0)); 3551 if (!area) 3552 return NULL; 3553 3554 addr = (unsigned long)area->addr; 3555 if (vmap_pages_range(addr, addr + size, pgprot_nx(prot), 3556 pages, PAGE_SHIFT) < 0) { 3557 vunmap(area->addr); 3558 return NULL; 3559 } 3560 3561 if (flags & VM_MAP_PUT_PAGES) { 3562 area->pages = pages; 3563 area->nr_pages = count; 3564 } 3565 return area->addr; 3566} 3567EXPORT_SYMBOL(vmap); 3568 3569#ifdef CONFIG_VMAP_PFN 3570struct vmap_pfn_data { 3571 unsigned long *pfns; 3572 pgprot_t prot; 3573 unsigned int idx; 3574}; 3575 3576static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private) 3577{ 3578 struct vmap_pfn_data *data = private; 3579 unsigned long pfn = data->pfns[data->idx]; 3580 pte_t ptent; 3581 3582 if (WARN_ON_ONCE(pfn_valid(pfn))) 3583 return -EINVAL; 3584 3585 ptent = pte_mkspecial(pfn_pte(pfn, data->prot)); 3586 set_pte_at(&init_mm, addr, pte, ptent); 3587 3588 data->idx++; 3589 return 0; 3590} 3591 3592/** 3593 * vmap_pfn - map an array of PFNs into virtually contiguous space 3594 * @pfns: array of PFNs 3595 * @count: number of pages to map 3596 * @prot: page protection for the mapping 3597 * 3598 * Maps @count PFNs from @pfns into contiguous kernel virtual space and returns 3599 * the start address of the mapping. 3600 */ 3601void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot) 3602{ 3603 struct vmap_pfn_data data = { .pfns = pfns, .prot = pgprot_nx(prot) }; 3604 struct vm_struct *area; 3605 3606 area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP, 3607 __builtin_return_address(0)); 3608 if (!area) 3609 return NULL; 3610 if (apply_to_page_range(&init_mm, (unsigned long)area->addr, 3611 count * PAGE_SIZE, vmap_pfn_apply, &data)) { 3612 free_vm_area(area); 3613 return NULL; 3614 } 3615 3616 flush_cache_vmap((unsigned long)area->addr, 3617 (unsigned long)area->addr + count * PAGE_SIZE); 3618 3619 return area->addr; 3620} 3621EXPORT_SYMBOL_GPL(vmap_pfn); 3622#endif /* CONFIG_VMAP_PFN */ 3623 3624/* 3625 * Helper for vmalloc to adjust the gfp flags for certain allocations. 3626 */ 3627static inline gfp_t vmalloc_gfp_adjust(gfp_t flags, const bool large) 3628{ 3629 flags |= __GFP_NOWARN; 3630 if (large) 3631 flags &= ~__GFP_NOFAIL; 3632 return flags; 3633} 3634 3635static inline unsigned int 3636vm_area_alloc_pages(gfp_t gfp, int nid, 3637 unsigned int order, unsigned int nr_pages, struct page **pages) 3638{ 3639 unsigned int nr_allocated = 0; 3640 unsigned int nr_remaining = nr_pages; 3641 unsigned int max_attempt_order = MAX_PAGE_ORDER; 3642 struct page *page; 3643 int i; 3644 unsigned int large_order = ilog2(nr_remaining); 3645 gfp_t large_gfp = vmalloc_gfp_adjust(gfp, large_order) & ~__GFP_DIRECT_RECLAIM; 3646 3647 large_order = min(max_attempt_order, large_order); 3648 3649 /* 3650 * Initially, attempt to have the page allocator give us large order 3651 * pages. Do not attempt allocating smaller than order chunks since 3652 * __vmap_pages_range() expects physically contigous pages of exactly 3653 * order long chunks. 3654 */ 3655 while (large_order > order && nr_remaining) { 3656 if (nid == NUMA_NO_NODE) 3657 page = alloc_pages_noprof(large_gfp, large_order); 3658 else 3659 page = alloc_pages_node_noprof(nid, large_gfp, large_order); 3660 3661 if (unlikely(!page)) { 3662 max_attempt_order = --large_order; 3663 continue; 3664 } 3665 3666 split_page(page, large_order); 3667 for (i = 0; i < (1U << large_order); i++) 3668 pages[nr_allocated + i] = page + i; 3669 3670 nr_allocated += 1U << large_order; 3671 nr_remaining = nr_pages - nr_allocated; 3672 3673 large_order = ilog2(nr_remaining); 3674 large_order = min(max_attempt_order, large_order); 3675 } 3676 3677 /* 3678 * For order-0 pages we make use of bulk allocator, if 3679 * the page array is partly or not at all populated due 3680 * to fails, fallback to a single page allocator that is 3681 * more permissive. 3682 */ 3683 if (!order) { 3684 while (nr_allocated < nr_pages) { 3685 unsigned int nr, nr_pages_request; 3686 3687 /* 3688 * A maximum allowed request is hard-coded and is 100 3689 * pages per call. That is done in order to prevent a 3690 * long preemption off scenario in the bulk-allocator 3691 * so the range is [1:100]. 3692 */ 3693 nr_pages_request = min(100U, nr_pages - nr_allocated); 3694 3695 /* memory allocation should consider mempolicy, we can't 3696 * wrongly use nearest node when nid == NUMA_NO_NODE, 3697 * otherwise memory may be allocated in only one node, 3698 * but mempolicy wants to alloc memory by interleaving. 3699 */ 3700 if (IS_ENABLED(CONFIG_NUMA) && nid == NUMA_NO_NODE) 3701 nr = alloc_pages_bulk_mempolicy_noprof(gfp, 3702 nr_pages_request, 3703 pages + nr_allocated); 3704 else 3705 nr = alloc_pages_bulk_node_noprof(gfp, nid, 3706 nr_pages_request, 3707 pages + nr_allocated); 3708 3709 nr_allocated += nr; 3710 3711 /* 3712 * If zero or pages were obtained partly, 3713 * fallback to a single page allocator. 3714 */ 3715 if (nr != nr_pages_request) 3716 break; 3717 } 3718 } 3719 3720 /* High-order pages or fallback path if "bulk" fails. */ 3721 while (nr_allocated < nr_pages) { 3722 if (!(gfp & __GFP_NOFAIL) && fatal_signal_pending(current)) 3723 break; 3724 3725 if (nid == NUMA_NO_NODE) 3726 page = alloc_pages_noprof(gfp, order); 3727 else 3728 page = alloc_pages_node_noprof(nid, gfp, order); 3729 3730 if (unlikely(!page)) 3731 break; 3732 3733 /* 3734 * High-order allocations must be able to be treated as 3735 * independent small pages by callers (as they can with 3736 * small-page vmallocs). Some drivers do their own refcounting 3737 * on vmalloc_to_page() pages, some use page->mapping, 3738 * page->lru, etc. 3739 */ 3740 if (order) 3741 split_page(page, order); 3742 3743 /* 3744 * Careful, we allocate and map page-order pages, but 3745 * tracking is done per PAGE_SIZE page so as to keep the 3746 * vm_struct APIs independent of the physical/mapped size. 3747 */ 3748 for (i = 0; i < (1U << order); i++) 3749 pages[nr_allocated + i] = page + i; 3750 3751 nr_allocated += 1U << order; 3752 } 3753 3754 return nr_allocated; 3755} 3756 3757static LLIST_HEAD(pending_vm_area_cleanup); 3758static void cleanup_vm_area_work(struct work_struct *work) 3759{ 3760 struct vm_struct *area, *tmp; 3761 struct llist_node *head; 3762 3763 head = llist_del_all(&pending_vm_area_cleanup); 3764 if (!head) 3765 return; 3766 3767 llist_for_each_entry_safe(area, tmp, head, llnode) { 3768 if (!area->pages) 3769 free_vm_area(area); 3770 else 3771 vfree(area->addr); 3772 } 3773} 3774 3775/* 3776 * Helper for __vmalloc_area_node() to defer cleanup 3777 * of partially initialized vm_struct in error paths. 3778 */ 3779static DECLARE_WORK(cleanup_vm_area, cleanup_vm_area_work); 3780static void defer_vm_area_cleanup(struct vm_struct *area) 3781{ 3782 if (llist_add(&area->llnode, &pending_vm_area_cleanup)) 3783 schedule_work(&cleanup_vm_area); 3784} 3785 3786/* 3787 * Page tables allocations ignore external GFP. Enforces it by 3788 * the memalloc scope API. It is used by vmalloc internals and 3789 * KASAN shadow population only. 3790 * 3791 * GFP to scope mapping: 3792 * 3793 * non-blocking (no __GFP_DIRECT_RECLAIM) - memalloc_noreclaim_save() 3794 * GFP_NOFS - memalloc_nofs_save() 3795 * GFP_NOIO - memalloc_noio_save() 3796 * 3797 * Returns a flag cookie to pair with restore. 3798 */ 3799unsigned int 3800memalloc_apply_gfp_scope(gfp_t gfp_mask) 3801{ 3802 unsigned int flags = 0; 3803 3804 if (!gfpflags_allow_blocking(gfp_mask)) 3805 flags = memalloc_noreclaim_save(); 3806 else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == __GFP_IO) 3807 flags = memalloc_nofs_save(); 3808 else if ((gfp_mask & (__GFP_FS | __GFP_IO)) == 0) 3809 flags = memalloc_noio_save(); 3810 3811 /* 0 - no scope applied. */ 3812 return flags; 3813} 3814 3815void 3816memalloc_restore_scope(unsigned int flags) 3817{ 3818 if (flags) 3819 memalloc_flags_restore(flags); 3820} 3821 3822static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, 3823 pgprot_t prot, unsigned int page_shift, 3824 int node) 3825{ 3826 const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; 3827 bool nofail = gfp_mask & __GFP_NOFAIL; 3828 unsigned long addr = (unsigned long)area->addr; 3829 unsigned long size = get_vm_area_size(area); 3830 unsigned long array_size; 3831 unsigned int nr_small_pages = size >> PAGE_SHIFT; 3832 unsigned int page_order; 3833 unsigned int flags; 3834 int ret; 3835 3836 array_size = (unsigned long)nr_small_pages * sizeof(struct page *); 3837 3838 /* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */ 3839 if (!gfpflags_allow_blocking(gfp_mask)) 3840 nofail = false; 3841 3842 if (!(gfp_mask & (GFP_DMA | GFP_DMA32))) 3843 gfp_mask |= __GFP_HIGHMEM; 3844 3845 /* Please note that the recursion is strictly bounded. */ 3846 if (array_size > PAGE_SIZE) { 3847 area->pages = __vmalloc_node_noprof(array_size, 1, nested_gfp, node, 3848 area->caller); 3849 } else { 3850 area->pages = kmalloc_node_noprof(array_size, nested_gfp, node); 3851 } 3852 3853 if (!area->pages) { 3854 warn_alloc(gfp_mask, NULL, 3855 "vmalloc error: size %lu, failed to allocated page array size %lu", 3856 nr_small_pages * PAGE_SIZE, array_size); 3857 goto fail; 3858 } 3859 3860 set_vm_area_page_order(area, page_shift - PAGE_SHIFT); 3861 page_order = vm_area_page_order(area); 3862 3863 /* 3864 * High-order nofail allocations are really expensive and 3865 * potentially dangerous (pre-mature OOM, disruptive reclaim 3866 * and compaction etc. 3867 * 3868 * Please note, the __vmalloc_node_range_noprof() falls-back 3869 * to order-0 pages if high-order attempt is unsuccessful. 3870 */ 3871 area->nr_pages = vm_area_alloc_pages( 3872 vmalloc_gfp_adjust(gfp_mask, page_order), node, 3873 page_order, nr_small_pages, area->pages); 3874 3875 atomic_long_add(area->nr_pages, &nr_vmalloc_pages); 3876 /* All pages of vm should be charged to same memcg, so use first one. */ 3877 if (gfp_mask & __GFP_ACCOUNT && area->nr_pages) 3878 mod_memcg_page_state(area->pages[0], MEMCG_VMALLOC, 3879 area->nr_pages); 3880 3881 /* 3882 * If not enough pages were obtained to accomplish an 3883 * allocation request, free them via vfree() if any. 3884 */ 3885 if (area->nr_pages != nr_small_pages) { 3886 /* 3887 * vm_area_alloc_pages() can fail due to insufficient memory but 3888 * also:- 3889 * 3890 * - a pending fatal signal 3891 * - insufficient huge page-order pages 3892 * 3893 * Since we always retry allocations at order-0 in the huge page 3894 * case a warning for either is spurious. 3895 */ 3896 if (!fatal_signal_pending(current) && page_order == 0) 3897 warn_alloc(gfp_mask, NULL, 3898 "vmalloc error: size %lu, failed to allocate pages", 3899 area->nr_pages * PAGE_SIZE); 3900 goto fail; 3901 } 3902 3903 /* 3904 * page tables allocations ignore external gfp mask, enforce it 3905 * by the scope API 3906 */ 3907 flags = memalloc_apply_gfp_scope(gfp_mask); 3908 do { 3909 ret = __vmap_pages_range(addr, addr + size, prot, area->pages, 3910 page_shift, nested_gfp); 3911 if (nofail && (ret < 0)) 3912 schedule_timeout_uninterruptible(1); 3913 } while (nofail && (ret < 0)); 3914 memalloc_restore_scope(flags); 3915 3916 if (ret < 0) { 3917 warn_alloc(gfp_mask, NULL, 3918 "vmalloc error: size %lu, failed to map pages", 3919 area->nr_pages * PAGE_SIZE); 3920 goto fail; 3921 } 3922 3923 return area->addr; 3924 3925fail: 3926 defer_vm_area_cleanup(area); 3927 return NULL; 3928} 3929 3930/* 3931 * See __vmalloc_node_range() for a clear list of supported vmalloc flags. 3932 * This gfp lists all flags currently passed through vmalloc. Currently, 3933 * __GFP_ZERO is used by BPF and __GFP_NORETRY is used by percpu. Both drm 3934 * and BPF also use GFP_USER. Additionally, various users pass 3935 * GFP_KERNEL_ACCOUNT. Xfs uses __GFP_NOLOCKDEP. 3936 */ 3937#define GFP_VMALLOC_SUPPORTED (GFP_KERNEL | GFP_ATOMIC | GFP_NOWAIT |\ 3938 __GFP_NOFAIL | __GFP_ZERO | __GFP_NORETRY |\ 3939 GFP_NOFS | GFP_NOIO | GFP_KERNEL_ACCOUNT |\ 3940 GFP_USER | __GFP_NOLOCKDEP) 3941 3942static gfp_t vmalloc_fix_flags(gfp_t flags) 3943{ 3944 gfp_t invalid_mask = flags & ~GFP_VMALLOC_SUPPORTED; 3945 3946 flags &= GFP_VMALLOC_SUPPORTED; 3947 WARN_ONCE(1, "Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n", 3948 invalid_mask, &invalid_mask, flags, &flags); 3949 return flags; 3950} 3951 3952/** 3953 * __vmalloc_node_range - allocate virtually contiguous memory 3954 * @size: allocation size 3955 * @align: desired alignment 3956 * @start: vm area range start 3957 * @end: vm area range end 3958 * @gfp_mask: flags for the page level allocator 3959 * @prot: protection mask for the allocated pages 3960 * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) 3961 * @node: node to use for allocation or NUMA_NO_NODE 3962 * @caller: caller's return address 3963 * 3964 * Allocate enough pages to cover @size from the page level 3965 * allocator with @gfp_mask flags and map them into contiguous 3966 * virtual range with protection @prot. 3967 * 3968 * Supported GFP classes: %GFP_KERNEL, %GFP_ATOMIC, %GFP_NOWAIT, 3969 * %GFP_NOFS and %GFP_NOIO. Zone modifiers are not supported. 3970 * Please note %GFP_ATOMIC and %GFP_NOWAIT are supported only 3971 * by __vmalloc(). 3972 * 3973 * Retry modifiers: only %__GFP_NOFAIL is supported; %__GFP_NORETRY 3974 * and %__GFP_RETRY_MAYFAIL are not supported. 3975 * 3976 * %__GFP_NOWARN can be used to suppress failure messages. 3977 * 3978 * Can not be called from interrupt nor NMI contexts. 3979 * Return: the address of the area or %NULL on failure 3980 */ 3981void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align, 3982 unsigned long start, unsigned long end, gfp_t gfp_mask, 3983 pgprot_t prot, unsigned long vm_flags, int node, 3984 const void *caller) 3985{ 3986 struct vm_struct *area; 3987 void *ret; 3988 kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_NONE; 3989 unsigned long original_align = align; 3990 unsigned int shift = PAGE_SHIFT; 3991 3992 if (WARN_ON_ONCE(!size)) 3993 return NULL; 3994 3995 if ((size >> PAGE_SHIFT) > totalram_pages()) { 3996 warn_alloc(gfp_mask, NULL, 3997 "vmalloc error: size %lu, exceeds total pages", 3998 size); 3999 return NULL; 4000 } 4001 4002 if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) { 4003 /* 4004 * Try huge pages. Only try for PAGE_KERNEL allocations, 4005 * others like modules don't yet expect huge pages in 4006 * their allocations due to apply_to_page_range not 4007 * supporting them. 4008 */ 4009 4010 if (arch_vmap_pmd_supported(prot) && size >= PMD_SIZE) 4011 shift = PMD_SHIFT; 4012 else 4013 shift = arch_vmap_pte_supported_shift(size); 4014 4015 align = max(original_align, 1UL << shift); 4016 } 4017 4018again: 4019 area = __get_vm_area_node(size, align, shift, VM_ALLOC | 4020 VM_UNINITIALIZED | vm_flags, start, end, node, 4021 gfp_mask, caller); 4022 if (!area) { 4023 bool nofail = gfp_mask & __GFP_NOFAIL; 4024 warn_alloc(gfp_mask, NULL, 4025 "vmalloc error: size %lu, vm_struct allocation failed%s", 4026 size, (nofail) ? ". Retrying." : ""); 4027 if (nofail) { 4028 schedule_timeout_uninterruptible(1); 4029 goto again; 4030 } 4031 goto fail; 4032 } 4033 4034 /* 4035 * Prepare arguments for __vmalloc_area_node() and 4036 * kasan_unpoison_vmalloc(). 4037 */ 4038 if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) { 4039 if (kasan_hw_tags_enabled()) { 4040 /* 4041 * Modify protection bits to allow tagging. 4042 * This must be done before mapping. 4043 */ 4044 prot = arch_vmap_pgprot_tagged(prot); 4045 4046 /* 4047 * Skip page_alloc poisoning and zeroing for physical 4048 * pages backing VM_ALLOC mapping. Memory is instead 4049 * poisoned and zeroed by kasan_unpoison_vmalloc(). 4050 */ 4051 gfp_mask |= __GFP_SKIP_KASAN | __GFP_SKIP_ZERO; 4052 } 4053 4054 /* Take note that the mapping is PAGE_KERNEL. */ 4055 kasan_flags |= KASAN_VMALLOC_PROT_NORMAL; 4056 } 4057 4058 /* Allocate physical pages and map them into vmalloc space. */ 4059 ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node); 4060 if (!ret) 4061 goto fail; 4062 4063 /* 4064 * Mark the pages as accessible, now that they are mapped. 4065 * The condition for setting KASAN_VMALLOC_INIT should complement the 4066 * one in post_alloc_hook() with regards to the __GFP_SKIP_ZERO check 4067 * to make sure that memory is initialized under the same conditions. 4068 * Tag-based KASAN modes only assign tags to normal non-executable 4069 * allocations, see __kasan_unpoison_vmalloc(). 4070 */ 4071 kasan_flags |= KASAN_VMALLOC_VM_ALLOC; 4072 if (!want_init_on_free() && want_init_on_alloc(gfp_mask) && 4073 (gfp_mask & __GFP_SKIP_ZERO)) 4074 kasan_flags |= KASAN_VMALLOC_INIT; 4075 /* KASAN_VMALLOC_PROT_NORMAL already set if required. */ 4076 area->addr = kasan_unpoison_vmalloc(area->addr, size, kasan_flags); 4077 4078 /* 4079 * In this function, newly allocated vm_struct has VM_UNINITIALIZED 4080 * flag. It means that vm_struct is not fully initialized. 4081 * Now, it is fully initialized, so remove this flag here. 4082 */ 4083 clear_vm_uninitialized_flag(area); 4084 4085 if (!(vm_flags & VM_DEFER_KMEMLEAK)) 4086 kmemleak_vmalloc(area, PAGE_ALIGN(size), gfp_mask); 4087 4088 return area->addr; 4089 4090fail: 4091 if (shift > PAGE_SHIFT) { 4092 shift = PAGE_SHIFT; 4093 align = original_align; 4094 goto again; 4095 } 4096 4097 return NULL; 4098} 4099 4100/** 4101 * __vmalloc_node - allocate virtually contiguous memory 4102 * @size: allocation size 4103 * @align: desired alignment 4104 * @gfp_mask: flags for the page level allocator 4105 * @node: node to use for allocation or NUMA_NO_NODE 4106 * @caller: caller's return address 4107 * 4108 * Allocate enough pages to cover @size from the page level allocator with 4109 * @gfp_mask flags. Map them into contiguous kernel virtual space. 4110 * 4111 * Semantics of @gfp_mask (including reclaim/retry modifiers such as 4112 * __GFP_NOFAIL) are the same as in __vmalloc_node_range_noprof(). 4113 * 4114 * Return: pointer to the allocated memory or %NULL on error 4115 */ 4116void *__vmalloc_node_noprof(unsigned long size, unsigned long align, 4117 gfp_t gfp_mask, int node, const void *caller) 4118{ 4119 return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END, 4120 gfp_mask, PAGE_KERNEL, 0, node, caller); 4121} 4122/* 4123 * This is only for performance analysis of vmalloc and stress purpose. 4124 * It is required by vmalloc test module, therefore do not use it other 4125 * than that. 4126 */ 4127#ifdef CONFIG_TEST_VMALLOC_MODULE 4128EXPORT_SYMBOL_GPL(__vmalloc_node_noprof); 4129#endif 4130 4131void *__vmalloc_noprof(unsigned long size, gfp_t gfp_mask) 4132{ 4133 if (unlikely(gfp_mask & ~GFP_VMALLOC_SUPPORTED)) 4134 gfp_mask = vmalloc_fix_flags(gfp_mask); 4135 return __vmalloc_node_noprof(size, 1, gfp_mask, NUMA_NO_NODE, 4136 __builtin_return_address(0)); 4137} 4138EXPORT_SYMBOL(__vmalloc_noprof); 4139 4140/** 4141 * vmalloc - allocate virtually contiguous memory 4142 * @size: allocation size 4143 * 4144 * Allocate enough pages to cover @size from the page level 4145 * allocator and map them into contiguous kernel virtual space. 4146 * 4147 * For tight control over page level allocator and protection flags 4148 * use __vmalloc() instead. 4149 * 4150 * Return: pointer to the allocated memory or %NULL on error 4151 */ 4152void *vmalloc_noprof(unsigned long size) 4153{ 4154 return __vmalloc_node_noprof(size, 1, GFP_KERNEL, NUMA_NO_NODE, 4155 __builtin_return_address(0)); 4156} 4157EXPORT_SYMBOL(vmalloc_noprof); 4158 4159/** 4160 * vmalloc_huge_node - allocate virtually contiguous memory, allow huge pages 4161 * @size: allocation size 4162 * @gfp_mask: flags for the page level allocator 4163 * @node: node to use for allocation or NUMA_NO_NODE 4164 * 4165 * Allocate enough pages to cover @size from the page level 4166 * allocator and map them into contiguous kernel virtual space. 4167 * If @size is greater than or equal to PMD_SIZE, allow using 4168 * huge pages for the memory 4169 * 4170 * Return: pointer to the allocated memory or %NULL on error 4171 */ 4172void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node) 4173{ 4174 if (unlikely(gfp_mask & ~GFP_VMALLOC_SUPPORTED)) 4175 gfp_mask = vmalloc_fix_flags(gfp_mask); 4176 return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, 4177 gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, 4178 node, __builtin_return_address(0)); 4179} 4180EXPORT_SYMBOL_GPL(vmalloc_huge_node_noprof); 4181 4182/** 4183 * vzalloc - allocate virtually contiguous memory with zero fill 4184 * @size: allocation size 4185 * 4186 * Allocate enough pages to cover @size from the page level 4187 * allocator and map them into contiguous kernel virtual space. 4188 * The memory allocated is set to zero. 4189 * 4190 * For tight control over page level allocator and protection flags 4191 * use __vmalloc() instead. 4192 * 4193 * Return: pointer to the allocated memory or %NULL on error 4194 */ 4195void *vzalloc_noprof(unsigned long size) 4196{ 4197 return __vmalloc_node_noprof(size, 1, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 4198 __builtin_return_address(0)); 4199} 4200EXPORT_SYMBOL(vzalloc_noprof); 4201 4202/** 4203 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace 4204 * @size: allocation size 4205 * 4206 * The resulting memory area is zeroed so it can be mapped to userspace 4207 * without leaking data. 4208 * 4209 * Return: pointer to the allocated memory or %NULL on error 4210 */ 4211void *vmalloc_user_noprof(unsigned long size) 4212{ 4213 return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, 4214 GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, 4215 VM_USERMAP, NUMA_NO_NODE, 4216 __builtin_return_address(0)); 4217} 4218EXPORT_SYMBOL(vmalloc_user_noprof); 4219 4220/** 4221 * vmalloc_node - allocate memory on a specific node 4222 * @size: allocation size 4223 * @node: numa node 4224 * 4225 * Allocate enough pages to cover @size from the page level 4226 * allocator and map them into contiguous kernel virtual space. 4227 * 4228 * For tight control over page level allocator and protection flags 4229 * use __vmalloc() instead. 4230 * 4231 * Return: pointer to the allocated memory or %NULL on error 4232 */ 4233void *vmalloc_node_noprof(unsigned long size, int node) 4234{ 4235 return __vmalloc_node_noprof(size, 1, GFP_KERNEL, node, 4236 __builtin_return_address(0)); 4237} 4238EXPORT_SYMBOL(vmalloc_node_noprof); 4239 4240/** 4241 * vzalloc_node - allocate memory on a specific node with zero fill 4242 * @size: allocation size 4243 * @node: numa node 4244 * 4245 * Allocate enough pages to cover @size from the page level 4246 * allocator and map them into contiguous kernel virtual space. 4247 * The memory allocated is set to zero. 4248 * 4249 * Return: pointer to the allocated memory or %NULL on error 4250 */ 4251void *vzalloc_node_noprof(unsigned long size, int node) 4252{ 4253 return __vmalloc_node_noprof(size, 1, GFP_KERNEL | __GFP_ZERO, node, 4254 __builtin_return_address(0)); 4255} 4256EXPORT_SYMBOL(vzalloc_node_noprof); 4257 4258/** 4259 * vrealloc_node_align - reallocate virtually contiguous memory; contents 4260 * remain unchanged 4261 * @p: object to reallocate memory for 4262 * @size: the size to reallocate 4263 * @align: requested alignment 4264 * @flags: the flags for the page level allocator 4265 * @nid: node number of the target node 4266 * 4267 * If @p is %NULL, vrealloc_XXX() behaves exactly like vmalloc_XXX(). If @size 4268 * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 4269 * 4270 * If the caller wants the new memory to be on specific node *only*, 4271 * __GFP_THISNODE flag should be set, otherwise the function will try to avoid 4272 * reallocation and possibly disregard the specified @nid. 4273 * 4274 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 4275 * initial memory allocation, every subsequent call to this API for the same 4276 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 4277 * __GFP_ZERO is not fully honored by this API. 4278 * 4279 * Requesting an alignment that is bigger than the alignment of the existing 4280 * allocation will fail. 4281 * 4282 * In any case, the contents of the object pointed to are preserved up to the 4283 * lesser of the new and old sizes. 4284 * 4285 * This function must not be called concurrently with itself or vfree() for the 4286 * same memory allocation. 4287 * 4288 * Return: pointer to the allocated memory; %NULL if @size is zero or in case of 4289 * failure 4290 */ 4291void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, 4292 gfp_t flags, int nid) 4293{ 4294 struct vm_struct *vm = NULL; 4295 size_t alloced_size = 0; 4296 size_t old_size = 0; 4297 void *n; 4298 4299 if (!size) { 4300 vfree(p); 4301 return NULL; 4302 } 4303 4304 if (p) { 4305 vm = find_vm_area(p); 4306 if (unlikely(!vm)) { 4307 WARN(1, "Trying to vrealloc() nonexistent vm area (%p)\n", p); 4308 return NULL; 4309 } 4310 4311 alloced_size = get_vm_area_size(vm); 4312 old_size = vm->requested_size; 4313 if (WARN(alloced_size < old_size, 4314 "vrealloc() has mismatched area vs requested sizes (%p)\n", p)) 4315 return NULL; 4316 if (WARN(!IS_ALIGNED((unsigned long)p, align), 4317 "will not reallocate with a bigger alignment (0x%lx)\n", align)) 4318 return NULL; 4319 if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE && 4320 nid != page_to_nid(vmalloc_to_page(p))) 4321 goto need_realloc; 4322 } 4323 4324 /* 4325 * TODO: Shrink the vm_area, i.e. unmap and free unused pages. What 4326 * would be a good heuristic for when to shrink the vm_area? 4327 */ 4328 if (size <= old_size) { 4329 /* Zero out "freed" memory, potentially for future realloc. */ 4330 if (want_init_on_free() || want_init_on_alloc(flags)) 4331 memset((void *)p + size, 0, old_size - size); 4332 vm->requested_size = size; 4333 kasan_vrealloc(p, old_size, size); 4334 return (void *)p; 4335 } 4336 4337 /* 4338 * We already have the bytes available in the allocation; use them. 4339 */ 4340 if (size <= alloced_size) { 4341 /* 4342 * No need to zero memory here, as unused memory will have 4343 * already been zeroed at initial allocation time or during 4344 * realloc shrink time. 4345 */ 4346 vm->requested_size = size; 4347 kasan_vrealloc(p, old_size, size); 4348 return (void *)p; 4349 } 4350 4351need_realloc: 4352 /* TODO: Grow the vm_area, i.e. allocate and map additional pages. */ 4353 n = __vmalloc_node_noprof(size, align, flags, nid, __builtin_return_address(0)); 4354 4355 if (!n) 4356 return NULL; 4357 4358 if (p) { 4359 memcpy(n, p, old_size); 4360 vfree(p); 4361 } 4362 4363 return n; 4364} 4365 4366#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) 4367#define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 4368#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) 4369#define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL) 4370#else 4371/* 4372 * 64b systems should always have either DMA or DMA32 zones. For others 4373 * GFP_DMA32 should do the right thing and use the normal zone. 4374 */ 4375#define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL) 4376#endif 4377 4378/** 4379 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 4380 * @size: allocation size 4381 * 4382 * Allocate enough 32bit PA addressable pages to cover @size from the 4383 * page level allocator and map them into contiguous kernel virtual space. 4384 * 4385 * Return: pointer to the allocated memory or %NULL on error 4386 */ 4387void *vmalloc_32_noprof(unsigned long size) 4388{ 4389 return __vmalloc_node_noprof(size, 1, GFP_VMALLOC32, NUMA_NO_NODE, 4390 __builtin_return_address(0)); 4391} 4392EXPORT_SYMBOL(vmalloc_32_noprof); 4393 4394/** 4395 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 4396 * @size: allocation size 4397 * 4398 * The resulting memory area is 32bit addressable and zeroed so it can be 4399 * mapped to userspace without leaking data. 4400 * 4401 * Return: pointer to the allocated memory or %NULL on error 4402 */ 4403void *vmalloc_32_user_noprof(unsigned long size) 4404{ 4405 return __vmalloc_node_range_noprof(size, SHMLBA, VMALLOC_START, VMALLOC_END, 4406 GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, 4407 VM_USERMAP, NUMA_NO_NODE, 4408 __builtin_return_address(0)); 4409} 4410EXPORT_SYMBOL(vmalloc_32_user_noprof); 4411 4412/* 4413 * Atomically zero bytes in the iterator. 4414 * 4415 * Returns the number of zeroed bytes. 4416 */ 4417static size_t zero_iter(struct iov_iter *iter, size_t count) 4418{ 4419 size_t remains = count; 4420 4421 while (remains > 0) { 4422 size_t num, copied; 4423 4424 num = min_t(size_t, remains, PAGE_SIZE); 4425 copied = copy_page_to_iter_nofault(ZERO_PAGE(0), 0, num, iter); 4426 remains -= copied; 4427 4428 if (copied < num) 4429 break; 4430 } 4431 4432 return count - remains; 4433} 4434 4435/* 4436 * small helper routine, copy contents to iter from addr. 4437 * If the page is not present, fill zero. 4438 * 4439 * Returns the number of copied bytes. 4440 */ 4441static size_t aligned_vread_iter(struct iov_iter *iter, 4442 const char *addr, size_t count) 4443{ 4444 size_t remains = count; 4445 struct page *page; 4446 4447 while (remains > 0) { 4448 unsigned long offset, length; 4449 size_t copied = 0; 4450 4451 offset = offset_in_page(addr); 4452 length = PAGE_SIZE - offset; 4453 if (length > remains) 4454 length = remains; 4455 page = vmalloc_to_page(addr); 4456 /* 4457 * To do safe access to this _mapped_ area, we need lock. But 4458 * adding lock here means that we need to add overhead of 4459 * vmalloc()/vfree() calls for this _debug_ interface, rarely 4460 * used. Instead of that, we'll use an local mapping via 4461 * copy_page_to_iter_nofault() and accept a small overhead in 4462 * this access function. 4463 */ 4464 if (page) 4465 copied = copy_page_to_iter_nofault(page, offset, 4466 length, iter); 4467 else 4468 copied = zero_iter(iter, length); 4469 4470 addr += copied; 4471 remains -= copied; 4472 4473 if (copied != length) 4474 break; 4475 } 4476 4477 return count - remains; 4478} 4479 4480/* 4481 * Read from a vm_map_ram region of memory. 4482 * 4483 * Returns the number of copied bytes. 4484 */ 4485static size_t vmap_ram_vread_iter(struct iov_iter *iter, const char *addr, 4486 size_t count, unsigned long flags) 4487{ 4488 char *start; 4489 struct vmap_block *vb; 4490 struct xarray *xa; 4491 unsigned long offset; 4492 unsigned int rs, re; 4493 size_t remains, n; 4494 4495 /* 4496 * If it's area created by vm_map_ram() interface directly, but 4497 * not further subdividing and delegating management to vmap_block, 4498 * handle it here. 4499 */ 4500 if (!(flags & VMAP_BLOCK)) 4501 return aligned_vread_iter(iter, addr, count); 4502 4503 remains = count; 4504 4505 /* 4506 * Area is split into regions and tracked with vmap_block, read out 4507 * each region and zero fill the hole between regions. 4508 */ 4509 xa = addr_to_vb_xa((unsigned long) addr); 4510 vb = xa_load(xa, addr_to_vb_idx((unsigned long)addr)); 4511 if (!vb) 4512 goto finished_zero; 4513 4514 spin_lock(&vb->lock); 4515 if (bitmap_empty(vb->used_map, VMAP_BBMAP_BITS)) { 4516 spin_unlock(&vb->lock); 4517 goto finished_zero; 4518 } 4519 4520 for_each_set_bitrange(rs, re, vb->used_map, VMAP_BBMAP_BITS) { 4521 size_t copied; 4522 4523 if (remains == 0) 4524 goto finished; 4525 4526 start = vmap_block_vaddr(vb->va->va_start, rs); 4527 4528 if (addr < start) { 4529 size_t to_zero = min_t(size_t, start - addr, remains); 4530 size_t zeroed = zero_iter(iter, to_zero); 4531 4532 addr += zeroed; 4533 remains -= zeroed; 4534 4535 if (remains == 0 || zeroed != to_zero) 4536 goto finished; 4537 } 4538 4539 /*it could start reading from the middle of used region*/ 4540 offset = offset_in_page(addr); 4541 n = ((re - rs + 1) << PAGE_SHIFT) - offset; 4542 if (n > remains) 4543 n = remains; 4544 4545 copied = aligned_vread_iter(iter, start + offset, n); 4546 4547 addr += copied; 4548 remains -= copied; 4549 4550 if (copied != n) 4551 goto finished; 4552 } 4553 4554 spin_unlock(&vb->lock); 4555 4556finished_zero: 4557 /* zero-fill the left dirty or free regions */ 4558 return count - remains + zero_iter(iter, remains); 4559finished: 4560 /* We couldn't copy/zero everything */ 4561 spin_unlock(&vb->lock); 4562 return count - remains; 4563} 4564 4565/** 4566 * vread_iter() - read vmalloc area in a safe way to an iterator. 4567 * @iter: the iterator to which data should be written. 4568 * @addr: vm address. 4569 * @count: number of bytes to be read. 4570 * 4571 * This function checks that addr is a valid vmalloc'ed area, and 4572 * copy data from that area to a given buffer. If the given memory range 4573 * of [addr...addr+count) includes some valid address, data is copied to 4574 * proper area of @buf. If there are memory holes, they'll be zero-filled. 4575 * IOREMAP area is treated as memory hole and no copy is done. 4576 * 4577 * If [addr...addr+count) doesn't includes any intersects with alive 4578 * vm_struct area, returns 0. @buf should be kernel's buffer. 4579 * 4580 * Note: In usual ops, vread() is never necessary because the caller 4581 * should know vmalloc() area is valid and can use memcpy(). 4582 * This is for routines which have to access vmalloc area without 4583 * any information, as /proc/kcore. 4584 * 4585 * Return: number of bytes for which addr and buf should be increased 4586 * (same number as @count) or %0 if [addr...addr+count) doesn't 4587 * include any intersection with valid vmalloc area 4588 */ 4589long vread_iter(struct iov_iter *iter, const char *addr, size_t count) 4590{ 4591 struct vmap_node *vn; 4592 struct vmap_area *va; 4593 struct vm_struct *vm; 4594 char *vaddr; 4595 size_t n, size, flags, remains; 4596 unsigned long next; 4597 4598 addr = kasan_reset_tag(addr); 4599 4600 /* Don't allow overflow */ 4601 if ((unsigned long) addr + count < count) 4602 count = -(unsigned long) addr; 4603 4604 remains = count; 4605 4606 vn = find_vmap_area_exceed_addr_lock((unsigned long) addr, &va); 4607 if (!vn) 4608 goto finished_zero; 4609 4610 /* no intersects with alive vmap_area */ 4611 if ((unsigned long)addr + remains <= va->va_start) 4612 goto finished_zero; 4613 4614 do { 4615 size_t copied; 4616 4617 if (remains == 0) 4618 goto finished; 4619 4620 vm = va->vm; 4621 flags = va->flags & VMAP_FLAGS_MASK; 4622 /* 4623 * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need 4624 * be set together with VMAP_RAM. 4625 */ 4626 WARN_ON(flags == VMAP_BLOCK); 4627 4628 if (!vm && !flags) 4629 goto next_va; 4630 4631 if (vm && (vm->flags & VM_UNINITIALIZED)) 4632 goto next_va; 4633 4634 /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 4635 smp_rmb(); 4636 4637 vaddr = (char *) va->va_start; 4638 size = vm ? get_vm_area_size(vm) : va_size(va); 4639 4640 if (addr >= vaddr + size) 4641 goto next_va; 4642 4643 if (addr < vaddr) { 4644 size_t to_zero = min_t(size_t, vaddr - addr, remains); 4645 size_t zeroed = zero_iter(iter, to_zero); 4646 4647 addr += zeroed; 4648 remains -= zeroed; 4649 4650 if (remains == 0 || zeroed != to_zero) 4651 goto finished; 4652 } 4653 4654 n = vaddr + size - addr; 4655 if (n > remains) 4656 n = remains; 4657 4658 if (flags & VMAP_RAM) 4659 copied = vmap_ram_vread_iter(iter, addr, n, flags); 4660 else if (!(vm && (vm->flags & (VM_IOREMAP | VM_SPARSE)))) 4661 copied = aligned_vread_iter(iter, addr, n); 4662 else /* IOREMAP | SPARSE area is treated as memory hole */ 4663 copied = zero_iter(iter, n); 4664 4665 addr += copied; 4666 remains -= copied; 4667 4668 if (copied != n) 4669 goto finished; 4670 4671 next_va: 4672 next = va->va_end; 4673 spin_unlock(&vn->busy.lock); 4674 } while ((vn = find_vmap_area_exceed_addr_lock(next, &va))); 4675 4676finished_zero: 4677 if (vn) 4678 spin_unlock(&vn->busy.lock); 4679 4680 /* zero-fill memory holes */ 4681 return count - remains + zero_iter(iter, remains); 4682finished: 4683 /* Nothing remains, or We couldn't copy/zero everything. */ 4684 if (vn) 4685 spin_unlock(&vn->busy.lock); 4686 4687 return count - remains; 4688} 4689 4690/** 4691 * remap_vmalloc_range_partial - map vmalloc pages to userspace 4692 * @vma: vma to cover 4693 * @uaddr: target user address to start at 4694 * @kaddr: virtual address of vmalloc kernel memory 4695 * @pgoff: offset from @kaddr to start at 4696 * @size: size of map area 4697 * 4698 * Returns: 0 for success, -Exxx on failure 4699 * 4700 * This function checks that @kaddr is a valid vmalloc'ed area, 4701 * and that it is big enough to cover the range starting at 4702 * @uaddr in @vma. Will return failure if that criteria isn't 4703 * met. 4704 * 4705 * Similar to remap_pfn_range() (see mm/memory.c) 4706 */ 4707int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, 4708 void *kaddr, unsigned long pgoff, 4709 unsigned long size) 4710{ 4711 struct vm_struct *area; 4712 unsigned long off; 4713 unsigned long end_index; 4714 4715 if (check_shl_overflow(pgoff, PAGE_SHIFT, &off)) 4716 return -EINVAL; 4717 4718 size = PAGE_ALIGN(size); 4719 4720 if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) 4721 return -EINVAL; 4722 4723 area = find_vm_area(kaddr); 4724 if (!area) 4725 return -EINVAL; 4726 4727 if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT))) 4728 return -EINVAL; 4729 4730 if (check_add_overflow(size, off, &end_index) || 4731 end_index > get_vm_area_size(area)) 4732 return -EINVAL; 4733 kaddr += off; 4734 4735 do { 4736 struct page *page = vmalloc_to_page(kaddr); 4737 int ret; 4738 4739 ret = vm_insert_page(vma, uaddr, page); 4740 if (ret) 4741 return ret; 4742 4743 uaddr += PAGE_SIZE; 4744 kaddr += PAGE_SIZE; 4745 size -= PAGE_SIZE; 4746 } while (size > 0); 4747 4748 vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); 4749 4750 return 0; 4751} 4752 4753/** 4754 * remap_vmalloc_range - map vmalloc pages to userspace 4755 * @vma: vma to cover (map full range of vma) 4756 * @addr: vmalloc memory 4757 * @pgoff: number of pages into addr before first page to map 4758 * 4759 * Returns: 0 for success, -Exxx on failure 4760 * 4761 * This function checks that addr is a valid vmalloc'ed area, and 4762 * that it is big enough to cover the vma. Will return failure if 4763 * that criteria isn't met. 4764 * 4765 * Similar to remap_pfn_range() (see mm/memory.c) 4766 */ 4767int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 4768 unsigned long pgoff) 4769{ 4770 return remap_vmalloc_range_partial(vma, vma->vm_start, 4771 addr, pgoff, 4772 vma->vm_end - vma->vm_start); 4773} 4774EXPORT_SYMBOL(remap_vmalloc_range); 4775 4776void free_vm_area(struct vm_struct *area) 4777{ 4778 struct vm_struct *ret; 4779 ret = remove_vm_area(area->addr); 4780 BUG_ON(ret != area); 4781 kfree(area); 4782} 4783EXPORT_SYMBOL_GPL(free_vm_area); 4784 4785#ifdef CONFIG_SMP 4786static struct vmap_area *node_to_va(struct rb_node *n) 4787{ 4788 return rb_entry_safe(n, struct vmap_area, rb_node); 4789} 4790 4791/** 4792 * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to 4793 * @addr: target address 4794 * 4795 * Returns: vmap_area if it is found. If there is no such area 4796 * the first highest(reverse order) vmap_area is returned 4797 * i.e. va->va_start < addr && va->va_end < addr or NULL 4798 * if there are no any areas before @addr. 4799 */ 4800static struct vmap_area * 4801pvm_find_va_enclose_addr(unsigned long addr) 4802{ 4803 struct vmap_area *va, *tmp; 4804 struct rb_node *n; 4805 4806 n = free_vmap_area_root.rb_node; 4807 va = NULL; 4808 4809 while (n) { 4810 tmp = rb_entry(n, struct vmap_area, rb_node); 4811 if (tmp->va_start <= addr) { 4812 va = tmp; 4813 if (tmp->va_end >= addr) 4814 break; 4815 4816 n = n->rb_right; 4817 } else { 4818 n = n->rb_left; 4819 } 4820 } 4821 4822 return va; 4823} 4824 4825/** 4826 * pvm_determine_end_from_reverse - find the highest aligned address 4827 * of free block below VMALLOC_END 4828 * @va: 4829 * in - the VA we start the search(reverse order); 4830 * out - the VA with the highest aligned end address. 4831 * @align: alignment for required highest address 4832 * 4833 * Returns: determined end address within vmap_area 4834 */ 4835static unsigned long 4836pvm_determine_end_from_reverse(struct vmap_area **va, unsigned long align) 4837{ 4838 unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 4839 unsigned long addr; 4840 4841 if (likely(*va)) { 4842 list_for_each_entry_from_reverse((*va), 4843 &free_vmap_area_list, list) { 4844 addr = min((*va)->va_end & ~(align - 1), vmalloc_end); 4845 if ((*va)->va_start < addr) 4846 return addr; 4847 } 4848 } 4849 4850 return 0; 4851} 4852 4853/** 4854 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator 4855 * @offsets: array containing offset of each area 4856 * @sizes: array containing size of each area 4857 * @nr_vms: the number of areas to allocate 4858 * @align: alignment, all entries in @offsets and @sizes must be aligned to this 4859 * 4860 * Returns: kmalloc'd vm_struct pointer array pointing to allocated 4861 * vm_structs on success, %NULL on failure 4862 * 4863 * Percpu allocator wants to use congruent vm areas so that it can 4864 * maintain the offsets among percpu areas. This function allocates 4865 * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to 4866 * be scattered pretty far, distance between two areas easily going up 4867 * to gigabytes. To avoid interacting with regular vmallocs, these 4868 * areas are allocated from top. 4869 * 4870 * Despite its complicated look, this allocator is rather simple. It 4871 * does everything top-down and scans free blocks from the end looking 4872 * for matching base. While scanning, if any of the areas do not fit the 4873 * base address is pulled down to fit the area. Scanning is repeated till 4874 * all the areas fit and then all necessary data structures are inserted 4875 * and the result is returned. 4876 */ 4877struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, 4878 const size_t *sizes, int nr_vms, 4879 size_t align) 4880{ 4881 const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); 4882 const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); 4883 struct vmap_area **vas, *va; 4884 struct vm_struct **vms; 4885 int area, area2, last_area, term_area; 4886 unsigned long base, start, size, end, last_end, orig_start, orig_end; 4887 bool purged = false; 4888 4889 /* verify parameters and allocate data structures */ 4890 BUG_ON(offset_in_page(align) || !is_power_of_2(align)); 4891 for (last_area = 0, area = 0; area < nr_vms; area++) { 4892 start = offsets[area]; 4893 end = start + sizes[area]; 4894 4895 /* is everything aligned properly? */ 4896 BUG_ON(!IS_ALIGNED(offsets[area], align)); 4897 BUG_ON(!IS_ALIGNED(sizes[area], align)); 4898 4899 /* detect the area with the highest address */ 4900 if (start > offsets[last_area]) 4901 last_area = area; 4902 4903 for (area2 = area + 1; area2 < nr_vms; area2++) { 4904 unsigned long start2 = offsets[area2]; 4905 unsigned long end2 = start2 + sizes[area2]; 4906 4907 BUG_ON(start2 < end && start < end2); 4908 } 4909 } 4910 last_end = offsets[last_area] + sizes[last_area]; 4911 4912 if (vmalloc_end - vmalloc_start < last_end) { 4913 WARN_ON(true); 4914 return NULL; 4915 } 4916 4917 vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); 4918 vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); 4919 if (!vas || !vms) 4920 goto err_free2; 4921 4922 for (area = 0; area < nr_vms; area++) { 4923 vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL); 4924 vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); 4925 if (!vas[area] || !vms[area]) 4926 goto err_free; 4927 } 4928retry: 4929 spin_lock(&free_vmap_area_lock); 4930 4931 /* start scanning - we scan from the top, begin with the last area */ 4932 area = term_area = last_area; 4933 start = offsets[area]; 4934 end = start + sizes[area]; 4935 4936 va = pvm_find_va_enclose_addr(vmalloc_end); 4937 base = pvm_determine_end_from_reverse(&va, align) - end; 4938 4939 while (true) { 4940 /* 4941 * base might have underflowed, add last_end before 4942 * comparing. 4943 */ 4944 if (base + last_end < vmalloc_start + last_end) 4945 goto overflow; 4946 4947 /* 4948 * Fitting base has not been found. 4949 */ 4950 if (va == NULL) 4951 goto overflow; 4952 4953 /* 4954 * If required width exceeds current VA block, move 4955 * base downwards and then recheck. 4956 */ 4957 if (base + end > va->va_end) { 4958 base = pvm_determine_end_from_reverse(&va, align) - end; 4959 term_area = area; 4960 continue; 4961 } 4962 4963 /* 4964 * If this VA does not fit, move base downwards and recheck. 4965 */ 4966 if (base + start < va->va_start) { 4967 va = node_to_va(rb_prev(&va->rb_node)); 4968 base = pvm_determine_end_from_reverse(&va, align) - end; 4969 term_area = area; 4970 continue; 4971 } 4972 4973 /* 4974 * This area fits, move on to the previous one. If 4975 * the previous one is the terminal one, we're done. 4976 */ 4977 area = (area + nr_vms - 1) % nr_vms; 4978 if (area == term_area) 4979 break; 4980 4981 start = offsets[area]; 4982 end = start + sizes[area]; 4983 va = pvm_find_va_enclose_addr(base + end); 4984 } 4985 4986 /* we've found a fitting base, insert all va's */ 4987 for (area = 0; area < nr_vms; area++) { 4988 int ret; 4989 4990 start = base + offsets[area]; 4991 size = sizes[area]; 4992 4993 va = pvm_find_va_enclose_addr(start); 4994 if (WARN_ON_ONCE(va == NULL)) 4995 /* It is a BUG(), but trigger recovery instead. */ 4996 goto recovery; 4997 4998 ret = va_clip(&free_vmap_area_root, 4999 &free_vmap_area_list, va, start, size); 5000 if (WARN_ON_ONCE(unlikely(ret))) 5001 /* It is a BUG(), but trigger recovery instead. */ 5002 goto recovery; 5003 5004 /* Allocated area. */ 5005 va = vas[area]; 5006 va->va_start = start; 5007 va->va_end = start + size; 5008 } 5009 5010 spin_unlock(&free_vmap_area_lock); 5011 5012 /* populate the kasan shadow space */ 5013 for (area = 0; area < nr_vms; area++) { 5014 if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area], GFP_KERNEL)) 5015 goto err_free_shadow; 5016 } 5017 5018 /* insert all vm's */ 5019 for (area = 0; area < nr_vms; area++) { 5020 struct vmap_node *vn = addr_to_node(vas[area]->va_start); 5021 5022 spin_lock(&vn->busy.lock); 5023 insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head); 5024 setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC, 5025 pcpu_get_vm_areas); 5026 spin_unlock(&vn->busy.lock); 5027 } 5028 5029 /* 5030 * Mark allocated areas as accessible. Do it now as a best-effort 5031 * approach, as they can be mapped outside of vmalloc code. 5032 * With hardware tag-based KASAN, marking is skipped for 5033 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc(). 5034 */ 5035 kasan_unpoison_vmap_areas(vms, nr_vms, KASAN_VMALLOC_PROT_NORMAL); 5036 5037 kfree(vas); 5038 return vms; 5039 5040recovery: 5041 /* 5042 * Remove previously allocated areas. There is no 5043 * need in removing these areas from the busy tree, 5044 * because they are inserted only on the final step 5045 * and when pcpu_get_vm_areas() is success. 5046 */ 5047 while (area--) { 5048 orig_start = vas[area]->va_start; 5049 orig_end = vas[area]->va_end; 5050 va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 5051 &free_vmap_area_list); 5052 if (va) 5053 kasan_release_vmalloc(orig_start, orig_end, 5054 va->va_start, va->va_end, 5055 KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH); 5056 vas[area] = NULL; 5057 } 5058 5059overflow: 5060 spin_unlock(&free_vmap_area_lock); 5061 if (!purged) { 5062 reclaim_and_purge_vmap_areas(); 5063 purged = true; 5064 5065 /* Before "retry", check if we recover. */ 5066 for (area = 0; area < nr_vms; area++) { 5067 if (vas[area]) 5068 continue; 5069 5070 vas[area] = kmem_cache_zalloc( 5071 vmap_area_cachep, GFP_KERNEL); 5072 if (!vas[area]) 5073 goto err_free; 5074 } 5075 5076 goto retry; 5077 } 5078 5079err_free: 5080 for (area = 0; area < nr_vms; area++) { 5081 if (vas[area]) 5082 kmem_cache_free(vmap_area_cachep, vas[area]); 5083 5084 kfree(vms[area]); 5085 } 5086err_free2: 5087 kfree(vas); 5088 kfree(vms); 5089 return NULL; 5090 5091err_free_shadow: 5092 spin_lock(&free_vmap_area_lock); 5093 /* 5094 * We release all the vmalloc shadows, even the ones for regions that 5095 * hadn't been successfully added. This relies on kasan_release_vmalloc 5096 * being able to tolerate this case. 5097 */ 5098 for (area = 0; area < nr_vms; area++) { 5099 orig_start = vas[area]->va_start; 5100 orig_end = vas[area]->va_end; 5101 va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root, 5102 &free_vmap_area_list); 5103 if (va) 5104 kasan_release_vmalloc(orig_start, orig_end, 5105 va->va_start, va->va_end, 5106 KASAN_VMALLOC_PAGE_RANGE | KASAN_VMALLOC_TLB_FLUSH); 5107 vas[area] = NULL; 5108 kfree(vms[area]); 5109 } 5110 spin_unlock(&free_vmap_area_lock); 5111 kfree(vas); 5112 kfree(vms); 5113 return NULL; 5114} 5115 5116/** 5117 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator 5118 * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() 5119 * @nr_vms: the number of allocated areas 5120 * 5121 * Free vm_structs and the array allocated by pcpu_get_vm_areas(). 5122 */ 5123void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) 5124{ 5125 int i; 5126 5127 for (i = 0; i < nr_vms; i++) 5128 free_vm_area(vms[i]); 5129 kfree(vms); 5130} 5131#endif /* CONFIG_SMP */ 5132 5133#ifdef CONFIG_PRINTK 5134bool vmalloc_dump_obj(void *object) 5135{ 5136 const void *caller; 5137 struct vm_struct *vm; 5138 struct vmap_area *va; 5139 struct vmap_node *vn; 5140 unsigned long addr; 5141 unsigned int nr_pages; 5142 5143 addr = PAGE_ALIGN((unsigned long) object); 5144 vn = addr_to_node(addr); 5145 5146 if (!spin_trylock(&vn->busy.lock)) 5147 return false; 5148 5149 va = __find_vmap_area(addr, &vn->busy.root); 5150 if (!va || !va->vm) { 5151 spin_unlock(&vn->busy.lock); 5152 return false; 5153 } 5154 5155 vm = va->vm; 5156 addr = (unsigned long) vm->addr; 5157 caller = vm->caller; 5158 nr_pages = vm->nr_pages; 5159 spin_unlock(&vn->busy.lock); 5160 5161 pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n", 5162 nr_pages, addr, caller); 5163 5164 return true; 5165} 5166#endif 5167 5168#ifdef CONFIG_PROC_FS 5169 5170/* 5171 * Print number of pages allocated on each memory node. 5172 * 5173 * This function can only be called if CONFIG_NUMA is enabled 5174 * and VM_UNINITIALIZED bit in v->flags is disabled. 5175 */ 5176static void show_numa_info(struct seq_file *m, struct vm_struct *v, 5177 unsigned int *counters) 5178{ 5179 unsigned int nr; 5180 unsigned int step = 1U << vm_area_page_order(v); 5181 5182 if (!counters) 5183 return; 5184 5185 memset(counters, 0, nr_node_ids * sizeof(unsigned int)); 5186 5187 for (nr = 0; nr < v->nr_pages; nr += step) 5188 counters[page_to_nid(v->pages[nr])] += step; 5189 for_each_node_state(nr, N_HIGH_MEMORY) 5190 if (counters[nr]) 5191 seq_printf(m, " N%u=%u", nr, counters[nr]); 5192} 5193 5194static void show_purge_info(struct seq_file *m) 5195{ 5196 struct vmap_node *vn; 5197 struct vmap_area *va; 5198 5199 for_each_vmap_node(vn) { 5200 spin_lock(&vn->lazy.lock); 5201 list_for_each_entry(va, &vn->lazy.head, list) { 5202 seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n", 5203 (void *)va->va_start, (void *)va->va_end, 5204 va_size(va)); 5205 } 5206 spin_unlock(&vn->lazy.lock); 5207 } 5208} 5209 5210static int vmalloc_info_show(struct seq_file *m, void *p) 5211{ 5212 struct vmap_node *vn; 5213 struct vmap_area *va; 5214 struct vm_struct *v; 5215 unsigned int *counters; 5216 5217 if (IS_ENABLED(CONFIG_NUMA)) 5218 counters = kmalloc_array(nr_node_ids, sizeof(unsigned int), GFP_KERNEL); 5219 5220 for_each_vmap_node(vn) { 5221 spin_lock(&vn->busy.lock); 5222 list_for_each_entry(va, &vn->busy.head, list) { 5223 if (!va->vm) { 5224 if (va->flags & VMAP_RAM) 5225 seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n", 5226 (void *)va->va_start, (void *)va->va_end, 5227 va_size(va)); 5228 5229 continue; 5230 } 5231 5232 v = va->vm; 5233 if (v->flags & VM_UNINITIALIZED) 5234 continue; 5235 5236 /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ 5237 smp_rmb(); 5238 5239 seq_printf(m, "0x%pK-0x%pK %7ld", 5240 v->addr, v->addr + v->size, v->size); 5241 5242 if (v->caller) 5243 seq_printf(m, " %pS", v->caller); 5244 5245 if (v->nr_pages) 5246 seq_printf(m, " pages=%d", v->nr_pages); 5247 5248 if (v->phys_addr) 5249 seq_printf(m, " phys=%pa", &v->phys_addr); 5250 5251 if (v->flags & VM_IOREMAP) 5252 seq_puts(m, " ioremap"); 5253 5254 if (v->flags & VM_SPARSE) 5255 seq_puts(m, " sparse"); 5256 5257 if (v->flags & VM_ALLOC) 5258 seq_puts(m, " vmalloc"); 5259 5260 if (v->flags & VM_MAP) 5261 seq_puts(m, " vmap"); 5262 5263 if (v->flags & VM_USERMAP) 5264 seq_puts(m, " user"); 5265 5266 if (v->flags & VM_DMA_COHERENT) 5267 seq_puts(m, " dma-coherent"); 5268 5269 if (is_vmalloc_addr(v->pages)) 5270 seq_puts(m, " vpages"); 5271 5272 if (IS_ENABLED(CONFIG_NUMA)) 5273 show_numa_info(m, v, counters); 5274 5275 seq_putc(m, '\n'); 5276 } 5277 spin_unlock(&vn->busy.lock); 5278 } 5279 5280 /* 5281 * As a final step, dump "unpurged" areas. 5282 */ 5283 show_purge_info(m); 5284 if (IS_ENABLED(CONFIG_NUMA)) 5285 kfree(counters); 5286 return 0; 5287} 5288 5289static int __init proc_vmalloc_init(void) 5290{ 5291 proc_create_single("vmallocinfo", 0400, NULL, vmalloc_info_show); 5292 return 0; 5293} 5294module_init(proc_vmalloc_init); 5295 5296#endif 5297 5298static void __init vmap_init_free_space(void) 5299{ 5300 unsigned long vmap_start = 1; 5301 const unsigned long vmap_end = ULONG_MAX; 5302 struct vmap_area *free; 5303 struct vm_struct *busy; 5304 5305 /* 5306 * B F B B B F 5307 * -|-----|.....|-----|-----|-----|.....|- 5308 * | The KVA space | 5309 * |<--------------------------------->| 5310 */ 5311 for (busy = vmlist; busy; busy = busy->next) { 5312 if ((unsigned long) busy->addr - vmap_start > 0) { 5313 free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 5314 if (!WARN_ON_ONCE(!free)) { 5315 free->va_start = vmap_start; 5316 free->va_end = (unsigned long) busy->addr; 5317 5318 insert_vmap_area_augment(free, NULL, 5319 &free_vmap_area_root, 5320 &free_vmap_area_list); 5321 } 5322 } 5323 5324 vmap_start = (unsigned long) busy->addr + busy->size; 5325 } 5326 5327 if (vmap_end - vmap_start > 0) { 5328 free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 5329 if (!WARN_ON_ONCE(!free)) { 5330 free->va_start = vmap_start; 5331 free->va_end = vmap_end; 5332 5333 insert_vmap_area_augment(free, NULL, 5334 &free_vmap_area_root, 5335 &free_vmap_area_list); 5336 } 5337 } 5338} 5339 5340static void vmap_init_nodes(void) 5341{ 5342 struct vmap_node *vn; 5343 int i; 5344 5345#if BITS_PER_LONG == 64 5346 /* 5347 * A high threshold of max nodes is fixed and bound to 128, 5348 * thus a scale factor is 1 for systems where number of cores 5349 * are less or equal to specified threshold. 5350 * 5351 * As for NUMA-aware notes. For bigger systems, for example 5352 * NUMA with multi-sockets, where we can end-up with thousands 5353 * of cores in total, a "sub-numa-clustering" should be added. 5354 * 5355 * In this case a NUMA domain is considered as a single entity 5356 * with dedicated sub-nodes in it which describe one group or 5357 * set of cores. Therefore a per-domain purging is supposed to 5358 * be added as well as a per-domain balancing. 5359 */ 5360 int n = clamp_t(unsigned int, num_possible_cpus(), 1, 128); 5361 5362 if (n > 1) { 5363 vn = kmalloc_array(n, sizeof(*vn), GFP_NOWAIT); 5364 if (vn) { 5365 /* Node partition is 16 pages. */ 5366 vmap_zone_size = (1 << 4) * PAGE_SIZE; 5367 nr_vmap_nodes = n; 5368 vmap_nodes = vn; 5369 } else { 5370 pr_err("Failed to allocate an array. Disable a node layer\n"); 5371 } 5372 } 5373#endif 5374 5375 for_each_vmap_node(vn) { 5376 vn->busy.root = RB_ROOT; 5377 INIT_LIST_HEAD(&vn->busy.head); 5378 spin_lock_init(&vn->busy.lock); 5379 5380 vn->lazy.root = RB_ROOT; 5381 INIT_LIST_HEAD(&vn->lazy.head); 5382 spin_lock_init(&vn->lazy.lock); 5383 5384 for (i = 0; i < MAX_VA_SIZE_PAGES; i++) { 5385 INIT_LIST_HEAD(&vn->pool[i].head); 5386 WRITE_ONCE(vn->pool[i].len, 0); 5387 } 5388 5389 spin_lock_init(&vn->pool_lock); 5390 } 5391} 5392 5393static unsigned long 5394vmap_node_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 5395{ 5396 unsigned long count = 0; 5397 struct vmap_node *vn; 5398 int i; 5399 5400 for_each_vmap_node(vn) { 5401 for (i = 0; i < MAX_VA_SIZE_PAGES; i++) 5402 count += READ_ONCE(vn->pool[i].len); 5403 } 5404 5405 return count ? count : SHRINK_EMPTY; 5406} 5407 5408static unsigned long 5409vmap_node_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 5410{ 5411 struct vmap_node *vn; 5412 5413 for_each_vmap_node(vn) 5414 decay_va_pool_node(vn, true); 5415 5416 return SHRINK_STOP; 5417} 5418 5419void __init vmalloc_init(void) 5420{ 5421 struct shrinker *vmap_node_shrinker; 5422 struct vmap_area *va; 5423 struct vmap_node *vn; 5424 struct vm_struct *tmp; 5425 int i; 5426 5427 /* 5428 * Create the cache for vmap_area objects. 5429 */ 5430 vmap_area_cachep = KMEM_CACHE(vmap_area, SLAB_PANIC); 5431 5432 for_each_possible_cpu(i) { 5433 struct vmap_block_queue *vbq; 5434 struct vfree_deferred *p; 5435 5436 vbq = &per_cpu(vmap_block_queue, i); 5437 spin_lock_init(&vbq->lock); 5438 INIT_LIST_HEAD(&vbq->free); 5439 p = &per_cpu(vfree_deferred, i); 5440 init_llist_head(&p->list); 5441 INIT_WORK(&p->wq, delayed_vfree_work); 5442 xa_init(&vbq->vmap_blocks); 5443 } 5444 5445 /* 5446 * Setup nodes before importing vmlist. 5447 */ 5448 vmap_init_nodes(); 5449 5450 /* Import existing vmlist entries. */ 5451 for (tmp = vmlist; tmp; tmp = tmp->next) { 5452 va = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT); 5453 if (WARN_ON_ONCE(!va)) 5454 continue; 5455 5456 va->va_start = (unsigned long)tmp->addr; 5457 va->va_end = va->va_start + tmp->size; 5458 va->vm = tmp; 5459 5460 vn = addr_to_node(va->va_start); 5461 insert_vmap_area(va, &vn->busy.root, &vn->busy.head); 5462 } 5463 5464 /* 5465 * Now we can initialize a free vmap space. 5466 */ 5467 vmap_init_free_space(); 5468 vmap_initialized = true; 5469 5470 vmap_node_shrinker = shrinker_alloc(0, "vmap-node"); 5471 if (!vmap_node_shrinker) { 5472 pr_err("Failed to allocate vmap-node shrinker!\n"); 5473 return; 5474 } 5475 5476 vmap_node_shrinker->count_objects = vmap_node_shrink_count; 5477 vmap_node_shrinker->scan_objects = vmap_node_shrink_scan; 5478 shrinker_register(vmap_node_shrinker); 5479}
~/
Read-only file preview.