FILE
slub.c
File preview with syntax highlighting.
linux-6.19.14/mm/slub.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * SLUB: A slab allocator that limits cache line use instead of queuing 4 * objects in per cpu and per node lists. 5 * 6 * The allocator synchronizes using per slab locks or atomic operations 7 * and only uses a centralized lock to manage a pool of partial slabs. 8 * 9 * (C) 2007 SGI, Christoph Lameter 10 * (C) 2011 Linux Foundation, Christoph Lameter 11 */ 12 13#include <linux/mm.h> 14#include <linux/swap.h> /* mm_account_reclaimed_pages() */ 15#include <linux/module.h> 16#include <linux/bit_spinlock.h> 17#include <linux/interrupt.h> 18#include <linux/swab.h> 19#include <linux/bitops.h> 20#include <linux/slab.h> 21#include "slab.h" 22#include <linux/vmalloc.h> 23#include <linux/proc_fs.h> 24#include <linux/seq_file.h> 25#include <linux/kasan.h> 26#include <linux/node.h> 27#include <linux/kmsan.h> 28#include <linux/cpu.h> 29#include <linux/cpuset.h> 30#include <linux/mempolicy.h> 31#include <linux/ctype.h> 32#include <linux/stackdepot.h> 33#include <linux/debugobjects.h> 34#include <linux/kallsyms.h> 35#include <linux/kfence.h> 36#include <linux/memory.h> 37#include <linux/math64.h> 38#include <linux/fault-inject.h> 39#include <linux/kmemleak.h> 40#include <linux/stacktrace.h> 41#include <linux/prefetch.h> 42#include <linux/memcontrol.h> 43#include <linux/random.h> 44#include <linux/prandom.h> 45#include <kunit/test.h> 46#include <kunit/test-bug.h> 47#include <linux/sort.h> 48#include <linux/irq_work.h> 49#include <linux/kprobes.h> 50#include <linux/debugfs.h> 51#include <trace/events/kmem.h> 52 53#include "internal.h" 54 55/* 56 * Lock order: 57 * 1. slab_mutex (Global Mutex) 58 * 2. node->list_lock (Spinlock) 59 * 3. kmem_cache->cpu_slab->lock (Local lock) 60 * 4. slab_lock(slab) (Only on some arches) 61 * 5. object_map_lock (Only for debugging) 62 * 63 * slab_mutex 64 * 65 * The role of the slab_mutex is to protect the list of all the slabs 66 * and to synchronize major metadata changes to slab cache structures. 67 * Also synchronizes memory hotplug callbacks. 68 * 69 * slab_lock 70 * 71 * The slab_lock is a wrapper around the page lock, thus it is a bit 72 * spinlock. 73 * 74 * The slab_lock is only used on arches that do not have the ability 75 * to do a cmpxchg_double. It only protects: 76 * 77 * A. slab->freelist -> List of free objects in a slab 78 * B. slab->inuse -> Number of objects in use 79 * C. slab->objects -> Number of objects in slab 80 * D. slab->frozen -> frozen state 81 * 82 * Frozen slabs 83 * 84 * If a slab is frozen then it is exempt from list management. It is 85 * the cpu slab which is actively allocated from by the processor that 86 * froze it and it is not on any list. The processor that froze the 87 * slab is the one who can perform list operations on the slab. Other 88 * processors may put objects onto the freelist but the processor that 89 * froze the slab is the only one that can retrieve the objects from the 90 * slab's freelist. 91 * 92 * CPU partial slabs 93 * 94 * The partially empty slabs cached on the CPU partial list are used 95 * for performance reasons, which speeds up the allocation process. 96 * These slabs are not frozen, but are also exempt from list management, 97 * by clearing the SL_partial flag when moving out of the node 98 * partial list. Please see __slab_free() for more details. 99 * 100 * To sum up, the current scheme is: 101 * - node partial slab: SL_partial && !frozen 102 * - cpu partial slab: !SL_partial && !frozen 103 * - cpu slab: !SL_partial && frozen 104 * - full slab: !SL_partial && !frozen 105 * 106 * list_lock 107 * 108 * The list_lock protects the partial and full list on each node and 109 * the partial slab counter. If taken then no new slabs may be added or 110 * removed from the lists nor make the number of partial slabs be modified. 111 * (Note that the total number of slabs is an atomic value that may be 112 * modified without taking the list lock). 113 * 114 * The list_lock is a centralized lock and thus we avoid taking it as 115 * much as possible. As long as SLUB does not have to handle partial 116 * slabs, operations can continue without any centralized lock. F.e. 117 * allocating a long series of objects that fill up slabs does not require 118 * the list lock. 119 * 120 * For debug caches, all allocations are forced to go through a list_lock 121 * protected region to serialize against concurrent validation. 122 * 123 * cpu_slab->lock local lock 124 * 125 * This locks protect slowpath manipulation of all kmem_cache_cpu fields 126 * except the stat counters. This is a percpu structure manipulated only by 127 * the local cpu, so the lock protects against being preempted or interrupted 128 * by an irq. Fast path operations rely on lockless operations instead. 129 * 130 * On PREEMPT_RT, the local lock neither disables interrupts nor preemption 131 * which means the lockless fastpath cannot be used as it might interfere with 132 * an in-progress slow path operations. In this case the local lock is always 133 * taken but it still utilizes the freelist for the common operations. 134 * 135 * lockless fastpaths 136 * 137 * The fast path allocation (slab_alloc_node()) and freeing (do_slab_free()) 138 * are fully lockless when satisfied from the percpu slab (and when 139 * cmpxchg_double is possible to use, otherwise slab_lock is taken). 140 * They also don't disable preemption or migration or irqs. They rely on 141 * the transaction id (tid) field to detect being preempted or moved to 142 * another cpu. 143 * 144 * irq, preemption, migration considerations 145 * 146 * Interrupts are disabled as part of list_lock or local_lock operations, or 147 * around the slab_lock operation, in order to make the slab allocator safe 148 * to use in the context of an irq. 149 * 150 * In addition, preemption (or migration on PREEMPT_RT) is disabled in the 151 * allocation slowpath, bulk allocation, and put_cpu_partial(), so that the 152 * local cpu doesn't change in the process and e.g. the kmem_cache_cpu pointer 153 * doesn't have to be revalidated in each section protected by the local lock. 154 * 155 * SLUB assigns one slab for allocation to each processor. 156 * Allocations only occur from these slabs called cpu slabs. 157 * 158 * Slabs with free elements are kept on a partial list and during regular 159 * operations no list for full slabs is used. If an object in a full slab is 160 * freed then the slab will show up again on the partial lists. 161 * We track full slabs for debugging purposes though because otherwise we 162 * cannot scan all objects. 163 * 164 * Slabs are freed when they become empty. Teardown and setup is 165 * minimal so we rely on the page allocators per cpu caches for 166 * fast frees and allocs. 167 * 168 * slab->frozen The slab is frozen and exempt from list processing. 169 * This means that the slab is dedicated to a purpose 170 * such as satisfying allocations for a specific 171 * processor. Objects may be freed in the slab while 172 * it is frozen but slab_free will then skip the usual 173 * list operations. It is up to the processor holding 174 * the slab to integrate the slab into the slab lists 175 * when the slab is no longer needed. 176 * 177 * One use of this flag is to mark slabs that are 178 * used for allocations. Then such a slab becomes a cpu 179 * slab. The cpu slab may be equipped with an additional 180 * freelist that allows lockless access to 181 * free objects in addition to the regular freelist 182 * that requires the slab lock. 183 * 184 * SLAB_DEBUG_FLAGS Slab requires special handling due to debug 185 * options set. This moves slab handling out of 186 * the fast path and disables lockless freelists. 187 */ 188 189/** 190 * enum slab_flags - How the slab flags bits are used. 191 * @SL_locked: Is locked with slab_lock() 192 * @SL_partial: On the per-node partial list 193 * @SL_pfmemalloc: Was allocated from PF_MEMALLOC reserves 194 * 195 * The slab flags share space with the page flags but some bits have 196 * different interpretations. The high bits are used for information 197 * like zone/node/section. 198 */ 199enum slab_flags { 200 SL_locked = PG_locked, 201 SL_partial = PG_workingset, /* Historical reasons for this bit */ 202 SL_pfmemalloc = PG_active, /* Historical reasons for this bit */ 203}; 204 205/* 206 * We could simply use migrate_disable()/enable() but as long as it's a 207 * function call even on !PREEMPT_RT, use inline preempt_disable() there. 208 */ 209#ifndef CONFIG_PREEMPT_RT 210#define slub_get_cpu_ptr(var) get_cpu_ptr(var) 211#define slub_put_cpu_ptr(var) put_cpu_ptr(var) 212#define USE_LOCKLESS_FAST_PATH() (true) 213#else 214#define slub_get_cpu_ptr(var) \ 215({ \ 216 migrate_disable(); \ 217 this_cpu_ptr(var); \ 218}) 219#define slub_put_cpu_ptr(var) \ 220do { \ 221 (void)(var); \ 222 migrate_enable(); \ 223} while (0) 224#define USE_LOCKLESS_FAST_PATH() (false) 225#endif 226 227#ifndef CONFIG_SLUB_TINY 228#define __fastpath_inline __always_inline 229#else 230#define __fastpath_inline 231#endif 232 233#ifdef CONFIG_SLUB_DEBUG 234#ifdef CONFIG_SLUB_DEBUG_ON 235DEFINE_STATIC_KEY_TRUE(slub_debug_enabled); 236#else 237DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); 238#endif 239#endif /* CONFIG_SLUB_DEBUG */ 240 241#ifdef CONFIG_NUMA 242static DEFINE_STATIC_KEY_FALSE(strict_numa); 243#endif 244 245/* Structure holding parameters for get_partial() call chain */ 246struct partial_context { 247 gfp_t flags; 248 unsigned int orig_size; 249 void *object; 250}; 251 252static inline bool kmem_cache_debug(struct kmem_cache *s) 253{ 254 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS); 255} 256 257void *fixup_red_left(struct kmem_cache *s, void *p) 258{ 259 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE)) 260 p += s->red_left_pad; 261 262 return p; 263} 264 265static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) 266{ 267#ifdef CONFIG_SLUB_CPU_PARTIAL 268 return !kmem_cache_debug(s); 269#else 270 return false; 271#endif 272} 273 274/* 275 * Issues still to be resolved: 276 * 277 * - Support PAGE_ALLOC_DEBUG. Should be easy to do. 278 * 279 * - Variable sizing of the per node arrays 280 */ 281 282/* Enable to log cmpxchg failures */ 283#undef SLUB_DEBUG_CMPXCHG 284 285#ifndef CONFIG_SLUB_TINY 286/* 287 * Minimum number of partial slabs. These will be left on the partial 288 * lists even if they are empty. kmem_cache_shrink may reclaim them. 289 */ 290#define MIN_PARTIAL 5 291 292/* 293 * Maximum number of desirable partial slabs. 294 * The existence of more partial slabs makes kmem_cache_shrink 295 * sort the partial list by the number of objects in use. 296 */ 297#define MAX_PARTIAL 10 298#else 299#define MIN_PARTIAL 0 300#define MAX_PARTIAL 0 301#endif 302 303#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \ 304 SLAB_POISON | SLAB_STORE_USER) 305 306/* 307 * These debug flags cannot use CMPXCHG because there might be consistency 308 * issues when checking or reading debug information 309 */ 310#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \ 311 SLAB_TRACE) 312 313 314/* 315 * Debugging flags that require metadata to be stored in the slab. These get 316 * disabled when slab_debug=O is used and a cache's min order increases with 317 * metadata. 318 */ 319#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) 320 321#define OO_SHIFT 16 322#define OO_MASK ((1 << OO_SHIFT) - 1) 323#define MAX_OBJS_PER_PAGE 32767 /* since slab.objects is u15 */ 324 325/* Internal SLUB flags */ 326/* Poison object */ 327#define __OBJECT_POISON __SLAB_FLAG_BIT(_SLAB_OBJECT_POISON) 328/* Use cmpxchg_double */ 329 330#ifdef system_has_freelist_aba 331#define __CMPXCHG_DOUBLE __SLAB_FLAG_BIT(_SLAB_CMPXCHG_DOUBLE) 332#else 333#define __CMPXCHG_DOUBLE __SLAB_FLAG_UNUSED 334#endif 335 336/* 337 * Tracking user of a slab. 338 */ 339#define TRACK_ADDRS_COUNT 16 340struct track { 341 unsigned long addr; /* Called from address */ 342#ifdef CONFIG_STACKDEPOT 343 depot_stack_handle_t handle; 344#endif 345 int cpu; /* Was running on cpu */ 346 int pid; /* Pid context */ 347 unsigned long when; /* When did the operation occur */ 348}; 349 350enum track_item { TRACK_ALLOC, TRACK_FREE }; 351 352#ifdef SLAB_SUPPORTS_SYSFS 353static int sysfs_slab_add(struct kmem_cache *); 354static int sysfs_slab_alias(struct kmem_cache *, const char *); 355#else 356static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } 357static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) 358 { return 0; } 359#endif 360 361#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) 362static void debugfs_slab_add(struct kmem_cache *); 363#else 364static inline void debugfs_slab_add(struct kmem_cache *s) { } 365#endif 366 367enum stat_item { 368 ALLOC_PCS, /* Allocation from percpu sheaf */ 369 ALLOC_FASTPATH, /* Allocation from cpu slab */ 370 ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */ 371 FREE_PCS, /* Free to percpu sheaf */ 372 FREE_RCU_SHEAF, /* Free to rcu_free sheaf */ 373 FREE_RCU_SHEAF_FAIL, /* Failed to free to a rcu_free sheaf */ 374 FREE_FASTPATH, /* Free to cpu slab */ 375 FREE_SLOWPATH, /* Freeing not to cpu slab */ 376 FREE_FROZEN, /* Freeing to frozen slab */ 377 FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */ 378 FREE_REMOVE_PARTIAL, /* Freeing removes last object */ 379 ALLOC_FROM_PARTIAL, /* Cpu slab acquired from node partial list */ 380 ALLOC_SLAB, /* Cpu slab acquired from page allocator */ 381 ALLOC_REFILL, /* Refill cpu slab from slab freelist */ 382 ALLOC_NODE_MISMATCH, /* Switching cpu slab */ 383 FREE_SLAB, /* Slab freed to the page allocator */ 384 CPUSLAB_FLUSH, /* Abandoning of the cpu slab */ 385 DEACTIVATE_FULL, /* Cpu slab was full when deactivated */ 386 DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */ 387 DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */ 388 DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */ 389 DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */ 390 DEACTIVATE_BYPASS, /* Implicit deactivation */ 391 ORDER_FALLBACK, /* Number of times fallback was necessary */ 392 CMPXCHG_DOUBLE_CPU_FAIL,/* Failures of this_cpu_cmpxchg_double */ 393 CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */ 394 CPU_PARTIAL_ALLOC, /* Used cpu partial on alloc */ 395 CPU_PARTIAL_FREE, /* Refill cpu partial on free */ 396 CPU_PARTIAL_NODE, /* Refill cpu partial from node partial */ 397 CPU_PARTIAL_DRAIN, /* Drain cpu partial to node partial */ 398 SHEAF_FLUSH, /* Objects flushed from a sheaf */ 399 SHEAF_REFILL, /* Objects refilled to a sheaf */ 400 SHEAF_ALLOC, /* Allocation of an empty sheaf */ 401 SHEAF_FREE, /* Freeing of an empty sheaf */ 402 BARN_GET, /* Got full sheaf from barn */ 403 BARN_GET_FAIL, /* Failed to get full sheaf from barn */ 404 BARN_PUT, /* Put full sheaf to barn */ 405 BARN_PUT_FAIL, /* Failed to put full sheaf to barn */ 406 SHEAF_PREFILL_FAST, /* Sheaf prefill grabbed the spare sheaf */ 407 SHEAF_PREFILL_SLOW, /* Sheaf prefill found no spare sheaf */ 408 SHEAF_PREFILL_OVERSIZE, /* Allocation of oversize sheaf for prefill */ 409 SHEAF_RETURN_FAST, /* Sheaf return reattached spare sheaf */ 410 SHEAF_RETURN_SLOW, /* Sheaf return could not reattach spare */ 411 NR_SLUB_STAT_ITEMS 412}; 413 414struct freelist_tid { 415 union { 416 struct { 417 void *freelist; /* Pointer to next available object */ 418 unsigned long tid; /* Globally unique transaction id */ 419 }; 420 freelist_full_t freelist_tid; 421 }; 422}; 423 424/* 425 * When changing the layout, make sure freelist and tid are still compatible 426 * with this_cpu_cmpxchg_double() alignment requirements. 427 */ 428struct kmem_cache_cpu { 429 struct freelist_tid; 430 struct slab *slab; /* The slab from which we are allocating */ 431#ifdef CONFIG_SLUB_CPU_PARTIAL 432 struct slab *partial; /* Partially allocated slabs */ 433#endif 434 local_trylock_t lock; /* Protects the fields above */ 435#ifdef CONFIG_SLUB_STATS 436 unsigned int stat[NR_SLUB_STAT_ITEMS]; 437#endif 438}; 439 440static inline void stat(const struct kmem_cache *s, enum stat_item si) 441{ 442#ifdef CONFIG_SLUB_STATS 443 /* 444 * The rmw is racy on a preemptible kernel but this is acceptable, so 445 * avoid this_cpu_add()'s irq-disable overhead. 446 */ 447 raw_cpu_inc(s->cpu_slab->stat[si]); 448#endif 449} 450 451static inline 452void stat_add(const struct kmem_cache *s, enum stat_item si, int v) 453{ 454#ifdef CONFIG_SLUB_STATS 455 raw_cpu_add(s->cpu_slab->stat[si], v); 456#endif 457} 458 459#define MAX_FULL_SHEAVES 10 460#define MAX_EMPTY_SHEAVES 10 461 462struct node_barn { 463 spinlock_t lock; 464 struct list_head sheaves_full; 465 struct list_head sheaves_empty; 466 unsigned int nr_full; 467 unsigned int nr_empty; 468}; 469 470struct slab_sheaf { 471 union { 472 struct rcu_head rcu_head; 473 struct list_head barn_list; 474 /* only used for prefilled sheafs */ 475 struct { 476 unsigned int capacity; 477 bool pfmemalloc; 478 }; 479 }; 480 struct kmem_cache *cache; 481 unsigned int size; 482 int node; /* only used for rcu_sheaf */ 483 void *objects[]; 484}; 485 486struct slub_percpu_sheaves { 487 local_trylock_t lock; 488 struct slab_sheaf *main; /* never NULL when unlocked */ 489 struct slab_sheaf *spare; /* empty or full, may be NULL */ 490 struct slab_sheaf *rcu_free; /* for batching kfree_rcu() */ 491}; 492 493/* 494 * The slab lists for all objects. 495 */ 496struct kmem_cache_node { 497 spinlock_t list_lock; 498 unsigned long nr_partial; 499 struct list_head partial; 500#ifdef CONFIG_SLUB_DEBUG 501 atomic_long_t nr_slabs; 502 atomic_long_t total_objects; 503 struct list_head full; 504#endif 505 struct node_barn *barn; 506}; 507 508static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) 509{ 510 return s->node[node]; 511} 512 513/* 514 * Get the barn of the current cpu's closest memory node. It may not exist on 515 * systems with memoryless nodes but without CONFIG_HAVE_MEMORYLESS_NODES 516 */ 517static inline struct node_barn *get_barn(struct kmem_cache *s) 518{ 519 struct kmem_cache_node *n = get_node(s, numa_mem_id()); 520 521 if (!n) 522 return NULL; 523 524 return n->barn; 525} 526 527/* 528 * Iterator over all nodes. The body will be executed for each node that has 529 * a kmem_cache_node structure allocated (which is true for all online nodes) 530 */ 531#define for_each_kmem_cache_node(__s, __node, __n) \ 532 for (__node = 0; __node < nr_node_ids; __node++) \ 533 if ((__n = get_node(__s, __node))) 534 535/* 536 * Tracks for which NUMA nodes we have kmem_cache_nodes allocated. 537 * Corresponds to node_state[N_MEMORY], but can temporarily 538 * differ during memory hotplug/hotremove operations. 539 * Protected by slab_mutex. 540 */ 541static nodemask_t slab_nodes; 542 543/* 544 * Workqueue used for flush_cpu_slab(). 545 */ 546static struct workqueue_struct *flushwq; 547 548struct slub_flush_work { 549 struct work_struct work; 550 struct kmem_cache *s; 551 bool skip; 552}; 553 554static DEFINE_MUTEX(flush_lock); 555static DEFINE_PER_CPU(struct slub_flush_work, slub_flush); 556 557/******************************************************************** 558 * Core slab cache functions 559 *******************************************************************/ 560 561/* 562 * Returns freelist pointer (ptr). With hardening, this is obfuscated 563 * with an XOR of the address where the pointer is held and a per-cache 564 * random number. 565 */ 566static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s, 567 void *ptr, unsigned long ptr_addr) 568{ 569 unsigned long encoded; 570 571#ifdef CONFIG_SLAB_FREELIST_HARDENED 572 encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr); 573#else 574 encoded = (unsigned long)ptr; 575#endif 576 return (freeptr_t){.v = encoded}; 577} 578 579static inline void *freelist_ptr_decode(const struct kmem_cache *s, 580 freeptr_t ptr, unsigned long ptr_addr) 581{ 582 void *decoded; 583 584#ifdef CONFIG_SLAB_FREELIST_HARDENED 585 decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr)); 586#else 587 decoded = (void *)ptr.v; 588#endif 589 return decoded; 590} 591 592static inline void *get_freepointer(struct kmem_cache *s, void *object) 593{ 594 unsigned long ptr_addr; 595 freeptr_t p; 596 597 object = kasan_reset_tag(object); 598 ptr_addr = (unsigned long)object + s->offset; 599 p = *(freeptr_t *)(ptr_addr); 600 return freelist_ptr_decode(s, p, ptr_addr); 601} 602 603static void prefetch_freepointer(const struct kmem_cache *s, void *object) 604{ 605 prefetchw(object + s->offset); 606} 607 608/* 609 * When running under KMSAN, get_freepointer_safe() may return an uninitialized 610 * pointer value in the case the current thread loses the race for the next 611 * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in 612 * slab_alloc_node() will fail, so the uninitialized value won't be used, but 613 * KMSAN will still check all arguments of cmpxchg because of imperfect 614 * handling of inline assembly. 615 * To work around this problem, we apply __no_kmsan_checks to ensure that 616 * get_freepointer_safe() returns initialized memory. 617 */ 618__no_kmsan_checks 619static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) 620{ 621 unsigned long freepointer_addr; 622 freeptr_t p; 623 624 if (!debug_pagealloc_enabled_static()) 625 return get_freepointer(s, object); 626 627 object = kasan_reset_tag(object); 628 freepointer_addr = (unsigned long)object + s->offset; 629 copy_from_kernel_nofault(&p, (freeptr_t *)freepointer_addr, sizeof(p)); 630 return freelist_ptr_decode(s, p, freepointer_addr); 631} 632 633static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) 634{ 635 unsigned long freeptr_addr = (unsigned long)object + s->offset; 636 637#ifdef CONFIG_SLAB_FREELIST_HARDENED 638 BUG_ON(object == fp); /* naive detection of double free or corruption */ 639#endif 640 641 freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); 642 *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); 643} 644 645/* 646 * See comment in calculate_sizes(). 647 */ 648static inline bool freeptr_outside_object(struct kmem_cache *s) 649{ 650 return s->offset >= s->inuse; 651} 652 653/* 654 * Return offset of the end of info block which is inuse + free pointer if 655 * not overlapping with object. 656 */ 657static inline unsigned int get_info_end(struct kmem_cache *s) 658{ 659 if (freeptr_outside_object(s)) 660 return s->inuse + sizeof(void *); 661 else 662 return s->inuse; 663} 664 665/* Loop over all objects in a slab */ 666#define for_each_object(__p, __s, __addr, __objects) \ 667 for (__p = fixup_red_left(__s, __addr); \ 668 __p < (__addr) + (__objects) * (__s)->size; \ 669 __p += (__s)->size) 670 671static inline unsigned int order_objects(unsigned int order, unsigned int size) 672{ 673 return ((unsigned int)PAGE_SIZE << order) / size; 674} 675 676static inline struct kmem_cache_order_objects oo_make(unsigned int order, 677 unsigned int size) 678{ 679 struct kmem_cache_order_objects x = { 680 (order << OO_SHIFT) + order_objects(order, size) 681 }; 682 683 return x; 684} 685 686static inline unsigned int oo_order(struct kmem_cache_order_objects x) 687{ 688 return x.x >> OO_SHIFT; 689} 690 691static inline unsigned int oo_objects(struct kmem_cache_order_objects x) 692{ 693 return x.x & OO_MASK; 694} 695 696#ifdef CONFIG_SLUB_CPU_PARTIAL 697static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 698{ 699 unsigned int nr_slabs; 700 701 s->cpu_partial = nr_objects; 702 703 /* 704 * We take the number of objects but actually limit the number of 705 * slabs on the per cpu partial list, in order to limit excessive 706 * growth of the list. For simplicity we assume that the slabs will 707 * be half-full. 708 */ 709 nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo)); 710 s->cpu_partial_slabs = nr_slabs; 711} 712 713static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 714{ 715 return s->cpu_partial_slabs; 716} 717#else 718#ifdef SLAB_SUPPORTS_SYSFS 719static inline void 720slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) 721{ 722} 723#endif 724 725static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s) 726{ 727 return 0; 728} 729#endif /* CONFIG_SLUB_CPU_PARTIAL */ 730 731/* 732 * If network-based swap is enabled, slub must keep track of whether memory 733 * were allocated from pfmemalloc reserves. 734 */ 735static inline bool slab_test_pfmemalloc(const struct slab *slab) 736{ 737 return test_bit(SL_pfmemalloc, &slab->flags.f); 738} 739 740static inline void slab_set_pfmemalloc(struct slab *slab) 741{ 742 set_bit(SL_pfmemalloc, &slab->flags.f); 743} 744 745static inline void __slab_clear_pfmemalloc(struct slab *slab) 746{ 747 __clear_bit(SL_pfmemalloc, &slab->flags.f); 748} 749 750/* 751 * Per slab locking using the pagelock 752 */ 753static __always_inline void slab_lock(struct slab *slab) 754{ 755 bit_spin_lock(SL_locked, &slab->flags.f); 756} 757 758static __always_inline void slab_unlock(struct slab *slab) 759{ 760 bit_spin_unlock(SL_locked, &slab->flags.f); 761} 762 763static inline bool 764__update_freelist_fast(struct slab *slab, struct freelist_counters *old, 765 struct freelist_counters *new) 766{ 767#ifdef system_has_freelist_aba 768 return try_cmpxchg_freelist(&slab->freelist_counters, 769 &old->freelist_counters, 770 new->freelist_counters); 771#else 772 return false; 773#endif 774} 775 776static inline bool 777__update_freelist_slow(struct slab *slab, struct freelist_counters *old, 778 struct freelist_counters *new) 779{ 780 bool ret = false; 781 782 slab_lock(slab); 783 if (slab->freelist == old->freelist && 784 slab->counters == old->counters) { 785 slab->freelist = new->freelist; 786 slab->counters = new->counters; 787 ret = true; 788 } 789 slab_unlock(slab); 790 791 return ret; 792} 793 794/* 795 * Interrupts must be disabled (for the fallback code to work right), typically 796 * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is 797 * part of bit_spin_lock(), is sufficient because the policy is not to allow any 798 * allocation/ free operation in hardirq context. Therefore nothing can 799 * interrupt the operation. 800 */ 801static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab, 802 struct freelist_counters *old, struct freelist_counters *new, const char *n) 803{ 804 bool ret; 805 806 if (USE_LOCKLESS_FAST_PATH()) 807 lockdep_assert_irqs_disabled(); 808 809 if (s->flags & __CMPXCHG_DOUBLE) 810 ret = __update_freelist_fast(slab, old, new); 811 else 812 ret = __update_freelist_slow(slab, old, new); 813 814 if (likely(ret)) 815 return true; 816 817 cpu_relax(); 818 stat(s, CMPXCHG_DOUBLE_FAIL); 819 820#ifdef SLUB_DEBUG_CMPXCHG 821 pr_info("%s %s: cmpxchg double redo ", n, s->name); 822#endif 823 824 return false; 825} 826 827static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab, 828 struct freelist_counters *old, struct freelist_counters *new, const char *n) 829{ 830 bool ret; 831 832 if (s->flags & __CMPXCHG_DOUBLE) { 833 ret = __update_freelist_fast(slab, old, new); 834 } else { 835 unsigned long flags; 836 837 local_irq_save(flags); 838 ret = __update_freelist_slow(slab, old, new); 839 local_irq_restore(flags); 840 } 841 if (likely(ret)) 842 return true; 843 844 cpu_relax(); 845 stat(s, CMPXCHG_DOUBLE_FAIL); 846 847#ifdef SLUB_DEBUG_CMPXCHG 848 pr_info("%s %s: cmpxchg double redo ", n, s->name); 849#endif 850 851 return false; 852} 853 854/* 855 * kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API 856 * family will round up the real request size to these fixed ones, so 857 * there could be an extra area than what is requested. Save the original 858 * request size in the meta data area, for better debug and sanity check. 859 */ 860static inline void set_orig_size(struct kmem_cache *s, 861 void *object, unsigned long orig_size) 862{ 863 void *p = kasan_reset_tag(object); 864 865 if (!slub_debug_orig_size(s)) 866 return; 867 868 p += get_info_end(s); 869 p += sizeof(struct track) * 2; 870 871 *(unsigned long *)p = orig_size; 872} 873 874static inline unsigned long get_orig_size(struct kmem_cache *s, void *object) 875{ 876 void *p = kasan_reset_tag(object); 877 878 if (is_kfence_address(object)) 879 return kfence_ksize(object); 880 881 if (!slub_debug_orig_size(s)) 882 return s->object_size; 883 884 p += get_info_end(s); 885 p += sizeof(struct track) * 2; 886 887 return *(unsigned long *)p; 888} 889 890#ifdef CONFIG_SLUB_DEBUG 891 892/* 893 * For debugging context when we want to check if the struct slab pointer 894 * appears to be valid. 895 */ 896static inline bool validate_slab_ptr(struct slab *slab) 897{ 898 return PageSlab(slab_page(slab)); 899} 900 901static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; 902static DEFINE_SPINLOCK(object_map_lock); 903 904static void __fill_map(unsigned long *obj_map, struct kmem_cache *s, 905 struct slab *slab) 906{ 907 void *addr = slab_address(slab); 908 void *p; 909 910 bitmap_zero(obj_map, slab->objects); 911 912 for (p = slab->freelist; p; p = get_freepointer(s, p)) 913 set_bit(__obj_to_index(s, addr, p), obj_map); 914} 915 916#if IS_ENABLED(CONFIG_KUNIT) 917static bool slab_add_kunit_errors(void) 918{ 919 struct kunit_resource *resource; 920 921 if (!kunit_get_current_test()) 922 return false; 923 924 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); 925 if (!resource) 926 return false; 927 928 (*(int *)resource->data)++; 929 kunit_put_resource(resource); 930 return true; 931} 932 933bool slab_in_kunit_test(void) 934{ 935 struct kunit_resource *resource; 936 937 if (!kunit_get_current_test()) 938 return false; 939 940 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); 941 if (!resource) 942 return false; 943 944 kunit_put_resource(resource); 945 return true; 946} 947#else 948static inline bool slab_add_kunit_errors(void) { return false; } 949#endif 950 951static inline unsigned int size_from_object(struct kmem_cache *s) 952{ 953 if (s->flags & SLAB_RED_ZONE) 954 return s->size - s->red_left_pad; 955 956 return s->size; 957} 958 959static inline void *restore_red_left(struct kmem_cache *s, void *p) 960{ 961 if (s->flags & SLAB_RED_ZONE) 962 p -= s->red_left_pad; 963 964 return p; 965} 966 967/* 968 * Debug settings: 969 */ 970#if defined(CONFIG_SLUB_DEBUG_ON) 971static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS; 972#else 973static slab_flags_t slub_debug; 974#endif 975 976static const char *slub_debug_string __ro_after_init; 977static int disable_higher_order_debug; 978 979/* 980 * slub is about to manipulate internal object metadata. This memory lies 981 * outside the range of the allocated object, so accessing it would normally 982 * be reported by kasan as a bounds error. metadata_access_enable() is used 983 * to tell kasan that these accesses are OK. 984 */ 985static inline void metadata_access_enable(void) 986{ 987 kasan_disable_current(); 988 kmsan_disable_current(); 989} 990 991static inline void metadata_access_disable(void) 992{ 993 kmsan_enable_current(); 994 kasan_enable_current(); 995} 996 997/* 998 * Object debugging 999 */ 1000 1001/* Verify that a pointer has an address that is valid within a slab page */ 1002static inline int check_valid_pointer(struct kmem_cache *s, 1003 struct slab *slab, void *object) 1004{ 1005 void *base; 1006 1007 if (!object) 1008 return 1; 1009 1010 base = slab_address(slab); 1011 object = kasan_reset_tag(object); 1012 object = restore_red_left(s, object); 1013 if (object < base || object >= base + slab->objects * s->size || 1014 (object - base) % s->size) { 1015 return 0; 1016 } 1017 1018 return 1; 1019} 1020 1021static void print_section(char *level, char *text, u8 *addr, 1022 unsigned int length) 1023{ 1024 metadata_access_enable(); 1025 print_hex_dump(level, text, DUMP_PREFIX_ADDRESS, 1026 16, 1, kasan_reset_tag((void *)addr), length, 1); 1027 metadata_access_disable(); 1028} 1029 1030static struct track *get_track(struct kmem_cache *s, void *object, 1031 enum track_item alloc) 1032{ 1033 struct track *p; 1034 1035 p = object + get_info_end(s); 1036 1037 return kasan_reset_tag(p + alloc); 1038} 1039 1040#ifdef CONFIG_STACKDEPOT 1041static noinline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) 1042{ 1043 depot_stack_handle_t handle; 1044 unsigned long entries[TRACK_ADDRS_COUNT]; 1045 unsigned int nr_entries; 1046 1047 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 3); 1048 handle = stack_depot_save(entries, nr_entries, gfp_flags); 1049 1050 return handle; 1051} 1052#else 1053static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) 1054{ 1055 return 0; 1056} 1057#endif 1058 1059static void set_track_update(struct kmem_cache *s, void *object, 1060 enum track_item alloc, unsigned long addr, 1061 depot_stack_handle_t handle) 1062{ 1063 struct track *p = get_track(s, object, alloc); 1064 1065#ifdef CONFIG_STACKDEPOT 1066 p->handle = handle; 1067#endif 1068 p->addr = addr; 1069 p->cpu = smp_processor_id(); 1070 p->pid = current->pid; 1071 p->when = jiffies; 1072} 1073 1074static __always_inline void set_track(struct kmem_cache *s, void *object, 1075 enum track_item alloc, unsigned long addr, gfp_t gfp_flags) 1076{ 1077 depot_stack_handle_t handle = set_track_prepare(gfp_flags); 1078 1079 set_track_update(s, object, alloc, addr, handle); 1080} 1081 1082static void init_tracking(struct kmem_cache *s, void *object) 1083{ 1084 struct track *p; 1085 1086 if (!(s->flags & SLAB_STORE_USER)) 1087 return; 1088 1089 p = get_track(s, object, TRACK_ALLOC); 1090 memset(p, 0, 2*sizeof(struct track)); 1091} 1092 1093static void print_track(const char *s, struct track *t, unsigned long pr_time) 1094{ 1095 depot_stack_handle_t handle __maybe_unused; 1096 1097 if (!t->addr) 1098 return; 1099 1100 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n", 1101 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid); 1102#ifdef CONFIG_STACKDEPOT 1103 handle = READ_ONCE(t->handle); 1104 if (handle) 1105 stack_depot_print(handle); 1106 else 1107 pr_err("object allocation/free stack trace missing\n"); 1108#endif 1109} 1110 1111void print_tracking(struct kmem_cache *s, void *object) 1112{ 1113 unsigned long pr_time = jiffies; 1114 if (!(s->flags & SLAB_STORE_USER)) 1115 return; 1116 1117 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time); 1118 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time); 1119} 1120 1121static void print_slab_info(const struct slab *slab) 1122{ 1123 pr_err("Slab 0x%p objects=%u used=%u fp=0x%p flags=%pGp\n", 1124 slab, slab->objects, slab->inuse, slab->freelist, 1125 &slab->flags.f); 1126} 1127 1128void skip_orig_size_check(struct kmem_cache *s, const void *object) 1129{ 1130 set_orig_size(s, (void *)object, s->object_size); 1131} 1132 1133static void __slab_bug(struct kmem_cache *s, const char *fmt, va_list argsp) 1134{ 1135 struct va_format vaf; 1136 va_list args; 1137 1138 va_copy(args, argsp); 1139 vaf.fmt = fmt; 1140 vaf.va = &args; 1141 pr_err("=============================================================================\n"); 1142 pr_err("BUG %s (%s): %pV\n", s ? s->name : "<unknown>", print_tainted(), &vaf); 1143 pr_err("-----------------------------------------------------------------------------\n\n"); 1144 va_end(args); 1145} 1146 1147static void slab_bug(struct kmem_cache *s, const char *fmt, ...) 1148{ 1149 va_list args; 1150 1151 va_start(args, fmt); 1152 __slab_bug(s, fmt, args); 1153 va_end(args); 1154} 1155 1156__printf(2, 3) 1157static void slab_fix(struct kmem_cache *s, const char *fmt, ...) 1158{ 1159 struct va_format vaf; 1160 va_list args; 1161 1162 if (slab_add_kunit_errors()) 1163 return; 1164 1165 va_start(args, fmt); 1166 vaf.fmt = fmt; 1167 vaf.va = &args; 1168 pr_err("FIX %s: %pV\n", s->name, &vaf); 1169 va_end(args); 1170} 1171 1172static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) 1173{ 1174 unsigned int off; /* Offset of last byte */ 1175 u8 *addr = slab_address(slab); 1176 1177 print_tracking(s, p); 1178 1179 print_slab_info(slab); 1180 1181 pr_err("Object 0x%p @offset=%tu fp=0x%p\n\n", 1182 p, p - addr, get_freepointer(s, p)); 1183 1184 if (s->flags & SLAB_RED_ZONE) 1185 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad, 1186 s->red_left_pad); 1187 else if (p > addr + 16) 1188 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16); 1189 1190 print_section(KERN_ERR, "Object ", p, 1191 min_t(unsigned int, s->object_size, PAGE_SIZE)); 1192 if (s->flags & SLAB_RED_ZONE) 1193 print_section(KERN_ERR, "Redzone ", p + s->object_size, 1194 s->inuse - s->object_size); 1195 1196 off = get_info_end(s); 1197 1198 if (s->flags & SLAB_STORE_USER) 1199 off += 2 * sizeof(struct track); 1200 1201 if (slub_debug_orig_size(s)) 1202 off += sizeof(unsigned long); 1203 1204 off += kasan_metadata_size(s, false); 1205 1206 if (off != size_from_object(s)) 1207 /* Beginning of the filler is the free pointer */ 1208 print_section(KERN_ERR, "Padding ", p + off, 1209 size_from_object(s) - off); 1210} 1211 1212static void object_err(struct kmem_cache *s, struct slab *slab, 1213 u8 *object, const char *reason) 1214{ 1215 if (slab_add_kunit_errors()) 1216 return; 1217 1218 slab_bug(s, reason); 1219 if (!object || !check_valid_pointer(s, slab, object)) { 1220 print_slab_info(slab); 1221 pr_err("Invalid pointer 0x%p\n", object); 1222 } else { 1223 print_trailer(s, slab, object); 1224 } 1225 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1226 1227 WARN_ON(1); 1228} 1229 1230static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 1231 void **freelist, void *nextfree) 1232{ 1233 if ((s->flags & SLAB_CONSISTENCY_CHECKS) && 1234 !check_valid_pointer(s, slab, nextfree) && freelist) { 1235 object_err(s, slab, *freelist, "Freechain corrupt"); 1236 *freelist = NULL; 1237 slab_fix(s, "Isolate corrupted freechain"); 1238 return true; 1239 } 1240 1241 return false; 1242} 1243 1244static void __slab_err(struct slab *slab) 1245{ 1246 if (slab_in_kunit_test()) 1247 return; 1248 1249 print_slab_info(slab); 1250 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 1251 1252 WARN_ON(1); 1253} 1254 1255static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab, 1256 const char *fmt, ...) 1257{ 1258 va_list args; 1259 1260 if (slab_add_kunit_errors()) 1261 return; 1262 1263 va_start(args, fmt); 1264 __slab_bug(s, fmt, args); 1265 va_end(args); 1266 1267 __slab_err(slab); 1268} 1269 1270static void init_object(struct kmem_cache *s, void *object, u8 val) 1271{ 1272 u8 *p = kasan_reset_tag(object); 1273 unsigned int poison_size = s->object_size; 1274 1275 if (s->flags & SLAB_RED_ZONE) { 1276 /* 1277 * Here and below, avoid overwriting the KMSAN shadow. Keeping 1278 * the shadow makes it possible to distinguish uninit-value 1279 * from use-after-free. 1280 */ 1281 memset_no_sanitize_memory(p - s->red_left_pad, val, 1282 s->red_left_pad); 1283 1284 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { 1285 /* 1286 * Redzone the extra allocated space by kmalloc than 1287 * requested, and the poison size will be limited to 1288 * the original request size accordingly. 1289 */ 1290 poison_size = get_orig_size(s, object); 1291 } 1292 } 1293 1294 if (s->flags & __OBJECT_POISON) { 1295 memset_no_sanitize_memory(p, POISON_FREE, poison_size - 1); 1296 memset_no_sanitize_memory(p + poison_size - 1, POISON_END, 1); 1297 } 1298 1299 if (s->flags & SLAB_RED_ZONE) 1300 memset_no_sanitize_memory(p + poison_size, val, 1301 s->inuse - poison_size); 1302} 1303 1304static void restore_bytes(struct kmem_cache *s, const char *message, u8 data, 1305 void *from, void *to) 1306{ 1307 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data); 1308 memset(from, data, to - from); 1309} 1310 1311#ifdef CONFIG_KMSAN 1312#define pad_check_attributes noinline __no_kmsan_checks 1313#else 1314#define pad_check_attributes 1315#endif 1316 1317static pad_check_attributes int 1318check_bytes_and_report(struct kmem_cache *s, struct slab *slab, 1319 u8 *object, const char *what, u8 *start, unsigned int value, 1320 unsigned int bytes, bool slab_obj_print) 1321{ 1322 u8 *fault; 1323 u8 *end; 1324 u8 *addr = slab_address(slab); 1325 1326 metadata_access_enable(); 1327 fault = memchr_inv(kasan_reset_tag(start), value, bytes); 1328 metadata_access_disable(); 1329 if (!fault) 1330 return 1; 1331 1332 end = start + bytes; 1333 while (end > fault && end[-1] == value) 1334 end--; 1335 1336 if (slab_add_kunit_errors()) 1337 goto skip_bug_print; 1338 1339 pr_err("[%s overwritten] 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", 1340 what, fault, end - 1, fault - addr, fault[0], value); 1341 1342 if (slab_obj_print) 1343 object_err(s, slab, object, "Object corrupt"); 1344 1345skip_bug_print: 1346 restore_bytes(s, what, value, fault, end); 1347 return 0; 1348} 1349 1350/* 1351 * Object layout: 1352 * 1353 * object address 1354 * Bytes of the object to be managed. 1355 * If the freepointer may overlay the object then the free 1356 * pointer is at the middle of the object. 1357 * 1358 * Poisoning uses 0x6b (POISON_FREE) and the last byte is 1359 * 0xa5 (POISON_END) 1360 * 1361 * object + s->object_size 1362 * Padding to reach word boundary. This is also used for Redzoning. 1363 * Padding is extended by another word if Redzoning is enabled and 1364 * object_size == inuse. 1365 * 1366 * We fill with 0xbb (SLUB_RED_INACTIVE) for inactive objects and with 1367 * 0xcc (SLUB_RED_ACTIVE) for objects in use. 1368 * 1369 * object + s->inuse 1370 * Meta data starts here. 1371 * 1372 * A. Free pointer (if we cannot overwrite object on free) 1373 * B. Tracking data for SLAB_STORE_USER 1374 * C. Original request size for kmalloc object (SLAB_STORE_USER enabled) 1375 * D. Padding to reach required alignment boundary or at minimum 1376 * one word if debugging is on to be able to detect writes 1377 * before the word boundary. 1378 * 1379 * Padding is done using 0x5a (POISON_INUSE) 1380 * 1381 * object + s->size 1382 * Nothing is used beyond s->size. 1383 * 1384 * If slabcaches are merged then the object_size and inuse boundaries are mostly 1385 * ignored. And therefore no slab options that rely on these boundaries 1386 * may be used with merged slabcaches. 1387 */ 1388 1389static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) 1390{ 1391 unsigned long off = get_info_end(s); /* The end of info */ 1392 1393 if (s->flags & SLAB_STORE_USER) { 1394 /* We also have user information there */ 1395 off += 2 * sizeof(struct track); 1396 1397 if (s->flags & SLAB_KMALLOC) 1398 off += sizeof(unsigned long); 1399 } 1400 1401 off += kasan_metadata_size(s, false); 1402 1403 if (size_from_object(s) == off) 1404 return 1; 1405 1406 return check_bytes_and_report(s, slab, p, "Object padding", 1407 p + off, POISON_INUSE, size_from_object(s) - off, true); 1408} 1409 1410/* Check the pad bytes at the end of a slab page */ 1411static pad_check_attributes void 1412slab_pad_check(struct kmem_cache *s, struct slab *slab) 1413{ 1414 u8 *start; 1415 u8 *fault; 1416 u8 *end; 1417 u8 *pad; 1418 int length; 1419 int remainder; 1420 1421 if (!(s->flags & SLAB_POISON)) 1422 return; 1423 1424 start = slab_address(slab); 1425 length = slab_size(slab); 1426 end = start + length; 1427 remainder = length % s->size; 1428 if (!remainder) 1429 return; 1430 1431 pad = end - remainder; 1432 metadata_access_enable(); 1433 fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder); 1434 metadata_access_disable(); 1435 if (!fault) 1436 return; 1437 while (end > fault && end[-1] == POISON_INUSE) 1438 end--; 1439 1440 slab_bug(s, "Padding overwritten. 0x%p-0x%p @offset=%tu", 1441 fault, end - 1, fault - start); 1442 print_section(KERN_ERR, "Padding ", pad, remainder); 1443 __slab_err(slab); 1444 1445 restore_bytes(s, "slab padding", POISON_INUSE, fault, end); 1446} 1447 1448static int check_object(struct kmem_cache *s, struct slab *slab, 1449 void *object, u8 val) 1450{ 1451 u8 *p = object; 1452 u8 *endobject = object + s->object_size; 1453 unsigned int orig_size, kasan_meta_size; 1454 int ret = 1; 1455 1456 if (s->flags & SLAB_RED_ZONE) { 1457 if (!check_bytes_and_report(s, slab, object, "Left Redzone", 1458 object - s->red_left_pad, val, s->red_left_pad, ret)) 1459 ret = 0; 1460 1461 if (!check_bytes_and_report(s, slab, object, "Right Redzone", 1462 endobject, val, s->inuse - s->object_size, ret)) 1463 ret = 0; 1464 1465 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { 1466 orig_size = get_orig_size(s, object); 1467 1468 if (s->object_size > orig_size && 1469 !check_bytes_and_report(s, slab, object, 1470 "kmalloc Redzone", p + orig_size, 1471 val, s->object_size - orig_size, ret)) { 1472 ret = 0; 1473 } 1474 } 1475 } else { 1476 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { 1477 if (!check_bytes_and_report(s, slab, p, "Alignment padding", 1478 endobject, POISON_INUSE, 1479 s->inuse - s->object_size, ret)) 1480 ret = 0; 1481 } 1482 } 1483 1484 if (s->flags & SLAB_POISON) { 1485 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) { 1486 /* 1487 * KASAN can save its free meta data inside of the 1488 * object at offset 0. Thus, skip checking the part of 1489 * the redzone that overlaps with the meta data. 1490 */ 1491 kasan_meta_size = kasan_metadata_size(s, true); 1492 if (kasan_meta_size < s->object_size - 1 && 1493 !check_bytes_and_report(s, slab, p, "Poison", 1494 p + kasan_meta_size, POISON_FREE, 1495 s->object_size - kasan_meta_size - 1, ret)) 1496 ret = 0; 1497 if (kasan_meta_size < s->object_size && 1498 !check_bytes_and_report(s, slab, p, "End Poison", 1499 p + s->object_size - 1, POISON_END, 1, ret)) 1500 ret = 0; 1501 } 1502 /* 1503 * check_pad_bytes cleans up on its own. 1504 */ 1505 if (!check_pad_bytes(s, slab, p)) 1506 ret = 0; 1507 } 1508 1509 /* 1510 * Cannot check freepointer while object is allocated if 1511 * object and freepointer overlap. 1512 */ 1513 if ((freeptr_outside_object(s) || val != SLUB_RED_ACTIVE) && 1514 !check_valid_pointer(s, slab, get_freepointer(s, p))) { 1515 object_err(s, slab, p, "Freepointer corrupt"); 1516 /* 1517 * No choice but to zap it and thus lose the remainder 1518 * of the free objects in this slab. May cause 1519 * another error because the object count is now wrong. 1520 */ 1521 set_freepointer(s, p, NULL); 1522 ret = 0; 1523 } 1524 1525 return ret; 1526} 1527 1528/* 1529 * Checks if the slab state looks sane. Assumes the struct slab pointer 1530 * was either obtained in a way that ensures it's valid, or validated 1531 * by validate_slab_ptr() 1532 */ 1533static int check_slab(struct kmem_cache *s, struct slab *slab) 1534{ 1535 int maxobj; 1536 1537 maxobj = order_objects(slab_order(slab), s->size); 1538 if (slab->objects > maxobj) { 1539 slab_err(s, slab, "objects %u > max %u", 1540 slab->objects, maxobj); 1541 return 0; 1542 } 1543 if (slab->inuse > slab->objects) { 1544 slab_err(s, slab, "inuse %u > max %u", 1545 slab->inuse, slab->objects); 1546 return 0; 1547 } 1548 if (slab->frozen) { 1549 slab_err(s, slab, "Slab disabled since SLUB metadata consistency check failed"); 1550 return 0; 1551 } 1552 1553 /* Slab_pad_check fixes things up after itself */ 1554 slab_pad_check(s, slab); 1555 return 1; 1556} 1557 1558/* 1559 * Determine if a certain object in a slab is on the freelist. Must hold the 1560 * slab lock to guarantee that the chains are in a consistent state. 1561 */ 1562static bool on_freelist(struct kmem_cache *s, struct slab *slab, void *search) 1563{ 1564 int nr = 0; 1565 void *fp; 1566 void *object = NULL; 1567 int max_objects; 1568 1569 fp = slab->freelist; 1570 while (fp && nr <= slab->objects) { 1571 if (fp == search) 1572 return true; 1573 if (!check_valid_pointer(s, slab, fp)) { 1574 if (object) { 1575 object_err(s, slab, object, 1576 "Freechain corrupt"); 1577 set_freepointer(s, object, NULL); 1578 break; 1579 } else { 1580 slab_err(s, slab, "Freepointer corrupt"); 1581 slab->freelist = NULL; 1582 slab->inuse = slab->objects; 1583 slab_fix(s, "Freelist cleared"); 1584 return false; 1585 } 1586 } 1587 object = fp; 1588 fp = get_freepointer(s, object); 1589 nr++; 1590 } 1591 1592 if (nr > slab->objects) { 1593 slab_err(s, slab, "Freelist cycle detected"); 1594 slab->freelist = NULL; 1595 slab->inuse = slab->objects; 1596 slab_fix(s, "Freelist cleared"); 1597 return false; 1598 } 1599 1600 max_objects = order_objects(slab_order(slab), s->size); 1601 if (max_objects > MAX_OBJS_PER_PAGE) 1602 max_objects = MAX_OBJS_PER_PAGE; 1603 1604 if (slab->objects != max_objects) { 1605 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d", 1606 slab->objects, max_objects); 1607 slab->objects = max_objects; 1608 slab_fix(s, "Number of objects adjusted"); 1609 } 1610 if (slab->inuse != slab->objects - nr) { 1611 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d", 1612 slab->inuse, slab->objects - nr); 1613 slab->inuse = slab->objects - nr; 1614 slab_fix(s, "Object count adjusted"); 1615 } 1616 return search == NULL; 1617} 1618 1619static void trace(struct kmem_cache *s, struct slab *slab, void *object, 1620 int alloc) 1621{ 1622 if (s->flags & SLAB_TRACE) { 1623 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", 1624 s->name, 1625 alloc ? "alloc" : "free", 1626 object, slab->inuse, 1627 slab->freelist); 1628 1629 if (!alloc) 1630 print_section(KERN_INFO, "Object ", (void *)object, 1631 s->object_size); 1632 1633 dump_stack(); 1634 } 1635} 1636 1637/* 1638 * Tracking of fully allocated slabs for debugging purposes. 1639 */ 1640static void add_full(struct kmem_cache *s, 1641 struct kmem_cache_node *n, struct slab *slab) 1642{ 1643 if (!(s->flags & SLAB_STORE_USER)) 1644 return; 1645 1646 lockdep_assert_held(&n->list_lock); 1647 list_add(&slab->slab_list, &n->full); 1648} 1649 1650static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) 1651{ 1652 if (!(s->flags & SLAB_STORE_USER)) 1653 return; 1654 1655 lockdep_assert_held(&n->list_lock); 1656 list_del(&slab->slab_list); 1657} 1658 1659static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 1660{ 1661 return atomic_long_read(&n->nr_slabs); 1662} 1663 1664static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) 1665{ 1666 struct kmem_cache_node *n = get_node(s, node); 1667 1668 atomic_long_inc(&n->nr_slabs); 1669 atomic_long_add(objects, &n->total_objects); 1670} 1671static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) 1672{ 1673 struct kmem_cache_node *n = get_node(s, node); 1674 1675 atomic_long_dec(&n->nr_slabs); 1676 atomic_long_sub(objects, &n->total_objects); 1677} 1678 1679/* Object debug checks for alloc/free paths */ 1680static void setup_object_debug(struct kmem_cache *s, void *object) 1681{ 1682 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)) 1683 return; 1684 1685 init_object(s, object, SLUB_RED_INACTIVE); 1686 init_tracking(s, object); 1687} 1688 1689static 1690void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) 1691{ 1692 if (!kmem_cache_debug_flags(s, SLAB_POISON)) 1693 return; 1694 1695 metadata_access_enable(); 1696 memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); 1697 metadata_access_disable(); 1698} 1699 1700static inline int alloc_consistency_checks(struct kmem_cache *s, 1701 struct slab *slab, void *object) 1702{ 1703 if (!check_slab(s, slab)) 1704 return 0; 1705 1706 if (!check_valid_pointer(s, slab, object)) { 1707 object_err(s, slab, object, "Freelist Pointer check fails"); 1708 return 0; 1709 } 1710 1711 if (!check_object(s, slab, object, SLUB_RED_INACTIVE)) 1712 return 0; 1713 1714 return 1; 1715} 1716 1717static noinline bool alloc_debug_processing(struct kmem_cache *s, 1718 struct slab *slab, void *object, int orig_size) 1719{ 1720 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 1721 if (!alloc_consistency_checks(s, slab, object)) 1722 goto bad; 1723 } 1724 1725 /* Success. Perform special debug activities for allocs */ 1726 trace(s, slab, object, 1); 1727 set_orig_size(s, object, orig_size); 1728 init_object(s, object, SLUB_RED_ACTIVE); 1729 return true; 1730 1731bad: 1732 /* 1733 * Let's do the best we can to avoid issues in the future. Marking all 1734 * objects as used avoids touching the remaining objects. 1735 */ 1736 slab_fix(s, "Marking all objects used"); 1737 slab->inuse = slab->objects; 1738 slab->freelist = NULL; 1739 slab->frozen = 1; /* mark consistency-failed slab as frozen */ 1740 1741 return false; 1742} 1743 1744static inline int free_consistency_checks(struct kmem_cache *s, 1745 struct slab *slab, void *object, unsigned long addr) 1746{ 1747 if (!check_valid_pointer(s, slab, object)) { 1748 slab_err(s, slab, "Invalid object pointer 0x%p", object); 1749 return 0; 1750 } 1751 1752 if (on_freelist(s, slab, object)) { 1753 object_err(s, slab, object, "Object already free"); 1754 return 0; 1755 } 1756 1757 if (!check_object(s, slab, object, SLUB_RED_ACTIVE)) 1758 return 0; 1759 1760 if (unlikely(s != slab->slab_cache)) { 1761 if (!slab->slab_cache) { 1762 slab_err(NULL, slab, "No slab cache for object 0x%p", 1763 object); 1764 } else { 1765 object_err(s, slab, object, 1766 "page slab pointer corrupt."); 1767 } 1768 return 0; 1769 } 1770 return 1; 1771} 1772 1773/* 1774 * Parse a block of slab_debug options. Blocks are delimited by ';' 1775 * 1776 * @str: start of block 1777 * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified 1778 * @slabs: return start of list of slabs, or NULL when there's no list 1779 * @init: assume this is initial parsing and not per-kmem-create parsing 1780 * 1781 * returns the start of next block if there's any, or NULL 1782 */ 1783static const char * 1784parse_slub_debug_flags(const char *str, slab_flags_t *flags, const char **slabs, bool init) 1785{ 1786 bool higher_order_disable = false; 1787 1788 /* Skip any completely empty blocks */ 1789 while (*str && *str == ';') 1790 str++; 1791 1792 if (*str == ',') { 1793 /* 1794 * No options but restriction on slabs. This means full 1795 * debugging for slabs matching a pattern. 1796 */ 1797 *flags = DEBUG_DEFAULT_FLAGS; 1798 goto check_slabs; 1799 } 1800 *flags = 0; 1801 1802 /* Determine which debug features should be switched on */ 1803 for (; *str && *str != ',' && *str != ';'; str++) { 1804 switch (tolower(*str)) { 1805 case '-': 1806 *flags = 0; 1807 break; 1808 case 'f': 1809 *flags |= SLAB_CONSISTENCY_CHECKS; 1810 break; 1811 case 'z': 1812 *flags |= SLAB_RED_ZONE; 1813 break; 1814 case 'p': 1815 *flags |= SLAB_POISON; 1816 break; 1817 case 'u': 1818 *flags |= SLAB_STORE_USER; 1819 break; 1820 case 't': 1821 *flags |= SLAB_TRACE; 1822 break; 1823 case 'a': 1824 *flags |= SLAB_FAILSLAB; 1825 break; 1826 case 'o': 1827 /* 1828 * Avoid enabling debugging on caches if its minimum 1829 * order would increase as a result. 1830 */ 1831 higher_order_disable = true; 1832 break; 1833 default: 1834 if (init) 1835 pr_err("slab_debug option '%c' unknown. skipped\n", *str); 1836 } 1837 } 1838check_slabs: 1839 if (*str == ',') 1840 *slabs = ++str; 1841 else 1842 *slabs = NULL; 1843 1844 /* Skip over the slab list */ 1845 while (*str && *str != ';') 1846 str++; 1847 1848 /* Skip any completely empty blocks */ 1849 while (*str && *str == ';') 1850 str++; 1851 1852 if (init && higher_order_disable) 1853 disable_higher_order_debug = 1; 1854 1855 if (*str) 1856 return str; 1857 else 1858 return NULL; 1859} 1860 1861static int __init setup_slub_debug(const char *str, const struct kernel_param *kp) 1862{ 1863 slab_flags_t flags; 1864 slab_flags_t global_flags; 1865 const char *saved_str; 1866 const char *slab_list; 1867 bool global_slub_debug_changed = false; 1868 bool slab_list_specified = false; 1869 1870 global_flags = DEBUG_DEFAULT_FLAGS; 1871 if (!str || !*str) 1872 /* 1873 * No options specified. Switch on full debugging. 1874 */ 1875 goto out; 1876 1877 saved_str = str; 1878 while (str) { 1879 str = parse_slub_debug_flags(str, &flags, &slab_list, true); 1880 1881 if (!slab_list) { 1882 global_flags = flags; 1883 global_slub_debug_changed = true; 1884 } else { 1885 slab_list_specified = true; 1886 if (flags & SLAB_STORE_USER) 1887 stack_depot_request_early_init(); 1888 } 1889 } 1890 1891 /* 1892 * For backwards compatibility, a single list of flags with list of 1893 * slabs means debugging is only changed for those slabs, so the global 1894 * slab_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending 1895 * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as 1896 * long as there is no option specifying flags without a slab list. 1897 */ 1898 if (slab_list_specified) { 1899 if (!global_slub_debug_changed) 1900 global_flags = slub_debug; 1901 slub_debug_string = saved_str; 1902 } 1903out: 1904 slub_debug = global_flags; 1905 if (slub_debug & SLAB_STORE_USER) 1906 stack_depot_request_early_init(); 1907 if (slub_debug != 0 || slub_debug_string) 1908 static_branch_enable(&slub_debug_enabled); 1909 else 1910 static_branch_disable(&slub_debug_enabled); 1911 if ((static_branch_unlikely(&init_on_alloc) || 1912 static_branch_unlikely(&init_on_free)) && 1913 (slub_debug & SLAB_POISON)) 1914 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n"); 1915 return 0; 1916} 1917 1918static const struct kernel_param_ops param_ops_slab_debug __initconst = { 1919 .flags = KERNEL_PARAM_OPS_FL_NOARG, 1920 .set = setup_slub_debug, 1921}; 1922__core_param_cb(slab_debug, ¶m_ops_slab_debug, NULL, 0); 1923__core_param_cb(slub_debug, ¶m_ops_slab_debug, NULL, 0); 1924 1925/* 1926 * kmem_cache_flags - apply debugging options to the cache 1927 * @flags: flags to set 1928 * @name: name of the cache 1929 * 1930 * Debug option(s) are applied to @flags. In addition to the debug 1931 * option(s), if a slab name (or multiple) is specified i.e. 1932 * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ... 1933 * then only the select slabs will receive the debug option(s). 1934 */ 1935slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) 1936{ 1937 const char *iter; 1938 size_t len; 1939 const char *next_block; 1940 slab_flags_t block_flags; 1941 slab_flags_t slub_debug_local = slub_debug; 1942 1943 if (flags & SLAB_NO_USER_FLAGS) 1944 return flags; 1945 1946 /* 1947 * If the slab cache is for debugging (e.g. kmemleak) then 1948 * don't store user (stack trace) information by default, 1949 * but let the user enable it via the command line below. 1950 */ 1951 if (flags & SLAB_NOLEAKTRACE) 1952 slub_debug_local &= ~SLAB_STORE_USER; 1953 1954 len = strlen(name); 1955 next_block = slub_debug_string; 1956 /* Go through all blocks of debug options, see if any matches our slab's name */ 1957 while (next_block) { 1958 next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false); 1959 if (!iter) 1960 continue; 1961 /* Found a block that has a slab list, search it */ 1962 while (*iter) { 1963 const char *end, *glob; 1964 size_t cmplen; 1965 1966 end = strchrnul(iter, ','); 1967 if (next_block && next_block < end) 1968 end = next_block - 1; 1969 1970 glob = strnchr(iter, end - iter, '*'); 1971 if (glob) 1972 cmplen = glob - iter; 1973 else 1974 cmplen = max_t(size_t, len, (end - iter)); 1975 1976 if (!strncmp(name, iter, cmplen)) { 1977 flags |= block_flags; 1978 return flags; 1979 } 1980 1981 if (!*end || *end == ';') 1982 break; 1983 iter = end + 1; 1984 } 1985 } 1986 1987 return flags | slub_debug_local; 1988} 1989#else /* !CONFIG_SLUB_DEBUG */ 1990static inline void setup_object_debug(struct kmem_cache *s, void *object) {} 1991static inline 1992void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {} 1993 1994static inline bool alloc_debug_processing(struct kmem_cache *s, 1995 struct slab *slab, void *object, int orig_size) { return true; } 1996 1997static inline bool free_debug_processing(struct kmem_cache *s, 1998 struct slab *slab, void *head, void *tail, int *bulk_cnt, 1999 unsigned long addr, depot_stack_handle_t handle) { return true; } 2000 2001static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {} 2002static inline int check_object(struct kmem_cache *s, struct slab *slab, 2003 void *object, u8 val) { return 1; } 2004static inline depot_stack_handle_t set_track_prepare(gfp_t gfp_flags) { return 0; } 2005static inline void set_track(struct kmem_cache *s, void *object, 2006 enum track_item alloc, unsigned long addr, gfp_t gfp_flags) {} 2007static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, 2008 struct slab *slab) {} 2009static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, 2010 struct slab *slab) {} 2011slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name) 2012{ 2013 return flags; 2014} 2015#define slub_debug 0 2016 2017#define disable_higher_order_debug 0 2018 2019static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 2020 { return 0; } 2021static inline void inc_slabs_node(struct kmem_cache *s, int node, 2022 int objects) {} 2023static inline void dec_slabs_node(struct kmem_cache *s, int node, 2024 int objects) {} 2025static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, 2026 void **freelist, void *nextfree) 2027{ 2028 return false; 2029} 2030#endif /* CONFIG_SLUB_DEBUG */ 2031 2032/* 2033 * The allocated objcg pointers array is not accounted directly. 2034 * Moreover, it should not come from DMA buffer and is not readily 2035 * reclaimable. So those GFP bits should be masked off. 2036 */ 2037#define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | \ 2038 __GFP_ACCOUNT | __GFP_NOFAIL) 2039 2040#ifdef CONFIG_SLAB_OBJ_EXT 2041 2042#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG 2043 2044static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) 2045{ 2046 struct slabobj_ext *slab_exts; 2047 struct slab *obj_exts_slab; 2048 2049 obj_exts_slab = virt_to_slab(obj_exts); 2050 slab_exts = slab_obj_exts(obj_exts_slab); 2051 if (slab_exts) { 2052 unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, 2053 obj_exts_slab, obj_exts); 2054 2055 if (unlikely(is_codetag_empty(&slab_exts[offs].ref))) 2056 return; 2057 2058 /* codetag should be NULL here */ 2059 WARN_ON(slab_exts[offs].ref.ct); 2060 set_codetag_empty(&slab_exts[offs].ref); 2061 } 2062} 2063 2064static inline bool mark_failed_objexts_alloc(struct slab *slab) 2065{ 2066 return cmpxchg(&slab->obj_exts, 0, OBJEXTS_ALLOC_FAIL) == 0; 2067} 2068 2069static inline void handle_failed_objexts_alloc(unsigned long obj_exts, 2070 struct slabobj_ext *vec, unsigned int objects) 2071{ 2072 /* 2073 * If vector previously failed to allocate then we have live 2074 * objects with no tag reference. Mark all references in this 2075 * vector as empty to avoid warnings later on. 2076 */ 2077 if (obj_exts == OBJEXTS_ALLOC_FAIL) { 2078 unsigned int i; 2079 2080 for (i = 0; i < objects; i++) 2081 set_codetag_empty(&vec[i].ref); 2082 } 2083} 2084 2085#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ 2086 2087static inline void mark_objexts_empty(struct slabobj_ext *obj_exts) {} 2088static inline bool mark_failed_objexts_alloc(struct slab *slab) { return false; } 2089static inline void handle_failed_objexts_alloc(unsigned long obj_exts, 2090 struct slabobj_ext *vec, unsigned int objects) {} 2091 2092#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ 2093 2094static inline void init_slab_obj_exts(struct slab *slab) 2095{ 2096 slab->obj_exts = 0; 2097} 2098 2099/* 2100 * Calculate the allocation size for slabobj_ext array. 2101 * 2102 * When memory allocation profiling is enabled, the obj_exts array 2103 * could be allocated from the same slab cache it's being allocated for. 2104 * This would prevent the slab from ever being freed because it would 2105 * always contain at least one allocated object (its own obj_exts array). 2106 * 2107 * To avoid this, increase the allocation size when we detect the array 2108 * may come from the same cache, forcing it to use a different cache. 2109 */ 2110static inline size_t obj_exts_alloc_size(struct kmem_cache *s, 2111 struct slab *slab, gfp_t gfp) 2112{ 2113 size_t sz = sizeof(struct slabobj_ext) * slab->objects; 2114 struct kmem_cache *obj_exts_cache; 2115 2116 if (sz > KMALLOC_MAX_CACHE_SIZE) 2117 return sz; 2118 2119 if (!is_kmalloc_normal(s)) 2120 return sz; 2121 2122 obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0); 2123 /* 2124 * We can't simply compare s with obj_exts_cache, because random kmalloc 2125 * caches have multiple caches per size, selected by caller address. 2126 * Since caller address may differ between kmalloc_slab() and actual 2127 * allocation, bump size when sizes are equal. 2128 */ 2129 if (s->object_size == obj_exts_cache->object_size) 2130 return obj_exts_cache->object_size + 1; 2131 2132 return sz; 2133} 2134 2135int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 2136 gfp_t gfp, bool new_slab) 2137{ 2138 bool allow_spin = gfpflags_allow_spinning(gfp); 2139 unsigned int objects = objs_per_slab(s, slab); 2140 unsigned long new_exts; 2141 unsigned long old_exts; 2142 struct slabobj_ext *vec; 2143 size_t sz; 2144 2145 gfp &= ~OBJCGS_CLEAR_MASK; 2146 /* Prevent recursive extension vector allocation */ 2147 gfp |= __GFP_NO_OBJ_EXT; 2148 2149 sz = obj_exts_alloc_size(s, slab, gfp); 2150 2151 /* 2152 * Note that allow_spin may be false during early boot and its 2153 * restricted GFP_BOOT_MASK. Due to kmalloc_nolock() only supporting 2154 * architectures with cmpxchg16b, early obj_exts will be missing for 2155 * very early allocations on those. 2156 */ 2157 if (unlikely(!allow_spin)) 2158 vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT, 2159 slab_nid(slab)); 2160 else 2161 vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab)); 2162 2163 if (!vec) { 2164 /* 2165 * Try to mark vectors which failed to allocate. 2166 * If this operation fails, there may be a racing process 2167 * that has already completed the allocation. 2168 */ 2169 if (!mark_failed_objexts_alloc(slab) && 2170 slab_obj_exts(slab)) 2171 return 0; 2172 2173 return -ENOMEM; 2174 } 2175 2176 VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL && 2177 virt_to_slab(vec)->slab_cache == s); 2178 2179 new_exts = (unsigned long)vec; 2180 if (unlikely(!allow_spin)) 2181 new_exts |= OBJEXTS_NOSPIN_ALLOC; 2182#ifdef CONFIG_MEMCG 2183 new_exts |= MEMCG_DATA_OBJEXTS; 2184#endif 2185retry: 2186 old_exts = READ_ONCE(slab->obj_exts); 2187 handle_failed_objexts_alloc(old_exts, vec, objects); 2188 if (new_slab) { 2189 /* 2190 * If the slab is brand new and nobody can yet access its 2191 * obj_exts, no synchronization is required and obj_exts can 2192 * be simply assigned. 2193 */ 2194 slab->obj_exts = new_exts; 2195 } else if (old_exts & ~OBJEXTS_FLAGS_MASK) { 2196 /* 2197 * If the slab is already in use, somebody can allocate and 2198 * assign slabobj_exts in parallel. In this case the existing 2199 * objcg vector should be reused. 2200 */ 2201 mark_objexts_empty(vec); 2202 if (unlikely(!allow_spin)) 2203 kfree_nolock(vec); 2204 else 2205 kfree(vec); 2206 return 0; 2207 } else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) { 2208 /* Retry if a racing thread changed slab->obj_exts from under us. */ 2209 goto retry; 2210 } 2211 2212 if (allow_spin) 2213 kmemleak_not_leak(vec); 2214 return 0; 2215} 2216 2217static inline void free_slab_obj_exts(struct slab *slab) 2218{ 2219 struct slabobj_ext *obj_exts; 2220 2221 obj_exts = slab_obj_exts(slab); 2222 if (!obj_exts) { 2223 /* 2224 * If obj_exts allocation failed, slab->obj_exts is set to 2225 * OBJEXTS_ALLOC_FAIL. In this case, we end up here and should 2226 * clear the flag. 2227 */ 2228 slab->obj_exts = 0; 2229 return; 2230 } 2231 2232 /* 2233 * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its 2234 * corresponding extension will be NULL. alloc_tag_sub() will throw a 2235 * warning if slab has extensions but the extension of an object is 2236 * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that 2237 * the extension for obj_exts is expected to be NULL. 2238 */ 2239 mark_objexts_empty(obj_exts); 2240 if (unlikely(READ_ONCE(slab->obj_exts) & OBJEXTS_NOSPIN_ALLOC)) 2241 kfree_nolock(obj_exts); 2242 else 2243 kfree(obj_exts); 2244 slab->obj_exts = 0; 2245} 2246 2247#else /* CONFIG_SLAB_OBJ_EXT */ 2248 2249static inline void init_slab_obj_exts(struct slab *slab) 2250{ 2251} 2252 2253static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, 2254 gfp_t gfp, bool new_slab) 2255{ 2256 return 0; 2257} 2258 2259static inline void free_slab_obj_exts(struct slab *slab) 2260{ 2261} 2262 2263#endif /* CONFIG_SLAB_OBJ_EXT */ 2264 2265#ifdef CONFIG_MEM_ALLOC_PROFILING 2266 2267static inline struct slabobj_ext * 2268prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p) 2269{ 2270 struct slab *slab; 2271 2272 slab = virt_to_slab(p); 2273 if (!slab_obj_exts(slab) && 2274 alloc_slab_obj_exts(slab, s, flags, false)) { 2275 pr_warn_once("%s, %s: Failed to create slab extension vector!\n", 2276 __func__, s->name); 2277 return NULL; 2278 } 2279 2280 return slab_obj_exts(slab) + obj_to_index(s, slab, p); 2281} 2282 2283/* Should be called only if mem_alloc_profiling_enabled() */ 2284static noinline void 2285__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2286{ 2287 struct slabobj_ext *obj_exts; 2288 2289 if (!object) 2290 return; 2291 2292 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) 2293 return; 2294 2295 if (flags & __GFP_NO_OBJ_EXT) 2296 return; 2297 2298 obj_exts = prepare_slab_obj_exts_hook(s, flags, object); 2299 /* 2300 * Currently obj_exts is used only for allocation profiling. 2301 * If other users appear then mem_alloc_profiling_enabled() 2302 * check should be added before alloc_tag_add(). 2303 */ 2304 if (likely(obj_exts)) 2305 alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size); 2306 else 2307 alloc_tag_set_inaccurate(current->alloc_tag); 2308} 2309 2310static inline void 2311alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2312{ 2313 if (mem_alloc_profiling_enabled()) 2314 __alloc_tagging_slab_alloc_hook(s, object, flags); 2315} 2316 2317/* Should be called only if mem_alloc_profiling_enabled() */ 2318static noinline void 2319__alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2320 int objects) 2321{ 2322 struct slabobj_ext *obj_exts; 2323 int i; 2324 2325 /* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */ 2326 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) 2327 return; 2328 2329 obj_exts = slab_obj_exts(slab); 2330 if (!obj_exts) 2331 return; 2332 2333 for (i = 0; i < objects; i++) { 2334 unsigned int off = obj_to_index(s, slab, p[i]); 2335 2336 alloc_tag_sub(&obj_exts[off].ref, s->size); 2337 } 2338} 2339 2340static inline void 2341alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2342 int objects) 2343{ 2344 if (mem_alloc_profiling_enabled()) 2345 __alloc_tagging_slab_free_hook(s, slab, p, objects); 2346} 2347 2348#else /* CONFIG_MEM_ALLOC_PROFILING */ 2349 2350static inline void 2351alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags) 2352{ 2353} 2354 2355static inline void 2356alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2357 int objects) 2358{ 2359} 2360 2361#endif /* CONFIG_MEM_ALLOC_PROFILING */ 2362 2363 2364#ifdef CONFIG_MEMCG 2365 2366static void memcg_alloc_abort_single(struct kmem_cache *s, void *object); 2367 2368static __fastpath_inline 2369bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 2370 gfp_t flags, size_t size, void **p) 2371{ 2372 if (likely(!memcg_kmem_online())) 2373 return true; 2374 2375 if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))) 2376 return true; 2377 2378 if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p))) 2379 return true; 2380 2381 if (likely(size == 1)) { 2382 memcg_alloc_abort_single(s, *p); 2383 *p = NULL; 2384 } else { 2385 kmem_cache_free_bulk(s, size, p); 2386 } 2387 2388 return false; 2389} 2390 2391static __fastpath_inline 2392void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p, 2393 int objects) 2394{ 2395 struct slabobj_ext *obj_exts; 2396 2397 if (!memcg_kmem_online()) 2398 return; 2399 2400 obj_exts = slab_obj_exts(slab); 2401 if (likely(!obj_exts)) 2402 return; 2403 2404 __memcg_slab_free_hook(s, slab, p, objects, obj_exts); 2405} 2406 2407static __fastpath_inline 2408bool memcg_slab_post_charge(void *p, gfp_t flags) 2409{ 2410 struct slabobj_ext *slab_exts; 2411 struct kmem_cache *s; 2412 struct page *page; 2413 struct slab *slab; 2414 unsigned long off; 2415 2416 page = virt_to_page(p); 2417 if (PageLargeKmalloc(page)) { 2418 unsigned int order; 2419 int size; 2420 2421 if (PageMemcgKmem(page)) 2422 return true; 2423 2424 order = large_kmalloc_order(page); 2425 if (__memcg_kmem_charge_page(page, flags, order)) 2426 return false; 2427 2428 /* 2429 * This page has already been accounted in the global stats but 2430 * not in the memcg stats. So, subtract from the global and use 2431 * the interface which adds to both global and memcg stats. 2432 */ 2433 size = PAGE_SIZE << order; 2434 mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE_B, -size); 2435 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, size); 2436 return true; 2437 } 2438 2439 slab = page_slab(page); 2440 s = slab->slab_cache; 2441 2442 /* 2443 * Ignore KMALLOC_NORMAL cache to avoid possible circular dependency 2444 * of slab_obj_exts being allocated from the same slab and thus the slab 2445 * becoming effectively unfreeable. 2446 */ 2447 if (is_kmalloc_normal(s)) 2448 return true; 2449 2450 /* Ignore already charged objects. */ 2451 slab_exts = slab_obj_exts(slab); 2452 if (slab_exts) { 2453 off = obj_to_index(s, slab, p); 2454 if (unlikely(slab_exts[off].objcg)) 2455 return true; 2456 } 2457 2458 return __memcg_slab_post_alloc_hook(s, NULL, flags, 1, &p); 2459} 2460 2461#else /* CONFIG_MEMCG */ 2462static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s, 2463 struct list_lru *lru, 2464 gfp_t flags, size_t size, 2465 void **p) 2466{ 2467 return true; 2468} 2469 2470static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, 2471 void **p, int objects) 2472{ 2473} 2474 2475static inline bool memcg_slab_post_charge(void *p, gfp_t flags) 2476{ 2477 return true; 2478} 2479#endif /* CONFIG_MEMCG */ 2480 2481#ifdef CONFIG_SLUB_RCU_DEBUG 2482static void slab_free_after_rcu_debug(struct rcu_head *rcu_head); 2483 2484struct rcu_delayed_free { 2485 struct rcu_head head; 2486 void *object; 2487}; 2488#endif 2489 2490/* 2491 * Hooks for other subsystems that check memory allocations. In a typical 2492 * production configuration these hooks all should produce no code at all. 2493 * 2494 * Returns true if freeing of the object can proceed, false if its reuse 2495 * was delayed by CONFIG_SLUB_RCU_DEBUG or KASAN quarantine, or it was returned 2496 * to KFENCE. 2497 */ 2498static __always_inline 2499bool slab_free_hook(struct kmem_cache *s, void *x, bool init, 2500 bool after_rcu_delay) 2501{ 2502 /* Are the object contents still accessible? */ 2503 bool still_accessible = (s->flags & SLAB_TYPESAFE_BY_RCU) && !after_rcu_delay; 2504 2505 kmemleak_free_recursive(x, s->flags); 2506 kmsan_slab_free(s, x); 2507 2508 debug_check_no_locks_freed(x, s->object_size); 2509 2510 if (!(s->flags & SLAB_DEBUG_OBJECTS)) 2511 debug_check_no_obj_freed(x, s->object_size); 2512 2513 /* Use KCSAN to help debug racy use-after-free. */ 2514 if (!still_accessible) 2515 __kcsan_check_access(x, s->object_size, 2516 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT); 2517 2518 if (kfence_free(x)) 2519 return false; 2520 2521 /* 2522 * Give KASAN a chance to notice an invalid free operation before we 2523 * modify the object. 2524 */ 2525 if (kasan_slab_pre_free(s, x)) 2526 return false; 2527 2528#ifdef CONFIG_SLUB_RCU_DEBUG 2529 if (still_accessible) { 2530 struct rcu_delayed_free *delayed_free; 2531 2532 delayed_free = kmalloc(sizeof(*delayed_free), GFP_NOWAIT); 2533 if (delayed_free) { 2534 /* 2535 * Let KASAN track our call stack as a "related work 2536 * creation", just like if the object had been freed 2537 * normally via kfree_rcu(). 2538 * We have to do this manually because the rcu_head is 2539 * not located inside the object. 2540 */ 2541 kasan_record_aux_stack(x); 2542 2543 delayed_free->object = x; 2544 call_rcu(&delayed_free->head, slab_free_after_rcu_debug); 2545 return false; 2546 } 2547 } 2548#endif /* CONFIG_SLUB_RCU_DEBUG */ 2549 2550 /* 2551 * As memory initialization might be integrated into KASAN, 2552 * kasan_slab_free and initialization memset's must be 2553 * kept together to avoid discrepancies in behavior. 2554 * 2555 * The initialization memset's clear the object and the metadata, 2556 * but don't touch the SLAB redzone. 2557 * 2558 * The object's freepointer is also avoided if stored outside the 2559 * object. 2560 */ 2561 if (unlikely(init)) { 2562 int rsize; 2563 unsigned int inuse, orig_size; 2564 2565 inuse = get_info_end(s); 2566 orig_size = get_orig_size(s, x); 2567 if (!kasan_has_integrated_init()) 2568 memset(kasan_reset_tag(x), 0, orig_size); 2569 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0; 2570 memset((char *)kasan_reset_tag(x) + inuse, 0, 2571 s->size - inuse - rsize); 2572 /* 2573 * Restore orig_size, otherwise kmalloc redzone overwritten 2574 * would be reported 2575 */ 2576 set_orig_size(s, x, orig_size); 2577 2578 } 2579 /* KASAN might put x into memory quarantine, delaying its reuse. */ 2580 return !kasan_slab_free(s, x, init, still_accessible, false); 2581} 2582 2583static __fastpath_inline 2584bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail, 2585 int *cnt) 2586{ 2587 2588 void *object; 2589 void *next = *head; 2590 void *old_tail = *tail; 2591 bool init; 2592 2593 if (is_kfence_address(next)) { 2594 slab_free_hook(s, next, false, false); 2595 return false; 2596 } 2597 2598 /* Head and tail of the reconstructed freelist */ 2599 *head = NULL; 2600 *tail = NULL; 2601 2602 init = slab_want_init_on_free(s); 2603 2604 do { 2605 object = next; 2606 next = get_freepointer(s, object); 2607 2608 /* If object's reuse doesn't have to be delayed */ 2609 if (likely(slab_free_hook(s, object, init, false))) { 2610 /* Move object to the new freelist */ 2611 set_freepointer(s, object, *head); 2612 *head = object; 2613 if (!*tail) 2614 *tail = object; 2615 } else { 2616 /* 2617 * Adjust the reconstructed freelist depth 2618 * accordingly if object's reuse is delayed. 2619 */ 2620 --(*cnt); 2621 } 2622 } while (object != old_tail); 2623 2624 return *head != NULL; 2625} 2626 2627static void *setup_object(struct kmem_cache *s, void *object) 2628{ 2629 setup_object_debug(s, object); 2630 object = kasan_init_slab_obj(s, object); 2631 if (unlikely(s->ctor)) { 2632 kasan_unpoison_new_object(s, object); 2633 s->ctor(object); 2634 kasan_poison_new_object(s, object); 2635 } 2636 return object; 2637} 2638 2639static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp) 2640{ 2641 struct slab_sheaf *sheaf; 2642 size_t sheaf_size; 2643 2644 if (gfp & __GFP_NO_OBJ_EXT) 2645 return NULL; 2646 2647 gfp &= ~OBJCGS_CLEAR_MASK; 2648 2649 /* 2650 * Prevent recursion to the same cache, or a deep stack of kmallocs of 2651 * varying sizes (sheaf capacity might differ for each kmalloc size 2652 * bucket) 2653 */ 2654 if (s->flags & SLAB_KMALLOC) 2655 gfp |= __GFP_NO_OBJ_EXT; 2656 2657 sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity); 2658 sheaf = kzalloc(sheaf_size, gfp); 2659 2660 if (unlikely(!sheaf)) 2661 return NULL; 2662 2663 sheaf->cache = s; 2664 2665 stat(s, SHEAF_ALLOC); 2666 2667 return sheaf; 2668} 2669 2670static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf) 2671{ 2672 kfree(sheaf); 2673 2674 stat(s, SHEAF_FREE); 2675} 2676 2677static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, 2678 size_t size, void **p); 2679 2680 2681static int refill_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf, 2682 gfp_t gfp) 2683{ 2684 int to_fill = s->sheaf_capacity - sheaf->size; 2685 int filled; 2686 2687 if (!to_fill) 2688 return 0; 2689 2690 filled = __kmem_cache_alloc_bulk(s, gfp, to_fill, 2691 &sheaf->objects[sheaf->size]); 2692 2693 sheaf->size += filled; 2694 2695 stat_add(s, SHEAF_REFILL, filled); 2696 2697 if (filled < to_fill) 2698 return -ENOMEM; 2699 2700 return 0; 2701} 2702 2703 2704static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp) 2705{ 2706 struct slab_sheaf *sheaf = alloc_empty_sheaf(s, gfp); 2707 2708 if (!sheaf) 2709 return NULL; 2710 2711 if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) { 2712 free_empty_sheaf(s, sheaf); 2713 return NULL; 2714 } 2715 2716 return sheaf; 2717} 2718 2719/* 2720 * Maximum number of objects freed during a single flush of main pcs sheaf. 2721 * Translates directly to an on-stack array size. 2722 */ 2723#define PCS_BATCH_MAX 32U 2724 2725static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); 2726 2727/* 2728 * Free all objects from the main sheaf. In order to perform 2729 * __kmem_cache_free_bulk() outside of cpu_sheaves->lock, work in batches where 2730 * object pointers are moved to a on-stack array under the lock. To bound the 2731 * stack usage, limit each batch to PCS_BATCH_MAX. 2732 * 2733 * Must be called with s->cpu_sheaves->lock locked, returns with the lock 2734 * unlocked. 2735 * 2736 * Returns how many objects are remaining to be flushed 2737 */ 2738static unsigned int __sheaf_flush_main_batch(struct kmem_cache *s) 2739{ 2740 struct slub_percpu_sheaves *pcs; 2741 unsigned int batch, remaining; 2742 void *objects[PCS_BATCH_MAX]; 2743 struct slab_sheaf *sheaf; 2744 2745 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 2746 2747 pcs = this_cpu_ptr(s->cpu_sheaves); 2748 sheaf = pcs->main; 2749 2750 batch = min(PCS_BATCH_MAX, sheaf->size); 2751 2752 sheaf->size -= batch; 2753 memcpy(objects, sheaf->objects + sheaf->size, batch * sizeof(void *)); 2754 2755 remaining = sheaf->size; 2756 2757 local_unlock(&s->cpu_sheaves->lock); 2758 2759 __kmem_cache_free_bulk(s, batch, &objects[0]); 2760 2761 stat_add(s, SHEAF_FLUSH, batch); 2762 2763 return remaining; 2764} 2765 2766static void sheaf_flush_main(struct kmem_cache *s) 2767{ 2768 unsigned int remaining; 2769 2770 do { 2771 local_lock(&s->cpu_sheaves->lock); 2772 2773 remaining = __sheaf_flush_main_batch(s); 2774 2775 } while (remaining); 2776} 2777 2778/* 2779 * Returns true if the main sheaf was at least partially flushed. 2780 */ 2781static bool sheaf_try_flush_main(struct kmem_cache *s) 2782{ 2783 unsigned int remaining; 2784 bool ret = false; 2785 2786 do { 2787 if (!local_trylock(&s->cpu_sheaves->lock)) 2788 return ret; 2789 2790 ret = true; 2791 remaining = __sheaf_flush_main_batch(s); 2792 2793 } while (remaining); 2794 2795 return ret; 2796} 2797 2798/* 2799 * Free all objects from a sheaf that's unused, i.e. not linked to any 2800 * cpu_sheaves, so we need no locking and batching. The locking is also not 2801 * necessary when flushing cpu's sheaves (both spare and main) during cpu 2802 * hotremove as the cpu is not executing anymore. 2803 */ 2804static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf) 2805{ 2806 if (!sheaf->size) 2807 return; 2808 2809 stat_add(s, SHEAF_FLUSH, sheaf->size); 2810 2811 __kmem_cache_free_bulk(s, sheaf->size, &sheaf->objects[0]); 2812 2813 sheaf->size = 0; 2814} 2815 2816static bool __rcu_free_sheaf_prepare(struct kmem_cache *s, 2817 struct slab_sheaf *sheaf) 2818{ 2819 bool init = slab_want_init_on_free(s); 2820 void **p = &sheaf->objects[0]; 2821 unsigned int i = 0; 2822 bool pfmemalloc = false; 2823 2824 while (i < sheaf->size) { 2825 struct slab *slab = virt_to_slab(p[i]); 2826 2827 memcg_slab_free_hook(s, slab, p + i, 1); 2828 alloc_tagging_slab_free_hook(s, slab, p + i, 1); 2829 2830 if (unlikely(!slab_free_hook(s, p[i], init, true))) { 2831 p[i] = p[--sheaf->size]; 2832 continue; 2833 } 2834 2835 if (slab_test_pfmemalloc(slab)) 2836 pfmemalloc = true; 2837 2838 i++; 2839 } 2840 2841 return pfmemalloc; 2842} 2843 2844static void rcu_free_sheaf_nobarn(struct rcu_head *head) 2845{ 2846 struct slab_sheaf *sheaf; 2847 struct kmem_cache *s; 2848 2849 sheaf = container_of(head, struct slab_sheaf, rcu_head); 2850 s = sheaf->cache; 2851 2852 __rcu_free_sheaf_prepare(s, sheaf); 2853 2854 sheaf_flush_unused(s, sheaf); 2855 2856 free_empty_sheaf(s, sheaf); 2857} 2858 2859/* 2860 * Caller needs to make sure migration is disabled in order to fully flush 2861 * single cpu's sheaves 2862 * 2863 * must not be called from an irq 2864 * 2865 * flushing operations are rare so let's keep it simple and flush to slabs 2866 * directly, skipping the barn 2867 */ 2868static void pcs_flush_all(struct kmem_cache *s) 2869{ 2870 struct slub_percpu_sheaves *pcs; 2871 struct slab_sheaf *spare, *rcu_free; 2872 2873 local_lock(&s->cpu_sheaves->lock); 2874 pcs = this_cpu_ptr(s->cpu_sheaves); 2875 2876 spare = pcs->spare; 2877 pcs->spare = NULL; 2878 2879 rcu_free = pcs->rcu_free; 2880 pcs->rcu_free = NULL; 2881 2882 local_unlock(&s->cpu_sheaves->lock); 2883 2884 if (spare) { 2885 sheaf_flush_unused(s, spare); 2886 free_empty_sheaf(s, spare); 2887 } 2888 2889 if (rcu_free) 2890 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn); 2891 2892 sheaf_flush_main(s); 2893} 2894 2895static void __pcs_flush_all_cpu(struct kmem_cache *s, unsigned int cpu) 2896{ 2897 struct slub_percpu_sheaves *pcs; 2898 2899 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 2900 2901 /* The cpu is not executing anymore so we don't need pcs->lock */ 2902 sheaf_flush_unused(s, pcs->main); 2903 if (pcs->spare) { 2904 sheaf_flush_unused(s, pcs->spare); 2905 free_empty_sheaf(s, pcs->spare); 2906 pcs->spare = NULL; 2907 } 2908 2909 if (pcs->rcu_free) { 2910 call_rcu(&pcs->rcu_free->rcu_head, rcu_free_sheaf_nobarn); 2911 pcs->rcu_free = NULL; 2912 } 2913} 2914 2915static void pcs_destroy(struct kmem_cache *s) 2916{ 2917 int cpu; 2918 2919 for_each_possible_cpu(cpu) { 2920 struct slub_percpu_sheaves *pcs; 2921 2922 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 2923 2924 /* can happen when unwinding failed create */ 2925 if (!pcs->main) 2926 continue; 2927 2928 /* 2929 * We have already passed __kmem_cache_shutdown() so everything 2930 * was flushed and there should be no objects allocated from 2931 * slabs, otherwise kmem_cache_destroy() would have aborted. 2932 * Therefore something would have to be really wrong if the 2933 * warnings here trigger, and we should rather leave objects and 2934 * sheaves to leak in that case. 2935 */ 2936 2937 WARN_ON(pcs->spare); 2938 WARN_ON(pcs->rcu_free); 2939 2940 if (!WARN_ON(pcs->main->size)) { 2941 free_empty_sheaf(s, pcs->main); 2942 pcs->main = NULL; 2943 } 2944 } 2945 2946 free_percpu(s->cpu_sheaves); 2947 s->cpu_sheaves = NULL; 2948} 2949 2950static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn) 2951{ 2952 struct slab_sheaf *empty = NULL; 2953 unsigned long flags; 2954 2955 if (!data_race(barn->nr_empty)) 2956 return NULL; 2957 2958 spin_lock_irqsave(&barn->lock, flags); 2959 2960 if (likely(barn->nr_empty)) { 2961 empty = list_first_entry(&barn->sheaves_empty, 2962 struct slab_sheaf, barn_list); 2963 list_del(&empty->barn_list); 2964 barn->nr_empty--; 2965 } 2966 2967 spin_unlock_irqrestore(&barn->lock, flags); 2968 2969 return empty; 2970} 2971 2972/* 2973 * The following two functions are used mainly in cases where we have to undo an 2974 * intended action due to a race or cpu migration. Thus they do not check the 2975 * empty or full sheaf limits for simplicity. 2976 */ 2977 2978static void barn_put_empty_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf) 2979{ 2980 unsigned long flags; 2981 2982 spin_lock_irqsave(&barn->lock, flags); 2983 2984 list_add(&sheaf->barn_list, &barn->sheaves_empty); 2985 barn->nr_empty++; 2986 2987 spin_unlock_irqrestore(&barn->lock, flags); 2988} 2989 2990static void barn_put_full_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf) 2991{ 2992 unsigned long flags; 2993 2994 spin_lock_irqsave(&barn->lock, flags); 2995 2996 list_add(&sheaf->barn_list, &barn->sheaves_full); 2997 barn->nr_full++; 2998 2999 spin_unlock_irqrestore(&barn->lock, flags); 3000} 3001 3002static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn) 3003{ 3004 struct slab_sheaf *sheaf = NULL; 3005 unsigned long flags; 3006 3007 if (!data_race(barn->nr_full) && !data_race(barn->nr_empty)) 3008 return NULL; 3009 3010 spin_lock_irqsave(&barn->lock, flags); 3011 3012 if (barn->nr_full) { 3013 sheaf = list_first_entry(&barn->sheaves_full, struct slab_sheaf, 3014 barn_list); 3015 list_del(&sheaf->barn_list); 3016 barn->nr_full--; 3017 } else if (barn->nr_empty) { 3018 sheaf = list_first_entry(&barn->sheaves_empty, 3019 struct slab_sheaf, barn_list); 3020 list_del(&sheaf->barn_list); 3021 barn->nr_empty--; 3022 } 3023 3024 spin_unlock_irqrestore(&barn->lock, flags); 3025 3026 return sheaf; 3027} 3028 3029/* 3030 * If a full sheaf is available, return it and put the supplied empty one to 3031 * barn. We ignore the limit on empty sheaves as the number of sheaves doesn't 3032 * change. 3033 */ 3034static struct slab_sheaf * 3035barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty) 3036{ 3037 struct slab_sheaf *full = NULL; 3038 unsigned long flags; 3039 3040 if (!data_race(barn->nr_full)) 3041 return NULL; 3042 3043 spin_lock_irqsave(&barn->lock, flags); 3044 3045 if (likely(barn->nr_full)) { 3046 full = list_first_entry(&barn->sheaves_full, struct slab_sheaf, 3047 barn_list); 3048 list_del(&full->barn_list); 3049 list_add(&empty->barn_list, &barn->sheaves_empty); 3050 barn->nr_full--; 3051 barn->nr_empty++; 3052 } 3053 3054 spin_unlock_irqrestore(&barn->lock, flags); 3055 3056 return full; 3057} 3058 3059/* 3060 * If an empty sheaf is available, return it and put the supplied full one to 3061 * barn. But if there are too many full sheaves, reject this with -E2BIG. 3062 */ 3063static struct slab_sheaf * 3064barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full) 3065{ 3066 struct slab_sheaf *empty; 3067 unsigned long flags; 3068 3069 /* we don't repeat this check under barn->lock as it's not critical */ 3070 if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES) 3071 return ERR_PTR(-E2BIG); 3072 if (!data_race(barn->nr_empty)) 3073 return ERR_PTR(-ENOMEM); 3074 3075 spin_lock_irqsave(&barn->lock, flags); 3076 3077 if (likely(barn->nr_empty)) { 3078 empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf, 3079 barn_list); 3080 list_del(&empty->barn_list); 3081 list_add(&full->barn_list, &barn->sheaves_full); 3082 barn->nr_empty--; 3083 barn->nr_full++; 3084 } else { 3085 empty = ERR_PTR(-ENOMEM); 3086 } 3087 3088 spin_unlock_irqrestore(&barn->lock, flags); 3089 3090 return empty; 3091} 3092 3093static void barn_init(struct node_barn *barn) 3094{ 3095 spin_lock_init(&barn->lock); 3096 INIT_LIST_HEAD(&barn->sheaves_full); 3097 INIT_LIST_HEAD(&barn->sheaves_empty); 3098 barn->nr_full = 0; 3099 barn->nr_empty = 0; 3100} 3101 3102static void barn_shrink(struct kmem_cache *s, struct node_barn *barn) 3103{ 3104 LIST_HEAD(empty_list); 3105 LIST_HEAD(full_list); 3106 struct slab_sheaf *sheaf, *sheaf2; 3107 unsigned long flags; 3108 3109 spin_lock_irqsave(&barn->lock, flags); 3110 3111 list_splice_init(&barn->sheaves_full, &full_list); 3112 barn->nr_full = 0; 3113 list_splice_init(&barn->sheaves_empty, &empty_list); 3114 barn->nr_empty = 0; 3115 3116 spin_unlock_irqrestore(&barn->lock, flags); 3117 3118 list_for_each_entry_safe(sheaf, sheaf2, &full_list, barn_list) { 3119 sheaf_flush_unused(s, sheaf); 3120 free_empty_sheaf(s, sheaf); 3121 } 3122 3123 list_for_each_entry_safe(sheaf, sheaf2, &empty_list, barn_list) 3124 free_empty_sheaf(s, sheaf); 3125} 3126 3127/* 3128 * Slab allocation and freeing 3129 */ 3130static inline struct slab *alloc_slab_page(gfp_t flags, int node, 3131 struct kmem_cache_order_objects oo, 3132 bool allow_spin) 3133{ 3134 struct page *page; 3135 struct slab *slab; 3136 unsigned int order = oo_order(oo); 3137 3138 if (unlikely(!allow_spin)) 3139 page = alloc_frozen_pages_nolock(0/* __GFP_COMP is implied */, 3140 node, order); 3141 else if (node == NUMA_NO_NODE) 3142 page = alloc_frozen_pages(flags, order); 3143 else 3144 page = __alloc_frozen_pages(flags, order, node, NULL); 3145 3146 if (!page) 3147 return NULL; 3148 3149 __SetPageSlab(page); 3150 slab = page_slab(page); 3151 if (page_is_pfmemalloc(page)) 3152 slab_set_pfmemalloc(slab); 3153 3154 return slab; 3155} 3156 3157#ifdef CONFIG_SLAB_FREELIST_RANDOM 3158/* Pre-initialize the random sequence cache */ 3159static int init_cache_random_seq(struct kmem_cache *s) 3160{ 3161 unsigned int count = oo_objects(s->oo); 3162 int err; 3163 3164 /* Bailout if already initialised */ 3165 if (s->random_seq) 3166 return 0; 3167 3168 err = cache_random_seq_create(s, count, GFP_KERNEL); 3169 if (err) { 3170 pr_err("SLUB: Unable to initialize free list for %s\n", 3171 s->name); 3172 return err; 3173 } 3174 3175 /* Transform to an offset on the set of pages */ 3176 if (s->random_seq) { 3177 unsigned int i; 3178 3179 for (i = 0; i < count; i++) 3180 s->random_seq[i] *= s->size; 3181 } 3182 return 0; 3183} 3184 3185/* Initialize each random sequence freelist per cache */ 3186static void __init init_freelist_randomization(void) 3187{ 3188 struct kmem_cache *s; 3189 3190 mutex_lock(&slab_mutex); 3191 3192 list_for_each_entry(s, &slab_caches, list) 3193 init_cache_random_seq(s); 3194 3195 mutex_unlock(&slab_mutex); 3196} 3197 3198/* Get the next entry on the pre-computed freelist randomized */ 3199static void *next_freelist_entry(struct kmem_cache *s, 3200 unsigned long *pos, void *start, 3201 unsigned long page_limit, 3202 unsigned long freelist_count) 3203{ 3204 unsigned int idx; 3205 3206 /* 3207 * If the target page allocation failed, the number of objects on the 3208 * page might be smaller than the usual size defined by the cache. 3209 */ 3210 do { 3211 idx = s->random_seq[*pos]; 3212 *pos += 1; 3213 if (*pos >= freelist_count) 3214 *pos = 0; 3215 } while (unlikely(idx >= page_limit)); 3216 3217 return (char *)start + idx; 3218} 3219 3220static DEFINE_PER_CPU(struct rnd_state, slab_rnd_state); 3221 3222/* Shuffle the single linked freelist based on a random pre-computed sequence */ 3223static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, 3224 bool allow_spin) 3225{ 3226 void *start; 3227 void *cur; 3228 void *next; 3229 unsigned long idx, pos, page_limit, freelist_count; 3230 3231 if (slab->objects < 2 || !s->random_seq) 3232 return false; 3233 3234 freelist_count = oo_objects(s->oo); 3235 if (allow_spin) { 3236 pos = get_random_u32_below(freelist_count); 3237 } else { 3238 struct rnd_state *state; 3239 3240 /* 3241 * An interrupt or NMI handler might interrupt and change 3242 * the state in the middle, but that's safe. 3243 */ 3244 state = &get_cpu_var(slab_rnd_state); 3245 pos = prandom_u32_state(state) % freelist_count; 3246 put_cpu_var(slab_rnd_state); 3247 } 3248 3249 page_limit = slab->objects * s->size; 3250 start = fixup_red_left(s, slab_address(slab)); 3251 3252 /* First entry is used as the base of the freelist */ 3253 cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count); 3254 cur = setup_object(s, cur); 3255 slab->freelist = cur; 3256 3257 for (idx = 1; idx < slab->objects; idx++) { 3258 next = next_freelist_entry(s, &pos, start, page_limit, 3259 freelist_count); 3260 next = setup_object(s, next); 3261 set_freepointer(s, cur, next); 3262 cur = next; 3263 } 3264 set_freepointer(s, cur, NULL); 3265 3266 return true; 3267} 3268#else 3269static inline int init_cache_random_seq(struct kmem_cache *s) 3270{ 3271 return 0; 3272} 3273static inline void init_freelist_randomization(void) { } 3274static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab, 3275 bool allow_spin) 3276{ 3277 return false; 3278} 3279#endif /* CONFIG_SLAB_FREELIST_RANDOM */ 3280 3281static __always_inline void account_slab(struct slab *slab, int order, 3282 struct kmem_cache *s, gfp_t gfp) 3283{ 3284 if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) 3285 alloc_slab_obj_exts(slab, s, gfp, true); 3286 3287 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 3288 PAGE_SIZE << order); 3289} 3290 3291static __always_inline void unaccount_slab(struct slab *slab, int order, 3292 struct kmem_cache *s) 3293{ 3294 /* 3295 * The slab object extensions should now be freed regardless of 3296 * whether mem_alloc_profiling_enabled() or not because profiling 3297 * might have been disabled after slab->obj_exts got allocated. 3298 */ 3299 free_slab_obj_exts(slab); 3300 3301 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s), 3302 -(PAGE_SIZE << order)); 3303} 3304 3305static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) 3306{ 3307 bool allow_spin = gfpflags_allow_spinning(flags); 3308 struct slab *slab; 3309 struct kmem_cache_order_objects oo = s->oo; 3310 gfp_t alloc_gfp; 3311 void *start, *p, *next; 3312 int idx; 3313 bool shuffle; 3314 3315 flags &= gfp_allowed_mask; 3316 3317 flags |= s->allocflags; 3318 3319 /* 3320 * Let the initial higher-order allocation fail under memory pressure 3321 * so we fall-back to the minimum order allocation. 3322 */ 3323 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; 3324 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) 3325 alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; 3326 3327 /* 3328 * __GFP_RECLAIM could be cleared on the first allocation attempt, 3329 * so pass allow_spin flag directly. 3330 */ 3331 slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin); 3332 if (unlikely(!slab)) { 3333 oo = s->min; 3334 alloc_gfp = flags; 3335 /* 3336 * Allocation may have failed due to fragmentation. 3337 * Try a lower order alloc if possible 3338 */ 3339 slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin); 3340 if (unlikely(!slab)) 3341 return NULL; 3342 stat(s, ORDER_FALLBACK); 3343 } 3344 3345 slab->objects = oo_objects(oo); 3346 slab->inuse = 0; 3347 slab->frozen = 0; 3348 init_slab_obj_exts(slab); 3349 3350 account_slab(slab, oo_order(oo), s, flags); 3351 3352 slab->slab_cache = s; 3353 3354 kasan_poison_slab(slab); 3355 3356 start = slab_address(slab); 3357 3358 setup_slab_debug(s, slab, start); 3359 3360 shuffle = shuffle_freelist(s, slab, allow_spin); 3361 3362 if (!shuffle) { 3363 start = fixup_red_left(s, start); 3364 start = setup_object(s, start); 3365 slab->freelist = start; 3366 for (idx = 0, p = start; idx < slab->objects - 1; idx++) { 3367 next = p + s->size; 3368 next = setup_object(s, next); 3369 set_freepointer(s, p, next); 3370 p = next; 3371 } 3372 set_freepointer(s, p, NULL); 3373 } 3374 3375 return slab; 3376} 3377 3378static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node) 3379{ 3380 if (unlikely(flags & GFP_SLAB_BUG_MASK)) 3381 flags = kmalloc_fix_flags(flags); 3382 3383 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO)); 3384 3385 return allocate_slab(s, 3386 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); 3387} 3388 3389static void __free_slab(struct kmem_cache *s, struct slab *slab) 3390{ 3391 struct page *page = slab_page(slab); 3392 int order = compound_order(page); 3393 int pages = 1 << order; 3394 3395 __slab_clear_pfmemalloc(slab); 3396 page->mapping = NULL; 3397 __ClearPageSlab(page); 3398 mm_account_reclaimed_pages(pages); 3399 unaccount_slab(slab, order, s); 3400 free_frozen_pages(page, order); 3401} 3402 3403static void rcu_free_slab(struct rcu_head *h) 3404{ 3405 struct slab *slab = container_of(h, struct slab, rcu_head); 3406 3407 __free_slab(slab->slab_cache, slab); 3408} 3409 3410static void free_slab(struct kmem_cache *s, struct slab *slab) 3411{ 3412 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) { 3413 void *p; 3414 3415 slab_pad_check(s, slab); 3416 for_each_object(p, s, slab_address(slab), slab->objects) 3417 check_object(s, slab, p, SLUB_RED_INACTIVE); 3418 } 3419 3420 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) 3421 call_rcu(&slab->rcu_head, rcu_free_slab); 3422 else 3423 __free_slab(s, slab); 3424} 3425 3426static void discard_slab(struct kmem_cache *s, struct slab *slab) 3427{ 3428 dec_slabs_node(s, slab_nid(slab), slab->objects); 3429 free_slab(s, slab); 3430} 3431 3432static inline bool slab_test_node_partial(const struct slab *slab) 3433{ 3434 return test_bit(SL_partial, &slab->flags.f); 3435} 3436 3437static inline void slab_set_node_partial(struct slab *slab) 3438{ 3439 set_bit(SL_partial, &slab->flags.f); 3440} 3441 3442static inline void slab_clear_node_partial(struct slab *slab) 3443{ 3444 clear_bit(SL_partial, &slab->flags.f); 3445} 3446 3447/* 3448 * Management of partially allocated slabs. 3449 */ 3450static inline void 3451__add_partial(struct kmem_cache_node *n, struct slab *slab, int tail) 3452{ 3453 n->nr_partial++; 3454 if (tail == DEACTIVATE_TO_TAIL) 3455 list_add_tail(&slab->slab_list, &n->partial); 3456 else 3457 list_add(&slab->slab_list, &n->partial); 3458 slab_set_node_partial(slab); 3459} 3460 3461static inline void add_partial(struct kmem_cache_node *n, 3462 struct slab *slab, int tail) 3463{ 3464 lockdep_assert_held(&n->list_lock); 3465 __add_partial(n, slab, tail); 3466} 3467 3468static inline void remove_partial(struct kmem_cache_node *n, 3469 struct slab *slab) 3470{ 3471 lockdep_assert_held(&n->list_lock); 3472 list_del(&slab->slab_list); 3473 slab_clear_node_partial(slab); 3474 n->nr_partial--; 3475} 3476 3477/* 3478 * Called only for kmem_cache_debug() caches instead of remove_partial(), with a 3479 * slab from the n->partial list. Remove only a single object from the slab, do 3480 * the alloc_debug_processing() checks and leave the slab on the list, or move 3481 * it to full list if it was the last free object. 3482 */ 3483static void *alloc_single_from_partial(struct kmem_cache *s, 3484 struct kmem_cache_node *n, struct slab *slab, int orig_size) 3485{ 3486 void *object; 3487 3488 lockdep_assert_held(&n->list_lock); 3489 3490#ifdef CONFIG_SLUB_DEBUG 3491 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 3492 if (!validate_slab_ptr(slab)) { 3493 slab_err(s, slab, "Not a valid slab page"); 3494 return NULL; 3495 } 3496 } 3497#endif 3498 3499 object = slab->freelist; 3500 slab->freelist = get_freepointer(s, object); 3501 slab->inuse++; 3502 3503 if (!alloc_debug_processing(s, slab, object, orig_size)) { 3504 remove_partial(n, slab); 3505 return NULL; 3506 } 3507 3508 if (slab->inuse == slab->objects) { 3509 remove_partial(n, slab); 3510 add_full(s, n, slab); 3511 } 3512 3513 return object; 3514} 3515 3516static void defer_deactivate_slab(struct slab *slab, void *flush_freelist); 3517 3518/* 3519 * Called only for kmem_cache_debug() caches to allocate from a freshly 3520 * allocated slab. Allocate a single object instead of whole freelist 3521 * and put the slab to the partial (or full) list. 3522 */ 3523static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab, 3524 int orig_size, gfp_t gfpflags) 3525{ 3526 bool allow_spin = gfpflags_allow_spinning(gfpflags); 3527 int nid = slab_nid(slab); 3528 struct kmem_cache_node *n = get_node(s, nid); 3529 unsigned long flags; 3530 void *object; 3531 3532 if (!allow_spin && !spin_trylock_irqsave(&n->list_lock, flags)) { 3533 /* Unlucky, discard newly allocated slab */ 3534 defer_deactivate_slab(slab, NULL); 3535 return NULL; 3536 } 3537 3538 object = slab->freelist; 3539 slab->freelist = get_freepointer(s, object); 3540 slab->inuse = 1; 3541 3542 if (!alloc_debug_processing(s, slab, object, orig_size)) { 3543 /* 3544 * It's not really expected that this would fail on a 3545 * freshly allocated slab, but a concurrent memory 3546 * corruption in theory could cause that. 3547 * Leak memory of allocated slab. 3548 */ 3549 if (!allow_spin) 3550 spin_unlock_irqrestore(&n->list_lock, flags); 3551 return NULL; 3552 } 3553 3554 if (allow_spin) 3555 spin_lock_irqsave(&n->list_lock, flags); 3556 3557 if (slab->inuse == slab->objects) 3558 add_full(s, n, slab); 3559 else 3560 add_partial(n, slab, DEACTIVATE_TO_HEAD); 3561 3562 inc_slabs_node(s, nid, slab->objects); 3563 spin_unlock_irqrestore(&n->list_lock, flags); 3564 3565 return object; 3566} 3567 3568#ifdef CONFIG_SLUB_CPU_PARTIAL 3569static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain); 3570#else 3571static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab, 3572 int drain) { } 3573#endif 3574static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags); 3575 3576/* 3577 * Try to allocate a partial slab from a specific node. 3578 */ 3579static struct slab *get_partial_node(struct kmem_cache *s, 3580 struct kmem_cache_node *n, 3581 struct partial_context *pc) 3582{ 3583 struct slab *slab, *slab2, *partial = NULL; 3584 unsigned long flags; 3585 unsigned int partial_slabs = 0; 3586 3587 /* 3588 * Racy check. If we mistakenly see no partial slabs then we 3589 * just allocate an empty slab. If we mistakenly try to get a 3590 * partial slab and there is none available then get_partial() 3591 * will return NULL. 3592 */ 3593 if (!n || !n->nr_partial) 3594 return NULL; 3595 3596 if (gfpflags_allow_spinning(pc->flags)) 3597 spin_lock_irqsave(&n->list_lock, flags); 3598 else if (!spin_trylock_irqsave(&n->list_lock, flags)) 3599 return NULL; 3600 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) { 3601 if (!pfmemalloc_match(slab, pc->flags)) 3602 continue; 3603 3604 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 3605 void *object = alloc_single_from_partial(s, n, slab, 3606 pc->orig_size); 3607 if (object) { 3608 partial = slab; 3609 pc->object = object; 3610 break; 3611 } 3612 continue; 3613 } 3614 3615 remove_partial(n, slab); 3616 3617 if (!partial) { 3618 partial = slab; 3619 stat(s, ALLOC_FROM_PARTIAL); 3620 3621 if ((slub_get_cpu_partial(s) == 0)) { 3622 break; 3623 } 3624 } else { 3625 put_cpu_partial(s, slab, 0); 3626 stat(s, CPU_PARTIAL_NODE); 3627 3628 if (++partial_slabs > slub_get_cpu_partial(s) / 2) { 3629 break; 3630 } 3631 } 3632 } 3633 spin_unlock_irqrestore(&n->list_lock, flags); 3634 return partial; 3635} 3636 3637/* 3638 * Get a slab from somewhere. Search in increasing NUMA distances. 3639 */ 3640static struct slab *get_any_partial(struct kmem_cache *s, 3641 struct partial_context *pc) 3642{ 3643#ifdef CONFIG_NUMA 3644 struct zonelist *zonelist; 3645 struct zoneref *z; 3646 struct zone *zone; 3647 enum zone_type highest_zoneidx = gfp_zone(pc->flags); 3648 struct slab *slab; 3649 unsigned int cpuset_mems_cookie; 3650 bool allow_spin = gfpflags_allow_spinning(pc->flags); 3651 3652 /* 3653 * The defrag ratio allows a configuration of the tradeoffs between 3654 * inter node defragmentation and node local allocations. A lower 3655 * defrag_ratio increases the tendency to do local allocations 3656 * instead of attempting to obtain partial slabs from other nodes. 3657 * 3658 * If the defrag_ratio is set to 0 then kmalloc() always 3659 * returns node local objects. If the ratio is higher then kmalloc() 3660 * may return off node objects because partial slabs are obtained 3661 * from other nodes and filled up. 3662 * 3663 * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100 3664 * (which makes defrag_ratio = 1000) then every (well almost) 3665 * allocation will first attempt to defrag slab caches on other nodes. 3666 * This means scanning over all nodes to look for partial slabs which 3667 * may be expensive if we do it every time we are trying to find a slab 3668 * with available objects. 3669 */ 3670 if (!s->remote_node_defrag_ratio || 3671 get_cycles() % 1024 > s->remote_node_defrag_ratio) 3672 return NULL; 3673 3674 do { 3675 /* 3676 * read_mems_allowed_begin() accesses current->mems_allowed_seq, 3677 * a seqcount_spinlock_t that is not NMI-safe. Do not access 3678 * current->mems_allowed_seq and avoid retry when GFP flags 3679 * indicate spinning is not allowed. 3680 */ 3681 if (allow_spin) 3682 cpuset_mems_cookie = read_mems_allowed_begin(); 3683 3684 zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); 3685 for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { 3686 struct kmem_cache_node *n; 3687 3688 n = get_node(s, zone_to_nid(zone)); 3689 3690 if (n && cpuset_zone_allowed(zone, pc->flags) && 3691 n->nr_partial > s->min_partial) { 3692 slab = get_partial_node(s, n, pc); 3693 if (slab) { 3694 /* 3695 * Don't check read_mems_allowed_retry() 3696 * here - if mems_allowed was updated in 3697 * parallel, that was a harmless race 3698 * between allocation and the cpuset 3699 * update 3700 */ 3701 return slab; 3702 } 3703 } 3704 } 3705 } while (allow_spin && read_mems_allowed_retry(cpuset_mems_cookie)); 3706#endif /* CONFIG_NUMA */ 3707 return NULL; 3708} 3709 3710/* 3711 * Get a partial slab, lock it and return it. 3712 */ 3713static struct slab *get_partial(struct kmem_cache *s, int node, 3714 struct partial_context *pc) 3715{ 3716 struct slab *slab; 3717 int searchnode = node; 3718 3719 if (node == NUMA_NO_NODE) 3720 searchnode = numa_mem_id(); 3721 3722 slab = get_partial_node(s, get_node(s, searchnode), pc); 3723 if (slab || (node != NUMA_NO_NODE && (pc->flags & __GFP_THISNODE))) 3724 return slab; 3725 3726 return get_any_partial(s, pc); 3727} 3728 3729#ifdef CONFIG_PREEMPTION 3730/* 3731 * Calculate the next globally unique transaction for disambiguation 3732 * during cmpxchg. The transactions start with the cpu number and are then 3733 * incremented by CONFIG_NR_CPUS. 3734 */ 3735#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) 3736#else 3737/* 3738 * No preemption supported therefore also no need to check for 3739 * different cpus. 3740 */ 3741#define TID_STEP 1 3742#endif /* CONFIG_PREEMPTION */ 3743 3744static inline unsigned long next_tid(unsigned long tid) 3745{ 3746 return tid + TID_STEP; 3747} 3748 3749#ifdef SLUB_DEBUG_CMPXCHG 3750static inline unsigned int tid_to_cpu(unsigned long tid) 3751{ 3752 return tid % TID_STEP; 3753} 3754 3755static inline unsigned long tid_to_event(unsigned long tid) 3756{ 3757 return tid / TID_STEP; 3758} 3759#endif 3760 3761static inline unsigned int init_tid(int cpu) 3762{ 3763 return cpu; 3764} 3765 3766static inline void note_cmpxchg_failure(const char *n, 3767 const struct kmem_cache *s, unsigned long tid) 3768{ 3769#ifdef SLUB_DEBUG_CMPXCHG 3770 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); 3771 3772 pr_info("%s %s: cmpxchg redo ", n, s->name); 3773 3774 if (IS_ENABLED(CONFIG_PREEMPTION) && 3775 tid_to_cpu(tid) != tid_to_cpu(actual_tid)) { 3776 pr_warn("due to cpu change %d -> %d\n", 3777 tid_to_cpu(tid), tid_to_cpu(actual_tid)); 3778 } else if (tid_to_event(tid) != tid_to_event(actual_tid)) { 3779 pr_warn("due to cpu running other code. Event %ld->%ld\n", 3780 tid_to_event(tid), tid_to_event(actual_tid)); 3781 } else { 3782 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", 3783 actual_tid, tid, next_tid(tid)); 3784 } 3785#endif 3786 stat(s, CMPXCHG_DOUBLE_CPU_FAIL); 3787} 3788 3789static void init_kmem_cache_cpus(struct kmem_cache *s) 3790{ 3791#ifdef CONFIG_PREEMPT_RT 3792 /* 3793 * Register lockdep key for non-boot kmem caches to avoid 3794 * WARN_ON_ONCE(static_obj(key))) in lockdep_register_key() 3795 */ 3796 bool finegrain_lockdep = !init_section_contains(s, 1); 3797#else 3798 /* 3799 * Don't bother with different lockdep classes for each 3800 * kmem_cache, since we only use local_trylock_irqsave(). 3801 */ 3802 bool finegrain_lockdep = false; 3803#endif 3804 int cpu; 3805 struct kmem_cache_cpu *c; 3806 3807 if (finegrain_lockdep) 3808 lockdep_register_key(&s->lock_key); 3809 for_each_possible_cpu(cpu) { 3810 c = per_cpu_ptr(s->cpu_slab, cpu); 3811 local_trylock_init(&c->lock); 3812 if (finegrain_lockdep) 3813 lockdep_set_class(&c->lock, &s->lock_key); 3814 c->tid = init_tid(cpu); 3815 } 3816} 3817 3818/* 3819 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist, 3820 * unfreezes the slabs and puts it on the proper list. 3821 * Assumes the slab has been already safely taken away from kmem_cache_cpu 3822 * by the caller. 3823 */ 3824static void deactivate_slab(struct kmem_cache *s, struct slab *slab, 3825 void *freelist) 3826{ 3827 struct kmem_cache_node *n = get_node(s, slab_nid(slab)); 3828 int free_delta = 0; 3829 void *nextfree, *freelist_iter, *freelist_tail; 3830 int tail = DEACTIVATE_TO_HEAD; 3831 unsigned long flags = 0; 3832 struct freelist_counters old, new; 3833 3834 if (READ_ONCE(slab->freelist)) { 3835 stat(s, DEACTIVATE_REMOTE_FREES); 3836 tail = DEACTIVATE_TO_TAIL; 3837 } 3838 3839 /* 3840 * Stage one: Count the objects on cpu's freelist as free_delta and 3841 * remember the last object in freelist_tail for later splicing. 3842 */ 3843 freelist_tail = NULL; 3844 freelist_iter = freelist; 3845 while (freelist_iter) { 3846 nextfree = get_freepointer(s, freelist_iter); 3847 3848 /* 3849 * If 'nextfree' is invalid, it is possible that the object at 3850 * 'freelist_iter' is already corrupted. So isolate all objects 3851 * starting at 'freelist_iter' by skipping them. 3852 */ 3853 if (freelist_corrupted(s, slab, &freelist_iter, nextfree)) 3854 break; 3855 3856 freelist_tail = freelist_iter; 3857 free_delta++; 3858 3859 freelist_iter = nextfree; 3860 } 3861 3862 /* 3863 * Stage two: Unfreeze the slab while splicing the per-cpu 3864 * freelist to the head of slab's freelist. 3865 */ 3866 do { 3867 old.freelist = READ_ONCE(slab->freelist); 3868 old.counters = READ_ONCE(slab->counters); 3869 VM_BUG_ON(!old.frozen); 3870 3871 /* Determine target state of the slab */ 3872 new.counters = old.counters; 3873 new.frozen = 0; 3874 if (freelist_tail) { 3875 new.inuse -= free_delta; 3876 set_freepointer(s, freelist_tail, old.freelist); 3877 new.freelist = freelist; 3878 } else { 3879 new.freelist = old.freelist; 3880 } 3881 } while (!slab_update_freelist(s, slab, &old, &new, "unfreezing slab")); 3882 3883 /* 3884 * Stage three: Manipulate the slab list based on the updated state. 3885 */ 3886 if (!new.inuse && n->nr_partial >= s->min_partial) { 3887 stat(s, DEACTIVATE_EMPTY); 3888 discard_slab(s, slab); 3889 stat(s, FREE_SLAB); 3890 } else if (new.freelist) { 3891 spin_lock_irqsave(&n->list_lock, flags); 3892 add_partial(n, slab, tail); 3893 spin_unlock_irqrestore(&n->list_lock, flags); 3894 stat(s, tail); 3895 } else { 3896 stat(s, DEACTIVATE_FULL); 3897 } 3898} 3899 3900/* 3901 * ___slab_alloc()'s caller is supposed to check if kmem_cache::kmem_cache_cpu::lock 3902 * can be acquired without a deadlock before invoking the function. 3903 * 3904 * Without LOCKDEP we trust the code to be correct. kmalloc_nolock() is 3905 * using local_lock_is_locked() properly before calling local_lock_cpu_slab(), 3906 * and kmalloc() is not used in an unsupported context. 3907 * 3908 * With LOCKDEP, on PREEMPT_RT lockdep does its checking in local_lock_irqsave(). 3909 * On !PREEMPT_RT we use trylock to avoid false positives in NMI, but 3910 * lockdep_assert() will catch a bug in case: 3911 * #1 3912 * kmalloc() -> ___slab_alloc() -> irqsave -> NMI -> bpf -> kmalloc_nolock() 3913 * or 3914 * #2 3915 * kmalloc() -> ___slab_alloc() -> irqsave -> tracepoint/kprobe -> bpf -> kmalloc_nolock() 3916 * 3917 * On PREEMPT_RT an invocation is not possible from IRQ-off or preempt 3918 * disabled context. The lock will always be acquired and if needed it 3919 * block and sleep until the lock is available. 3920 * #1 is possible in !PREEMPT_RT only. 3921 * #2 is possible in both with a twist that irqsave is replaced with rt_spinlock: 3922 * kmalloc() -> ___slab_alloc() -> rt_spin_lock(kmem_cache_A) -> 3923 * tracepoint/kprobe -> bpf -> kmalloc_nolock() -> rt_spin_lock(kmem_cache_B) 3924 * 3925 * local_lock_is_locked() prevents the case kmem_cache_A == kmem_cache_B 3926 */ 3927#if defined(CONFIG_PREEMPT_RT) || !defined(CONFIG_LOCKDEP) 3928#define local_lock_cpu_slab(s, flags) \ 3929 local_lock_irqsave(&(s)->cpu_slab->lock, flags) 3930#else 3931#define local_lock_cpu_slab(s, flags) \ 3932 do { \ 3933 bool __l = local_trylock_irqsave(&(s)->cpu_slab->lock, flags); \ 3934 lockdep_assert(__l); \ 3935 } while (0) 3936#endif 3937 3938#define local_unlock_cpu_slab(s, flags) \ 3939 local_unlock_irqrestore(&(s)->cpu_slab->lock, flags) 3940 3941#ifdef CONFIG_SLUB_CPU_PARTIAL 3942static void __put_partials(struct kmem_cache *s, struct slab *partial_slab) 3943{ 3944 struct kmem_cache_node *n = NULL, *n2 = NULL; 3945 struct slab *slab, *slab_to_discard = NULL; 3946 unsigned long flags = 0; 3947 3948 while (partial_slab) { 3949 slab = partial_slab; 3950 partial_slab = slab->next; 3951 3952 n2 = get_node(s, slab_nid(slab)); 3953 if (n != n2) { 3954 if (n) 3955 spin_unlock_irqrestore(&n->list_lock, flags); 3956 3957 n = n2; 3958 spin_lock_irqsave(&n->list_lock, flags); 3959 } 3960 3961 if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) { 3962 slab->next = slab_to_discard; 3963 slab_to_discard = slab; 3964 } else { 3965 add_partial(n, slab, DEACTIVATE_TO_TAIL); 3966 stat(s, FREE_ADD_PARTIAL); 3967 } 3968 } 3969 3970 if (n) 3971 spin_unlock_irqrestore(&n->list_lock, flags); 3972 3973 while (slab_to_discard) { 3974 slab = slab_to_discard; 3975 slab_to_discard = slab_to_discard->next; 3976 3977 stat(s, DEACTIVATE_EMPTY); 3978 discard_slab(s, slab); 3979 stat(s, FREE_SLAB); 3980 } 3981} 3982 3983/* 3984 * Put all the cpu partial slabs to the node partial list. 3985 */ 3986static void put_partials(struct kmem_cache *s) 3987{ 3988 struct slab *partial_slab; 3989 unsigned long flags; 3990 3991 local_lock_irqsave(&s->cpu_slab->lock, flags); 3992 partial_slab = this_cpu_read(s->cpu_slab->partial); 3993 this_cpu_write(s->cpu_slab->partial, NULL); 3994 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 3995 3996 if (partial_slab) 3997 __put_partials(s, partial_slab); 3998} 3999 4000static void put_partials_cpu(struct kmem_cache *s, 4001 struct kmem_cache_cpu *c) 4002{ 4003 struct slab *partial_slab; 4004 4005 partial_slab = slub_percpu_partial(c); 4006 c->partial = NULL; 4007 4008 if (partial_slab) 4009 __put_partials(s, partial_slab); 4010} 4011 4012/* 4013 * Put a slab into a partial slab slot if available. 4014 * 4015 * If we did not find a slot then simply move all the partials to the 4016 * per node partial list. 4017 */ 4018static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain) 4019{ 4020 struct slab *oldslab; 4021 struct slab *slab_to_put = NULL; 4022 unsigned long flags; 4023 int slabs = 0; 4024 4025 local_lock_cpu_slab(s, flags); 4026 4027 oldslab = this_cpu_read(s->cpu_slab->partial); 4028 4029 if (oldslab) { 4030 if (drain && oldslab->slabs >= s->cpu_partial_slabs) { 4031 /* 4032 * Partial array is full. Move the existing set to the 4033 * per node partial list. Postpone the actual unfreezing 4034 * outside of the critical section. 4035 */ 4036 slab_to_put = oldslab; 4037 oldslab = NULL; 4038 } else { 4039 slabs = oldslab->slabs; 4040 } 4041 } 4042 4043 slabs++; 4044 4045 slab->slabs = slabs; 4046 slab->next = oldslab; 4047 4048 this_cpu_write(s->cpu_slab->partial, slab); 4049 4050 local_unlock_cpu_slab(s, flags); 4051 4052 if (slab_to_put) { 4053 __put_partials(s, slab_to_put); 4054 stat(s, CPU_PARTIAL_DRAIN); 4055 } 4056} 4057 4058#else /* CONFIG_SLUB_CPU_PARTIAL */ 4059 4060static inline void put_partials(struct kmem_cache *s) { } 4061static inline void put_partials_cpu(struct kmem_cache *s, 4062 struct kmem_cache_cpu *c) { } 4063 4064#endif /* CONFIG_SLUB_CPU_PARTIAL */ 4065 4066static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) 4067{ 4068 unsigned long flags; 4069 struct slab *slab; 4070 void *freelist; 4071 4072 local_lock_irqsave(&s->cpu_slab->lock, flags); 4073 4074 slab = c->slab; 4075 freelist = c->freelist; 4076 4077 c->slab = NULL; 4078 c->freelist = NULL; 4079 c->tid = next_tid(c->tid); 4080 4081 local_unlock_irqrestore(&s->cpu_slab->lock, flags); 4082 4083 if (slab) { 4084 deactivate_slab(s, slab, freelist); 4085 stat(s, CPUSLAB_FLUSH); 4086 } 4087} 4088 4089static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) 4090{ 4091 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 4092 void *freelist = c->freelist; 4093 struct slab *slab = c->slab; 4094 4095 c->slab = NULL; 4096 c->freelist = NULL; 4097 c->tid = next_tid(c->tid); 4098 4099 if (slab) { 4100 deactivate_slab(s, slab, freelist); 4101 stat(s, CPUSLAB_FLUSH); 4102 } 4103 4104 put_partials_cpu(s, c); 4105} 4106 4107static inline void flush_this_cpu_slab(struct kmem_cache *s) 4108{ 4109 struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); 4110 4111 if (c->slab) 4112 flush_slab(s, c); 4113 4114 put_partials(s); 4115} 4116 4117static bool has_cpu_slab(int cpu, struct kmem_cache *s) 4118{ 4119 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 4120 4121 return c->slab || slub_percpu_partial(c); 4122} 4123 4124static bool has_pcs_used(int cpu, struct kmem_cache *s) 4125{ 4126 struct slub_percpu_sheaves *pcs; 4127 4128 if (!s->cpu_sheaves) 4129 return false; 4130 4131 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 4132 4133 return (pcs->spare || pcs->rcu_free || pcs->main->size); 4134} 4135 4136/* 4137 * Flush cpu slab. 4138 * 4139 * Called from CPU work handler with migration disabled. 4140 */ 4141static void flush_cpu_slab(struct work_struct *w) 4142{ 4143 struct kmem_cache *s; 4144 struct slub_flush_work *sfw; 4145 4146 sfw = container_of(w, struct slub_flush_work, work); 4147 4148 s = sfw->s; 4149 4150 if (s->cpu_sheaves) 4151 pcs_flush_all(s); 4152 4153 flush_this_cpu_slab(s); 4154} 4155 4156static void flush_all_cpus_locked(struct kmem_cache *s) 4157{ 4158 struct slub_flush_work *sfw; 4159 unsigned int cpu; 4160 4161 lockdep_assert_cpus_held(); 4162 mutex_lock(&flush_lock); 4163 4164 for_each_online_cpu(cpu) { 4165 sfw = &per_cpu(slub_flush, cpu); 4166 if (!has_cpu_slab(cpu, s) && !has_pcs_used(cpu, s)) { 4167 sfw->skip = true; 4168 continue; 4169 } 4170 INIT_WORK(&sfw->work, flush_cpu_slab); 4171 sfw->skip = false; 4172 sfw->s = s; 4173 queue_work_on(cpu, flushwq, &sfw->work); 4174 } 4175 4176 for_each_online_cpu(cpu) { 4177 sfw = &per_cpu(slub_flush, cpu); 4178 if (sfw->skip) 4179 continue; 4180 flush_work(&sfw->work); 4181 } 4182 4183 mutex_unlock(&flush_lock); 4184} 4185 4186static void flush_all(struct kmem_cache *s) 4187{ 4188 cpus_read_lock(); 4189 flush_all_cpus_locked(s); 4190 cpus_read_unlock(); 4191} 4192 4193static void flush_rcu_sheaf(struct work_struct *w) 4194{ 4195 struct slub_percpu_sheaves *pcs; 4196 struct slab_sheaf *rcu_free; 4197 struct slub_flush_work *sfw; 4198 struct kmem_cache *s; 4199 4200 sfw = container_of(w, struct slub_flush_work, work); 4201 s = sfw->s; 4202 4203 local_lock(&s->cpu_sheaves->lock); 4204 pcs = this_cpu_ptr(s->cpu_sheaves); 4205 4206 rcu_free = pcs->rcu_free; 4207 pcs->rcu_free = NULL; 4208 4209 local_unlock(&s->cpu_sheaves->lock); 4210 4211 if (rcu_free) 4212 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn); 4213} 4214 4215 4216/* needed for kvfree_rcu_barrier() */ 4217void flush_rcu_sheaves_on_cache(struct kmem_cache *s) 4218{ 4219 struct slub_flush_work *sfw; 4220 unsigned int cpu; 4221 4222 mutex_lock(&flush_lock); 4223 4224 for_each_online_cpu(cpu) { 4225 sfw = &per_cpu(slub_flush, cpu); 4226 4227 /* 4228 * we don't check if rcu_free sheaf exists - racing 4229 * __kfree_rcu_sheaf() might have just removed it. 4230 * by executing flush_rcu_sheaf() on the cpu we make 4231 * sure the __kfree_rcu_sheaf() finished its call_rcu() 4232 */ 4233 4234 INIT_WORK(&sfw->work, flush_rcu_sheaf); 4235 sfw->s = s; 4236 queue_work_on(cpu, flushwq, &sfw->work); 4237 } 4238 4239 for_each_online_cpu(cpu) { 4240 sfw = &per_cpu(slub_flush, cpu); 4241 flush_work(&sfw->work); 4242 } 4243 4244 mutex_unlock(&flush_lock); 4245} 4246 4247void flush_all_rcu_sheaves(void) 4248{ 4249 struct kmem_cache *s; 4250 4251 cpus_read_lock(); 4252 mutex_lock(&slab_mutex); 4253 4254 list_for_each_entry(s, &slab_caches, list) { 4255 if (!s->cpu_sheaves) 4256 continue; 4257 flush_rcu_sheaves_on_cache(s); 4258 } 4259 4260 mutex_unlock(&slab_mutex); 4261 cpus_read_unlock(); 4262 4263 rcu_barrier(); 4264} 4265 4266/* 4267 * Use the cpu notifier to insure that the cpu slabs are flushed when 4268 * necessary. 4269 */ 4270static int slub_cpu_dead(unsigned int cpu) 4271{ 4272 struct kmem_cache *s; 4273 4274 mutex_lock(&slab_mutex); 4275 list_for_each_entry(s, &slab_caches, list) { 4276 __flush_cpu_slab(s, cpu); 4277 if (s->cpu_sheaves) 4278 __pcs_flush_all_cpu(s, cpu); 4279 } 4280 mutex_unlock(&slab_mutex); 4281 return 0; 4282} 4283 4284/* 4285 * Check if the objects in a per cpu structure fit numa 4286 * locality expectations. 4287 */ 4288static inline int node_match(struct slab *slab, int node) 4289{ 4290#ifdef CONFIG_NUMA 4291 if (node != NUMA_NO_NODE && slab_nid(slab) != node) 4292 return 0; 4293#endif 4294 return 1; 4295} 4296 4297#ifdef CONFIG_SLUB_DEBUG 4298static int count_free(struct slab *slab) 4299{ 4300 return slab->objects - slab->inuse; 4301} 4302 4303static inline unsigned long node_nr_objs(struct kmem_cache_node *n) 4304{ 4305 return atomic_long_read(&n->total_objects); 4306} 4307 4308/* Supports checking bulk free of a constructed freelist */ 4309static inline bool free_debug_processing(struct kmem_cache *s, 4310 struct slab *slab, void *head, void *tail, int *bulk_cnt, 4311 unsigned long addr, depot_stack_handle_t handle) 4312{ 4313 bool checks_ok = false; 4314 void *object = head; 4315 int cnt = 0; 4316 4317 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 4318 if (!check_slab(s, slab)) 4319 goto out; 4320 } 4321 4322 if (slab->inuse < *bulk_cnt) { 4323 slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n", 4324 slab->inuse, *bulk_cnt); 4325 goto out; 4326 } 4327 4328next_object: 4329 4330 if (++cnt > *bulk_cnt) 4331 goto out_cnt; 4332 4333 if (s->flags & SLAB_CONSISTENCY_CHECKS) { 4334 if (!free_consistency_checks(s, slab, object, addr)) 4335 goto out; 4336 } 4337 4338 if (s->flags & SLAB_STORE_USER) 4339 set_track_update(s, object, TRACK_FREE, addr, handle); 4340 trace(s, slab, object, 0); 4341 /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */ 4342 init_object(s, object, SLUB_RED_INACTIVE); 4343 4344 /* Reached end of constructed freelist yet? */ 4345 if (object != tail) { 4346 object = get_freepointer(s, object); 4347 goto next_object; 4348 } 4349 checks_ok = true; 4350 4351out_cnt: 4352 if (cnt != *bulk_cnt) { 4353 slab_err(s, slab, "Bulk free expected %d objects but found %d\n", 4354 *bulk_cnt, cnt); 4355 *bulk_cnt = cnt; 4356 } 4357 4358out: 4359 4360 if (!checks_ok) 4361 slab_fix(s, "Object at 0x%p not freed", object); 4362 4363 return checks_ok; 4364} 4365#endif /* CONFIG_SLUB_DEBUG */ 4366 4367#if defined(CONFIG_SLUB_DEBUG) || defined(SLAB_SUPPORTS_SYSFS) 4368static unsigned long count_partial(struct kmem_cache_node *n, 4369 int (*get_count)(struct slab *)) 4370{ 4371 unsigned long flags; 4372 unsigned long x = 0; 4373 struct slab *slab; 4374 4375 spin_lock_irqsave(&n->list_lock, flags); 4376 list_for_each_entry(slab, &n->partial, slab_list) 4377 x += get_count(slab); 4378 spin_unlock_irqrestore(&n->list_lock, flags); 4379 return x; 4380} 4381#endif /* CONFIG_SLUB_DEBUG || SLAB_SUPPORTS_SYSFS */ 4382 4383#ifdef CONFIG_SLUB_DEBUG 4384#define MAX_PARTIAL_TO_SCAN 10000 4385 4386static unsigned long count_partial_free_approx(struct kmem_cache_node *n) 4387{ 4388 unsigned long flags; 4389 unsigned long x = 0; 4390 struct slab *slab; 4391 4392 spin_lock_irqsave(&n->list_lock, flags); 4393 if (n->nr_partial <= MAX_PARTIAL_TO_SCAN) { 4394 list_for_each_entry(slab, &n->partial, slab_list) 4395 x += slab->objects - slab->inuse; 4396 } else { 4397 /* 4398 * For a long list, approximate the total count of objects in 4399 * it to meet the limit on the number of slabs to scan. 4400 * Scan from both the list's head and tail for better accuracy. 4401 */ 4402 unsigned long scanned = 0; 4403 4404 list_for_each_entry(slab, &n->partial, slab_list) { 4405 x += slab->objects - slab->inuse; 4406 if (++scanned == MAX_PARTIAL_TO_SCAN / 2) 4407 break; 4408 } 4409 list_for_each_entry_reverse(slab, &n->partial, slab_list) { 4410 x += slab->objects - slab->inuse; 4411 if (++scanned == MAX_PARTIAL_TO_SCAN) 4412 break; 4413 } 4414 x = mult_frac(x, n->nr_partial, scanned); 4415 x = min(x, node_nr_objs(n)); 4416 } 4417 spin_unlock_irqrestore(&n->list_lock, flags); 4418 return x; 4419} 4420 4421static noinline void 4422slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) 4423{ 4424 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, 4425 DEFAULT_RATELIMIT_BURST); 4426 int cpu = raw_smp_processor_id(); 4427 int node; 4428 struct kmem_cache_node *n; 4429 4430 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) 4431 return; 4432 4433 pr_warn("SLUB: Unable to allocate memory on CPU %u (of node %d) on node %d, gfp=%#x(%pGg)\n", 4434 cpu, cpu_to_node(cpu), nid, gfpflags, &gfpflags); 4435 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n", 4436 s->name, s->object_size, s->size, oo_order(s->oo), 4437 oo_order(s->min)); 4438 4439 if (oo_order(s->min) > get_order(s->object_size)) 4440 pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n", 4441 s->name); 4442 4443 for_each_kmem_cache_node(s, node, n) { 4444 unsigned long nr_slabs; 4445 unsigned long nr_objs; 4446 unsigned long nr_free; 4447 4448 nr_free = count_partial_free_approx(n); 4449 nr_slabs = node_nr_slabs(n); 4450 nr_objs = node_nr_objs(n); 4451 4452 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", 4453 node, nr_slabs, nr_objs, nr_free); 4454 } 4455} 4456#else /* CONFIG_SLUB_DEBUG */ 4457static inline void 4458slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { } 4459#endif 4460 4461static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) 4462{ 4463 if (unlikely(slab_test_pfmemalloc(slab))) 4464 return gfp_pfmemalloc_allowed(gfpflags); 4465 4466 return true; 4467} 4468 4469static inline bool 4470__update_cpu_freelist_fast(struct kmem_cache *s, 4471 void *freelist_old, void *freelist_new, 4472 unsigned long tid) 4473{ 4474 struct freelist_tid old = { .freelist = freelist_old, .tid = tid }; 4475 struct freelist_tid new = { .freelist = freelist_new, .tid = next_tid(tid) }; 4476 4477 return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid, 4478 &old.freelist_tid, new.freelist_tid); 4479} 4480 4481/* 4482 * Check the slab->freelist and either transfer the freelist to the 4483 * per cpu freelist or deactivate the slab. 4484 * 4485 * The slab is still frozen if the return value is not NULL. 4486 * 4487 * If this function returns NULL then the slab has been unfrozen. 4488 */ 4489static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) 4490{ 4491 struct freelist_counters old, new; 4492 4493 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); 4494 4495 do { 4496 old.freelist = slab->freelist; 4497 old.counters = slab->counters; 4498 4499 new.freelist = NULL; 4500 new.counters = old.counters; 4501 4502 new.inuse = old.objects; 4503 new.frozen = old.freelist != NULL; 4504 4505 4506 } while (!__slab_update_freelist(s, slab, &old, &new, "get_freelist")); 4507 4508 return old.freelist; 4509} 4510 4511/* 4512 * Freeze the partial slab and return the pointer to the freelist. 4513 */ 4514static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab) 4515{ 4516 struct freelist_counters old, new; 4517 4518 do { 4519 old.freelist = slab->freelist; 4520 old.counters = slab->counters; 4521 4522 new.freelist = NULL; 4523 new.counters = old.counters; 4524 VM_BUG_ON(new.frozen); 4525 4526 new.inuse = old.objects; 4527 new.frozen = 1; 4528 4529 } while (!slab_update_freelist(s, slab, &old, &new, "freeze_slab")); 4530 4531 return old.freelist; 4532} 4533 4534/* 4535 * Slow path. The lockless freelist is empty or we need to perform 4536 * debugging duties. 4537 * 4538 * Processing is still very fast if new objects have been freed to the 4539 * regular freelist. In that case we simply take over the regular freelist 4540 * as the lockless freelist and zap the regular freelist. 4541 * 4542 * If that is not working then we fall back to the partial lists. We take the 4543 * first element of the freelist as the object to allocate now and move the 4544 * rest of the freelist to the lockless freelist. 4545 * 4546 * And if we were unable to get a new slab from the partial slab lists then 4547 * we need to allocate a new slab. This is the slowest path since it involves 4548 * a call to the page allocator and the setup of a new slab. 4549 * 4550 * Version of __slab_alloc to use when we know that preemption is 4551 * already disabled (which is the case for bulk allocation). 4552 */ 4553static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 4554 unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) 4555{ 4556 bool allow_spin = gfpflags_allow_spinning(gfpflags); 4557 void *freelist; 4558 struct slab *slab; 4559 unsigned long flags; 4560 struct partial_context pc; 4561 bool try_thisnode = true; 4562 4563 stat(s, ALLOC_SLOWPATH); 4564 4565reread_slab: 4566 4567 slab = READ_ONCE(c->slab); 4568 if (!slab) { 4569 /* 4570 * if the node is not online or has no normal memory, just 4571 * ignore the node constraint 4572 */ 4573 if (unlikely(node != NUMA_NO_NODE && 4574 !node_isset(node, slab_nodes))) 4575 node = NUMA_NO_NODE; 4576 goto new_slab; 4577 } 4578 4579 if (unlikely(!node_match(slab, node))) { 4580 /* 4581 * same as above but node_match() being false already 4582 * implies node != NUMA_NO_NODE. 4583 * 4584 * We don't strictly honor pfmemalloc and NUMA preferences 4585 * when !allow_spin because: 4586 * 4587 * 1. Most kmalloc() users allocate objects on the local node, 4588 * so kmalloc_nolock() tries not to interfere with them by 4589 * deactivating the cpu slab. 4590 * 4591 * 2. Deactivating due to NUMA or pfmemalloc mismatch may cause 4592 * unnecessary slab allocations even when n->partial list 4593 * is not empty. 4594 */ 4595 if (!node_isset(node, slab_nodes) || 4596 !allow_spin) { 4597 node = NUMA_NO_NODE; 4598 } else { 4599 stat(s, ALLOC_NODE_MISMATCH); 4600 goto deactivate_slab; 4601 } 4602 } 4603 4604 /* 4605 * By rights, we should be searching for a slab page that was 4606 * PFMEMALLOC but right now, we are losing the pfmemalloc 4607 * information when the page leaves the per-cpu allocator 4608 */ 4609 if (unlikely(!pfmemalloc_match(slab, gfpflags) && allow_spin)) 4610 goto deactivate_slab; 4611 4612 /* must check again c->slab in case we got preempted and it changed */ 4613 local_lock_cpu_slab(s, flags); 4614 4615 if (unlikely(slab != c->slab)) { 4616 local_unlock_cpu_slab(s, flags); 4617 goto reread_slab; 4618 } 4619 freelist = c->freelist; 4620 if (freelist) 4621 goto load_freelist; 4622 4623 freelist = get_freelist(s, slab); 4624 4625 if (!freelist) { 4626 c->slab = NULL; 4627 c->tid = next_tid(c->tid); 4628 local_unlock_cpu_slab(s, flags); 4629 stat(s, DEACTIVATE_BYPASS); 4630 goto new_slab; 4631 } 4632 4633 stat(s, ALLOC_REFILL); 4634 4635load_freelist: 4636 4637 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); 4638 4639 /* 4640 * freelist is pointing to the list of objects to be used. 4641 * slab is pointing to the slab from which the objects are obtained. 4642 * That slab must be frozen for per cpu allocations to work. 4643 */ 4644 VM_BUG_ON(!c->slab->frozen); 4645 c->freelist = get_freepointer(s, freelist); 4646 c->tid = next_tid(c->tid); 4647 local_unlock_cpu_slab(s, flags); 4648 return freelist; 4649 4650deactivate_slab: 4651 4652 local_lock_cpu_slab(s, flags); 4653 if (slab != c->slab) { 4654 local_unlock_cpu_slab(s, flags); 4655 goto reread_slab; 4656 } 4657 freelist = c->freelist; 4658 c->slab = NULL; 4659 c->freelist = NULL; 4660 c->tid = next_tid(c->tid); 4661 local_unlock_cpu_slab(s, flags); 4662 deactivate_slab(s, slab, freelist); 4663 4664new_slab: 4665 4666#ifdef CONFIG_SLUB_CPU_PARTIAL 4667 while (slub_percpu_partial(c)) { 4668 local_lock_cpu_slab(s, flags); 4669 if (unlikely(c->slab)) { 4670 local_unlock_cpu_slab(s, flags); 4671 goto reread_slab; 4672 } 4673 if (unlikely(!slub_percpu_partial(c))) { 4674 local_unlock_cpu_slab(s, flags); 4675 /* we were preempted and partial list got empty */ 4676 goto new_objects; 4677 } 4678 4679 slab = slub_percpu_partial(c); 4680 slub_set_percpu_partial(c, slab); 4681 4682 if (likely(node_match(slab, node) && 4683 pfmemalloc_match(slab, gfpflags)) || 4684 !allow_spin) { 4685 c->slab = slab; 4686 freelist = get_freelist(s, slab); 4687 VM_BUG_ON(!freelist); 4688 stat(s, CPU_PARTIAL_ALLOC); 4689 goto load_freelist; 4690 } 4691 4692 local_unlock_cpu_slab(s, flags); 4693 4694 slab->next = NULL; 4695 __put_partials(s, slab); 4696 } 4697#endif 4698 4699new_objects: 4700 4701 pc.flags = gfpflags; 4702 /* 4703 * When a preferred node is indicated but no __GFP_THISNODE 4704 * 4705 * 1) try to get a partial slab from target node only by having 4706 * __GFP_THISNODE in pc.flags for get_partial() 4707 * 2) if 1) failed, try to allocate a new slab from target node with 4708 * GPF_NOWAIT | __GFP_THISNODE opportunistically 4709 * 3) if 2) failed, retry with original gfpflags which will allow 4710 * get_partial() try partial lists of other nodes before potentially 4711 * allocating new page from other nodes 4712 */ 4713 if (unlikely(node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) 4714 && try_thisnode)) { 4715 if (unlikely(!allow_spin)) 4716 /* Do not upgrade gfp to NOWAIT from more restrictive mode */ 4717 pc.flags = gfpflags | __GFP_THISNODE; 4718 else 4719 pc.flags = GFP_NOWAIT | __GFP_THISNODE; 4720 } 4721 4722 pc.orig_size = orig_size; 4723 slab = get_partial(s, node, &pc); 4724 if (slab) { 4725 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4726 freelist = pc.object; 4727 /* 4728 * For debug caches here we had to go through 4729 * alloc_single_from_partial() so just store the 4730 * tracking info and return the object. 4731 * 4732 * Due to disabled preemption we need to disallow 4733 * blocking. The flags are further adjusted by 4734 * gfp_nested_mask() in stack_depot itself. 4735 */ 4736 if (s->flags & SLAB_STORE_USER) 4737 set_track(s, freelist, TRACK_ALLOC, addr, 4738 gfpflags & ~(__GFP_DIRECT_RECLAIM)); 4739 4740 return freelist; 4741 } 4742 4743 freelist = freeze_slab(s, slab); 4744 goto retry_load_slab; 4745 } 4746 4747 slub_put_cpu_ptr(s->cpu_slab); 4748 slab = new_slab(s, pc.flags, node); 4749 c = slub_get_cpu_ptr(s->cpu_slab); 4750 4751 if (unlikely(!slab)) { 4752 if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE) 4753 && try_thisnode) { 4754 try_thisnode = false; 4755 goto new_objects; 4756 } 4757 slab_out_of_memory(s, gfpflags, node); 4758 return NULL; 4759 } 4760 4761 stat(s, ALLOC_SLAB); 4762 4763 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 4764 freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags); 4765 4766 if (unlikely(!freelist)) { 4767 /* This could cause an endless loop. Fail instead. */ 4768 if (!allow_spin) 4769 return NULL; 4770 goto new_objects; 4771 } 4772 4773 if (s->flags & SLAB_STORE_USER) 4774 set_track(s, freelist, TRACK_ALLOC, addr, 4775 gfpflags & ~(__GFP_DIRECT_RECLAIM)); 4776 4777 return freelist; 4778 } 4779 4780 /* 4781 * No other reference to the slab yet so we can 4782 * muck around with it freely without cmpxchg 4783 */ 4784 freelist = slab->freelist; 4785 slab->freelist = NULL; 4786 slab->inuse = slab->objects; 4787 slab->frozen = 1; 4788 4789 inc_slabs_node(s, slab_nid(slab), slab->objects); 4790 4791 if (unlikely(!pfmemalloc_match(slab, gfpflags) && allow_spin)) { 4792 /* 4793 * For !pfmemalloc_match() case we don't load freelist so that 4794 * we don't make further mismatched allocations easier. 4795 */ 4796 deactivate_slab(s, slab, get_freepointer(s, freelist)); 4797 return freelist; 4798 } 4799 4800retry_load_slab: 4801 4802 local_lock_cpu_slab(s, flags); 4803 if (unlikely(c->slab)) { 4804 void *flush_freelist = c->freelist; 4805 struct slab *flush_slab = c->slab; 4806 4807 c->slab = NULL; 4808 c->freelist = NULL; 4809 c->tid = next_tid(c->tid); 4810 4811 local_unlock_cpu_slab(s, flags); 4812 4813 if (unlikely(!allow_spin)) { 4814 /* Reentrant slub cannot take locks, defer */ 4815 defer_deactivate_slab(flush_slab, flush_freelist); 4816 } else { 4817 deactivate_slab(s, flush_slab, flush_freelist); 4818 } 4819 4820 stat(s, CPUSLAB_FLUSH); 4821 4822 goto retry_load_slab; 4823 } 4824 c->slab = slab; 4825 4826 goto load_freelist; 4827} 4828/* 4829 * We disallow kprobes in ___slab_alloc() to prevent reentrance 4830 * 4831 * kmalloc() -> ___slab_alloc() -> local_lock_cpu_slab() protected part of 4832 * ___slab_alloc() manipulating c->freelist -> kprobe -> bpf -> 4833 * kmalloc_nolock() or kfree_nolock() -> __update_cpu_freelist_fast() 4834 * manipulating c->freelist without lock. 4835 * 4836 * This does not prevent kprobe in functions called from ___slab_alloc() such as 4837 * local_lock_irqsave() itself, and that is fine, we only need to protect the 4838 * c->freelist manipulation in ___slab_alloc() itself. 4839 */ 4840NOKPROBE_SYMBOL(___slab_alloc); 4841 4842/* 4843 * A wrapper for ___slab_alloc() for contexts where preemption is not yet 4844 * disabled. Compensates for possible cpu changes by refetching the per cpu area 4845 * pointer. 4846 */ 4847static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 4848 unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) 4849{ 4850 void *p; 4851 4852#ifdef CONFIG_PREEMPT_COUNT 4853 /* 4854 * We may have been preempted and rescheduled on a different 4855 * cpu before disabling preemption. Need to reload cpu area 4856 * pointer. 4857 */ 4858 c = slub_get_cpu_ptr(s->cpu_slab); 4859#endif 4860 if (unlikely(!gfpflags_allow_spinning(gfpflags))) { 4861 if (local_lock_is_locked(&s->cpu_slab->lock)) { 4862 /* 4863 * EBUSY is an internal signal to kmalloc_nolock() to 4864 * retry a different bucket. It's not propagated 4865 * to the caller. 4866 */ 4867 p = ERR_PTR(-EBUSY); 4868 goto out; 4869 } 4870 } 4871 p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size); 4872out: 4873#ifdef CONFIG_PREEMPT_COUNT 4874 slub_put_cpu_ptr(s->cpu_slab); 4875#endif 4876 return p; 4877} 4878 4879static __always_inline void *__slab_alloc_node(struct kmem_cache *s, 4880 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 4881{ 4882 struct kmem_cache_cpu *c; 4883 struct slab *slab; 4884 unsigned long tid; 4885 void *object; 4886 4887redo: 4888 /* 4889 * Must read kmem_cache cpu data via this cpu ptr. Preemption is 4890 * enabled. We may switch back and forth between cpus while 4891 * reading from one cpu area. That does not matter as long 4892 * as we end up on the original cpu again when doing the cmpxchg. 4893 * 4894 * We must guarantee that tid and kmem_cache_cpu are retrieved on the 4895 * same cpu. We read first the kmem_cache_cpu pointer and use it to read 4896 * the tid. If we are preempted and switched to another cpu between the 4897 * two reads, it's OK as the two are still associated with the same cpu 4898 * and cmpxchg later will validate the cpu. 4899 */ 4900 c = raw_cpu_ptr(s->cpu_slab); 4901 tid = READ_ONCE(c->tid); 4902 4903 /* 4904 * Irqless object alloc/free algorithm used here depends on sequence 4905 * of fetching cpu_slab's data. tid should be fetched before anything 4906 * on c to guarantee that object and slab associated with previous tid 4907 * won't be used with current tid. If we fetch tid first, object and 4908 * slab could be one associated with next tid and our alloc/free 4909 * request will be failed. In this case, we will retry. So, no problem. 4910 */ 4911 barrier(); 4912 4913 /* 4914 * The transaction ids are globally unique per cpu and per operation on 4915 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double 4916 * occurs on the right processor and that there was no operation on the 4917 * linked list in between. 4918 */ 4919 4920 object = c->freelist; 4921 slab = c->slab; 4922 4923#ifdef CONFIG_NUMA 4924 if (static_branch_unlikely(&strict_numa) && 4925 node == NUMA_NO_NODE) { 4926 4927 struct mempolicy *mpol = current->mempolicy; 4928 4929 if (mpol) { 4930 /* 4931 * Special BIND rule support. If existing slab 4932 * is in permitted set then do not redirect 4933 * to a particular node. 4934 * Otherwise we apply the memory policy to get 4935 * the node we need to allocate on. 4936 */ 4937 if (mpol->mode != MPOL_BIND || !slab || 4938 !node_isset(slab_nid(slab), mpol->nodes)) 4939 4940 node = mempolicy_slab_node(); 4941 } 4942 } 4943#endif 4944 4945 if (!USE_LOCKLESS_FAST_PATH() || 4946 unlikely(!object || !slab || !node_match(slab, node))) { 4947 object = __slab_alloc(s, gfpflags, node, addr, c, orig_size); 4948 } else { 4949 void *next_object = get_freepointer_safe(s, object); 4950 4951 /* 4952 * The cmpxchg will only match if there was no additional 4953 * operation and if we are on the right processor. 4954 * 4955 * The cmpxchg does the following atomically (without lock 4956 * semantics!) 4957 * 1. Relocate first pointer to the current per cpu area. 4958 * 2. Verify that tid and freelist have not been changed 4959 * 3. If they were not changed replace tid and freelist 4960 * 4961 * Since this is without lock semantics the protection is only 4962 * against code executing on this cpu *not* from access by 4963 * other cpus. 4964 */ 4965 if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) { 4966 note_cmpxchg_failure("slab_alloc", s, tid); 4967 goto redo; 4968 } 4969 prefetch_freepointer(s, next_object); 4970 stat(s, ALLOC_FASTPATH); 4971 } 4972 4973 return object; 4974} 4975 4976/* 4977 * If the object has been wiped upon free, make sure it's fully initialized by 4978 * zeroing out freelist pointer. 4979 * 4980 * Note that we also wipe custom freelist pointers. 4981 */ 4982static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s, 4983 void *obj) 4984{ 4985 if (unlikely(slab_want_init_on_free(s)) && obj && 4986 !freeptr_outside_object(s)) 4987 memset((void *)((char *)kasan_reset_tag(obj) + s->offset), 4988 0, sizeof(void *)); 4989} 4990 4991static __fastpath_inline 4992struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) 4993{ 4994 flags &= gfp_allowed_mask; 4995 4996 might_alloc(flags); 4997 4998 if (unlikely(should_failslab(s, flags))) 4999 return NULL; 5000 5001 return s; 5002} 5003 5004static __fastpath_inline 5005bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, 5006 gfp_t flags, size_t size, void **p, bool init, 5007 unsigned int orig_size) 5008{ 5009 unsigned int zero_size = s->object_size; 5010 bool kasan_init = init; 5011 size_t i; 5012 gfp_t init_flags = flags & gfp_allowed_mask; 5013 5014 /* 5015 * For kmalloc object, the allocated memory size(object_size) is likely 5016 * larger than the requested size(orig_size). If redzone check is 5017 * enabled for the extra space, don't zero it, as it will be redzoned 5018 * soon. The redzone operation for this extra space could be seen as a 5019 * replacement of current poisoning under certain debug option, and 5020 * won't break other sanity checks. 5021 */ 5022 if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) && 5023 (s->flags & SLAB_KMALLOC)) 5024 zero_size = orig_size; 5025 5026 /* 5027 * When slab_debug is enabled, avoid memory initialization integrated 5028 * into KASAN and instead zero out the memory via the memset below with 5029 * the proper size. Otherwise, KASAN might overwrite SLUB redzones and 5030 * cause false-positive reports. This does not lead to a performance 5031 * penalty on production builds, as slab_debug is not intended to be 5032 * enabled there. 5033 */ 5034 if (__slub_debug_enabled()) 5035 kasan_init = false; 5036 5037 /* 5038 * As memory initialization might be integrated into KASAN, 5039 * kasan_slab_alloc and initialization memset must be 5040 * kept together to avoid discrepancies in behavior. 5041 * 5042 * As p[i] might get tagged, memset and kmemleak hook come after KASAN. 5043 */ 5044 for (i = 0; i < size; i++) { 5045 p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init); 5046 if (p[i] && init && (!kasan_init || 5047 !kasan_has_integrated_init())) 5048 memset(p[i], 0, zero_size); 5049 if (gfpflags_allow_spinning(flags)) 5050 kmemleak_alloc_recursive(p[i], s->object_size, 1, 5051 s->flags, init_flags); 5052 kmsan_slab_alloc(s, p[i], init_flags); 5053 alloc_tagging_slab_alloc_hook(s, p[i], flags); 5054 } 5055 5056 return memcg_slab_post_alloc_hook(s, lru, flags, size, p); 5057} 5058 5059/* 5060 * Replace the empty main sheaf with a (at least partially) full sheaf. 5061 * 5062 * Must be called with the cpu_sheaves local lock locked. If successful, returns 5063 * the pcs pointer and the local lock locked (possibly on a different cpu than 5064 * initially called). If not successful, returns NULL and the local lock 5065 * unlocked. 5066 */ 5067static struct slub_percpu_sheaves * 5068__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp) 5069{ 5070 struct slab_sheaf *empty = NULL; 5071 struct slab_sheaf *full; 5072 struct node_barn *barn; 5073 bool can_alloc; 5074 5075 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 5076 5077 if (pcs->spare && pcs->spare->size > 0) { 5078 swap(pcs->main, pcs->spare); 5079 return pcs; 5080 } 5081 5082 barn = get_barn(s); 5083 if (!barn) { 5084 local_unlock(&s->cpu_sheaves->lock); 5085 return NULL; 5086 } 5087 5088 full = barn_replace_empty_sheaf(barn, pcs->main); 5089 5090 if (full) { 5091 stat(s, BARN_GET); 5092 pcs->main = full; 5093 return pcs; 5094 } 5095 5096 stat(s, BARN_GET_FAIL); 5097 5098 can_alloc = gfpflags_allow_blocking(gfp); 5099 5100 if (can_alloc) { 5101 if (pcs->spare) { 5102 empty = pcs->spare; 5103 pcs->spare = NULL; 5104 } else { 5105 empty = barn_get_empty_sheaf(barn); 5106 } 5107 } 5108 5109 local_unlock(&s->cpu_sheaves->lock); 5110 5111 if (!can_alloc) 5112 return NULL; 5113 5114 if (empty) { 5115 if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) { 5116 full = empty; 5117 } else { 5118 /* 5119 * we must be very low on memory so don't bother 5120 * with the barn 5121 */ 5122 free_empty_sheaf(s, empty); 5123 } 5124 } else { 5125 full = alloc_full_sheaf(s, gfp); 5126 } 5127 5128 if (!full) 5129 return NULL; 5130 5131 /* 5132 * we can reach here only when gfpflags_allow_blocking 5133 * so this must not be an irq 5134 */ 5135 local_lock(&s->cpu_sheaves->lock); 5136 pcs = this_cpu_ptr(s->cpu_sheaves); 5137 5138 /* 5139 * If we are returning empty sheaf, we either got it from the 5140 * barn or had to allocate one. If we are returning a full 5141 * sheaf, it's due to racing or being migrated to a different 5142 * cpu. Breaching the barn's sheaf limits should be thus rare 5143 * enough so just ignore them to simplify the recovery. 5144 */ 5145 5146 if (pcs->main->size == 0) { 5147 barn_put_empty_sheaf(barn, pcs->main); 5148 pcs->main = full; 5149 return pcs; 5150 } 5151 5152 if (!pcs->spare) { 5153 pcs->spare = full; 5154 return pcs; 5155 } 5156 5157 if (pcs->spare->size == 0) { 5158 barn_put_empty_sheaf(barn, pcs->spare); 5159 pcs->spare = full; 5160 return pcs; 5161 } 5162 5163 barn_put_full_sheaf(barn, full); 5164 stat(s, BARN_PUT); 5165 5166 return pcs; 5167} 5168 5169static __fastpath_inline 5170void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node) 5171{ 5172 struct slub_percpu_sheaves *pcs; 5173 bool node_requested; 5174 void *object; 5175 5176#ifdef CONFIG_NUMA 5177 if (static_branch_unlikely(&strict_numa) && 5178 node == NUMA_NO_NODE) { 5179 5180 struct mempolicy *mpol = current->mempolicy; 5181 5182 if (mpol) { 5183 /* 5184 * Special BIND rule support. If the local node 5185 * is in permitted set then do not redirect 5186 * to a particular node. 5187 * Otherwise we apply the memory policy to get 5188 * the node we need to allocate on. 5189 */ 5190 if (mpol->mode != MPOL_BIND || 5191 !node_isset(numa_mem_id(), mpol->nodes)) 5192 5193 node = mempolicy_slab_node(); 5194 } 5195 } 5196#endif 5197 5198 node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE; 5199 5200 /* 5201 * We assume the percpu sheaves contain only local objects although it's 5202 * not completely guaranteed, so we verify later. 5203 */ 5204 if (unlikely(node_requested && node != numa_mem_id())) 5205 return NULL; 5206 5207 if (!local_trylock(&s->cpu_sheaves->lock)) 5208 return NULL; 5209 5210 pcs = this_cpu_ptr(s->cpu_sheaves); 5211 5212 if (unlikely(pcs->main->size == 0)) { 5213 pcs = __pcs_replace_empty_main(s, pcs, gfp); 5214 if (unlikely(!pcs)) 5215 return NULL; 5216 } 5217 5218 object = pcs->main->objects[pcs->main->size - 1]; 5219 5220 if (unlikely(node_requested)) { 5221 /* 5222 * Verify that the object was from the node we want. This could 5223 * be false because of cpu migration during an unlocked part of 5224 * the current allocation or previous freeing process. 5225 */ 5226 if (page_to_nid(virt_to_page(object)) != node) { 5227 local_unlock(&s->cpu_sheaves->lock); 5228 return NULL; 5229 } 5230 } 5231 5232 pcs->main->size--; 5233 5234 local_unlock(&s->cpu_sheaves->lock); 5235 5236 stat(s, ALLOC_PCS); 5237 5238 return object; 5239} 5240 5241static __fastpath_inline 5242unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 5243{ 5244 struct slub_percpu_sheaves *pcs; 5245 struct slab_sheaf *main; 5246 unsigned int allocated = 0; 5247 unsigned int batch; 5248 5249next_batch: 5250 if (!local_trylock(&s->cpu_sheaves->lock)) 5251 return allocated; 5252 5253 pcs = this_cpu_ptr(s->cpu_sheaves); 5254 5255 if (unlikely(pcs->main->size == 0)) { 5256 5257 struct slab_sheaf *full; 5258 struct node_barn *barn; 5259 5260 if (pcs->spare && pcs->spare->size > 0) { 5261 swap(pcs->main, pcs->spare); 5262 goto do_alloc; 5263 } 5264 5265 barn = get_barn(s); 5266 if (!barn) { 5267 local_unlock(&s->cpu_sheaves->lock); 5268 return allocated; 5269 } 5270 5271 full = barn_replace_empty_sheaf(barn, pcs->main); 5272 5273 if (full) { 5274 stat(s, BARN_GET); 5275 pcs->main = full; 5276 goto do_alloc; 5277 } 5278 5279 stat(s, BARN_GET_FAIL); 5280 5281 local_unlock(&s->cpu_sheaves->lock); 5282 5283 /* 5284 * Once full sheaves in barn are depleted, let the bulk 5285 * allocation continue from slab pages, otherwise we would just 5286 * be copying arrays of pointers twice. 5287 */ 5288 return allocated; 5289 } 5290 5291do_alloc: 5292 5293 main = pcs->main; 5294 batch = min(size, main->size); 5295 5296 main->size -= batch; 5297 memcpy(p, main->objects + main->size, batch * sizeof(void *)); 5298 5299 local_unlock(&s->cpu_sheaves->lock); 5300 5301 stat_add(s, ALLOC_PCS, batch); 5302 5303 allocated += batch; 5304 5305 if (batch < size) { 5306 p += batch; 5307 size -= batch; 5308 goto next_batch; 5309 } 5310 5311 return allocated; 5312} 5313 5314 5315/* 5316 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) 5317 * have the fastpath folded into their functions. So no function call 5318 * overhead for requests that can be satisfied on the fastpath. 5319 * 5320 * The fastpath works by first checking if the lockless freelist can be used. 5321 * If not then __slab_alloc is called for slow processing. 5322 * 5323 * Otherwise we can simply pick the next object from the lockless free list. 5324 */ 5325static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru, 5326 gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) 5327{ 5328 void *object; 5329 bool init = false; 5330 5331 s = slab_pre_alloc_hook(s, gfpflags); 5332 if (unlikely(!s)) 5333 return NULL; 5334 5335 object = kfence_alloc(s, orig_size, gfpflags); 5336 if (unlikely(object)) 5337 goto out; 5338 5339 if (s->cpu_sheaves) 5340 object = alloc_from_pcs(s, gfpflags, node); 5341 5342 if (!object) 5343 object = __slab_alloc_node(s, gfpflags, node, addr, orig_size); 5344 5345 maybe_wipe_obj_freeptr(s, object); 5346 init = slab_want_init_on_alloc(gfpflags, s); 5347 5348out: 5349 /* 5350 * When init equals 'true', like for kzalloc() family, only 5351 * @orig_size bytes might be zeroed instead of s->object_size 5352 * In case this fails due to memcg_slab_post_alloc_hook(), 5353 * object is set to NULL 5354 */ 5355 slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size); 5356 5357 return object; 5358} 5359 5360void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags) 5361{ 5362 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, 5363 s->object_size); 5364 5365 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); 5366 5367 return ret; 5368} 5369EXPORT_SYMBOL(kmem_cache_alloc_noprof); 5370 5371void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, 5372 gfp_t gfpflags) 5373{ 5374 void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_, 5375 s->object_size); 5376 5377 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); 5378 5379 return ret; 5380} 5381EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof); 5382 5383bool kmem_cache_charge(void *objp, gfp_t gfpflags) 5384{ 5385 if (!memcg_kmem_online()) 5386 return true; 5387 5388 return memcg_slab_post_charge(objp, gfpflags); 5389} 5390EXPORT_SYMBOL(kmem_cache_charge); 5391 5392/** 5393 * kmem_cache_alloc_node - Allocate an object on the specified node 5394 * @s: The cache to allocate from. 5395 * @gfpflags: See kmalloc(). 5396 * @node: node number of the target node. 5397 * 5398 * Identical to kmem_cache_alloc but it will allocate memory on the given 5399 * node, which can improve the performance for cpu bound structures. 5400 * 5401 * Fallback to other node is possible if __GFP_THISNODE is not set. 5402 * 5403 * Return: pointer to the new object or %NULL in case of error 5404 */ 5405void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node) 5406{ 5407 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); 5408 5409 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node); 5410 5411 return ret; 5412} 5413EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); 5414 5415static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s, 5416 struct slab_sheaf *sheaf, gfp_t gfp) 5417{ 5418 gfp_t gfp_nomemalloc; 5419 int ret; 5420 5421 gfp_nomemalloc = gfp | __GFP_NOMEMALLOC; 5422 if (gfp_pfmemalloc_allowed(gfp)) 5423 gfp_nomemalloc |= __GFP_NOWARN; 5424 5425 ret = refill_sheaf(s, sheaf, gfp_nomemalloc); 5426 5427 if (likely(!ret || !gfp_pfmemalloc_allowed(gfp))) 5428 return ret; 5429 5430 /* 5431 * if we are allowed to, refill sheaf with pfmemalloc but then remember 5432 * it for when it's returned 5433 */ 5434 ret = refill_sheaf(s, sheaf, gfp); 5435 sheaf->pfmemalloc = true; 5436 5437 return ret; 5438} 5439 5440/* 5441 * returns a sheaf that has at least the requested size 5442 * when prefilling is needed, do so with given gfp flags 5443 * 5444 * return NULL if sheaf allocation or prefilling failed 5445 */ 5446struct slab_sheaf * 5447kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) 5448{ 5449 struct slub_percpu_sheaves *pcs; 5450 struct slab_sheaf *sheaf = NULL; 5451 struct node_barn *barn; 5452 5453 if (unlikely(size > s->sheaf_capacity)) { 5454 5455 /* 5456 * slab_debug disables cpu sheaves intentionally so all 5457 * prefilled sheaves become "oversize" and we give up on 5458 * performance for the debugging. Same with SLUB_TINY. 5459 * Creating a cache without sheaves and then requesting a 5460 * prefilled sheaf is however not expected, so warn. 5461 */ 5462 WARN_ON_ONCE(s->sheaf_capacity == 0 && 5463 !IS_ENABLED(CONFIG_SLUB_TINY) && 5464 !(s->flags & SLAB_DEBUG_FLAGS)); 5465 5466 sheaf = kzalloc(struct_size(sheaf, objects, size), gfp); 5467 if (!sheaf) 5468 return NULL; 5469 5470 stat(s, SHEAF_PREFILL_OVERSIZE); 5471 sheaf->cache = s; 5472 sheaf->capacity = size; 5473 5474 /* 5475 * we do not need to care about pfmemalloc here because oversize 5476 * sheaves area always flushed and freed when returned 5477 */ 5478 if (!__kmem_cache_alloc_bulk(s, gfp, size, 5479 &sheaf->objects[0])) { 5480 kfree(sheaf); 5481 return NULL; 5482 } 5483 5484 sheaf->size = size; 5485 5486 return sheaf; 5487 } 5488 5489 local_lock(&s->cpu_sheaves->lock); 5490 pcs = this_cpu_ptr(s->cpu_sheaves); 5491 5492 if (pcs->spare) { 5493 sheaf = pcs->spare; 5494 pcs->spare = NULL; 5495 stat(s, SHEAF_PREFILL_FAST); 5496 } else { 5497 barn = get_barn(s); 5498 5499 stat(s, SHEAF_PREFILL_SLOW); 5500 if (barn) 5501 sheaf = barn_get_full_or_empty_sheaf(barn); 5502 if (sheaf && sheaf->size) 5503 stat(s, BARN_GET); 5504 else 5505 stat(s, BARN_GET_FAIL); 5506 } 5507 5508 local_unlock(&s->cpu_sheaves->lock); 5509 5510 5511 if (!sheaf) 5512 sheaf = alloc_empty_sheaf(s, gfp); 5513 5514 if (sheaf) { 5515 sheaf->capacity = s->sheaf_capacity; 5516 sheaf->pfmemalloc = false; 5517 5518 if (sheaf->size < size && 5519 __prefill_sheaf_pfmemalloc(s, sheaf, gfp)) { 5520 sheaf_flush_unused(s, sheaf); 5521 free_empty_sheaf(s, sheaf); 5522 sheaf = NULL; 5523 } 5524 } 5525 5526 return sheaf; 5527} 5528 5529/* 5530 * Use this to return a sheaf obtained by kmem_cache_prefill_sheaf() 5531 * 5532 * If the sheaf cannot simply become the percpu spare sheaf, but there's space 5533 * for a full sheaf in the barn, we try to refill the sheaf back to the cache's 5534 * sheaf_capacity to avoid handling partially full sheaves. 5535 * 5536 * If the refill fails because gfp is e.g. GFP_NOWAIT, or the barn is full, the 5537 * sheaf is instead flushed and freed. 5538 */ 5539void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp, 5540 struct slab_sheaf *sheaf) 5541{ 5542 struct slub_percpu_sheaves *pcs; 5543 struct node_barn *barn; 5544 5545 if (unlikely((sheaf->capacity != s->sheaf_capacity) 5546 || sheaf->pfmemalloc)) { 5547 sheaf_flush_unused(s, sheaf); 5548 kfree(sheaf); 5549 return; 5550 } 5551 5552 local_lock(&s->cpu_sheaves->lock); 5553 pcs = this_cpu_ptr(s->cpu_sheaves); 5554 barn = get_barn(s); 5555 5556 if (!pcs->spare) { 5557 pcs->spare = sheaf; 5558 sheaf = NULL; 5559 stat(s, SHEAF_RETURN_FAST); 5560 } 5561 5562 local_unlock(&s->cpu_sheaves->lock); 5563 5564 if (!sheaf) 5565 return; 5566 5567 stat(s, SHEAF_RETURN_SLOW); 5568 5569 /* 5570 * If the barn has too many full sheaves or we fail to refill the sheaf, 5571 * simply flush and free it. 5572 */ 5573 if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES || 5574 refill_sheaf(s, sheaf, gfp)) { 5575 sheaf_flush_unused(s, sheaf); 5576 free_empty_sheaf(s, sheaf); 5577 return; 5578 } 5579 5580 barn_put_full_sheaf(barn, sheaf); 5581 stat(s, BARN_PUT); 5582} 5583 5584/* 5585 * refill a sheaf previously returned by kmem_cache_prefill_sheaf to at least 5586 * the given size 5587 * 5588 * the sheaf might be replaced by a new one when requesting more than 5589 * s->sheaf_capacity objects if such replacement is necessary, but the refill 5590 * fails (returning -ENOMEM), the existing sheaf is left intact 5591 * 5592 * In practice we always refill to full sheaf's capacity. 5593 */ 5594int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, 5595 struct slab_sheaf **sheafp, unsigned int size) 5596{ 5597 struct slab_sheaf *sheaf; 5598 5599 /* 5600 * TODO: do we want to support *sheaf == NULL to be equivalent of 5601 * kmem_cache_prefill_sheaf() ? 5602 */ 5603 if (!sheafp || !(*sheafp)) 5604 return -EINVAL; 5605 5606 sheaf = *sheafp; 5607 if (sheaf->size >= size) 5608 return 0; 5609 5610 if (likely(sheaf->capacity >= size)) { 5611 if (likely(sheaf->capacity == s->sheaf_capacity)) 5612 return __prefill_sheaf_pfmemalloc(s, sheaf, gfp); 5613 5614 if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size, 5615 &sheaf->objects[sheaf->size])) { 5616 return -ENOMEM; 5617 } 5618 sheaf->size = sheaf->capacity; 5619 5620 return 0; 5621 } 5622 5623 /* 5624 * We had a regular sized sheaf and need an oversize one, or we had an 5625 * oversize one already but need a larger one now. 5626 * This should be a very rare path so let's not complicate it. 5627 */ 5628 sheaf = kmem_cache_prefill_sheaf(s, gfp, size); 5629 if (!sheaf) 5630 return -ENOMEM; 5631 5632 kmem_cache_return_sheaf(s, gfp, *sheafp); 5633 *sheafp = sheaf; 5634 return 0; 5635} 5636 5637/* 5638 * Allocate from a sheaf obtained by kmem_cache_prefill_sheaf() 5639 * 5640 * Guaranteed not to fail as many allocations as was the requested size. 5641 * After the sheaf is emptied, it fails - no fallback to the slab cache itself. 5642 * 5643 * The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT 5644 * memcg charging is forced over limit if necessary, to avoid failure. 5645 * 5646 * It is possible that the allocation comes from kfence and then the sheaf 5647 * size is not decreased. 5648 */ 5649void * 5650kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, 5651 struct slab_sheaf *sheaf) 5652{ 5653 void *ret = NULL; 5654 bool init; 5655 5656 if (sheaf->size == 0) 5657 goto out; 5658 5659 ret = kfence_alloc(s, s->object_size, gfp); 5660 5661 if (likely(!ret)) 5662 ret = sheaf->objects[--sheaf->size]; 5663 5664 init = slab_want_init_on_alloc(gfp, s); 5665 5666 /* add __GFP_NOFAIL to force successful memcg charging */ 5667 slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, init, s->object_size); 5668out: 5669 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE); 5670 5671 return ret; 5672} 5673 5674unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf) 5675{ 5676 return sheaf->size; 5677} 5678/* 5679 * To avoid unnecessary overhead, we pass through large allocation requests 5680 * directly to the page allocator. We use __GFP_COMP, because we will need to 5681 * know the allocation order to free the pages properly in kfree. 5682 */ 5683static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) 5684{ 5685 struct page *page; 5686 void *ptr = NULL; 5687 unsigned int order = get_order(size); 5688 5689 if (unlikely(flags & GFP_SLAB_BUG_MASK)) 5690 flags = kmalloc_fix_flags(flags); 5691 5692 flags |= __GFP_COMP; 5693 5694 if (node == NUMA_NO_NODE) 5695 page = alloc_frozen_pages_noprof(flags, order); 5696 else 5697 page = __alloc_frozen_pages_noprof(flags, order, node, NULL); 5698 5699 if (page) { 5700 ptr = page_address(page); 5701 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, 5702 PAGE_SIZE << order); 5703 __SetPageLargeKmalloc(page); 5704 } 5705 5706 ptr = kasan_kmalloc_large(ptr, size, flags); 5707 /* As ptr might get tagged, call kmemleak hook after KASAN. */ 5708 kmemleak_alloc(ptr, size, 1, flags); 5709 kmsan_kmalloc_large(ptr, size, flags); 5710 5711 return ptr; 5712} 5713 5714void *__kmalloc_large_noprof(size_t size, gfp_t flags) 5715{ 5716 void *ret = ___kmalloc_large_node(size, flags, NUMA_NO_NODE); 5717 5718 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 5719 flags, NUMA_NO_NODE); 5720 return ret; 5721} 5722EXPORT_SYMBOL(__kmalloc_large_noprof); 5723 5724void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) 5725{ 5726 void *ret = ___kmalloc_large_node(size, flags, node); 5727 5728 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), 5729 flags, node); 5730 return ret; 5731} 5732EXPORT_SYMBOL(__kmalloc_large_node_noprof); 5733 5734static __always_inline 5735void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node, 5736 unsigned long caller) 5737{ 5738 struct kmem_cache *s; 5739 void *ret; 5740 5741 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 5742 ret = __kmalloc_large_node_noprof(size, flags, node); 5743 trace_kmalloc(caller, ret, size, 5744 PAGE_SIZE << get_order(size), flags, node); 5745 return ret; 5746 } 5747 5748 if (unlikely(!size)) 5749 return ZERO_SIZE_PTR; 5750 5751 s = kmalloc_slab(size, b, flags, caller); 5752 5753 ret = slab_alloc_node(s, NULL, flags, node, caller, size); 5754 ret = kasan_kmalloc(s, ret, size, flags); 5755 trace_kmalloc(caller, ret, size, s->size, flags, node); 5756 return ret; 5757} 5758void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) 5759{ 5760 return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, _RET_IP_); 5761} 5762EXPORT_SYMBOL(__kmalloc_node_noprof); 5763 5764void *__kmalloc_noprof(size_t size, gfp_t flags) 5765{ 5766 return __do_kmalloc_node(size, NULL, flags, NUMA_NO_NODE, _RET_IP_); 5767} 5768EXPORT_SYMBOL(__kmalloc_noprof); 5769 5770/** 5771 * kmalloc_nolock - Allocate an object of given size from any context. 5772 * @size: size to allocate 5773 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, __GFP_ZERO, __GFP_NO_OBJ_EXT 5774 * allowed. 5775 * @node: node number of the target node. 5776 * 5777 * Return: pointer to the new object or NULL in case of error. 5778 * NULL does not mean EBUSY or EAGAIN. It means ENOMEM. 5779 * There is no reason to call it again and expect !NULL. 5780 */ 5781void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) 5782{ 5783 gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags; 5784 struct kmem_cache *s; 5785 bool can_retry = true; 5786 void *ret = ERR_PTR(-EBUSY); 5787 5788 VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO | 5789 __GFP_NO_OBJ_EXT)); 5790 5791 if (unlikely(!size)) 5792 return ZERO_SIZE_PTR; 5793 5794 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !preemptible()) 5795 /* 5796 * kmalloc_nolock() in PREEMPT_RT is not supported from 5797 * non-preemptible context because local_lock becomes a 5798 * sleeping lock on RT. 5799 */ 5800 return NULL; 5801retry: 5802 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) 5803 return NULL; 5804 s = kmalloc_slab(size, NULL, alloc_gfp, _RET_IP_); 5805 5806 if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s)) 5807 /* 5808 * kmalloc_nolock() is not supported on architectures that 5809 * don't implement cmpxchg16b, but debug caches don't use 5810 * per-cpu slab and per-cpu partial slabs. They rely on 5811 * kmem_cache_node->list_lock, so kmalloc_nolock() can 5812 * attempt to allocate from debug caches by 5813 * spin_trylock_irqsave(&n->list_lock, ...) 5814 */ 5815 return NULL; 5816 5817 /* 5818 * Do not call slab_alloc_node(), since trylock mode isn't 5819 * compatible with slab_pre_alloc_hook/should_failslab and 5820 * kfence_alloc. Hence call __slab_alloc_node() (at most twice) 5821 * and slab_post_alloc_hook() directly. 5822 * 5823 * In !PREEMPT_RT ___slab_alloc() manipulates (freelist,tid) pair 5824 * in irq saved region. It assumes that the same cpu will not 5825 * __update_cpu_freelist_fast() into the same (freelist,tid) pair. 5826 * Therefore use in_nmi() to check whether particular bucket is in 5827 * irq protected section. 5828 * 5829 * If in_nmi() && local_lock_is_locked(s->cpu_slab) then it means that 5830 * this cpu was interrupted somewhere inside ___slab_alloc() after 5831 * it did local_lock_irqsave(&s->cpu_slab->lock, flags). 5832 * In this case fast path with __update_cpu_freelist_fast() is not safe. 5833 */ 5834 if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) 5835 ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size); 5836 5837 if (PTR_ERR(ret) == -EBUSY) { 5838 if (can_retry) { 5839 /* pick the next kmalloc bucket */ 5840 size = s->object_size + 1; 5841 /* 5842 * Another alternative is to 5843 * if (memcg) alloc_gfp &= ~__GFP_ACCOUNT; 5844 * else if (!memcg) alloc_gfp |= __GFP_ACCOUNT; 5845 * to retry from bucket of the same size. 5846 */ 5847 can_retry = false; 5848 goto retry; 5849 } 5850 ret = NULL; 5851 } 5852 5853 maybe_wipe_obj_freeptr(s, ret); 5854 slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, 5855 slab_want_init_on_alloc(alloc_gfp, s), size); 5856 5857 ret = kasan_kmalloc(s, ret, size, alloc_gfp); 5858 return ret; 5859} 5860EXPORT_SYMBOL_GPL(kmalloc_nolock_noprof); 5861 5862void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, 5863 int node, unsigned long caller) 5864{ 5865 return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, caller); 5866 5867} 5868EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof); 5869 5870void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) 5871{ 5872 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, 5873 _RET_IP_, size); 5874 5875 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE); 5876 5877 ret = kasan_kmalloc(s, ret, size, gfpflags); 5878 return ret; 5879} 5880EXPORT_SYMBOL(__kmalloc_cache_noprof); 5881 5882void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, 5883 int node, size_t size) 5884{ 5885 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); 5886 5887 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node); 5888 5889 ret = kasan_kmalloc(s, ret, size, gfpflags); 5890 return ret; 5891} 5892EXPORT_SYMBOL(__kmalloc_cache_node_noprof); 5893 5894static noinline void free_to_partial_list( 5895 struct kmem_cache *s, struct slab *slab, 5896 void *head, void *tail, int bulk_cnt, 5897 unsigned long addr) 5898{ 5899 struct kmem_cache_node *n = get_node(s, slab_nid(slab)); 5900 struct slab *slab_free = NULL; 5901 int cnt = bulk_cnt; 5902 unsigned long flags; 5903 depot_stack_handle_t handle = 0; 5904 5905 /* 5906 * We cannot use GFP_NOWAIT as there are callsites where waking up 5907 * kswapd could deadlock 5908 */ 5909 if (s->flags & SLAB_STORE_USER) 5910 handle = set_track_prepare(__GFP_NOWARN); 5911 5912 spin_lock_irqsave(&n->list_lock, flags); 5913 5914 if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) { 5915 void *prior = slab->freelist; 5916 5917 /* Perform the actual freeing while we still hold the locks */ 5918 slab->inuse -= cnt; 5919 set_freepointer(s, tail, prior); 5920 slab->freelist = head; 5921 5922 /* 5923 * If the slab is empty, and node's partial list is full, 5924 * it should be discarded anyway no matter it's on full or 5925 * partial list. 5926 */ 5927 if (slab->inuse == 0 && n->nr_partial >= s->min_partial) 5928 slab_free = slab; 5929 5930 if (!prior) { 5931 /* was on full list */ 5932 remove_full(s, n, slab); 5933 if (!slab_free) { 5934 add_partial(n, slab, DEACTIVATE_TO_TAIL); 5935 stat(s, FREE_ADD_PARTIAL); 5936 } 5937 } else if (slab_free) { 5938 remove_partial(n, slab); 5939 stat(s, FREE_REMOVE_PARTIAL); 5940 } 5941 } 5942 5943 if (slab_free) { 5944 /* 5945 * Update the counters while still holding n->list_lock to 5946 * prevent spurious validation warnings 5947 */ 5948 dec_slabs_node(s, slab_nid(slab_free), slab_free->objects); 5949 } 5950 5951 spin_unlock_irqrestore(&n->list_lock, flags); 5952 5953 if (slab_free) { 5954 stat(s, FREE_SLAB); 5955 free_slab(s, slab_free); 5956 } 5957} 5958 5959/* 5960 * Slow path handling. This may still be called frequently since objects 5961 * have a longer lifetime than the cpu slabs in most processing loads. 5962 * 5963 * So we still attempt to reduce cache line usage. Just take the slab 5964 * lock and free the item. If there is no additional partial slab 5965 * handling required then we can return immediately. 5966 */ 5967static void __slab_free(struct kmem_cache *s, struct slab *slab, 5968 void *head, void *tail, int cnt, 5969 unsigned long addr) 5970 5971{ 5972 bool was_frozen, was_full; 5973 struct freelist_counters old, new; 5974 struct kmem_cache_node *n = NULL; 5975 unsigned long flags; 5976 bool on_node_partial; 5977 5978 stat(s, FREE_SLOWPATH); 5979 5980 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { 5981 free_to_partial_list(s, slab, head, tail, cnt, addr); 5982 return; 5983 } 5984 5985 /* 5986 * It is enough to test IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) below 5987 * instead of kmem_cache_has_cpu_partial(s), because kmem_cache_debug(s) 5988 * is the only other reason it can be false, and it is already handled 5989 * above. 5990 */ 5991 5992 do { 5993 if (unlikely(n)) { 5994 spin_unlock_irqrestore(&n->list_lock, flags); 5995 n = NULL; 5996 } 5997 5998 old.freelist = slab->freelist; 5999 old.counters = slab->counters; 6000 6001 was_full = (old.freelist == NULL); 6002 was_frozen = old.frozen; 6003 6004 set_freepointer(s, tail, old.freelist); 6005 6006 new.freelist = head; 6007 new.counters = old.counters; 6008 new.inuse -= cnt; 6009 6010 /* 6011 * Might need to be taken off (due to becoming empty) or added 6012 * to (due to not being full anymore) the partial list. 6013 * Unless it's frozen. 6014 */ 6015 if ((!new.inuse || was_full) && !was_frozen) { 6016 /* 6017 * If slab becomes non-full and we have cpu partial 6018 * lists, we put it there unconditionally to avoid 6019 * taking the list_lock. Otherwise we need it. 6020 */ 6021 if (!(IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full)) { 6022 6023 n = get_node(s, slab_nid(slab)); 6024 /* 6025 * Speculatively acquire the list_lock. 6026 * If the cmpxchg does not succeed then we may 6027 * drop the list_lock without any processing. 6028 * 6029 * Otherwise the list_lock will synchronize with 6030 * other processors updating the list of slabs. 6031 */ 6032 spin_lock_irqsave(&n->list_lock, flags); 6033 6034 on_node_partial = slab_test_node_partial(slab); 6035 } 6036 } 6037 6038 } while (!slab_update_freelist(s, slab, &old, &new, "__slab_free")); 6039 6040 if (likely(!n)) { 6041 6042 if (likely(was_frozen)) { 6043 /* 6044 * The list lock was not taken therefore no list 6045 * activity can be necessary. 6046 */ 6047 stat(s, FREE_FROZEN); 6048 } else if (IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full) { 6049 /* 6050 * If we started with a full slab then put it onto the 6051 * per cpu partial list. 6052 */ 6053 put_cpu_partial(s, slab, 1); 6054 stat(s, CPU_PARTIAL_FREE); 6055 } 6056 6057 /* 6058 * In other cases we didn't take the list_lock because the slab 6059 * was already on the partial list and will remain there. 6060 */ 6061 6062 return; 6063 } 6064 6065 /* 6066 * This slab was partially empty but not on the per-node partial list, 6067 * in which case we shouldn't manipulate its list, just return. 6068 */ 6069 if (!was_full && !on_node_partial) { 6070 spin_unlock_irqrestore(&n->list_lock, flags); 6071 return; 6072 } 6073 6074 /* 6075 * If slab became empty, should we add/keep it on the partial list or we 6076 * have enough? 6077 */ 6078 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 6079 goto slab_empty; 6080 6081 /* 6082 * Objects left in the slab. If it was not on the partial list before 6083 * then add it. This can only happen when cache has no per cpu partial 6084 * list otherwise we would have put it there. 6085 */ 6086 if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) { 6087 add_partial(n, slab, DEACTIVATE_TO_TAIL); 6088 stat(s, FREE_ADD_PARTIAL); 6089 } 6090 spin_unlock_irqrestore(&n->list_lock, flags); 6091 return; 6092 6093slab_empty: 6094 /* 6095 * The slab could have a single object and thus go from full to empty in 6096 * a single free, but more likely it was on the partial list. Remove it. 6097 */ 6098 if (likely(!was_full)) { 6099 remove_partial(n, slab); 6100 stat(s, FREE_REMOVE_PARTIAL); 6101 } 6102 6103 spin_unlock_irqrestore(&n->list_lock, flags); 6104 stat(s, FREE_SLAB); 6105 discard_slab(s, slab); 6106} 6107 6108/* 6109 * pcs is locked. We should have get rid of the spare sheaf and obtained an 6110 * empty sheaf, while the main sheaf is full. We want to install the empty sheaf 6111 * as a main sheaf, and make the current main sheaf a spare sheaf. 6112 * 6113 * However due to having relinquished the cpu_sheaves lock when obtaining 6114 * the empty sheaf, we need to handle some unlikely but possible cases. 6115 * 6116 * If we put any sheaf to barn here, it's because we were interrupted or have 6117 * been migrated to a different cpu, which should be rare enough so just ignore 6118 * the barn's limits to simplify the handling. 6119 * 6120 * An alternative scenario that gets us here is when we fail 6121 * barn_replace_full_sheaf(), because there's no empty sheaf available in the 6122 * barn, so we had to allocate it by alloc_empty_sheaf(). But because we saw the 6123 * limit on full sheaves was not exceeded, we assume it didn't change and just 6124 * put the full sheaf there. 6125 */ 6126static void __pcs_install_empty_sheaf(struct kmem_cache *s, 6127 struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty, 6128 struct node_barn *barn) 6129{ 6130 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 6131 6132 /* This is what we expect to find if nobody interrupted us. */ 6133 if (likely(!pcs->spare)) { 6134 pcs->spare = pcs->main; 6135 pcs->main = empty; 6136 return; 6137 } 6138 6139 /* 6140 * Unlikely because if the main sheaf had space, we would have just 6141 * freed to it. Get rid of our empty sheaf. 6142 */ 6143 if (pcs->main->size < s->sheaf_capacity) { 6144 barn_put_empty_sheaf(barn, empty); 6145 return; 6146 } 6147 6148 /* Also unlikely for the same reason */ 6149 if (pcs->spare->size < s->sheaf_capacity) { 6150 swap(pcs->main, pcs->spare); 6151 barn_put_empty_sheaf(barn, empty); 6152 return; 6153 } 6154 6155 /* 6156 * We probably failed barn_replace_full_sheaf() due to no empty sheaf 6157 * available there, but we allocated one, so finish the job. 6158 */ 6159 barn_put_full_sheaf(barn, pcs->main); 6160 stat(s, BARN_PUT); 6161 pcs->main = empty; 6162} 6163 6164/* 6165 * Replace the full main sheaf with a (at least partially) empty sheaf. 6166 * 6167 * Must be called with the cpu_sheaves local lock locked. If successful, returns 6168 * the pcs pointer and the local lock locked (possibly on a different cpu than 6169 * initially called). If not successful, returns NULL and the local lock 6170 * unlocked. 6171 */ 6172static struct slub_percpu_sheaves * 6173__pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) 6174{ 6175 struct slab_sheaf *empty; 6176 struct node_barn *barn; 6177 bool put_fail; 6178 6179restart: 6180 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); 6181 6182 barn = get_barn(s); 6183 if (!barn) { 6184 local_unlock(&s->cpu_sheaves->lock); 6185 return NULL; 6186 } 6187 6188 put_fail = false; 6189 6190 if (!pcs->spare) { 6191 empty = barn_get_empty_sheaf(barn); 6192 if (empty) { 6193 pcs->spare = pcs->main; 6194 pcs->main = empty; 6195 return pcs; 6196 } 6197 goto alloc_empty; 6198 } 6199 6200 if (pcs->spare->size < s->sheaf_capacity) { 6201 swap(pcs->main, pcs->spare); 6202 return pcs; 6203 } 6204 6205 empty = barn_replace_full_sheaf(barn, pcs->main); 6206 6207 if (!IS_ERR(empty)) { 6208 stat(s, BARN_PUT); 6209 pcs->main = empty; 6210 return pcs; 6211 } 6212 6213 if (PTR_ERR(empty) == -E2BIG) { 6214 /* Since we got here, spare exists and is full */ 6215 struct slab_sheaf *to_flush = pcs->spare; 6216 6217 stat(s, BARN_PUT_FAIL); 6218 6219 pcs->spare = NULL; 6220 local_unlock(&s->cpu_sheaves->lock); 6221 6222 sheaf_flush_unused(s, to_flush); 6223 empty = to_flush; 6224 goto got_empty; 6225 } 6226 6227 /* 6228 * We could not replace full sheaf because barn had no empty 6229 * sheaves. We can still allocate it and put the full sheaf in 6230 * __pcs_install_empty_sheaf(), but if we fail to allocate it, 6231 * make sure to count the fail. 6232 */ 6233 put_fail = true; 6234 6235alloc_empty: 6236 local_unlock(&s->cpu_sheaves->lock); 6237 6238 empty = alloc_empty_sheaf(s, GFP_NOWAIT); 6239 if (empty) 6240 goto got_empty; 6241 6242 if (put_fail) 6243 stat(s, BARN_PUT_FAIL); 6244 6245 if (!sheaf_try_flush_main(s)) 6246 return NULL; 6247 6248 if (!local_trylock(&s->cpu_sheaves->lock)) 6249 return NULL; 6250 6251 pcs = this_cpu_ptr(s->cpu_sheaves); 6252 6253 /* 6254 * we flushed the main sheaf so it should be empty now, 6255 * but in case we got preempted or migrated, we need to 6256 * check again 6257 */ 6258 if (pcs->main->size == s->sheaf_capacity) 6259 goto restart; 6260 6261 return pcs; 6262 6263got_empty: 6264 if (!local_trylock(&s->cpu_sheaves->lock)) { 6265 barn_put_empty_sheaf(barn, empty); 6266 return NULL; 6267 } 6268 6269 pcs = this_cpu_ptr(s->cpu_sheaves); 6270 __pcs_install_empty_sheaf(s, pcs, empty, barn); 6271 6272 return pcs; 6273} 6274 6275/* 6276 * Free an object to the percpu sheaves. 6277 * The object is expected to have passed slab_free_hook() already. 6278 */ 6279static __fastpath_inline 6280bool free_to_pcs(struct kmem_cache *s, void *object) 6281{ 6282 struct slub_percpu_sheaves *pcs; 6283 6284 if (!local_trylock(&s->cpu_sheaves->lock)) 6285 return false; 6286 6287 pcs = this_cpu_ptr(s->cpu_sheaves); 6288 6289 if (unlikely(pcs->main->size == s->sheaf_capacity)) { 6290 6291 pcs = __pcs_replace_full_main(s, pcs); 6292 if (unlikely(!pcs)) 6293 return false; 6294 } 6295 6296 pcs->main->objects[pcs->main->size++] = object; 6297 6298 local_unlock(&s->cpu_sheaves->lock); 6299 6300 stat(s, FREE_PCS); 6301 6302 return true; 6303} 6304 6305static void rcu_free_sheaf(struct rcu_head *head) 6306{ 6307 struct kmem_cache_node *n; 6308 struct slab_sheaf *sheaf; 6309 struct node_barn *barn = NULL; 6310 struct kmem_cache *s; 6311 6312 sheaf = container_of(head, struct slab_sheaf, rcu_head); 6313 6314 s = sheaf->cache; 6315 6316 /* 6317 * This may remove some objects due to slab_free_hook() returning false, 6318 * so that the sheaf might no longer be completely full. But it's easier 6319 * to handle it as full (unless it became completely empty), as the code 6320 * handles it fine. The only downside is that sheaf will serve fewer 6321 * allocations when reused. It only happens due to debugging, which is a 6322 * performance hit anyway. 6323 * 6324 * If it returns true, there was at least one object from pfmemalloc 6325 * slab so simply flush everything. 6326 */ 6327 if (__rcu_free_sheaf_prepare(s, sheaf)) 6328 goto flush; 6329 6330 n = get_node(s, sheaf->node); 6331 if (!n) 6332 goto flush; 6333 6334 barn = n->barn; 6335 6336 /* due to slab_free_hook() */ 6337 if (unlikely(sheaf->size == 0)) 6338 goto empty; 6339 6340 /* 6341 * Checking nr_full/nr_empty outside lock avoids contention in case the 6342 * barn is at the respective limit. Due to the race we might go over the 6343 * limit but that should be rare and harmless. 6344 */ 6345 6346 if (data_race(barn->nr_full) < MAX_FULL_SHEAVES) { 6347 stat(s, BARN_PUT); 6348 barn_put_full_sheaf(barn, sheaf); 6349 return; 6350 } 6351 6352flush: 6353 stat(s, BARN_PUT_FAIL); 6354 sheaf_flush_unused(s, sheaf); 6355 6356empty: 6357 if (barn && data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) { 6358 barn_put_empty_sheaf(barn, sheaf); 6359 return; 6360 } 6361 6362 free_empty_sheaf(s, sheaf); 6363} 6364 6365/* 6366 * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since 6367 * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), 6368 * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids 6369 * this problem by bypassing the sheaves layer entirely on PREEMPT_RT. 6370 * 6371 * However, lockdep still complains that it is invalid to acquire spinlock_t 6372 * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a 6373 * spinning lock. Tell lockdep that acquiring spinlock_t is valid here 6374 * by temporarily raising the wait-type to LD_WAIT_CONFIG. 6375 */ 6376static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); 6377 6378bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj) 6379{ 6380 struct slub_percpu_sheaves *pcs; 6381 struct slab_sheaf *rcu_sheaf; 6382 6383 if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) 6384 return false; 6385 6386 lock_map_acquire_try(&kfree_rcu_sheaf_map); 6387 6388 if (!local_trylock(&s->cpu_sheaves->lock)) 6389 goto fail; 6390 6391 pcs = this_cpu_ptr(s->cpu_sheaves); 6392 6393 if (unlikely(!pcs->rcu_free)) { 6394 6395 struct slab_sheaf *empty; 6396 struct node_barn *barn; 6397 6398 if (pcs->spare && pcs->spare->size == 0) { 6399 pcs->rcu_free = pcs->spare; 6400 pcs->spare = NULL; 6401 goto do_free; 6402 } 6403 6404 barn = get_barn(s); 6405 if (!barn) { 6406 local_unlock(&s->cpu_sheaves->lock); 6407 goto fail; 6408 } 6409 6410 empty = barn_get_empty_sheaf(barn); 6411 6412 if (empty) { 6413 pcs->rcu_free = empty; 6414 goto do_free; 6415 } 6416 6417 local_unlock(&s->cpu_sheaves->lock); 6418 6419 empty = alloc_empty_sheaf(s, GFP_NOWAIT); 6420 6421 if (!empty) 6422 goto fail; 6423 6424 if (!local_trylock(&s->cpu_sheaves->lock)) { 6425 barn_put_empty_sheaf(barn, empty); 6426 goto fail; 6427 } 6428 6429 pcs = this_cpu_ptr(s->cpu_sheaves); 6430 6431 if (unlikely(pcs->rcu_free)) 6432 barn_put_empty_sheaf(barn, empty); 6433 else 6434 pcs->rcu_free = empty; 6435 } 6436 6437do_free: 6438 6439 rcu_sheaf = pcs->rcu_free; 6440 6441 /* 6442 * Since we flush immediately when size reaches capacity, we never reach 6443 * this with size already at capacity, so no OOB write is possible. 6444 */ 6445 rcu_sheaf->objects[rcu_sheaf->size++] = obj; 6446 6447 if (likely(rcu_sheaf->size < s->sheaf_capacity)) { 6448 rcu_sheaf = NULL; 6449 } else { 6450 pcs->rcu_free = NULL; 6451 rcu_sheaf->node = numa_mem_id(); 6452 } 6453 6454 /* 6455 * we flush before local_unlock to make sure a racing 6456 * flush_all_rcu_sheaves() doesn't miss this sheaf 6457 */ 6458 if (rcu_sheaf) 6459 call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); 6460 6461 local_unlock(&s->cpu_sheaves->lock); 6462 6463 stat(s, FREE_RCU_SHEAF); 6464 lock_map_release(&kfree_rcu_sheaf_map); 6465 return true; 6466 6467fail: 6468 stat(s, FREE_RCU_SHEAF_FAIL); 6469 lock_map_release(&kfree_rcu_sheaf_map); 6470 return false; 6471} 6472 6473/* 6474 * Bulk free objects to the percpu sheaves. 6475 * Unlike free_to_pcs() this includes the calls to all necessary hooks 6476 * and the fallback to freeing to slab pages. 6477 */ 6478static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) 6479{ 6480 struct slub_percpu_sheaves *pcs; 6481 struct slab_sheaf *main, *empty; 6482 bool init = slab_want_init_on_free(s); 6483 unsigned int batch, i = 0; 6484 struct node_barn *barn; 6485 void *remote_objects[PCS_BATCH_MAX]; 6486 unsigned int remote_nr = 0; 6487 int node = numa_mem_id(); 6488 6489next_remote_batch: 6490 while (i < size) { 6491 struct slab *slab = virt_to_slab(p[i]); 6492 6493 memcg_slab_free_hook(s, slab, p + i, 1); 6494 alloc_tagging_slab_free_hook(s, slab, p + i, 1); 6495 6496 if (unlikely(!slab_free_hook(s, p[i], init, false))) { 6497 p[i] = p[--size]; 6498 continue; 6499 } 6500 6501 if (unlikely((IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != node) 6502 || slab_test_pfmemalloc(slab))) { 6503 remote_objects[remote_nr] = p[i]; 6504 p[i] = p[--size]; 6505 if (++remote_nr >= PCS_BATCH_MAX) 6506 goto flush_remote; 6507 continue; 6508 } 6509 6510 i++; 6511 } 6512 6513 if (!size) 6514 goto flush_remote; 6515 6516next_batch: 6517 if (!local_trylock(&s->cpu_sheaves->lock)) 6518 goto fallback; 6519 6520 pcs = this_cpu_ptr(s->cpu_sheaves); 6521 6522 if (likely(pcs->main->size < s->sheaf_capacity)) 6523 goto do_free; 6524 6525 barn = get_barn(s); 6526 if (!barn) 6527 goto no_empty; 6528 6529 if (!pcs->spare) { 6530 empty = barn_get_empty_sheaf(barn); 6531 if (!empty) 6532 goto no_empty; 6533 6534 pcs->spare = pcs->main; 6535 pcs->main = empty; 6536 goto do_free; 6537 } 6538 6539 if (pcs->spare->size < s->sheaf_capacity) { 6540 swap(pcs->main, pcs->spare); 6541 goto do_free; 6542 } 6543 6544 empty = barn_replace_full_sheaf(barn, pcs->main); 6545 if (IS_ERR(empty)) { 6546 stat(s, BARN_PUT_FAIL); 6547 goto no_empty; 6548 } 6549 6550 stat(s, BARN_PUT); 6551 pcs->main = empty; 6552 6553do_free: 6554 main = pcs->main; 6555 batch = min(size, s->sheaf_capacity - main->size); 6556 6557 memcpy(main->objects + main->size, p, batch * sizeof(void *)); 6558 main->size += batch; 6559 6560 local_unlock(&s->cpu_sheaves->lock); 6561 6562 stat_add(s, FREE_PCS, batch); 6563 6564 if (batch < size) { 6565 p += batch; 6566 size -= batch; 6567 goto next_batch; 6568 } 6569 6570 if (remote_nr) 6571 goto flush_remote; 6572 6573 return; 6574 6575no_empty: 6576 local_unlock(&s->cpu_sheaves->lock); 6577 6578 /* 6579 * if we depleted all empty sheaves in the barn or there are too 6580 * many full sheaves, free the rest to slab pages 6581 */ 6582fallback: 6583 __kmem_cache_free_bulk(s, size, p); 6584 6585flush_remote: 6586 if (remote_nr) { 6587 __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); 6588 if (i < size) { 6589 remote_nr = 0; 6590 goto next_remote_batch; 6591 } 6592 } 6593} 6594 6595struct defer_free { 6596 struct llist_head objects; 6597 struct llist_head slabs; 6598 struct irq_work work; 6599}; 6600 6601static void free_deferred_objects(struct irq_work *work); 6602 6603static DEFINE_PER_CPU(struct defer_free, defer_free_objects) = { 6604 .objects = LLIST_HEAD_INIT(objects), 6605 .slabs = LLIST_HEAD_INIT(slabs), 6606 .work = IRQ_WORK_INIT(free_deferred_objects), 6607}; 6608 6609/* 6610 * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe 6611 * to take sleeping spin_locks from __slab_free() and deactivate_slab(). 6612 * In !PREEMPT_RT irq_work will run after local_unlock_irqrestore(). 6613 */ 6614static void free_deferred_objects(struct irq_work *work) 6615{ 6616 struct defer_free *df = container_of(work, struct defer_free, work); 6617 struct llist_head *objs = &df->objects; 6618 struct llist_head *slabs = &df->slabs; 6619 struct llist_node *llnode, *pos, *t; 6620 6621 if (llist_empty(objs) && llist_empty(slabs)) 6622 return; 6623 6624 llnode = llist_del_all(objs); 6625 llist_for_each_safe(pos, t, llnode) { 6626 struct kmem_cache *s; 6627 struct slab *slab; 6628 void *x = pos; 6629 6630 slab = virt_to_slab(x); 6631 s = slab->slab_cache; 6632 6633 /* Point 'x' back to the beginning of allocated object */ 6634 x -= s->offset; 6635 6636 /* 6637 * We used freepointer in 'x' to link 'x' into df->objects. 6638 * Clear it to NULL to avoid false positive detection 6639 * of "Freepointer corruption". 6640 */ 6641 set_freepointer(s, x, NULL); 6642 6643 __slab_free(s, slab, x, x, 1, _THIS_IP_); 6644 } 6645 6646 llnode = llist_del_all(slabs); 6647 llist_for_each_safe(pos, t, llnode) { 6648 struct slab *slab = container_of(pos, struct slab, llnode); 6649 6650 if (slab->frozen) 6651 deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); 6652 else 6653 free_slab(slab->slab_cache, slab); 6654 } 6655} 6656 6657static void defer_free(struct kmem_cache *s, void *head) 6658{ 6659 struct defer_free *df; 6660 6661 guard(preempt)(); 6662 6663 head = kasan_reset_tag(head); 6664 6665 df = this_cpu_ptr(&defer_free_objects); 6666 if (llist_add(head + s->offset, &df->objects)) 6667 irq_work_queue(&df->work); 6668} 6669 6670static void defer_deactivate_slab(struct slab *slab, void *flush_freelist) 6671{ 6672 struct defer_free *df; 6673 6674 slab->flush_freelist = flush_freelist; 6675 6676 guard(preempt)(); 6677 6678 df = this_cpu_ptr(&defer_free_objects); 6679 if (llist_add(&slab->llnode, &df->slabs)) 6680 irq_work_queue(&df->work); 6681} 6682 6683void defer_free_barrier(void) 6684{ 6685 int cpu; 6686 6687 for_each_possible_cpu(cpu) 6688 irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); 6689} 6690 6691/* 6692 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 6693 * can perform fastpath freeing without additional function calls. 6694 * 6695 * The fastpath is only possible if we are freeing to the current cpu slab 6696 * of this processor. This typically the case if we have just allocated 6697 * the item before. 6698 * 6699 * If fastpath is not possible then fall back to __slab_free where we deal 6700 * with all sorts of special processing. 6701 * 6702 * Bulk free of a freelist with several objects (all pointing to the 6703 * same slab) possible by specifying head and tail ptr, plus objects 6704 * count (cnt). Bulk free indicated by tail pointer being set. 6705 */ 6706static __always_inline void do_slab_free(struct kmem_cache *s, 6707 struct slab *slab, void *head, void *tail, 6708 int cnt, unsigned long addr) 6709{ 6710 /* cnt == 0 signals that it's called from kfree_nolock() */ 6711 bool allow_spin = cnt; 6712 struct kmem_cache_cpu *c; 6713 unsigned long tid; 6714 void **freelist; 6715 6716redo: 6717 /* 6718 * Determine the currently cpus per cpu slab. 6719 * The cpu may change afterward. However that does not matter since 6720 * data is retrieved via this pointer. If we are on the same cpu 6721 * during the cmpxchg then the free will succeed. 6722 */ 6723 c = raw_cpu_ptr(s->cpu_slab); 6724 tid = READ_ONCE(c->tid); 6725 6726 /* Same with comment on barrier() in __slab_alloc_node() */ 6727 barrier(); 6728 6729 if (unlikely(slab != c->slab)) { 6730 if (unlikely(!allow_spin)) { 6731 /* 6732 * __slab_free() can locklessly cmpxchg16 into a slab, 6733 * but then it might need to take spin_lock or local_lock 6734 * in put_cpu_partial() for further processing. 6735 * Avoid the complexity and simply add to a deferred list. 6736 */ 6737 defer_free(s, head); 6738 } else { 6739 __slab_free(s, slab, head, tail, cnt, addr); 6740 } 6741 return; 6742 } 6743 6744 if (unlikely(!allow_spin)) { 6745 if ((in_nmi() || !USE_LOCKLESS_FAST_PATH()) && 6746 local_lock_is_locked(&s->cpu_slab->lock)) { 6747 defer_free(s, head); 6748 return; 6749 } 6750 cnt = 1; /* restore cnt. kfree_nolock() frees one object at a time */ 6751 } 6752 6753 if (USE_LOCKLESS_FAST_PATH()) { 6754 freelist = READ_ONCE(c->freelist); 6755 6756 set_freepointer(s, tail, freelist); 6757 6758 if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) { 6759 note_cmpxchg_failure("slab_free", s, tid); 6760 goto redo; 6761 } 6762 } else { 6763 __maybe_unused unsigned long flags = 0; 6764 6765 /* Update the free list under the local lock */ 6766 local_lock_cpu_slab(s, flags); 6767 c = this_cpu_ptr(s->cpu_slab); 6768 if (unlikely(slab != c->slab)) { 6769 local_unlock_cpu_slab(s, flags); 6770 goto redo; 6771 } 6772 tid = c->tid; 6773 freelist = c->freelist; 6774 6775 set_freepointer(s, tail, freelist); 6776 c->freelist = head; 6777 c->tid = next_tid(tid); 6778 6779 local_unlock_cpu_slab(s, flags); 6780 } 6781 stat_add(s, FREE_FASTPATH, cnt); 6782} 6783 6784static __fastpath_inline 6785void slab_free(struct kmem_cache *s, struct slab *slab, void *object, 6786 unsigned long addr) 6787{ 6788 memcg_slab_free_hook(s, slab, &object, 1); 6789 alloc_tagging_slab_free_hook(s, slab, &object, 1); 6790 6791 if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false))) 6792 return; 6793 6794 if (s->cpu_sheaves && likely(!IS_ENABLED(CONFIG_NUMA) || 6795 slab_nid(slab) == numa_mem_id()) 6796 && likely(!slab_test_pfmemalloc(slab))) { 6797 if (likely(free_to_pcs(s, object))) 6798 return; 6799 } 6800 6801 do_slab_free(s, slab, object, object, 1, addr); 6802} 6803 6804#ifdef CONFIG_MEMCG 6805/* Do not inline the rare memcg charging failed path into the allocation path */ 6806static noinline 6807void memcg_alloc_abort_single(struct kmem_cache *s, void *object) 6808{ 6809 struct slab *slab = virt_to_slab(object); 6810 6811 alloc_tagging_slab_free_hook(s, slab, &object, 1); 6812 6813 if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false))) 6814 do_slab_free(s, slab, object, object, 1, _RET_IP_); 6815} 6816#endif 6817 6818static __fastpath_inline 6819void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head, 6820 void *tail, void **p, int cnt, unsigned long addr) 6821{ 6822 memcg_slab_free_hook(s, slab, p, cnt); 6823 alloc_tagging_slab_free_hook(s, slab, p, cnt); 6824 /* 6825 * With KASAN enabled slab_free_freelist_hook modifies the freelist 6826 * to remove objects, whose reuse must be delayed. 6827 */ 6828 if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt))) 6829 do_slab_free(s, slab, head, tail, cnt, addr); 6830} 6831 6832#ifdef CONFIG_SLUB_RCU_DEBUG 6833static void slab_free_after_rcu_debug(struct rcu_head *rcu_head) 6834{ 6835 struct rcu_delayed_free *delayed_free = 6836 container_of(rcu_head, struct rcu_delayed_free, head); 6837 void *object = delayed_free->object; 6838 struct slab *slab = virt_to_slab(object); 6839 struct kmem_cache *s; 6840 6841 kfree(delayed_free); 6842 6843 if (WARN_ON(is_kfence_address(object))) 6844 return; 6845 6846 /* find the object and the cache again */ 6847 if (WARN_ON(!slab)) 6848 return; 6849 s = slab->slab_cache; 6850 if (WARN_ON(!(s->flags & SLAB_TYPESAFE_BY_RCU))) 6851 return; 6852 6853 /* resume freeing */ 6854 if (slab_free_hook(s, object, slab_want_init_on_free(s), true)) 6855 do_slab_free(s, slab, object, object, 1, _THIS_IP_); 6856} 6857#endif /* CONFIG_SLUB_RCU_DEBUG */ 6858 6859#ifdef CONFIG_KASAN_GENERIC 6860void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) 6861{ 6862 do_slab_free(cache, virt_to_slab(x), x, x, 1, addr); 6863} 6864#endif 6865 6866static inline struct kmem_cache *virt_to_cache(const void *obj) 6867{ 6868 struct slab *slab; 6869 6870 slab = virt_to_slab(obj); 6871 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__)) 6872 return NULL; 6873 return slab->slab_cache; 6874} 6875 6876static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) 6877{ 6878 struct kmem_cache *cachep; 6879 6880 if (!IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) && 6881 !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) 6882 return s; 6883 6884 cachep = virt_to_cache(x); 6885 if (WARN(cachep && cachep != s, 6886 "%s: Wrong slab cache. %s but object is from %s\n", 6887 __func__, s->name, cachep->name)) 6888 print_tracking(cachep, x); 6889 return cachep; 6890} 6891 6892/** 6893 * kmem_cache_free - Deallocate an object 6894 * @s: The cache the allocation was from. 6895 * @x: The previously allocated object. 6896 * 6897 * Free an object which was previously allocated from this 6898 * cache. 6899 */ 6900void kmem_cache_free(struct kmem_cache *s, void *x) 6901{ 6902 s = cache_from_obj(s, x); 6903 if (!s) 6904 return; 6905 trace_kmem_cache_free(_RET_IP_, x, s); 6906 slab_free(s, virt_to_slab(x), x, _RET_IP_); 6907} 6908EXPORT_SYMBOL(kmem_cache_free); 6909 6910static void free_large_kmalloc(struct page *page, void *object) 6911{ 6912 unsigned int order = compound_order(page); 6913 6914 if (WARN_ON_ONCE(!PageLargeKmalloc(page))) { 6915 dump_page(page, "Not a kmalloc allocation"); 6916 return; 6917 } 6918 6919 if (WARN_ON_ONCE(order == 0)) 6920 pr_warn_once("object pointer: 0x%p\n", object); 6921 6922 kmemleak_free(object); 6923 kasan_kfree_large(object); 6924 kmsan_kfree_large(object); 6925 6926 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, 6927 -(PAGE_SIZE << order)); 6928 __ClearPageLargeKmalloc(page); 6929 free_frozen_pages(page, order); 6930} 6931 6932/* 6933 * Given an rcu_head embedded within an object obtained from kvmalloc at an 6934 * offset < 4k, free the object in question. 6935 */ 6936void kvfree_rcu_cb(struct rcu_head *head) 6937{ 6938 void *obj = head; 6939 struct page *page; 6940 struct slab *slab; 6941 struct kmem_cache *s; 6942 void *slab_addr; 6943 6944 if (is_vmalloc_addr(obj)) { 6945 obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); 6946 vfree(obj); 6947 return; 6948 } 6949 6950 page = virt_to_page(obj); 6951 slab = page_slab(page); 6952 if (!slab) { 6953 /* 6954 * rcu_head offset can be only less than page size so no need to 6955 * consider allocation order 6956 */ 6957 obj = (void *) PAGE_ALIGN_DOWN((unsigned long)obj); 6958 free_large_kmalloc(page, obj); 6959 return; 6960 } 6961 6962 s = slab->slab_cache; 6963 slab_addr = slab_address(slab); 6964 6965 if (is_kfence_address(obj)) { 6966 obj = kfence_object_start(obj); 6967 } else { 6968 unsigned int idx = __obj_to_index(s, slab_addr, obj); 6969 6970 obj = slab_addr + s->size * idx; 6971 obj = fixup_red_left(s, obj); 6972 } 6973 6974 slab_free(s, slab, obj, _RET_IP_); 6975} 6976 6977/** 6978 * kfree - free previously allocated memory 6979 * @object: pointer returned by kmalloc() or kmem_cache_alloc() 6980 * 6981 * If @object is NULL, no operation is performed. 6982 */ 6983void kfree(const void *object) 6984{ 6985 struct page *page; 6986 struct slab *slab; 6987 struct kmem_cache *s; 6988 void *x = (void *)object; 6989 6990 trace_kfree(_RET_IP_, object); 6991 6992 if (unlikely(ZERO_OR_NULL_PTR(object))) 6993 return; 6994 6995 page = virt_to_page(object); 6996 slab = page_slab(page); 6997 if (!slab) { 6998 free_large_kmalloc(page, (void *)object); 6999 return; 7000 } 7001 7002 s = slab->slab_cache; 7003 slab_free(s, slab, x, _RET_IP_); 7004} 7005EXPORT_SYMBOL(kfree); 7006 7007/* 7008 * Can be called while holding raw_spinlock_t or from IRQ and NMI, 7009 * but ONLY for objects allocated by kmalloc_nolock(). 7010 * Debug checks (like kmemleak and kfence) were skipped on allocation, 7011 * hence 7012 * obj = kmalloc(); kfree_nolock(obj); 7013 * will miss kmemleak/kfence book keeping and will cause false positives. 7014 * large_kmalloc is not supported either. 7015 */ 7016void kfree_nolock(const void *object) 7017{ 7018 struct slab *slab; 7019 struct kmem_cache *s; 7020 void *x = (void *)object; 7021 7022 if (unlikely(ZERO_OR_NULL_PTR(object))) 7023 return; 7024 7025 slab = virt_to_slab(object); 7026 if (unlikely(!slab)) { 7027 WARN_ONCE(1, "large_kmalloc is not supported by kfree_nolock()"); 7028 return; 7029 } 7030 7031 s = slab->slab_cache; 7032 7033 memcg_slab_free_hook(s, slab, &x, 1); 7034 alloc_tagging_slab_free_hook(s, slab, &x, 1); 7035 /* 7036 * Unlike slab_free() do NOT call the following: 7037 * kmemleak_free_recursive(x, s->flags); 7038 * debug_check_no_locks_freed(x, s->object_size); 7039 * debug_check_no_obj_freed(x, s->object_size); 7040 * __kcsan_check_access(x, s->object_size, ..); 7041 * kfence_free(x); 7042 * since they take spinlocks or not safe from any context. 7043 */ 7044 kmsan_slab_free(s, x); 7045 /* 7046 * If KASAN finds a kernel bug it will do kasan_report_invalid_free() 7047 * which will call raw_spin_lock_irqsave() which is technically 7048 * unsafe from NMI, but take chance and report kernel bug. 7049 * The sequence of 7050 * kasan_report_invalid_free() -> raw_spin_lock_irqsave() -> NMI 7051 * -> kfree_nolock() -> kasan_report_invalid_free() on the same CPU 7052 * is double buggy and deserves to deadlock. 7053 */ 7054 if (kasan_slab_pre_free(s, x)) 7055 return; 7056 /* 7057 * memcg, kasan_slab_pre_free are done for 'x'. 7058 * The only thing left is kasan_poison without quarantine, 7059 * since kasan quarantine takes locks and not supported from NMI. 7060 */ 7061 kasan_slab_free(s, x, false, false, /* skip quarantine */true); 7062 do_slab_free(s, slab, x, x, 0, _RET_IP_); 7063} 7064EXPORT_SYMBOL_GPL(kfree_nolock); 7065 7066static __always_inline __realloc_size(2) void * 7067__do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid) 7068{ 7069 void *ret; 7070 size_t ks = 0; 7071 int orig_size = 0; 7072 struct kmem_cache *s = NULL; 7073 7074 if (unlikely(ZERO_OR_NULL_PTR(p))) 7075 goto alloc_new; 7076 7077 /* Check for double-free. */ 7078 if (!kasan_check_byte(p)) 7079 return NULL; 7080 7081 /* 7082 * If reallocation is not necessary (e. g. the new size is less 7083 * than the current allocated size), the current allocation will be 7084 * preserved unless __GFP_THISNODE is set. In the latter case a new 7085 * allocation on the requested node will be attempted. 7086 */ 7087 if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE && 7088 nid != page_to_nid(virt_to_page(p))) 7089 goto alloc_new; 7090 7091 if (is_kfence_address(p)) { 7092 ks = orig_size = kfence_ksize(p); 7093 } else { 7094 struct page *page = virt_to_page(p); 7095 struct slab *slab = page_slab(page); 7096 7097 if (!slab) { 7098 /* Big kmalloc object */ 7099 ks = page_size(page); 7100 WARN_ON(ks <= KMALLOC_MAX_CACHE_SIZE); 7101 WARN_ON(p != page_address(page)); 7102 } else { 7103 s = slab->slab_cache; 7104 orig_size = get_orig_size(s, (void *)p); 7105 ks = s->object_size; 7106 } 7107 } 7108 7109 /* If the old object doesn't fit, allocate a bigger one */ 7110 if (new_size > ks) 7111 goto alloc_new; 7112 7113 /* If the old object doesn't satisfy the new alignment, allocate a new one */ 7114 if (!IS_ALIGNED((unsigned long)p, align)) 7115 goto alloc_new; 7116 7117 /* Zero out spare memory. */ 7118 if (want_init_on_alloc(flags)) { 7119 kasan_disable_current(); 7120 if (orig_size && orig_size < new_size) 7121 memset(kasan_reset_tag(p) + orig_size, 0, new_size - orig_size); 7122 else 7123 memset(kasan_reset_tag(p) + new_size, 0, ks - new_size); 7124 kasan_enable_current(); 7125 } 7126 7127 /* Setup kmalloc redzone when needed */ 7128 if (s && slub_debug_orig_size(s)) { 7129 set_orig_size(s, (void *)p, new_size); 7130 if (s->flags & SLAB_RED_ZONE && new_size < ks) 7131 memset_no_sanitize_memory(kasan_reset_tag(p) + new_size, 7132 SLUB_RED_ACTIVE, ks - new_size); 7133 } 7134 7135 p = kasan_krealloc(p, new_size, flags); 7136 return (void *)p; 7137 7138alloc_new: 7139 ret = kmalloc_node_track_caller_noprof(new_size, flags, nid, _RET_IP_); 7140 if (ret && p) { 7141 /* Disable KASAN checks as the object's redzone is accessed. */ 7142 kasan_disable_current(); 7143 memcpy(ret, kasan_reset_tag(p), orig_size ?: ks); 7144 kasan_enable_current(); 7145 } 7146 7147 return ret; 7148} 7149 7150/** 7151 * krealloc_node_align - reallocate memory. The contents will remain unchanged. 7152 * @p: object to reallocate memory for. 7153 * @new_size: how many bytes of memory are required. 7154 * @align: desired alignment. 7155 * @flags: the type of memory to allocate. 7156 * @nid: NUMA node or NUMA_NO_NODE 7157 * 7158 * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size 7159 * is 0 and @p is not a %NULL pointer, the object pointed to is freed. 7160 * 7161 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 7162 * Documentation/core-api/memory-allocation.rst for more details. 7163 * 7164 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 7165 * initial memory allocation, every subsequent call to this API for the same 7166 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 7167 * __GFP_ZERO is not fully honored by this API. 7168 * 7169 * When slub_debug_orig_size() is off, krealloc() only knows about the bucket 7170 * size of an allocation (but not the exact size it was allocated with) and 7171 * hence implements the following semantics for shrinking and growing buffers 7172 * with __GFP_ZERO:: 7173 * 7174 * new bucket 7175 * 0 size size 7176 * |--------|----------------| 7177 * | keep | zero | 7178 * 7179 * Otherwise, the original allocation size 'orig_size' could be used to 7180 * precisely clear the requested size, and the new size will also be stored 7181 * as the new 'orig_size'. 7182 * 7183 * In any case, the contents of the object pointed to are preserved up to the 7184 * lesser of the new and old sizes. 7185 * 7186 * Return: pointer to the allocated memory or %NULL in case of error 7187 */ 7188void *krealloc_node_align_noprof(const void *p, size_t new_size, unsigned long align, 7189 gfp_t flags, int nid) 7190{ 7191 void *ret; 7192 7193 if (unlikely(!new_size)) { 7194 kfree(p); 7195 return ZERO_SIZE_PTR; 7196 } 7197 7198 ret = __do_krealloc(p, new_size, align, flags, nid); 7199 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret)) 7200 kfree(p); 7201 7202 return ret; 7203} 7204EXPORT_SYMBOL(krealloc_node_align_noprof); 7205 7206static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) 7207{ 7208 /* 7209 * We want to attempt a large physically contiguous block first because 7210 * it is less likely to fragment multiple larger blocks and therefore 7211 * contribute to a long term fragmentation less than vmalloc fallback. 7212 * However make sure that larger requests are not too disruptive - i.e. 7213 * do not direct reclaim unless physically continuous memory is preferred 7214 * (__GFP_RETRY_MAYFAIL mode). We still kick in kswapd/kcompactd to 7215 * start working in the background 7216 */ 7217 if (size > PAGE_SIZE) { 7218 flags |= __GFP_NOWARN; 7219 7220 if (!(flags & __GFP_RETRY_MAYFAIL)) 7221 flags &= ~__GFP_DIRECT_RECLAIM; 7222 7223 /* nofail semantic is implemented by the vmalloc fallback */ 7224 flags &= ~__GFP_NOFAIL; 7225 } 7226 7227 return flags; 7228} 7229 7230/** 7231 * __kvmalloc_node - attempt to allocate physically contiguous memory, but upon 7232 * failure, fall back to non-contiguous (vmalloc) allocation. 7233 * @size: size of the request. 7234 * @b: which set of kmalloc buckets to allocate from. 7235 * @align: desired alignment. 7236 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. 7237 * @node: numa node to allocate from 7238 * 7239 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 7240 * Documentation/core-api/memory-allocation.rst for more details. 7241 * 7242 * Uses kmalloc to get the memory but if the allocation fails then falls back 7243 * to the vmalloc allocator. Use kvfree for freeing the memory. 7244 * 7245 * GFP_NOWAIT and GFP_ATOMIC are supported, the __GFP_NORETRY modifier is not. 7246 * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is 7247 * preferable to the vmalloc fallback, due to visible performance drawbacks. 7248 * 7249 * Return: pointer to the allocated memory of %NULL in case of failure 7250 */ 7251void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align, 7252 gfp_t flags, int node) 7253{ 7254 bool allow_block; 7255 void *ret; 7256 7257 /* 7258 * It doesn't really make sense to fallback to vmalloc for sub page 7259 * requests 7260 */ 7261 ret = __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), 7262 kmalloc_gfp_adjust(flags, size), 7263 node, _RET_IP_); 7264 if (ret || size <= PAGE_SIZE) 7265 return ret; 7266 7267 /* Don't even allow crazy sizes */ 7268 if (unlikely(size > INT_MAX)) { 7269 WARN_ON_ONCE(!(flags & __GFP_NOWARN)); 7270 return NULL; 7271 } 7272 7273 /* 7274 * For non-blocking the VM_ALLOW_HUGE_VMAP is not used 7275 * because the huge-mapping path in vmalloc contains at 7276 * least one might_sleep() call. 7277 * 7278 * TODO: Revise huge-mapping path to support non-blocking 7279 * flags. 7280 */ 7281 allow_block = gfpflags_allow_blocking(flags); 7282 7283 /* 7284 * kvmalloc() can always use VM_ALLOW_HUGE_VMAP, 7285 * since the callers already cannot assume anything 7286 * about the resulting pointer, and cannot play 7287 * protection games. 7288 */ 7289 return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_END, 7290 flags, PAGE_KERNEL, allow_block ? VM_ALLOW_HUGE_VMAP:0, 7291 node, __builtin_return_address(0)); 7292} 7293EXPORT_SYMBOL(__kvmalloc_node_noprof); 7294 7295/** 7296 * kvfree() - Free memory. 7297 * @addr: Pointer to allocated memory. 7298 * 7299 * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc(). 7300 * It is slightly more efficient to use kfree() or vfree() if you are certain 7301 * that you know which one to use. 7302 * 7303 * Context: Either preemptible task context or not-NMI interrupt. 7304 */ 7305void kvfree(const void *addr) 7306{ 7307 if (is_vmalloc_addr(addr)) 7308 vfree(addr); 7309 else 7310 kfree(addr); 7311} 7312EXPORT_SYMBOL(kvfree); 7313 7314/** 7315 * kvfree_sensitive - Free a data object containing sensitive information. 7316 * @addr: address of the data object to be freed. 7317 * @len: length of the data object. 7318 * 7319 * Use the special memzero_explicit() function to clear the content of a 7320 * kvmalloc'ed object containing sensitive data to make sure that the 7321 * compiler won't optimize out the data clearing. 7322 */ 7323void kvfree_sensitive(const void *addr, size_t len) 7324{ 7325 if (likely(!ZERO_OR_NULL_PTR(addr))) { 7326 memzero_explicit((void *)addr, len); 7327 kvfree(addr); 7328 } 7329} 7330EXPORT_SYMBOL(kvfree_sensitive); 7331 7332/** 7333 * kvrealloc_node_align - reallocate memory; contents remain unchanged 7334 * @p: object to reallocate memory for 7335 * @size: the size to reallocate 7336 * @align: desired alignment 7337 * @flags: the flags for the page level allocator 7338 * @nid: NUMA node id 7339 * 7340 * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 7341 * and @p is not a %NULL pointer, the object pointed to is freed. 7342 * 7343 * Only alignments up to those guaranteed by kmalloc() will be honored. Please see 7344 * Documentation/core-api/memory-allocation.rst for more details. 7345 * 7346 * If __GFP_ZERO logic is requested, callers must ensure that, starting with the 7347 * initial memory allocation, every subsequent call to this API for the same 7348 * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that 7349 * __GFP_ZERO is not fully honored by this API. 7350 * 7351 * In any case, the contents of the object pointed to are preserved up to the 7352 * lesser of the new and old sizes. 7353 * 7354 * This function must not be called concurrently with itself or kvfree() for the 7355 * same memory allocation. 7356 * 7357 * Return: pointer to the allocated memory or %NULL in case of error 7358 */ 7359void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, 7360 gfp_t flags, int nid) 7361{ 7362 void *n; 7363 7364 if (is_vmalloc_addr(p)) 7365 return vrealloc_node_align_noprof(p, size, align, flags, nid); 7366 7367 n = krealloc_node_align_noprof(p, size, align, kmalloc_gfp_adjust(flags, size), nid); 7368 if (!n) { 7369 /* We failed to krealloc(), fall back to kvmalloc(). */ 7370 n = kvmalloc_node_align_noprof(size, align, flags, nid); 7371 if (!n) 7372 return NULL; 7373 7374 if (p) { 7375 /* We already know that `p` is not a vmalloc address. */ 7376 kasan_disable_current(); 7377 memcpy(n, kasan_reset_tag(p), ksize(p)); 7378 kasan_enable_current(); 7379 7380 kfree(p); 7381 } 7382 } 7383 7384 return n; 7385} 7386EXPORT_SYMBOL(kvrealloc_node_align_noprof); 7387 7388struct detached_freelist { 7389 struct slab *slab; 7390 void *tail; 7391 void *freelist; 7392 int cnt; 7393 struct kmem_cache *s; 7394}; 7395 7396/* 7397 * This function progressively scans the array with free objects (with 7398 * a limited look ahead) and extract objects belonging to the same 7399 * slab. It builds a detached freelist directly within the given 7400 * slab/objects. This can happen without any need for 7401 * synchronization, because the objects are owned by running process. 7402 * The freelist is build up as a single linked list in the objects. 7403 * The idea is, that this detached freelist can then be bulk 7404 * transferred to the real freelist(s), but only requiring a single 7405 * synchronization primitive. Look ahead in the array is limited due 7406 * to performance reasons. 7407 */ 7408static inline 7409int build_detached_freelist(struct kmem_cache *s, size_t size, 7410 void **p, struct detached_freelist *df) 7411{ 7412 int lookahead = 3; 7413 void *object; 7414 struct page *page; 7415 struct slab *slab; 7416 size_t same; 7417 7418 object = p[--size]; 7419 page = virt_to_page(object); 7420 slab = page_slab(page); 7421 if (!s) { 7422 /* Handle kalloc'ed objects */ 7423 if (!slab) { 7424 free_large_kmalloc(page, object); 7425 df->slab = NULL; 7426 return size; 7427 } 7428 /* Derive kmem_cache from object */ 7429 df->slab = slab; 7430 df->s = slab->slab_cache; 7431 } else { 7432 df->slab = slab; 7433 df->s = cache_from_obj(s, object); /* Support for memcg */ 7434 } 7435 7436 /* Start new detached freelist */ 7437 df->tail = object; 7438 df->freelist = object; 7439 df->cnt = 1; 7440 7441 if (is_kfence_address(object)) 7442 return size; 7443 7444 set_freepointer(df->s, object, NULL); 7445 7446 same = size; 7447 while (size) { 7448 object = p[--size]; 7449 /* df->slab is always set at this point */ 7450 if (df->slab == virt_to_slab(object)) { 7451 /* Opportunity build freelist */ 7452 set_freepointer(df->s, object, df->freelist); 7453 df->freelist = object; 7454 df->cnt++; 7455 same--; 7456 if (size != same) 7457 swap(p[size], p[same]); 7458 continue; 7459 } 7460 7461 /* Limit look ahead search */ 7462 if (!--lookahead) 7463 break; 7464 } 7465 7466 return same; 7467} 7468 7469/* 7470 * Internal bulk free of objects that were not initialised by the post alloc 7471 * hooks and thus should not be processed by the free hooks 7472 */ 7473static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 7474{ 7475 if (!size) 7476 return; 7477 7478 do { 7479 struct detached_freelist df; 7480 7481 size = build_detached_freelist(s, size, p, &df); 7482 if (!df.slab) 7483 continue; 7484 7485 if (kfence_free(df.freelist)) 7486 continue; 7487 7488 do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, 7489 _RET_IP_); 7490 } while (likely(size)); 7491} 7492 7493/* Note that interrupts must be enabled when calling this function. */ 7494void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) 7495{ 7496 if (!size) 7497 return; 7498 7499 /* 7500 * freeing to sheaves is so incompatible with the detached freelist so 7501 * once we go that way, we have to do everything differently 7502 */ 7503 if (s && s->cpu_sheaves) { 7504 free_to_pcs_bulk(s, size, p); 7505 return; 7506 } 7507 7508 do { 7509 struct detached_freelist df; 7510 7511 size = build_detached_freelist(s, size, p, &df); 7512 if (!df.slab) 7513 continue; 7514 7515 slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size], 7516 df.cnt, _RET_IP_); 7517 } while (likely(size)); 7518} 7519EXPORT_SYMBOL(kmem_cache_free_bulk); 7520 7521static inline 7522int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, 7523 void **p) 7524{ 7525 struct kmem_cache_cpu *c; 7526 unsigned long irqflags; 7527 int i; 7528 7529 /* 7530 * Drain objects in the per cpu slab, while disabling local 7531 * IRQs, which protects against PREEMPT and interrupts 7532 * handlers invoking normal fastpath. 7533 */ 7534 c = slub_get_cpu_ptr(s->cpu_slab); 7535 local_lock_irqsave(&s->cpu_slab->lock, irqflags); 7536 7537 for (i = 0; i < size; i++) { 7538 void *object = c->freelist; 7539 7540 if (unlikely(!object)) { 7541 /* 7542 * We may have removed an object from c->freelist using 7543 * the fastpath in the previous iteration; in that case, 7544 * c->tid has not been bumped yet. 7545 * Since ___slab_alloc() may reenable interrupts while 7546 * allocating memory, we should bump c->tid now. 7547 */ 7548 c->tid = next_tid(c->tid); 7549 7550 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); 7551 7552 /* 7553 * Invoking slow path likely have side-effect 7554 * of re-populating per CPU c->freelist 7555 */ 7556 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, 7557 _RET_IP_, c, s->object_size); 7558 if (unlikely(!p[i])) 7559 goto error; 7560 7561 c = this_cpu_ptr(s->cpu_slab); 7562 maybe_wipe_obj_freeptr(s, p[i]); 7563 7564 local_lock_irqsave(&s->cpu_slab->lock, irqflags); 7565 7566 continue; /* goto for-loop */ 7567 } 7568 c->freelist = get_freepointer(s, object); 7569 p[i] = object; 7570 maybe_wipe_obj_freeptr(s, p[i]); 7571 stat(s, ALLOC_FASTPATH); 7572 } 7573 c->tid = next_tid(c->tid); 7574 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); 7575 slub_put_cpu_ptr(s->cpu_slab); 7576 7577 return i; 7578 7579error: 7580 slub_put_cpu_ptr(s->cpu_slab); 7581 __kmem_cache_free_bulk(s, i, p); 7582 return 0; 7583 7584} 7585 7586/* Note that interrupts must be enabled when calling this function. */ 7587int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, 7588 void **p) 7589{ 7590 unsigned int i = 0; 7591 void *kfence_obj; 7592 7593 if (!size) 7594 return 0; 7595 7596 s = slab_pre_alloc_hook(s, flags); 7597 if (unlikely(!s)) 7598 return 0; 7599 7600 /* 7601 * to make things simpler, only assume at most once kfence allocated 7602 * object per bulk allocation and choose its index randomly 7603 */ 7604 kfence_obj = kfence_alloc(s, s->object_size, flags); 7605 7606 if (unlikely(kfence_obj)) { 7607 if (unlikely(size == 1)) { 7608 p[0] = kfence_obj; 7609 goto out; 7610 } 7611 size--; 7612 } 7613 7614 if (s->cpu_sheaves) 7615 i = alloc_from_pcs_bulk(s, size, p); 7616 7617 if (i < size) { 7618 /* 7619 * If we ran out of memory, don't bother with freeing back to 7620 * the percpu sheaves, we have bigger problems. 7621 */ 7622 if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) { 7623 if (i > 0) 7624 __kmem_cache_free_bulk(s, i, p); 7625 if (kfence_obj) 7626 __kfence_free(kfence_obj); 7627 return 0; 7628 } 7629 } 7630 7631 if (unlikely(kfence_obj)) { 7632 int idx = get_random_u32_below(size + 1); 7633 7634 if (idx != size) 7635 p[size] = p[idx]; 7636 p[idx] = kfence_obj; 7637 7638 size++; 7639 } 7640 7641out: 7642 /* 7643 * memcg and kmem_cache debug support and memory initialization. 7644 * Done outside of the IRQ disabled fastpath loop. 7645 */ 7646 if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p, 7647 slab_want_init_on_alloc(flags, s), s->object_size))) { 7648 return 0; 7649 } 7650 7651 return size; 7652} 7653EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof); 7654 7655/* 7656 * Object placement in a slab is made very easy because we always start at 7657 * offset 0. If we tune the size of the object to the alignment then we can 7658 * get the required alignment by putting one properly sized object after 7659 * another. 7660 * 7661 * Notice that the allocation order determines the sizes of the per cpu 7662 * caches. Each processor has always one slab available for allocations. 7663 * Increasing the allocation order reduces the number of times that slabs 7664 * must be moved on and off the partial lists and is therefore a factor in 7665 * locking overhead. 7666 */ 7667 7668/* 7669 * Minimum / Maximum order of slab pages. This influences locking overhead 7670 * and slab fragmentation. A higher order reduces the number of partial slabs 7671 * and increases the number of allocations possible without having to 7672 * take the list_lock. 7673 */ 7674static unsigned int slub_min_order; 7675static unsigned int slub_max_order = 7676 IS_ENABLED(CONFIG_SLUB_TINY) ? 1 : PAGE_ALLOC_COSTLY_ORDER; 7677static unsigned int slub_min_objects; 7678 7679/* 7680 * Calculate the order of allocation given an slab object size. 7681 * 7682 * The order of allocation has significant impact on performance and other 7683 * system components. Generally order 0 allocations should be preferred since 7684 * order 0 does not cause fragmentation in the page allocator. Larger objects 7685 * be problematic to put into order 0 slabs because there may be too much 7686 * unused space left. We go to a higher order if more than 1/16th of the slab 7687 * would be wasted. 7688 * 7689 * In order to reach satisfactory performance we must ensure that a minimum 7690 * number of objects is in one slab. Otherwise we may generate too much 7691 * activity on the partial lists which requires taking the list_lock. This is 7692 * less a concern for large slabs though which are rarely used. 7693 * 7694 * slab_max_order specifies the order where we begin to stop considering the 7695 * number of objects in a slab as critical. If we reach slab_max_order then 7696 * we try to keep the page order as low as possible. So we accept more waste 7697 * of space in favor of a small page order. 7698 * 7699 * Higher order allocations also allow the placement of more objects in a 7700 * slab and thereby reduce object handling overhead. If the user has 7701 * requested a higher minimum order then we start with that one instead of 7702 * the smallest order which will fit the object. 7703 */ 7704static inline unsigned int calc_slab_order(unsigned int size, 7705 unsigned int min_order, unsigned int max_order, 7706 unsigned int fract_leftover) 7707{ 7708 unsigned int order; 7709 7710 for (order = min_order; order <= max_order; order++) { 7711 7712 unsigned int slab_size = (unsigned int)PAGE_SIZE << order; 7713 unsigned int rem; 7714 7715 rem = slab_size % size; 7716 7717 if (rem <= slab_size / fract_leftover) 7718 break; 7719 } 7720 7721 return order; 7722} 7723 7724static inline int calculate_order(unsigned int size) 7725{ 7726 unsigned int order; 7727 unsigned int min_objects; 7728 unsigned int max_objects; 7729 unsigned int min_order; 7730 7731 min_objects = slub_min_objects; 7732 if (!min_objects) { 7733 /* 7734 * Some architectures will only update present cpus when 7735 * onlining them, so don't trust the number if it's just 1. But 7736 * we also don't want to use nr_cpu_ids always, as on some other 7737 * architectures, there can be many possible cpus, but never 7738 * onlined. Here we compromise between trying to avoid too high 7739 * order on systems that appear larger than they are, and too 7740 * low order on systems that appear smaller than they are. 7741 */ 7742 unsigned int nr_cpus = num_present_cpus(); 7743 if (nr_cpus <= 1) 7744 nr_cpus = nr_cpu_ids; 7745 min_objects = 4 * (fls(nr_cpus) + 1); 7746 } 7747 /* min_objects can't be 0 because get_order(0) is undefined */ 7748 max_objects = max(order_objects(slub_max_order, size), 1U); 7749 min_objects = min(min_objects, max_objects); 7750 7751 min_order = max_t(unsigned int, slub_min_order, 7752 get_order(min_objects * size)); 7753 if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE) 7754 return get_order(size * MAX_OBJS_PER_PAGE) - 1; 7755 7756 /* 7757 * Attempt to find best configuration for a slab. This works by first 7758 * attempting to generate a layout with the best possible configuration 7759 * and backing off gradually. 7760 * 7761 * We start with accepting at most 1/16 waste and try to find the 7762 * smallest order from min_objects-derived/slab_min_order up to 7763 * slab_max_order that will satisfy the constraint. Note that increasing 7764 * the order can only result in same or less fractional waste, not more. 7765 * 7766 * If that fails, we increase the acceptable fraction of waste and try 7767 * again. The last iteration with fraction of 1/2 would effectively 7768 * accept any waste and give us the order determined by min_objects, as 7769 * long as at least single object fits within slab_max_order. 7770 */ 7771 for (unsigned int fraction = 16; fraction > 1; fraction /= 2) { 7772 order = calc_slab_order(size, min_order, slub_max_order, 7773 fraction); 7774 if (order <= slub_max_order) 7775 return order; 7776 } 7777 7778 /* 7779 * Doh this slab cannot be placed using slab_max_order. 7780 */ 7781 order = get_order(size); 7782 if (order <= MAX_PAGE_ORDER) 7783 return order; 7784 return -ENOSYS; 7785} 7786 7787static void 7788init_kmem_cache_node(struct kmem_cache_node *n, struct node_barn *barn) 7789{ 7790 n->nr_partial = 0; 7791 spin_lock_init(&n->list_lock); 7792 INIT_LIST_HEAD(&n->partial); 7793#ifdef CONFIG_SLUB_DEBUG 7794 atomic_long_set(&n->nr_slabs, 0); 7795 atomic_long_set(&n->total_objects, 0); 7796 INIT_LIST_HEAD(&n->full); 7797#endif 7798 n->barn = barn; 7799 if (barn) 7800 barn_init(barn); 7801} 7802 7803static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 7804{ 7805 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < 7806 NR_KMALLOC_TYPES * KMALLOC_SHIFT_HIGH * 7807 sizeof(struct kmem_cache_cpu)); 7808 7809 /* 7810 * Must align to double word boundary for the double cmpxchg 7811 * instructions to work; see __pcpu_double_call_return_bool(). 7812 */ 7813 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), 7814 2 * sizeof(void *)); 7815 7816 if (!s->cpu_slab) 7817 return 0; 7818 7819 init_kmem_cache_cpus(s); 7820 7821 return 1; 7822} 7823 7824static int init_percpu_sheaves(struct kmem_cache *s) 7825{ 7826 int cpu; 7827 7828 for_each_possible_cpu(cpu) { 7829 struct slub_percpu_sheaves *pcs; 7830 7831 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); 7832 7833 local_trylock_init(&pcs->lock); 7834 7835 pcs->main = alloc_empty_sheaf(s, GFP_KERNEL); 7836 7837 if (!pcs->main) 7838 return -ENOMEM; 7839 } 7840 7841 return 0; 7842} 7843 7844static struct kmem_cache *kmem_cache_node; 7845 7846/* 7847 * No kmalloc_node yet so do it by hand. We know that this is the first 7848 * slab on the node for this slabcache. There are no concurrent accesses 7849 * possible. 7850 * 7851 * Note that this function only works on the kmem_cache_node 7852 * when allocating for the kmem_cache_node. This is used for bootstrapping 7853 * memory on a fresh node that has no slab structures yet. 7854 */ 7855static void early_kmem_cache_node_alloc(int node) 7856{ 7857 struct slab *slab; 7858 struct kmem_cache_node *n; 7859 7860 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); 7861 7862 slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); 7863 7864 BUG_ON(!slab); 7865 if (slab_nid(slab) != node) { 7866 pr_err("SLUB: Unable to allocate memory from node %d\n", node); 7867 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); 7868 } 7869 7870 n = slab->freelist; 7871 BUG_ON(!n); 7872#ifdef CONFIG_SLUB_DEBUG 7873 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); 7874#endif 7875 n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false); 7876 slab->freelist = get_freepointer(kmem_cache_node, n); 7877 slab->inuse = 1; 7878 kmem_cache_node->node[node] = n; 7879 init_kmem_cache_node(n, NULL); 7880 inc_slabs_node(kmem_cache_node, node, slab->objects); 7881 7882 /* 7883 * No locks need to be taken here as it has just been 7884 * initialized and there is no concurrent access. 7885 */ 7886 __add_partial(n, slab, DEACTIVATE_TO_HEAD); 7887} 7888 7889static void free_kmem_cache_nodes(struct kmem_cache *s) 7890{ 7891 int node; 7892 struct kmem_cache_node *n; 7893 7894 for_each_kmem_cache_node(s, node, n) { 7895 if (n->barn) { 7896 WARN_ON(n->barn->nr_full); 7897 WARN_ON(n->barn->nr_empty); 7898 kfree(n->barn); 7899 n->barn = NULL; 7900 } 7901 7902 s->node[node] = NULL; 7903 kmem_cache_free(kmem_cache_node, n); 7904 } 7905} 7906 7907void __kmem_cache_release(struct kmem_cache *s) 7908{ 7909 cache_random_seq_destroy(s); 7910 if (s->cpu_sheaves) 7911 pcs_destroy(s); 7912#ifdef CONFIG_PREEMPT_RT 7913 if (s->cpu_slab) 7914 lockdep_unregister_key(&s->lock_key); 7915#endif 7916 free_percpu(s->cpu_slab); 7917 free_kmem_cache_nodes(s); 7918} 7919 7920static int init_kmem_cache_nodes(struct kmem_cache *s) 7921{ 7922 int node; 7923 7924 for_each_node_mask(node, slab_nodes) { 7925 struct kmem_cache_node *n; 7926 struct node_barn *barn = NULL; 7927 7928 if (slab_state == DOWN) { 7929 early_kmem_cache_node_alloc(node); 7930 continue; 7931 } 7932 7933 if (s->cpu_sheaves) { 7934 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node); 7935 7936 if (!barn) 7937 return 0; 7938 } 7939 7940 n = kmem_cache_alloc_node(kmem_cache_node, 7941 GFP_KERNEL, node); 7942 if (!n) { 7943 kfree(barn); 7944 return 0; 7945 } 7946 7947 init_kmem_cache_node(n, barn); 7948 7949 s->node[node] = n; 7950 } 7951 return 1; 7952} 7953 7954static void set_cpu_partial(struct kmem_cache *s) 7955{ 7956#ifdef CONFIG_SLUB_CPU_PARTIAL 7957 unsigned int nr_objects; 7958 7959 /* 7960 * cpu_partial determined the maximum number of objects kept in the 7961 * per cpu partial lists of a processor. 7962 * 7963 * Per cpu partial lists mainly contain slabs that just have one 7964 * object freed. If they are used for allocation then they can be 7965 * filled up again with minimal effort. The slab will never hit the 7966 * per node partial lists and therefore no locking will be required. 7967 * 7968 * For backwards compatibility reasons, this is determined as number 7969 * of objects, even though we now limit maximum number of pages, see 7970 * slub_set_cpu_partial() 7971 */ 7972 if (!kmem_cache_has_cpu_partial(s)) 7973 nr_objects = 0; 7974 else if (s->size >= PAGE_SIZE) 7975 nr_objects = 6; 7976 else if (s->size >= 1024) 7977 nr_objects = 24; 7978 else if (s->size >= 256) 7979 nr_objects = 52; 7980 else 7981 nr_objects = 120; 7982 7983 slub_set_cpu_partial(s, nr_objects); 7984#endif 7985} 7986 7987/* 7988 * calculate_sizes() determines the order and the distribution of data within 7989 * a slab object. 7990 */ 7991static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s) 7992{ 7993 slab_flags_t flags = s->flags; 7994 unsigned int size = s->object_size; 7995 unsigned int order; 7996 7997 /* 7998 * Round up object size to the next word boundary. We can only 7999 * place the free pointer at word boundaries and this determines 8000 * the possible location of the free pointer. 8001 */ 8002 size = ALIGN(size, sizeof(void *)); 8003 8004#ifdef CONFIG_SLUB_DEBUG 8005 /* 8006 * Determine if we can poison the object itself. If the user of 8007 * the slab may touch the object after free or before allocation 8008 * then we should never poison the object itself. 8009 */ 8010 if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) && 8011 !s->ctor) 8012 s->flags |= __OBJECT_POISON; 8013 else 8014 s->flags &= ~__OBJECT_POISON; 8015 8016 8017 /* 8018 * If we are Redzoning then check if there is some space between the 8019 * end of the object and the free pointer. If not then add an 8020 * additional word to have some bytes to store Redzone information. 8021 */ 8022 if ((flags & SLAB_RED_ZONE) && size == s->object_size) 8023 size += sizeof(void *); 8024#endif 8025 8026 /* 8027 * With that we have determined the number of bytes in actual use 8028 * by the object and redzoning. 8029 */ 8030 s->inuse = size; 8031 8032 if (((flags & SLAB_TYPESAFE_BY_RCU) && !args->use_freeptr_offset) || 8033 (flags & SLAB_POISON) || s->ctor || 8034 ((flags & SLAB_RED_ZONE) && 8035 (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) { 8036 /* 8037 * Relocate free pointer after the object if it is not 8038 * permitted to overwrite the first word of the object on 8039 * kmem_cache_free. 8040 * 8041 * This is the case if we do RCU, have a constructor, are 8042 * poisoning the objects, or are redzoning an object smaller 8043 * than sizeof(void *) or are redzoning an object with 8044 * slub_debug_orig_size() enabled, in which case the right 8045 * redzone may be extended. 8046 * 8047 * The assumption that s->offset >= s->inuse means free 8048 * pointer is outside of the object is used in the 8049 * freeptr_outside_object() function. If that is no 8050 * longer true, the function needs to be modified. 8051 */ 8052 s->offset = size; 8053 size += sizeof(void *); 8054 } else if ((flags & SLAB_TYPESAFE_BY_RCU) && args->use_freeptr_offset) { 8055 s->offset = args->freeptr_offset; 8056 } else { 8057 /* 8058 * Store freelist pointer near middle of object to keep 8059 * it away from the edges of the object to avoid small 8060 * sized over/underflows from neighboring allocations. 8061 */ 8062 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *)); 8063 } 8064 8065#ifdef CONFIG_SLUB_DEBUG 8066 if (flags & SLAB_STORE_USER) { 8067 /* 8068 * Need to store information about allocs and frees after 8069 * the object. 8070 */ 8071 size += 2 * sizeof(struct track); 8072 8073 /* Save the original kmalloc request size */ 8074 if (flags & SLAB_KMALLOC) 8075 size += sizeof(unsigned long); 8076 } 8077#endif 8078 8079 kasan_cache_create(s, &size, &s->flags); 8080#ifdef CONFIG_SLUB_DEBUG 8081 if (flags & SLAB_RED_ZONE) { 8082 /* 8083 * Add some empty padding so that we can catch 8084 * overwrites from earlier objects rather than let 8085 * tracking information or the free pointer be 8086 * corrupted if a user writes before the start 8087 * of the object. 8088 */ 8089 size += sizeof(void *); 8090 8091 s->red_left_pad = sizeof(void *); 8092 s->red_left_pad = ALIGN(s->red_left_pad, s->align); 8093 size += s->red_left_pad; 8094 } 8095#endif 8096 8097 /* 8098 * SLUB stores one object immediately after another beginning from 8099 * offset 0. In order to align the objects we have to simply size 8100 * each object to conform to the alignment. 8101 */ 8102 size = ALIGN(size, s->align); 8103 s->size = size; 8104 s->reciprocal_size = reciprocal_value(size); 8105 order = calculate_order(size); 8106 8107 if ((int)order < 0) 8108 return 0; 8109 8110 s->allocflags = __GFP_COMP; 8111 8112 if (s->flags & SLAB_CACHE_DMA) 8113 s->allocflags |= GFP_DMA; 8114 8115 if (s->flags & SLAB_CACHE_DMA32) 8116 s->allocflags |= GFP_DMA32; 8117 8118 if (s->flags & SLAB_RECLAIM_ACCOUNT) 8119 s->allocflags |= __GFP_RECLAIMABLE; 8120 8121 /* 8122 * Determine the number of objects per slab 8123 */ 8124 s->oo = oo_make(order, size); 8125 s->min = oo_make(get_order(size), size); 8126 8127 return !!oo_objects(s->oo); 8128} 8129 8130static void list_slab_objects(struct kmem_cache *s, struct slab *slab) 8131{ 8132#ifdef CONFIG_SLUB_DEBUG 8133 void *addr = slab_address(slab); 8134 void *p; 8135 8136 if (!slab_add_kunit_errors()) 8137 slab_bug(s, "Objects remaining on __kmem_cache_shutdown()"); 8138 8139 spin_lock(&object_map_lock); 8140 __fill_map(object_map, s, slab); 8141 8142 for_each_object(p, s, addr, slab->objects) { 8143 8144 if (!test_bit(__obj_to_index(s, addr, p), object_map)) { 8145 if (slab_add_kunit_errors()) 8146 continue; 8147 pr_err("Object 0x%p @offset=%tu\n", p, p - addr); 8148 print_tracking(s, p); 8149 } 8150 } 8151 spin_unlock(&object_map_lock); 8152 8153 __slab_err(slab); 8154#endif 8155} 8156 8157/* 8158 * Attempt to free all partial slabs on a node. 8159 * This is called from __kmem_cache_shutdown(). We must take list_lock 8160 * because sysfs file might still access partial list after the shutdowning. 8161 */ 8162static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) 8163{ 8164 LIST_HEAD(discard); 8165 struct slab *slab, *h; 8166 8167 BUG_ON(irqs_disabled()); 8168 spin_lock_irq(&n->list_lock); 8169 list_for_each_entry_safe(slab, h, &n->partial, slab_list) { 8170 if (!slab->inuse) { 8171 remove_partial(n, slab); 8172 list_add(&slab->slab_list, &discard); 8173 } else { 8174 list_slab_objects(s, slab); 8175 } 8176 } 8177 spin_unlock_irq(&n->list_lock); 8178 8179 list_for_each_entry_safe(slab, h, &discard, slab_list) 8180 discard_slab(s, slab); 8181} 8182 8183bool __kmem_cache_empty(struct kmem_cache *s) 8184{ 8185 int node; 8186 struct kmem_cache_node *n; 8187 8188 for_each_kmem_cache_node(s, node, n) 8189 if (n->nr_partial || node_nr_slabs(n)) 8190 return false; 8191 return true; 8192} 8193 8194/* 8195 * Release all resources used by a slab cache. 8196 */ 8197int __kmem_cache_shutdown(struct kmem_cache *s) 8198{ 8199 int node; 8200 struct kmem_cache_node *n; 8201 8202 flush_all_cpus_locked(s); 8203 8204 /* we might have rcu sheaves in flight */ 8205 if (s->cpu_sheaves) 8206 rcu_barrier(); 8207 8208 /* Attempt to free all objects */ 8209 for_each_kmem_cache_node(s, node, n) { 8210 if (n->barn) 8211 barn_shrink(s, n->barn); 8212 free_partial(s, n); 8213 if (n->nr_partial || node_nr_slabs(n)) 8214 return 1; 8215 } 8216 return 0; 8217} 8218 8219#ifdef CONFIG_PRINTK 8220void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) 8221{ 8222 void *base; 8223 int __maybe_unused i; 8224 unsigned int objnr; 8225 void *objp; 8226 void *objp0; 8227 struct kmem_cache *s = slab->slab_cache; 8228 struct track __maybe_unused *trackp; 8229 8230 kpp->kp_ptr = object; 8231 kpp->kp_slab = slab; 8232 kpp->kp_slab_cache = s; 8233 base = slab_address(slab); 8234 objp0 = kasan_reset_tag(object); 8235#ifdef CONFIG_SLUB_DEBUG 8236 objp = restore_red_left(s, objp0); 8237#else 8238 objp = objp0; 8239#endif 8240 objnr = obj_to_index(s, slab, objp); 8241 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp); 8242 objp = base + s->size * objnr; 8243 kpp->kp_objp = objp; 8244 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size 8245 || (objp - base) % s->size) || 8246 !(s->flags & SLAB_STORE_USER)) 8247 return; 8248#ifdef CONFIG_SLUB_DEBUG 8249 objp = fixup_red_left(s, objp); 8250 trackp = get_track(s, objp, TRACK_ALLOC); 8251 kpp->kp_ret = (void *)trackp->addr; 8252#ifdef CONFIG_STACKDEPOT 8253 { 8254 depot_stack_handle_t handle; 8255 unsigned long *entries; 8256 unsigned int nr_entries; 8257 8258 handle = READ_ONCE(trackp->handle); 8259 if (handle) { 8260 nr_entries = stack_depot_fetch(handle, &entries); 8261 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) 8262 kpp->kp_stack[i] = (void *)entries[i]; 8263 } 8264 8265 trackp = get_track(s, objp, TRACK_FREE); 8266 handle = READ_ONCE(trackp->handle); 8267 if (handle) { 8268 nr_entries = stack_depot_fetch(handle, &entries); 8269 for (i = 0; i < KS_ADDRS_COUNT && i < nr_entries; i++) 8270 kpp->kp_free_stack[i] = (void *)entries[i]; 8271 } 8272 } 8273#endif 8274#endif 8275} 8276#endif 8277 8278/******************************************************************** 8279 * Kmalloc subsystem 8280 *******************************************************************/ 8281 8282static int __init setup_slub_min_order(const char *str, const struct kernel_param *kp) 8283{ 8284 int ret; 8285 8286 ret = kstrtouint(str, 0, &slub_min_order); 8287 if (ret) 8288 return ret; 8289 8290 if (slub_min_order > slub_max_order) 8291 slub_max_order = slub_min_order; 8292 8293 return 0; 8294} 8295 8296static const struct kernel_param_ops param_ops_slab_min_order __initconst = { 8297 .set = setup_slub_min_order, 8298}; 8299__core_param_cb(slab_min_order, ¶m_ops_slab_min_order, &slub_min_order, 0); 8300__core_param_cb(slub_min_order, ¶m_ops_slab_min_order, &slub_min_order, 0); 8301 8302static int __init setup_slub_max_order(const char *str, const struct kernel_param *kp) 8303{ 8304 int ret; 8305 8306 ret = kstrtouint(str, 0, &slub_max_order); 8307 if (ret) 8308 return ret; 8309 8310 slub_max_order = min_t(unsigned int, slub_max_order, MAX_PAGE_ORDER); 8311 8312 if (slub_min_order > slub_max_order) 8313 slub_min_order = slub_max_order; 8314 8315 return 0; 8316} 8317 8318static const struct kernel_param_ops param_ops_slab_max_order __initconst = { 8319 .set = setup_slub_max_order, 8320}; 8321__core_param_cb(slab_max_order, ¶m_ops_slab_max_order, &slub_max_order, 0); 8322__core_param_cb(slub_max_order, ¶m_ops_slab_max_order, &slub_max_order, 0); 8323 8324core_param(slab_min_objects, slub_min_objects, uint, 0); 8325core_param(slub_min_objects, slub_min_objects, uint, 0); 8326 8327#ifdef CONFIG_NUMA 8328static int __init setup_slab_strict_numa(const char *str, const struct kernel_param *kp) 8329{ 8330 if (nr_node_ids > 1) { 8331 static_branch_enable(&strict_numa); 8332 pr_info("SLUB: Strict NUMA enabled.\n"); 8333 } else { 8334 pr_warn("slab_strict_numa parameter set on non NUMA system.\n"); 8335 } 8336 8337 return 0; 8338} 8339 8340static const struct kernel_param_ops param_ops_slab_strict_numa __initconst = { 8341 .flags = KERNEL_PARAM_OPS_FL_NOARG, 8342 .set = setup_slab_strict_numa, 8343}; 8344__core_param_cb(slab_strict_numa, ¶m_ops_slab_strict_numa, NULL, 0); 8345#endif 8346 8347 8348#ifdef CONFIG_HARDENED_USERCOPY 8349/* 8350 * Rejects incorrectly sized objects and objects that are to be copied 8351 * to/from userspace but do not fall entirely within the containing slab 8352 * cache's usercopy region. 8353 * 8354 * Returns NULL if check passes, otherwise const char * to name of cache 8355 * to indicate an error. 8356 */ 8357void __check_heap_object(const void *ptr, unsigned long n, 8358 const struct slab *slab, bool to_user) 8359{ 8360 struct kmem_cache *s; 8361 unsigned int offset; 8362 bool is_kfence = is_kfence_address(ptr); 8363 8364 ptr = kasan_reset_tag(ptr); 8365 8366 /* Find object and usable object size. */ 8367 s = slab->slab_cache; 8368 8369 /* Reject impossible pointers. */ 8370 if (ptr < slab_address(slab)) 8371 usercopy_abort("SLUB object not in SLUB page?!", NULL, 8372 to_user, 0, n); 8373 8374 /* Find offset within object. */ 8375 if (is_kfence) 8376 offset = ptr - kfence_object_start(ptr); 8377 else 8378 offset = (ptr - slab_address(slab)) % s->size; 8379 8380 /* Adjust for redzone and reject if within the redzone. */ 8381 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) { 8382 if (offset < s->red_left_pad) 8383 usercopy_abort("SLUB object in left red zone", 8384 s->name, to_user, offset, n); 8385 offset -= s->red_left_pad; 8386 } 8387 8388 /* Allow address range falling entirely within usercopy region. */ 8389 if (offset >= s->useroffset && 8390 offset - s->useroffset <= s->usersize && 8391 n <= s->useroffset - offset + s->usersize) 8392 return; 8393 8394 usercopy_abort("SLUB object", s->name, to_user, offset, n); 8395} 8396#endif /* CONFIG_HARDENED_USERCOPY */ 8397 8398#define SHRINK_PROMOTE_MAX 32 8399 8400/* 8401 * kmem_cache_shrink discards empty slabs and promotes the slabs filled 8402 * up most to the head of the partial lists. New allocations will then 8403 * fill those up and thus they can be removed from the partial lists. 8404 * 8405 * The slabs with the least items are placed last. This results in them 8406 * being allocated from last increasing the chance that the last objects 8407 * are freed in them. 8408 */ 8409static int __kmem_cache_do_shrink(struct kmem_cache *s) 8410{ 8411 int node; 8412 int i; 8413 struct kmem_cache_node *n; 8414 struct slab *slab; 8415 struct slab *t; 8416 struct list_head discard; 8417 struct list_head promote[SHRINK_PROMOTE_MAX]; 8418 unsigned long flags; 8419 int ret = 0; 8420 8421 for_each_kmem_cache_node(s, node, n) { 8422 INIT_LIST_HEAD(&discard); 8423 for (i = 0; i < SHRINK_PROMOTE_MAX; i++) 8424 INIT_LIST_HEAD(promote + i); 8425 8426 if (n->barn) 8427 barn_shrink(s, n->barn); 8428 8429 spin_lock_irqsave(&n->list_lock, flags); 8430 8431 /* 8432 * Build lists of slabs to discard or promote. 8433 * 8434 * Note that concurrent frees may occur while we hold the 8435 * list_lock. slab->inuse here is the upper limit. 8436 */ 8437 list_for_each_entry_safe(slab, t, &n->partial, slab_list) { 8438 int free = slab->objects - slab->inuse; 8439 8440 /* Do not reread slab->inuse */ 8441 barrier(); 8442 8443 /* We do not keep full slabs on the list */ 8444 BUG_ON(free <= 0); 8445 8446 if (free == slab->objects) { 8447 list_move(&slab->slab_list, &discard); 8448 slab_clear_node_partial(slab); 8449 n->nr_partial--; 8450 dec_slabs_node(s, node, slab->objects); 8451 } else if (free <= SHRINK_PROMOTE_MAX) 8452 list_move(&slab->slab_list, promote + free - 1); 8453 } 8454 8455 /* 8456 * Promote the slabs filled up most to the head of the 8457 * partial list. 8458 */ 8459 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) 8460 list_splice(promote + i, &n->partial); 8461 8462 spin_unlock_irqrestore(&n->list_lock, flags); 8463 8464 /* Release empty slabs */ 8465 list_for_each_entry_safe(slab, t, &discard, slab_list) 8466 free_slab(s, slab); 8467 8468 if (node_nr_slabs(n)) 8469 ret = 1; 8470 } 8471 8472 return ret; 8473} 8474 8475int __kmem_cache_shrink(struct kmem_cache *s) 8476{ 8477 flush_all(s); 8478 return __kmem_cache_do_shrink(s); 8479} 8480 8481static int slab_mem_going_offline_callback(void) 8482{ 8483 struct kmem_cache *s; 8484 8485 mutex_lock(&slab_mutex); 8486 list_for_each_entry(s, &slab_caches, list) { 8487 flush_all_cpus_locked(s); 8488 __kmem_cache_do_shrink(s); 8489 } 8490 mutex_unlock(&slab_mutex); 8491 8492 return 0; 8493} 8494 8495static int slab_mem_going_online_callback(int nid) 8496{ 8497 struct kmem_cache_node *n; 8498 struct kmem_cache *s; 8499 int ret = 0; 8500 8501 /* 8502 * We are bringing a node online. No memory is available yet. We must 8503 * allocate a kmem_cache_node structure in order to bring the node 8504 * online. 8505 */ 8506 mutex_lock(&slab_mutex); 8507 list_for_each_entry(s, &slab_caches, list) { 8508 struct node_barn *barn = NULL; 8509 8510 /* 8511 * The structure may already exist if the node was previously 8512 * onlined and offlined. 8513 */ 8514 if (get_node(s, nid)) 8515 continue; 8516 8517 if (s->cpu_sheaves) { 8518 barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, nid); 8519 8520 if (!barn) { 8521 ret = -ENOMEM; 8522 goto out; 8523 } 8524 } 8525 8526 /* 8527 * XXX: kmem_cache_alloc_node will fallback to other nodes 8528 * since memory is not yet available from the node that 8529 * is brought up. 8530 */ 8531 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); 8532 if (!n) { 8533 kfree(barn); 8534 ret = -ENOMEM; 8535 goto out; 8536 } 8537 8538 init_kmem_cache_node(n, barn); 8539 8540 s->node[nid] = n; 8541 } 8542 /* 8543 * Any cache created after this point will also have kmem_cache_node 8544 * initialized for the new node. 8545 */ 8546 node_set(nid, slab_nodes); 8547out: 8548 mutex_unlock(&slab_mutex); 8549 return ret; 8550} 8551 8552static int slab_memory_callback(struct notifier_block *self, 8553 unsigned long action, void *arg) 8554{ 8555 struct node_notify *nn = arg; 8556 int nid = nn->nid; 8557 int ret = 0; 8558 8559 switch (action) { 8560 case NODE_ADDING_FIRST_MEMORY: 8561 ret = slab_mem_going_online_callback(nid); 8562 break; 8563 case NODE_REMOVING_LAST_MEMORY: 8564 ret = slab_mem_going_offline_callback(); 8565 break; 8566 } 8567 if (ret) 8568 ret = notifier_from_errno(ret); 8569 else 8570 ret = NOTIFY_OK; 8571 return ret; 8572} 8573 8574/******************************************************************** 8575 * Basic setup of slabs 8576 *******************************************************************/ 8577 8578/* 8579 * Used for early kmem_cache structures that were allocated using 8580 * the page allocator. Allocate them properly then fix up the pointers 8581 * that may be pointing to the wrong kmem_cache structure. 8582 */ 8583 8584static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) 8585{ 8586 int node; 8587 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 8588 struct kmem_cache_node *n; 8589 8590 memcpy(s, static_cache, kmem_cache->object_size); 8591 8592 /* 8593 * This runs very early, and only the boot processor is supposed to be 8594 * up. Even if it weren't true, IRQs are not up so we couldn't fire 8595 * IPIs around. 8596 */ 8597 __flush_cpu_slab(s, smp_processor_id()); 8598 for_each_kmem_cache_node(s, node, n) { 8599 struct slab *p; 8600 8601 list_for_each_entry(p, &n->partial, slab_list) 8602 p->slab_cache = s; 8603 8604#ifdef CONFIG_SLUB_DEBUG 8605 list_for_each_entry(p, &n->full, slab_list) 8606 p->slab_cache = s; 8607#endif 8608 } 8609 list_add(&s->list, &slab_caches); 8610 return s; 8611} 8612 8613void __init kmem_cache_init(void) 8614{ 8615 static __initdata struct kmem_cache boot_kmem_cache, 8616 boot_kmem_cache_node; 8617 int node; 8618 8619 if (debug_guardpage_minorder()) 8620 slub_max_order = 0; 8621 8622 /* Inform pointer hashing choice about slub debugging state. */ 8623 hash_pointers_finalize(__slub_debug_enabled()); 8624 8625 kmem_cache_node = &boot_kmem_cache_node; 8626 kmem_cache = &boot_kmem_cache; 8627 8628 /* 8629 * Initialize the nodemask for which we will allocate per node 8630 * structures. Here we don't need taking slab_mutex yet. 8631 */ 8632 for_each_node_state(node, N_MEMORY) 8633 node_set(node, slab_nodes); 8634 8635 create_boot_cache(kmem_cache_node, "kmem_cache_node", 8636 sizeof(struct kmem_cache_node), 8637 SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); 8638 8639 hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 8640 8641 /* Able to allocate the per node structures */ 8642 slab_state = PARTIAL; 8643 8644 create_boot_cache(kmem_cache, "kmem_cache", 8645 offsetof(struct kmem_cache, node) + 8646 nr_node_ids * sizeof(struct kmem_cache_node *), 8647 SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0); 8648 8649 kmem_cache = bootstrap(&boot_kmem_cache); 8650 kmem_cache_node = bootstrap(&boot_kmem_cache_node); 8651 8652 /* Now we can use the kmem_cache to allocate kmalloc slabs */ 8653 setup_kmalloc_cache_index_table(); 8654 create_kmalloc_caches(); 8655 8656 /* Setup random freelists for each cache */ 8657 init_freelist_randomization(); 8658 8659 cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL, 8660 slub_cpu_dead); 8661 8662 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n", 8663 cache_line_size(), 8664 slub_min_order, slub_max_order, slub_min_objects, 8665 nr_cpu_ids, nr_node_ids); 8666} 8667 8668void __init kmem_cache_init_late(void) 8669{ 8670 flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); 8671 WARN_ON(!flushwq); 8672#ifdef CONFIG_SLAB_FREELIST_RANDOM 8673 prandom_init_once(&slab_rnd_state); 8674#endif 8675} 8676 8677struct kmem_cache * 8678__kmem_cache_alias(const char *name, unsigned int size, unsigned int align, 8679 slab_flags_t flags, void (*ctor)(void *)) 8680{ 8681 struct kmem_cache *s; 8682 8683 s = find_mergeable(size, align, flags, name, ctor); 8684 if (s) { 8685 if (sysfs_slab_alias(s, name)) 8686 pr_err("SLUB: Unable to add cache alias %s to sysfs\n", 8687 name); 8688 8689 s->refcount++; 8690 8691 /* 8692 * Adjust the object sizes so that we clear 8693 * the complete object on kzalloc. 8694 */ 8695 s->object_size = max(s->object_size, size); 8696 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); 8697 } 8698 8699 return s; 8700} 8701 8702int do_kmem_cache_create(struct kmem_cache *s, const char *name, 8703 unsigned int size, struct kmem_cache_args *args, 8704 slab_flags_t flags) 8705{ 8706 int err = -EINVAL; 8707 8708 s->name = name; 8709 s->size = s->object_size = size; 8710 8711 s->flags = kmem_cache_flags(flags, s->name); 8712#ifdef CONFIG_SLAB_FREELIST_HARDENED 8713 s->random = get_random_long(); 8714#endif 8715 s->align = args->align; 8716 s->ctor = args->ctor; 8717#ifdef CONFIG_HARDENED_USERCOPY 8718 s->useroffset = args->useroffset; 8719 s->usersize = args->usersize; 8720#endif 8721 8722 if (!calculate_sizes(args, s)) 8723 goto out; 8724 if (disable_higher_order_debug) { 8725 /* 8726 * Disable debugging flags that store metadata if the min slab 8727 * order increased. 8728 */ 8729 if (get_order(s->size) > get_order(s->object_size)) { 8730 s->flags &= ~DEBUG_METADATA_FLAGS; 8731 s->offset = 0; 8732 if (!calculate_sizes(args, s)) 8733 goto out; 8734 } 8735 } 8736 8737#ifdef system_has_freelist_aba 8738 if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) { 8739 /* Enable fast mode */ 8740 s->flags |= __CMPXCHG_DOUBLE; 8741 } 8742#endif 8743 8744 /* 8745 * The larger the object size is, the more slabs we want on the partial 8746 * list to avoid pounding the page allocator excessively. 8747 */ 8748 s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); 8749 s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial); 8750 8751 set_cpu_partial(s); 8752 8753 if (args->sheaf_capacity && !IS_ENABLED(CONFIG_SLUB_TINY) 8754 && !(s->flags & SLAB_DEBUG_FLAGS)) { 8755 s->cpu_sheaves = alloc_percpu(struct slub_percpu_sheaves); 8756 if (!s->cpu_sheaves) { 8757 err = -ENOMEM; 8758 goto out; 8759 } 8760 // TODO: increase capacity to grow slab_sheaf up to next kmalloc size? 8761 s->sheaf_capacity = args->sheaf_capacity; 8762 } 8763 8764#ifdef CONFIG_NUMA 8765 s->remote_node_defrag_ratio = 1000; 8766#endif 8767 8768 /* Initialize the pre-computed randomized freelist if slab is up */ 8769 if (slab_state >= UP) { 8770 if (init_cache_random_seq(s)) 8771 goto out; 8772 } 8773 8774 if (!init_kmem_cache_nodes(s)) 8775 goto out; 8776 8777 if (!alloc_kmem_cache_cpus(s)) 8778 goto out; 8779 8780 if (s->cpu_sheaves) { 8781 err = init_percpu_sheaves(s); 8782 if (err) 8783 goto out; 8784 } 8785 8786 err = 0; 8787 8788 /* Mutex is not taken during early boot */ 8789 if (slab_state <= UP) 8790 goto out; 8791 8792 /* 8793 * Failing to create sysfs files is not critical to SLUB functionality. 8794 * If it fails, proceed with cache creation without these files. 8795 */ 8796 if (sysfs_slab_add(s)) 8797 pr_err("SLUB: Unable to add cache %s to sysfs\n", s->name); 8798 8799 if (s->flags & SLAB_STORE_USER) 8800 debugfs_slab_add(s); 8801 8802out: 8803 if (err) 8804 __kmem_cache_release(s); 8805 return err; 8806} 8807 8808#ifdef SLAB_SUPPORTS_SYSFS 8809static int count_inuse(struct slab *slab) 8810{ 8811 return slab->inuse; 8812} 8813 8814static int count_total(struct slab *slab) 8815{ 8816 return slab->objects; 8817} 8818#endif 8819 8820#ifdef CONFIG_SLUB_DEBUG 8821static void validate_slab(struct kmem_cache *s, struct slab *slab, 8822 unsigned long *obj_map) 8823{ 8824 void *p; 8825 void *addr = slab_address(slab); 8826 8827 if (!validate_slab_ptr(slab)) { 8828 slab_err(s, slab, "Not a valid slab page"); 8829 return; 8830 } 8831 8832 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL)) 8833 return; 8834 8835 /* Now we know that a valid freelist exists */ 8836 __fill_map(obj_map, s, slab); 8837 for_each_object(p, s, addr, slab->objects) { 8838 u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ? 8839 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE; 8840 8841 if (!check_object(s, slab, p, val)) 8842 break; 8843 } 8844} 8845 8846static int validate_slab_node(struct kmem_cache *s, 8847 struct kmem_cache_node *n, unsigned long *obj_map) 8848{ 8849 unsigned long count = 0; 8850 struct slab *slab; 8851 unsigned long flags; 8852 8853 spin_lock_irqsave(&n->list_lock, flags); 8854 8855 list_for_each_entry(slab, &n->partial, slab_list) { 8856 validate_slab(s, slab, obj_map); 8857 count++; 8858 } 8859 if (count != n->nr_partial) { 8860 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", 8861 s->name, count, n->nr_partial); 8862 slab_add_kunit_errors(); 8863 } 8864 8865 if (!(s->flags & SLAB_STORE_USER)) 8866 goto out; 8867 8868 list_for_each_entry(slab, &n->full, slab_list) { 8869 validate_slab(s, slab, obj_map); 8870 count++; 8871 } 8872 if (count != node_nr_slabs(n)) { 8873 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", 8874 s->name, count, node_nr_slabs(n)); 8875 slab_add_kunit_errors(); 8876 } 8877 8878out: 8879 spin_unlock_irqrestore(&n->list_lock, flags); 8880 return count; 8881} 8882 8883long validate_slab_cache(struct kmem_cache *s) 8884{ 8885 int node; 8886 unsigned long count = 0; 8887 struct kmem_cache_node *n; 8888 unsigned long *obj_map; 8889 8890 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); 8891 if (!obj_map) 8892 return -ENOMEM; 8893 8894 flush_all(s); 8895 for_each_kmem_cache_node(s, node, n) 8896 count += validate_slab_node(s, n, obj_map); 8897 8898 bitmap_free(obj_map); 8899 8900 return count; 8901} 8902EXPORT_SYMBOL(validate_slab_cache); 8903 8904#ifdef CONFIG_DEBUG_FS 8905/* 8906 * Generate lists of code addresses where slabcache objects are allocated 8907 * and freed. 8908 */ 8909 8910struct location { 8911 depot_stack_handle_t handle; 8912 unsigned long count; 8913 unsigned long addr; 8914 unsigned long waste; 8915 long long sum_time; 8916 long min_time; 8917 long max_time; 8918 long min_pid; 8919 long max_pid; 8920 DECLARE_BITMAP(cpus, NR_CPUS); 8921 nodemask_t nodes; 8922}; 8923 8924struct loc_track { 8925 unsigned long max; 8926 unsigned long count; 8927 struct location *loc; 8928 loff_t idx; 8929}; 8930 8931static struct dentry *slab_debugfs_root; 8932 8933static void free_loc_track(struct loc_track *t) 8934{ 8935 if (t->max) 8936 free_pages((unsigned long)t->loc, 8937 get_order(sizeof(struct location) * t->max)); 8938} 8939 8940static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) 8941{ 8942 struct location *l; 8943 int order; 8944 8945 order = get_order(sizeof(struct location) * max); 8946 8947 l = (void *)__get_free_pages(flags, order); 8948 if (!l) 8949 return 0; 8950 8951 if (t->count) { 8952 memcpy(l, t->loc, sizeof(struct location) * t->count); 8953 free_loc_track(t); 8954 } 8955 t->max = max; 8956 t->loc = l; 8957 return 1; 8958} 8959 8960static int add_location(struct loc_track *t, struct kmem_cache *s, 8961 const struct track *track, 8962 unsigned int orig_size) 8963{ 8964 long start, end, pos; 8965 struct location *l; 8966 unsigned long caddr, chandle, cwaste; 8967 unsigned long age = jiffies - track->when; 8968 depot_stack_handle_t handle = 0; 8969 unsigned int waste = s->object_size - orig_size; 8970 8971#ifdef CONFIG_STACKDEPOT 8972 handle = READ_ONCE(track->handle); 8973#endif 8974 start = -1; 8975 end = t->count; 8976 8977 for ( ; ; ) { 8978 pos = start + (end - start + 1) / 2; 8979 8980 /* 8981 * There is nothing at "end". If we end up there 8982 * we need to add something to before end. 8983 */ 8984 if (pos == end) 8985 break; 8986 8987 l = &t->loc[pos]; 8988 caddr = l->addr; 8989 chandle = l->handle; 8990 cwaste = l->waste; 8991 if ((track->addr == caddr) && (handle == chandle) && 8992 (waste == cwaste)) { 8993 8994 l->count++; 8995 if (track->when) { 8996 l->sum_time += age; 8997 if (age < l->min_time) 8998 l->min_time = age; 8999 if (age > l->max_time) 9000 l->max_time = age; 9001 9002 if (track->pid < l->min_pid) 9003 l->min_pid = track->pid; 9004 if (track->pid > l->max_pid) 9005 l->max_pid = track->pid; 9006 9007 cpumask_set_cpu(track->cpu, 9008 to_cpumask(l->cpus)); 9009 } 9010 node_set(page_to_nid(virt_to_page(track)), l->nodes); 9011 return 1; 9012 } 9013 9014 if (track->addr < caddr) 9015 end = pos; 9016 else if (track->addr == caddr && handle < chandle) 9017 end = pos; 9018 else if (track->addr == caddr && handle == chandle && 9019 waste < cwaste) 9020 end = pos; 9021 else 9022 start = pos; 9023 } 9024 9025 /* 9026 * Not found. Insert new tracking element. 9027 */ 9028 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) 9029 return 0; 9030 9031 l = t->loc + pos; 9032 if (pos < t->count) 9033 memmove(l + 1, l, 9034 (t->count - pos) * sizeof(struct location)); 9035 t->count++; 9036 l->count = 1; 9037 l->addr = track->addr; 9038 l->sum_time = age; 9039 l->min_time = age; 9040 l->max_time = age; 9041 l->min_pid = track->pid; 9042 l->max_pid = track->pid; 9043 l->handle = handle; 9044 l->waste = waste; 9045 cpumask_clear(to_cpumask(l->cpus)); 9046 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); 9047 nodes_clear(l->nodes); 9048 node_set(page_to_nid(virt_to_page(track)), l->nodes); 9049 return 1; 9050} 9051 9052static void process_slab(struct loc_track *t, struct kmem_cache *s, 9053 struct slab *slab, enum track_item alloc, 9054 unsigned long *obj_map) 9055{ 9056 void *addr = slab_address(slab); 9057 bool is_alloc = (alloc == TRACK_ALLOC); 9058 void *p; 9059 9060 __fill_map(obj_map, s, slab); 9061 9062 for_each_object(p, s, addr, slab->objects) 9063 if (!test_bit(__obj_to_index(s, addr, p), obj_map)) 9064 add_location(t, s, get_track(s, p, alloc), 9065 is_alloc ? get_orig_size(s, p) : 9066 s->object_size); 9067} 9068#endif /* CONFIG_DEBUG_FS */ 9069#endif /* CONFIG_SLUB_DEBUG */ 9070 9071#ifdef SLAB_SUPPORTS_SYSFS 9072enum slab_stat_type { 9073 SL_ALL, /* All slabs */ 9074 SL_PARTIAL, /* Only partially allocated slabs */ 9075 SL_CPU, /* Only slabs used for cpu caches */ 9076 SL_OBJECTS, /* Determine allocated objects not slabs */ 9077 SL_TOTAL /* Determine object capacity not slabs */ 9078}; 9079 9080#define SO_ALL (1 << SL_ALL) 9081#define SO_PARTIAL (1 << SL_PARTIAL) 9082#define SO_CPU (1 << SL_CPU) 9083#define SO_OBJECTS (1 << SL_OBJECTS) 9084#define SO_TOTAL (1 << SL_TOTAL) 9085 9086static ssize_t show_slab_objects(struct kmem_cache *s, 9087 char *buf, unsigned long flags) 9088{ 9089 unsigned long total = 0; 9090 int node; 9091 int x; 9092 unsigned long *nodes; 9093 int len = 0; 9094 9095 nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL); 9096 if (!nodes) 9097 return -ENOMEM; 9098 9099 if (flags & SO_CPU) { 9100 int cpu; 9101 9102 for_each_possible_cpu(cpu) { 9103 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, 9104 cpu); 9105 int node; 9106 struct slab *slab; 9107 9108 slab = READ_ONCE(c->slab); 9109 if (!slab) 9110 continue; 9111 9112 node = slab_nid(slab); 9113 if (flags & SO_TOTAL) 9114 x = slab->objects; 9115 else if (flags & SO_OBJECTS) 9116 x = slab->inuse; 9117 else 9118 x = 1; 9119 9120 total += x; 9121 nodes[node] += x; 9122 9123#ifdef CONFIG_SLUB_CPU_PARTIAL 9124 slab = slub_percpu_partial_read_once(c); 9125 if (slab) { 9126 node = slab_nid(slab); 9127 if (flags & SO_TOTAL) 9128 WARN_ON_ONCE(1); 9129 else if (flags & SO_OBJECTS) 9130 WARN_ON_ONCE(1); 9131 else 9132 x = data_race(slab->slabs); 9133 total += x; 9134 nodes[node] += x; 9135 } 9136#endif 9137 } 9138 } 9139 9140 /* 9141 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex" 9142 * already held which will conflict with an existing lock order: 9143 * 9144 * mem_hotplug_lock->slab_mutex->kernfs_mutex 9145 * 9146 * We don't really need mem_hotplug_lock (to hold off 9147 * slab_mem_going_offline_callback) here because slab's memory hot 9148 * unplug code doesn't destroy the kmem_cache->node[] data. 9149 */ 9150 9151#ifdef CONFIG_SLUB_DEBUG 9152 if (flags & SO_ALL) { 9153 struct kmem_cache_node *n; 9154 9155 for_each_kmem_cache_node(s, node, n) { 9156 9157 if (flags & SO_TOTAL) 9158 x = node_nr_objs(n); 9159 else if (flags & SO_OBJECTS) 9160 x = node_nr_objs(n) - count_partial(n, count_free); 9161 else 9162 x = node_nr_slabs(n); 9163 total += x; 9164 nodes[node] += x; 9165 } 9166 9167 } else 9168#endif 9169 if (flags & SO_PARTIAL) { 9170 struct kmem_cache_node *n; 9171 9172 for_each_kmem_cache_node(s, node, n) { 9173 if (flags & SO_TOTAL) 9174 x = count_partial(n, count_total); 9175 else if (flags & SO_OBJECTS) 9176 x = count_partial(n, count_inuse); 9177 else 9178 x = n->nr_partial; 9179 total += x; 9180 nodes[node] += x; 9181 } 9182 } 9183 9184 len += sysfs_emit_at(buf, len, "%lu", total); 9185#ifdef CONFIG_NUMA 9186 for (node = 0; node < nr_node_ids; node++) { 9187 if (nodes[node]) 9188 len += sysfs_emit_at(buf, len, " N%d=%lu", 9189 node, nodes[node]); 9190 } 9191#endif 9192 len += sysfs_emit_at(buf, len, "\n"); 9193 kfree(nodes); 9194 9195 return len; 9196} 9197 9198#define to_slab_attr(n) container_of(n, struct slab_attribute, attr) 9199#define to_slab(n) container_of(n, struct kmem_cache, kobj) 9200 9201struct slab_attribute { 9202 struct attribute attr; 9203 ssize_t (*show)(struct kmem_cache *s, char *buf); 9204 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); 9205}; 9206 9207#define SLAB_ATTR_RO(_name) \ 9208 static struct slab_attribute _name##_attr = __ATTR_RO_MODE(_name, 0400) 9209 9210#define SLAB_ATTR(_name) \ 9211 static struct slab_attribute _name##_attr = __ATTR_RW_MODE(_name, 0600) 9212 9213static ssize_t slab_size_show(struct kmem_cache *s, char *buf) 9214{ 9215 return sysfs_emit(buf, "%u\n", s->size); 9216} 9217SLAB_ATTR_RO(slab_size); 9218 9219static ssize_t align_show(struct kmem_cache *s, char *buf) 9220{ 9221 return sysfs_emit(buf, "%u\n", s->align); 9222} 9223SLAB_ATTR_RO(align); 9224 9225static ssize_t object_size_show(struct kmem_cache *s, char *buf) 9226{ 9227 return sysfs_emit(buf, "%u\n", s->object_size); 9228} 9229SLAB_ATTR_RO(object_size); 9230 9231static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) 9232{ 9233 return sysfs_emit(buf, "%u\n", oo_objects(s->oo)); 9234} 9235SLAB_ATTR_RO(objs_per_slab); 9236 9237static ssize_t order_show(struct kmem_cache *s, char *buf) 9238{ 9239 return sysfs_emit(buf, "%u\n", oo_order(s->oo)); 9240} 9241SLAB_ATTR_RO(order); 9242 9243static ssize_t sheaf_capacity_show(struct kmem_cache *s, char *buf) 9244{ 9245 return sysfs_emit(buf, "%u\n", s->sheaf_capacity); 9246} 9247SLAB_ATTR_RO(sheaf_capacity); 9248 9249static ssize_t min_partial_show(struct kmem_cache *s, char *buf) 9250{ 9251 return sysfs_emit(buf, "%lu\n", s->min_partial); 9252} 9253 9254static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, 9255 size_t length) 9256{ 9257 unsigned long min; 9258 int err; 9259 9260 err = kstrtoul(buf, 10, &min); 9261 if (err) 9262 return err; 9263 9264 s->min_partial = min; 9265 return length; 9266} 9267SLAB_ATTR(min_partial); 9268 9269static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) 9270{ 9271 unsigned int nr_partial = 0; 9272#ifdef CONFIG_SLUB_CPU_PARTIAL 9273 nr_partial = s->cpu_partial; 9274#endif 9275 9276 return sysfs_emit(buf, "%u\n", nr_partial); 9277} 9278 9279static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, 9280 size_t length) 9281{ 9282 unsigned int objects; 9283 int err; 9284 9285 err = kstrtouint(buf, 10, &objects); 9286 if (err) 9287 return err; 9288 if (objects && !kmem_cache_has_cpu_partial(s)) 9289 return -EINVAL; 9290 9291 slub_set_cpu_partial(s, objects); 9292 flush_all(s); 9293 return length; 9294} 9295SLAB_ATTR(cpu_partial); 9296 9297static ssize_t ctor_show(struct kmem_cache *s, char *buf) 9298{ 9299 if (!s->ctor) 9300 return 0; 9301 return sysfs_emit(buf, "%pS\n", s->ctor); 9302} 9303SLAB_ATTR_RO(ctor); 9304 9305static ssize_t aliases_show(struct kmem_cache *s, char *buf) 9306{ 9307 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); 9308} 9309SLAB_ATTR_RO(aliases); 9310 9311static ssize_t partial_show(struct kmem_cache *s, char *buf) 9312{ 9313 return show_slab_objects(s, buf, SO_PARTIAL); 9314} 9315SLAB_ATTR_RO(partial); 9316 9317static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) 9318{ 9319 return show_slab_objects(s, buf, SO_CPU); 9320} 9321SLAB_ATTR_RO(cpu_slabs); 9322 9323static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) 9324{ 9325 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); 9326} 9327SLAB_ATTR_RO(objects_partial); 9328 9329static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) 9330{ 9331 int objects = 0; 9332 int slabs = 0; 9333 int cpu __maybe_unused; 9334 int len = 0; 9335 9336#ifdef CONFIG_SLUB_CPU_PARTIAL 9337 for_each_online_cpu(cpu) { 9338 struct slab *slab; 9339 9340 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); 9341 9342 if (slab) 9343 slabs += data_race(slab->slabs); 9344 } 9345#endif 9346 9347 /* Approximate half-full slabs, see slub_set_cpu_partial() */ 9348 objects = (slabs * oo_objects(s->oo)) / 2; 9349 len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs); 9350 9351#ifdef CONFIG_SLUB_CPU_PARTIAL 9352 for_each_online_cpu(cpu) { 9353 struct slab *slab; 9354 9355 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); 9356 if (slab) { 9357 slabs = data_race(slab->slabs); 9358 objects = (slabs * oo_objects(s->oo)) / 2; 9359 len += sysfs_emit_at(buf, len, " C%d=%d(%d)", 9360 cpu, objects, slabs); 9361 } 9362 } 9363#endif 9364 len += sysfs_emit_at(buf, len, "\n"); 9365 9366 return len; 9367} 9368SLAB_ATTR_RO(slabs_cpu_partial); 9369 9370static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) 9371{ 9372 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); 9373} 9374SLAB_ATTR_RO(reclaim_account); 9375 9376static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) 9377{ 9378 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); 9379} 9380SLAB_ATTR_RO(hwcache_align); 9381 9382#ifdef CONFIG_ZONE_DMA 9383static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) 9384{ 9385 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); 9386} 9387SLAB_ATTR_RO(cache_dma); 9388#endif 9389 9390#ifdef CONFIG_HARDENED_USERCOPY 9391static ssize_t usersize_show(struct kmem_cache *s, char *buf) 9392{ 9393 return sysfs_emit(buf, "%u\n", s->usersize); 9394} 9395SLAB_ATTR_RO(usersize); 9396#endif 9397 9398static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) 9399{ 9400 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU)); 9401} 9402SLAB_ATTR_RO(destroy_by_rcu); 9403 9404#ifdef CONFIG_SLUB_DEBUG 9405static ssize_t slabs_show(struct kmem_cache *s, char *buf) 9406{ 9407 return show_slab_objects(s, buf, SO_ALL); 9408} 9409SLAB_ATTR_RO(slabs); 9410 9411static ssize_t total_objects_show(struct kmem_cache *s, char *buf) 9412{ 9413 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); 9414} 9415SLAB_ATTR_RO(total_objects); 9416 9417static ssize_t objects_show(struct kmem_cache *s, char *buf) 9418{ 9419 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); 9420} 9421SLAB_ATTR_RO(objects); 9422 9423static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) 9424{ 9425 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS)); 9426} 9427SLAB_ATTR_RO(sanity_checks); 9428 9429static ssize_t trace_show(struct kmem_cache *s, char *buf) 9430{ 9431 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE)); 9432} 9433SLAB_ATTR_RO(trace); 9434 9435static ssize_t red_zone_show(struct kmem_cache *s, char *buf) 9436{ 9437 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); 9438} 9439 9440SLAB_ATTR_RO(red_zone); 9441 9442static ssize_t poison_show(struct kmem_cache *s, char *buf) 9443{ 9444 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON)); 9445} 9446 9447SLAB_ATTR_RO(poison); 9448 9449static ssize_t store_user_show(struct kmem_cache *s, char *buf) 9450{ 9451 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); 9452} 9453 9454SLAB_ATTR_RO(store_user); 9455 9456static ssize_t validate_show(struct kmem_cache *s, char *buf) 9457{ 9458 return 0; 9459} 9460 9461static ssize_t validate_store(struct kmem_cache *s, 9462 const char *buf, size_t length) 9463{ 9464 int ret = -EINVAL; 9465 9466 if (buf[0] == '1' && kmem_cache_debug(s)) { 9467 ret = validate_slab_cache(s); 9468 if (ret >= 0) 9469 ret = length; 9470 } 9471 return ret; 9472} 9473SLAB_ATTR(validate); 9474 9475#endif /* CONFIG_SLUB_DEBUG */ 9476 9477#ifdef CONFIG_FAILSLAB 9478static ssize_t failslab_show(struct kmem_cache *s, char *buf) 9479{ 9480 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); 9481} 9482 9483static ssize_t failslab_store(struct kmem_cache *s, const char *buf, 9484 size_t length) 9485{ 9486 if (s->refcount > 1) 9487 return -EINVAL; 9488 9489 if (buf[0] == '1') 9490 WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB); 9491 else 9492 WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB); 9493 9494 return length; 9495} 9496SLAB_ATTR(failslab); 9497#endif 9498 9499static ssize_t shrink_show(struct kmem_cache *s, char *buf) 9500{ 9501 return 0; 9502} 9503 9504static ssize_t shrink_store(struct kmem_cache *s, 9505 const char *buf, size_t length) 9506{ 9507 if (buf[0] == '1') 9508 kmem_cache_shrink(s); 9509 else 9510 return -EINVAL; 9511 return length; 9512} 9513SLAB_ATTR(shrink); 9514 9515#ifdef CONFIG_NUMA 9516static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) 9517{ 9518 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10); 9519} 9520 9521static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, 9522 const char *buf, size_t length) 9523{ 9524 unsigned int ratio; 9525 int err; 9526 9527 err = kstrtouint(buf, 10, &ratio); 9528 if (err) 9529 return err; 9530 if (ratio > 100) 9531 return -ERANGE; 9532 9533 s->remote_node_defrag_ratio = ratio * 10; 9534 9535 return length; 9536} 9537SLAB_ATTR(remote_node_defrag_ratio); 9538#endif 9539 9540#ifdef CONFIG_SLUB_STATS 9541static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) 9542{ 9543 unsigned long sum = 0; 9544 int cpu; 9545 int len = 0; 9546 int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL); 9547 9548 if (!data) 9549 return -ENOMEM; 9550 9551 for_each_online_cpu(cpu) { 9552 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; 9553 9554 data[cpu] = x; 9555 sum += x; 9556 } 9557 9558 len += sysfs_emit_at(buf, len, "%lu", sum); 9559 9560#ifdef CONFIG_SMP 9561 for_each_online_cpu(cpu) { 9562 if (data[cpu]) 9563 len += sysfs_emit_at(buf, len, " C%d=%u", 9564 cpu, data[cpu]); 9565 } 9566#endif 9567 kfree(data); 9568 len += sysfs_emit_at(buf, len, "\n"); 9569 9570 return len; 9571} 9572 9573static void clear_stat(struct kmem_cache *s, enum stat_item si) 9574{ 9575 int cpu; 9576 9577 for_each_online_cpu(cpu) 9578 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; 9579} 9580 9581#define STAT_ATTR(si, text) \ 9582static ssize_t text##_show(struct kmem_cache *s, char *buf) \ 9583{ \ 9584 return show_stat(s, buf, si); \ 9585} \ 9586static ssize_t text##_store(struct kmem_cache *s, \ 9587 const char *buf, size_t length) \ 9588{ \ 9589 if (buf[0] != '0') \ 9590 return -EINVAL; \ 9591 clear_stat(s, si); \ 9592 return length; \ 9593} \ 9594SLAB_ATTR(text); \ 9595 9596STAT_ATTR(ALLOC_PCS, alloc_cpu_sheaf); 9597STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); 9598STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); 9599STAT_ATTR(FREE_PCS, free_cpu_sheaf); 9600STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf); 9601STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail); 9602STAT_ATTR(FREE_FASTPATH, free_fastpath); 9603STAT_ATTR(FREE_SLOWPATH, free_slowpath); 9604STAT_ATTR(FREE_FROZEN, free_frozen); 9605STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); 9606STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); 9607STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); 9608STAT_ATTR(ALLOC_SLAB, alloc_slab); 9609STAT_ATTR(ALLOC_REFILL, alloc_refill); 9610STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); 9611STAT_ATTR(FREE_SLAB, free_slab); 9612STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); 9613STAT_ATTR(DEACTIVATE_FULL, deactivate_full); 9614STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); 9615STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); 9616STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); 9617STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); 9618STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); 9619STAT_ATTR(ORDER_FALLBACK, order_fallback); 9620STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); 9621STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); 9622STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); 9623STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); 9624STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); 9625STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); 9626STAT_ATTR(SHEAF_FLUSH, sheaf_flush); 9627STAT_ATTR(SHEAF_REFILL, sheaf_refill); 9628STAT_ATTR(SHEAF_ALLOC, sheaf_alloc); 9629STAT_ATTR(SHEAF_FREE, sheaf_free); 9630STAT_ATTR(BARN_GET, barn_get); 9631STAT_ATTR(BARN_GET_FAIL, barn_get_fail); 9632STAT_ATTR(BARN_PUT, barn_put); 9633STAT_ATTR(BARN_PUT_FAIL, barn_put_fail); 9634STAT_ATTR(SHEAF_PREFILL_FAST, sheaf_prefill_fast); 9635STAT_ATTR(SHEAF_PREFILL_SLOW, sheaf_prefill_slow); 9636STAT_ATTR(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize); 9637STAT_ATTR(SHEAF_RETURN_FAST, sheaf_return_fast); 9638STAT_ATTR(SHEAF_RETURN_SLOW, sheaf_return_slow); 9639#endif /* CONFIG_SLUB_STATS */ 9640 9641#ifdef CONFIG_KFENCE 9642static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf) 9643{ 9644 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE)); 9645} 9646 9647static ssize_t skip_kfence_store(struct kmem_cache *s, 9648 const char *buf, size_t length) 9649{ 9650 int ret = length; 9651 9652 if (buf[0] == '0') 9653 s->flags &= ~SLAB_SKIP_KFENCE; 9654 else if (buf[0] == '1') 9655 s->flags |= SLAB_SKIP_KFENCE; 9656 else 9657 ret = -EINVAL; 9658 9659 return ret; 9660} 9661SLAB_ATTR(skip_kfence); 9662#endif 9663 9664static struct attribute *slab_attrs[] = { 9665 &slab_size_attr.attr, 9666 &object_size_attr.attr, 9667 &objs_per_slab_attr.attr, 9668 &order_attr.attr, 9669 &sheaf_capacity_attr.attr, 9670 &min_partial_attr.attr, 9671 &cpu_partial_attr.attr, 9672 &objects_partial_attr.attr, 9673 &partial_attr.attr, 9674 &cpu_slabs_attr.attr, 9675 &ctor_attr.attr, 9676 &aliases_attr.attr, 9677 &align_attr.attr, 9678 &hwcache_align_attr.attr, 9679 &reclaim_account_attr.attr, 9680 &destroy_by_rcu_attr.attr, 9681 &shrink_attr.attr, 9682 &slabs_cpu_partial_attr.attr, 9683#ifdef CONFIG_SLUB_DEBUG 9684 &total_objects_attr.attr, 9685 &objects_attr.attr, 9686 &slabs_attr.attr, 9687 &sanity_checks_attr.attr, 9688 &trace_attr.attr, 9689 &red_zone_attr.attr, 9690 &poison_attr.attr, 9691 &store_user_attr.attr, 9692 &validate_attr.attr, 9693#endif 9694#ifdef CONFIG_ZONE_DMA 9695 &cache_dma_attr.attr, 9696#endif 9697#ifdef CONFIG_NUMA 9698 &remote_node_defrag_ratio_attr.attr, 9699#endif 9700#ifdef CONFIG_SLUB_STATS 9701 &alloc_cpu_sheaf_attr.attr, 9702 &alloc_fastpath_attr.attr, 9703 &alloc_slowpath_attr.attr, 9704 &free_cpu_sheaf_attr.attr, 9705 &free_rcu_sheaf_attr.attr, 9706 &free_rcu_sheaf_fail_attr.attr, 9707 &free_fastpath_attr.attr, 9708 &free_slowpath_attr.attr, 9709 &free_frozen_attr.attr, 9710 &free_add_partial_attr.attr, 9711 &free_remove_partial_attr.attr, 9712 &alloc_from_partial_attr.attr, 9713 &alloc_slab_attr.attr, 9714 &alloc_refill_attr.attr, 9715 &alloc_node_mismatch_attr.attr, 9716 &free_slab_attr.attr, 9717 &cpuslab_flush_attr.attr, 9718 &deactivate_full_attr.attr, 9719 &deactivate_empty_attr.attr, 9720 &deactivate_to_head_attr.attr, 9721 &deactivate_to_tail_attr.attr, 9722 &deactivate_remote_frees_attr.attr, 9723 &deactivate_bypass_attr.attr, 9724 &order_fallback_attr.attr, 9725 &cmpxchg_double_fail_attr.attr, 9726 &cmpxchg_double_cpu_fail_attr.attr, 9727 &cpu_partial_alloc_attr.attr, 9728 &cpu_partial_free_attr.attr, 9729 &cpu_partial_node_attr.attr, 9730 &cpu_partial_drain_attr.attr, 9731 &sheaf_flush_attr.attr, 9732 &sheaf_refill_attr.attr, 9733 &sheaf_alloc_attr.attr, 9734 &sheaf_free_attr.attr, 9735 &barn_get_attr.attr, 9736 &barn_get_fail_attr.attr, 9737 &barn_put_attr.attr, 9738 &barn_put_fail_attr.attr, 9739 &sheaf_prefill_fast_attr.attr, 9740 &sheaf_prefill_slow_attr.attr, 9741 &sheaf_prefill_oversize_attr.attr, 9742 &sheaf_return_fast_attr.attr, 9743 &sheaf_return_slow_attr.attr, 9744#endif 9745#ifdef CONFIG_FAILSLAB 9746 &failslab_attr.attr, 9747#endif 9748#ifdef CONFIG_HARDENED_USERCOPY 9749 &usersize_attr.attr, 9750#endif 9751#ifdef CONFIG_KFENCE 9752 &skip_kfence_attr.attr, 9753#endif 9754 9755 NULL 9756}; 9757 9758static const struct attribute_group slab_attr_group = { 9759 .attrs = slab_attrs, 9760}; 9761 9762static ssize_t slab_attr_show(struct kobject *kobj, 9763 struct attribute *attr, 9764 char *buf) 9765{ 9766 struct slab_attribute *attribute; 9767 struct kmem_cache *s; 9768 9769 attribute = to_slab_attr(attr); 9770 s = to_slab(kobj); 9771 9772 if (!attribute->show) 9773 return -EIO; 9774 9775 return attribute->show(s, buf); 9776} 9777 9778static ssize_t slab_attr_store(struct kobject *kobj, 9779 struct attribute *attr, 9780 const char *buf, size_t len) 9781{ 9782 struct slab_attribute *attribute; 9783 struct kmem_cache *s; 9784 9785 attribute = to_slab_attr(attr); 9786 s = to_slab(kobj); 9787 9788 if (!attribute->store) 9789 return -EIO; 9790 9791 return attribute->store(s, buf, len); 9792} 9793 9794static void kmem_cache_release(struct kobject *k) 9795{ 9796 slab_kmem_cache_release(to_slab(k)); 9797} 9798 9799static const struct sysfs_ops slab_sysfs_ops = { 9800 .show = slab_attr_show, 9801 .store = slab_attr_store, 9802}; 9803 9804static const struct kobj_type slab_ktype = { 9805 .sysfs_ops = &slab_sysfs_ops, 9806 .release = kmem_cache_release, 9807}; 9808 9809static struct kset *slab_kset; 9810 9811static inline struct kset *cache_kset(struct kmem_cache *s) 9812{ 9813 return slab_kset; 9814} 9815 9816#define ID_STR_LENGTH 32 9817 9818/* Create a unique string id for a slab cache: 9819 * 9820 * Format :[flags-]size 9821 */ 9822static char *create_unique_id(struct kmem_cache *s) 9823{ 9824 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); 9825 char *p = name; 9826 9827 if (!name) 9828 return ERR_PTR(-ENOMEM); 9829 9830 *p++ = ':'; 9831 /* 9832 * First flags affecting slabcache operations. We will only 9833 * get here for aliasable slabs so we do not need to support 9834 * too many flags. The flags here must cover all flags that 9835 * are matched during merging to guarantee that the id is 9836 * unique. 9837 */ 9838 if (s->flags & SLAB_CACHE_DMA) 9839 *p++ = 'd'; 9840 if (s->flags & SLAB_CACHE_DMA32) 9841 *p++ = 'D'; 9842 if (s->flags & SLAB_RECLAIM_ACCOUNT) 9843 *p++ = 'a'; 9844 if (s->flags & SLAB_CONSISTENCY_CHECKS) 9845 *p++ = 'F'; 9846 if (s->flags & SLAB_ACCOUNT) 9847 *p++ = 'A'; 9848 if (p != name + 1) 9849 *p++ = '-'; 9850 p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size); 9851 9852 if (WARN_ON(p > name + ID_STR_LENGTH - 1)) { 9853 kfree(name); 9854 return ERR_PTR(-EINVAL); 9855 } 9856 kmsan_unpoison_memory(name, p - name); 9857 return name; 9858} 9859 9860static int sysfs_slab_add(struct kmem_cache *s) 9861{ 9862 int err; 9863 const char *name; 9864 struct kset *kset = cache_kset(s); 9865 int unmergeable = slab_unmergeable(s); 9866 9867 if (!unmergeable && disable_higher_order_debug && 9868 (slub_debug & DEBUG_METADATA_FLAGS)) 9869 unmergeable = 1; 9870 9871 if (unmergeable) { 9872 /* 9873 * Slabcache can never be merged so we can use the name proper. 9874 * This is typically the case for debug situations. In that 9875 * case we can catch duplicate names easily. 9876 */ 9877 sysfs_remove_link(&slab_kset->kobj, s->name); 9878 name = s->name; 9879 } else { 9880 /* 9881 * Create a unique name for the slab as a target 9882 * for the symlinks. 9883 */ 9884 name = create_unique_id(s); 9885 if (IS_ERR(name)) 9886 return PTR_ERR(name); 9887 } 9888 9889 s->kobj.kset = kset; 9890 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); 9891 if (err) 9892 goto out; 9893 9894 err = sysfs_create_group(&s->kobj, &slab_attr_group); 9895 if (err) 9896 goto out_del_kobj; 9897 9898 if (!unmergeable) { 9899 /* Setup first alias */ 9900 sysfs_slab_alias(s, s->name); 9901 } 9902out: 9903 if (!unmergeable) 9904 kfree(name); 9905 return err; 9906out_del_kobj: 9907 kobject_del(&s->kobj); 9908 goto out; 9909} 9910 9911void sysfs_slab_unlink(struct kmem_cache *s) 9912{ 9913 if (s->kobj.state_in_sysfs) 9914 kobject_del(&s->kobj); 9915} 9916 9917void sysfs_slab_release(struct kmem_cache *s) 9918{ 9919 kobject_put(&s->kobj); 9920} 9921 9922/* 9923 * Need to buffer aliases during bootup until sysfs becomes 9924 * available lest we lose that information. 9925 */ 9926struct saved_alias { 9927 struct kmem_cache *s; 9928 const char *name; 9929 struct saved_alias *next; 9930}; 9931 9932static struct saved_alias *alias_list; 9933 9934static int sysfs_slab_alias(struct kmem_cache *s, const char *name) 9935{ 9936 struct saved_alias *al; 9937 9938 if (slab_state == FULL) { 9939 /* 9940 * If we have a leftover link then remove it. 9941 */ 9942 sysfs_remove_link(&slab_kset->kobj, name); 9943 /* 9944 * The original cache may have failed to generate sysfs file. 9945 * In that case, sysfs_create_link() returns -ENOENT and 9946 * symbolic link creation is skipped. 9947 */ 9948 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); 9949 } 9950 9951 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); 9952 if (!al) 9953 return -ENOMEM; 9954 9955 al->s = s; 9956 al->name = name; 9957 al->next = alias_list; 9958 alias_list = al; 9959 kmsan_unpoison_memory(al, sizeof(*al)); 9960 return 0; 9961} 9962 9963static int __init slab_sysfs_init(void) 9964{ 9965 struct kmem_cache *s; 9966 int err; 9967 9968 mutex_lock(&slab_mutex); 9969 9970 slab_kset = kset_create_and_add("slab", NULL, kernel_kobj); 9971 if (!slab_kset) { 9972 mutex_unlock(&slab_mutex); 9973 pr_err("Cannot register slab subsystem.\n"); 9974 return -ENOMEM; 9975 } 9976 9977 slab_state = FULL; 9978 9979 list_for_each_entry(s, &slab_caches, list) { 9980 err = sysfs_slab_add(s); 9981 if (err) 9982 pr_err("SLUB: Unable to add boot slab %s to sysfs\n", 9983 s->name); 9984 } 9985 9986 while (alias_list) { 9987 struct saved_alias *al = alias_list; 9988 9989 alias_list = alias_list->next; 9990 err = sysfs_slab_alias(al->s, al->name); 9991 if (err) 9992 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", 9993 al->name); 9994 kfree(al); 9995 } 9996 9997 mutex_unlock(&slab_mutex); 9998 return 0; 9999} 10000late_initcall(slab_sysfs_init); 10001#endif /* SLAB_SUPPORTS_SYSFS */ 10002 10003#if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS) 10004static int slab_debugfs_show(struct seq_file *seq, void *v) 10005{ 10006 struct loc_track *t = seq->private; 10007 struct location *l; 10008 unsigned long idx; 10009 10010 idx = (unsigned long) t->idx; 10011 if (idx < t->count) { 10012 l = &t->loc[idx]; 10013 10014 seq_printf(seq, "%7ld ", l->count); 10015 10016 if (l->addr) 10017 seq_printf(seq, "%pS", (void *)l->addr); 10018 else 10019 seq_puts(seq, "<not-available>"); 10020 10021 if (l->waste) 10022 seq_printf(seq, " waste=%lu/%lu", 10023 l->count * l->waste, l->waste); 10024 10025 if (l->sum_time != l->min_time) { 10026 seq_printf(seq, " age=%ld/%llu/%ld", 10027 l->min_time, div_u64(l->sum_time, l->count), 10028 l->max_time); 10029 } else 10030 seq_printf(seq, " age=%ld", l->min_time); 10031 10032 if (l->min_pid != l->max_pid) 10033 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid); 10034 else 10035 seq_printf(seq, " pid=%ld", 10036 l->min_pid); 10037 10038 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus))) 10039 seq_printf(seq, " cpus=%*pbl", 10040 cpumask_pr_args(to_cpumask(l->cpus))); 10041 10042 if (nr_online_nodes > 1 && !nodes_empty(l->nodes)) 10043 seq_printf(seq, " nodes=%*pbl", 10044 nodemask_pr_args(&l->nodes)); 10045 10046#ifdef CONFIG_STACKDEPOT 10047 { 10048 depot_stack_handle_t handle; 10049 unsigned long *entries; 10050 unsigned int nr_entries, j; 10051 10052 handle = READ_ONCE(l->handle); 10053 if (handle) { 10054 nr_entries = stack_depot_fetch(handle, &entries); 10055 seq_puts(seq, "\n"); 10056 for (j = 0; j < nr_entries; j++) 10057 seq_printf(seq, " %pS\n", (void *)entries[j]); 10058 } 10059 } 10060#endif 10061 seq_puts(seq, "\n"); 10062 } 10063 10064 if (!idx && !t->count) 10065 seq_puts(seq, "No data\n"); 10066 10067 return 0; 10068} 10069 10070static void slab_debugfs_stop(struct seq_file *seq, void *v) 10071{ 10072} 10073 10074static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos) 10075{ 10076 struct loc_track *t = seq->private; 10077 10078 t->idx = ++(*ppos); 10079 if (*ppos <= t->count) 10080 return ppos; 10081 10082 return NULL; 10083} 10084 10085static int cmp_loc_by_count(const void *a, const void *b) 10086{ 10087 struct location *loc1 = (struct location *)a; 10088 struct location *loc2 = (struct location *)b; 10089 10090 return cmp_int(loc2->count, loc1->count); 10091} 10092 10093static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos) 10094{ 10095 struct loc_track *t = seq->private; 10096 10097 t->idx = *ppos; 10098 return ppos; 10099} 10100 10101static const struct seq_operations slab_debugfs_sops = { 10102 .start = slab_debugfs_start, 10103 .next = slab_debugfs_next, 10104 .stop = slab_debugfs_stop, 10105 .show = slab_debugfs_show, 10106}; 10107 10108static int slab_debug_trace_open(struct inode *inode, struct file *filep) 10109{ 10110 10111 struct kmem_cache_node *n; 10112 enum track_item alloc; 10113 int node; 10114 struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops, 10115 sizeof(struct loc_track)); 10116 struct kmem_cache *s = file_inode(filep)->i_private; 10117 unsigned long *obj_map; 10118 10119 if (!t) 10120 return -ENOMEM; 10121 10122 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); 10123 if (!obj_map) { 10124 seq_release_private(inode, filep); 10125 return -ENOMEM; 10126 } 10127 10128 alloc = debugfs_get_aux_num(filep); 10129 10130 if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) { 10131 bitmap_free(obj_map); 10132 seq_release_private(inode, filep); 10133 return -ENOMEM; 10134 } 10135 10136 for_each_kmem_cache_node(s, node, n) { 10137 unsigned long flags; 10138 struct slab *slab; 10139 10140 if (!node_nr_slabs(n)) 10141 continue; 10142 10143 spin_lock_irqsave(&n->list_lock, flags); 10144 list_for_each_entry(slab, &n->partial, slab_list) 10145 process_slab(t, s, slab, alloc, obj_map); 10146 list_for_each_entry(slab, &n->full, slab_list) 10147 process_slab(t, s, slab, alloc, obj_map); 10148 spin_unlock_irqrestore(&n->list_lock, flags); 10149 } 10150 10151 /* Sort locations by count */ 10152 sort(t->loc, t->count, sizeof(struct location), 10153 cmp_loc_by_count, NULL); 10154 10155 bitmap_free(obj_map); 10156 return 0; 10157} 10158 10159static int slab_debug_trace_release(struct inode *inode, struct file *file) 10160{ 10161 struct seq_file *seq = file->private_data; 10162 struct loc_track *t = seq->private; 10163 10164 free_loc_track(t); 10165 return seq_release_private(inode, file); 10166} 10167 10168static const struct file_operations slab_debugfs_fops = { 10169 .open = slab_debug_trace_open, 10170 .read = seq_read, 10171 .llseek = seq_lseek, 10172 .release = slab_debug_trace_release, 10173}; 10174 10175static void debugfs_slab_add(struct kmem_cache *s) 10176{ 10177 struct dentry *slab_cache_dir; 10178 10179 if (unlikely(!slab_debugfs_root)) 10180 return; 10181 10182 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root); 10183 10184 debugfs_create_file_aux_num("alloc_traces", 0400, slab_cache_dir, s, 10185 TRACK_ALLOC, &slab_debugfs_fops); 10186 10187 debugfs_create_file_aux_num("free_traces", 0400, slab_cache_dir, s, 10188 TRACK_FREE, &slab_debugfs_fops); 10189} 10190 10191void debugfs_slab_release(struct kmem_cache *s) 10192{ 10193 debugfs_lookup_and_remove(s->name, slab_debugfs_root); 10194} 10195 10196static int __init slab_debugfs_init(void) 10197{ 10198 struct kmem_cache *s; 10199 10200 slab_debugfs_root = debugfs_create_dir("slab", NULL); 10201 10202 list_for_each_entry(s, &slab_caches, list) 10203 if (s->flags & SLAB_STORE_USER) 10204 debugfs_slab_add(s); 10205 10206 return 0; 10207 10208} 10209__initcall(slab_debugfs_init); 10210#endif 10211/* 10212 * The /proc/slabinfo ABI 10213 */ 10214#ifdef CONFIG_SLUB_DEBUG 10215void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) 10216{ 10217 unsigned long nr_slabs = 0; 10218 unsigned long nr_objs = 0; 10219 unsigned long nr_free = 0; 10220 int node; 10221 struct kmem_cache_node *n; 10222 10223 for_each_kmem_cache_node(s, node, n) { 10224 nr_slabs += node_nr_slabs(n); 10225 nr_objs += node_nr_objs(n); 10226 nr_free += count_partial_free_approx(n); 10227 } 10228 10229 sinfo->active_objs = nr_objs - nr_free; 10230 sinfo->num_objs = nr_objs; 10231 sinfo->active_slabs = nr_slabs; 10232 sinfo->num_slabs = nr_slabs; 10233 sinfo->objects_per_slab = oo_objects(s->oo); 10234 sinfo->cache_order = oo_order(s->oo); 10235} 10236#endif /* CONFIG_SLUB_DEBUG */
~/
Read-only file preview.