FILE
rwsem.c
File preview with syntax highlighting.
linux-6.19.14/kernel/locking/rwsem.c
1// SPDX-License-Identifier: GPL-2.0 2/* kernel/rwsem.c: R/W semaphores, public implementation 3 * 4 * Written by David Howells (dhowells@redhat.com). 5 * Derived from asm-i386/semaphore.h 6 * 7 * Writer lock-stealing by Alex Shi <alex.shi@intel.com> 8 * and Michel Lespinasse <walken@google.com> 9 * 10 * Optimistic spinning by Tim Chen <tim.c.chen@intel.com> 11 * and Davidlohr Bueso <davidlohr@hp.com>. Based on mutexes. 12 * 13 * Rwsem count bit fields re-definition and rwsem rearchitecture by 14 * Waiman Long <longman@redhat.com> and 15 * Peter Zijlstra <peterz@infradead.org>. 16 */ 17 18#include <linux/types.h> 19#include <linux/kernel.h> 20#include <linux/sched.h> 21#include <linux/sched/rt.h> 22#include <linux/sched/task.h> 23#include <linux/sched/debug.h> 24#include <linux/sched/wake_q.h> 25#include <linux/sched/signal.h> 26#include <linux/sched/clock.h> 27#include <linux/export.h> 28#include <linux/rwsem.h> 29#include <linux/atomic.h> 30#include <linux/hung_task.h> 31#include <trace/events/lock.h> 32 33#ifndef CONFIG_PREEMPT_RT 34#include "lock_events.h" 35 36/* 37 * The least significant 2 bits of the owner value has the following 38 * meanings when set. 39 * - Bit 0: RWSEM_READER_OWNED - rwsem may be owned by readers (just a hint) 40 * - Bit 1: RWSEM_NONSPINNABLE - Cannot spin on a reader-owned lock 41 * 42 * When the rwsem is reader-owned and a spinning writer has timed out, 43 * the nonspinnable bit will be set to disable optimistic spinning. 44 45 * When a writer acquires a rwsem, it puts its task_struct pointer 46 * into the owner field. It is cleared after an unlock. 47 * 48 * When a reader acquires a rwsem, it will also puts its task_struct 49 * pointer into the owner field with the RWSEM_READER_OWNED bit set. 50 * On unlock, the owner field will largely be left untouched. So 51 * for a free or reader-owned rwsem, the owner value may contain 52 * information about the last reader that acquires the rwsem. 53 * 54 * That information may be helpful in debugging cases where the system 55 * seems to hang on a reader owned rwsem especially if only one reader 56 * is involved. Ideally we would like to track all the readers that own 57 * a rwsem, but the overhead is simply too big. 58 * 59 * A fast path reader optimistic lock stealing is supported when the rwsem 60 * is previously owned by a writer and the following conditions are met: 61 * - rwsem is not currently writer owned 62 * - the handoff isn't set. 63 */ 64#define RWSEM_READER_OWNED (1UL << 0) 65#define RWSEM_NONSPINNABLE (1UL << 1) 66#define RWSEM_OWNER_FLAGS_MASK (RWSEM_READER_OWNED | RWSEM_NONSPINNABLE) 67 68#ifdef CONFIG_DEBUG_RWSEMS 69# define DEBUG_RWSEMS_WARN_ON(c, sem) do { \ 70 if (!debug_locks_silent && \ 71 WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, magic = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\ 72 #c, atomic_long_read(&(sem)->count), \ 73 (unsigned long) sem->magic, \ 74 atomic_long_read(&(sem)->owner), (long)current, \ 75 list_empty(&(sem)->wait_list) ? "" : "not ")) \ 76 debug_locks_off(); \ 77 } while (0) 78#else 79# define DEBUG_RWSEMS_WARN_ON(c, sem) 80#endif 81 82/* 83 * On 64-bit architectures, the bit definitions of the count are: 84 * 85 * Bit 0 - writer locked bit 86 * Bit 1 - waiters present bit 87 * Bit 2 - lock handoff bit 88 * Bits 3-7 - reserved 89 * Bits 8-62 - 55-bit reader count 90 * Bit 63 - read fail bit 91 * 92 * On 32-bit architectures, the bit definitions of the count are: 93 * 94 * Bit 0 - writer locked bit 95 * Bit 1 - waiters present bit 96 * Bit 2 - lock handoff bit 97 * Bits 3-7 - reserved 98 * Bits 8-30 - 23-bit reader count 99 * Bit 31 - read fail bit 100 * 101 * It is not likely that the most significant bit (read fail bit) will ever 102 * be set. This guard bit is still checked anyway in the down_read() fastpath 103 * just in case we need to use up more of the reader bits for other purpose 104 * in the future. 105 * 106 * atomic_long_fetch_add() is used to obtain reader lock, whereas 107 * atomic_long_cmpxchg() will be used to obtain writer lock. 108 * 109 * There are three places where the lock handoff bit may be set or cleared. 110 * 1) rwsem_mark_wake() for readers -- set, clear 111 * 2) rwsem_try_write_lock() for writers -- set, clear 112 * 3) rwsem_del_waiter() -- clear 113 * 114 * For all the above cases, wait_lock will be held. A writer must also 115 * be the first one in the wait_list to be eligible for setting the handoff 116 * bit. So concurrent setting/clearing of handoff bit is not possible. 117 */ 118#define RWSEM_WRITER_LOCKED (1UL << 0) 119#define RWSEM_FLAG_WAITERS (1UL << 1) 120#define RWSEM_FLAG_HANDOFF (1UL << 2) 121#define RWSEM_FLAG_READFAIL (1UL << (BITS_PER_LONG - 1)) 122 123#define RWSEM_READER_SHIFT 8 124#define RWSEM_READER_BIAS (1UL << RWSEM_READER_SHIFT) 125#define RWSEM_READER_MASK (~(RWSEM_READER_BIAS - 1)) 126#define RWSEM_WRITER_MASK RWSEM_WRITER_LOCKED 127#define RWSEM_LOCK_MASK (RWSEM_WRITER_MASK|RWSEM_READER_MASK) 128#define RWSEM_READ_FAILED_MASK (RWSEM_WRITER_MASK|RWSEM_FLAG_WAITERS|\ 129 RWSEM_FLAG_HANDOFF|RWSEM_FLAG_READFAIL) 130 131/* 132 * All writes to owner are protected by WRITE_ONCE() to make sure that 133 * store tearing can't happen as optimistic spinners may read and use 134 * the owner value concurrently without lock. Read from owner, however, 135 * may not need READ_ONCE() as long as the pointer value is only used 136 * for comparison and isn't being dereferenced. 137 * 138 * Both rwsem_{set,clear}_owner() functions should be in the same 139 * preempt disable section as the atomic op that changes sem->count. 140 */ 141static inline void rwsem_set_owner(struct rw_semaphore *sem) 142{ 143 lockdep_assert_preemption_disabled(); 144 atomic_long_set(&sem->owner, (long)current); 145} 146 147static inline void rwsem_clear_owner(struct rw_semaphore *sem) 148{ 149 lockdep_assert_preemption_disabled(); 150 atomic_long_set(&sem->owner, 0); 151} 152 153/* 154 * Test the flags in the owner field. 155 */ 156static inline bool rwsem_test_oflags(struct rw_semaphore *sem, long flags) 157{ 158 return atomic_long_read(&sem->owner) & flags; 159} 160 161/* 162 * The task_struct pointer of the last owning reader will be left in 163 * the owner field. 164 * 165 * Note that the owner value just indicates the task has owned the rwsem 166 * previously, it may not be the real owner or one of the real owners 167 * anymore when that field is examined, so take it with a grain of salt. 168 * 169 * The reader non-spinnable bit is preserved. 170 */ 171static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, 172 struct task_struct *owner) 173{ 174 unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED | 175 (atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE); 176 177 atomic_long_set(&sem->owner, val); 178} 179 180static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) 181{ 182 __rwsem_set_reader_owned(sem, current); 183} 184 185#if defined(CONFIG_DEBUG_RWSEMS) || defined(CONFIG_DETECT_HUNG_TASK_BLOCKER) 186/* 187 * Return just the real task structure pointer of the owner 188 */ 189struct task_struct *rwsem_owner(struct rw_semaphore *sem) 190{ 191 return (struct task_struct *) 192 (atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK); 193} 194 195/* 196 * Return true if the rwsem is owned by a reader. 197 */ 198bool is_rwsem_reader_owned(struct rw_semaphore *sem) 199{ 200 /* 201 * Check the count to see if it is write-locked. 202 */ 203 long count = atomic_long_read(&sem->count); 204 205 if (count & RWSEM_WRITER_MASK) 206 return false; 207 return rwsem_test_oflags(sem, RWSEM_READER_OWNED); 208} 209 210/* 211 * With CONFIG_DEBUG_RWSEMS or CONFIG_DETECT_HUNG_TASK_BLOCKER configured, 212 * it will make sure that the owner field of a reader-owned rwsem either 213 * points to a real reader-owner(s) or gets cleared. The only exception is 214 * when the unlock is done by up_read_non_owner(). 215 */ 216static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) 217{ 218 unsigned long val = atomic_long_read(&sem->owner); 219 220 while ((val & ~RWSEM_OWNER_FLAGS_MASK) == (unsigned long)current) { 221 if (atomic_long_try_cmpxchg(&sem->owner, &val, 222 val & RWSEM_OWNER_FLAGS_MASK)) 223 return; 224 } 225} 226#else 227static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) 228{ 229} 230#endif 231 232/* 233 * Set the RWSEM_NONSPINNABLE bits if the RWSEM_READER_OWNED flag 234 * remains set. Otherwise, the operation will be aborted. 235 */ 236static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) 237{ 238 unsigned long owner = atomic_long_read(&sem->owner); 239 240 do { 241 if (!(owner & RWSEM_READER_OWNED)) 242 break; 243 if (owner & RWSEM_NONSPINNABLE) 244 break; 245 } while (!atomic_long_try_cmpxchg(&sem->owner, &owner, 246 owner | RWSEM_NONSPINNABLE)); 247} 248 249static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) 250{ 251 *cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count); 252 253 if (WARN_ON_ONCE(*cntp < 0)) 254 rwsem_set_nonspinnable(sem); 255 256 if (!(*cntp & RWSEM_READ_FAILED_MASK)) { 257 rwsem_set_reader_owned(sem); 258 return true; 259 } 260 261 return false; 262} 263 264static inline bool rwsem_write_trylock(struct rw_semaphore *sem) 265{ 266 long tmp = RWSEM_UNLOCKED_VALUE; 267 268 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) { 269 rwsem_set_owner(sem); 270 return true; 271 } 272 273 return false; 274} 275 276/* 277 * Return the real task structure pointer of the owner and the embedded 278 * flags in the owner. pflags must be non-NULL. 279 */ 280static inline struct task_struct * 281rwsem_owner_flags(struct rw_semaphore *sem, unsigned long *pflags) 282{ 283 unsigned long owner = atomic_long_read(&sem->owner); 284 285 *pflags = owner & RWSEM_OWNER_FLAGS_MASK; 286 return (struct task_struct *)(owner & ~RWSEM_OWNER_FLAGS_MASK); 287} 288 289/* 290 * Guide to the rw_semaphore's count field. 291 * 292 * When the RWSEM_WRITER_LOCKED bit in count is set, the lock is owned 293 * by a writer. 294 * 295 * The lock is owned by readers when 296 * (1) the RWSEM_WRITER_LOCKED isn't set in count, 297 * (2) some of the reader bits are set in count, and 298 * (3) the owner field has RWSEM_READ_OWNED bit set. 299 * 300 * Having some reader bits set is not enough to guarantee a readers owned 301 * lock as the readers may be in the process of backing out from the count 302 * and a writer has just released the lock. So another writer may steal 303 * the lock immediately after that. 304 */ 305 306/* 307 * Initialize an rwsem: 308 */ 309void __init_rwsem(struct rw_semaphore *sem, const char *name, 310 struct lock_class_key *key) 311{ 312#ifdef CONFIG_DEBUG_LOCK_ALLOC 313 /* 314 * Make sure we are not reinitializing a held semaphore: 315 */ 316 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); 317 lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); 318#endif 319#ifdef CONFIG_DEBUG_RWSEMS 320 sem->magic = sem; 321#endif 322 atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); 323 raw_spin_lock_init(&sem->wait_lock); 324 INIT_LIST_HEAD(&sem->wait_list); 325 atomic_long_set(&sem->owner, 0L); 326#ifdef CONFIG_RWSEM_SPIN_ON_OWNER 327 osq_lock_init(&sem->osq); 328#endif 329} 330EXPORT_SYMBOL(__init_rwsem); 331 332enum rwsem_waiter_type { 333 RWSEM_WAITING_FOR_WRITE, 334 RWSEM_WAITING_FOR_READ 335}; 336 337struct rwsem_waiter { 338 struct list_head list; 339 struct task_struct *task; 340 enum rwsem_waiter_type type; 341 unsigned long timeout; 342 bool handoff_set; 343}; 344#define rwsem_first_waiter(sem) \ 345 list_first_entry(&sem->wait_list, struct rwsem_waiter, list) 346 347enum rwsem_wake_type { 348 RWSEM_WAKE_ANY, /* Wake whatever's at head of wait list */ 349 RWSEM_WAKE_READERS, /* Wake readers only */ 350 RWSEM_WAKE_READ_OWNED /* Waker thread holds the read lock */ 351}; 352 353/* 354 * The typical HZ value is either 250 or 1000. So set the minimum waiting 355 * time to at least 4ms or 1 jiffy (if it is higher than 4ms) in the wait 356 * queue before initiating the handoff protocol. 357 */ 358#define RWSEM_WAIT_TIMEOUT DIV_ROUND_UP(HZ, 250) 359 360/* 361 * Magic number to batch-wakeup waiting readers, even when writers are 362 * also present in the queue. This both limits the amount of work the 363 * waking thread must do and also prevents any potential counter overflow, 364 * however unlikely. 365 */ 366#define MAX_READERS_WAKEUP 0x100 367 368static inline void 369rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) 370{ 371 lockdep_assert_held(&sem->wait_lock); 372 list_add_tail(&waiter->list, &sem->wait_list); 373 /* caller will set RWSEM_FLAG_WAITERS */ 374} 375 376/* 377 * Remove a waiter from the wait_list and clear flags. 378 * 379 * Both rwsem_mark_wake() and rwsem_try_write_lock() contain a full 'copy' of 380 * this function. Modify with care. 381 * 382 * Return: true if wait_list isn't empty and false otherwise 383 */ 384static inline bool 385rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) 386{ 387 lockdep_assert_held(&sem->wait_lock); 388 list_del(&waiter->list); 389 if (likely(!list_empty(&sem->wait_list))) 390 return true; 391 392 atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count); 393 return false; 394} 395 396/* 397 * handle the lock release when processes blocked on it that can now run 398 * - if we come here from up_xxxx(), then the RWSEM_FLAG_WAITERS bit must 399 * have been set. 400 * - there must be someone on the queue 401 * - the wait_lock must be held by the caller 402 * - tasks are marked for wakeup, the caller must later invoke wake_up_q() 403 * to actually wakeup the blocked task(s) and drop the reference count, 404 * preferably when the wait_lock is released 405 * - woken process blocks are discarded from the list after having task zeroed 406 * - writers are only marked woken if downgrading is false 407 * 408 * Implies rwsem_del_waiter() for all woken readers. 409 */ 410static void rwsem_mark_wake(struct rw_semaphore *sem, 411 enum rwsem_wake_type wake_type, 412 struct wake_q_head *wake_q) 413{ 414 struct rwsem_waiter *waiter, *tmp; 415 long oldcount, woken = 0, adjustment = 0; 416 struct list_head wlist; 417 418 lockdep_assert_held(&sem->wait_lock); 419 420 /* 421 * Take a peek at the queue head waiter such that we can determine 422 * the wakeup(s) to perform. 423 */ 424 waiter = rwsem_first_waiter(sem); 425 426 if (waiter->type == RWSEM_WAITING_FOR_WRITE) { 427 if (wake_type == RWSEM_WAKE_ANY) { 428 /* 429 * Mark writer at the front of the queue for wakeup. 430 * Until the task is actually later awoken later by 431 * the caller, other writers are able to steal it. 432 * Readers, on the other hand, will block as they 433 * will notice the queued writer. 434 */ 435 wake_q_add(wake_q, waiter->task); 436 lockevent_inc(rwsem_wake_writer); 437 } 438 439 return; 440 } 441 442 /* 443 * No reader wakeup if there are too many of them already. 444 */ 445 if (unlikely(atomic_long_read(&sem->count) < 0)) 446 return; 447 448 /* 449 * Writers might steal the lock before we grant it to the next reader. 450 * We prefer to do the first reader grant before counting readers 451 * so we can bail out early if a writer stole the lock. 452 */ 453 if (wake_type != RWSEM_WAKE_READ_OWNED) { 454 struct task_struct *owner; 455 456 adjustment = RWSEM_READER_BIAS; 457 oldcount = atomic_long_fetch_add(adjustment, &sem->count); 458 if (unlikely(oldcount & RWSEM_WRITER_MASK)) { 459 /* 460 * When we've been waiting "too" long (for writers 461 * to give up the lock), request a HANDOFF to 462 * force the issue. 463 */ 464 if (time_after(jiffies, waiter->timeout)) { 465 if (!(oldcount & RWSEM_FLAG_HANDOFF)) { 466 adjustment -= RWSEM_FLAG_HANDOFF; 467 lockevent_inc(rwsem_rlock_handoff); 468 } 469 waiter->handoff_set = true; 470 } 471 472 atomic_long_add(-adjustment, &sem->count); 473 return; 474 } 475 /* 476 * Set it to reader-owned to give spinners an early 477 * indication that readers now have the lock. 478 * The reader nonspinnable bit seen at slowpath entry of 479 * the reader is copied over. 480 */ 481 owner = waiter->task; 482 __rwsem_set_reader_owned(sem, owner); 483 } 484 485 /* 486 * Grant up to MAX_READERS_WAKEUP read locks to all the readers in the 487 * queue. We know that the woken will be at least 1 as we accounted 488 * for above. Note we increment the 'active part' of the count by the 489 * number of readers before waking any processes up. 490 * 491 * This is an adaptation of the phase-fair R/W locks where at the 492 * reader phase (first waiter is a reader), all readers are eligible 493 * to acquire the lock at the same time irrespective of their order 494 * in the queue. The writers acquire the lock according to their 495 * order in the queue. 496 * 497 * We have to do wakeup in 2 passes to prevent the possibility that 498 * the reader count may be decremented before it is incremented. It 499 * is because the to-be-woken waiter may not have slept yet. So it 500 * may see waiter->task got cleared, finish its critical section and 501 * do an unlock before the reader count increment. 502 * 503 * 1) Collect the read-waiters in a separate list, count them and 504 * fully increment the reader count in rwsem. 505 * 2) For each waiters in the new list, clear waiter->task and 506 * put them into wake_q to be woken up later. 507 */ 508 INIT_LIST_HEAD(&wlist); 509 list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { 510 if (waiter->type == RWSEM_WAITING_FOR_WRITE) 511 continue; 512 513 woken++; 514 list_move_tail(&waiter->list, &wlist); 515 516 /* 517 * Limit # of readers that can be woken up per wakeup call. 518 */ 519 if (unlikely(woken >= MAX_READERS_WAKEUP)) 520 break; 521 } 522 523 adjustment = woken * RWSEM_READER_BIAS - adjustment; 524 lockevent_cond_inc(rwsem_wake_reader, woken); 525 526 oldcount = atomic_long_read(&sem->count); 527 if (list_empty(&sem->wait_list)) { 528 /* 529 * Combined with list_move_tail() above, this implies 530 * rwsem_del_waiter(). 531 */ 532 adjustment -= RWSEM_FLAG_WAITERS; 533 if (oldcount & RWSEM_FLAG_HANDOFF) 534 adjustment -= RWSEM_FLAG_HANDOFF; 535 } else if (woken) { 536 /* 537 * When we've woken a reader, we no longer need to force 538 * writers to give up the lock and we can clear HANDOFF. 539 */ 540 if (oldcount & RWSEM_FLAG_HANDOFF) 541 adjustment -= RWSEM_FLAG_HANDOFF; 542 } 543 544 if (adjustment) 545 atomic_long_add(adjustment, &sem->count); 546 547 /* 2nd pass */ 548 list_for_each_entry_safe(waiter, tmp, &wlist, list) { 549 struct task_struct *tsk; 550 551 tsk = waiter->task; 552 get_task_struct(tsk); 553 554 /* 555 * Ensure calling get_task_struct() before setting the reader 556 * waiter to nil such that rwsem_down_read_slowpath() cannot 557 * race with do_exit() by always holding a reference count 558 * to the task to wakeup. 559 */ 560 smp_store_release(&waiter->task, NULL); 561 /* 562 * Ensure issuing the wakeup (either by us or someone else) 563 * after setting the reader waiter to nil. 564 */ 565 wake_q_add_safe(wake_q, tsk); 566 } 567} 568 569/* 570 * Remove a waiter and try to wake up other waiters in the wait queue 571 * This function is called from the out_nolock path of both the reader and 572 * writer slowpaths with wait_lock held. It releases the wait_lock and 573 * optionally wake up waiters before it returns. 574 */ 575static inline void 576rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter, 577 struct wake_q_head *wake_q) 578 __releases(&sem->wait_lock) 579{ 580 bool first = rwsem_first_waiter(sem) == waiter; 581 582 wake_q_init(wake_q); 583 584 /* 585 * If the wait_list isn't empty and the waiter to be deleted is 586 * the first waiter, we wake up the remaining waiters as they may 587 * be eligible to acquire or spin on the lock. 588 */ 589 if (rwsem_del_waiter(sem, waiter) && first) 590 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, wake_q); 591 raw_spin_unlock_irq(&sem->wait_lock); 592 if (!wake_q_empty(wake_q)) 593 wake_up_q(wake_q); 594} 595 596/* 597 * This function must be called with the sem->wait_lock held to prevent 598 * race conditions between checking the rwsem wait list and setting the 599 * sem->count accordingly. 600 * 601 * Implies rwsem_del_waiter() on success. 602 */ 603static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, 604 struct rwsem_waiter *waiter) 605{ 606 struct rwsem_waiter *first = rwsem_first_waiter(sem); 607 long count, new; 608 609 lockdep_assert_held(&sem->wait_lock); 610 611 count = atomic_long_read(&sem->count); 612 do { 613 bool has_handoff = !!(count & RWSEM_FLAG_HANDOFF); 614 615 if (has_handoff) { 616 /* 617 * Honor handoff bit and yield only when the first 618 * waiter is the one that set it. Otherwisee, we 619 * still try to acquire the rwsem. 620 */ 621 if (first->handoff_set && (waiter != first)) 622 return false; 623 } 624 625 new = count; 626 627 if (count & RWSEM_LOCK_MASK) { 628 /* 629 * A waiter (first or not) can set the handoff bit 630 * if it is an RT task or wait in the wait queue 631 * for too long. 632 */ 633 if (has_handoff || (!rt_or_dl_task(waiter->task) && 634 !time_after(jiffies, waiter->timeout))) 635 return false; 636 637 new |= RWSEM_FLAG_HANDOFF; 638 } else { 639 new |= RWSEM_WRITER_LOCKED; 640 new &= ~RWSEM_FLAG_HANDOFF; 641 642 if (list_is_singular(&sem->wait_list)) 643 new &= ~RWSEM_FLAG_WAITERS; 644 } 645 } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new)); 646 647 /* 648 * We have either acquired the lock with handoff bit cleared or set 649 * the handoff bit. Only the first waiter can have its handoff_set 650 * set here to enable optimistic spinning in slowpath loop. 651 */ 652 if (new & RWSEM_FLAG_HANDOFF) { 653 first->handoff_set = true; 654 lockevent_inc(rwsem_wlock_handoff); 655 return false; 656 } 657 658 /* 659 * Have rwsem_try_write_lock() fully imply rwsem_del_waiter() on 660 * success. 661 */ 662 list_del(&waiter->list); 663 rwsem_set_owner(sem); 664 return true; 665} 666 667/* 668 * The rwsem_spin_on_owner() function returns the following 4 values 669 * depending on the lock owner state. 670 * OWNER_NULL : owner is currently NULL 671 * OWNER_WRITER: when owner changes and is a writer 672 * OWNER_READER: when owner changes and the new owner may be a reader. 673 * OWNER_NONSPINNABLE: 674 * when optimistic spinning has to stop because either the 675 * owner stops running, is unknown, or its timeslice has 676 * been used up. 677 */ 678enum owner_state { 679 OWNER_NULL = 1 << 0, 680 OWNER_WRITER = 1 << 1, 681 OWNER_READER = 1 << 2, 682 OWNER_NONSPINNABLE = 1 << 3, 683}; 684 685#ifdef CONFIG_RWSEM_SPIN_ON_OWNER 686/* 687 * Try to acquire write lock before the writer has been put on wait queue. 688 */ 689static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) 690{ 691 long count = atomic_long_read(&sem->count); 692 693 while (!(count & (RWSEM_LOCK_MASK|RWSEM_FLAG_HANDOFF))) { 694 if (atomic_long_try_cmpxchg_acquire(&sem->count, &count, 695 count | RWSEM_WRITER_LOCKED)) { 696 rwsem_set_owner(sem); 697 lockevent_inc(rwsem_opt_lock); 698 return true; 699 } 700 } 701 return false; 702} 703 704static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) 705{ 706 struct task_struct *owner; 707 unsigned long flags; 708 bool ret = true; 709 710 if (need_resched()) { 711 lockevent_inc(rwsem_opt_fail); 712 return false; 713 } 714 715 /* 716 * Disable preemption is equal to the RCU read-side crital section, 717 * thus the task_strcut structure won't go away. 718 */ 719 owner = rwsem_owner_flags(sem, &flags); 720 /* 721 * Don't check the read-owner as the entry may be stale. 722 */ 723 if ((flags & RWSEM_NONSPINNABLE) || 724 (owner && !(flags & RWSEM_READER_OWNED) && !owner_on_cpu(owner))) 725 ret = false; 726 727 lockevent_cond_inc(rwsem_opt_fail, !ret); 728 return ret; 729} 730 731static inline enum owner_state 732rwsem_owner_state(struct task_struct *owner, unsigned long flags) 733{ 734 if (flags & RWSEM_NONSPINNABLE) 735 return OWNER_NONSPINNABLE; 736 737 if (flags & RWSEM_READER_OWNED) 738 return OWNER_READER; 739 740 return owner ? OWNER_WRITER : OWNER_NULL; 741} 742 743static noinline enum owner_state 744rwsem_spin_on_owner(struct rw_semaphore *sem) 745{ 746 struct task_struct *new, *owner; 747 unsigned long flags, new_flags; 748 enum owner_state state; 749 750 lockdep_assert_preemption_disabled(); 751 752 owner = rwsem_owner_flags(sem, &flags); 753 state = rwsem_owner_state(owner, flags); 754 if (state != OWNER_WRITER) 755 return state; 756 757 for (;;) { 758 /* 759 * When a waiting writer set the handoff flag, it may spin 760 * on the owner as well. Once that writer acquires the lock, 761 * we can spin on it. So we don't need to quit even when the 762 * handoff bit is set. 763 */ 764 new = rwsem_owner_flags(sem, &new_flags); 765 if ((new != owner) || (new_flags != flags)) { 766 state = rwsem_owner_state(new, new_flags); 767 break; 768 } 769 770 /* 771 * Ensure we emit the owner->on_cpu, dereference _after_ 772 * checking sem->owner still matches owner, if that fails, 773 * owner might point to free()d memory, if it still matches, 774 * our spinning context already disabled preemption which is 775 * equal to RCU read-side crital section ensures the memory 776 * stays valid. 777 */ 778 barrier(); 779 780 if (need_resched() || !owner_on_cpu(owner)) { 781 state = OWNER_NONSPINNABLE; 782 break; 783 } 784 785 cpu_relax(); 786 } 787 788 return state; 789} 790 791/* 792 * Calculate reader-owned rwsem spinning threshold for writer 793 * 794 * The more readers own the rwsem, the longer it will take for them to 795 * wind down and free the rwsem. So the empirical formula used to 796 * determine the actual spinning time limit here is: 797 * 798 * Spinning threshold = (10 + nr_readers/2)us 799 * 800 * The limit is capped to a maximum of 25us (30 readers). This is just 801 * a heuristic and is subjected to change in the future. 802 */ 803static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem) 804{ 805 long count = atomic_long_read(&sem->count); 806 int readers = count >> RWSEM_READER_SHIFT; 807 u64 delta; 808 809 if (readers > 30) 810 readers = 30; 811 delta = (20 + readers) * NSEC_PER_USEC / 2; 812 813 return sched_clock() + delta; 814} 815 816static bool rwsem_optimistic_spin(struct rw_semaphore *sem) 817{ 818 bool taken = false; 819 int prev_owner_state = OWNER_NULL; 820 int loop = 0; 821 u64 rspin_threshold = 0; 822 823 /* sem->wait_lock should not be held when doing optimistic spinning */ 824 if (!osq_lock(&sem->osq)) 825 goto done; 826 827 /* 828 * Optimistically spin on the owner field and attempt to acquire the 829 * lock whenever the owner changes. Spinning will be stopped when: 830 * 1) the owning writer isn't running; or 831 * 2) readers own the lock and spinning time has exceeded limit. 832 */ 833 for (;;) { 834 enum owner_state owner_state; 835 836 owner_state = rwsem_spin_on_owner(sem); 837 if (owner_state == OWNER_NONSPINNABLE) 838 break; 839 840 /* 841 * Try to acquire the lock 842 */ 843 taken = rwsem_try_write_lock_unqueued(sem); 844 845 if (taken) 846 break; 847 848 /* 849 * Time-based reader-owned rwsem optimistic spinning 850 */ 851 if (owner_state == OWNER_READER) { 852 /* 853 * Re-initialize rspin_threshold every time when 854 * the owner state changes from non-reader to reader. 855 * This allows a writer to steal the lock in between 856 * 2 reader phases and have the threshold reset at 857 * the beginning of the 2nd reader phase. 858 */ 859 if (prev_owner_state != OWNER_READER) { 860 if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE)) 861 break; 862 rspin_threshold = rwsem_rspin_threshold(sem); 863 loop = 0; 864 } 865 866 /* 867 * Check time threshold once every 16 iterations to 868 * avoid calling sched_clock() too frequently so 869 * as to reduce the average latency between the times 870 * when the lock becomes free and when the spinner 871 * is ready to do a trylock. 872 */ 873 else if (!(++loop & 0xf) && (sched_clock() > rspin_threshold)) { 874 rwsem_set_nonspinnable(sem); 875 lockevent_inc(rwsem_opt_nospin); 876 break; 877 } 878 } 879 880 /* 881 * An RT task cannot do optimistic spinning if it cannot 882 * be sure the lock holder is running or live-lock may 883 * happen if the current task and the lock holder happen 884 * to run in the same CPU. However, aborting optimistic 885 * spinning while a NULL owner is detected may miss some 886 * opportunity where spinning can continue without causing 887 * problem. 888 * 889 * There are 2 possible cases where an RT task may be able 890 * to continue spinning. 891 * 892 * 1) The lock owner is in the process of releasing the 893 * lock, sem->owner is cleared but the lock has not 894 * been released yet. 895 * 2) The lock was free and owner cleared, but another 896 * task just comes in and acquire the lock before 897 * we try to get it. The new owner may be a spinnable 898 * writer. 899 * 900 * To take advantage of two scenarios listed above, the RT 901 * task is made to retry one more time to see if it can 902 * acquire the lock or continue spinning on the new owning 903 * writer. Of course, if the time lag is long enough or the 904 * new owner is not a writer or spinnable, the RT task will 905 * quit spinning. 906 * 907 * If the owner is a writer, the need_resched() check is 908 * done inside rwsem_spin_on_owner(). If the owner is not 909 * a writer, need_resched() check needs to be done here. 910 */ 911 if (owner_state != OWNER_WRITER) { 912 if (need_resched()) 913 break; 914 if (rt_or_dl_task(current) && 915 (prev_owner_state != OWNER_WRITER)) 916 break; 917 } 918 prev_owner_state = owner_state; 919 920 /* 921 * The cpu_relax() call is a compiler barrier which forces 922 * everything in this loop to be re-loaded. We don't need 923 * memory barriers as we'll eventually observe the right 924 * values at the cost of a few extra spins. 925 */ 926 cpu_relax(); 927 } 928 osq_unlock(&sem->osq); 929done: 930 lockevent_cond_inc(rwsem_opt_fail, !taken); 931 return taken; 932} 933 934/* 935 * Clear the owner's RWSEM_NONSPINNABLE bit if it is set. This should 936 * only be called when the reader count reaches 0. 937 */ 938static inline void clear_nonspinnable(struct rw_semaphore *sem) 939{ 940 if (unlikely(rwsem_test_oflags(sem, RWSEM_NONSPINNABLE))) 941 atomic_long_andnot(RWSEM_NONSPINNABLE, &sem->owner); 942} 943 944#else 945static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) 946{ 947 return false; 948} 949 950static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem) 951{ 952 return false; 953} 954 955static inline void clear_nonspinnable(struct rw_semaphore *sem) { } 956 957static inline enum owner_state 958rwsem_spin_on_owner(struct rw_semaphore *sem) 959{ 960 return OWNER_NONSPINNABLE; 961} 962#endif 963 964/* 965 * Prepare to wake up waiter(s) in the wait queue by putting them into the 966 * given wake_q if the rwsem lock owner isn't a writer. If rwsem is likely 967 * reader-owned, wake up read lock waiters in queue front or wake up any 968 * front waiter otherwise. 969 970 * This is being called from both reader and writer slow paths. 971 */ 972static inline void rwsem_cond_wake_waiter(struct rw_semaphore *sem, long count, 973 struct wake_q_head *wake_q) 974{ 975 enum rwsem_wake_type wake_type; 976 977 if (count & RWSEM_WRITER_MASK) 978 return; 979 980 if (count & RWSEM_READER_MASK) { 981 wake_type = RWSEM_WAKE_READERS; 982 } else { 983 wake_type = RWSEM_WAKE_ANY; 984 clear_nonspinnable(sem); 985 } 986 rwsem_mark_wake(sem, wake_type, wake_q); 987} 988 989/* 990 * Wait for the read lock to be granted 991 */ 992static struct rw_semaphore __sched * 993rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int state) 994{ 995 long adjustment = -RWSEM_READER_BIAS; 996 long rcnt = (count >> RWSEM_READER_SHIFT); 997 struct rwsem_waiter waiter; 998 DEFINE_WAKE_Q(wake_q); 999 1000 /* 1001 * To prevent a constant stream of readers from starving a sleeping 1002 * writer, don't attempt optimistic lock stealing if the lock is 1003 * very likely owned by readers. 1004 */ 1005 if ((atomic_long_read(&sem->owner) & RWSEM_READER_OWNED) && 1006 (rcnt > 1) && !(count & RWSEM_WRITER_LOCKED)) 1007 goto queue; 1008 1009 /* 1010 * Reader optimistic lock stealing. 1011 */ 1012 if (!(count & (RWSEM_WRITER_LOCKED | RWSEM_FLAG_HANDOFF))) { 1013 rwsem_set_reader_owned(sem); 1014 lockevent_inc(rwsem_rlock_steal); 1015 1016 /* 1017 * Wake up other readers in the wait queue if it is 1018 * the first reader. 1019 */ 1020 if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) { 1021 raw_spin_lock_irq(&sem->wait_lock); 1022 if (!list_empty(&sem->wait_list)) 1023 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, 1024 &wake_q); 1025 raw_spin_unlock_irq(&sem->wait_lock); 1026 wake_up_q(&wake_q); 1027 } 1028 return sem; 1029 } 1030 1031queue: 1032 waiter.task = current; 1033 waiter.type = RWSEM_WAITING_FOR_READ; 1034 waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT; 1035 waiter.handoff_set = false; 1036 1037 raw_spin_lock_irq(&sem->wait_lock); 1038 if (list_empty(&sem->wait_list)) { 1039 /* 1040 * In case the wait queue is empty and the lock isn't owned 1041 * by a writer, this reader can exit the slowpath and return 1042 * immediately as its RWSEM_READER_BIAS has already been set 1043 * in the count. 1044 */ 1045 if (!(atomic_long_read(&sem->count) & RWSEM_WRITER_MASK)) { 1046 /* Provide lock ACQUIRE */ 1047 smp_acquire__after_ctrl_dep(); 1048 raw_spin_unlock_irq(&sem->wait_lock); 1049 rwsem_set_reader_owned(sem); 1050 lockevent_inc(rwsem_rlock_fast); 1051 return sem; 1052 } 1053 adjustment += RWSEM_FLAG_WAITERS; 1054 } 1055 rwsem_add_waiter(sem, &waiter); 1056 1057 /* we're now waiting on the lock, but no longer actively locking */ 1058 count = atomic_long_add_return(adjustment, &sem->count); 1059 1060 rwsem_cond_wake_waiter(sem, count, &wake_q); 1061 raw_spin_unlock_irq(&sem->wait_lock); 1062 1063 if (!wake_q_empty(&wake_q)) 1064 wake_up_q(&wake_q); 1065 1066 trace_contention_begin(sem, LCB_F_READ); 1067 set_current_state(state); 1068 1069 if (state == TASK_UNINTERRUPTIBLE) 1070 hung_task_set_blocker(sem, BLOCKER_TYPE_RWSEM_READER); 1071 1072 /* wait to be given the lock */ 1073 for (;;) { 1074 if (!smp_load_acquire(&waiter.task)) { 1075 /* Matches rwsem_mark_wake()'s smp_store_release(). */ 1076 break; 1077 } 1078 if (signal_pending_state(state, current)) { 1079 raw_spin_lock_irq(&sem->wait_lock); 1080 if (waiter.task) 1081 goto out_nolock; 1082 raw_spin_unlock_irq(&sem->wait_lock); 1083 /* Ordered by sem->wait_lock against rwsem_mark_wake(). */ 1084 break; 1085 } 1086 schedule_preempt_disabled(); 1087 lockevent_inc(rwsem_sleep_reader); 1088 set_current_state(state); 1089 } 1090 1091 if (state == TASK_UNINTERRUPTIBLE) 1092 hung_task_clear_blocker(); 1093 1094 __set_current_state(TASK_RUNNING); 1095 lockevent_inc(rwsem_rlock); 1096 trace_contention_end(sem, 0); 1097 return sem; 1098 1099out_nolock: 1100 rwsem_del_wake_waiter(sem, &waiter, &wake_q); 1101 __set_current_state(TASK_RUNNING); 1102 lockevent_inc(rwsem_rlock_fail); 1103 trace_contention_end(sem, -EINTR); 1104 return ERR_PTR(-EINTR); 1105} 1106 1107/* 1108 * Wait until we successfully acquire the write lock 1109 */ 1110static struct rw_semaphore __sched * 1111rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) 1112{ 1113 struct rwsem_waiter waiter; 1114 DEFINE_WAKE_Q(wake_q); 1115 1116 /* do optimistic spinning and steal lock if possible */ 1117 if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) { 1118 /* rwsem_optimistic_spin() implies ACQUIRE on success */ 1119 return sem; 1120 } 1121 1122 /* 1123 * Optimistic spinning failed, proceed to the slowpath 1124 * and block until we can acquire the sem. 1125 */ 1126 waiter.task = current; 1127 waiter.type = RWSEM_WAITING_FOR_WRITE; 1128 waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT; 1129 waiter.handoff_set = false; 1130 1131 raw_spin_lock_irq(&sem->wait_lock); 1132 rwsem_add_waiter(sem, &waiter); 1133 1134 /* we're now waiting on the lock */ 1135 if (rwsem_first_waiter(sem) != &waiter) { 1136 rwsem_cond_wake_waiter(sem, atomic_long_read(&sem->count), 1137 &wake_q); 1138 if (!wake_q_empty(&wake_q)) { 1139 /* 1140 * We want to minimize wait_lock hold time especially 1141 * when a large number of readers are to be woken up. 1142 */ 1143 raw_spin_unlock_irq(&sem->wait_lock); 1144 wake_up_q(&wake_q); 1145 raw_spin_lock_irq(&sem->wait_lock); 1146 } 1147 } else { 1148 atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count); 1149 } 1150 1151 /* wait until we successfully acquire the lock */ 1152 set_current_state(state); 1153 trace_contention_begin(sem, LCB_F_WRITE); 1154 1155 if (state == TASK_UNINTERRUPTIBLE) 1156 hung_task_set_blocker(sem, BLOCKER_TYPE_RWSEM_WRITER); 1157 1158 for (;;) { 1159 if (rwsem_try_write_lock(sem, &waiter)) { 1160 /* rwsem_try_write_lock() implies ACQUIRE on success */ 1161 break; 1162 } 1163 1164 raw_spin_unlock_irq(&sem->wait_lock); 1165 1166 if (signal_pending_state(state, current)) 1167 goto out_nolock; 1168 1169 /* 1170 * After setting the handoff bit and failing to acquire 1171 * the lock, attempt to spin on owner to accelerate lock 1172 * transfer. If the previous owner is a on-cpu writer and it 1173 * has just released the lock, OWNER_NULL will be returned. 1174 * In this case, we attempt to acquire the lock again 1175 * without sleeping. 1176 */ 1177 if (waiter.handoff_set) { 1178 enum owner_state owner_state; 1179 1180 owner_state = rwsem_spin_on_owner(sem); 1181 if (owner_state == OWNER_NULL) 1182 goto trylock_again; 1183 } 1184 1185 schedule_preempt_disabled(); 1186 lockevent_inc(rwsem_sleep_writer); 1187 set_current_state(state); 1188trylock_again: 1189 raw_spin_lock_irq(&sem->wait_lock); 1190 } 1191 1192 if (state == TASK_UNINTERRUPTIBLE) 1193 hung_task_clear_blocker(); 1194 1195 __set_current_state(TASK_RUNNING); 1196 raw_spin_unlock_irq(&sem->wait_lock); 1197 lockevent_inc(rwsem_wlock); 1198 trace_contention_end(sem, 0); 1199 return sem; 1200 1201out_nolock: 1202 __set_current_state(TASK_RUNNING); 1203 raw_spin_lock_irq(&sem->wait_lock); 1204 rwsem_del_wake_waiter(sem, &waiter, &wake_q); 1205 lockevent_inc(rwsem_wlock_fail); 1206 trace_contention_end(sem, -EINTR); 1207 return ERR_PTR(-EINTR); 1208} 1209 1210/* 1211 * handle waking up a waiter on the semaphore 1212 * - up_read/up_write has decremented the active part of count if we come here 1213 */ 1214static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem) 1215{ 1216 unsigned long flags; 1217 DEFINE_WAKE_Q(wake_q); 1218 1219 raw_spin_lock_irqsave(&sem->wait_lock, flags); 1220 1221 if (!list_empty(&sem->wait_list)) 1222 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); 1223 1224 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); 1225 wake_up_q(&wake_q); 1226 1227 return sem; 1228} 1229 1230/* 1231 * downgrade a write lock into a read lock 1232 * - caller incremented waiting part of count and discovered it still negative 1233 * - just wake up any readers at the front of the queue 1234 */ 1235static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) 1236{ 1237 unsigned long flags; 1238 DEFINE_WAKE_Q(wake_q); 1239 1240 raw_spin_lock_irqsave(&sem->wait_lock, flags); 1241 1242 if (!list_empty(&sem->wait_list)) 1243 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); 1244 1245 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); 1246 wake_up_q(&wake_q); 1247 1248 return sem; 1249} 1250 1251/* 1252 * lock for reading 1253 */ 1254static __always_inline int __down_read_common(struct rw_semaphore *sem, int state) 1255{ 1256 int ret = 0; 1257 long count; 1258 1259 preempt_disable(); 1260 if (!rwsem_read_trylock(sem, &count)) { 1261 if (IS_ERR(rwsem_down_read_slowpath(sem, count, state))) { 1262 ret = -EINTR; 1263 goto out; 1264 } 1265 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); 1266 } 1267out: 1268 preempt_enable(); 1269 return ret; 1270} 1271 1272static __always_inline void __down_read(struct rw_semaphore *sem) 1273{ 1274 __down_read_common(sem, TASK_UNINTERRUPTIBLE); 1275} 1276 1277static __always_inline int __down_read_interruptible(struct rw_semaphore *sem) 1278{ 1279 return __down_read_common(sem, TASK_INTERRUPTIBLE); 1280} 1281 1282static __always_inline int __down_read_killable(struct rw_semaphore *sem) 1283{ 1284 return __down_read_common(sem, TASK_KILLABLE); 1285} 1286 1287static inline int __down_read_trylock(struct rw_semaphore *sem) 1288{ 1289 int ret = 0; 1290 long tmp; 1291 1292 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); 1293 1294 preempt_disable(); 1295 tmp = atomic_long_read(&sem->count); 1296 while (!(tmp & RWSEM_READ_FAILED_MASK)) { 1297 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, 1298 tmp + RWSEM_READER_BIAS)) { 1299 rwsem_set_reader_owned(sem); 1300 ret = 1; 1301 break; 1302 } 1303 } 1304 preempt_enable(); 1305 return ret; 1306} 1307 1308/* 1309 * lock for writing 1310 */ 1311static __always_inline int __down_write_common(struct rw_semaphore *sem, int state) 1312{ 1313 int ret = 0; 1314 1315 preempt_disable(); 1316 if (unlikely(!rwsem_write_trylock(sem))) { 1317 if (IS_ERR(rwsem_down_write_slowpath(sem, state))) 1318 ret = -EINTR; 1319 } 1320 preempt_enable(); 1321 return ret; 1322} 1323 1324static __always_inline void __down_write(struct rw_semaphore *sem) 1325{ 1326 __down_write_common(sem, TASK_UNINTERRUPTIBLE); 1327} 1328 1329static __always_inline int __down_write_killable(struct rw_semaphore *sem) 1330{ 1331 return __down_write_common(sem, TASK_KILLABLE); 1332} 1333 1334static inline int __down_write_trylock(struct rw_semaphore *sem) 1335{ 1336 int ret; 1337 1338 preempt_disable(); 1339 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); 1340 ret = rwsem_write_trylock(sem); 1341 preempt_enable(); 1342 1343 return ret; 1344} 1345 1346/* 1347 * unlock after reading 1348 */ 1349static inline void __up_read(struct rw_semaphore *sem) 1350{ 1351 long tmp; 1352 1353 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); 1354 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); 1355 1356 preempt_disable(); 1357 rwsem_clear_reader_owned(sem); 1358 tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); 1359 DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); 1360 if (unlikely((tmp & (RWSEM_LOCK_MASK|RWSEM_FLAG_WAITERS)) == 1361 RWSEM_FLAG_WAITERS)) { 1362 clear_nonspinnable(sem); 1363 rwsem_wake(sem); 1364 } 1365 preempt_enable(); 1366} 1367 1368/* 1369 * unlock after writing 1370 */ 1371static inline void __up_write(struct rw_semaphore *sem) 1372{ 1373 long tmp; 1374 1375 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); 1376 /* 1377 * sem->owner may differ from current if the ownership is transferred 1378 * to an anonymous writer by setting the RWSEM_NONSPINNABLE bits. 1379 */ 1380 DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) && 1381 !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem); 1382 1383 preempt_disable(); 1384 rwsem_clear_owner(sem); 1385 tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); 1386 if (unlikely(tmp & RWSEM_FLAG_WAITERS)) 1387 rwsem_wake(sem); 1388 preempt_enable(); 1389} 1390 1391/* 1392 * downgrade write lock to read lock 1393 */ 1394static inline void __downgrade_write(struct rw_semaphore *sem) 1395{ 1396 long tmp; 1397 1398 /* 1399 * When downgrading from exclusive to shared ownership, 1400 * anything inside the write-locked region cannot leak 1401 * into the read side. In contrast, anything in the 1402 * read-locked region is ok to be re-ordered into the 1403 * write side. As such, rely on RELEASE semantics. 1404 */ 1405 DEBUG_RWSEMS_WARN_ON(rwsem_owner(sem) != current, sem); 1406 preempt_disable(); 1407 tmp = atomic_long_fetch_add_release( 1408 -RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count); 1409 rwsem_set_reader_owned(sem); 1410 if (tmp & RWSEM_FLAG_WAITERS) 1411 rwsem_downgrade_wake(sem); 1412 preempt_enable(); 1413} 1414 1415#else /* !CONFIG_PREEMPT_RT */ 1416 1417#define RT_MUTEX_BUILD_MUTEX 1418#include "rtmutex.c" 1419 1420#define rwbase_set_and_save_current_state(state) \ 1421 set_current_state(state) 1422 1423#define rwbase_restore_current_state() \ 1424 __set_current_state(TASK_RUNNING) 1425 1426#define rwbase_rtmutex_lock_state(rtm, state) \ 1427 __rt_mutex_lock(rtm, state) 1428 1429#define rwbase_rtmutex_slowlock_locked(rtm, state, wq) \ 1430 __rt_mutex_slowlock_locked(rtm, NULL, state, wq) 1431 1432#define rwbase_rtmutex_unlock(rtm) \ 1433 __rt_mutex_unlock(rtm) 1434 1435#define rwbase_rtmutex_trylock(rtm) \ 1436 __rt_mutex_trylock(rtm) 1437 1438#define rwbase_signal_pending_state(state, current) \ 1439 signal_pending_state(state, current) 1440 1441#define rwbase_pre_schedule() \ 1442 rt_mutex_pre_schedule() 1443 1444#define rwbase_schedule() \ 1445 rt_mutex_schedule() 1446 1447#define rwbase_post_schedule() \ 1448 rt_mutex_post_schedule() 1449 1450#include "rwbase_rt.c" 1451 1452void __init_rwsem(struct rw_semaphore *sem, const char *name, 1453 struct lock_class_key *key) 1454{ 1455 init_rwbase_rt(&(sem)->rwbase); 1456 1457#ifdef CONFIG_DEBUG_LOCK_ALLOC 1458 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); 1459 lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); 1460#endif 1461} 1462EXPORT_SYMBOL(__init_rwsem); 1463 1464static inline void __down_read(struct rw_semaphore *sem) 1465{ 1466 rwbase_read_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); 1467} 1468 1469static inline int __down_read_interruptible(struct rw_semaphore *sem) 1470{ 1471 return rwbase_read_lock(&sem->rwbase, TASK_INTERRUPTIBLE); 1472} 1473 1474static inline int __down_read_killable(struct rw_semaphore *sem) 1475{ 1476 return rwbase_read_lock(&sem->rwbase, TASK_KILLABLE); 1477} 1478 1479static inline int __down_read_trylock(struct rw_semaphore *sem) 1480{ 1481 return rwbase_read_trylock(&sem->rwbase); 1482} 1483 1484static inline void __up_read(struct rw_semaphore *sem) 1485{ 1486 rwbase_read_unlock(&sem->rwbase, TASK_NORMAL); 1487} 1488 1489static inline void __sched __down_write(struct rw_semaphore *sem) 1490{ 1491 rwbase_write_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); 1492} 1493 1494static inline int __sched __down_write_killable(struct rw_semaphore *sem) 1495{ 1496 return rwbase_write_lock(&sem->rwbase, TASK_KILLABLE); 1497} 1498 1499static inline int __down_write_trylock(struct rw_semaphore *sem) 1500{ 1501 return rwbase_write_trylock(&sem->rwbase); 1502} 1503 1504static inline void __up_write(struct rw_semaphore *sem) 1505{ 1506 rwbase_write_unlock(&sem->rwbase); 1507} 1508 1509static inline void __downgrade_write(struct rw_semaphore *sem) 1510{ 1511 rwbase_write_downgrade(&sem->rwbase); 1512} 1513 1514/* Debug stubs for the common API */ 1515#define DEBUG_RWSEMS_WARN_ON(c, sem) 1516 1517static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, 1518 struct task_struct *owner) 1519{ 1520} 1521 1522static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem) 1523{ 1524 int count = atomic_read(&sem->rwbase.readers); 1525 1526 return count < 0 && count != READER_BIAS; 1527} 1528 1529#endif /* CONFIG_PREEMPT_RT */ 1530 1531/* 1532 * lock for reading 1533 */ 1534void __sched down_read(struct rw_semaphore *sem) 1535{ 1536 might_sleep(); 1537 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); 1538 1539 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); 1540} 1541EXPORT_SYMBOL(down_read); 1542 1543int __sched down_read_interruptible(struct rw_semaphore *sem) 1544{ 1545 might_sleep(); 1546 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); 1547 1548 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_interruptible)) { 1549 rwsem_release(&sem->dep_map, _RET_IP_); 1550 return -EINTR; 1551 } 1552 1553 return 0; 1554} 1555EXPORT_SYMBOL(down_read_interruptible); 1556 1557int __sched down_read_killable(struct rw_semaphore *sem) 1558{ 1559 might_sleep(); 1560 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); 1561 1562 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { 1563 rwsem_release(&sem->dep_map, _RET_IP_); 1564 return -EINTR; 1565 } 1566 1567 return 0; 1568} 1569EXPORT_SYMBOL(down_read_killable); 1570 1571/* 1572 * trylock for reading -- returns 1 if successful, 0 if contention 1573 */ 1574int down_read_trylock(struct rw_semaphore *sem) 1575{ 1576 int ret = __down_read_trylock(sem); 1577 1578 if (ret == 1) 1579 rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); 1580 return ret; 1581} 1582EXPORT_SYMBOL(down_read_trylock); 1583 1584/* 1585 * lock for writing 1586 */ 1587void __sched down_write(struct rw_semaphore *sem) 1588{ 1589 might_sleep(); 1590 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); 1591 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); 1592} 1593EXPORT_SYMBOL(down_write); 1594 1595/* 1596 * lock for writing 1597 */ 1598int __sched down_write_killable(struct rw_semaphore *sem) 1599{ 1600 might_sleep(); 1601 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); 1602 1603 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, 1604 __down_write_killable)) { 1605 rwsem_release(&sem->dep_map, _RET_IP_); 1606 return -EINTR; 1607 } 1608 1609 return 0; 1610} 1611EXPORT_SYMBOL(down_write_killable); 1612 1613/* 1614 * trylock for writing -- returns 1 if successful, 0 if contention 1615 */ 1616int down_write_trylock(struct rw_semaphore *sem) 1617{ 1618 int ret = __down_write_trylock(sem); 1619 1620 if (ret == 1) 1621 rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_); 1622 1623 return ret; 1624} 1625EXPORT_SYMBOL(down_write_trylock); 1626 1627/* 1628 * release a read lock 1629 */ 1630void up_read(struct rw_semaphore *sem) 1631{ 1632 rwsem_release(&sem->dep_map, _RET_IP_); 1633 __up_read(sem); 1634} 1635EXPORT_SYMBOL(up_read); 1636 1637/* 1638 * release a write lock 1639 */ 1640void up_write(struct rw_semaphore *sem) 1641{ 1642 rwsem_release(&sem->dep_map, _RET_IP_); 1643 __up_write(sem); 1644} 1645EXPORT_SYMBOL(up_write); 1646 1647/* 1648 * downgrade write lock to read lock 1649 */ 1650void downgrade_write(struct rw_semaphore *sem) 1651{ 1652 lock_downgrade(&sem->dep_map, _RET_IP_); 1653 __downgrade_write(sem); 1654} 1655EXPORT_SYMBOL(downgrade_write); 1656 1657#ifdef CONFIG_DEBUG_LOCK_ALLOC 1658 1659void down_read_nested(struct rw_semaphore *sem, int subclass) 1660{ 1661 might_sleep(); 1662 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); 1663 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); 1664} 1665EXPORT_SYMBOL(down_read_nested); 1666 1667int down_read_killable_nested(struct rw_semaphore *sem, int subclass) 1668{ 1669 might_sleep(); 1670 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); 1671 1672 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { 1673 rwsem_release(&sem->dep_map, _RET_IP_); 1674 return -EINTR; 1675 } 1676 1677 return 0; 1678} 1679EXPORT_SYMBOL(down_read_killable_nested); 1680 1681void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest) 1682{ 1683 might_sleep(); 1684 rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_); 1685 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); 1686} 1687EXPORT_SYMBOL(_down_write_nest_lock); 1688 1689void down_read_non_owner(struct rw_semaphore *sem) 1690{ 1691 might_sleep(); 1692 __down_read(sem); 1693 /* 1694 * The owner value for a reader-owned lock is mostly for debugging 1695 * purpose only and is not critical to the correct functioning of 1696 * rwsem. So it is perfectly fine to set it in a preempt-enabled 1697 * context here. 1698 */ 1699 __rwsem_set_reader_owned(sem, NULL); 1700} 1701EXPORT_SYMBOL(down_read_non_owner); 1702 1703void down_write_nested(struct rw_semaphore *sem, int subclass) 1704{ 1705 might_sleep(); 1706 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); 1707 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); 1708} 1709EXPORT_SYMBOL(down_write_nested); 1710 1711int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass) 1712{ 1713 might_sleep(); 1714 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); 1715 1716 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, 1717 __down_write_killable)) { 1718 rwsem_release(&sem->dep_map, _RET_IP_); 1719 return -EINTR; 1720 } 1721 1722 return 0; 1723} 1724EXPORT_SYMBOL(down_write_killable_nested); 1725 1726void up_read_non_owner(struct rw_semaphore *sem) 1727{ 1728 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); 1729 __up_read(sem); 1730} 1731EXPORT_SYMBOL(up_read_non_owner); 1732 1733#endif
~/
Read-only file preview.