notifier.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2#include <linux/kdebug.h>
  3#include <linux/kprobes.h>
  4#include <linux/export.h>
  5#include <linux/notifier.h>
  6#include <linux/rcupdate.h>
  7#include <linux/vmalloc.h>
  8
  9#define CREATE_TRACE_POINTS
 10#include <trace/events/notifier.h>
 11
 12/*
 13 *	Notifier chain core routines.  The exported routines below
 14 *	are layered on top of these, with appropriate locking added.
 15 */
 16
 17static int notifier_chain_register(struct notifier_block **nl,
 18				   struct notifier_block *n,
 19				   bool unique_priority)
 20{
 21	while ((*nl) != NULL) {
 22		if (unlikely((*nl) == n)) {
 23			WARN(1, "notifier callback %ps already registered",
 24			     n->notifier_call);
 25			return -EEXIST;
 26		}
 27		if (n->priority > (*nl)->priority)
 28			break;
 29		if (n->priority == (*nl)->priority && unique_priority)
 30			return -EBUSY;
 31		nl = &((*nl)->next);
 32	}
 33	n->next = *nl;
 34	rcu_assign_pointer(*nl, n);
 35	trace_notifier_register((void *)n->notifier_call);
 36	return 0;
 37}
 38
 39static int notifier_chain_unregister(struct notifier_block **nl,
 40		struct notifier_block *n)
 41{
 42	while ((*nl) != NULL) {
 43		if ((*nl) == n) {
 44			rcu_assign_pointer(*nl, n->next);
 45			trace_notifier_unregister((void *)n->notifier_call);
 46			return 0;
 47		}
 48		nl = &((*nl)->next);
 49	}
 50	return -ENOENT;
 51}
 52
 53/**
 54 * notifier_call_chain - Informs the registered notifiers about an event.
 55 *	@nl:		Pointer to head of the blocking notifier chain
 56 *	@val:		Value passed unmodified to notifier function
 57 *	@v:		Pointer passed unmodified to notifier function
 58 *	@nr_to_call:	Number of notifier functions to be called. Don't care
 59 *			value of this parameter is -1.
 60 *	@nr_calls:	Records the number of notifications sent. Don't care
 61 *			value of this field is NULL.
 62 *	Return:		notifier_call_chain returns the value returned by the
 63 *			last notifier function called.
 64 */
 65static int notifier_call_chain(struct notifier_block **nl,
 66			       unsigned long val, void *v,
 67			       int nr_to_call, int *nr_calls)
 68{
 69	int ret = NOTIFY_DONE;
 70	struct notifier_block *nb, *next_nb;
 71
 72	nb = rcu_dereference_raw(*nl);
 73
 74	while (nb && nr_to_call) {
 75		next_nb = rcu_dereference_raw(nb->next);
 76
 77#ifdef CONFIG_DEBUG_NOTIFIERS
 78		if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
 79			WARN(1, "Invalid notifier called!");
 80			nb = next_nb;
 81			continue;
 82		}
 83#endif
 84		trace_notifier_run((void *)nb->notifier_call);
 85		ret = nb->notifier_call(nb, val, v);
 86
 87		if (nr_calls)
 88			(*nr_calls)++;
 89
 90		if (ret & NOTIFY_STOP_MASK)
 91			break;
 92		nb = next_nb;
 93		nr_to_call--;
 94	}
 95	return ret;
 96}
 97NOKPROBE_SYMBOL(notifier_call_chain);
 98
 99/**
100 * notifier_call_chain_robust - Inform the registered notifiers about an event
101 *                              and rollback on error.
102 * @nl:		Pointer to head of the blocking notifier chain
103 * @val_up:	Value passed unmodified to the notifier function
104 * @val_down:	Value passed unmodified to the notifier function when recovering
105 *              from an error on @val_up
106 * @v:		Pointer passed unmodified to the notifier function
107 *
108 * NOTE:	It is important the @nl chain doesn't change between the two
109 *		invocations of notifier_call_chain() such that we visit the
110 *		exact same notifier callbacks; this rules out any RCU usage.
111 *
112 * Return:	the return value of the @val_up call.
113 */
114static int notifier_call_chain_robust(struct notifier_block **nl,
115				     unsigned long val_up, unsigned long val_down,
116				     void *v)
117{
118	int ret, nr = 0;
119
120	ret = notifier_call_chain(nl, val_up, v, -1, &nr);
121	if (ret & NOTIFY_STOP_MASK)
122		notifier_call_chain(nl, val_down, v, nr-1, NULL);
123
124	return ret;
125}
126
127/*
128 *	Atomic notifier chain routines.  Registration and unregistration
129 *	use a spinlock, and call_chain is synchronized by RCU (no locks).
130 */
131
132/**
133 *	atomic_notifier_chain_register - Add notifier to an atomic notifier chain
134 *	@nh: Pointer to head of the atomic notifier chain
135 *	@n: New entry in notifier chain
136 *
137 *	Adds a notifier to an atomic notifier chain.
138 *
139 *	Returns 0 on success, %-EEXIST on error.
140 */
141int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
142		struct notifier_block *n)
143{
144	unsigned long flags;
145	int ret;
146
147	spin_lock_irqsave(&nh->lock, flags);
148	ret = notifier_chain_register(&nh->head, n, false);
149	spin_unlock_irqrestore(&nh->lock, flags);
150	return ret;
151}
152EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
153
154/**
155 *	atomic_notifier_chain_register_unique_prio - Add notifier to an atomic notifier chain
156 *	@nh: Pointer to head of the atomic notifier chain
157 *	@n: New entry in notifier chain
158 *
159 *	Adds a notifier to an atomic notifier chain if there is no other
160 *	notifier registered using the same priority.
161 *
162 *	Returns 0 on success, %-EEXIST or %-EBUSY on error.
163 */
164int atomic_notifier_chain_register_unique_prio(struct atomic_notifier_head *nh,
165					       struct notifier_block *n)
166{
167	unsigned long flags;
168	int ret;
169
170	spin_lock_irqsave(&nh->lock, flags);
171	ret = notifier_chain_register(&nh->head, n, true);
172	spin_unlock_irqrestore(&nh->lock, flags);
173	return ret;
174}
175EXPORT_SYMBOL_GPL(atomic_notifier_chain_register_unique_prio);
176
177/**
178 *	atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
179 *	@nh: Pointer to head of the atomic notifier chain
180 *	@n: Entry to remove from notifier chain
181 *
182 *	Removes a notifier from an atomic notifier chain.
183 *
184 *	Returns zero on success or %-ENOENT on failure.
185 */
186int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
187		struct notifier_block *n)
188{
189	unsigned long flags;
190	int ret;
191
192	spin_lock_irqsave(&nh->lock, flags);
193	ret = notifier_chain_unregister(&nh->head, n);
194	spin_unlock_irqrestore(&nh->lock, flags);
195	synchronize_rcu();
196	return ret;
197}
198EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
199
200/**
201 *	atomic_notifier_call_chain - Call functions in an atomic notifier chain
202 *	@nh: Pointer to head of the atomic notifier chain
203 *	@val: Value passed unmodified to notifier function
204 *	@v: Pointer passed unmodified to notifier function
205 *
206 *	Calls each function in a notifier chain in turn.  The functions
207 *	run in an atomic context, so they must not block.
208 *	This routine uses RCU to synchronize with changes to the chain.
209 *
210 *	If the return value of the notifier can be and'ed
211 *	with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
212 *	will return immediately, with the return value of
213 *	the notifier function which halted execution.
214 *	Otherwise the return value is the return value
215 *	of the last notifier function called.
216 */
217int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
218			       unsigned long val, void *v)
219{
220	int ret;
221
222	rcu_read_lock();
223	ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
224	rcu_read_unlock();
225
226	return ret;
227}
228EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
229NOKPROBE_SYMBOL(atomic_notifier_call_chain);
230
231/**
232 *	atomic_notifier_call_chain_is_empty - Check whether notifier chain is empty
233 *	@nh: Pointer to head of the atomic notifier chain
234 *
235 *	Checks whether notifier chain is empty.
236 *
237 *	Returns true is notifier chain is empty, false otherwise.
238 */
239bool atomic_notifier_call_chain_is_empty(struct atomic_notifier_head *nh)
240{
241	return !rcu_access_pointer(nh->head);
242}
243
244/*
245 *	Blocking notifier chain routines.  All access to the chain is
246 *	synchronized by an rwsem.
247 */
248
249static int __blocking_notifier_chain_register(struct blocking_notifier_head *nh,
250					      struct notifier_block *n,
251					      bool unique_priority)
252{
253	int ret;
254
255	/*
256	 * This code gets used during boot-up, when task switching is
257	 * not yet working and interrupts must remain disabled.  At
258	 * such times we must not call down_write().
259	 */
260	if (unlikely(system_state == SYSTEM_BOOTING))
261		return notifier_chain_register(&nh->head, n, unique_priority);
262
263	down_write(&nh->rwsem);
264	ret = notifier_chain_register(&nh->head, n, unique_priority);
265	up_write(&nh->rwsem);
266	return ret;
267}
268
269/**
270 *	blocking_notifier_chain_register - Add notifier to a blocking notifier chain
271 *	@nh: Pointer to head of the blocking notifier chain
272 *	@n: New entry in notifier chain
273 *
274 *	Adds a notifier to a blocking notifier chain.
275 *	Must be called in process context.
276 *
277 *	Returns 0 on success, %-EEXIST on error.
278 */
279int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
280		struct notifier_block *n)
281{
282	return __blocking_notifier_chain_register(nh, n, false);
283}
284EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
285
286/**
287 *	blocking_notifier_chain_register_unique_prio - Add notifier to a blocking notifier chain
288 *	@nh: Pointer to head of the blocking notifier chain
289 *	@n: New entry in notifier chain
290 *
291 *	Adds a notifier to an blocking notifier chain if there is no other
292 *	notifier registered using the same priority.
293 *
294 *	Returns 0 on success, %-EEXIST or %-EBUSY on error.
295 */
296int blocking_notifier_chain_register_unique_prio(struct blocking_notifier_head *nh,
297						 struct notifier_block *n)
298{
299	return __blocking_notifier_chain_register(nh, n, true);
300}
301EXPORT_SYMBOL_GPL(blocking_notifier_chain_register_unique_prio);
302
303/**
304 *	blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
305 *	@nh: Pointer to head of the blocking notifier chain
306 *	@n: Entry to remove from notifier chain
307 *
308 *	Removes a notifier from a blocking notifier chain.
309 *	Must be called from process context.
310 *
311 *	Returns zero on success or %-ENOENT on failure.
312 */
313int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
314		struct notifier_block *n)
315{
316	int ret;
317
318	/*
319	 * This code gets used during boot-up, when task switching is
320	 * not yet working and interrupts must remain disabled.  At
321	 * such times we must not call down_write().
322	 */
323	if (unlikely(system_state == SYSTEM_BOOTING))
324		return notifier_chain_unregister(&nh->head, n);
325
326	down_write(&nh->rwsem);
327	ret = notifier_chain_unregister(&nh->head, n);
328	up_write(&nh->rwsem);
329	return ret;
330}
331EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
332
333int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
334		unsigned long val_up, unsigned long val_down, void *v)
335{
336	int ret = NOTIFY_DONE;
337
338	/*
339	 * We check the head outside the lock, but if this access is
340	 * racy then it does not matter what the result of the test
341	 * is, we re-check the list after having taken the lock anyway:
342	 */
343	if (rcu_access_pointer(nh->head)) {
344		down_read(&nh->rwsem);
345		ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
346		up_read(&nh->rwsem);
347	}
348	return ret;
349}
350EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
351
352/**
353 *	blocking_notifier_call_chain - Call functions in a blocking notifier chain
354 *	@nh: Pointer to head of the blocking notifier chain
355 *	@val: Value passed unmodified to notifier function
356 *	@v: Pointer passed unmodified to notifier function
357 *
358 *	Calls each function in a notifier chain in turn.  The functions
359 *	run in a process context, so they are allowed to block.
360 *
361 *	If the return value of the notifier can be and'ed
362 *	with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
363 *	will return immediately, with the return value of
364 *	the notifier function which halted execution.
365 *	Otherwise the return value is the return value
366 *	of the last notifier function called.
367 */
368int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
369		unsigned long val, void *v)
370{
371	int ret = NOTIFY_DONE;
372
373	/*
374	 * We check the head outside the lock, but if this access is
375	 * racy then it does not matter what the result of the test
376	 * is, we re-check the list after having taken the lock anyway:
377	 */
378	if (rcu_access_pointer(nh->head)) {
379		down_read(&nh->rwsem);
380		ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
381		up_read(&nh->rwsem);
382	}
383	return ret;
384}
385EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
386
387/*
388 *	Raw notifier chain routines.  There is no protection;
389 *	the caller must provide it.  Use at your own risk!
390 */
391
392/**
393 *	raw_notifier_chain_register - Add notifier to a raw notifier chain
394 *	@nh: Pointer to head of the raw notifier chain
395 *	@n: New entry in notifier chain
396 *
397 *	Adds a notifier to a raw notifier chain.
398 *	All locking must be provided by the caller.
399 *
400 *	Returns 0 on success, %-EEXIST on error.
401 */
402int raw_notifier_chain_register(struct raw_notifier_head *nh,
403		struct notifier_block *n)
404{
405	return notifier_chain_register(&nh->head, n, false);
406}
407EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
408
409/**
410 *	raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
411 *	@nh: Pointer to head of the raw notifier chain
412 *	@n: Entry to remove from notifier chain
413 *
414 *	Removes a notifier from a raw notifier chain.
415 *	All locking must be provided by the caller.
416 *
417 *	Returns zero on success or %-ENOENT on failure.
418 */
419int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
420		struct notifier_block *n)
421{
422	return notifier_chain_unregister(&nh->head, n);
423}
424EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
425
426int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
427		unsigned long val_up, unsigned long val_down, void *v)
428{
429	return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
430}
431EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
432
433/**
434 *	raw_notifier_call_chain - Call functions in a raw notifier chain
435 *	@nh: Pointer to head of the raw notifier chain
436 *	@val: Value passed unmodified to notifier function
437 *	@v: Pointer passed unmodified to notifier function
438 *
439 *	Calls each function in a notifier chain in turn.  The functions
440 *	run in an undefined context.
441 *	All locking must be provided by the caller.
442 *
443 *	If the return value of the notifier can be and'ed
444 *	with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
445 *	will return immediately, with the return value of
446 *	the notifier function which halted execution.
447 *	Otherwise the return value is the return value
448 *	of the last notifier function called.
449 */
450int raw_notifier_call_chain(struct raw_notifier_head *nh,
451		unsigned long val, void *v)
452{
453	return notifier_call_chain(&nh->head, val, v, -1, NULL);
454}
455EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
456
457/*
458 *	SRCU notifier chain routines.    Registration and unregistration
459 *	use a mutex, and call_chain is synchronized by SRCU (no locks).
460 */
461
462/**
463 *	srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
464 *	@nh: Pointer to head of the SRCU notifier chain
465 *	@n: New entry in notifier chain
466 *
467 *	Adds a notifier to an SRCU notifier chain.
468 *	Must be called in process context.
469 *
470 *	Returns 0 on success, %-EEXIST on error.
471 */
472int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
473		struct notifier_block *n)
474{
475	int ret;
476
477	/*
478	 * This code gets used during boot-up, when task switching is
479	 * not yet working and interrupts must remain disabled.  At
480	 * such times we must not call mutex_lock().
481	 */
482	if (unlikely(system_state == SYSTEM_BOOTING))
483		return notifier_chain_register(&nh->head, n, false);
484
485	mutex_lock(&nh->mutex);
486	ret = notifier_chain_register(&nh->head, n, false);
487	mutex_unlock(&nh->mutex);
488	return ret;
489}
490EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
491
492/**
493 *	srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
494 *	@nh: Pointer to head of the SRCU notifier chain
495 *	@n: Entry to remove from notifier chain
496 *
497 *	Removes a notifier from an SRCU notifier chain.
498 *	Must be called from process context.
499 *
500 *	Returns zero on success or %-ENOENT on failure.
501 */
502int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
503		struct notifier_block *n)
504{
505	int ret;
506
507	/*
508	 * This code gets used during boot-up, when task switching is
509	 * not yet working and interrupts must remain disabled.  At
510	 * such times we must not call mutex_lock().
511	 */
512	if (unlikely(system_state == SYSTEM_BOOTING))
513		return notifier_chain_unregister(&nh->head, n);
514
515	mutex_lock(&nh->mutex);
516	ret = notifier_chain_unregister(&nh->head, n);
517	mutex_unlock(&nh->mutex);
518	synchronize_srcu(&nh->srcu);
519	return ret;
520}
521EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
522
523/**
524 *	srcu_notifier_call_chain - Call functions in an SRCU notifier chain
525 *	@nh: Pointer to head of the SRCU notifier chain
526 *	@val: Value passed unmodified to notifier function
527 *	@v: Pointer passed unmodified to notifier function
528 *
529 *	Calls each function in a notifier chain in turn.  The functions
530 *	run in a process context, so they are allowed to block.
531 *
532 *	If the return value of the notifier can be and'ed
533 *	with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
534 *	will return immediately, with the return value of
535 *	the notifier function which halted execution.
536 *	Otherwise the return value is the return value
537 *	of the last notifier function called.
538 */
539int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
540		unsigned long val, void *v)
541{
542	int ret;
543	int idx;
544
545	idx = srcu_read_lock(&nh->srcu);
546	ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
547	srcu_read_unlock(&nh->srcu, idx);
548	return ret;
549}
550EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
551
552/**
553 *	srcu_init_notifier_head - Initialize an SRCU notifier head
554 *	@nh: Pointer to head of the srcu notifier chain
555 *
556 *	Unlike other sorts of notifier heads, SRCU notifier heads require
557 *	dynamic initialization.  Be sure to call this routine before
558 *	calling any of the other SRCU notifier routines for this head.
559 *
560 *	If an SRCU notifier head is deallocated, it must first be cleaned
561 *	up by calling srcu_cleanup_notifier_head().  Otherwise the head's
562 *	per-cpu data (used by the SRCU mechanism) will leak.
563 */
564void srcu_init_notifier_head(struct srcu_notifier_head *nh)
565{
566	mutex_init(&nh->mutex);
567	if (init_srcu_struct(&nh->srcu) < 0)
568		BUG();
569	nh->head = NULL;
570}
571EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
572
573static ATOMIC_NOTIFIER_HEAD(die_chain);
574
575int notrace notify_die(enum die_val val, const char *str,
576	       struct pt_regs *regs, long err, int trap, int sig)
577{
578	struct die_args args = {
579		.regs	= regs,
580		.str	= str,
581		.err	= err,
582		.trapnr	= trap,
583		.signr	= sig,
584
585	};
586	RCU_LOCKDEP_WARN(!rcu_is_watching(),
587			   "notify_die called but RCU thinks we're quiescent");
588	return atomic_notifier_call_chain(&die_chain, val, &args);
589}
590NOKPROBE_SYMBOL(notify_die);
591
592int register_die_notifier(struct notifier_block *nb)
593{
594	return atomic_notifier_chain_register(&die_chain, nb);
595}
596EXPORT_SYMBOL_GPL(register_die_notifier);
597
598int unregister_die_notifier(struct notifier_block *nb)
599{
600	return atomic_notifier_chain_unregister(&die_chain, nb);
601}
602EXPORT_SYMBOL_GPL(unregister_die_notifier);