| // SPDX-License-Identifier: GPL-2.0-only |
| #include <linux/kdebug.h> |
| #include <linux/kprobes.h> |
| #include <linux/export.h> |
| #include <linux/notifier.h> |
| #include <linux/rcupdate.h> |
| #include <linux/vmalloc.h> |
| #include <linux/reboot.h> |
| |
| /* |
| * Notifier list for kernel code which wants to be called |
| * at shutdown. This is used to stop any idling DMA operations |
| * and the like. |
| */ |
| BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); |
| |
| /* |
| * Notifier chain core routines. The exported routines below |
| * are layered on top of these, with appropriate locking added. |
| */ |
| |
| static int notifier_chain_register(struct notifier_block **nl, |
| struct notifier_block *n) |
| { |
| while ((*nl) != NULL) { |
| if (unlikely((*nl) == n)) { |
| WARN(1, "double register detected"); |
| return 0; |
| } |
| if (n->priority > (*nl)->priority) |
| break; |
| nl = &((*nl)->next); |
| } |
| n->next = *nl; |
| rcu_assign_pointer(*nl, n); |
| return 0; |
| } |
| |
| static int notifier_chain_cond_register(struct notifier_block **nl, |
| struct notifier_block *n) |
| { |
| while ((*nl) != NULL) { |
| if ((*nl) == n) |
| return 0; |
| if (n->priority > (*nl)->priority) |
| break; |
| nl = &((*nl)->next); |
| } |
| n->next = *nl; |
| rcu_assign_pointer(*nl, n); |
| return 0; |
| } |
| |
| static int notifier_chain_unregister(struct notifier_block **nl, |
| struct notifier_block *n) |
| { |
| while ((*nl) != NULL) { |
| if ((*nl) == n) { |
| rcu_assign_pointer(*nl, n->next); |
| return 0; |
| } |
| nl = &((*nl)->next); |
| } |
| return -ENOENT; |
| } |
| |
| /** |
| * notifier_call_chain - Informs the registered notifiers about an event. |
| * @nl: Pointer to head of the blocking notifier chain |
| * @val: Value passed unmodified to notifier function |
| * @v: Pointer passed unmodified to notifier function |
| * @nr_to_call: Number of notifier functions to be called. Don't care |
| * value of this parameter is -1. |
| * @nr_calls: Records the number of notifications sent. Don't care |
| * value of this field is NULL. |
| * @returns: notifier_call_chain returns the value returned by the |
| * last notifier function called. |
| */ |
| static int notifier_call_chain(struct notifier_block **nl, |
| unsigned long val, void *v, |
| int nr_to_call, int *nr_calls) |
| { |
| int ret = NOTIFY_DONE; |
| struct notifier_block *nb, *next_nb; |
| |
| nb = rcu_dereference_raw(*nl); |
| |
| while (nb && nr_to_call) { |
| next_nb = rcu_dereference_raw(nb->next); |
| |
| #ifdef CONFIG_DEBUG_NOTIFIERS |
| if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { |
| WARN(1, "Invalid notifier called!"); |
| nb = next_nb; |
| continue; |
| } |
| #endif |
| ret = nb->notifier_call(nb, val, v); |
| |
| if (nr_calls) |
| (*nr_calls)++; |
| |
| if (ret & NOTIFY_STOP_MASK) |
| break; |
| nb = next_nb; |
| nr_to_call--; |
| } |
| return ret; |
| } |
| NOKPROBE_SYMBOL(notifier_call_chain); |
| |
| /* |
| * Atomic notifier chain routines. Registration and unregistration |
| * use a spinlock, and call_chain is synchronized by RCU (no locks). |
| */ |
| |
| /** |
| * atomic_notifier_chain_register - Add notifier to an atomic notifier chain |
| * @nh: Pointer to head of the atomic notifier chain |
| * @n: New entry in notifier chain |
| * |
| * Adds a notifier to an atomic notifier chain. |
| * |
| * Currently always returns zero. |
| */ |
| int atomic_notifier_chain_register(struct atomic_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| unsigned long flags; |
| int ret; |
| |
| spin_lock_irqsave(&nh->lock, flags); |
| ret = notifier_chain_register(&nh->head, n); |
| spin_unlock_irqrestore(&nh->lock, flags); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); |
| |
| /** |
| * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain |
| * @nh: Pointer to head of the atomic notifier chain |
| * @n: Entry to remove from notifier chain |
| * |
| * Removes a notifier from an atomic notifier chain. |
| * |
| * Returns zero on success or %-ENOENT on failure. |
| */ |
| int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| unsigned long flags; |
| int ret; |
| |
| spin_lock_irqsave(&nh->lock, flags); |
| ret = notifier_chain_unregister(&nh->head, n); |
| spin_unlock_irqrestore(&nh->lock, flags); |
| synchronize_rcu(); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); |
| |
| /** |
| * __atomic_notifier_call_chain - Call functions in an atomic notifier chain |
| * @nh: Pointer to head of the atomic notifier chain |
| * @val: Value passed unmodified to notifier function |
| * @v: Pointer passed unmodified to notifier function |
| * @nr_to_call: See the comment for notifier_call_chain. |
| * @nr_calls: See the comment for notifier_call_chain. |
| * |
| * Calls each function in a notifier chain in turn. The functions |
| * run in an atomic context, so they must not block. |
| * This routine uses RCU to synchronize with changes to the chain. |
| * |
| * If the return value of the notifier can be and'ed |
| * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() |
| * will return immediately, with the return value of |
| * the notifier function which halted execution. |
| * Otherwise the return value is the return value |
| * of the last notifier function called. |
| */ |
| int __atomic_notifier_call_chain(struct atomic_notifier_head *nh, |
| unsigned long val, void *v, |
| int nr_to_call, int *nr_calls) |
| { |
| int ret; |
| |
| rcu_read_lock(); |
| ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); |
| rcu_read_unlock(); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain); |
| NOKPROBE_SYMBOL(__atomic_notifier_call_chain); |
| |
| int atomic_notifier_call_chain(struct atomic_notifier_head *nh, |
| unsigned long val, void *v) |
| { |
| return __atomic_notifier_call_chain(nh, val, v, -1, NULL); |
| } |
| EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); |
| NOKPROBE_SYMBOL(atomic_notifier_call_chain); |
| |
| /* |
| * Blocking notifier chain routines. All access to the chain is |
| * synchronized by an rwsem. |
| */ |
| |
| /** |
| * blocking_notifier_chain_register - Add notifier to a blocking notifier chain |
| * @nh: Pointer to head of the blocking notifier chain |
| * @n: New entry in notifier chain |
| * |
| * Adds a notifier to a blocking notifier chain. |
| * Must be called in process context. |
| * |
| * Currently always returns zero. |
| */ |
| int blocking_notifier_chain_register(struct blocking_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| int ret; |
| |
| /* |
| * This code gets used during boot-up, when task switching is |
| * not yet working and interrupts must remain disabled. At |
| * such times we must not call down_write(). |
| */ |
| if (unlikely(system_state == SYSTEM_BOOTING)) |
| return notifier_chain_register(&nh->head, n); |
| |
| down_write(&nh->rwsem); |
| ret = notifier_chain_register(&nh->head, n); |
| up_write(&nh->rwsem); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); |
| |
| /** |
| * blocking_notifier_chain_cond_register - Cond add notifier to a blocking notifier chain |
| * @nh: Pointer to head of the blocking notifier chain |
| * @n: New entry in notifier chain |
| * |
| * Adds a notifier to a blocking notifier chain, only if not already |
| * present in the chain. |
| * Must be called in process context. |
| * |
| * Currently always returns zero. |
| */ |
| int blocking_notifier_chain_cond_register(struct blocking_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| int ret; |
| |
| down_write(&nh->rwsem); |
| ret = notifier_chain_cond_register(&nh->head, n); |
| up_write(&nh->rwsem); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(blocking_notifier_chain_cond_register); |
| |
| /** |
| * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain |
| * @nh: Pointer to head of the blocking notifier chain |
| * @n: Entry to remove from notifier chain |
| * |
| * Removes a notifier from a blocking notifier chain. |
| * Must be called from process context. |
| * |
| * Returns zero on success or %-ENOENT on failure. |
| */ |
| int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| int ret; |
| |
| /* |
| * This code gets used during boot-up, when task switching is |
| * not yet working and interrupts must remain disabled. At |
| * such times we must not call down_write(). |
| */ |
| if (unlikely(system_state == SYSTEM_BOOTING)) |
| return notifier_chain_unregister(&nh->head, n); |
| |
| down_write(&nh->rwsem); |
| ret = notifier_chain_unregister(&nh->head, n); |
| up_write(&nh->rwsem); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); |
| |
| /** |
| * __blocking_notifier_call_chain - Call functions in a blocking notifier chain |
| * @nh: Pointer to head of the blocking notifier chain |
| * @val: Value passed unmodified to notifier function |
| * @v: Pointer passed unmodified to notifier function |
| * @nr_to_call: See comment for notifier_call_chain. |
| * @nr_calls: See comment for notifier_call_chain. |
| * |
| * Calls each function in a notifier chain in turn. The functions |
| * run in a process context, so they are allowed to block. |
| * |
| * If the return value of the notifier can be and'ed |
| * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() |
| * will return immediately, with the return value of |
| * the notifier function which halted execution. |
| * Otherwise the return value is the return value |
| * of the last notifier function called. |
| */ |
| int __blocking_notifier_call_chain(struct blocking_notifier_head *nh, |
| unsigned long val, void *v, |
| int nr_to_call, int *nr_calls) |
| { |
| int ret = NOTIFY_DONE; |
| |
| /* |
| * We check the head outside the lock, but if this access is |
| * racy then it does not matter what the result of the test |
| * is, we re-check the list after having taken the lock anyway: |
| */ |
| if (rcu_access_pointer(nh->head)) { |
| down_read(&nh->rwsem); |
| ret = notifier_call_chain(&nh->head, val, v, nr_to_call, |
| nr_calls); |
| up_read(&nh->rwsem); |
| } |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain); |
| |
| int blocking_notifier_call_chain(struct blocking_notifier_head *nh, |
| unsigned long val, void *v) |
| { |
| return __blocking_notifier_call_chain(nh, val, v, -1, NULL); |
| } |
| EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); |
| |
| /* |
| * Raw notifier chain routines. There is no protection; |
| * the caller must provide it. Use at your own risk! |
| */ |
| |
| /** |
| * raw_notifier_chain_register - Add notifier to a raw notifier chain |
| * @nh: Pointer to head of the raw notifier chain |
| * @n: New entry in notifier chain |
| * |
| * Adds a notifier to a raw notifier chain. |
| * All locking must be provided by the caller. |
| * |
| * Currently always returns zero. |
| */ |
| int raw_notifier_chain_register(struct raw_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| return notifier_chain_register(&nh->head, n); |
| } |
| EXPORT_SYMBOL_GPL(raw_notifier_chain_register); |
| |
| /** |
| * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain |
| * @nh: Pointer to head of the raw notifier chain |
| * @n: Entry to remove from notifier chain |
| * |
| * Removes a notifier from a raw notifier chain. |
| * All locking must be provided by the caller. |
| * |
| * Returns zero on success or %-ENOENT on failure. |
| */ |
| int raw_notifier_chain_unregister(struct raw_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| return notifier_chain_unregister(&nh->head, n); |
| } |
| EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); |
| |
| /** |
| * __raw_notifier_call_chain - Call functions in a raw notifier chain |
| * @nh: Pointer to head of the raw notifier chain |
| * @val: Value passed unmodified to notifier function |
| * @v: Pointer passed unmodified to notifier function |
| * @nr_to_call: See comment for notifier_call_chain. |
| * @nr_calls: See comment for notifier_call_chain |
| * |
| * Calls each function in a notifier chain in turn. The functions |
| * run in an undefined context. |
| * All locking must be provided by the caller. |
| * |
| * If the return value of the notifier can be and'ed |
| * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() |
| * will return immediately, with the return value of |
| * the notifier function which halted execution. |
| * Otherwise the return value is the return value |
| * of the last notifier function called. |
| */ |
| int __raw_notifier_call_chain(struct raw_notifier_head *nh, |
| unsigned long val, void *v, |
| int nr_to_call, int *nr_calls) |
| { |
| return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); |
| } |
| EXPORT_SYMBOL_GPL(__raw_notifier_call_chain); |
| |
| int raw_notifier_call_chain(struct raw_notifier_head *nh, |
| unsigned long val, void *v) |
| { |
| return __raw_notifier_call_chain(nh, val, v, -1, NULL); |
| } |
| EXPORT_SYMBOL_GPL(raw_notifier_call_chain); |
| |
| #ifdef CONFIG_SRCU |
| /* |
| * SRCU notifier chain routines. Registration and unregistration |
| * use a mutex, and call_chain is synchronized by SRCU (no locks). |
| */ |
| |
| /** |
| * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain |
| * @nh: Pointer to head of the SRCU notifier chain |
| * @n: New entry in notifier chain |
| * |
| * Adds a notifier to an SRCU notifier chain. |
| * Must be called in process context. |
| * |
| * Currently always returns zero. |
| */ |
| int srcu_notifier_chain_register(struct srcu_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| int ret; |
| |
| /* |
| * This code gets used during boot-up, when task switching is |
| * not yet working and interrupts must remain disabled. At |
| * such times we must not call mutex_lock(). |
| */ |
| if (unlikely(system_state == SYSTEM_BOOTING)) |
| return notifier_chain_register(&nh->head, n); |
| |
| mutex_lock(&nh->mutex); |
| ret = notifier_chain_register(&nh->head, n); |
| mutex_unlock(&nh->mutex); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); |
| |
| /** |
| * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain |
| * @nh: Pointer to head of the SRCU notifier chain |
| * @n: Entry to remove from notifier chain |
| * |
| * Removes a notifier from an SRCU notifier chain. |
| * Must be called from process context. |
| * |
| * Returns zero on success or %-ENOENT on failure. |
| */ |
| int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, |
| struct notifier_block *n) |
| { |
| int ret; |
| |
| /* |
| * This code gets used during boot-up, when task switching is |
| * not yet working and interrupts must remain disabled. At |
| * such times we must not call mutex_lock(). |
| */ |
| if (unlikely(system_state == SYSTEM_BOOTING)) |
| return notifier_chain_unregister(&nh->head, n); |
| |
| mutex_lock(&nh->mutex); |
| ret = notifier_chain_unregister(&nh->head, n); |
| mutex_unlock(&nh->mutex); |
| synchronize_srcu(&nh->srcu); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); |
| |
| /** |
| * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain |
| * @nh: Pointer to head of the SRCU notifier chain |
| * @val: Value passed unmodified to notifier function |
| * @v: Pointer passed unmodified to notifier function |
| * @nr_to_call: See comment for notifier_call_chain. |
| * @nr_calls: See comment for notifier_call_chain |
| * |
| * Calls each function in a notifier chain in turn. The functions |
| * run in a process context, so they are allowed to block. |
| * |
| * If the return value of the notifier can be and'ed |
| * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() |
| * will return immediately, with the return value of |
| * the notifier function which halted execution. |
| * Otherwise the return value is the return value |
| * of the last notifier function called. |
| */ |
| int __srcu_notifier_call_chain(struct srcu_notifier_head *nh, |
| unsigned long val, void *v, |
| int nr_to_call, int *nr_calls) |
| { |
| int ret; |
| int idx; |
| |
| idx = srcu_read_lock(&nh->srcu); |
| ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls); |
| srcu_read_unlock(&nh->srcu, idx); |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain); |
| |
| int srcu_notifier_call_chain(struct srcu_notifier_head *nh, |
| unsigned long val, void *v) |
| { |
| return __srcu_notifier_call_chain(nh, val, v, -1, NULL); |
| } |
| EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); |
| |
| /** |
| * srcu_init_notifier_head - Initialize an SRCU notifier head |
| * @nh: Pointer to head of the srcu notifier chain |
| * |
| * Unlike other sorts of notifier heads, SRCU notifier heads require |
| * dynamic initialization. Be sure to call this routine before |
| * calling any of the other SRCU notifier routines for this head. |
| * |
| * If an SRCU notifier head is deallocated, it must first be cleaned |
| * up by calling srcu_cleanup_notifier_head(). Otherwise the head's |
| * per-cpu data (used by the SRCU mechanism) will leak. |
| */ |
| void srcu_init_notifier_head(struct srcu_notifier_head *nh) |
| { |
| mutex_init(&nh->mutex); |
| if (init_srcu_struct(&nh->srcu) < 0) |
| BUG(); |
| nh->head = NULL; |
| } |
| EXPORT_SYMBOL_GPL(srcu_init_notifier_head); |
| |
| #endif /* CONFIG_SRCU */ |
| |
| static ATOMIC_NOTIFIER_HEAD(die_chain); |
| |
| int notrace notify_die(enum die_val val, const char *str, |
| struct pt_regs *regs, long err, int trap, int sig) |
| { |
| struct die_args args = { |
| .regs = regs, |
| .str = str, |
| .err = err, |
| .trapnr = trap, |
| .signr = sig, |
| |
| }; |
| RCU_LOCKDEP_WARN(!rcu_is_watching(), |
| "notify_die called but RCU thinks we're quiescent"); |
| return atomic_notifier_call_chain(&die_chain, val, &args); |
| } |
| NOKPROBE_SYMBOL(notify_die); |
| |
| int register_die_notifier(struct notifier_block *nb) |
| { |
| vmalloc_sync_mappings(); |
| return atomic_notifier_chain_register(&die_chain, nb); |
| } |
| EXPORT_SYMBOL_GPL(register_die_notifier); |
| |
| int unregister_die_notifier(struct notifier_block *nb) |
| { |
| return atomic_notifier_chain_unregister(&die_chain, nb); |
| } |
| EXPORT_SYMBOL_GPL(unregister_die_notifier); |