common.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3#include <linux/irq-entry-common.h>
  4#include <linux/resume_user_mode.h>
  5#include <linux/highmem.h>
  6#include <linux/jump_label.h>
  7#include <linux/kmsan.h>
  8#include <linux/livepatch.h>
  9#include <linux/tick.h>
 10
 11/* Workaround to allow gradual conversion of architecture code */
 12void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }
 13
 14#ifdef CONFIG_HAVE_GENERIC_TIF_BITS
 15#define EXIT_TO_USER_MODE_WORK_LOOP	(EXIT_TO_USER_MODE_WORK & ~_TIF_RSEQ)
 16#else
 17#define EXIT_TO_USER_MODE_WORK_LOOP	(EXIT_TO_USER_MODE_WORK)
 18#endif
 19
 20static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs,
 21							      unsigned long ti_work)
 22{
 23	/*
 24	 * Before returning to user space ensure that all pending work
 25	 * items have been completed.
 26	 */
 27	while (ti_work & EXIT_TO_USER_MODE_WORK_LOOP) {
 28
 29		local_irq_enable_exit_to_user(ti_work);
 30
 31		if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY))
 32			schedule();
 33
 34		if (ti_work & _TIF_UPROBE)
 35			uprobe_notify_resume(regs);
 36
 37		if (ti_work & _TIF_PATCH_PENDING)
 38			klp_update_patch_state(current);
 39
 40		if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
 41			arch_do_signal_or_restart(regs);
 42
 43		if (ti_work & _TIF_NOTIFY_RESUME)
 44			resume_user_mode_work(regs);
 45
 46		/* Architecture specific TIF work */
 47		arch_exit_to_user_mode_work(regs, ti_work);
 48
 49		/*
 50		 * Disable interrupts and reevaluate the work flags as they
 51		 * might have changed while interrupts and preemption was
 52		 * enabled above.
 53		 */
 54		local_irq_disable_exit_to_user();
 55
 56		/* Check if any of the above work has queued a deferred wakeup */
 57		tick_nohz_user_enter_prepare();
 58
 59		ti_work = read_thread_flags();
 60	}
 61
 62	/* Return the latest work state for arch_exit_to_user_mode() */
 63	return ti_work;
 64}
 65
 66/**
 67 * exit_to_user_mode_loop - do any pending work before leaving to user space
 68 * @regs:	Pointer to pt_regs on entry stack
 69 * @ti_work:	TIF work flags as read by the caller
 70 */
 71__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
 72						     unsigned long ti_work)
 73{
 74	for (;;) {
 75		ti_work = __exit_to_user_mode_loop(regs, ti_work);
 76
 77		if (likely(!rseq_exit_to_user_mode_restart(regs, ti_work)))
 78			return ti_work;
 79		ti_work = read_thread_flags();
 80	}
 81}
 82
 83noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
 84{
 85	irqentry_state_t ret = {
 86		.exit_rcu = false,
 87	};
 88
 89	if (user_mode(regs)) {
 90		irqentry_enter_from_user_mode(regs);
 91		return ret;
 92	}
 93
 94	/*
 95	 * If this entry hit the idle task invoke ct_irq_enter() whether
 96	 * RCU is watching or not.
 97	 *
 98	 * Interrupts can nest when the first interrupt invokes softirq
 99	 * processing on return which enables interrupts.
100	 *
101	 * Scheduler ticks in the idle task can mark quiescent state and
102	 * terminate a grace period, if and only if the timer interrupt is
103	 * not nested into another interrupt.
104	 *
105	 * Checking for rcu_is_watching() here would prevent the nesting
106	 * interrupt to invoke ct_irq_enter(). If that nested interrupt is
107	 * the tick then rcu_flavor_sched_clock_irq() would wrongfully
108	 * assume that it is the first interrupt and eventually claim
109	 * quiescent state and end grace periods prematurely.
110	 *
111	 * Unconditionally invoke ct_irq_enter() so RCU state stays
112	 * consistent.
113	 *
114	 * TINY_RCU does not support EQS, so let the compiler eliminate
115	 * this part when enabled.
116	 */
117	if (!IS_ENABLED(CONFIG_TINY_RCU) &&
118	    (is_idle_task(current) || arch_in_rcu_eqs())) {
119		/*
120		 * If RCU is not watching then the same careful
121		 * sequence vs. lockdep and tracing is required
122		 * as in irqentry_enter_from_user_mode().
123		 */
124		lockdep_hardirqs_off(CALLER_ADDR0);
125		ct_irq_enter();
126		instrumentation_begin();
127		kmsan_unpoison_entry_regs(regs);
128		trace_hardirqs_off_finish();
129		instrumentation_end();
130
131		ret.exit_rcu = true;
132		return ret;
133	}
134
135	/*
136	 * If RCU is watching then RCU only wants to check whether it needs
137	 * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick()
138	 * already contains a warning when RCU is not watching, so no point
139	 * in having another one here.
140	 */
141	lockdep_hardirqs_off(CALLER_ADDR0);
142	instrumentation_begin();
143	kmsan_unpoison_entry_regs(regs);
144	rcu_irq_enter_check_tick();
145	trace_hardirqs_off_finish();
146	instrumentation_end();
147
148	return ret;
149}
150
151/**
152 * arch_irqentry_exit_need_resched - Architecture specific need resched function
153 *
154 * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed.
155 * Defaults return true.
156 *
157 * The main purpose is to permit arch to avoid preemption of a task from an IRQ.
158 */
159static inline bool arch_irqentry_exit_need_resched(void);
160
161#ifndef arch_irqentry_exit_need_resched
162static inline bool arch_irqentry_exit_need_resched(void) { return true; }
163#endif
164
165void raw_irqentry_exit_cond_resched(void)
166{
167	if (!preempt_count()) {
168		/* Sanity check RCU and thread stack */
169		rcu_irq_exit_check_preempt();
170		if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
171			WARN_ON_ONCE(!on_thread_stack());
172		if (need_resched() && arch_irqentry_exit_need_resched())
173			preempt_schedule_irq();
174	}
175}
176#ifdef CONFIG_PREEMPT_DYNAMIC
177#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
178DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
179#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
180DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
181void dynamic_irqentry_exit_cond_resched(void)
182{
183	if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
184		return;
185	raw_irqentry_exit_cond_resched();
186}
187#endif
188#endif
189
190noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
191{
192	lockdep_assert_irqs_disabled();
193
194	/* Check whether this returns to user mode */
195	if (user_mode(regs)) {
196		irqentry_exit_to_user_mode(regs);
197	} else if (!regs_irqs_disabled(regs)) {
198		/*
199		 * If RCU was not watching on entry this needs to be done
200		 * carefully and needs the same ordering of lockdep/tracing
201		 * and RCU as the return to user mode path.
202		 */
203		if (state.exit_rcu) {
204			instrumentation_begin();
205			/* Tell the tracer that IRET will enable interrupts */
206			trace_hardirqs_on_prepare();
207			lockdep_hardirqs_on_prepare();
208			instrumentation_end();
209			ct_irq_exit();
210			lockdep_hardirqs_on(CALLER_ADDR0);
211			return;
212		}
213
214		instrumentation_begin();
215		if (IS_ENABLED(CONFIG_PREEMPTION))
216			irqentry_exit_cond_resched();
217
218		/* Covers both tracing and lockdep */
219		trace_hardirqs_on();
220		instrumentation_end();
221	} else {
222		/*
223		 * IRQ flags state is correct already. Just tell RCU if it
224		 * was not watching on entry.
225		 */
226		if (state.exit_rcu)
227			ct_irq_exit();
228	}
229}
230
231irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
232{
233	irqentry_state_t irq_state;
234
235	irq_state.lockdep = lockdep_hardirqs_enabled();
236
237	__nmi_enter();
238	lockdep_hardirqs_off(CALLER_ADDR0);
239	lockdep_hardirq_enter();
240	ct_nmi_enter();
241
242	instrumentation_begin();
243	kmsan_unpoison_entry_regs(regs);
244	trace_hardirqs_off_finish();
245	ftrace_nmi_enter();
246	instrumentation_end();
247
248	return irq_state;
249}
250
251void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
252{
253	instrumentation_begin();
254	ftrace_nmi_exit();
255	if (irq_state.lockdep) {
256		trace_hardirqs_on_prepare();
257		lockdep_hardirqs_on_prepare();
258	}
259	instrumentation_end();
260
261	ct_nmi_exit();
262	lockdep_hardirq_exit();
263	if (irq_state.lockdep)
264		lockdep_hardirqs_on(CALLER_ADDR0);
265	__nmi_exit();
266}