watchdog.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Watchdog support on powerpc systems.
  4 *
  5 * Copyright 2017, IBM Corporation.
  6 *
  7 * This uses code from arch/sparc/kernel/nmi.c and kernel/watchdog.c
  8 */
  9
 10#define pr_fmt(fmt) "watchdog: " fmt
 11
 12#include <linux/kernel.h>
 13#include <linux/param.h>
 14#include <linux/init.h>
 15#include <linux/percpu.h>
 16#include <linux/cpu.h>
 17#include <linux/nmi.h>
 18#include <linux/module.h>
 19#include <linux/export.h>
 20#include <linux/kprobes.h>
 21#include <linux/hardirq.h>
 22#include <linux/reboot.h>
 23#include <linux/slab.h>
 24#include <linux/kdebug.h>
 25#include <linux/sched/debug.h>
 26#include <linux/delay.h>
 27#include <linux/processor.h>
 28#include <linux/smp.h>
 29#include <linux/sys_info.h>
 30
 31#include <asm/interrupt.h>
 32#include <asm/paca.h>
 33#include <asm/nmi.h>
 34
 35/*
 36 * The powerpc watchdog ensures that each CPU is able to service timers.
 37 * The watchdog sets up a simple timer on each CPU to run once per timer
 38 * period, and updates a per-cpu timestamp and a "pending" cpumask. This is
 39 * the heartbeat.
 40 *
 41 * Then there are two systems to check that the heartbeat is still running.
 42 * The local soft-NMI, and the SMP checker.
 43 *
 44 * The soft-NMI checker can detect lockups on the local CPU. When interrupts
 45 * are disabled with local_irq_disable(), platforms that use soft-masking
 46 * can leave hardware interrupts enabled and handle them with a masked
 47 * interrupt handler. The masked handler can send the timer interrupt to the
 48 * watchdog's soft_nmi_interrupt(), which appears to Linux as an NMI
 49 * interrupt, and can be used to detect CPUs stuck with IRQs disabled.
 50 *
 51 * The soft-NMI checker will compare the heartbeat timestamp for this CPU
 52 * with the current time, and take action if the difference exceeds the
 53 * watchdog threshold.
 54 *
 55 * The limitation of the soft-NMI watchdog is that it does not work when
 56 * interrupts are hard disabled or otherwise not being serviced. This is
 57 * solved by also having a SMP watchdog where all CPUs check all other
 58 * CPUs heartbeat.
 59 *
 60 * The SMP checker can detect lockups on other CPUs. A global "pending"
 61 * cpumask is kept, containing all CPUs which enable the watchdog. Each
 62 * CPU clears their pending bit in their heartbeat timer. When the bitmask
 63 * becomes empty, the last CPU to clear its pending bit updates a global
 64 * timestamp and refills the pending bitmask.
 65 *
 66 * In the heartbeat timer, if any CPU notices that the global timestamp has
 67 * not been updated for a period exceeding the watchdog threshold, then it
 68 * means the CPU(s) with their bit still set in the pending mask have had
 69 * their heartbeat stop, and action is taken.
 70 *
 71 * Some platforms implement true NMI IPIs, which can be used by the SMP
 72 * watchdog to detect an unresponsive CPU and pull it out of its stuck
 73 * state with the NMI IPI, to get crash/debug data from it. This way the
 74 * SMP watchdog can detect hardware interrupts off lockups.
 75 */
 76
 77static cpumask_t wd_cpus_enabled __read_mostly;
 78
 79static u64 wd_panic_timeout_tb __read_mostly; /* timebase ticks until panic */
 80static u64 wd_smp_panic_timeout_tb __read_mostly; /* panic other CPUs */
 81
 82static u64 wd_timer_period_ms __read_mostly;  /* interval between heartbeat */
 83
 84static DEFINE_PER_CPU(struct hrtimer, wd_hrtimer);
 85static DEFINE_PER_CPU(u64, wd_timer_tb);
 86
 87/* SMP checker bits */
 88static unsigned long __wd_smp_lock;
 89static unsigned long __wd_reporting;
 90static unsigned long __wd_nmi_output;
 91static cpumask_t wd_smp_cpus_pending;
 92static cpumask_t wd_smp_cpus_stuck;
 93static u64 wd_smp_last_reset_tb;
 94
 95#ifdef CONFIG_PPC_PSERIES
 96static u64 wd_timeout_pct;
 97#endif
 98
 99/*
100 * Try to take the exclusive watchdog action / NMI IPI / printing lock.
101 * wd_smp_lock must be held. If this fails, we should return and wait
102 * for the watchdog to kick in again (or another CPU to trigger it).
103 *
104 * Importantly, if hardlockup_panic is set, wd_try_report failure should
105 * not delay the panic, because whichever other CPU is reporting will
106 * call panic.
107 */
108static bool wd_try_report(void)
109{
110	if (__wd_reporting)
111		return false;
112	__wd_reporting = 1;
113	return true;
114}
115
116/* End printing after successful wd_try_report. wd_smp_lock not required. */
117static void wd_end_reporting(void)
118{
119	smp_mb(); /* End printing "critical section" */
120	WARN_ON_ONCE(__wd_reporting == 0);
121	WRITE_ONCE(__wd_reporting, 0);
122}
123
124static inline void wd_smp_lock(unsigned long *flags)
125{
126	/*
127	 * Avoid locking layers if possible.
128	 * This may be called from low level interrupt handlers at some
129	 * point in future.
130	 */
131	raw_local_irq_save(*flags);
132	hard_irq_disable(); /* Make it soft-NMI safe */
133	while (unlikely(test_and_set_bit_lock(0, &__wd_smp_lock))) {
134		raw_local_irq_restore(*flags);
135		spin_until_cond(!test_bit(0, &__wd_smp_lock));
136		raw_local_irq_save(*flags);
137		hard_irq_disable();
138	}
139}
140
141static inline void wd_smp_unlock(unsigned long *flags)
142{
143	clear_bit_unlock(0, &__wd_smp_lock);
144	raw_local_irq_restore(*flags);
145}
146
147static void wd_lockup_ipi(struct pt_regs *regs)
148{
149	int cpu = raw_smp_processor_id();
150	u64 tb = get_tb();
151
152	pr_emerg("CPU %d Hard LOCKUP\n", cpu);
153	pr_emerg("CPU %d TB:%lld, last heartbeat TB:%lld (%lldms ago)\n",
154		 cpu, tb, per_cpu(wd_timer_tb, cpu),
155		 tb_to_ns(tb - per_cpu(wd_timer_tb, cpu)) / 1000000);
156	print_modules();
157	print_irqtrace_events(current);
158	if (regs)
159		show_regs(regs);
160	else
161		dump_stack();
162
163	/*
164	 * __wd_nmi_output must be set after we printk from NMI context.
165	 *
166	 * printk from NMI context defers printing to the console to irq_work.
167	 * If that NMI was taken in some code that is hard-locked, then irqs
168	 * are disabled so irq_work will never fire. That can result in the
169	 * hard lockup messages being delayed (indefinitely, until something
170	 * else kicks the console drivers).
171	 *
172	 * Setting __wd_nmi_output will cause another CPU to notice and kick
173	 * the console drivers for us.
174	 *
175	 * xchg is not needed here (it could be a smp_mb and store), but xchg
176	 * gives the memory ordering and atomicity required.
177	 */
178	xchg(&__wd_nmi_output, 1);
179
180	/* Do not panic from here because that can recurse into NMI IPI layer */
181}
182
183static bool set_cpu_stuck(int cpu)
184{
185	cpumask_set_cpu(cpu, &wd_smp_cpus_stuck);
186	cpumask_clear_cpu(cpu, &wd_smp_cpus_pending);
187	/*
188	 * See wd_smp_clear_cpu_pending()
189	 */
190	smp_mb();
191	if (cpumask_empty(&wd_smp_cpus_pending)) {
192		wd_smp_last_reset_tb = get_tb();
193		cpumask_andnot(&wd_smp_cpus_pending,
194				&wd_cpus_enabled,
195				&wd_smp_cpus_stuck);
196		return true;
197	}
198	return false;
199}
200
201static void watchdog_smp_panic(int cpu)
202{
203	static cpumask_t wd_smp_cpus_ipi; // protected by reporting
204	unsigned long flags;
205	u64 tb, last_reset;
206	int c;
207
208	wd_smp_lock(&flags);
209	/* Double check some things under lock */
210	tb = get_tb();
211	last_reset = wd_smp_last_reset_tb;
212	if ((s64)(tb - last_reset) < (s64)wd_smp_panic_timeout_tb)
213		goto out;
214	if (cpumask_test_cpu(cpu, &wd_smp_cpus_pending))
215		goto out;
216	if (!wd_try_report())
217		goto out;
218	for_each_online_cpu(c) {
219		if (!cpumask_test_cpu(c, &wd_smp_cpus_pending))
220			continue;
221		if (c == cpu)
222			continue; // should not happen
223
224		__cpumask_set_cpu(c, &wd_smp_cpus_ipi);
225		if (set_cpu_stuck(c))
226			break;
227	}
228	if (cpumask_empty(&wd_smp_cpus_ipi)) {
229		wd_end_reporting();
230		goto out;
231	}
232	wd_smp_unlock(&flags);
233
234	pr_emerg("CPU %d detected hard LOCKUP on other CPUs %*pbl\n",
235		 cpu, cpumask_pr_args(&wd_smp_cpus_ipi));
236	pr_emerg("CPU %d TB:%lld, last SMP heartbeat TB:%lld (%lldms ago)\n",
237		 cpu, tb, last_reset, tb_to_ns(tb - last_reset) / 1000000);
238
239	if (sysctl_hardlockup_all_cpu_backtrace ||
240	    (hardlockup_si_mask & SYS_INFO_ALL_BT)) {
241		trigger_allbutcpu_cpu_backtrace(cpu);
242		cpumask_clear(&wd_smp_cpus_ipi);
243	} else {
244		/*
245		 * Try to trigger the stuck CPUs, unless we are going to
246		 * get a backtrace on all of them anyway.
247		 */
248		for_each_cpu(c, &wd_smp_cpus_ipi) {
249			smp_send_nmi_ipi(c, wd_lockup_ipi, 1000000);
250			__cpumask_clear_cpu(c, &wd_smp_cpus_ipi);
251		}
252	}
253
254	sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
255	if (hardlockup_panic)
256		nmi_panic(NULL, "Hard LOCKUP");
257
258	wd_end_reporting();
259
260	return;
261
262out:
263	wd_smp_unlock(&flags);
264}
265
266static void wd_smp_clear_cpu_pending(int cpu)
267{
268	if (!cpumask_test_cpu(cpu, &wd_smp_cpus_pending)) {
269		if (unlikely(cpumask_test_cpu(cpu, &wd_smp_cpus_stuck))) {
270			struct pt_regs *regs = get_irq_regs();
271			unsigned long flags;
272
273			pr_emerg("CPU %d became unstuck TB:%lld\n",
274				 cpu, get_tb());
275			print_irqtrace_events(current);
276			if (regs)
277				show_regs(regs);
278			else
279				dump_stack();
280
281			wd_smp_lock(&flags);
282			cpumask_clear_cpu(cpu, &wd_smp_cpus_stuck);
283			wd_smp_unlock(&flags);
284		} else {
285			/*
286			 * The last CPU to clear pending should have reset the
287			 * watchdog so we generally should not find it empty
288			 * here if our CPU was clear. However it could happen
289			 * due to a rare race with another CPU taking the
290			 * last CPU out of the mask concurrently.
291			 *
292			 * We can't add a warning for it. But just in case
293			 * there is a problem with the watchdog that is causing
294			 * the mask to not be reset, try to kick it along here.
295			 */
296			if (unlikely(cpumask_empty(&wd_smp_cpus_pending)))
297				goto none_pending;
298		}
299		return;
300	}
301
302	/*
303	 * All other updates to wd_smp_cpus_pending are performed under
304	 * wd_smp_lock. All of them are atomic except the case where the
305	 * mask becomes empty and is reset. This will not happen here because
306	 * cpu was tested to be in the bitmap (above), and a CPU only clears
307	 * its own bit. _Except_ in the case where another CPU has detected a
308	 * hard lockup on our CPU and takes us out of the pending mask. So in
309	 * normal operation there will be no race here, no problem.
310	 *
311	 * In the lockup case, this atomic clear-bit vs a store that refills
312	 * other bits in the accessed word wll not be a problem. The bit clear
313	 * is atomic so it will not cause the store to get lost, and the store
314	 * will never set this bit so it will not overwrite the bit clear. The
315	 * only way for a stuck CPU to return to the pending bitmap is to
316	 * become unstuck itself.
317	 */
318	cpumask_clear_cpu(cpu, &wd_smp_cpus_pending);
319
320	/*
321	 * Order the store to clear pending with the load(s) to check all
322	 * words in the pending mask to check they are all empty. This orders
323	 * with the same barrier on another CPU. This prevents two CPUs
324	 * clearing the last 2 pending bits, but neither seeing the other's
325	 * store when checking if the mask is empty, and missing an empty
326	 * mask, which ends with a false positive.
327	 */
328	smp_mb();
329	if (cpumask_empty(&wd_smp_cpus_pending)) {
330		unsigned long flags;
331
332none_pending:
333		/*
334		 * Double check under lock because more than one CPU could see
335		 * a clear mask with the lockless check after clearing their
336		 * pending bits.
337		 */
338		wd_smp_lock(&flags);
339		if (cpumask_empty(&wd_smp_cpus_pending)) {
340			wd_smp_last_reset_tb = get_tb();
341			cpumask_andnot(&wd_smp_cpus_pending,
342					&wd_cpus_enabled,
343					&wd_smp_cpus_stuck);
344		}
345		wd_smp_unlock(&flags);
346	}
347}
348
349static void watchdog_timer_interrupt(int cpu)
350{
351	u64 tb = get_tb();
352
353	per_cpu(wd_timer_tb, cpu) = tb;
354
355	wd_smp_clear_cpu_pending(cpu);
356
357	if ((s64)(tb - wd_smp_last_reset_tb) >= (s64)wd_smp_panic_timeout_tb)
358		watchdog_smp_panic(cpu);
359
360	if (__wd_nmi_output && xchg(&__wd_nmi_output, 0)) {
361		/*
362		 * Something has called printk from NMI context. It might be
363		 * stuck, so this triggers a flush that will get that
364		 * printk output to the console.
365		 *
366		 * See wd_lockup_ipi.
367		 */
368		printk_trigger_flush();
369	}
370}
371
372DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)
373{
374	unsigned long flags;
375	int cpu = raw_smp_processor_id();
376	u64 tb;
377
378	/* should only arrive from kernel, with irqs disabled */
379	WARN_ON_ONCE(!arch_irq_disabled_regs(regs));
380
381	if (!cpumask_test_cpu(cpu, &wd_cpus_enabled))
382		return 0;
383
384	__this_cpu_inc(irq_stat.soft_nmi_irqs);
385
386	tb = get_tb();
387	if (tb - per_cpu(wd_timer_tb, cpu) >= wd_panic_timeout_tb) {
388		/*
389		 * Taking wd_smp_lock here means it is a soft-NMI lock, which
390		 * means we can't take any regular or irqsafe spin locks while
391		 * holding this lock. This is why timers can't printk while
392		 * holding the lock.
393		 */
394		wd_smp_lock(&flags);
395		if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) {
396			wd_smp_unlock(&flags);
397			return 0;
398		}
399		if (!wd_try_report()) {
400			wd_smp_unlock(&flags);
401			/* Couldn't report, try again in 100ms */
402			mtspr(SPRN_DEC, 100 * tb_ticks_per_usec * 1000);
403			return 0;
404		}
405
406		set_cpu_stuck(cpu);
407
408		wd_smp_unlock(&flags);
409
410		pr_emerg("CPU %d self-detected hard LOCKUP @ %pS\n",
411			 cpu, (void *)regs->nip);
412		pr_emerg("CPU %d TB:%lld, last heartbeat TB:%lld (%lldms ago)\n",
413			 cpu, tb, per_cpu(wd_timer_tb, cpu),
414			 tb_to_ns(tb - per_cpu(wd_timer_tb, cpu)) / 1000000);
415		print_modules();
416		print_irqtrace_events(current);
417		show_regs(regs);
418
419		xchg(&__wd_nmi_output, 1); // see wd_lockup_ipi
420
421		if (sysctl_hardlockup_all_cpu_backtrace ||
422		    (hardlockup_si_mask & SYS_INFO_ALL_BT))
423			trigger_allbutcpu_cpu_backtrace(cpu);
424
425		sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT);
426		if (hardlockup_panic)
427			nmi_panic(regs, "Hard LOCKUP");
428
429		wd_end_reporting();
430	}
431	/*
432	 * We are okay to change DEC in soft_nmi_interrupt because the masked
433	 * handler has marked a DEC as pending, so the timer interrupt will be
434	 * replayed as soon as local irqs are enabled again.
435	 */
436	if (wd_panic_timeout_tb < 0x7fffffff)
437		mtspr(SPRN_DEC, wd_panic_timeout_tb);
438
439	return 0;
440}
441
442static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
443{
444	int cpu = smp_processor_id();
445
446	if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED))
447		return HRTIMER_NORESTART;
448
449	if (!cpumask_test_cpu(cpu, &watchdog_cpumask))
450		return HRTIMER_NORESTART;
451
452	watchdog_timer_interrupt(cpu);
453
454	hrtimer_forward_now(hrtimer, ms_to_ktime(wd_timer_period_ms));
455
456	return HRTIMER_RESTART;
457}
458
459void arch_touch_nmi_watchdog(void)
460{
461	unsigned long ticks = tb_ticks_per_usec * wd_timer_period_ms * 1000;
462	int cpu = smp_processor_id();
463	u64 tb;
464
465	if (!cpumask_test_cpu(cpu, &watchdog_cpumask))
466		return;
467
468	tb = get_tb();
469	if (tb - per_cpu(wd_timer_tb, cpu) >= ticks) {
470		per_cpu(wd_timer_tb, cpu) = tb;
471		wd_smp_clear_cpu_pending(cpu);
472	}
473}
474EXPORT_SYMBOL(arch_touch_nmi_watchdog);
475
476static void start_watchdog(void *arg)
477{
478	struct hrtimer *hrtimer = this_cpu_ptr(&wd_hrtimer);
479	int cpu = smp_processor_id();
480	unsigned long flags;
481
482	if (cpumask_test_cpu(cpu, &wd_cpus_enabled)) {
483		WARN_ON(1);
484		return;
485	}
486
487	if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED))
488		return;
489
490	if (!cpumask_test_cpu(cpu, &watchdog_cpumask))
491		return;
492
493	wd_smp_lock(&flags);
494	cpumask_set_cpu(cpu, &wd_cpus_enabled);
495	if (cpumask_weight(&wd_cpus_enabled) == 1) {
496		cpumask_set_cpu(cpu, &wd_smp_cpus_pending);
497		wd_smp_last_reset_tb = get_tb();
498	}
499	wd_smp_unlock(&flags);
500
501	*this_cpu_ptr(&wd_timer_tb) = get_tb();
502
503	hrtimer_setup(hrtimer, watchdog_timer_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
504	hrtimer_start(hrtimer, ms_to_ktime(wd_timer_period_ms),
505		      HRTIMER_MODE_REL_PINNED);
506}
507
508static int start_watchdog_on_cpu(unsigned int cpu)
509{
510	return smp_call_function_single(cpu, start_watchdog, NULL, true);
511}
512
513static void stop_watchdog(void *arg)
514{
515	struct hrtimer *hrtimer = this_cpu_ptr(&wd_hrtimer);
516	int cpu = smp_processor_id();
517	unsigned long flags;
518
519	if (!cpumask_test_cpu(cpu, &wd_cpus_enabled))
520		return; /* Can happen in CPU unplug case */
521
522	hrtimer_cancel(hrtimer);
523
524	wd_smp_lock(&flags);
525	cpumask_clear_cpu(cpu, &wd_cpus_enabled);
526	wd_smp_unlock(&flags);
527
528	wd_smp_clear_cpu_pending(cpu);
529}
530
531static int stop_watchdog_on_cpu(unsigned int cpu)
532{
533	return smp_call_function_single(cpu, stop_watchdog, NULL, true);
534}
535
536static void watchdog_calc_timeouts(void)
537{
538	u64 threshold = watchdog_thresh;
539
540#ifdef CONFIG_PPC_PSERIES
541	threshold += (READ_ONCE(wd_timeout_pct) * threshold) / 100;
542#endif
543
544	wd_panic_timeout_tb = threshold * ppc_tb_freq;
545
546	/* Have the SMP detector trigger a bit later */
547	wd_smp_panic_timeout_tb = wd_panic_timeout_tb * 3 / 2;
548
549	/* 2/5 is the factor that the perf based detector uses */
550	wd_timer_period_ms = watchdog_thresh * 1000 * 2 / 5;
551}
552
553void watchdog_hardlockup_stop(void)
554{
555	int cpu;
556
557	for_each_cpu(cpu, &wd_cpus_enabled)
558		stop_watchdog_on_cpu(cpu);
559}
560
561void watchdog_hardlockup_start(void)
562{
563	int cpu;
564
565	watchdog_calc_timeouts();
566	for_each_cpu_and(cpu, cpu_online_mask, &watchdog_cpumask)
567		start_watchdog_on_cpu(cpu);
568}
569
570/*
571 * Invoked from core watchdog init.
572 */
573int __init watchdog_hardlockup_probe(void)
574{
575	int err;
576
577	err = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
578					"powerpc/watchdog:online",
579					start_watchdog_on_cpu,
580					stop_watchdog_on_cpu);
581	if (err < 0) {
582		pr_warn("could not be initialized");
583		return err;
584	}
585	return 0;
586}
587
588#ifdef CONFIG_PPC_PSERIES
589void watchdog_hardlockup_set_timeout_pct(u64 pct)
590{
591	pr_info("Set the NMI watchdog timeout factor to %llu%%\n", pct);
592	WRITE_ONCE(wd_timeout_pct, pct);
593	lockup_detector_reconfigure();
594}
595#endif