watchdog_perf.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Detect hard lockups on a system using perf
  4 *
  5 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
  6 *
  7 * Note: Most of this code is borrowed heavily from the original softlockup
  8 * detector, so thanks to Ingo for the initial implementation.
  9 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
 10 * to those contributors as well.
 11 */
 12
 13#define pr_fmt(fmt) "NMI watchdog: " fmt
 14
 15#include <linux/panic.h>
 16#include <linux/nmi.h>
 17#include <linux/atomic.h>
 18#include <linux/module.h>
 19#include <linux/sched/debug.h>
 20
 21#include <asm/irq_regs.h>
 22#include <linux/perf_event.h>
 23
 24static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
 25
 26static atomic_t watchdog_cpus = ATOMIC_INIT(0);
 27
 28#ifdef CONFIG_HARDLOCKUP_CHECK_TIMESTAMP
 29static DEFINE_PER_CPU(ktime_t, last_timestamp);
 30static DEFINE_PER_CPU(unsigned int, nmi_rearmed);
 31static ktime_t watchdog_hrtimer_sample_threshold __read_mostly;
 32
 33void watchdog_update_hrtimer_threshold(u64 period)
 34{
 35	/*
 36	 * The hrtimer runs with a period of (watchdog_threshold * 2) / 5
 37	 *
 38	 * So it runs effectively with 2.5 times the rate of the NMI
 39	 * watchdog. That means the hrtimer should fire 2-3 times before
 40	 * the NMI watchdog expires. The NMI watchdog on x86 is based on
 41	 * unhalted CPU cycles, so if Turbo-Mode is enabled the CPU cycles
 42	 * might run way faster than expected and the NMI fires in a
 43	 * smaller period than the one deduced from the nominal CPU
 44	 * frequency. Depending on the Turbo-Mode factor this might be fast
 45	 * enough to get the NMI period smaller than the hrtimer watchdog
 46	 * period and trigger false positives.
 47	 *
 48	 * The sample threshold is used to check in the NMI handler whether
 49	 * the minimum time between two NMI samples has elapsed. That
 50	 * prevents false positives.
 51	 *
 52	 * Set this to 4/5 of the actual watchdog threshold period so the
 53	 * hrtimer is guaranteed to fire at least once within the real
 54	 * watchdog threshold.
 55	 */
 56	watchdog_hrtimer_sample_threshold = period * 2;
 57}
 58
 59static bool watchdog_check_timestamp(void)
 60{
 61	ktime_t delta, now = ktime_get_mono_fast_ns();
 62
 63	delta = now - __this_cpu_read(last_timestamp);
 64	if (delta < watchdog_hrtimer_sample_threshold) {
 65		/*
 66		 * If ktime is jiffies based, a stalled timer would prevent
 67		 * jiffies from being incremented and the filter would look
 68		 * at a stale timestamp and never trigger.
 69		 */
 70		if (__this_cpu_inc_return(nmi_rearmed) < 10)
 71			return false;
 72	}
 73	__this_cpu_write(nmi_rearmed, 0);
 74	__this_cpu_write(last_timestamp, now);
 75	return true;
 76}
 77
 78static void watchdog_init_timestamp(void)
 79{
 80	__this_cpu_write(nmi_rearmed, 0);
 81	__this_cpu_write(last_timestamp, ktime_get_mono_fast_ns());
 82}
 83#else
 84static inline bool watchdog_check_timestamp(void) { return true; }
 85static inline void watchdog_init_timestamp(void) { }
 86#endif
 87
 88static struct perf_event_attr wd_hw_attr = {
 89	.type		= PERF_TYPE_HARDWARE,
 90	.config		= PERF_COUNT_HW_CPU_CYCLES,
 91	.size		= sizeof(struct perf_event_attr),
 92	.pinned		= 1,
 93	.disabled	= 1,
 94};
 95
 96static struct perf_event_attr fallback_wd_hw_attr = {
 97	.type		= PERF_TYPE_HARDWARE,
 98	.config		= PERF_COUNT_HW_CPU_CYCLES,
 99	.size		= sizeof(struct perf_event_attr),
100	.pinned		= 1,
101	.disabled	= 1,
102};
103
104/* Callback function for perf event subsystem */
105static void watchdog_overflow_callback(struct perf_event *event,
106				       struct perf_sample_data *data,
107				       struct pt_regs *regs)
108{
109	/* Ensure the watchdog never gets throttled */
110	event->hw.interrupts = 0;
111
112	if (panic_in_progress())
113		return;
114
115	if (!watchdog_check_timestamp())
116		return;
117
118	watchdog_hardlockup_check(smp_processor_id(), regs);
119}
120
121static int hardlockup_detector_event_create(void)
122{
123	unsigned int cpu;
124	struct perf_event_attr *wd_attr;
125	struct perf_event *evt;
126
127	/*
128	 * Preemption is not disabled because memory will be allocated.
129	 * Ensure CPU-locality by calling this in per-CPU kthread.
130	 */
131	WARN_ON(!is_percpu_thread());
132	cpu = raw_smp_processor_id();
133	wd_attr = &wd_hw_attr;
134	wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh);
135
136	/* Try to register using hardware perf events */
137	evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL,
138					       watchdog_overflow_callback, NULL);
139	if (IS_ERR(evt)) {
140		wd_attr = &fallback_wd_hw_attr;
141		wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh);
142		evt = perf_event_create_kernel_counter(wd_attr, cpu, NULL,
143						       watchdog_overflow_callback, NULL);
144	}
145
146	if (IS_ERR(evt)) {
147		pr_debug("Perf event create on CPU %d failed with %ld\n", cpu,
148			 PTR_ERR(evt));
149		return PTR_ERR(evt);
150	}
151	WARN_ONCE(this_cpu_read(watchdog_ev), "unexpected watchdog_ev leak");
152	this_cpu_write(watchdog_ev, evt);
153	return 0;
154}
155
156/**
157 * watchdog_hardlockup_enable - Enable the local event
158 * @cpu: The CPU to enable hard lockup on.
159 */
160void watchdog_hardlockup_enable(unsigned int cpu)
161{
162	WARN_ON_ONCE(cpu != smp_processor_id());
163
164	if (hardlockup_detector_event_create())
165		return;
166
167	/* use original value for check */
168	if (!atomic_fetch_inc(&watchdog_cpus))
169		pr_info("Enabled. Permanently consumes one hw-PMU counter.\n");
170
171	watchdog_init_timestamp();
172	perf_event_enable(this_cpu_read(watchdog_ev));
173}
174
175/**
176 * watchdog_hardlockup_disable - Disable the local event
177 * @cpu: The CPU to enable hard lockup on.
178 */
179void watchdog_hardlockup_disable(unsigned int cpu)
180{
181	struct perf_event *event = this_cpu_read(watchdog_ev);
182
183	WARN_ON_ONCE(cpu != smp_processor_id());
184
185	if (event) {
186		perf_event_disable(event);
187		perf_event_release_kernel(event);
188		this_cpu_write(watchdog_ev, NULL);
189		atomic_dec(&watchdog_cpus);
190	}
191}
192
193/**
194 * hardlockup_detector_perf_adjust_period - Adjust the event period due
195 *                                          to current cpu frequency change
196 * @period: The target period to be set
197 */
198void hardlockup_detector_perf_adjust_period(u64 period)
199{
200	struct perf_event *event = this_cpu_read(watchdog_ev);
201
202	if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED))
203		return;
204
205	if (!event)
206		return;
207
208	if (event->attr.sample_period == period)
209		return;
210
211	if (perf_event_period(event, period))
212		pr_err("failed to change period to %llu\n", period);
213}
214
215/**
216 * hardlockup_detector_perf_stop - Globally stop watchdog events
217 *
218 * Special interface for x86 to handle the perf HT bug.
219 */
220void __init hardlockup_detector_perf_stop(void)
221{
222	int cpu;
223
224	lockdep_assert_cpus_held();
225
226	for_each_online_cpu(cpu) {
227		struct perf_event *event = per_cpu(watchdog_ev, cpu);
228
229		if (event)
230			perf_event_disable(event);
231	}
232}
233
234/**
235 * hardlockup_detector_perf_restart - Globally restart watchdog events
236 *
237 * Special interface for x86 to handle the perf HT bug.
238 */
239void __init hardlockup_detector_perf_restart(void)
240{
241	int cpu;
242
243	lockdep_assert_cpus_held();
244
245	if (!(watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED))
246		return;
247
248	for_each_online_cpu(cpu) {
249		struct perf_event *event = per_cpu(watchdog_ev, cpu);
250
251		if (event)
252			perf_event_enable(event);
253	}
254}
255
256bool __weak __init arch_perf_nmi_is_available(void)
257{
258	return true;
259}
260
261/**
262 * watchdog_hardlockup_probe - Probe whether NMI event is available at all
263 */
264int __init watchdog_hardlockup_probe(void)
265{
266	int ret;
267
268	if (!arch_perf_nmi_is_available())
269		return -ENODEV;
270
271	ret = hardlockup_detector_event_create();
272
273	if (ret) {
274		pr_info("Perf NMI watchdog permanently disabled\n");
275	} else {
276		perf_event_release_kernel(this_cpu_read(watchdog_ev));
277		this_cpu_write(watchdog_ev, NULL);
278	}
279	return ret;
280}
281
282/**
283 * hardlockup_config_perf_event - Overwrite config of wd_hw_attr.
284 * @str: number which identifies the raw perf event to use
285 */
286void __init hardlockup_config_perf_event(const char *str)
287{
288	u64 config;
289	char buf[24];
290	char *comma = strchr(str, ',');
291
292	if (!comma) {
293		if (kstrtoull(str, 16, &config))
294			return;
295	} else {
296		unsigned int len = comma - str;
297
298		if (len > sizeof(buf))
299			return;
300
301		strscpy(buf, str, len);
302		if (kstrtoull(buf, 16, &config))
303			return;
304	}
305
306	wd_hw_attr.type = PERF_TYPE_RAW;
307	wd_hw_attr.config = config;
308}