bpf_test_rqspinlock.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
  3#include <linux/sched.h>
  4#include <linux/smp.h>
  5#include <linux/delay.h>
  6#include <linux/module.h>
  7#include <linux/prandom.h>
  8#include <linux/ktime.h>
  9#include <asm/rqspinlock.h>
 10#include <linux/perf_event.h>
 11#include <linux/kthread.h>
 12#include <linux/atomic.h>
 13#include <linux/slab.h>
 14
 15static struct perf_event_attr hw_attr = {
 16	.type		= PERF_TYPE_HARDWARE,
 17	.config		= PERF_COUNT_HW_CPU_CYCLES,
 18	.size		= sizeof(struct perf_event_attr),
 19	.pinned		= 1,
 20	.disabled	= 1,
 21	.sample_period	= 100000,
 22};
 23
 24static rqspinlock_t lock_a;
 25static rqspinlock_t lock_b;
 26static rqspinlock_t lock_c;
 27
 28#define RQSL_SLOW_THRESHOLD_MS 10
 29static const unsigned int rqsl_hist_ms[] = {
 30	1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
 31	12, 14, 16, 18, 20, 25, 30, 40, 50, 75,
 32	100, 150, 200, 250, 1000,
 33};
 34#define RQSL_NR_HIST_BUCKETS ARRAY_SIZE(rqsl_hist_ms)
 35
 36enum rqsl_context {
 37	RQSL_CTX_NORMAL = 0,
 38	RQSL_CTX_NMI,
 39	RQSL_CTX_MAX,
 40};
 41
 42struct rqsl_cpu_hist {
 43	atomic64_t hist[RQSL_CTX_MAX][RQSL_NR_HIST_BUCKETS];
 44	atomic64_t success[RQSL_CTX_MAX];
 45	atomic64_t failure[RQSL_CTX_MAX];
 46};
 47
 48static DEFINE_PER_CPU(struct rqsl_cpu_hist, rqsl_cpu_hists);
 49
 50enum rqsl_mode {
 51	RQSL_MODE_AA = 0,
 52	RQSL_MODE_ABBA,
 53	RQSL_MODE_ABBCCA,
 54};
 55
 56static int test_mode = RQSL_MODE_AA;
 57module_param(test_mode, int, 0644);
 58MODULE_PARM_DESC(test_mode,
 59		 "rqspinlock test mode: 0 = AA, 1 = ABBA, 2 = ABBCCA");
 60
 61static int normal_delay = 20;
 62module_param(normal_delay, int, 0644);
 63MODULE_PARM_DESC(normal_delay,
 64		 "rqspinlock critical section length for normal context (20ms default)");
 65
 66static int nmi_delay = 10;
 67module_param(nmi_delay, int, 0644);
 68MODULE_PARM_DESC(nmi_delay,
 69		 "rqspinlock critical section length for NMI context (10ms default)");
 70
 71static struct perf_event **rqsl_evts;
 72static int rqsl_nevts;
 73
 74static struct task_struct **rqsl_threads;
 75static int rqsl_nthreads;
 76static atomic_t rqsl_ready_cpus = ATOMIC_INIT(0);
 77
 78static int pause = 0;
 79
 80static const char *rqsl_mode_names[] = {
 81	[RQSL_MODE_AA] = "AA",
 82	[RQSL_MODE_ABBA] = "ABBA",
 83	[RQSL_MODE_ABBCCA] = "ABBCCA",
 84};
 85
 86struct rqsl_lock_pair {
 87	rqspinlock_t *worker_lock;
 88	rqspinlock_t *nmi_lock;
 89};
 90
 91static struct rqsl_lock_pair rqsl_get_lock_pair(int cpu)
 92{
 93	int mode = READ_ONCE(test_mode);
 94
 95	switch (mode) {
 96	default:
 97	case RQSL_MODE_AA:
 98		return (struct rqsl_lock_pair){ &lock_a, &lock_a };
 99	case RQSL_MODE_ABBA:
100		if (cpu & 1)
101			return (struct rqsl_lock_pair){ &lock_b, &lock_a };
102		return (struct rqsl_lock_pair){ &lock_a, &lock_b };
103	case RQSL_MODE_ABBCCA:
104		switch (cpu % 3) {
105		case 0:
106			return (struct rqsl_lock_pair){ &lock_a, &lock_b };
107		case 1:
108			return (struct rqsl_lock_pair){ &lock_b, &lock_c };
109		default:
110			return (struct rqsl_lock_pair){ &lock_c, &lock_a };
111		}
112	}
113}
114
115static u32 rqsl_hist_bucket_idx(u32 delta_ms)
116{
117	int i;
118
119	for (i = 0; i < RQSL_NR_HIST_BUCKETS; i++) {
120		if (delta_ms <= rqsl_hist_ms[i])
121			return i;
122	}
123
124	return RQSL_NR_HIST_BUCKETS - 1;
125}
126
127static void rqsl_record_lock_result(u64 delta_ns, enum rqsl_context ctx, int ret)
128{
129	struct rqsl_cpu_hist *hist = this_cpu_ptr(&rqsl_cpu_hists);
130	u32 delta_ms = DIV_ROUND_UP_ULL(delta_ns, NSEC_PER_MSEC);
131	u32 bucket = rqsl_hist_bucket_idx(delta_ms);
132	atomic64_t *buckets = hist->hist[ctx];
133
134	atomic64_inc(&buckets[bucket]);
135	if (!ret)
136		atomic64_inc(&hist->success[ctx]);
137	else
138		atomic64_inc(&hist->failure[ctx]);
139}
140
141static int rqspinlock_worker_fn(void *arg)
142{
143	int cpu = smp_processor_id();
144	unsigned long flags;
145	u64 start_ns;
146	int ret;
147
148	if (cpu) {
149		atomic_inc(&rqsl_ready_cpus);
150
151		while (!kthread_should_stop()) {
152			struct rqsl_lock_pair locks = rqsl_get_lock_pair(cpu);
153			rqspinlock_t *worker_lock = locks.worker_lock;
154
155			if (READ_ONCE(pause)) {
156				msleep(1000);
157				continue;
158			}
159			start_ns = ktime_get_mono_fast_ns();
160			ret = raw_res_spin_lock_irqsave(worker_lock, flags);
161			rqsl_record_lock_result(ktime_get_mono_fast_ns() - start_ns,
162						RQSL_CTX_NORMAL, ret);
163			mdelay(normal_delay);
164			if (!ret)
165				raw_res_spin_unlock_irqrestore(worker_lock, flags);
166			cpu_relax();
167		}
168		return 0;
169	}
170
171	while (!kthread_should_stop()) {
172		int expected = rqsl_nthreads > 0 ? rqsl_nthreads - 1 : 0;
173		int ready = atomic_read(&rqsl_ready_cpus);
174
175		if (ready == expected && !READ_ONCE(pause)) {
176			for (int i = 0; i < rqsl_nevts; i++)
177				perf_event_enable(rqsl_evts[i]);
178			pr_err("Waiting 5 secs to pause the test\n");
179			msleep(1000 * 5);
180			WRITE_ONCE(pause, 1);
181			pr_err("Paused the test\n");
182		} else {
183			msleep(1000);
184			cpu_relax();
185		}
186	}
187	return 0;
188}
189
190static void nmi_cb(struct perf_event *event, struct perf_sample_data *data,
191		   struct pt_regs *regs)
192{
193	struct rqsl_lock_pair locks;
194	int cpu = smp_processor_id();
195	unsigned long flags;
196	u64 start_ns;
197	int ret;
198
199	if (!cpu || READ_ONCE(pause))
200		return;
201
202	locks = rqsl_get_lock_pair(cpu);
203	start_ns = ktime_get_mono_fast_ns();
204	ret = raw_res_spin_lock_irqsave(locks.nmi_lock, flags);
205	rqsl_record_lock_result(ktime_get_mono_fast_ns() - start_ns,
206				RQSL_CTX_NMI, ret);
207
208	mdelay(nmi_delay);
209
210	if (!ret)
211		raw_res_spin_unlock_irqrestore(locks.nmi_lock, flags);
212}
213
214static void free_rqsl_threads(void)
215{
216	int i;
217
218	if (rqsl_threads) {
219		for_each_online_cpu(i) {
220			if (rqsl_threads[i])
221				kthread_stop(rqsl_threads[i]);
222		}
223		kfree(rqsl_threads);
224	}
225}
226
227static void free_rqsl_evts(void)
228{
229	int i;
230
231	if (rqsl_evts) {
232		for (i = 0; i < rqsl_nevts; i++) {
233			if (rqsl_evts[i])
234				perf_event_release_kernel(rqsl_evts[i]);
235		}
236		kfree(rqsl_evts);
237	}
238}
239
240static int bpf_test_rqspinlock_init(void)
241{
242	int i, ret;
243	int ncpus = num_online_cpus();
244
245	if (test_mode < RQSL_MODE_AA || test_mode > RQSL_MODE_ABBCCA) {
246		pr_err("Invalid mode %d\n", test_mode);
247		return -EINVAL;
248	}
249
250	pr_err("Mode = %s\n", rqsl_mode_names[test_mode]);
251
252	if (ncpus < test_mode + 2)
253		return -ENOTSUPP;
254
255	raw_res_spin_lock_init(&lock_a);
256	raw_res_spin_lock_init(&lock_b);
257	raw_res_spin_lock_init(&lock_c);
258
259	rqsl_evts = kcalloc(ncpus - 1, sizeof(*rqsl_evts), GFP_KERNEL);
260	if (!rqsl_evts)
261		return -ENOMEM;
262	rqsl_nevts = ncpus - 1;
263
264	for (i = 1; i < ncpus; i++) {
265		struct perf_event *e;
266
267		e = perf_event_create_kernel_counter(&hw_attr, i, NULL, nmi_cb, NULL);
268		if (IS_ERR(e)) {
269			ret = PTR_ERR(e);
270			goto err_perf_events;
271		}
272		rqsl_evts[i - 1] = e;
273	}
274
275	rqsl_threads = kcalloc(ncpus, sizeof(*rqsl_threads), GFP_KERNEL);
276	if (!rqsl_threads) {
277		ret = -ENOMEM;
278		goto err_perf_events;
279	}
280	rqsl_nthreads = ncpus;
281
282	for_each_online_cpu(i) {
283		struct task_struct *t;
284
285		t = kthread_create(rqspinlock_worker_fn, NULL, "rqsl_w/%d", i);
286		if (IS_ERR(t)) {
287			ret = PTR_ERR(t);
288			goto err_threads_create;
289		}
290		kthread_bind(t, i);
291		rqsl_threads[i] = t;
292		wake_up_process(t);
293	}
294	return 0;
295
296err_threads_create:
297	free_rqsl_threads();
298err_perf_events:
299	free_rqsl_evts();
300	return ret;
301}
302
303module_init(bpf_test_rqspinlock_init);
304
305static void rqsl_print_histograms(void)
306{
307	int cpu, i;
308
309	pr_err("rqspinlock acquisition latency histogram (ms):\n");
310
311	for_each_online_cpu(cpu) {
312		struct rqsl_cpu_hist *hist = per_cpu_ptr(&rqsl_cpu_hists, cpu);
313		u64 norm_counts[RQSL_NR_HIST_BUCKETS];
314		u64 nmi_counts[RQSL_NR_HIST_BUCKETS];
315		u64 total_counts[RQSL_NR_HIST_BUCKETS];
316		u64 norm_success, nmi_success, success_total;
317		u64 norm_failure, nmi_failure, failure_total;
318		u64 norm_total = 0, nmi_total = 0, total = 0;
319		bool has_slow = false;
320
321		for (i = 0; i < RQSL_NR_HIST_BUCKETS; i++) {
322			norm_counts[i] = atomic64_read(&hist->hist[RQSL_CTX_NORMAL][i]);
323			nmi_counts[i] = atomic64_read(&hist->hist[RQSL_CTX_NMI][i]);
324			total_counts[i] = norm_counts[i] + nmi_counts[i];
325			norm_total += norm_counts[i];
326			nmi_total += nmi_counts[i];
327			total += total_counts[i];
328			if (rqsl_hist_ms[i] > RQSL_SLOW_THRESHOLD_MS &&
329			    total_counts[i])
330				has_slow = true;
331		}
332
333		norm_success = atomic64_read(&hist->success[RQSL_CTX_NORMAL]);
334		nmi_success = atomic64_read(&hist->success[RQSL_CTX_NMI]);
335		norm_failure = atomic64_read(&hist->failure[RQSL_CTX_NORMAL]);
336		nmi_failure = atomic64_read(&hist->failure[RQSL_CTX_NMI]);
337		success_total = norm_success + nmi_success;
338		failure_total = norm_failure + nmi_failure;
339
340		if (!total)
341			continue;
342
343		if (!has_slow) {
344			pr_err(" cpu%d: total %llu (normal %llu, nmi %llu) | "
345			       "success %llu (normal %llu, nmi %llu) | "
346			       "failure %llu (normal %llu, nmi %llu), all within 0-%ums\n",
347			       cpu, total, norm_total, nmi_total,
348			       success_total, norm_success, nmi_success,
349			       failure_total, norm_failure, nmi_failure,
350			       RQSL_SLOW_THRESHOLD_MS);
351			continue;
352		}
353
354		pr_err(" cpu%d: total %llu (normal %llu, nmi %llu) | "
355		       "success %llu (normal %llu, nmi %llu) | "
356		       "failure %llu (normal %llu, nmi %llu)\n",
357		       cpu, total, norm_total, nmi_total,
358		       success_total, norm_success, nmi_success,
359		       failure_total, norm_failure, nmi_failure);
360		for (i = 0; i < RQSL_NR_HIST_BUCKETS; i++) {
361			unsigned int start_ms;
362
363			if (!total_counts[i])
364				continue;
365
366			start_ms = i == 0 ? 0 : rqsl_hist_ms[i - 1] + 1;
367			if (i == RQSL_NR_HIST_BUCKETS - 1) {
368				pr_err("   >= %ums: total %llu (normal %llu, nmi %llu)\n",
369				       start_ms, total_counts[i],
370				       norm_counts[i], nmi_counts[i]);
371			} else {
372				pr_err("   %u-%ums: total %llu (normal %llu, nmi %llu)\n",
373				       start_ms, rqsl_hist_ms[i],
374				       total_counts[i],
375				       norm_counts[i], nmi_counts[i]);
376			}
377		}
378	}
379}
380
381static void bpf_test_rqspinlock_exit(void)
382{
383	WRITE_ONCE(pause, 1);
384	free_rqsl_threads();
385	free_rqsl_evts();
386	rqsl_print_histograms();
387}
388
389module_exit(bpf_test_rqspinlock_exit);
390
391MODULE_AUTHOR("Kumar Kartikeya Dwivedi");
392MODULE_DESCRIPTION("BPF rqspinlock stress test module");
393MODULE_LICENSE("GPL");