cbo.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2023 Ventana Micro Systems Inc.
  4 *
  5 * Run with 'taskset -c <cpu-list> cbo' to only execute hwprobe on a
  6 * subset of cpus, as well as only executing the tests on those cpus.
  7 */
  8#define _GNU_SOURCE
  9#include <stdbool.h>
 10#include <stdint.h>
 11#include <string.h>
 12#include <sched.h>
 13#include <signal.h>
 14#include <assert.h>
 15#include <linux/compiler.h>
 16#include <linux/kernel.h>
 17#include <asm/ucontext.h>
 18#include <getopt.h>
 19
 20#include "hwprobe.h"
 21#include "kselftest.h"
 22
 23#define MK_CBO(fn) le32_bswap((uint32_t)(fn) << 20 | 10 << 15 | 2 << 12 | 0 << 7 | 15)
 24#define MK_PREFETCH(fn) \
 25	le32_bswap(0 << 25 | (uint32_t)(fn) << 20 | 10 << 15 | 6 << 12 | 0 << 7 | 19)
 26
 27static char mem[4096] __aligned(4096) = { [0 ... 4095] = 0xa5 };
 28
 29static bool got_fault;
 30
 31static void fault_handler(int sig, siginfo_t *info, void *context)
 32{
 33	unsigned long *regs = (unsigned long *)&((ucontext_t *)context)->uc_mcontext;
 34	uint32_t insn = *(uint32_t *)regs[0];
 35
 36	if (sig == SIGILL)
 37		assert(insn == MK_CBO(regs[11]));
 38
 39	if (sig == SIGSEGV || sig == SIGBUS)
 40		assert(insn == MK_PREFETCH(regs[11]));
 41
 42	got_fault = true;
 43	regs[0] += 4;
 44}
 45
 46#define cbo_insn(base, fn)							\
 47({										\
 48	asm volatile(								\
 49	"mv	a0, %0\n"							\
 50	"li	a1, %1\n"							\
 51	".4byte	%2\n"								\
 52	: : "r" (base), "i" (fn), "i" (MK_CBO(fn)) : "a0", "a1", "memory");	\
 53})
 54
 55#define prefetch_insn(base, fn)							\
 56({										\
 57	asm volatile(								\
 58	"mv	a0, %0\n"							\
 59	"li	a1, %1\n"							\
 60	".4byte	%2\n"								\
 61	: : "r" (base), "i" (fn), "i" (MK_PREFETCH(fn)) : "a0", "a1");		\
 62})
 63
 64static void cbo_inval(char *base) { cbo_insn(base, 0); }
 65static void cbo_clean(char *base) { cbo_insn(base, 1); }
 66static void cbo_flush(char *base) { cbo_insn(base, 2); }
 67static void cbo_zero(char *base)  { cbo_insn(base, 4); }
 68static void prefetch_i(char *base) { prefetch_insn(base, 0); }
 69static void prefetch_r(char *base) { prefetch_insn(base, 1); }
 70static void prefetch_w(char *base) { prefetch_insn(base, 3); }
 71
 72static void test_no_cbo_inval(void *arg)
 73{
 74	ksft_print_msg("Testing cbo.inval instruction remain privileged\n");
 75	got_fault = false;
 76	cbo_inval(&mem[0]);
 77	ksft_test_result(got_fault, "No cbo.inval\n");
 78}
 79
 80static void test_no_zicbom(void *arg)
 81{
 82	ksft_print_msg("Testing Zicbom instructions remain privileged\n");
 83
 84	got_fault = false;
 85	cbo_clean(&mem[0]);
 86	ksft_test_result(got_fault, "No cbo.clean\n");
 87
 88	got_fault = false;
 89	cbo_flush(&mem[0]);
 90	ksft_test_result(got_fault, "No cbo.flush\n");
 91}
 92
 93static void test_no_zicboz(void *arg)
 94{
 95	ksft_print_msg("No Zicboz, testing cbo.zero remains privileged\n");
 96
 97	got_fault = false;
 98	cbo_zero(&mem[0]);
 99	ksft_test_result(got_fault, "No cbo.zero\n");
100}
101
102static bool is_power_of_2(__u64 n)
103{
104	return n != 0 && (n & (n - 1)) == 0;
105}
106
107static void test_zicbop(void *arg)
108{
109	struct riscv_hwprobe pair = {
110		.key = RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE,
111	};
112	struct sigaction act = {
113		.sa_sigaction = &fault_handler,
114		.sa_flags = SA_SIGINFO
115	};
116	struct sigaction dfl = {
117		.sa_handler = SIG_DFL
118	};
119	cpu_set_t *cpus = (cpu_set_t *)arg;
120	__u64 block_size;
121	long rc;
122
123	rc = sigaction(SIGSEGV, &act, NULL);
124	assert(rc == 0);
125	rc = sigaction(SIGBUS, &act, NULL);
126	assert(rc == 0);
127
128	rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
129	block_size = pair.value;
130	ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOP_BLOCK_SIZE &&
131			 is_power_of_2(block_size), "Zicbop block size\n");
132	ksft_print_msg("Zicbop block size: %llu\n", block_size);
133
134	got_fault = false;
135	prefetch_i(&mem[0]);
136	prefetch_r(&mem[0]);
137	prefetch_w(&mem[0]);
138	ksft_test_result(!got_fault, "Zicbop prefetch.* on valid address\n");
139
140	got_fault = false;
141	prefetch_i(NULL);
142	prefetch_r(NULL);
143	prefetch_w(NULL);
144	ksft_test_result(!got_fault, "Zicbop prefetch.* on NULL\n");
145
146	rc = sigaction(SIGBUS, &dfl, NULL);
147	assert(rc == 0);
148	rc = sigaction(SIGSEGV, &dfl, NULL);
149	assert(rc == 0);
150}
151
152static void test_zicbom(void *arg)
153{
154	struct riscv_hwprobe pair = {
155		.key = RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE,
156	};
157	cpu_set_t *cpus = (cpu_set_t *)arg;
158	__u64 block_size;
159	long rc;
160
161	rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
162	block_size = pair.value;
163	ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOM_BLOCK_SIZE &&
164			 is_power_of_2(block_size), "Zicbom block size\n");
165	ksft_print_msg("Zicbom block size: %llu\n", block_size);
166
167	got_fault = false;
168	cbo_clean(&mem[block_size]);
169	ksft_test_result(!got_fault, "cbo.clean\n");
170
171	got_fault = false;
172	cbo_flush(&mem[block_size]);
173	ksft_test_result(!got_fault, "cbo.flush\n");
174}
175
176static void test_zicboz(void *arg)
177{
178	struct riscv_hwprobe pair = {
179		.key = RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE,
180	};
181	cpu_set_t *cpus = (cpu_set_t *)arg;
182	__u64 block_size;
183	int i, j;
184	long rc;
185
186	rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)cpus, 0);
187	block_size = pair.value;
188	ksft_test_result(rc == 0 && pair.key == RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE &&
189			 is_power_of_2(block_size), "Zicboz block size\n");
190	ksft_print_msg("Zicboz block size: %llu\n", block_size);
191
192	got_fault = false;
193	cbo_zero(&mem[block_size]);
194	ksft_test_result(!got_fault, "cbo.zero\n");
195
196	if (got_fault || !is_power_of_2(block_size)) {
197		ksft_test_result_skip("cbo.zero check\n");
198		return;
199	}
200
201	assert(block_size <= 1024);
202
203	for (i = 0; i < 4096 / block_size; ++i) {
204		if (i % 2)
205			cbo_zero(&mem[i * block_size]);
206	}
207
208	for (i = 0; i < 4096 / block_size; ++i) {
209		char expected = i % 2 ? 0x0 : 0xa5;
210
211		for (j = 0; j < block_size; ++j) {
212			if (mem[i * block_size + j] != expected) {
213				ksft_test_result_fail("cbo.zero check\n");
214				ksft_print_msg("cbo.zero check: mem[%llu] != 0x%x\n",
215					       i * block_size + j, expected);
216				return;
217			}
218		}
219	}
220
221	ksft_test_result_pass("cbo.zero check\n");
222}
223
224static void check_no_zicbo_cpus(cpu_set_t *cpus, __u64 cbo)
225{
226	struct riscv_hwprobe pair = {
227		.key = RISCV_HWPROBE_KEY_IMA_EXT_0,
228	};
229	cpu_set_t one_cpu;
230	int i = 0, c = 0;
231	long rc;
232	char *cbostr;
233
234	while (i++ < CPU_COUNT(cpus)) {
235		while (!CPU_ISSET(c, cpus))
236			++c;
237
238		CPU_ZERO(&one_cpu);
239		CPU_SET(c, &one_cpu);
240
241		rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&one_cpu, 0);
242		assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0);
243
244		switch (cbo) {
245		case RISCV_HWPROBE_EXT_ZICBOZ:
246			cbostr = "Zicboz";
247			break;
248		case RISCV_HWPROBE_EXT_ZICBOM:
249			cbostr = "Zicbom";
250			break;
251		case RISCV_HWPROBE_EXT_ZICBOP:
252			cbostr = "Zicbop";
253			break;
254		default:
255			ksft_exit_fail_msg("Internal error: invalid cbo %llu\n", cbo);
256		}
257
258		if (pair.value & cbo)
259			ksft_exit_fail_msg("%s is only present on a subset of harts.\n"
260					   "Use taskset to select a set of harts where %s\n"
261					   "presence (present or not) is consistent for each hart\n",
262					   cbostr, cbostr);
263		++c;
264	}
265}
266
267enum {
268	TEST_ZICBOZ,
269	TEST_NO_ZICBOZ,
270	TEST_ZICBOM,
271	TEST_NO_ZICBOM,
272	TEST_NO_CBO_INVAL,
273	TEST_ZICBOP,
274};
275
276static struct test_info {
277	bool enabled;
278	unsigned int nr_tests;
279	void (*test_fn)(void *arg);
280} tests[] = {
281	[TEST_ZICBOZ]		= { .nr_tests = 3, test_zicboz },
282	[TEST_NO_ZICBOZ]	= { .nr_tests = 1, test_no_zicboz },
283	[TEST_ZICBOM]		= { .nr_tests = 3, test_zicbom },
284	[TEST_NO_ZICBOM]	= { .nr_tests = 2, test_no_zicbom },
285	[TEST_NO_CBO_INVAL]	= { .nr_tests = 1, test_no_cbo_inval },
286	[TEST_ZICBOP]		= { .nr_tests = 3, test_zicbop },
287};
288
289static const struct option long_opts[] = {
290	{"zicbom-raises-sigill", no_argument, 0, 'm'},
291	{"zicboz-raises-sigill", no_argument, 0, 'z'},
292	{0, 0, 0, 0}
293};
294
295int main(int argc, char **argv)
296{
297	struct sigaction act = {
298		.sa_sigaction = &fault_handler,
299		.sa_flags = SA_SIGINFO,
300	};
301	struct riscv_hwprobe pair;
302	unsigned int plan = 0;
303	cpu_set_t cpus;
304	long rc;
305	int i, opt, long_index;
306
307	long_index = 0;
308
309	while ((opt = getopt_long(argc, argv, "mz", long_opts, &long_index)) != -1) {
310		switch (opt) {
311		case 'm':
312			tests[TEST_NO_ZICBOM].enabled = true;
313			tests[TEST_NO_CBO_INVAL].enabled = true;
314			rc = sigaction(SIGILL, &act, NULL);
315			assert(rc == 0);
316			break;
317		case 'z':
318			tests[TEST_NO_ZICBOZ].enabled = true;
319			tests[TEST_NO_CBO_INVAL].enabled = true;
320			rc = sigaction(SIGILL, &act, NULL);
321			assert(rc == 0);
322			break;
323		case '?':
324			fprintf(stderr,
325				"Usage: %s [--zicbom-raises-sigill|-m] [--zicboz-raises-sigill|-z]\n",
326				argv[0]);
327			exit(1);
328		default:
329			break;
330		}
331	}
332
333	rc = sched_getaffinity(0, sizeof(cpu_set_t), &cpus);
334	assert(rc == 0);
335
336	ksft_print_header();
337
338	pair.key = RISCV_HWPROBE_KEY_IMA_EXT_0;
339	rc = riscv_hwprobe(&pair, 1, sizeof(cpu_set_t), (unsigned long *)&cpus, 0);
340	if (rc < 0)
341		ksft_exit_fail_msg("hwprobe() failed with %ld\n", rc);
342	assert(rc == 0 && pair.key == RISCV_HWPROBE_KEY_IMA_EXT_0);
343
344	if (pair.value & RISCV_HWPROBE_EXT_ZICBOZ) {
345		tests[TEST_ZICBOZ].enabled = true;
346		tests[TEST_NO_ZICBOZ].enabled = false;
347	} else {
348		check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOZ);
349	}
350
351	if (pair.value & RISCV_HWPROBE_EXT_ZICBOM) {
352		tests[TEST_ZICBOM].enabled = true;
353		tests[TEST_NO_ZICBOM].enabled = false;
354	} else {
355		check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOM);
356	}
357
358	if (pair.value & RISCV_HWPROBE_EXT_ZICBOP)
359		tests[TEST_ZICBOP].enabled = true;
360	else
361		check_no_zicbo_cpus(&cpus, RISCV_HWPROBE_EXT_ZICBOP);
362
363	for (i = 0; i < ARRAY_SIZE(tests); ++i)
364		plan += tests[i].enabled ? tests[i].nr_tests : 0;
365
366	if (plan == 0)
367		ksft_print_msg("No tests enabled.\n");
368	else
369		ksft_set_plan(plan);
370
371	for (i = 0; i < ARRAY_SIZE(tests); ++i) {
372		if (tests[i].enabled)
373			tests[i].test_fn(&cpus);
374	}
375
376	ksft_finished();
377}