cacheflush.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2017 SiFive
  4 */
  5
  6#include <linux/acpi.h>
  7#include <linux/of.h>
  8#include <linux/prctl.h>
  9#include <asm/acpi.h>
 10#include <asm/cacheflush.h>
 11
 12#ifdef CONFIG_SMP
 13
 14#include <asm/sbi.h>
 15
 16static void ipi_remote_fence_i(void *info)
 17{
 18	return local_flush_icache_all();
 19}
 20
 21void flush_icache_all(void)
 22{
 23	local_flush_icache_all();
 24
 25	if (num_online_cpus() < 2)
 26		return;
 27
 28	/*
 29	 * Make sure all previous writes to the D$ are ordered before making
 30	 * the IPI. The RISC-V spec states that a hart must execute a data fence
 31	 * before triggering a remote fence.i in order to make the modification
 32	 * visable for remote harts.
 33	 *
 34	 * IPIs on RISC-V are triggered by MMIO writes to either CLINT or
 35	 * S-IMSIC, so the fence ensures previous data writes "happen before"
 36	 * the MMIO.
 37	 */
 38	RISCV_FENCE(w, o);
 39
 40	if (riscv_use_sbi_for_rfence())
 41		sbi_remote_fence_i(NULL);
 42	else
 43		on_each_cpu(ipi_remote_fence_i, NULL, 1);
 44}
 45EXPORT_SYMBOL(flush_icache_all);
 46
 47/*
 48 * Performs an icache flush for the given MM context.  RISC-V has no direct
 49 * mechanism for instruction cache shoot downs, so instead we send an IPI that
 50 * informs the remote harts they need to flush their local instruction caches.
 51 * To avoid pathologically slow behavior in a common case (a bunch of
 52 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
 53 * IPIs for harts that are not currently executing a MM context and instead
 54 * schedule a deferred local instruction cache flush to be performed before
 55 * execution resumes on each hart.
 56 */
 57void flush_icache_mm(struct mm_struct *mm, bool local)
 58{
 59	unsigned int cpu;
 60	cpumask_t others, *mask;
 61
 62	preempt_disable();
 63
 64	/* Mark every hart's icache as needing a flush for this MM. */
 65	mask = &mm->context.icache_stale_mask;
 66	cpumask_setall(mask);
 67	/* Flush this hart's I$ now, and mark it as flushed. */
 68	cpu = smp_processor_id();
 69	cpumask_clear_cpu(cpu, mask);
 70	local_flush_icache_all();
 71
 72	/*
 73	 * Flush the I$ of other harts concurrently executing, and mark them as
 74	 * flushed.
 75	 */
 76	cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
 77	local |= cpumask_empty(&others);
 78	if (mm == current->active_mm && local) {
 79		/*
 80		 * It's assumed that at least one strongly ordered operation is
 81		 * performed on this hart between setting a hart's cpumask bit
 82		 * and scheduling this MM context on that hart.  Sending an SBI
 83		 * remote message will do this, but in the case where no
 84		 * messages are sent we still need to order this hart's writes
 85		 * with flush_icache_deferred().
 86		 */
 87		smp_mb();
 88	} else if (riscv_use_sbi_for_rfence()) {
 89		sbi_remote_fence_i(&others);
 90	} else {
 91		on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1);
 92	}
 93
 94	preempt_enable();
 95}
 96
 97#endif /* CONFIG_SMP */
 98
 99#ifdef CONFIG_MMU
100void flush_icache_pte(struct mm_struct *mm, pte_t pte)
101{
102	struct folio *folio = page_folio(pte_page(pte));
103
104	if (!test_bit(PG_dcache_clean, &folio->flags.f)) {
105		flush_icache_mm(mm, false);
106		set_bit(PG_dcache_clean, &folio->flags.f);
107	}
108}
109#endif /* CONFIG_MMU */
110
111unsigned int riscv_cbom_block_size;
112EXPORT_SYMBOL_GPL(riscv_cbom_block_size);
113
114unsigned int riscv_cboz_block_size;
115EXPORT_SYMBOL_GPL(riscv_cboz_block_size);
116
117unsigned int riscv_cbop_block_size;
118EXPORT_SYMBOL_GPL(riscv_cbop_block_size);
119
120static void __init cbo_get_block_size(struct device_node *node,
121				      const char *name, u32 *block_size,
122				      unsigned long *first_hartid)
123{
124	unsigned long hartid;
125	u32 val;
126
127	if (riscv_of_processor_hartid(node, &hartid))
128		return;
129
130	if (of_property_read_u32(node, name, &val))
131		return;
132
133	if (!*block_size) {
134		*block_size = val;
135		*first_hartid = hartid;
136	} else if (*block_size != val) {
137		pr_warn("%s mismatched between harts %lu and %lu\n",
138			name, *first_hartid, hartid);
139	}
140}
141
142void __init riscv_init_cbo_blocksizes(void)
143{
144	unsigned long cbom_hartid, cboz_hartid, cbop_hartid;
145	u32 cbom_block_size = 0, cboz_block_size = 0, cbop_block_size = 0;
146	struct device_node *node;
147	struct acpi_table_header *rhct;
148	acpi_status status;
149
150	if (acpi_disabled) {
151		for_each_of_cpu_node(node) {
152			/* set block-size for cbom and/or cboz extension if available */
153			cbo_get_block_size(node, "riscv,cbom-block-size",
154					   &cbom_block_size, &cbom_hartid);
155			cbo_get_block_size(node, "riscv,cboz-block-size",
156					   &cboz_block_size, &cboz_hartid);
157			cbo_get_block_size(node, "riscv,cbop-block-size",
158					   &cbop_block_size, &cbop_hartid);
159		}
160	} else {
161		status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
162		if (ACPI_FAILURE(status))
163			return;
164
165		acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, &cbop_block_size);
166		acpi_put_table((struct acpi_table_header *)rhct);
167	}
168
169	if (cbom_block_size)
170		riscv_cbom_block_size = cbom_block_size;
171
172	if (cboz_block_size)
173		riscv_cboz_block_size = cboz_block_size;
174
175	if (cbop_block_size)
176		riscv_cbop_block_size = cbop_block_size;
177}
178
179#ifdef CONFIG_SMP
180static void set_icache_stale_mask(void)
181{
182	int cpu = get_cpu();
183	cpumask_t *mask;
184	bool stale_cpu;
185
186	/*
187	 * Mark every other hart's icache as needing a flush for
188	 * this MM. Maintain the previous value of the current
189	 * cpu to handle the case when this function is called
190	 * concurrently on different harts.
191	 */
192	mask = &current->mm->context.icache_stale_mask;
193	stale_cpu = cpumask_test_cpu(cpu, mask);
194
195	cpumask_setall(mask);
196	__assign_cpu(cpu, mask, stale_cpu);
197	put_cpu();
198}
199#endif
200
201/**
202 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in
203 * userspace.
204 * @ctx: Set the type of icache flushing instructions permitted/prohibited in
205 *	 userspace. Supported values described below.
206 *
207 * Supported values for ctx:
208 *
209 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space.
210 *
211 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in
212 *   a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``.
213 *   Therefore, caution must be taken; use this flag only when you can guarantee
214 *   that no thread in the process will emit fence.i from this point onward.
215 *
216 * @scope: Set scope of where icache flushing instructions are allowed to be
217 *	   emitted. Supported values described below.
218 *
219 * Supported values for scope:
220 *
221 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process
222 *                               is coherent with instruction storage upon
223 *                               migration.
224 *
225 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is
226 *                              coherent with instruction storage upon
227 *                              migration.
228 *
229 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are
230 * permitted to emit icache flushing instructions. Whenever any thread in the
231 * process is migrated, the corresponding hart's icache will be guaranteed to be
232 * consistent with instruction storage. This does not enforce any guarantees
233 * outside of migration. If a thread modifies an instruction that another thread
234 * may attempt to execute, the other thread must still emit an icache flushing
235 * instruction before attempting to execute the potentially modified
236 * instruction. This must be performed by the user-space program.
237 *
238 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the
239 * thread calling this function is permitted to emit icache flushing
240 * instructions. When the thread is migrated, the corresponding hart's icache
241 * will be guaranteed to be consistent with instruction storage.
242 *
243 * On kernels configured without SMP, this function is a nop as migrations
244 * across harts will not occur.
245 */
246int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope)
247{
248#ifdef CONFIG_SMP
249	switch (ctx) {
250	case PR_RISCV_CTX_SW_FENCEI_ON:
251		switch (scope) {
252		case PR_RISCV_SCOPE_PER_PROCESS:
253			current->mm->context.force_icache_flush = true;
254			break;
255		case PR_RISCV_SCOPE_PER_THREAD:
256			current->thread.force_icache_flush = true;
257			break;
258		default:
259			return -EINVAL;
260		}
261		break;
262	case PR_RISCV_CTX_SW_FENCEI_OFF:
263		switch (scope) {
264		case PR_RISCV_SCOPE_PER_PROCESS:
265			set_icache_stale_mask();
266			current->mm->context.force_icache_flush = false;
267			break;
268		case PR_RISCV_SCOPE_PER_THREAD:
269			set_icache_stale_mask();
270			current->thread.force_icache_flush = false;
271			break;
272		default:
273			return -EINVAL;
274		}
275		break;
276	default:
277		return -EINVAL;
278	}
279	return 0;
280#else
281	switch (ctx) {
282	case PR_RISCV_CTX_SW_FENCEI_ON:
283	case PR_RISCV_CTX_SW_FENCEI_OFF:
284		return 0;
285	default:
286		return -EINVAL;
287	}
288#endif
289}