errata.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2021 Heiko Stuebner <heiko@sntech.de>
  4 */
  5
  6#include <linux/bug.h>
  7#include <linux/kernel.h>
  8#include <linux/memory.h>
  9#include <linux/module.h>
 10#include <linux/string.h>
 11#include <linux/uaccess.h>
 12#include <asm/alternative.h>
 13#include <asm/bugs.h>
 14#include <asm/cacheflush.h>
 15#include <asm/cpufeature.h>
 16#include <asm/dma-noncoherent.h>
 17#include <asm/errata_list.h>
 18#include <asm/hwprobe.h>
 19#include <asm/io.h>
 20#include <asm/text-patching.h>
 21#include <asm/vendorid_list.h>
 22#include <asm/vendor_extensions.h>
 23
 24#define CSR_TH_SXSTATUS		0x5c0
 25#define SXSTATUS_MAEE		_AC(0x200000, UL)
 26
 27static bool errata_probe_mae(unsigned int stage,
 28			     unsigned long arch_id, unsigned long impid)
 29{
 30	if (!IS_ENABLED(CONFIG_ERRATA_THEAD_MAE))
 31		return false;
 32
 33	if (arch_id != 0 || impid != 0)
 34		return false;
 35
 36	if (stage != RISCV_ALTERNATIVES_EARLY_BOOT &&
 37	    stage != RISCV_ALTERNATIVES_MODULE)
 38		return false;
 39
 40	if (!(csr_read(CSR_TH_SXSTATUS) & SXSTATUS_MAEE))
 41		return false;
 42
 43	return true;
 44}
 45
 46/*
 47 * th.dcache.ipa rs1 (invalidate, physical address)
 48 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
 49 *   0000001    01010      rs1       000      00000  0001011
 50 * th.dcache.iva rs1 (invalidate, virtual address)
 51 *   0000001    00110      rs1       000      00000  0001011
 52 *
 53 * th.dcache.cpa rs1 (clean, physical address)
 54 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
 55 *   0000001    01001      rs1       000      00000  0001011
 56 * th.dcache.cva rs1 (clean, virtual address)
 57 *   0000001    00101      rs1       000      00000  0001011
 58 *
 59 * th.dcache.cipa rs1 (clean then invalidate, physical address)
 60 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
 61 *   0000001    01011      rs1       000      00000  0001011
 62 * th.dcache.civa rs1 (clean then invalidate, virtual address)
 63 *   0000001    00111      rs1       000      00000  0001011
 64 *
 65 * th.sync.s (make sure all cache operations finished)
 66 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
 67 *   0000000    11001     00000      000      00000  0001011
 68 */
 69#define THEAD_INVAL_A0	".long 0x02a5000b"
 70#define THEAD_CLEAN_A0	".long 0x0295000b"
 71#define THEAD_FLUSH_A0	".long 0x02b5000b"
 72#define THEAD_SYNC_S	".long 0x0190000b"
 73
 74#define THEAD_CMO_OP(_op, _start, _size, _cachesize)			\
 75asm volatile("mv a0, %1\n\t"						\
 76	     "j 2f\n\t"							\
 77	     "3:\n\t"							\
 78	     THEAD_##_op##_A0 "\n\t"					\
 79	     "add a0, a0, %0\n\t"					\
 80	     "2:\n\t"							\
 81	     "bltu a0, %2, 3b\n\t"					\
 82	     THEAD_SYNC_S						\
 83	     : : "r"(_cachesize),					\
 84		 "r"((unsigned long)(_start) & ~((_cachesize) - 1UL)),	\
 85		 "r"((unsigned long)(_start) + (_size))			\
 86	     : "a0")
 87
 88static void thead_errata_cache_inv(phys_addr_t paddr, size_t size)
 89{
 90	THEAD_CMO_OP(INVAL, paddr, size, riscv_cbom_block_size);
 91}
 92
 93static void thead_errata_cache_wback(phys_addr_t paddr, size_t size)
 94{
 95	THEAD_CMO_OP(CLEAN, paddr, size, riscv_cbom_block_size);
 96}
 97
 98static void thead_errata_cache_wback_inv(phys_addr_t paddr, size_t size)
 99{
100	THEAD_CMO_OP(FLUSH, paddr, size, riscv_cbom_block_size);
101}
102
103static const struct riscv_nonstd_cache_ops thead_errata_cmo_ops = {
104	.wback = &thead_errata_cache_wback,
105	.inv = &thead_errata_cache_inv,
106	.wback_inv = &thead_errata_cache_wback_inv,
107};
108
109static bool errata_probe_cmo(unsigned int stage,
110			     unsigned long arch_id, unsigned long impid)
111{
112	if (!IS_ENABLED(CONFIG_ERRATA_THEAD_CMO))
113		return false;
114
115	if (arch_id != 0 || impid != 0)
116		return false;
117
118	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
119		return false;
120
121	if (stage == RISCV_ALTERNATIVES_BOOT) {
122		riscv_cbom_block_size = L1_CACHE_BYTES;
123		riscv_noncoherent_supported();
124		riscv_noncoherent_register_cache_ops(&thead_errata_cmo_ops);
125	}
126
127	return true;
128}
129
130static bool errata_probe_pmu(unsigned int stage,
131			     unsigned long arch_id, unsigned long impid)
132{
133	if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PMU))
134		return false;
135
136	/* target-c9xx cores report arch_id and impid as 0 */
137	if (arch_id != 0 || impid != 0)
138		return false;
139
140	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
141		return false;
142
143	return true;
144}
145
146static bool errata_probe_ghostwrite(unsigned int stage,
147				    unsigned long arch_id, unsigned long impid)
148{
149	if (!IS_ENABLED(CONFIG_ERRATA_THEAD_GHOSTWRITE))
150		return false;
151
152	/*
153	 * target-c9xx cores report arch_id and impid as 0
154	 *
155	 * While ghostwrite may not affect all c9xx cores that implement
156	 * xtheadvector, there is no futher granularity than c9xx. Assume
157	 * vulnerable for this entire class of processors when xtheadvector is
158	 * enabled.
159	 */
160	if (arch_id != 0 || impid != 0)
161		return false;
162
163	if (stage != RISCV_ALTERNATIVES_EARLY_BOOT)
164		return false;
165
166	ghostwrite_set_vulnerable();
167
168	return true;
169}
170
171static u32 thead_errata_probe(unsigned int stage,
172			      unsigned long archid, unsigned long impid)
173{
174	u32 cpu_req_errata = 0;
175
176	if (errata_probe_mae(stage, archid, impid))
177		cpu_req_errata |= BIT(ERRATA_THEAD_MAE);
178
179	errata_probe_cmo(stage, archid, impid);
180
181	if (errata_probe_pmu(stage, archid, impid))
182		cpu_req_errata |= BIT(ERRATA_THEAD_PMU);
183
184	errata_probe_ghostwrite(stage, archid, impid);
185
186	return cpu_req_errata;
187}
188
189void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
190			     unsigned long archid, unsigned long impid,
191			     unsigned int stage)
192{
193	struct alt_entry *alt;
194	u32 cpu_req_errata = thead_errata_probe(stage, archid, impid);
195	u32 tmp;
196	void *oldptr, *altptr;
197
198	BUILD_BUG_ON(ERRATA_THEAD_NUMBER >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE);
199
200	for (alt = begin; alt < end; alt++) {
201		if (alt->vendor_id != THEAD_VENDOR_ID)
202			continue;
203		if (alt->patch_id >= ERRATA_THEAD_NUMBER)
204			continue;
205
206		tmp = (1U << alt->patch_id);
207		if (cpu_req_errata & tmp) {
208			oldptr = ALT_OLD_PTR(alt);
209			altptr = ALT_ALT_PTR(alt);
210
211			/* On vm-alternatives, the mmu isn't running yet */
212			if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) {
213				memcpy(oldptr, altptr, alt->alt_len);
214			} else {
215				mutex_lock(&text_mutex);
216				patch_text_nosync(oldptr, altptr, alt->alt_len);
217				mutex_unlock(&text_mutex);
218			}
219		}
220	}
221
222	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
223		local_flush_icache_all();
224}