cpu_errata.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Contains CPU specific errata definitions
  4 *
  5 * Copyright (C) 2014 ARM Ltd.
  6 */
  7
  8#include <linux/arm-smccc.h>
  9#include <linux/types.h>
 10#include <linux/cpu.h>
 11#include <asm/cpu.h>
 12#include <asm/cputype.h>
 13#include <asm/cpufeature.h>
 14#include <asm/kvm_asm.h>
 15#include <asm/smp_plat.h>
 16
 17static u64 target_impl_cpu_num;
 18static struct target_impl_cpu *target_impl_cpus;
 19
 20bool cpu_errata_set_target_impl(u64 num, void *impl_cpus)
 21{
 22	if (target_impl_cpu_num || !num || !impl_cpus)
 23		return false;
 24
 25	target_impl_cpu_num = num;
 26	target_impl_cpus = impl_cpus;
 27	return true;
 28}
 29
 30static inline bool is_midr_in_range(struct midr_range const *range)
 31{
 32	int i;
 33
 34	if (!target_impl_cpu_num)
 35		return midr_is_cpu_model_range(read_cpuid_id(), range->model,
 36					       range->rv_min, range->rv_max);
 37
 38	for (i = 0; i < target_impl_cpu_num; i++) {
 39		if (midr_is_cpu_model_range(target_impl_cpus[i].midr,
 40					    range->model,
 41					    range->rv_min, range->rv_max))
 42			return true;
 43	}
 44	return false;
 45}
 46
 47bool is_midr_in_range_list(struct midr_range const *ranges)
 48{
 49	while (ranges->model)
 50		if (is_midr_in_range(ranges++))
 51			return true;
 52	return false;
 53}
 54EXPORT_SYMBOL_GPL(is_midr_in_range_list);
 55
 56static bool __maybe_unused
 57__is_affected_midr_range(const struct arm64_cpu_capabilities *entry,
 58			 u32 midr, u32 revidr)
 59{
 60	const struct arm64_midr_revidr *fix;
 61	if (!is_midr_in_range(&entry->midr_range))
 62		return false;
 63
 64	midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
 65	for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
 66		if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
 67			return false;
 68	return true;
 69}
 70
 71static bool __maybe_unused
 72is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
 73{
 74	int i;
 75
 76	if (!target_impl_cpu_num) {
 77		WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 78		return __is_affected_midr_range(entry, read_cpuid_id(),
 79						read_cpuid(REVIDR_EL1));
 80	}
 81
 82	for (i = 0; i < target_impl_cpu_num; i++) {
 83		if (__is_affected_midr_range(entry, target_impl_cpus[i].midr,
 84					     target_impl_cpus[i].midr))
 85			return true;
 86	}
 87	return false;
 88}
 89
 90static bool __maybe_unused
 91is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
 92			    int scope)
 93{
 94	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 95	return is_midr_in_range_list(entry->midr_range_list);
 96}
 97
 98static bool __maybe_unused
 99is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
100{
101	u32 model;
102
103	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
104
105	model = read_cpuid_id();
106	model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
107		 MIDR_ARCHITECTURE_MASK;
108
109	return model == entry->midr_range.model;
110}
111
112static bool
113has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
114			  int scope)
115{
116	u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
117	u64 sys = arm64_ftr_reg_ctrel0.sys_val & mask;
118	u64 ctr_raw, ctr_real;
119
120	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
121
122	/*
123	 * We want to make sure that all the CPUs in the system expose
124	 * a consistent CTR_EL0 to make sure that applications behaves
125	 * correctly with migration.
126	 *
127	 * If a CPU has CTR_EL0.IDC but does not advertise it via CTR_EL0 :
128	 *
129	 * 1) It is safe if the system doesn't support IDC, as CPU anyway
130	 *    reports IDC = 0, consistent with the rest.
131	 *
132	 * 2) If the system has IDC, it is still safe as we trap CTR_EL0
133	 *    access on this CPU via the ARM64_HAS_CACHE_IDC capability.
134	 *
135	 * So, we need to make sure either the raw CTR_EL0 or the effective
136	 * CTR_EL0 matches the system's copy to allow a secondary CPU to boot.
137	 */
138	ctr_raw = read_cpuid_cachetype() & mask;
139	ctr_real = read_cpuid_effective_cachetype() & mask;
140
141	return (ctr_real != sys) && (ctr_raw != sys);
142}
143
144static void
145cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap)
146{
147	u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
148	bool enable_uct_trap = false;
149
150	/* Trap CTR_EL0 access on this CPU, only if it has a mismatch */
151	if ((read_cpuid_cachetype() & mask) !=
152	    (arm64_ftr_reg_ctrel0.sys_val & mask))
153		enable_uct_trap = true;
154
155	/* ... or if the system is affected by an erratum */
156	if (cap->capability == ARM64_WORKAROUND_1542419)
157		enable_uct_trap = true;
158
159	if (enable_uct_trap)
160		sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
161}
162
163#ifdef CONFIG_ARM64_ERRATUM_1463225
164static bool
165has_cortex_a76_erratum_1463225(const struct arm64_cpu_capabilities *entry,
166			       int scope)
167{
168	return is_affected_midr_range_list(entry, scope) && is_kernel_in_hyp_mode();
169}
170#endif
171
172static void __maybe_unused
173cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
174{
175	sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0);
176}
177
178#define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)	\
179	.matches = is_affected_midr_range,			\
180	.midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
181
182#define CAP_MIDR_ALL_VERSIONS(model)					\
183	.matches = is_affected_midr_range,				\
184	.midr_range = MIDR_ALL_VERSIONS(model)
185
186#define MIDR_FIXED(rev, revidr_mask) \
187	.fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
188
189#define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)		\
190	.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,				\
191	CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
192
193#define CAP_MIDR_RANGE_LIST(list)				\
194	.matches = is_affected_midr_range_list,			\
195	.midr_range_list = list
196
197/* Errata affecting a range of revisions of  given model variant */
198#define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)	 \
199	ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
200
201/* Errata affecting a single variant/revision of a model */
202#define ERRATA_MIDR_REV(model, var, rev)	\
203	ERRATA_MIDR_RANGE(model, var, rev, var, rev)
204
205/* Errata affecting all variants/revisions of a given a model */
206#define ERRATA_MIDR_ALL_VERSIONS(model)				\
207	.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,			\
208	CAP_MIDR_ALL_VERSIONS(model)
209
210/* Errata affecting a list of midr ranges, with same work around */
211#define ERRATA_MIDR_RANGE_LIST(midr_list)			\
212	.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,			\
213	CAP_MIDR_RANGE_LIST(midr_list)
214
215static const __maybe_unused struct midr_range tx2_family_cpus[] = {
216	MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
217	MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
218	{},
219};
220
221static bool __maybe_unused
222needs_tx2_tvm_workaround(const struct arm64_cpu_capabilities *entry,
223			 int scope)
224{
225	int i;
226
227	if (!is_affected_midr_range_list(entry, scope) ||
228	    !is_hyp_mode_available())
229		return false;
230
231	for_each_possible_cpu(i) {
232		if (MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0) != 0)
233			return true;
234	}
235
236	return false;
237}
238
239static bool __maybe_unused
240has_neoverse_n1_erratum_1542419(const struct arm64_cpu_capabilities *entry,
241				int scope)
242{
243	bool has_dic = read_cpuid_cachetype() & BIT(CTR_EL0_DIC_SHIFT);
244	const struct midr_range range = MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1);
245
246	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
247	return is_midr_in_range(&range) && has_dic;
248}
249
250static const struct midr_range impdef_pmuv3_cpus[] = {
251	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM),
252	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM),
253	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO),
254	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_PRO),
255	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_MAX),
256	MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_MAX),
257	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD),
258	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE),
259	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_PRO),
260	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_PRO),
261	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX),
262	MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX),
263	{},
264};
265
266static bool has_impdef_pmuv3(const struct arm64_cpu_capabilities *entry, int scope)
267{
268	u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
269	unsigned int pmuver;
270
271	if (!is_kernel_in_hyp_mode())
272		return false;
273
274	pmuver = cpuid_feature_extract_unsigned_field(dfr0,
275						      ID_AA64DFR0_EL1_PMUVer_SHIFT);
276	if (pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
277		return false;
278
279	return is_midr_in_range_list(impdef_pmuv3_cpus);
280}
281
282static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *__unused)
283{
284	sysreg_clear_set_s(SYS_HACR_EL2, 0, BIT(56));
285}
286
287#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
288static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = {
289#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
290	{
291		ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0)
292	},
293	{
294		.midr_range.model = MIDR_QCOM_KRYO,
295		.matches = is_kryo_midr,
296	},
297#endif
298#ifdef CONFIG_ARM64_ERRATUM_1286807
299	{
300		ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 0),
301	},
302	{
303		/* Kryo4xx Gold (rcpe to rfpe) => (r0p0 to r3p0) */
304		ERRATA_MIDR_RANGE(MIDR_QCOM_KRYO_4XX_GOLD, 0xc, 0xe, 0xf, 0xe),
305	},
306#endif
307#ifdef CONFIG_ARM64_ERRATUM_2441007
308	{
309		ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
310	},
311#endif
312#ifdef CONFIG_ARM64_ERRATUM_2441009
313	{
314		/* Cortex-A510 r0p0 -> r1p1. Fixed in r1p2 */
315		ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1),
316	},
317#endif
318	{},
319};
320#endif
321
322#ifdef CONFIG_CAVIUM_ERRATUM_23154
323static const struct midr_range cavium_erratum_23154_cpus[] = {
324	MIDR_ALL_VERSIONS(MIDR_THUNDERX),
325	MIDR_ALL_VERSIONS(MIDR_THUNDERX_81XX),
326	MIDR_ALL_VERSIONS(MIDR_THUNDERX_83XX),
327	MIDR_ALL_VERSIONS(MIDR_OCTX2_98XX),
328	MIDR_ALL_VERSIONS(MIDR_OCTX2_96XX),
329	MIDR_ALL_VERSIONS(MIDR_OCTX2_95XX),
330	MIDR_ALL_VERSIONS(MIDR_OCTX2_95XXN),
331	MIDR_ALL_VERSIONS(MIDR_OCTX2_95XXMM),
332	MIDR_ALL_VERSIONS(MIDR_OCTX2_95XXO),
333	{},
334};
335#endif
336
337#ifdef CONFIG_CAVIUM_ERRATUM_27456
338static const struct midr_range cavium_erratum_27456_cpus[] = {
339	/* Cavium ThunderX, T88 pass 1.x - 2.1 */
340	MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 1),
341	/* Cavium ThunderX, T81 pass 1.0 */
342	MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
343	{},
344};
345#endif
346
347#ifdef CONFIG_CAVIUM_ERRATUM_30115
348static const struct midr_range cavium_erratum_30115_cpus[] = {
349	/* Cavium ThunderX, T88 pass 1.x - 2.2 */
350	MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 2),
351	/* Cavium ThunderX, T81 pass 1.0 - 1.2 */
352	MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
353	/* Cavium ThunderX, T83 pass 1.0 */
354	MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
355	{},
356};
357#endif
358
359#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
360static const struct arm64_cpu_capabilities qcom_erratum_1003_list[] = {
361	{
362		ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
363	},
364	{
365		.midr_range.model = MIDR_QCOM_KRYO,
366		.matches = is_kryo_midr,
367	},
368	{},
369};
370#endif
371
372#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
373static const struct midr_range workaround_clean_cache[] = {
374#if	defined(CONFIG_ARM64_ERRATUM_826319) || \
375	defined(CONFIG_ARM64_ERRATUM_827319) || \
376	defined(CONFIG_ARM64_ERRATUM_824069)
377	/* Cortex-A53 r0p[012]: ARM errata 826319, 827319, 824069 */
378	MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
379#endif
380#ifdef	CONFIG_ARM64_ERRATUM_819472
381	/* Cortex-A53 r0p[01] : ARM errata 819472 */
382	MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
383#endif
384	{},
385};
386#endif
387
388#ifdef CONFIG_ARM64_ERRATUM_1418040
389/*
390 * - 1188873 affects r0p0 to r2p0
391 * - 1418040 affects r0p0 to r3p1
392 */
393static const struct midr_range erratum_1418040_list[] = {
394	/* Cortex-A76 r0p0 to r3p1 */
395	MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 1),
396	/* Neoverse-N1 r0p0 to r3p1 */
397	MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 0, 3, 1),
398	/* Kryo4xx Gold (rcpe to rfpf) => (r0p0 to r3p1) */
399	MIDR_RANGE(MIDR_QCOM_KRYO_4XX_GOLD, 0xc, 0xe, 0xf, 0xf),
400	{},
401};
402#endif
403
404#ifdef CONFIG_ARM64_ERRATUM_845719
405static const struct midr_range erratum_845719_list[] = {
406	/* Cortex-A53 r0p[01234] */
407	MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
408	/* Brahma-B53 r0p[0] */
409	MIDR_REV(MIDR_BRAHMA_B53, 0, 0),
410	/* Kryo2XX Silver rAp4 */
411	MIDR_REV(MIDR_QCOM_KRYO_2XX_SILVER, 0xa, 0x4),
412	{},
413};
414#endif
415
416#ifdef CONFIG_ARM64_ERRATUM_843419
417static const struct arm64_cpu_capabilities erratum_843419_list[] = {
418	{
419		/* Cortex-A53 r0p[01234] */
420		.matches = is_affected_midr_range,
421		ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
422		MIDR_FIXED(0x4, BIT(8)),
423	},
424	{
425		/* Brahma-B53 r0p[0] */
426		.matches = is_affected_midr_range,
427		ERRATA_MIDR_REV(MIDR_BRAHMA_B53, 0, 0),
428	},
429	{},
430};
431#endif
432
433#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_AT
434static const struct midr_range erratum_speculative_at_list[] = {
435#ifdef CONFIG_ARM64_ERRATUM_1165522
436	/* Cortex A76 r0p0 to r2p0 */
437	MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
438#endif
439#ifdef CONFIG_ARM64_ERRATUM_1319367
440	MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
441	MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
442#endif
443#ifdef CONFIG_ARM64_ERRATUM_1530923
444	/* Cortex A55 r0p0 to r2p0 */
445	MIDR_RANGE(MIDR_CORTEX_A55, 0, 0, 2, 0),
446	/* Kryo4xx Silver (rdpe => r1p0) */
447	MIDR_REV(MIDR_QCOM_KRYO_4XX_SILVER, 0xd, 0xe),
448#endif
449	{},
450};
451#endif
452
453#ifdef CONFIG_ARM64_ERRATUM_1463225
454static const struct midr_range erratum_1463225[] = {
455	/* Cortex-A76 r0p0 - r3p1 */
456	MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 1),
457	/* Kryo4xx Gold (rcpe to rfpf) => (r0p0 to r3p1) */
458	MIDR_RANGE(MIDR_QCOM_KRYO_4XX_GOLD, 0xc, 0xe, 0xf, 0xf),
459	{},
460};
461#endif
462
463#ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE
464static const struct midr_range trbe_overwrite_fill_mode_cpus[] = {
465#ifdef CONFIG_ARM64_ERRATUM_2139208
466	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
467	MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100),
468#endif
469#ifdef CONFIG_ARM64_ERRATUM_2119858
470	MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
471	MIDR_RANGE(MIDR_CORTEX_X2, 0, 0, 2, 0),
472#endif
473	{},
474};
475#endif	/* CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE */
476
477#ifdef CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE
478static const struct midr_range tsb_flush_fail_cpus[] = {
479#ifdef CONFIG_ARM64_ERRATUM_2067961
480	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
481	MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100),
482#endif
483#ifdef CONFIG_ARM64_ERRATUM_2054223
484	MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
485#endif
486	{},
487};
488#endif	/* CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE */
489
490#ifdef CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE
491static struct midr_range trbe_write_out_of_range_cpus[] = {
492#ifdef CONFIG_ARM64_ERRATUM_2253138
493	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
494	MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100),
495#endif
496#ifdef CONFIG_ARM64_ERRATUM_2224489
497	MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
498	MIDR_RANGE(MIDR_CORTEX_X2, 0, 0, 2, 0),
499#endif
500	{},
501};
502#endif /* CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE */
503
504#ifdef CONFIG_ARM64_ERRATUM_1742098
505static struct midr_range broken_aarch32_aes[] = {
506	MIDR_RANGE(MIDR_CORTEX_A57, 0, 1, 0xf, 0xf),
507	MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
508	{},
509};
510#endif /* CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE */
511
512#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
513static const struct midr_range erratum_spec_unpriv_load_list[] = {
514#ifdef CONFIG_ARM64_ERRATUM_3117295
515	MIDR_ALL_VERSIONS(MIDR_CORTEX_A510),
516#endif
517#ifdef CONFIG_ARM64_ERRATUM_2966298
518	/* Cortex-A520 r0p0 to r0p1 */
519	MIDR_REV_RANGE(MIDR_CORTEX_A520, 0, 0, 1),
520#endif
521	{},
522};
523#endif
524
525#ifdef CONFIG_ARM64_ERRATUM_3194386
526static const struct midr_range erratum_spec_ssbs_list[] = {
527	MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
528	MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
529	MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
530	MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
531	MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
532	MIDR_ALL_VERSIONS(MIDR_CORTEX_A715),
533	MIDR_ALL_VERSIONS(MIDR_CORTEX_A720),
534	MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE),
535	MIDR_ALL_VERSIONS(MIDR_CORTEX_A725),
536	MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
537	MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C),
538	MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
539	MIDR_ALL_VERSIONS(MIDR_CORTEX_X3),
540	MIDR_ALL_VERSIONS(MIDR_CORTEX_X4),
541	MIDR_ALL_VERSIONS(MIDR_CORTEX_X925),
542	MIDR_ALL_VERSIONS(MIDR_MICROSOFT_AZURE_COBALT_100),
543	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
544	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
545	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N3),
546	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
547	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2),
548	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3),
549	MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3AE),
550	{}
551};
552#endif
553
554#ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38
555static const struct midr_range erratum_ac03_cpu_38_list[] = {
556	MIDR_ALL_VERSIONS(MIDR_AMPERE1),
557	MIDR_ALL_VERSIONS(MIDR_AMPERE1A),
558	{},
559};
560#endif
561
562#ifdef CONFIG_AMPERE_ERRATUM_AC04_CPU_23
563static const struct midr_range erratum_ac04_cpu_23_list[] = {
564	MIDR_ALL_VERSIONS(MIDR_AMPERE1A),
565	{},
566};
567#endif
568
569const struct arm64_cpu_capabilities arm64_errata[] = {
570#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
571	{
572		.desc = "ARM errata 826319, 827319, 824069, or 819472",
573		.capability = ARM64_WORKAROUND_CLEAN_CACHE,
574		ERRATA_MIDR_RANGE_LIST(workaround_clean_cache),
575		.cpu_enable = cpu_enable_cache_maint_trap,
576	},
577#endif
578#ifdef CONFIG_ARM64_ERRATUM_832075
579	{
580	/* Cortex-A57 r0p0 - r1p2 */
581		.desc = "ARM erratum 832075",
582		.capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
583		ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
584				  0, 0,
585				  1, 2),
586	},
587#endif
588#ifdef CONFIG_ARM64_ERRATUM_834220
589	{
590	/* Cortex-A57 r0p0 - r1p2 */
591		.desc = "ARM erratum 834220",
592		.capability = ARM64_WORKAROUND_834220,
593		ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
594				  0, 0,
595				  1, 2),
596	},
597#endif
598#ifdef CONFIG_ARM64_ERRATUM_843419
599	{
600		.desc = "ARM erratum 843419",
601		.capability = ARM64_WORKAROUND_843419,
602		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
603		.matches = cpucap_multi_entry_cap_matches,
604		.match_list = erratum_843419_list,
605	},
606#endif
607#ifdef CONFIG_ARM64_ERRATUM_845719
608	{
609		.desc = "ARM erratum 845719",
610		.capability = ARM64_WORKAROUND_845719,
611		ERRATA_MIDR_RANGE_LIST(erratum_845719_list),
612	},
613#endif
614#ifdef CONFIG_CAVIUM_ERRATUM_23154
615	{
616		.desc = "Cavium errata 23154 and 38545",
617		.capability = ARM64_WORKAROUND_CAVIUM_23154,
618		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
619		ERRATA_MIDR_RANGE_LIST(cavium_erratum_23154_cpus),
620	},
621#endif
622#ifdef CONFIG_CAVIUM_ERRATUM_27456
623	{
624		.desc = "Cavium erratum 27456",
625		.capability = ARM64_WORKAROUND_CAVIUM_27456,
626		ERRATA_MIDR_RANGE_LIST(cavium_erratum_27456_cpus),
627	},
628#endif
629#ifdef CONFIG_CAVIUM_ERRATUM_30115
630	{
631		.desc = "Cavium erratum 30115",
632		.capability = ARM64_WORKAROUND_CAVIUM_30115,
633		ERRATA_MIDR_RANGE_LIST(cavium_erratum_30115_cpus),
634	},
635#endif
636	{
637		.desc = "Mismatched cache type (CTR_EL0)",
638		.capability = ARM64_MISMATCHED_CACHE_TYPE,
639		.matches = has_mismatched_cache_type,
640		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
641		.cpu_enable = cpu_enable_trap_ctr_access,
642	},
643#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
644	{
645		.desc = "Qualcomm Technologies Falkor/Kryo erratum 1003",
646		.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
647		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
648		.matches = cpucap_multi_entry_cap_matches,
649		.match_list = qcom_erratum_1003_list,
650	},
651#endif
652#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
653	{
654		.desc = "Qualcomm erratum 1009, or ARM erratum 1286807, 2441009",
655		.capability = ARM64_WORKAROUND_REPEAT_TLBI,
656		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
657		.matches = cpucap_multi_entry_cap_matches,
658		.match_list = arm64_repeat_tlbi_list,
659	},
660#endif
661#ifdef CONFIG_ARM64_ERRATUM_858921
662	{
663	/* Cortex-A73 all versions */
664		.desc = "ARM erratum 858921",
665		.capability = ARM64_WORKAROUND_858921,
666		ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
667	},
668#endif
669	{
670		.desc = "Spectre-v2",
671		.capability = ARM64_SPECTRE_V2,
672		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
673		.matches = has_spectre_v2,
674		.cpu_enable = spectre_v2_enable_mitigation,
675	},
676#ifdef CONFIG_RANDOMIZE_BASE
677	{
678	/* Must come after the Spectre-v2 entry */
679		.desc = "Spectre-v3a",
680		.capability = ARM64_SPECTRE_V3A,
681		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
682		.matches = has_spectre_v3a,
683		.cpu_enable = spectre_v3a_enable_mitigation,
684	},
685#endif
686	{
687		.desc = "Spectre-v4",
688		.capability = ARM64_SPECTRE_V4,
689		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
690		.matches = has_spectre_v4,
691		.cpu_enable = spectre_v4_enable_mitigation,
692	},
693	{
694		.desc = "Spectre-BHB",
695		.capability = ARM64_SPECTRE_BHB,
696		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
697		.matches = is_spectre_bhb_affected,
698		.cpu_enable = spectre_bhb_enable_mitigation,
699	},
700#ifdef CONFIG_ARM64_ERRATUM_1418040
701	{
702		.desc = "ARM erratum 1418040",
703		.capability = ARM64_WORKAROUND_1418040,
704		ERRATA_MIDR_RANGE_LIST(erratum_1418040_list),
705		/*
706		 * We need to allow affected CPUs to come in late, but
707		 * also need the non-affected CPUs to be able to come
708		 * in at any point in time. Wonderful.
709		 */
710		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
711	},
712#endif
713#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_AT
714	{
715		.desc = "ARM errata 1165522, 1319367, or 1530923",
716		.capability = ARM64_WORKAROUND_SPECULATIVE_AT,
717		ERRATA_MIDR_RANGE_LIST(erratum_speculative_at_list),
718	},
719#endif
720#ifdef CONFIG_ARM64_ERRATUM_1463225
721	{
722		.desc = "ARM erratum 1463225",
723		.capability = ARM64_WORKAROUND_1463225,
724		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
725		.matches = has_cortex_a76_erratum_1463225,
726		.midr_range_list = erratum_1463225,
727	},
728#endif
729#ifdef CONFIG_CAVIUM_TX2_ERRATUM_219
730	{
731		.desc = "Cavium ThunderX2 erratum 219 (KVM guest sysreg trapping)",
732		.capability = ARM64_WORKAROUND_CAVIUM_TX2_219_TVM,
733		ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
734		.matches = needs_tx2_tvm_workaround,
735	},
736	{
737		.desc = "Cavium ThunderX2 erratum 219 (PRFM removal)",
738		.capability = ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM,
739		ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
740	},
741#endif
742#ifdef CONFIG_ARM64_ERRATUM_1542419
743	{
744		/* we depend on the firmware portion for correctness */
745		.desc = "ARM erratum 1542419 (kernel portion)",
746		.capability = ARM64_WORKAROUND_1542419,
747		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
748		.matches = has_neoverse_n1_erratum_1542419,
749		.cpu_enable = cpu_enable_trap_ctr_access,
750	},
751#endif
752#ifdef CONFIG_ARM64_ERRATUM_1508412
753	{
754		/* we depend on the firmware portion for correctness */
755		.desc = "ARM erratum 1508412 (kernel portion)",
756		.capability = ARM64_WORKAROUND_1508412,
757		ERRATA_MIDR_RANGE(MIDR_CORTEX_A77,
758				  0, 0,
759				  1, 0),
760	},
761#endif
762#ifdef CONFIG_NVIDIA_CARMEL_CNP_ERRATUM
763	{
764		/* NVIDIA Carmel */
765		.desc = "NVIDIA Carmel CNP erratum",
766		.capability = ARM64_WORKAROUND_NVIDIA_CARMEL_CNP,
767		ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
768	},
769#endif
770#ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE
771	{
772		/*
773		 * The erratum work around is handled within the TRBE
774		 * driver and can be applied per-cpu. So, we can allow
775		 * a late CPU to come online with this erratum.
776		 */
777		.desc = "ARM erratum 2119858 or 2139208",
778		.capability = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
779		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
780		CAP_MIDR_RANGE_LIST(trbe_overwrite_fill_mode_cpus),
781	},
782#endif
783#ifdef CONFIG_ARM64_WORKAROUND_TSB_FLUSH_FAILURE
784	{
785		.desc = "ARM erratum 2067961 or 2054223",
786		.capability = ARM64_WORKAROUND_TSB_FLUSH_FAILURE,
787		ERRATA_MIDR_RANGE_LIST(tsb_flush_fail_cpus),
788	},
789#endif
790#ifdef CONFIG_ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE
791	{
792		.desc = "ARM erratum 2253138 or 2224489",
793		.capability = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
794		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
795		CAP_MIDR_RANGE_LIST(trbe_write_out_of_range_cpus),
796	},
797#endif
798#ifdef CONFIG_ARM64_ERRATUM_2645198
799	{
800		.desc = "ARM erratum 2645198",
801		.capability = ARM64_WORKAROUND_2645198,
802		ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A715)
803	},
804#endif
805#ifdef CONFIG_ARM64_ERRATUM_2077057
806	{
807		.desc = "ARM erratum 2077057",
808		.capability = ARM64_WORKAROUND_2077057,
809		ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2),
810	},
811#endif
812#ifdef CONFIG_ARM64_ERRATUM_2064142
813	{
814		.desc = "ARM erratum 2064142",
815		.capability = ARM64_WORKAROUND_2064142,
816
817		/* Cortex-A510 r0p0 - r0p2 */
818		ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2)
819	},
820#endif
821#ifdef CONFIG_ARM64_ERRATUM_2457168
822	{
823		.desc = "ARM erratum 2457168",
824		.capability = ARM64_WORKAROUND_2457168,
825		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
826
827		/* Cortex-A510 r0p0-r1p1 */
828		CAP_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1)
829	},
830#endif
831#ifdef CONFIG_ARM64_ERRATUM_2038923
832	{
833		.desc = "ARM erratum 2038923",
834		.capability = ARM64_WORKAROUND_2038923,
835
836		/* Cortex-A510 r0p0 - r0p2 */
837		ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2)
838	},
839#endif
840#ifdef CONFIG_ARM64_ERRATUM_1902691
841	{
842		.desc = "ARM erratum 1902691",
843		.capability = ARM64_WORKAROUND_1902691,
844
845		/* Cortex-A510 r0p0 - r0p1 */
846		ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 1)
847	},
848#endif
849#ifdef CONFIG_ARM64_ERRATUM_1742098
850	{
851		.desc = "ARM erratum 1742098",
852		.capability = ARM64_WORKAROUND_1742098,
853		CAP_MIDR_RANGE_LIST(broken_aarch32_aes),
854		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
855	},
856#endif
857#ifdef CONFIG_ARM64_ERRATUM_2658417
858	{
859		.desc = "ARM erratum 2658417",
860		.capability = ARM64_WORKAROUND_2658417,
861		/* Cortex-A510 r0p0 - r1p1 */
862		ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1),
863		MIDR_FIXED(MIDR_CPU_VAR_REV(1,1), BIT(25)),
864	},
865#endif
866#ifdef CONFIG_ARM64_ERRATUM_3194386
867	{
868		.desc = "SSBS not fully self-synchronizing",
869		.capability = ARM64_WORKAROUND_SPECULATIVE_SSBS,
870		ERRATA_MIDR_RANGE_LIST(erratum_spec_ssbs_list),
871	},
872#endif
873#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
874	{
875		.desc = "ARM errata 2966298, 3117295",
876		.capability = ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD,
877		/* Cortex-A520 r0p0 - r0p1 */
878		ERRATA_MIDR_RANGE_LIST(erratum_spec_unpriv_load_list),
879	},
880#endif
881#ifdef CONFIG_AMPERE_ERRATUM_AC03_CPU_38
882	{
883		.desc = "AmpereOne erratum AC03_CPU_38",
884		.capability = ARM64_WORKAROUND_AMPERE_AC03_CPU_38,
885		ERRATA_MIDR_RANGE_LIST(erratum_ac03_cpu_38_list),
886	},
887#endif
888#ifdef CONFIG_AMPERE_ERRATUM_AC04_CPU_23
889	{
890		.desc = "AmpereOne erratum AC04_CPU_23",
891		.capability = ARM64_WORKAROUND_AMPERE_AC04_CPU_23,
892		ERRATA_MIDR_RANGE_LIST(erratum_ac04_cpu_23_list),
893	},
894#endif
895	{
896		.desc = "Broken CNTVOFF_EL2",
897		.capability = ARM64_WORKAROUND_QCOM_ORYON_CNTVOFF,
898		ERRATA_MIDR_RANGE_LIST(((const struct midr_range[]) {
899					MIDR_ALL_VERSIONS(MIDR_QCOM_ORYON_X1),
900					{}
901				})),
902	},
903	{
904		.desc = "Apple IMPDEF PMUv3 Traps",
905		.capability = ARM64_WORKAROUND_PMUV3_IMPDEF_TRAPS,
906		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
907		.matches = has_impdef_pmuv3,
908		.cpu_enable = cpu_enable_impdef_pmuv3_traps,
909	},
910	{
911	}
912};