cpufeature.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copied from arch/arm64/kernel/cpufeature.c
   4 *
   5 * Copyright (C) 2015 ARM Ltd.
   6 * Copyright (C) 2017 SiFive
   7 */
   8
   9#include <linux/acpi.h>
  10#include <linux/bitmap.h>
  11#include <linux/cpu.h>
  12#include <linux/cpuhotplug.h>
  13#include <linux/ctype.h>
  14#include <linux/log2.h>
  15#include <linux/memory.h>
  16#include <linux/module.h>
  17#include <linux/of.h>
  18#include <asm/acpi.h>
  19#include <asm/alternative.h>
  20#include <asm/bugs.h>
  21#include <asm/cacheflush.h>
  22#include <asm/cpufeature.h>
  23#include <asm/hwcap.h>
  24#include <asm/text-patching.h>
  25#include <asm/hwprobe.h>
  26#include <asm/processor.h>
  27#include <asm/sbi.h>
  28#include <asm/vector.h>
  29#include <asm/vendor_extensions.h>
  30#include <asm/vendor_extensions/thead.h>
  31
  32#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
  33
  34static bool any_cpu_has_zicboz;
  35static bool any_cpu_has_zicbop;
  36static bool any_cpu_has_zicbom;
  37
  38unsigned long elf_hwcap __read_mostly;
  39
  40/* Host ISA bitmap */
  41static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
  42
  43/* Per-cpu ISA extensions. */
  44struct riscv_isainfo hart_isa[NR_CPUS];
  45
  46u32 thead_vlenb_of;
  47
  48/**
  49 * riscv_isa_extension_base() - Get base extension word
  50 *
  51 * @isa_bitmap: ISA bitmap to use
  52 * Return: base extension word as unsigned long value
  53 *
  54 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  55 */
  56unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap)
  57{
  58	return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0];
  59}
  60EXPORT_SYMBOL_GPL(riscv_isa_extension_base);
  61
  62/**
  63 * __riscv_isa_extension_available() - Check whether given extension
  64 * is available or not
  65 *
  66 * @isa_bitmap: ISA bitmap to use
  67 * @bit: bit position of the desired extension
  68 * Return: true or false
  69 *
  70 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  71 */
  72bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit)
  73{
  74	const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa;
  75
  76	if (bit >= RISCV_ISA_EXT_MAX)
  77		return false;
  78
  79	return test_bit(bit, bmap);
  80}
  81EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
  82
  83static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data,
  84			       const unsigned long *isa_bitmap)
  85{
  86	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f))
  87		return 0;
  88
  89	return -EPROBE_DEFER;
  90}
  91
  92static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data,
  93				     const unsigned long *isa_bitmap)
  94{
  95	if (!riscv_cbom_block_size) {
  96		pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n");
  97		return -EINVAL;
  98	}
  99	if (!is_power_of_2(riscv_cbom_block_size)) {
 100		pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n");
 101		return -EINVAL;
 102	}
 103
 104	any_cpu_has_zicbom = true;
 105	return 0;
 106}
 107
 108static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data,
 109				     const unsigned long *isa_bitmap)
 110{
 111	if (!riscv_cboz_block_size) {
 112		pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n");
 113		return -EINVAL;
 114	}
 115	if (!is_power_of_2(riscv_cboz_block_size)) {
 116		pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");
 117		return -EINVAL;
 118	}
 119	any_cpu_has_zicboz = true;
 120	return 0;
 121}
 122
 123static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data,
 124				     const unsigned long *isa_bitmap)
 125{
 126	if (!riscv_cbop_block_size) {
 127		pr_err("Zicbop detected in ISA string, disabling as no cbop-block-size found\n");
 128		return -EINVAL;
 129	}
 130	if (!is_power_of_2(riscv_cbop_block_size)) {
 131		pr_err("Zicbop disabled as cbop-block-size present, but is not a power-of-2\n");
 132		return -EINVAL;
 133	}
 134	any_cpu_has_zicbop = true;
 135	return 0;
 136}
 137
 138static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data,
 139				const unsigned long *isa_bitmap)
 140{
 141	if (!IS_ENABLED(CONFIG_FPU))
 142		return -EINVAL;
 143
 144	/*
 145	 * Due to extension ordering, d is checked before f, so no deferral
 146	 * is required.
 147	 */
 148	if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d)) {
 149		pr_warn_once("This kernel does not support systems with F but not D\n");
 150		return -EINVAL;
 151	}
 152
 153	return 0;
 154}
 155
 156static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data,
 157				const unsigned long *isa_bitmap)
 158{
 159	if (!IS_ENABLED(CONFIG_FPU))
 160		return -EINVAL;
 161
 162	return 0;
 163}
 164
 165static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data,
 166				       const unsigned long *isa_bitmap)
 167{
 168	if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
 169		return -EINVAL;
 170
 171	return 0;
 172}
 173
 174static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data,
 175					   const unsigned long *isa_bitmap)
 176{
 177	if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
 178		return -EINVAL;
 179
 180	if (!IS_ENABLED(CONFIG_FPU))
 181		return -EINVAL;
 182
 183	/*
 184	 * The kernel doesn't support systems that don't implement both of
 185	 * F and D, so if any of the vector extensions that do floating point
 186	 * are to be usable, both floating point extensions need to be usable.
 187	 *
 188	 * Since this function validates vector only, and v/Zve* are probed
 189	 * after f/d, there's no need for a deferral here.
 190	 */
 191	if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d))
 192		return -EINVAL;
 193
 194	return 0;
 195}
 196
 197static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data,
 198					    const unsigned long *isa_bitmap)
 199{
 200	if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
 201		return -EINVAL;
 202
 203	/*
 204	 * It isn't the kernel's job to check that the binding is correct, so
 205	 * it should be enough to check that any of the vector extensions are
 206	 * enabled, which in-turn means that vector is usable in this kernel
 207	 */
 208	if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X))
 209		return -EPROBE_DEFER;
 210
 211	return 0;
 212}
 213
 214static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data,
 215				 const unsigned long *isa_bitmap)
 216{
 217	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA))
 218		return 0;
 219
 220	return -EPROBE_DEFER;
 221}
 222static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data,
 223				  const unsigned long *isa_bitmap)
 224{
 225	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) &&
 226	    __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_d))
 227		return 0;
 228
 229	return -EPROBE_DEFER;
 230}
 231
 232static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data,
 233				  const unsigned long *isa_bitmap)
 234{
 235	if (IS_ENABLED(CONFIG_64BIT))
 236		return -EINVAL;
 237
 238	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) &&
 239	    __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_f))
 240		return 0;
 241
 242	return -EPROBE_DEFER;
 243}
 244
 245static int riscv_ext_zilsd_validate(const struct riscv_isa_ext_data *data,
 246				    const unsigned long *isa_bitmap)
 247{
 248	if (IS_ENABLED(CONFIG_64BIT))
 249		return -EINVAL;
 250
 251	return 0;
 252}
 253
 254static int riscv_ext_zclsd_validate(const struct riscv_isa_ext_data *data,
 255				    const unsigned long *isa_bitmap)
 256{
 257	if (IS_ENABLED(CONFIG_64BIT))
 258		return -EINVAL;
 259
 260	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZILSD) &&
 261	    __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA))
 262		return 0;
 263
 264	return -EPROBE_DEFER;
 265}
 266
 267static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data,
 268				   const unsigned long *isa_bitmap)
 269{
 270	if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
 271		return -EINVAL;
 272
 273	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F))
 274		return 0;
 275
 276	return -EPROBE_DEFER;
 277}
 278
 279static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data,
 280				       const unsigned long *isa_bitmap)
 281{
 282	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) &&
 283	    __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN))
 284		return 0;
 285
 286	return -EPROBE_DEFER;
 287}
 288
 289static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data,
 290				    const unsigned long *isa_bitmap)
 291{
 292	/* SVADE has already been detected, use SVADE only */
 293	if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SVADE))
 294		return -EOPNOTSUPP;
 295
 296	return 0;
 297}
 298
 299static const unsigned int riscv_a_exts[] = {
 300	RISCV_ISA_EXT_ZAAMO,
 301	RISCV_ISA_EXT_ZALRSC,
 302};
 303
 304#define RISCV_ISA_EXT_ZKN	\
 305	RISCV_ISA_EXT_ZBKB,	\
 306	RISCV_ISA_EXT_ZBKC,	\
 307	RISCV_ISA_EXT_ZBKX,	\
 308	RISCV_ISA_EXT_ZKND,	\
 309	RISCV_ISA_EXT_ZKNE,	\
 310	RISCV_ISA_EXT_ZKNH
 311
 312static const unsigned int riscv_zk_bundled_exts[] = {
 313	RISCV_ISA_EXT_ZKN,
 314	RISCV_ISA_EXT_ZKR,
 315	RISCV_ISA_EXT_ZKT
 316};
 317
 318static const unsigned int riscv_zkn_bundled_exts[] = {
 319	RISCV_ISA_EXT_ZKN
 320};
 321
 322static const unsigned int riscv_zks_bundled_exts[] = {
 323	RISCV_ISA_EXT_ZBKB,
 324	RISCV_ISA_EXT_ZBKC,
 325	RISCV_ISA_EXT_ZKSED,
 326	RISCV_ISA_EXT_ZKSH
 327};
 328
 329#define RISCV_ISA_EXT_ZVKN	\
 330	RISCV_ISA_EXT_ZVKNED,	\
 331	RISCV_ISA_EXT_ZVKNHB,	\
 332	RISCV_ISA_EXT_ZVKB,	\
 333	RISCV_ISA_EXT_ZVKT
 334
 335static const unsigned int riscv_zvkn_bundled_exts[] = {
 336	RISCV_ISA_EXT_ZVKN
 337};
 338
 339static const unsigned int riscv_zvknc_bundled_exts[] = {
 340	RISCV_ISA_EXT_ZVKN,
 341	RISCV_ISA_EXT_ZVBC
 342};
 343
 344static const unsigned int riscv_zvkng_bundled_exts[] = {
 345	RISCV_ISA_EXT_ZVKN,
 346	RISCV_ISA_EXT_ZVKG
 347};
 348
 349#define RISCV_ISA_EXT_ZVKS	\
 350	RISCV_ISA_EXT_ZVKSED,	\
 351	RISCV_ISA_EXT_ZVKSH,	\
 352	RISCV_ISA_EXT_ZVKB,	\
 353	RISCV_ISA_EXT_ZVKT
 354
 355static const unsigned int riscv_zvks_bundled_exts[] = {
 356	RISCV_ISA_EXT_ZVKS
 357};
 358
 359static const unsigned int riscv_zvksc_bundled_exts[] = {
 360	RISCV_ISA_EXT_ZVKS,
 361	RISCV_ISA_EXT_ZVBC
 362};
 363
 364static const unsigned int riscv_zvksg_bundled_exts[] = {
 365	RISCV_ISA_EXT_ZVKS,
 366	RISCV_ISA_EXT_ZVKG
 367};
 368
 369static const unsigned int riscv_zvbb_exts[] = {
 370	RISCV_ISA_EXT_ZVKB
 371};
 372
 373#define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST	\
 374	RISCV_ISA_EXT_ZVE64X,		\
 375	RISCV_ISA_EXT_ZVE32F,		\
 376	RISCV_ISA_EXT_ZVE32X
 377
 378#define RISCV_ISA_EXT_ZVE64D_IMPLY_LIST	\
 379	RISCV_ISA_EXT_ZVE64F,		\
 380	RISCV_ISA_EXT_ZVE64F_IMPLY_LIST
 381
 382#define RISCV_ISA_EXT_V_IMPLY_LIST	\
 383	RISCV_ISA_EXT_ZVE64D,		\
 384	RISCV_ISA_EXT_ZVE64D_IMPLY_LIST
 385
 386static const unsigned int riscv_zve32f_exts[] = {
 387	RISCV_ISA_EXT_ZVE32X
 388};
 389
 390static const unsigned int riscv_zve64f_exts[] = {
 391	RISCV_ISA_EXT_ZVE64F_IMPLY_LIST
 392};
 393
 394static const unsigned int riscv_zve64d_exts[] = {
 395	RISCV_ISA_EXT_ZVE64D_IMPLY_LIST
 396};
 397
 398static const unsigned int riscv_v_exts[] = {
 399	RISCV_ISA_EXT_V_IMPLY_LIST
 400};
 401
 402static const unsigned int riscv_zve64x_exts[] = {
 403	RISCV_ISA_EXT_ZVE32X,
 404	RISCV_ISA_EXT_ZVE64X
 405};
 406
 407/*
 408 * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V
 409 * privileged ISA, the existence of the CSRs is implied by any extension which
 410 * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the
 411 * existence of the CSR, and treat it as a subset of those other extensions.
 412 */
 413static const unsigned int riscv_xlinuxenvcfg_exts[] = {
 414	RISCV_ISA_EXT_XLINUXENVCFG
 415};
 416
 417/*
 418 * Zc* spec states that:
 419 * - C always implies Zca
 420 * - C+F implies Zcf (RV32 only)
 421 * - C+D implies Zcd
 422 *
 423 * These extensions will be enabled and then validated depending on the
 424 * availability of F/D RV32.
 425 */
 426static const unsigned int riscv_c_exts[] = {
 427	RISCV_ISA_EXT_ZCA,
 428	RISCV_ISA_EXT_ZCF,
 429	RISCV_ISA_EXT_ZCD,
 430};
 431
 432/*
 433 * The canonical order of ISA extension names in the ISA string is defined in
 434 * chapter 27 of the unprivileged specification.
 435 *
 436 * Ordinarily, for in-kernel data structures, this order is unimportant but
 437 * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo.
 438 *
 439 * The specification uses vague wording, such as should, when it comes to
 440 * ordering, so for our purposes the following rules apply:
 441 *
 442 * 1. All multi-letter extensions must be separated from other extensions by an
 443 *    underscore.
 444 *
 445 * 2. Additional standard extensions (starting with 'Z') must be sorted after
 446 *    single-letter extensions and before any higher-privileged extensions.
 447 *
 448 * 3. The first letter following the 'Z' conventionally indicates the most
 449 *    closely related alphabetical extension category, IMAFDQLCBKJTPVH.
 450 *    If multiple 'Z' extensions are named, they must be ordered first by
 451 *    category, then alphabetically within a category.
 452 *
 453 * 3. Standard supervisor-level extensions (starting with 'S') must be listed
 454 *    after standard unprivileged extensions.  If multiple supervisor-level
 455 *    extensions are listed, they must be ordered alphabetically.
 456 *
 457 * 4. Standard machine-level extensions (starting with 'Zxm') must be listed
 458 *    after any lower-privileged, standard extensions.  If multiple
 459 *    machine-level extensions are listed, they must be ordered
 460 *    alphabetically.
 461 *
 462 * 5. Non-standard extensions (starting with 'X') must be listed after all
 463 *    standard extensions. If multiple non-standard extensions are listed, they
 464 *    must be ordered alphabetically.
 465 *
 466 * An example string following the order is:
 467 *    rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux
 468 *
 469 * New entries to this struct should follow the ordering rules described above.
 470 */
 471const struct riscv_isa_ext_data riscv_isa_ext[] = {
 472	__RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i),
 473	__RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m),
 474	__RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_a, riscv_a_exts),
 475	__RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_f, riscv_ext_f_validate),
 476	__RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_d, riscv_ext_d_validate),
 477	__RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q),
 478	__RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_c, riscv_c_exts),
 479	__RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_v, riscv_v_exts, riscv_ext_vector_float_validate),
 480	__RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h),
 481	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate),
 482	__RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate),
 483	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate),
 484	__RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE),
 485	__RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
 486	__RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND),
 487	__RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),
 488	__RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI),
 489	__RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL),
 490	__RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),
 491	__RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),
 492	__RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP),
 493	__RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO),
 494	__RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA),
 495	__RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS),
 496	__RISCV_ISA_EXT_DATA(zalasr, RISCV_ISA_EXT_ZALASR),
 497	__RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC),
 498	__RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS),
 499	__RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends),
 500	__RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends),
 501	__RISCV_ISA_EXT_DATA_VALIDATE(zfh, RISCV_ISA_EXT_ZFH, riscv_ext_f_depends),
 502	__RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends),
 503	__RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA),
 504	__RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends),
 505	__RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate),
 506	__RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate),
 507	__RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends),
 508	__RISCV_ISA_EXT_DATA_VALIDATE(zclsd, RISCV_ISA_EXT_ZCLSD, riscv_ext_zclsd_validate),
 509	__RISCV_ISA_EXT_DATA_VALIDATE(zilsd, RISCV_ISA_EXT_ZILSD, riscv_ext_zilsd_validate),
 510	__RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA),
 511	__RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB),
 512	__RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC),
 513	__RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB),
 514	__RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC),
 515	__RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX),
 516	__RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS),
 517	__RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts),
 518	__RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts),
 519	__RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND),
 520	__RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE),
 521	__RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH),
 522	__RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR),
 523	__RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts),
 524	__RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT),
 525	__RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED),
 526	__RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH),
 527	__RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO),
 528	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate),
 529	__RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate),
 530	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate),
 531	__RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate),
 532	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate),
 533	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate),
 534	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate),
 535	__RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate),
 536	__RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate),
 537	__RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH),
 538	__RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN),
 539	__RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate),
 540	__RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate),
 541	__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate),
 542	__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate),
 543	__RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate),
 544	__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate),
 545	__RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate),
 546	__RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate),
 547	__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate),
 548	__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate),
 549	__RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate),
 550	__RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate),
 551	__RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate),
 552	__RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate),
 553	__RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
 554	__RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM),
 555	__RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts),
 556	__RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),
 557	__RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
 558	__RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
 559	__RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts),
 560	__RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
 561	__RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE),
 562	__RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate),
 563	__RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
 564	__RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
 565	__RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
 566	__RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B),
 567	__RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC),
 568};
 569
 570const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext);
 571
 572static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap)
 573{
 574	if (ext->id != RISCV_ISA_EXT_INVALID)
 575		set_bit(ext->id, bitmap);
 576
 577	for (int i = 0; i < ext->subset_ext_size; i++) {
 578		if (ext->subset_ext_ids[i] != RISCV_ISA_EXT_INVALID)
 579			set_bit(ext->subset_ext_ids[i], bitmap);
 580	}
 581}
 582
 583static const struct riscv_isa_ext_data *riscv_get_isa_ext_data(unsigned int ext_id)
 584{
 585	for (int i = 0; i < riscv_isa_ext_count; i++) {
 586		if (riscv_isa_ext[i].id == ext_id)
 587			return &riscv_isa_ext[i];
 588	}
 589
 590	return NULL;
 591}
 592
 593/*
 594 * "Resolve" a source ISA bitmap into one that matches kernel configuration as
 595 * well as correct extension dependencies. Some extensions depends on specific
 596 * kernel configuration to be usable (V needs CONFIG_RISCV_ISA_V for instance)
 597 * and this function will actually validate all the extensions provided in
 598 * source_isa into the resolved_isa based on extensions validate() callbacks.
 599 */
 600static void __init riscv_resolve_isa(unsigned long *source_isa,
 601				     unsigned long *resolved_isa, unsigned long *this_hwcap,
 602				     unsigned long *isa2hwcap)
 603{
 604	bool loop;
 605	const struct riscv_isa_ext_data *ext;
 606	DECLARE_BITMAP(prev_resolved_isa, RISCV_ISA_EXT_MAX);
 607	int max_loop_count = riscv_isa_ext_count, ret;
 608	unsigned int bit;
 609
 610	do {
 611		loop = false;
 612		if (max_loop_count-- < 0) {
 613			pr_err("Failed to reach a stable ISA state\n");
 614			return;
 615		}
 616		bitmap_copy(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX);
 617		for_each_set_bit(bit, source_isa, RISCV_ISA_EXT_MAX) {
 618			ext = riscv_get_isa_ext_data(bit);
 619
 620			if (ext && ext->validate) {
 621				ret = ext->validate(ext, resolved_isa);
 622				if (ret == -EPROBE_DEFER) {
 623					loop = true;
 624					continue;
 625				} else if (ret) {
 626					/* Disable the extension entirely */
 627					clear_bit(bit, source_isa);
 628					continue;
 629				}
 630			}
 631
 632			set_bit(bit, resolved_isa);
 633			/* No need to keep it in source isa now that it is enabled */
 634			clear_bit(bit, source_isa);
 635
 636			/* Single letter extensions get set in hwcap */
 637			if (bit < RISCV_ISA_EXT_BASE)
 638				*this_hwcap |= isa2hwcap[bit];
 639		}
 640	} while (loop && !bitmap_equal(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX));
 641}
 642
 643static void __init match_isa_ext(const char *name, const char *name_end, unsigned long *bitmap)
 644{
 645	for (int i = 0; i < riscv_isa_ext_count; i++) {
 646		const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i];
 647
 648		if ((name_end - name == strlen(ext->name)) &&
 649		    !strncasecmp(name, ext->name, name_end - name)) {
 650			riscv_isa_set_ext(ext, bitmap);
 651			break;
 652		}
 653	}
 654}
 655
 656static void __init riscv_parse_isa_string(const char *isa, unsigned long *bitmap)
 657{
 658	/*
 659	 * For all possible cpus, we have already validated in
 660	 * the boot process that they at least contain "rv" and
 661	 * whichever of "32"/"64" this kernel supports, and so this
 662	 * section can be skipped.
 663	 */
 664	isa += 4;
 665
 666	while (*isa) {
 667		const char *ext = isa++;
 668		const char *ext_end = isa;
 669		bool ext_err = false;
 670
 671		switch (*ext) {
 672		case 'x':
 673		case 'X':
 674			if (acpi_disabled)
 675				pr_warn_once("Vendor extensions are ignored in riscv,isa. Use riscv,isa-extensions instead.");
 676			/*
 677			 * To skip an extension, we find its end.
 678			 * As multi-letter extensions must be split from other multi-letter
 679			 * extensions with an "_", the end of a multi-letter extension will
 680			 * either be the null character or the "_" at the start of the next
 681			 * multi-letter extension.
 682			 */
 683			for (; *isa && *isa != '_'; ++isa)
 684				;
 685			ext_err = true;
 686			break;
 687		case 's':
 688			/*
 689			 * Workaround for invalid single-letter 's' & 'u' (QEMU).
 690			 * No need to set the bit in riscv_isa as 's' & 'u' are
 691			 * not valid ISA extensions. It works unless the first
 692			 * multi-letter extension in the ISA string begins with
 693			 * "Su" and is not prefixed with an underscore.
 694			 */
 695			if (ext[-1] != '_' && ext[1] == 'u') {
 696				++isa;
 697				ext_err = true;
 698				break;
 699			}
 700			fallthrough;
 701		case 'S':
 702		case 'z':
 703		case 'Z':
 704			/*
 705			 * Before attempting to parse the extension itself, we find its end.
 706			 * As multi-letter extensions must be split from other multi-letter
 707			 * extensions with an "_", the end of a multi-letter extension will
 708			 * either be the null character or the "_" at the start of the next
 709			 * multi-letter extension.
 710			 *
 711			 * Next, as the extensions version is currently ignored, we
 712			 * eliminate that portion. This is done by parsing backwards from
 713			 * the end of the extension, removing any numbers. This may be a
 714			 * major or minor number however, so the process is repeated if a
 715			 * minor number was found.
 716			 *
 717			 * ext_end is intended to represent the first character *after* the
 718			 * name portion of an extension, but will be decremented to the last
 719			 * character itself while eliminating the extensions version number.
 720			 * A simple re-increment solves this problem.
 721			 */
 722			for (; *isa && *isa != '_'; ++isa)
 723				if (unlikely(!isalnum(*isa)))
 724					ext_err = true;
 725
 726			ext_end = isa;
 727			if (unlikely(ext_err))
 728				break;
 729
 730			if (!isdigit(ext_end[-1]))
 731				break;
 732
 733			while (isdigit(*--ext_end))
 734				;
 735
 736			if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) {
 737				++ext_end;
 738				break;
 739			}
 740
 741			while (isdigit(*--ext_end))
 742				;
 743
 744			++ext_end;
 745			break;
 746		default:
 747			/*
 748			 * Things are a little easier for single-letter extensions, as they
 749			 * are parsed forwards.
 750			 *
 751			 * After checking that our starting position is valid, we need to
 752			 * ensure that, when isa was incremented at the start of the loop,
 753			 * that it arrived at the start of the next extension.
 754			 *
 755			 * If we are already on a non-digit, there is nothing to do. Either
 756			 * we have a multi-letter extension's _, or the start of an
 757			 * extension.
 758			 *
 759			 * Otherwise we have found the current extension's major version
 760			 * number. Parse past it, and a subsequent p/minor version number
 761			 * if present. The `p` extension must not appear immediately after
 762			 * a number, so there is no fear of missing it.
 763			 *
 764			 */
 765			if (unlikely(!isalpha(*ext))) {
 766				ext_err = true;
 767				break;
 768			}
 769
 770			if (!isdigit(*isa))
 771				break;
 772
 773			while (isdigit(*++isa))
 774				;
 775
 776			if (tolower(*isa) != 'p')
 777				break;
 778
 779			if (!isdigit(*++isa)) {
 780				--isa;
 781				break;
 782			}
 783
 784			while (isdigit(*++isa))
 785				;
 786
 787			break;
 788		}
 789
 790		/*
 791		 * The parser expects that at the start of an iteration isa points to the
 792		 * first character of the next extension. As we stop parsing an extension
 793		 * on meeting a non-alphanumeric character, an extra increment is needed
 794		 * where the succeeding extension is a multi-letter prefixed with an "_".
 795		 */
 796		if (*isa == '_')
 797			++isa;
 798
 799		if (unlikely(ext_err))
 800			continue;
 801
 802		match_isa_ext(ext, ext_end, bitmap);
 803	}
 804}
 805
 806static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap)
 807{
 808	struct device_node *node;
 809	const char *isa;
 810	int rc;
 811	struct acpi_table_header *rhct;
 812	acpi_status status;
 813	unsigned int cpu;
 814	u64 boot_vendorid;
 815	u64 boot_archid;
 816
 817	if (!acpi_disabled) {
 818		status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
 819		if (ACPI_FAILURE(status))
 820			return;
 821	}
 822
 823	boot_vendorid = riscv_get_mvendorid();
 824	boot_archid = riscv_get_marchid();
 825
 826	for_each_possible_cpu(cpu) {
 827		struct riscv_isainfo *isainfo = &hart_isa[cpu];
 828		unsigned long this_hwcap = 0;
 829		DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 };
 830
 831		if (acpi_disabled) {
 832			node = of_cpu_device_node_get(cpu);
 833			if (!node) {
 834				pr_warn("Unable to find cpu node\n");
 835				continue;
 836			}
 837
 838			rc = of_property_read_string(node, "riscv,isa", &isa);
 839			of_node_put(node);
 840			if (rc) {
 841				pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
 842				continue;
 843			}
 844		} else {
 845			rc = acpi_get_riscv_isa(rhct, cpu, &isa);
 846			if (rc < 0) {
 847				pr_warn("Unable to get ISA for the hart - %d\n", cpu);
 848				continue;
 849			}
 850		}
 851
 852		riscv_parse_isa_string(isa, source_isa);
 853
 854		/*
 855		 * These ones were as they were part of the base ISA when the
 856		 * port & dt-bindings were upstreamed, and so can be set
 857		 * unconditionally where `i` is in riscv,isa on DT systems.
 858		 */
 859		if (acpi_disabled) {
 860			set_bit(RISCV_ISA_EXT_ZICSR, source_isa);
 861			set_bit(RISCV_ISA_EXT_ZIFENCEI, source_isa);
 862			set_bit(RISCV_ISA_EXT_ZICNTR, source_isa);
 863			set_bit(RISCV_ISA_EXT_ZIHPM, source_isa);
 864		}
 865
 866		/*
 867		 * "V" in ISA strings is ambiguous in practice: it should mean
 868		 * just the standard V-1.0 but vendors aren't well behaved.
 869		 * Many vendors with T-Head CPU cores which implement the 0.7.1
 870		 * version of the vector specification put "v" into their DTs.
 871		 * CPU cores with the ratified spec will contain non-zero
 872		 * marchid.
 873		 */
 874		if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0) {
 875			this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v];
 876			clear_bit(RISCV_ISA_EXT_v, source_isa);
 877		}
 878
 879		riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
 880
 881		/*
 882		 * All "okay" hart should have same isa. Set HWCAP based on
 883		 * common capabilities of every "okay" hart, in case they don't
 884		 * have.
 885		 */
 886		if (elf_hwcap)
 887			elf_hwcap &= this_hwcap;
 888		else
 889			elf_hwcap = this_hwcap;
 890
 891		if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
 892			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
 893		else
 894			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
 895	}
 896
 897	if (!acpi_disabled && rhct)
 898		acpi_put_table((struct acpi_table_header *)rhct);
 899}
 900
 901static void __init riscv_fill_cpu_vendor_ext(struct device_node *cpu_node, int cpu)
 902{
 903	if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT))
 904		return;
 905
 906	for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) {
 907		struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i];
 908
 909		for (int j = 0; j < ext_list->ext_data_count; j++) {
 910			const struct riscv_isa_ext_data ext = ext_list->ext_data[j];
 911			struct riscv_isavendorinfo *isavendorinfo = &ext_list->per_hart_isa_bitmap[cpu];
 912
 913			if (of_property_match_string(cpu_node, "riscv,isa-extensions",
 914						     ext.property) < 0)
 915				continue;
 916
 917			/*
 918			 * Assume that subset extensions are all members of the
 919			 * same vendor.
 920			 */
 921			if (ext.subset_ext_size)
 922				for (int k = 0; k < ext.subset_ext_size; k++)
 923					set_bit(ext.subset_ext_ids[k], isavendorinfo->isa);
 924
 925			set_bit(ext.id, isavendorinfo->isa);
 926		}
 927	}
 928}
 929
 930/*
 931 * Populate all_harts_isa_bitmap for each vendor with all of the extensions that
 932 * are shared across CPUs for that vendor.
 933 */
 934static void __init riscv_fill_vendor_ext_list(int cpu)
 935{
 936	if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT))
 937		return;
 938
 939	for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) {
 940		struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i];
 941
 942		if (!ext_list->is_initialized) {
 943			bitmap_copy(ext_list->all_harts_isa_bitmap.isa,
 944				    ext_list->per_hart_isa_bitmap[cpu].isa,
 945				    RISCV_ISA_VENDOR_EXT_MAX);
 946			ext_list->is_initialized = true;
 947		} else {
 948			bitmap_and(ext_list->all_harts_isa_bitmap.isa,
 949				   ext_list->all_harts_isa_bitmap.isa,
 950				   ext_list->per_hart_isa_bitmap[cpu].isa,
 951				   RISCV_ISA_VENDOR_EXT_MAX);
 952		}
 953	}
 954}
 955
 956static int has_thead_homogeneous_vlenb(void)
 957{
 958	int cpu;
 959	u32 prev_vlenb = 0;
 960	u32 vlenb = 0;
 961
 962	/* Ignore thead,vlenb property if xtheadvector is not enabled in the kernel */
 963	if (!IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR))
 964		return 0;
 965
 966	for_each_possible_cpu(cpu) {
 967		struct device_node *cpu_node;
 968
 969		cpu_node = of_cpu_device_node_get(cpu);
 970		if (!cpu_node) {
 971			pr_warn("Unable to find cpu node\n");
 972			return -ENOENT;
 973		}
 974
 975		if (of_property_read_u32(cpu_node, "thead,vlenb", &vlenb)) {
 976			of_node_put(cpu_node);
 977
 978			if (prev_vlenb)
 979				return -ENOENT;
 980			continue;
 981		}
 982
 983		if (prev_vlenb && vlenb != prev_vlenb) {
 984			of_node_put(cpu_node);
 985			return -ENOENT;
 986		}
 987
 988		prev_vlenb = vlenb;
 989		of_node_put(cpu_node);
 990	}
 991
 992	thead_vlenb_of = vlenb;
 993	return 0;
 994}
 995
 996static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap)
 997{
 998	unsigned int cpu;
 999	bool mitigated;
1000
1001	for_each_possible_cpu(cpu) {
1002		unsigned long this_hwcap = 0;
1003		struct device_node *cpu_node;
1004		struct riscv_isainfo *isainfo = &hart_isa[cpu];
1005		DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 };
1006
1007		cpu_node = of_cpu_device_node_get(cpu);
1008		if (!cpu_node) {
1009			pr_warn("Unable to find cpu node\n");
1010			continue;
1011		}
1012
1013		if (!of_property_present(cpu_node, "riscv,isa-extensions")) {
1014			of_node_put(cpu_node);
1015			continue;
1016		}
1017
1018		for (int i = 0; i < riscv_isa_ext_count; i++) {
1019			const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i];
1020
1021			if (of_property_match_string(cpu_node, "riscv,isa-extensions",
1022						     ext->property) < 0)
1023				continue;
1024
1025			riscv_isa_set_ext(ext, source_isa);
1026		}
1027
1028		riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
1029		riscv_fill_cpu_vendor_ext(cpu_node, cpu);
1030
1031		of_node_put(cpu_node);
1032
1033		/*
1034		 * All "okay" harts should have same isa. Set HWCAP based on
1035		 * common capabilities of every "okay" hart, in case they don't.
1036		 */
1037		if (elf_hwcap)
1038			elf_hwcap &= this_hwcap;
1039		else
1040			elf_hwcap = this_hwcap;
1041
1042		if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
1043			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
1044		else
1045			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
1046
1047		riscv_fill_vendor_ext_list(cpu);
1048	}
1049
1050	/*
1051	 * Execute ghostwrite mitigation immediately after detecting extensions
1052	 * to disable xtheadvector if necessary.
1053	 */
1054	mitigated = ghostwrite_enable_mitigation();
1055
1056	if (!mitigated && has_xtheadvector_no_alternatives() && has_thead_homogeneous_vlenb() < 0) {
1057		pr_warn("Unsupported heterogeneous vlenb detected, vector extension disabled.\n");
1058		disable_xtheadvector();
1059	}
1060
1061	if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
1062		return -ENOENT;
1063
1064	return 0;
1065}
1066
1067#ifdef CONFIG_RISCV_ISA_FALLBACK
1068bool __initdata riscv_isa_fallback = true;
1069#else
1070bool __initdata riscv_isa_fallback;
1071static int __init riscv_isa_fallback_setup(char *__unused)
1072{
1073	riscv_isa_fallback = true;
1074	return 1;
1075}
1076early_param("riscv_isa_fallback", riscv_isa_fallback_setup);
1077#endif
1078
1079void __init riscv_fill_hwcap(void)
1080{
1081	char print_str[NUM_ALPHA_EXTS + 1];
1082	unsigned long isa2hwcap[26] = {0};
1083	int i, j;
1084
1085	isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;
1086	isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;
1087	isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A;
1088	isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;
1089	isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;
1090	isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;
1091	isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V;
1092
1093	if (!acpi_disabled) {
1094		riscv_fill_hwcap_from_isa_string(isa2hwcap);
1095	} else {
1096		int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap);
1097
1098		if (ret && riscv_isa_fallback) {
1099			pr_info("Falling back to deprecated \"riscv,isa\"\n");
1100			riscv_fill_hwcap_from_isa_string(isa2hwcap);
1101		}
1102	}
1103
1104	/*
1105	 * We don't support systems with F but without D, so mask those out
1106	 * here.
1107	 */
1108	if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) {
1109		pr_info("This kernel does not support systems with F but not D\n");
1110		elf_hwcap &= ~COMPAT_HWCAP_ISA_F;
1111	}
1112
1113	if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_ZVE32X) ||
1114	    has_xtheadvector_no_alternatives()) {
1115		/*
1116		 * This cannot fail when called on the boot hart
1117		 */
1118		riscv_v_setup_vsize();
1119	}
1120
1121	memset(print_str, 0, sizeof(print_str));
1122	for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
1123		if (riscv_isa[0] & BIT_MASK(i))
1124			print_str[j++] = (char)('a' + i);
1125	pr_info("riscv: base ISA extensions %s\n", print_str);
1126
1127	memset(print_str, 0, sizeof(print_str));
1128	for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
1129		if (elf_hwcap & BIT_MASK(i))
1130			print_str[j++] = (char)('a' + i);
1131	pr_info("riscv: ELF capabilities %s\n", print_str);
1132}
1133
1134unsigned long riscv_get_elf_hwcap(void)
1135{
1136	unsigned long hwcap;
1137
1138	hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1));
1139
1140	if (!riscv_v_vstate_ctrl_user_allowed())
1141		hwcap &= ~COMPAT_HWCAP_ISA_V;
1142
1143	return hwcap;
1144}
1145
1146void __init riscv_user_isa_enable(void)
1147{
1148	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ))
1149		current->thread.envcfg |= ENVCFG_CBZE;
1150	else if (any_cpu_has_zicboz)
1151		pr_warn("Zicboz disabled as it is unavailable on some harts\n");
1152
1153	if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM))
1154		current->thread.envcfg |= ENVCFG_CBCFE;
1155	else if (any_cpu_has_zicbom)
1156		pr_warn("Zicbom disabled as it is unavailable on some harts\n");
1157
1158	if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOP) &&
1159	    any_cpu_has_zicbop)
1160		pr_warn("Zicbop disabled as it is unavailable on some harts\n");
1161}
1162
1163#ifdef CONFIG_RISCV_ALTERNATIVE
1164/*
1165 * Alternative patch sites consider 48 bits when determining when to patch
1166 * the old instruction sequence with the new. These bits are broken into a
1167 * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the
1168 * patch site is for an erratum, identified by the 32-bit patch ID. When
1169 * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures
1170 * further break down patch ID into two 16-bit numbers. The lower 16 bits
1171 * are the cpufeature ID and the upper 16 bits are used for a value specific
1172 * to the cpufeature and patch site. If the upper 16 bits are zero, then it
1173 * implies no specific value is specified. cpufeatures that want to control
1174 * patching on a per-site basis will provide non-zero values and implement
1175 * checks here. The checks return true when patching should be done, and
1176 * false otherwise.
1177 */
1178static bool riscv_cpufeature_patch_check(u16 id, u16 value)
1179{
1180	if (!value)
1181		return true;
1182
1183	switch (id) {
1184	case RISCV_ISA_EXT_ZICBOZ:
1185		/*
1186		 * Zicboz alternative applications provide the maximum
1187		 * supported block size order, or zero when it doesn't
1188		 * matter. If the current block size exceeds the maximum,
1189		 * then the alternative cannot be applied.
1190		 */
1191		return riscv_cboz_block_size <= (1U << value);
1192	}
1193
1194	return false;
1195}
1196
1197void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
1198						  struct alt_entry *end,
1199						  unsigned int stage)
1200{
1201	struct alt_entry *alt;
1202	void *oldptr, *altptr;
1203	u16 id, value, vendor;
1204
1205	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
1206		return;
1207
1208	for (alt = begin; alt < end; alt++) {
1209		id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);
1210		vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id);
1211
1212		/*
1213		 * Any alternative with a patch_id that is less than
1214		 * RISCV_ISA_EXT_MAX is interpreted as a standard extension.
1215		 *
1216		 * Any alternative with patch_id that is greater than or equal
1217		 * to RISCV_VENDOR_EXT_ALTERNATIVES_BASE is interpreted as a
1218		 * vendor extension.
1219		 */
1220		if (id < RISCV_ISA_EXT_MAX) {
1221			/*
1222			 * This patch should be treated as errata so skip
1223			 * processing here.
1224			 */
1225			if (alt->vendor_id != 0)
1226				continue;
1227
1228			if (!__riscv_isa_extension_available(NULL, id))
1229				continue;
1230
1231			value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);
1232			if (!riscv_cpufeature_patch_check(id, value))
1233				continue;
1234		} else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) {
1235			if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor,
1236								    id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE))
1237				continue;
1238		} else {
1239			WARN(1, "This extension id:%d is not in ISA extension list", id);
1240			continue;
1241		}
1242
1243		oldptr = ALT_OLD_PTR(alt);
1244		altptr = ALT_ALT_PTR(alt);
1245
1246		mutex_lock(&text_mutex);
1247		patch_text_nosync(oldptr, altptr, alt->alt_len);
1248		riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);
1249		mutex_unlock(&text_mutex);
1250	}
1251}
1252#endif