pgtable.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Stand-alone page-table allocator for hyp stage-1 and guest stage-2.
   4 * No bombay mix was harmed in the writing of this file.
   5 *
   6 * Copyright (C) 2020 Google LLC
   7 * Author: Will Deacon <will@kernel.org>
   8 */
   9
  10#include <linux/bitfield.h>
  11#include <asm/kvm_pgtable.h>
  12#include <asm/stage2_pgtable.h>
  13
  14struct kvm_pgtable_walk_data {
  15	struct kvm_pgtable_walker	*walker;
  16
  17	const u64			start;
  18	u64				addr;
  19	const u64			end;
  20};
  21
  22static bool kvm_pgtable_walk_skip_bbm_tlbi(const struct kvm_pgtable_visit_ctx *ctx)
  23{
  24	return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_BBM_TLBI);
  25}
  26
  27static bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx)
  28{
  29	return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO);
  30}
  31
  32static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, u64 phys)
  33{
  34	u64 granule = kvm_granule_size(ctx->level);
  35
  36	if (!kvm_level_supports_block_mapping(ctx->level))
  37		return false;
  38
  39	if (granule > (ctx->end - ctx->addr))
  40		return false;
  41
  42	if (!IS_ALIGNED(phys, granule))
  43		return false;
  44
  45	return IS_ALIGNED(ctx->addr, granule);
  46}
  47
  48static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, s8 level)
  49{
  50	u64 shift = kvm_granule_shift(level);
  51	u64 mask = BIT(PAGE_SHIFT - 3) - 1;
  52
  53	return (data->addr >> shift) & mask;
  54}
  55
  56static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)
  57{
  58	u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */
  59	u64 mask = BIT(pgt->ia_bits) - 1;
  60
  61	return (addr & mask) >> shift;
  62}
  63
  64static u32 kvm_pgd_pages(u32 ia_bits, s8 start_level)
  65{
  66	struct kvm_pgtable pgt = {
  67		.ia_bits	= ia_bits,
  68		.start_level	= start_level,
  69	};
  70
  71	return kvm_pgd_page_idx(&pgt, -1ULL) + 1;
  72}
  73
  74static bool kvm_pte_table(kvm_pte_t pte, s8 level)
  75{
  76	if (level == KVM_PGTABLE_LAST_LEVEL)
  77		return false;
  78
  79	if (!kvm_pte_valid(pte))
  80		return false;
  81
  82	return FIELD_GET(KVM_PTE_TYPE, pte) == KVM_PTE_TYPE_TABLE;
  83}
  84
  85static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)
  86{
  87	return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));
  88}
  89
  90static void kvm_clear_pte(kvm_pte_t *ptep)
  91{
  92	WRITE_ONCE(*ptep, 0);
  93}
  94
  95static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)
  96{
  97	kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));
  98
  99	pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE);
 100	pte |= KVM_PTE_VALID;
 101	return pte;
 102}
 103
 104static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, s8 level)
 105{
 106	kvm_pte_t pte = kvm_phys_to_pte(pa);
 107	u64 type = (level == KVM_PGTABLE_LAST_LEVEL) ? KVM_PTE_TYPE_PAGE :
 108						       KVM_PTE_TYPE_BLOCK;
 109
 110	pte |= attr & (KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI);
 111	pte |= FIELD_PREP(KVM_PTE_TYPE, type);
 112	pte |= KVM_PTE_VALID;
 113
 114	return pte;
 115}
 116
 117static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id)
 118{
 119	return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);
 120}
 121
 122static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data,
 123				  const struct kvm_pgtable_visit_ctx *ctx,
 124				  enum kvm_pgtable_walk_flags visit)
 125{
 126	struct kvm_pgtable_walker *walker = data->walker;
 127
 128	/* Ensure the appropriate lock is held (e.g. RCU lock for stage-2 MMU) */
 129	WARN_ON_ONCE(kvm_pgtable_walk_shared(ctx) && !kvm_pgtable_walk_lock_held());
 130	return walker->cb(ctx, visit);
 131}
 132
 133static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker,
 134				      int r)
 135{
 136	/*
 137	 * Visitor callbacks return EAGAIN when the conditions that led to a
 138	 * fault are no longer reflected in the page tables due to a race to
 139	 * update a PTE. In the context of a fault handler this is interpreted
 140	 * as a signal to retry guest execution.
 141	 *
 142	 * Ignore the return code altogether for walkers outside a fault handler
 143	 * (e.g. write protecting a range of memory) and chug along with the
 144	 * page table walk.
 145	 */
 146	if (r == -EAGAIN)
 147		return walker->flags & KVM_PGTABLE_WALK_IGNORE_EAGAIN;
 148
 149	return !r;
 150}
 151
 152static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
 153			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level);
 154
 155static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
 156				      struct kvm_pgtable_mm_ops *mm_ops,
 157				      kvm_pteref_t pteref, s8 level)
 158{
 159	enum kvm_pgtable_walk_flags flags = data->walker->flags;
 160	kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref);
 161	struct kvm_pgtable_visit_ctx ctx = {
 162		.ptep	= ptep,
 163		.old	= READ_ONCE(*ptep),
 164		.arg	= data->walker->arg,
 165		.mm_ops	= mm_ops,
 166		.start	= data->start,
 167		.addr	= data->addr,
 168		.end	= data->end,
 169		.level	= level,
 170		.flags	= flags,
 171	};
 172	int ret = 0;
 173	bool reload = false;
 174	kvm_pteref_t childp;
 175	bool table = kvm_pte_table(ctx.old, level);
 176
 177	if (table && (ctx.flags & KVM_PGTABLE_WALK_TABLE_PRE)) {
 178		ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_PRE);
 179		reload = true;
 180	}
 181
 182	if (!table && (ctx.flags & KVM_PGTABLE_WALK_LEAF)) {
 183		ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_LEAF);
 184		reload = true;
 185	}
 186
 187	/*
 188	 * Reload the page table after invoking the walker callback for leaf
 189	 * entries or after pre-order traversal, to allow the walker to descend
 190	 * into a newly installed or replaced table.
 191	 */
 192	if (reload) {
 193		ctx.old = READ_ONCE(*ptep);
 194		table = kvm_pte_table(ctx.old, level);
 195	}
 196
 197	if (!kvm_pgtable_walk_continue(data->walker, ret))
 198		goto out;
 199
 200	if (!table) {
 201		data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level));
 202		data->addr += kvm_granule_size(level);
 203		goto out;
 204	}
 205
 206	childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);
 207	ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);
 208	if (!kvm_pgtable_walk_continue(data->walker, ret))
 209		goto out;
 210
 211	if (ctx.flags & KVM_PGTABLE_WALK_TABLE_POST)
 212		ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_POST);
 213
 214out:
 215	if (kvm_pgtable_walk_continue(data->walker, ret))
 216		return 0;
 217
 218	return ret;
 219}
 220
 221static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
 222			      struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level)
 223{
 224	u32 idx;
 225	int ret = 0;
 226
 227	if (WARN_ON_ONCE(level < KVM_PGTABLE_FIRST_LEVEL ||
 228			 level > KVM_PGTABLE_LAST_LEVEL))
 229		return -EINVAL;
 230
 231	for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) {
 232		kvm_pteref_t pteref = &pgtable[idx];
 233
 234		if (data->addr >= data->end)
 235			break;
 236
 237		ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);
 238		if (ret)
 239			break;
 240	}
 241
 242	return ret;
 243}
 244
 245static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data)
 246{
 247	u32 idx;
 248	int ret = 0;
 249	u64 limit = BIT(pgt->ia_bits);
 250
 251	if (data->addr > limit || data->end > limit)
 252		return -ERANGE;
 253
 254	if (!pgt->pgd)
 255		return -EINVAL;
 256
 257	for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {
 258		kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];
 259
 260		ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);
 261		if (ret)
 262			break;
 263	}
 264
 265	return ret;
 266}
 267
 268int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
 269		     struct kvm_pgtable_walker *walker)
 270{
 271	struct kvm_pgtable_walk_data walk_data = {
 272		.start	= ALIGN_DOWN(addr, PAGE_SIZE),
 273		.addr	= ALIGN_DOWN(addr, PAGE_SIZE),
 274		.end	= PAGE_ALIGN(walk_data.addr + size),
 275		.walker	= walker,
 276	};
 277	int r;
 278
 279	r = kvm_pgtable_walk_begin(walker);
 280	if (r)
 281		return r;
 282
 283	r = _kvm_pgtable_walk(pgt, &walk_data);
 284	kvm_pgtable_walk_end(walker);
 285
 286	return r;
 287}
 288
 289struct leaf_walk_data {
 290	kvm_pte_t	pte;
 291	s8		level;
 292};
 293
 294static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx,
 295		       enum kvm_pgtable_walk_flags visit)
 296{
 297	struct leaf_walk_data *data = ctx->arg;
 298
 299	data->pte   = ctx->old;
 300	data->level = ctx->level;
 301
 302	return 0;
 303}
 304
 305int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
 306			 kvm_pte_t *ptep, s8 *level)
 307{
 308	struct leaf_walk_data data;
 309	struct kvm_pgtable_walker walker = {
 310		.cb	= leaf_walker,
 311		.flags	= KVM_PGTABLE_WALK_LEAF,
 312		.arg	= &data,
 313	};
 314	int ret;
 315
 316	ret = kvm_pgtable_walk(pgt, ALIGN_DOWN(addr, PAGE_SIZE),
 317			       PAGE_SIZE, &walker);
 318	if (!ret) {
 319		if (ptep)
 320			*ptep  = data.pte;
 321		if (level)
 322			*level = data.level;
 323	}
 324
 325	return ret;
 326}
 327
 328struct hyp_map_data {
 329	const u64			phys;
 330	kvm_pte_t			attr;
 331};
 332
 333static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)
 334{
 335	bool device = prot & KVM_PGTABLE_PROT_DEVICE;
 336	u32 mtype = device ? MT_DEVICE_nGnRE : MT_NORMAL;
 337	kvm_pte_t attr = FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_ATTRIDX, mtype);
 338	u32 sh = KVM_PTE_LEAF_ATTR_LO_S1_SH_IS;
 339	u32 ap = (prot & KVM_PGTABLE_PROT_W) ? KVM_PTE_LEAF_ATTR_LO_S1_AP_RW :
 340					       KVM_PTE_LEAF_ATTR_LO_S1_AP_RO;
 341
 342	if (!(prot & KVM_PGTABLE_PROT_R))
 343		return -EINVAL;
 344
 345	if (prot & KVM_PGTABLE_PROT_X) {
 346		if (prot & KVM_PGTABLE_PROT_W)
 347			return -EINVAL;
 348
 349		if (device)
 350			return -EINVAL;
 351
 352		if (system_supports_bti_kernel())
 353			attr |= KVM_PTE_LEAF_ATTR_HI_S1_GP;
 354	} else {
 355		attr |= KVM_PTE_LEAF_ATTR_HI_S1_XN;
 356	}
 357
 358	attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_AP, ap);
 359	if (!kvm_lpa2_is_enabled())
 360		attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_SH, sh);
 361	attr |= KVM_PTE_LEAF_ATTR_LO_S1_AF;
 362	attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW;
 363	*ptep = attr;
 364
 365	return 0;
 366}
 367
 368enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte)
 369{
 370	enum kvm_pgtable_prot prot = pte & KVM_PTE_LEAF_ATTR_HI_SW;
 371	u32 ap;
 372
 373	if (!kvm_pte_valid(pte))
 374		return prot;
 375
 376	if (!(pte & KVM_PTE_LEAF_ATTR_HI_S1_XN))
 377		prot |= KVM_PGTABLE_PROT_X;
 378
 379	ap = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_AP, pte);
 380	if (ap == KVM_PTE_LEAF_ATTR_LO_S1_AP_RO)
 381		prot |= KVM_PGTABLE_PROT_R;
 382	else if (ap == KVM_PTE_LEAF_ATTR_LO_S1_AP_RW)
 383		prot |= KVM_PGTABLE_PROT_RW;
 384
 385	return prot;
 386}
 387
 388static bool hyp_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
 389				    struct hyp_map_data *data)
 390{
 391	u64 phys = data->phys + (ctx->addr - ctx->start);
 392	kvm_pte_t new;
 393
 394	if (!kvm_block_mapping_supported(ctx, phys))
 395		return false;
 396
 397	new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
 398	if (ctx->old == new)
 399		return true;
 400	if (!kvm_pte_valid(ctx->old))
 401		ctx->mm_ops->get_page(ctx->ptep);
 402	else if (WARN_ON((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
 403		return false;
 404
 405	smp_store_release(ctx->ptep, new);
 406	return true;
 407}
 408
 409static int hyp_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
 410			  enum kvm_pgtable_walk_flags visit)
 411{
 412	kvm_pte_t *childp, new;
 413	struct hyp_map_data *data = ctx->arg;
 414	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
 415
 416	if (hyp_map_walker_try_leaf(ctx, data))
 417		return 0;
 418
 419	if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL))
 420		return -EINVAL;
 421
 422	childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
 423	if (!childp)
 424		return -ENOMEM;
 425
 426	new = kvm_init_table_pte(childp, mm_ops);
 427	mm_ops->get_page(ctx->ptep);
 428	smp_store_release(ctx->ptep, new);
 429
 430	return 0;
 431}
 432
 433int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
 434			enum kvm_pgtable_prot prot)
 435{
 436	int ret;
 437	struct hyp_map_data map_data = {
 438		.phys	= ALIGN_DOWN(phys, PAGE_SIZE),
 439	};
 440	struct kvm_pgtable_walker walker = {
 441		.cb	= hyp_map_walker,
 442		.flags	= KVM_PGTABLE_WALK_LEAF,
 443		.arg	= &map_data,
 444	};
 445
 446	ret = hyp_set_prot_attr(prot, &map_data.attr);
 447	if (ret)
 448		return ret;
 449
 450	ret = kvm_pgtable_walk(pgt, addr, size, &walker);
 451	dsb(ishst);
 452	isb();
 453	return ret;
 454}
 455
 456static int hyp_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
 457			    enum kvm_pgtable_walk_flags visit)
 458{
 459	kvm_pte_t *childp = NULL;
 460	u64 granule = kvm_granule_size(ctx->level);
 461	u64 *unmapped = ctx->arg;
 462	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
 463
 464	if (!kvm_pte_valid(ctx->old))
 465		return -EINVAL;
 466
 467	if (kvm_pte_table(ctx->old, ctx->level)) {
 468		childp = kvm_pte_follow(ctx->old, mm_ops);
 469
 470		if (mm_ops->page_count(childp) != 1)
 471			return 0;
 472
 473		kvm_clear_pte(ctx->ptep);
 474		dsb(ishst);
 475		__tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), TLBI_TTL_UNKNOWN);
 476	} else {
 477		if (ctx->end - ctx->addr < granule)
 478			return -EINVAL;
 479
 480		kvm_clear_pte(ctx->ptep);
 481		dsb(ishst);
 482		__tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level);
 483		*unmapped += granule;
 484	}
 485
 486	dsb(ish);
 487	isb();
 488	mm_ops->put_page(ctx->ptep);
 489
 490	if (childp)
 491		mm_ops->put_page(childp);
 492
 493	return 0;
 494}
 495
 496u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
 497{
 498	u64 unmapped = 0;
 499	struct kvm_pgtable_walker walker = {
 500		.cb	= hyp_unmap_walker,
 501		.arg	= &unmapped,
 502		.flags	= KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
 503	};
 504
 505	if (!pgt->mm_ops->page_count)
 506		return 0;
 507
 508	kvm_pgtable_walk(pgt, addr, size, &walker);
 509	return unmapped;
 510}
 511
 512int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
 513			 struct kvm_pgtable_mm_ops *mm_ops)
 514{
 515	s8 start_level = KVM_PGTABLE_LAST_LEVEL + 1 -
 516			 ARM64_HW_PGTABLE_LEVELS(va_bits);
 517
 518	if (start_level < KVM_PGTABLE_FIRST_LEVEL ||
 519	    start_level > KVM_PGTABLE_LAST_LEVEL)
 520		return -EINVAL;
 521
 522	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);
 523	if (!pgt->pgd)
 524		return -ENOMEM;
 525
 526	pgt->ia_bits		= va_bits;
 527	pgt->start_level	= start_level;
 528	pgt->mm_ops		= mm_ops;
 529	pgt->mmu		= NULL;
 530	pgt->force_pte_cb	= NULL;
 531
 532	return 0;
 533}
 534
 535static int hyp_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
 536			   enum kvm_pgtable_walk_flags visit)
 537{
 538	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
 539
 540	if (!kvm_pte_valid(ctx->old))
 541		return 0;
 542
 543	mm_ops->put_page(ctx->ptep);
 544
 545	if (kvm_pte_table(ctx->old, ctx->level))
 546		mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));
 547
 548	return 0;
 549}
 550
 551void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)
 552{
 553	struct kvm_pgtable_walker walker = {
 554		.cb	= hyp_free_walker,
 555		.flags	= KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
 556	};
 557
 558	WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
 559	pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));
 560	pgt->pgd = NULL;
 561}
 562
 563struct stage2_map_data {
 564	const u64			phys;
 565	kvm_pte_t			attr;
 566	u8				owner_id;
 567
 568	kvm_pte_t			*anchor;
 569	kvm_pte_t			*childp;
 570
 571	struct kvm_s2_mmu		*mmu;
 572	void				*memcache;
 573
 574	/* Force mappings to page granularity */
 575	bool				force_pte;
 576
 577	/* Walk should update owner_id only */
 578	bool				annotation;
 579};
 580
 581u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift)
 582{
 583	u64 vtcr = VTCR_EL2_FLAGS;
 584	s8 lvls;
 585
 586	vtcr |= kvm_get_parange(mmfr0) << VTCR_EL2_PS_SHIFT;
 587	vtcr |= VTCR_EL2_T0SZ(phys_shift);
 588	/*
 589	 * Use a minimum 2 level page table to prevent splitting
 590	 * host PMD huge pages at stage2.
 591	 */
 592	lvls = stage2_pgtable_levels(phys_shift);
 593	if (lvls < 2)
 594		lvls = 2;
 595
 596	/*
 597	 * When LPA2 is enabled, the HW supports an extra level of translation
 598	 * (for 5 in total) when using 4K pages. It also introduces VTCR_EL2.SL2
 599	 * to as an addition to SL0 to enable encoding this extra start level.
 600	 * However, since we always use concatenated pages for the first level
 601	 * lookup, we will never need this extra level and therefore do not need
 602	 * to touch SL2.
 603	 */
 604	vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls);
 605
 606#ifdef CONFIG_ARM64_HW_AFDBM
 607	/*
 608	 * Enable the Hardware Access Flag management, unconditionally
 609	 * on all CPUs. In systems that have asymmetric support for the feature
 610	 * this allows KVM to leverage hardware support on the subset of cores
 611	 * that implement the feature.
 612	 *
 613	 * The architecture requires VTCR_EL2.HA to be RES0 (thus ignored by
 614	 * hardware) on implementations that do not advertise support for the
 615	 * feature. As such, setting HA unconditionally is safe, unless you
 616	 * happen to be running on a design that has unadvertised support for
 617	 * HAFDBS. Here be dragons.
 618	 */
 619	if (!cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38))
 620		vtcr |= VTCR_EL2_HA;
 621#endif /* CONFIG_ARM64_HW_AFDBM */
 622
 623	if (kvm_lpa2_is_enabled())
 624		vtcr |= VTCR_EL2_DS;
 625
 626	/* Set the vmid bits */
 627	vtcr |= (get_vmid_bits(mmfr1) == 16) ?
 628		VTCR_EL2_VS_16BIT :
 629		VTCR_EL2_VS_8BIT;
 630
 631	return vtcr;
 632}
 633
 634static bool stage2_has_fwb(struct kvm_pgtable *pgt)
 635{
 636	if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
 637		return false;
 638
 639	return !(pgt->flags & KVM_PGTABLE_S2_NOFWB);
 640}
 641
 642void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu,
 643				phys_addr_t addr, size_t size)
 644{
 645	unsigned long pages, inval_pages;
 646
 647	if (!system_supports_tlb_range()) {
 648		kvm_call_hyp(__kvm_tlb_flush_vmid, mmu);
 649		return;
 650	}
 651
 652	pages = size >> PAGE_SHIFT;
 653	while (pages > 0) {
 654		inval_pages = min(pages, MAX_TLBI_RANGE_PAGES);
 655		kvm_call_hyp(__kvm_tlb_flush_vmid_range, mmu, addr, inval_pages);
 656
 657		addr += inval_pages << PAGE_SHIFT;
 658		pages -= inval_pages;
 659	}
 660}
 661
 662#define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt))
 663
 664static int stage2_set_xn_attr(enum kvm_pgtable_prot prot, kvm_pte_t *attr)
 665{
 666	bool px, ux;
 667	u8 xn;
 668
 669	px = prot & KVM_PGTABLE_PROT_PX;
 670	ux = prot & KVM_PGTABLE_PROT_UX;
 671
 672	if (!cpus_have_final_cap(ARM64_HAS_XNX) && px != ux)
 673		return -EINVAL;
 674
 675	if (px && ux)
 676		xn = 0b00;
 677	else if (!px && ux)
 678		xn = 0b01;
 679	else if (!px && !ux)
 680		xn = 0b10;
 681	else
 682		xn = 0b11;
 683
 684	*attr &= ~KVM_PTE_LEAF_ATTR_HI_S2_XN;
 685	*attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, xn);
 686	return 0;
 687}
 688
 689static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,
 690				kvm_pte_t *ptep)
 691{
 692	kvm_pte_t attr;
 693	u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS;
 694	int r;
 695
 696	switch (prot & (KVM_PGTABLE_PROT_DEVICE |
 697			KVM_PGTABLE_PROT_NORMAL_NC)) {
 698	case KVM_PGTABLE_PROT_DEVICE | KVM_PGTABLE_PROT_NORMAL_NC:
 699		return -EINVAL;
 700	case KVM_PGTABLE_PROT_DEVICE:
 701		if (prot & KVM_PGTABLE_PROT_X)
 702			return -EINVAL;
 703		attr = KVM_S2_MEMATTR(pgt, DEVICE_nGnRE);
 704		break;
 705	case KVM_PGTABLE_PROT_NORMAL_NC:
 706		if (prot & KVM_PGTABLE_PROT_X)
 707			return -EINVAL;
 708		attr = KVM_S2_MEMATTR(pgt, NORMAL_NC);
 709		break;
 710	default:
 711		attr = KVM_S2_MEMATTR(pgt, NORMAL);
 712	}
 713
 714	r = stage2_set_xn_attr(prot, &attr);
 715	if (r)
 716		return r;
 717
 718	if (prot & KVM_PGTABLE_PROT_R)
 719		attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
 720
 721	if (prot & KVM_PGTABLE_PROT_W)
 722		attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
 723
 724	if (!kvm_lpa2_is_enabled())
 725		attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh);
 726
 727	attr |= KVM_PTE_LEAF_ATTR_LO_S2_AF;
 728	attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW;
 729	*ptep = attr;
 730
 731	return 0;
 732}
 733
 734enum kvm_pgtable_prot kvm_pgtable_stage2_pte_prot(kvm_pte_t pte)
 735{
 736	enum kvm_pgtable_prot prot = pte & KVM_PTE_LEAF_ATTR_HI_SW;
 737
 738	if (!kvm_pte_valid(pte))
 739		return prot;
 740
 741	if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R)
 742		prot |= KVM_PGTABLE_PROT_R;
 743	if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W)
 744		prot |= KVM_PGTABLE_PROT_W;
 745
 746	switch (FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, pte)) {
 747	case 0b00:
 748		prot |= KVM_PGTABLE_PROT_PX | KVM_PGTABLE_PROT_UX;
 749		break;
 750	case 0b01:
 751		prot |= KVM_PGTABLE_PROT_UX;
 752		break;
 753	case 0b11:
 754		prot |= KVM_PGTABLE_PROT_PX;
 755		break;
 756	default:
 757		break;
 758	}
 759
 760	return prot;
 761}
 762
 763static bool stage2_pte_needs_update(kvm_pte_t old, kvm_pte_t new)
 764{
 765	if (!kvm_pte_valid(old) || !kvm_pte_valid(new))
 766		return true;
 767
 768	return ((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS));
 769}
 770
 771static bool stage2_pte_is_counted(kvm_pte_t pte)
 772{
 773	/*
 774	 * The refcount tracks valid entries as well as invalid entries if they
 775	 * encode ownership of a page to another entity than the page-table
 776	 * owner, whose id is 0.
 777	 */
 778	return !!pte;
 779}
 780
 781static bool stage2_pte_is_locked(kvm_pte_t pte)
 782{
 783	return !kvm_pte_valid(pte) && (pte & KVM_INVALID_PTE_LOCKED);
 784}
 785
 786static bool stage2_try_set_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
 787{
 788	if (!kvm_pgtable_walk_shared(ctx)) {
 789		WRITE_ONCE(*ctx->ptep, new);
 790		return true;
 791	}
 792
 793	return cmpxchg(ctx->ptep, ctx->old, new) == ctx->old;
 794}
 795
 796/**
 797 * stage2_try_break_pte() - Invalidates a pte according to the
 798 *			    'break-before-make' requirements of the
 799 *			    architecture.
 800 *
 801 * @ctx: context of the visited pte.
 802 * @mmu: stage-2 mmu
 803 *
 804 * Returns: true if the pte was successfully broken.
 805 *
 806 * If the removed pte was valid, performs the necessary serialization and TLB
 807 * invalidation for the old value. For counted ptes, drops the reference count
 808 * on the containing table page.
 809 */
 810static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx,
 811				 struct kvm_s2_mmu *mmu)
 812{
 813	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
 814
 815	if (stage2_pte_is_locked(ctx->old)) {
 816		/*
 817		 * Should never occur if this walker has exclusive access to the
 818		 * page tables.
 819		 */
 820		WARN_ON(!kvm_pgtable_walk_shared(ctx));
 821		return false;
 822	}
 823
 824	if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED))
 825		return false;
 826
 827	if (!kvm_pgtable_walk_skip_bbm_tlbi(ctx)) {
 828		/*
 829		 * Perform the appropriate TLB invalidation based on the
 830		 * evicted pte value (if any).
 831		 */
 832		if (kvm_pte_table(ctx->old, ctx->level)) {
 833			u64 size = kvm_granule_size(ctx->level);
 834			u64 addr = ALIGN_DOWN(ctx->addr, size);
 835
 836			kvm_tlb_flush_vmid_range(mmu, addr, size);
 837		} else if (kvm_pte_valid(ctx->old)) {
 838			kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu,
 839				     ctx->addr, ctx->level);
 840		}
 841	}
 842
 843	if (stage2_pte_is_counted(ctx->old))
 844		mm_ops->put_page(ctx->ptep);
 845
 846	return true;
 847}
 848
 849static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
 850{
 851	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
 852
 853	WARN_ON(!stage2_pte_is_locked(*ctx->ptep));
 854
 855	if (stage2_pte_is_counted(new))
 856		mm_ops->get_page(ctx->ptep);
 857
 858	smp_store_release(ctx->ptep, new);
 859}
 860
 861static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt)
 862{
 863	/*
 864	 * If FEAT_TLBIRANGE is implemented, defer the individual
 865	 * TLB invalidations until the entire walk is finished, and
 866	 * then use the range-based TLBI instructions to do the
 867	 * invalidations. Condition deferred TLB invalidation on the
 868	 * system supporting FWB as the optimization is entirely
 869	 * pointless when the unmap walker needs to perform CMOs.
 870	 */
 871	return system_supports_tlb_range() && stage2_has_fwb(pgt);
 872}
 873
 874static void stage2_unmap_put_pte(const struct kvm_pgtable_visit_ctx *ctx,
 875				struct kvm_s2_mmu *mmu,
 876				struct kvm_pgtable_mm_ops *mm_ops)
 877{
 878	struct kvm_pgtable *pgt = ctx->arg;
 879
 880	/*
 881	 * Clear the existing PTE, and perform break-before-make if it was
 882	 * valid. Depending on the system support, defer the TLB maintenance
 883	 * for the same until the entire unmap walk is completed.
 884	 */
 885	if (kvm_pte_valid(ctx->old)) {
 886		kvm_clear_pte(ctx->ptep);
 887
 888		if (kvm_pte_table(ctx->old, ctx->level)) {
 889			kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
 890				     TLBI_TTL_UNKNOWN);
 891		} else if (!stage2_unmap_defer_tlb_flush(pgt)) {
 892			kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
 893				     ctx->level);
 894		}
 895	}
 896
 897	mm_ops->put_page(ctx->ptep);
 898}
 899
 900static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)
 901{
 902	u64 memattr = pte & KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR;
 903	return kvm_pte_valid(pte) && memattr == KVM_S2_MEMATTR(pgt, NORMAL);
 904}
 905
 906static bool stage2_pte_executable(kvm_pte_t pte)
 907{
 908	return kvm_pte_valid(pte) && !(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN);
 909}
 910
 911static u64 stage2_map_walker_phys_addr(const struct kvm_pgtable_visit_ctx *ctx,
 912				       const struct stage2_map_data *data)
 913{
 914	u64 phys = data->phys;
 915
 916	/* Work out the correct PA based on how far the walk has gotten */
 917	return phys + (ctx->addr - ctx->start);
 918}
 919
 920static bool stage2_leaf_mapping_allowed(const struct kvm_pgtable_visit_ctx *ctx,
 921					struct stage2_map_data *data)
 922{
 923	u64 phys = stage2_map_walker_phys_addr(ctx, data);
 924
 925	if (data->force_pte && ctx->level < KVM_PGTABLE_LAST_LEVEL)
 926		return false;
 927
 928	if (data->annotation)
 929		return true;
 930
 931	return kvm_block_mapping_supported(ctx, phys);
 932}
 933
 934static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
 935				      struct stage2_map_data *data)
 936{
 937	kvm_pte_t new;
 938	u64 phys = stage2_map_walker_phys_addr(ctx, data);
 939	u64 granule = kvm_granule_size(ctx->level);
 940	struct kvm_pgtable *pgt = data->mmu->pgt;
 941	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
 942
 943	if (!stage2_leaf_mapping_allowed(ctx, data))
 944		return -E2BIG;
 945
 946	if (!data->annotation)
 947		new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
 948	else
 949		new = kvm_init_invalid_leaf_owner(data->owner_id);
 950
 951	/*
 952	 * Skip updating the PTE if we are trying to recreate the exact
 953	 * same mapping or only change the access permissions. Instead,
 954	 * the vCPU will exit one more time from guest if still needed
 955	 * and then go through the path of relaxing permissions.
 956	 */
 957	if (!stage2_pte_needs_update(ctx->old, new))
 958		return -EAGAIN;
 959
 960	/* If we're only changing software bits, then store them and go! */
 961	if (!kvm_pgtable_walk_shared(ctx) &&
 962	    !((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW)) {
 963		bool old_is_counted = stage2_pte_is_counted(ctx->old);
 964
 965		if (old_is_counted != stage2_pte_is_counted(new)) {
 966			if (old_is_counted)
 967				mm_ops->put_page(ctx->ptep);
 968			else
 969				mm_ops->get_page(ctx->ptep);
 970		}
 971		WARN_ON_ONCE(!stage2_try_set_pte(ctx, new));
 972		return 0;
 973	}
 974
 975	if (!stage2_try_break_pte(ctx, data->mmu))
 976		return -EAGAIN;
 977
 978	/* Perform CMOs before installation of the guest stage-2 PTE */
 979	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc &&
 980	    stage2_pte_cacheable(pgt, new))
 981		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops),
 982					       granule);
 983
 984	if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou &&
 985	    stage2_pte_executable(new))
 986		mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);
 987
 988	stage2_make_pte(ctx, new);
 989
 990	return 0;
 991}
 992
 993static int stage2_map_walk_table_pre(const struct kvm_pgtable_visit_ctx *ctx,
 994				     struct stage2_map_data *data)
 995{
 996	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
 997	kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
 998	int ret;
 999
1000	if (!stage2_leaf_mapping_allowed(ctx, data))
1001		return 0;
1002
1003	ret = stage2_map_walker_try_leaf(ctx, data);
1004	if (ret)
1005		return ret;
1006
1007	mm_ops->free_unlinked_table(childp, ctx->level);
1008	return 0;
1009}
1010
1011static int stage2_map_walk_leaf(const struct kvm_pgtable_visit_ctx *ctx,
1012				struct stage2_map_data *data)
1013{
1014	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1015	kvm_pte_t *childp, new;
1016	int ret;
1017
1018	ret = stage2_map_walker_try_leaf(ctx, data);
1019	if (ret != -E2BIG)
1020		return ret;
1021
1022	if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL))
1023		return -EINVAL;
1024
1025	if (!data->memcache)
1026		return -ENOMEM;
1027
1028	childp = mm_ops->zalloc_page(data->memcache);
1029	if (!childp)
1030		return -ENOMEM;
1031
1032	if (!stage2_try_break_pte(ctx, data->mmu)) {
1033		mm_ops->put_page(childp);
1034		return -EAGAIN;
1035	}
1036
1037	/*
1038	 * If we've run into an existing block mapping then replace it with
1039	 * a table. Accesses beyond 'end' that fall within the new table
1040	 * will be mapped lazily.
1041	 */
1042	new = kvm_init_table_pte(childp, mm_ops);
1043	stage2_make_pte(ctx, new);
1044
1045	return 0;
1046}
1047
1048/*
1049 * The TABLE_PRE callback runs for table entries on the way down, looking
1050 * for table entries which we could conceivably replace with a block entry
1051 * for this mapping. If it finds one it replaces the entry and calls
1052 * kvm_pgtable_mm_ops::free_unlinked_table() to tear down the detached table.
1053 *
1054 * Otherwise, the LEAF callback performs the mapping at the existing leaves
1055 * instead.
1056 */
1057static int stage2_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
1058			     enum kvm_pgtable_walk_flags visit)
1059{
1060	struct stage2_map_data *data = ctx->arg;
1061
1062	switch (visit) {
1063	case KVM_PGTABLE_WALK_TABLE_PRE:
1064		return stage2_map_walk_table_pre(ctx, data);
1065	case KVM_PGTABLE_WALK_LEAF:
1066		return stage2_map_walk_leaf(ctx, data);
1067	default:
1068		return -EINVAL;
1069	}
1070}
1071
1072int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
1073			   u64 phys, enum kvm_pgtable_prot prot,
1074			   void *mc, enum kvm_pgtable_walk_flags flags)
1075{
1076	int ret;
1077	struct stage2_map_data map_data = {
1078		.phys		= ALIGN_DOWN(phys, PAGE_SIZE),
1079		.mmu		= pgt->mmu,
1080		.memcache	= mc,
1081		.force_pte	= pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),
1082	};
1083	struct kvm_pgtable_walker walker = {
1084		.cb		= stage2_map_walker,
1085		.flags		= flags |
1086				  KVM_PGTABLE_WALK_TABLE_PRE |
1087				  KVM_PGTABLE_WALK_LEAF,
1088		.arg		= &map_data,
1089	};
1090
1091	if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys)))
1092		return -EINVAL;
1093
1094	ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);
1095	if (ret)
1096		return ret;
1097
1098	ret = kvm_pgtable_walk(pgt, addr, size, &walker);
1099	dsb(ishst);
1100	return ret;
1101}
1102
1103int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
1104				 void *mc, u8 owner_id)
1105{
1106	int ret;
1107	struct stage2_map_data map_data = {
1108		.mmu		= pgt->mmu,
1109		.memcache	= mc,
1110		.owner_id	= owner_id,
1111		.force_pte	= true,
1112		.annotation	= true,
1113	};
1114	struct kvm_pgtable_walker walker = {
1115		.cb		= stage2_map_walker,
1116		.flags		= KVM_PGTABLE_WALK_TABLE_PRE |
1117				  KVM_PGTABLE_WALK_LEAF,
1118		.arg		= &map_data,
1119	};
1120
1121	if (owner_id > KVM_MAX_OWNER_ID)
1122		return -EINVAL;
1123
1124	ret = kvm_pgtable_walk(pgt, addr, size, &walker);
1125	return ret;
1126}
1127
1128static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
1129			       enum kvm_pgtable_walk_flags visit)
1130{
1131	struct kvm_pgtable *pgt = ctx->arg;
1132	struct kvm_s2_mmu *mmu = pgt->mmu;
1133	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1134	kvm_pte_t *childp = NULL;
1135	bool need_flush = false;
1136
1137	if (!kvm_pte_valid(ctx->old)) {
1138		if (stage2_pte_is_counted(ctx->old)) {
1139			kvm_clear_pte(ctx->ptep);
1140			mm_ops->put_page(ctx->ptep);
1141		}
1142		return 0;
1143	}
1144
1145	if (kvm_pte_table(ctx->old, ctx->level)) {
1146		childp = kvm_pte_follow(ctx->old, mm_ops);
1147
1148		if (mm_ops->page_count(childp) != 1)
1149			return 0;
1150	} else if (stage2_pte_cacheable(pgt, ctx->old)) {
1151		need_flush = !stage2_has_fwb(pgt);
1152	}
1153
1154	/*
1155	 * This is similar to the map() path in that we unmap the entire
1156	 * block entry and rely on the remaining portions being faulted
1157	 * back lazily.
1158	 */
1159	stage2_unmap_put_pte(ctx, mmu, mm_ops);
1160
1161	if (need_flush && mm_ops->dcache_clean_inval_poc)
1162		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1163					       kvm_granule_size(ctx->level));
1164
1165	if (childp)
1166		mm_ops->put_page(childp);
1167
1168	return 0;
1169}
1170
1171int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
1172{
1173	int ret;
1174	struct kvm_pgtable_walker walker = {
1175		.cb	= stage2_unmap_walker,
1176		.arg	= pgt,
1177		.flags	= KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
1178	};
1179
1180	ret = kvm_pgtable_walk(pgt, addr, size, &walker);
1181	if (stage2_unmap_defer_tlb_flush(pgt))
1182		/* Perform the deferred TLB invalidations */
1183		kvm_tlb_flush_vmid_range(pgt->mmu, addr, size);
1184
1185	return ret;
1186}
1187
1188struct stage2_attr_data {
1189	kvm_pte_t			attr_set;
1190	kvm_pte_t			attr_clr;
1191	kvm_pte_t			pte;
1192	s8				level;
1193};
1194
1195static int stage2_attr_walker(const struct kvm_pgtable_visit_ctx *ctx,
1196			      enum kvm_pgtable_walk_flags visit)
1197{
1198	kvm_pte_t pte = ctx->old;
1199	struct stage2_attr_data *data = ctx->arg;
1200	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1201
1202	if (!kvm_pte_valid(ctx->old))
1203		return -EAGAIN;
1204
1205	data->level = ctx->level;
1206	data->pte = pte;
1207	pte &= ~data->attr_clr;
1208	pte |= data->attr_set;
1209
1210	/*
1211	 * We may race with the CPU trying to set the access flag here,
1212	 * but worst-case the access flag update gets lost and will be
1213	 * set on the next access instead.
1214	 */
1215	if (data->pte != pte) {
1216		/*
1217		 * Invalidate instruction cache before updating the guest
1218		 * stage-2 PTE if we are going to add executable permission.
1219		 */
1220		if (mm_ops->icache_inval_pou &&
1221		    stage2_pte_executable(pte) && !stage2_pte_executable(ctx->old))
1222			mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),
1223						  kvm_granule_size(ctx->level));
1224
1225		if (!stage2_try_set_pte(ctx, pte))
1226			return -EAGAIN;
1227	}
1228
1229	return 0;
1230}
1231
1232static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,
1233				    u64 size, kvm_pte_t attr_set,
1234				    kvm_pte_t attr_clr, kvm_pte_t *orig_pte,
1235				    s8 *level, enum kvm_pgtable_walk_flags flags)
1236{
1237	int ret;
1238	kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI;
1239	struct stage2_attr_data data = {
1240		.attr_set	= attr_set & attr_mask,
1241		.attr_clr	= attr_clr & attr_mask,
1242	};
1243	struct kvm_pgtable_walker walker = {
1244		.cb		= stage2_attr_walker,
1245		.arg		= &data,
1246		.flags		= flags | KVM_PGTABLE_WALK_LEAF,
1247	};
1248
1249	ret = kvm_pgtable_walk(pgt, addr, size, &walker);
1250	if (ret)
1251		return ret;
1252
1253	if (orig_pte)
1254		*orig_pte = data.pte;
1255
1256	if (level)
1257		*level = data.level;
1258	return 0;
1259}
1260
1261int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)
1262{
1263	return stage2_update_leaf_attrs(pgt, addr, size, 0,
1264					KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
1265					NULL, NULL,
1266					KVM_PGTABLE_WALK_IGNORE_EAGAIN);
1267}
1268
1269void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,
1270				enum kvm_pgtable_walk_flags flags)
1271{
1272	int ret;
1273
1274	ret = stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,
1275				       NULL, NULL, flags);
1276	if (!ret)
1277		dsb(ishst);
1278}
1279
1280struct stage2_age_data {
1281	bool	mkold;
1282	bool	young;
1283};
1284
1285static int stage2_age_walker(const struct kvm_pgtable_visit_ctx *ctx,
1286			     enum kvm_pgtable_walk_flags visit)
1287{
1288	kvm_pte_t new = ctx->old & ~KVM_PTE_LEAF_ATTR_LO_S2_AF;
1289	struct stage2_age_data *data = ctx->arg;
1290
1291	if (!kvm_pte_valid(ctx->old) || new == ctx->old)
1292		return 0;
1293
1294	data->young = true;
1295
1296	/*
1297	 * stage2_age_walker() is always called while holding the MMU lock for
1298	 * write, so this will always succeed. Nonetheless, this deliberately
1299	 * follows the race detection pattern of the other stage-2 walkers in
1300	 * case the locking mechanics of the MMU notifiers is ever changed.
1301	 */
1302	if (data->mkold && !stage2_try_set_pte(ctx, new))
1303		return -EAGAIN;
1304
1305	/*
1306	 * "But where's the TLBI?!", you scream.
1307	 * "Over in the core code", I sigh.
1308	 *
1309	 * See the '->clear_flush_young()' callback on the KVM mmu notifier.
1310	 */
1311	return 0;
1312}
1313
1314bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr,
1315					 u64 size, bool mkold)
1316{
1317	struct stage2_age_data data = {
1318		.mkold		= mkold,
1319	};
1320	struct kvm_pgtable_walker walker = {
1321		.cb		= stage2_age_walker,
1322		.arg		= &data,
1323		.flags		= KVM_PGTABLE_WALK_LEAF,
1324	};
1325
1326	WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));
1327	return data.young;
1328}
1329
1330int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,
1331				   enum kvm_pgtable_prot prot, enum kvm_pgtable_walk_flags flags)
1332{
1333	kvm_pte_t xn = 0, set = 0, clr = 0;
1334	s8 level;
1335	int ret;
1336
1337	if (prot & KVM_PTE_LEAF_ATTR_HI_SW)
1338		return -EINVAL;
1339
1340	if (prot & KVM_PGTABLE_PROT_R)
1341		set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;
1342
1343	if (prot & KVM_PGTABLE_PROT_W)
1344		set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;
1345
1346	ret = stage2_set_xn_attr(prot, &xn);
1347	if (ret)
1348		return ret;
1349
1350	set |= xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
1351	clr |= ~xn & KVM_PTE_LEAF_ATTR_HI_S2_XN;
1352
1353	ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);
1354	if (!ret || ret == -EAGAIN)
1355		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level);
1356	return ret;
1357}
1358
1359static int stage2_flush_walker(const struct kvm_pgtable_visit_ctx *ctx,
1360			       enum kvm_pgtable_walk_flags visit)
1361{
1362	struct kvm_pgtable *pgt = ctx->arg;
1363	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1364
1365	if (!stage2_pte_cacheable(pgt, ctx->old))
1366		return 0;
1367
1368	if (mm_ops->dcache_clean_inval_poc)
1369		mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
1370					       kvm_granule_size(ctx->level));
1371	return 0;
1372}
1373
1374int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)
1375{
1376	struct kvm_pgtable_walker walker = {
1377		.cb	= stage2_flush_walker,
1378		.flags	= KVM_PGTABLE_WALK_LEAF,
1379		.arg	= pgt,
1380	};
1381
1382	if (stage2_has_fwb(pgt))
1383		return 0;
1384
1385	return kvm_pgtable_walk(pgt, addr, size, &walker);
1386}
1387
1388kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,
1389					      u64 phys, s8 level,
1390					      enum kvm_pgtable_prot prot,
1391					      void *mc, bool force_pte)
1392{
1393	struct stage2_map_data map_data = {
1394		.phys		= phys,
1395		.mmu		= pgt->mmu,
1396		.memcache	= mc,
1397		.force_pte	= force_pte,
1398	};
1399	struct kvm_pgtable_walker walker = {
1400		.cb		= stage2_map_walker,
1401		.flags		= KVM_PGTABLE_WALK_LEAF |
1402				  KVM_PGTABLE_WALK_SKIP_BBM_TLBI |
1403				  KVM_PGTABLE_WALK_SKIP_CMO,
1404		.arg		= &map_data,
1405	};
1406	/*
1407	 * The input address (.addr) is irrelevant for walking an
1408	 * unlinked table. Construct an ambiguous IA range to map
1409	 * kvm_granule_size(level) worth of memory.
1410	 */
1411	struct kvm_pgtable_walk_data data = {
1412		.walker	= &walker,
1413		.addr	= 0,
1414		.end	= kvm_granule_size(level),
1415	};
1416	struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
1417	kvm_pte_t *pgtable;
1418	int ret;
1419
1420	if (!IS_ALIGNED(phys, kvm_granule_size(level)))
1421		return ERR_PTR(-EINVAL);
1422
1423	ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);
1424	if (ret)
1425		return ERR_PTR(ret);
1426
1427	pgtable = mm_ops->zalloc_page(mc);
1428	if (!pgtable)
1429		return ERR_PTR(-ENOMEM);
1430
1431	ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable,
1432				 level + 1);
1433	if (ret) {
1434		kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level);
1435		return ERR_PTR(ret);
1436	}
1437
1438	return pgtable;
1439}
1440
1441/*
1442 * Get the number of page-tables needed to replace a block with a
1443 * fully populated tree up to the PTE entries. Note that @level is
1444 * interpreted as in "level @level entry".
1445 */
1446static int stage2_block_get_nr_page_tables(s8 level)
1447{
1448	switch (level) {
1449	case 1:
1450		return PTRS_PER_PTE + 1;
1451	case 2:
1452		return 1;
1453	case 3:
1454		return 0;
1455	default:
1456		WARN_ON_ONCE(level < KVM_PGTABLE_MIN_BLOCK_LEVEL ||
1457			     level > KVM_PGTABLE_LAST_LEVEL);
1458		return -EINVAL;
1459	};
1460}
1461
1462static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx,
1463			       enum kvm_pgtable_walk_flags visit)
1464{
1465	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1466	struct kvm_mmu_memory_cache *mc = ctx->arg;
1467	struct kvm_s2_mmu *mmu;
1468	kvm_pte_t pte = ctx->old, new, *childp;
1469	enum kvm_pgtable_prot prot;
1470	s8 level = ctx->level;
1471	bool force_pte;
1472	int nr_pages;
1473	u64 phys;
1474
1475	/* No huge-pages exist at the last level */
1476	if (level == KVM_PGTABLE_LAST_LEVEL)
1477		return 0;
1478
1479	/* We only split valid block mappings */
1480	if (!kvm_pte_valid(pte))
1481		return 0;
1482
1483	nr_pages = stage2_block_get_nr_page_tables(level);
1484	if (nr_pages < 0)
1485		return nr_pages;
1486
1487	if (mc->nobjs >= nr_pages) {
1488		/* Build a tree mapped down to the PTE granularity. */
1489		force_pte = true;
1490	} else {
1491		/*
1492		 * Don't force PTEs, so create_unlinked() below does
1493		 * not populate the tree up to the PTE level. The
1494		 * consequence is that the call will require a single
1495		 * page of level 2 entries at level 1, or a single
1496		 * page of PTEs at level 2. If we are at level 1, the
1497		 * PTEs will be created recursively.
1498		 */
1499		force_pte = false;
1500		nr_pages = 1;
1501	}
1502
1503	if (mc->nobjs < nr_pages)
1504		return -ENOMEM;
1505
1506	mmu = container_of(mc, struct kvm_s2_mmu, split_page_cache);
1507	phys = kvm_pte_to_phys(pte);
1508	prot = kvm_pgtable_stage2_pte_prot(pte);
1509
1510	childp = kvm_pgtable_stage2_create_unlinked(mmu->pgt, phys,
1511						    level, prot, mc, force_pte);
1512	if (IS_ERR(childp))
1513		return PTR_ERR(childp);
1514
1515	if (!stage2_try_break_pte(ctx, mmu)) {
1516		kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level);
1517		return -EAGAIN;
1518	}
1519
1520	/*
1521	 * Note, the contents of the page table are guaranteed to be made
1522	 * visible before the new PTE is assigned because stage2_make_pte()
1523	 * writes the PTE using smp_store_release().
1524	 */
1525	new = kvm_init_table_pte(childp, mm_ops);
1526	stage2_make_pte(ctx, new);
1527	return 0;
1528}
1529
1530int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,
1531			     struct kvm_mmu_memory_cache *mc)
1532{
1533	struct kvm_pgtable_walker walker = {
1534		.cb	= stage2_split_walker,
1535		.flags	= KVM_PGTABLE_WALK_LEAF,
1536		.arg	= mc,
1537	};
1538	int ret;
1539
1540	ret = kvm_pgtable_walk(pgt, addr, size, &walker);
1541	dsb(ishst);
1542	return ret;
1543}
1544
1545int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
1546			      struct kvm_pgtable_mm_ops *mm_ops,
1547			      enum kvm_pgtable_stage2_flags flags,
1548			      kvm_pgtable_force_pte_cb_t force_pte_cb)
1549{
1550	size_t pgd_sz;
1551	u64 vtcr = mmu->vtcr;
1552	u32 ia_bits = VTCR_EL2_IPA(vtcr);
1553	u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
1554	s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
1555
1556	pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
1557	pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);
1558	if (!pgt->pgd)
1559		return -ENOMEM;
1560
1561	pgt->ia_bits		= ia_bits;
1562	pgt->start_level	= start_level;
1563	pgt->mm_ops		= mm_ops;
1564	pgt->mmu		= mmu;
1565	pgt->flags		= flags;
1566	pgt->force_pte_cb	= force_pte_cb;
1567
1568	/* Ensure zeroed PGD pages are visible to the hardware walker */
1569	dsb(ishst);
1570	return 0;
1571}
1572
1573size_t kvm_pgtable_stage2_pgd_size(u64 vtcr)
1574{
1575	u32 ia_bits = VTCR_EL2_IPA(vtcr);
1576	u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
1577	s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
1578
1579	return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
1580}
1581
1582static int stage2_free_leaf(const struct kvm_pgtable_visit_ctx *ctx)
1583{
1584	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1585
1586	mm_ops->put_page(ctx->ptep);
1587	return 0;
1588}
1589
1590static int stage2_free_table_post(const struct kvm_pgtable_visit_ctx *ctx)
1591{
1592	struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
1593	kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
1594
1595	if (mm_ops->page_count(childp) != 1)
1596		return 0;
1597
1598	/*
1599	 * Drop references and clear the now stale PTE to avoid rewalking the
1600	 * freed page table.
1601	 */
1602	mm_ops->put_page(ctx->ptep);
1603	mm_ops->put_page(childp);
1604	kvm_clear_pte(ctx->ptep);
1605	return 0;
1606}
1607
1608static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
1609			      enum kvm_pgtable_walk_flags visit)
1610{
1611	if (!stage2_pte_is_counted(ctx->old))
1612		return 0;
1613
1614	switch (visit) {
1615	case KVM_PGTABLE_WALK_LEAF:
1616		return stage2_free_leaf(ctx);
1617	case KVM_PGTABLE_WALK_TABLE_POST:
1618		return stage2_free_table_post(ctx);
1619	default:
1620		return -EINVAL;
1621	}
1622}
1623
1624void kvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,
1625				       u64 addr, u64 size)
1626{
1627	struct kvm_pgtable_walker walker = {
1628		.cb	= stage2_free_walker,
1629		.flags	= KVM_PGTABLE_WALK_LEAF |
1630			  KVM_PGTABLE_WALK_TABLE_POST,
1631	};
1632
1633	WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));
1634}
1635
1636void kvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt)
1637{
1638	size_t pgd_sz;
1639
1640	pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;
1641
1642	/*
1643	 * Since the pgtable is unlinked at this point, and not shared with
1644	 * other walkers, safely deference pgd with kvm_dereference_pteref_raw()
1645	 */
1646	pgt->mm_ops->free_pages_exact(kvm_dereference_pteref_raw(pgt->pgd), pgd_sz);
1647	pgt->pgd = NULL;
1648}
1649
1650void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)
1651{
1652	kvm_pgtable_stage2_destroy_range(pgt, 0, BIT(pgt->ia_bits));
1653	kvm_pgtable_stage2_destroy_pgd(pgt);
1654}
1655
1656void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level)
1657{
1658	kvm_pteref_t ptep = (kvm_pteref_t)pgtable;
1659	struct kvm_pgtable_walker walker = {
1660		.cb	= stage2_free_walker,
1661		.flags	= KVM_PGTABLE_WALK_LEAF |
1662			  KVM_PGTABLE_WALK_TABLE_POST,
1663	};
1664	struct kvm_pgtable_walk_data data = {
1665		.walker	= &walker,
1666
1667		/*
1668		 * At this point the IPA really doesn't matter, as the page
1669		 * table being traversed has already been removed from the stage
1670		 * 2. Set an appropriate range to cover the entire page table.
1671		 */
1672		.addr	= 0,
1673		.end	= kvm_granule_size(level),
1674	};
1675
1676	WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));
1677
1678	WARN_ON(mm_ops->page_count(pgtable) != 1);
1679	mm_ops->put_page(pgtable);
1680}