tdp_iter.h

  1// SPDX-License-Identifier: GPL-2.0
  2
  3#ifndef __KVM_X86_MMU_TDP_ITER_H
  4#define __KVM_X86_MMU_TDP_ITER_H
  5
  6#include <linux/kvm_host.h>
  7
  8#include "mmu.h"
  9#include "spte.h"
 10
 11/*
 12 * TDP MMU SPTEs are RCU protected to allow paging structures (non-leaf SPTEs)
 13 * to be zapped while holding mmu_lock for read, and to allow TLB flushes to be
 14 * batched without having to collect the list of zapped SPs.  Flows that can
 15 * remove SPs must service pending TLB flushes prior to dropping RCU protection.
 16 */
 17static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep)
 18{
 19	return READ_ONCE(*rcu_dereference(sptep));
 20}
 21
 22static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte)
 23{
 24	KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));
 25	return xchg(rcu_dereference(sptep), new_spte);
 26}
 27
 28static inline u64 tdp_mmu_clear_spte_bits_atomic(tdp_ptep_t sptep, u64 mask)
 29{
 30	atomic64_t *sptep_atomic = (atomic64_t *)rcu_dereference(sptep);
 31
 32	return (u64)atomic64_fetch_and(~mask, sptep_atomic);
 33}
 34
 35static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte)
 36{
 37	KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));
 38	WRITE_ONCE(*rcu_dereference(sptep), new_spte);
 39}
 40
 41/*
 42 * SPTEs must be modified atomically if they are shadow-present, leaf SPTEs,
 43 * and have volatile bits (bits that can be set outside of mmu_lock) that
 44 * must not be clobbered.
 45 */
 46static inline bool kvm_tdp_mmu_spte_need_atomic_update(u64 old_spte, int level)
 47{
 48	return is_shadow_present_pte(old_spte) &&
 49	       is_last_spte(old_spte, level) &&
 50	       spte_needs_atomic_update(old_spte);
 51}
 52
 53static inline u64 kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 old_spte,
 54					 u64 new_spte, int level)
 55{
 56	if (kvm_tdp_mmu_spte_need_atomic_update(old_spte, level))
 57		return kvm_tdp_mmu_write_spte_atomic(sptep, new_spte);
 58
 59	__kvm_tdp_mmu_write_spte(sptep, new_spte);
 60	return old_spte;
 61}
 62
 63static inline u64 tdp_mmu_clear_spte_bits(tdp_ptep_t sptep, u64 old_spte,
 64					  u64 mask, int level)
 65{
 66	if (kvm_tdp_mmu_spte_need_atomic_update(old_spte, level))
 67		return tdp_mmu_clear_spte_bits_atomic(sptep, mask);
 68
 69	__kvm_tdp_mmu_write_spte(sptep, old_spte & ~mask);
 70	return old_spte;
 71}
 72
 73/*
 74 * A TDP iterator performs a pre-order walk over a TDP paging structure.
 75 */
 76struct tdp_iter {
 77	/*
 78	 * The iterator will traverse the paging structure towards the mapping
 79	 * for this GFN.
 80	 */
 81	gfn_t next_last_level_gfn;
 82	/*
 83	 * The next_last_level_gfn at the time when the thread last
 84	 * yielded. Only yielding when the next_last_level_gfn !=
 85	 * yielded_gfn helps ensure forward progress.
 86	 */
 87	gfn_t yielded_gfn;
 88	/* Pointers to the page tables traversed to reach the current SPTE */
 89	tdp_ptep_t pt_path[PT64_ROOT_MAX_LEVEL];
 90	/* A pointer to the current SPTE */
 91	tdp_ptep_t sptep;
 92	/* The lowest GFN (mask bits excluded) mapped by the current SPTE */
 93	gfn_t gfn;
 94	/* Mask applied to convert the GFN to the mapping GPA */
 95	gfn_t gfn_bits;
 96	/* The level of the root page given to the iterator */
 97	int root_level;
 98	/* The lowest level the iterator should traverse to */
 99	int min_level;
100	/* The iterator's current level within the paging structure */
101	int level;
102	/* The address space ID, i.e. SMM vs. regular. */
103	int as_id;
104	/* A snapshot of the value at sptep */
105	u64 old_spte;
106	/*
107	 * Whether the iterator has a valid state. This will be false if the
108	 * iterator walks off the end of the paging structure.
109	 */
110	bool valid;
111	/*
112	 * True if KVM dropped mmu_lock and yielded in the middle of a walk, in
113	 * which case tdp_iter_next() needs to restart the walk at the root
114	 * level instead of advancing to the next entry.
115	 */
116	bool yielded;
117};
118
119/*
120 * Iterates over every SPTE mapping the GFN range [start, end) in a
121 * preorder traversal.
122 */
123#define for_each_tdp_pte_min_level(iter, kvm, root, min_level, start, end)		  \
124	for (tdp_iter_start(&iter, root, min_level, start, kvm_gfn_root_bits(kvm, root)); \
125	     iter.valid && iter.gfn < end;						  \
126	     tdp_iter_next(&iter))
127
128#define for_each_tdp_pte_min_level_all(iter, root, min_level)		\
129	for (tdp_iter_start(&iter, root, min_level, 0, 0);		\
130		iter.valid && iter.gfn < tdp_mmu_max_gfn_exclusive();	\
131		tdp_iter_next(&iter))
132
133#define for_each_tdp_pte(iter, kvm, root, start, end)				\
134	for_each_tdp_pte_min_level(iter, kvm, root, PG_LEVEL_4K, start, end)
135
136tdp_ptep_t spte_to_child_pt(u64 pte, int level);
137
138void tdp_iter_start(struct tdp_iter *iter, struct kvm_mmu_page *root,
139		    int min_level, gfn_t next_last_level_gfn, gfn_t gfn_bits);
140void tdp_iter_next(struct tdp_iter *iter);
141void tdp_iter_restart(struct tdp_iter *iter);
142
143#endif /* __KVM_X86_MMU_TDP_ITER_H */