mmu.c

  1#define pr_fmt(fmt)  "Hyper-V: " fmt
  2
  3#include <linux/log2.h>
  4#include <linux/slab.h>
  5#include <linux/types.h>
  6
  7#include <asm/fpu/api.h>
  8#include <asm/mshyperv.h>
  9#include <asm/msr.h>
 10#include <asm/tlbflush.h>
 11#include <asm/tlb.h>
 12
 13#define CREATE_TRACE_POINTS
 14#include <asm/trace/hyperv.h>
 15
 16/* Each gva in gva_list encodes up to 4096 pages to flush */
 17#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
 18
 19static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
 20				      const struct flush_tlb_info *info);
 21
 22/*
 23 * Fills in gva_list starting from offset. Returns the number of items added.
 24 */
 25static inline int fill_gva_list(u64 gva_list[], int offset,
 26				unsigned long start, unsigned long end)
 27{
 28	int gva_n = offset;
 29	unsigned long cur = start, diff;
 30
 31	do {
 32		diff = end > cur ? end - cur : 0;
 33
 34		gva_list[gva_n] = cur & PAGE_MASK;
 35		/*
 36		 * Lower 12 bits encode the number of additional
 37		 * pages to flush (in addition to the 'cur' page).
 38		 */
 39		if (diff >= HV_TLB_FLUSH_UNIT) {
 40			gva_list[gva_n] |= ~PAGE_MASK;
 41			cur += HV_TLB_FLUSH_UNIT;
 42		}  else if (diff) {
 43			gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
 44			cur = end;
 45		}
 46
 47		gva_n++;
 48
 49	} while (cur < end);
 50
 51	return gva_n - offset;
 52}
 53
 54static bool cpu_is_lazy(int cpu)
 55{
 56	return per_cpu(cpu_tlbstate_shared.is_lazy, cpu);
 57}
 58
 59static void hyperv_flush_tlb_multi(const struct cpumask *cpus,
 60				   const struct flush_tlb_info *info)
 61{
 62	int cpu, vcpu, gva_n, max_gvas;
 63	struct hv_tlb_flush *flush;
 64	u64 status;
 65	unsigned long flags;
 66	bool do_lazy = !info->freed_tables;
 67
 68	trace_hyperv_mmu_flush_tlb_multi(cpus, info);
 69
 70	if (!hv_hypercall_pg)
 71		goto do_native;
 72
 73	local_irq_save(flags);
 74
 75	flush = *this_cpu_ptr(hyperv_pcpu_input_arg);
 76
 77	if (unlikely(!flush)) {
 78		local_irq_restore(flags);
 79		goto do_native;
 80	}
 81
 82	if (info->mm) {
 83		/*
 84		 * AddressSpace argument must match the CR3 with PCID bits
 85		 * stripped out.
 86		 */
 87		flush->address_space = virt_to_phys(info->mm->pgd);
 88		flush->address_space &= CR3_ADDR_MASK;
 89		flush->flags = 0;
 90	} else {
 91		flush->address_space = 0;
 92		flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
 93	}
 94
 95	flush->processor_mask = 0;
 96	if (cpumask_equal(cpus, cpu_present_mask)) {
 97		flush->flags |= HV_FLUSH_ALL_PROCESSORS;
 98	} else {
 99		/*
100		 * From the supplied CPU set we need to figure out if we can get
101		 * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}
102		 * hypercalls. This is possible when the highest VP number in
103		 * the set is < 64. As VP numbers are usually in ascending order
104		 * and match Linux CPU ids, here is an optimization: we check
105		 * the VP number for the highest bit in the supplied set first
106		 * so we can quickly find out if using *_EX hypercalls is a
107		 * must. We will also check all VP numbers when walking the
108		 * supplied CPU set to remain correct in all cases.
109		 */
110		cpu = cpumask_last(cpus);
111
112		if (cpu < nr_cpumask_bits && hv_cpu_number_to_vp_number(cpu) >= 64)
113			goto do_ex_hypercall;
114
115		for_each_cpu(cpu, cpus) {
116			if (do_lazy && cpu_is_lazy(cpu))
117				continue;
118			vcpu = hv_cpu_number_to_vp_number(cpu);
119			if (vcpu == VP_INVAL) {
120				local_irq_restore(flags);
121				goto do_native;
122			}
123
124			if (vcpu >= 64)
125				goto do_ex_hypercall;
126
127			__set_bit(vcpu, (unsigned long *)
128				  &flush->processor_mask);
129		}
130
131		/* nothing to flush if 'processor_mask' ends up being empty */
132		if (!flush->processor_mask) {
133			local_irq_restore(flags);
134			return;
135		}
136	}
137
138	/*
139	 * We can flush not more than max_gvas with one hypercall. Flush the
140	 * whole address space if we were asked to do more.
141	 */
142	max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
143
144	if (info->end == TLB_FLUSH_ALL) {
145		flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
146		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
147					 flush, NULL);
148	} else if (info->end &&
149		   ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
150		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
151					 flush, NULL);
152	} else {
153		gva_n = fill_gva_list(flush->gva_list, 0,
154				      info->start, info->end);
155		status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
156					     gva_n, 0, flush, NULL);
157	}
158	goto check_status;
159
160do_ex_hypercall:
161	status = hyperv_flush_tlb_others_ex(cpus, info);
162
163check_status:
164	local_irq_restore(flags);
165
166	if (hv_result_success(status))
167		return;
168do_native:
169	native_flush_tlb_multi(cpus, info);
170}
171
172static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
173				      const struct flush_tlb_info *info)
174{
175	int nr_bank = 0, max_gvas, gva_n;
176	struct hv_tlb_flush_ex *flush;
177	u64 status;
178
179	if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
180		return HV_STATUS_INVALID_PARAMETER;
181
182	flush = *this_cpu_ptr(hyperv_pcpu_input_arg);
183
184	if (info->mm) {
185		/*
186		 * AddressSpace argument must match the CR3 with PCID bits
187		 * stripped out.
188		 */
189		flush->address_space = virt_to_phys(info->mm->pgd);
190		flush->address_space &= CR3_ADDR_MASK;
191		flush->flags = 0;
192	} else {
193		flush->address_space = 0;
194		flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
195	}
196
197	flush->hv_vp_set.valid_bank_mask = 0;
198
199	flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
200	nr_bank = cpumask_to_vpset_skip(&flush->hv_vp_set, cpus,
201			info->freed_tables ? NULL : cpu_is_lazy);
202	if (nr_bank < 0)
203		return HV_STATUS_INVALID_PARAMETER;
204
205	/*
206	 * We can flush not more than max_gvas with one hypercall. Flush the
207	 * whole address space if we were asked to do more.
208	 *
209	 * For these hypercalls, Hyper-V treats the valid_bank_mask field
210	 * of flush->hv_vp_set as part of the fixed size input header.
211	 * So the variable input header size is equal to nr_bank.
212	 */
213	max_gvas =
214		(PAGE_SIZE - sizeof(*flush) - nr_bank *
215		 sizeof(flush->hv_vp_set.bank_contents[0])) /
216		sizeof(flush->gva_list[0]);
217
218	if (info->end == TLB_FLUSH_ALL) {
219		flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
220		status = hv_do_rep_hypercall(
221			HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
222			0, nr_bank, flush, NULL);
223	} else if (info->end &&
224		   ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
225		status = hv_do_rep_hypercall(
226			HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
227			0, nr_bank, flush, NULL);
228	} else {
229		gva_n = fill_gva_list(flush->gva_list, nr_bank,
230				      info->start, info->end);
231		status = hv_do_rep_hypercall(
232			HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
233			gva_n, nr_bank, flush, NULL);
234	}
235
236	return status;
237}
238
239void hyperv_setup_mmu_ops(void)
240{
241	if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
242		return;
243
244	pr_info("Using hypercall for remote TLB flush\n");
245	pv_ops.mmu.flush_tlb_multi = hyperv_flush_tlb_multi;
246}