paravirt.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/types.h>
  3#include <linux/interrupt.h>
  4#include <linux/irq_work.h>
  5#include <linux/jump_label.h>
  6#include <linux/kvm_para.h>
  7#include <linux/reboot.h>
  8#include <linux/static_call.h>
  9#include <asm/paravirt.h>
 10
 11static int has_steal_clock;
 12struct static_key paravirt_steal_enabled;
 13struct static_key paravirt_steal_rq_enabled;
 14static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64);
 15DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key);
 16
 17static u64 native_steal_clock(int cpu)
 18{
 19	return 0;
 20}
 21
 22DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
 23
 24static bool steal_acc = true;
 25
 26static int __init parse_no_stealacc(char *arg)
 27{
 28	steal_acc = false;
 29	return 0;
 30}
 31early_param("no-steal-acc", parse_no_stealacc);
 32
 33static u64 paravt_steal_clock(int cpu)
 34{
 35	int version;
 36	u64 steal;
 37	struct kvm_steal_time *src;
 38
 39	src = &per_cpu(steal_time, cpu);
 40	do {
 41
 42		version = src->version;
 43		virt_rmb(); /* Make sure that the version is read before the steal */
 44		steal = src->steal;
 45		virt_rmb(); /* Make sure that the steal is read before the next version */
 46
 47	} while ((version & 1) || (version != src->version));
 48
 49	return steal;
 50}
 51
 52#ifdef CONFIG_SMP
 53static struct smp_ops native_ops;
 54
 55static void pv_send_ipi_single(int cpu, unsigned int action)
 56{
 57	int min, old;
 58	irq_cpustat_t *info = &per_cpu(irq_stat, cpu);
 59
 60	if (unlikely(action == ACTION_BOOT_CPU)) {
 61		native_ops.send_ipi_single(cpu, action);
 62		return;
 63	}
 64
 65	old = atomic_fetch_or(BIT(action), &info->message);
 66	if (old)
 67		return;
 68
 69	min = cpu_logical_map(cpu);
 70	kvm_hypercall3(KVM_HCALL_FUNC_IPI, 1, 0, min);
 71}
 72
 73#define KVM_IPI_CLUSTER_SIZE	(2 * BITS_PER_LONG)
 74
 75static void pv_send_ipi_mask(const struct cpumask *mask, unsigned int action)
 76{
 77	int i, cpu, min = 0, max = 0, old;
 78	__uint128_t bitmap = 0;
 79	irq_cpustat_t *info;
 80
 81	if (cpumask_empty(mask))
 82		return;
 83
 84	if (unlikely(action == ACTION_BOOT_CPU)) {
 85		native_ops.send_ipi_mask(mask, action);
 86		return;
 87	}
 88
 89	action = BIT(action);
 90	for_each_cpu(i, mask) {
 91		info = &per_cpu(irq_stat, i);
 92		old = atomic_fetch_or(action, &info->message);
 93		if (old)
 94			continue;
 95
 96		cpu = cpu_logical_map(i);
 97		if (!bitmap) {
 98			min = max = cpu;
 99		} else if (cpu < min && cpu > (max - KVM_IPI_CLUSTER_SIZE)) {
100			/* cpu < min, and bitmap still enough */
101			bitmap <<= min - cpu;
102			min = cpu;
103		} else if (cpu > min && cpu < (min + KVM_IPI_CLUSTER_SIZE)) {
104			/* cpu > min, and bitmap still enough */
105			max = cpu > max ? cpu : max;
106		} else {
107			/*
108			 * With cpu, bitmap will exceed KVM_IPI_CLUSTER_SIZE,
109			 * send IPI here directly and skip the remaining CPUs.
110			 */
111			kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap,
112				      (unsigned long)(bitmap >> BITS_PER_LONG), min);
113			min = max = cpu;
114			bitmap = 0;
115		}
116		__set_bit(cpu - min, (unsigned long *)&bitmap);
117	}
118
119	if (bitmap)
120		kvm_hypercall3(KVM_HCALL_FUNC_IPI, (unsigned long)bitmap,
121			      (unsigned long)(bitmap >> BITS_PER_LONG), min);
122}
123
124static irqreturn_t pv_ipi_interrupt(int irq, void *dev)
125{
126	u32 action;
127	irq_cpustat_t *info;
128
129	/* Clear SWI interrupt */
130	clear_csr_estat(1 << INT_SWI0);
131	info = this_cpu_ptr(&irq_stat);
132	action = atomic_xchg(&info->message, 0);
133
134	if (action & SMP_RESCHEDULE) {
135		scheduler_ipi();
136		info->ipi_irqs[IPI_RESCHEDULE]++;
137	}
138
139	if (action & SMP_CALL_FUNCTION) {
140		generic_smp_call_function_interrupt();
141		info->ipi_irqs[IPI_CALL_FUNCTION]++;
142	}
143
144	if (action & SMP_IRQ_WORK) {
145		irq_work_run();
146		info->ipi_irqs[IPI_IRQ_WORK]++;
147	}
148
149	if (action & SMP_CLEAR_VECTOR) {
150		complete_irq_moving();
151		info->ipi_irqs[IPI_CLEAR_VECTOR]++;
152	}
153
154	return IRQ_HANDLED;
155}
156
157static void pv_init_ipi(void)
158{
159	int r, swi;
160
161	/* Init native ipi irq for ACTION_BOOT_CPU */
162	native_ops.init_ipi();
163	swi = get_percpu_irq(INT_SWI0);
164	if (swi < 0)
165		panic("SWI0 IRQ mapping failed\n");
166	irq_set_percpu_devid(swi);
167	r = request_percpu_irq(swi, pv_ipi_interrupt, "SWI0-IPI", &irq_stat);
168	if (r < 0)
169		panic("SWI0 IRQ request failed\n");
170}
171#endif
172
173bool kvm_para_available(void)
174{
175	int config;
176	static int hypervisor_type;
177
178	if (!cpu_has_hypervisor)
179		return false;
180
181	if (!hypervisor_type) {
182		config = read_cpucfg(CPUCFG_KVM_SIG);
183		if (!memcmp(&config, KVM_SIGNATURE, 4))
184			hypervisor_type = HYPERVISOR_KVM;
185	}
186
187	return hypervisor_type == HYPERVISOR_KVM;
188}
189
190unsigned int kvm_arch_para_features(void)
191{
192	static unsigned int feature;
193
194	if (!kvm_para_available())
195		return 0;
196
197	if (!feature)
198		feature = read_cpucfg(CPUCFG_KVM_FEATURE);
199
200	return feature;
201}
202
203int __init pv_ipi_init(void)
204{
205	if (!kvm_para_has_feature(KVM_FEATURE_IPI))
206		return 0;
207
208#ifdef CONFIG_SMP
209	native_ops		= mp_ops;
210	mp_ops.init_ipi		= pv_init_ipi;
211	mp_ops.send_ipi_single	= pv_send_ipi_single;
212	mp_ops.send_ipi_mask	= pv_send_ipi_mask;
213#endif
214
215	return 0;
216}
217
218static int pv_enable_steal_time(void)
219{
220	int cpu = smp_processor_id();
221	unsigned long addr;
222	struct kvm_steal_time *st;
223
224	if (!has_steal_clock)
225		return -EPERM;
226
227	st = &per_cpu(steal_time, cpu);
228	addr = per_cpu_ptr_to_phys(st);
229
230	/* The whole structure kvm_steal_time should be in one page */
231	if (PFN_DOWN(addr) != PFN_DOWN(addr + sizeof(*st))) {
232		pr_warn("Illegal PV steal time addr %lx\n", addr);
233		return -EFAULT;
234	}
235
236	addr |= KVM_STEAL_PHYS_VALID;
237	kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), addr);
238
239	return 0;
240}
241
242static void pv_disable_steal_time(void)
243{
244	if (has_steal_clock)
245		kvm_hypercall2(KVM_HCALL_FUNC_NOTIFY, BIT(KVM_FEATURE_STEAL_TIME), 0);
246}
247
248#ifdef CONFIG_SMP
249static int pv_time_cpu_online(unsigned int cpu)
250{
251	unsigned long flags;
252
253	local_irq_save(flags);
254	pv_enable_steal_time();
255	local_irq_restore(flags);
256
257	return 0;
258}
259
260static int pv_time_cpu_down_prepare(unsigned int cpu)
261{
262	unsigned long flags;
263
264	local_irq_save(flags);
265	pv_disable_steal_time();
266	local_irq_restore(flags);
267
268	return 0;
269}
270#endif
271
272static void pv_cpu_reboot(void *unused)
273{
274	pv_disable_steal_time();
275}
276
277static int pv_reboot_notify(struct notifier_block *nb, unsigned long code, void *unused)
278{
279	on_each_cpu(pv_cpu_reboot, NULL, 1);
280	return NOTIFY_DONE;
281}
282
283static struct notifier_block pv_reboot_nb = {
284	.notifier_call  = pv_reboot_notify,
285};
286
287int __init pv_time_init(void)
288{
289	int r;
290
291	if (!kvm_para_has_feature(KVM_FEATURE_STEAL_TIME))
292		return 0;
293
294	has_steal_clock = 1;
295	r = pv_enable_steal_time();
296	if (r < 0) {
297		has_steal_clock = 0;
298		return 0;
299	}
300	register_reboot_notifier(&pv_reboot_nb);
301
302#ifdef CONFIG_SMP
303	r = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
304				      "loongarch/pv_time:online",
305				      pv_time_cpu_online, pv_time_cpu_down_prepare);
306	if (r < 0) {
307		has_steal_clock = 0;
308		pr_err("Failed to install cpu hotplug callbacks\n");
309		return r;
310	}
311#endif
312
313	static_call_update(pv_steal_clock, paravt_steal_clock);
314
315	static_key_slow_inc(&paravirt_steal_enabled);
316#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
317	if (steal_acc)
318		static_key_slow_inc(&paravirt_steal_rq_enabled);
319#endif
320
321	pr_info("Using paravirt steal-time\n");
322
323	return 0;
324}
325
326int __init pv_spinlock_init(void)
327{
328	if (!cpu_has_hypervisor)
329		return 0;
330
331	static_branch_enable(&virt_spin_lock_key);
332
333	return 0;
334}