smp.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  4 *
  5 * Derived from MIPS:
  6 * Copyright (C) 2000, 2001 Kanoj Sarcar
  7 * Copyright (C) 2000, 2001 Ralf Baechle
  8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc.
  9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation
 10 */
 11#include <linux/acpi.h>
 12#include <linux/cpu.h>
 13#include <linux/cpumask.h>
 14#include <linux/init.h>
 15#include <linux/interrupt.h>
 16#include <linux/irq_work.h>
 17#include <linux/profile.h>
 18#include <linux/seq_file.h>
 19#include <linux/smp.h>
 20#include <linux/threads.h>
 21#include <linux/export.h>
 22#include <linux/suspend.h>
 23#include <linux/syscore_ops.h>
 24#include <linux/time.h>
 25#include <linux/tracepoint.h>
 26#include <linux/sched/hotplug.h>
 27#include <linux/sched/task_stack.h>
 28
 29#include <asm/cpu.h>
 30#include <asm/idle.h>
 31#include <asm/loongson.h>
 32#include <asm/mmu_context.h>
 33#include <asm/numa.h>
 34#include <asm/paravirt.h>
 35#include <asm/processor.h>
 36#include <asm/setup.h>
 37#include <asm/time.h>
 38
 39int __cpu_number_map[NR_CPUS];   /* Map physical to logical */
 40EXPORT_SYMBOL(__cpu_number_map);
 41
 42int __cpu_logical_map[NR_CPUS];		/* Map logical to physical */
 43EXPORT_SYMBOL(__cpu_logical_map);
 44
 45/* Representing the threads (siblings) of each logical CPU */
 46cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
 47EXPORT_SYMBOL(cpu_sibling_map);
 48
 49/* Representing the last level cache shared map of each logical CPU */
 50cpumask_t cpu_llc_shared_map[NR_CPUS] __read_mostly;
 51EXPORT_SYMBOL(cpu_llc_shared_map);
 52
 53/* Representing the core map of multi-core chips of each logical CPU */
 54cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
 55EXPORT_SYMBOL(cpu_core_map);
 56
 57static DECLARE_COMPLETION(cpu_starting);
 58static DECLARE_COMPLETION(cpu_running);
 59
 60/*
 61 * A logcal cpu mask containing only one VPE per core to
 62 * reduce the number of IPIs on large MT systems.
 63 */
 64cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly;
 65EXPORT_SYMBOL(cpu_foreign_map);
 66
 67/* representing cpus for which sibling maps can be computed */
 68static cpumask_t cpu_sibling_setup_map;
 69
 70/* representing cpus for which llc shared maps can be computed */
 71static cpumask_t cpu_llc_shared_setup_map;
 72
 73/* representing cpus for which core maps can be computed */
 74static cpumask_t cpu_core_setup_map;
 75
 76struct secondary_data cpuboot_data;
 77static DEFINE_PER_CPU(int, cpu_state);
 78
 79static const char *ipi_types[NR_IPI] __tracepoint_string = {
 80	[IPI_RESCHEDULE] = "Rescheduling interrupts",
 81	[IPI_CALL_FUNCTION] = "Function call interrupts",
 82	[IPI_IRQ_WORK] = "IRQ work interrupts",
 83	[IPI_CLEAR_VECTOR] = "Clear vector interrupts",
 84};
 85
 86void show_ipi_list(struct seq_file *p, int prec)
 87{
 88	unsigned int cpu, i;
 89
 90	for (i = 0; i < NR_IPI; i++) {
 91		seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : "");
 92		for_each_online_cpu(cpu)
 93			seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, cpu).ipi_irqs[i], 10);
 94		seq_printf(p, " LoongArch  %d  %s\n", i + 1, ipi_types[i]);
 95	}
 96}
 97
 98static inline void set_cpu_core_map(int cpu)
 99{
100	int i;
101
102	cpumask_set_cpu(cpu, &cpu_core_setup_map);
103
104	for_each_cpu(i, &cpu_core_setup_map) {
105		if (cpu_data[cpu].package == cpu_data[i].package) {
106			cpumask_set_cpu(i, &cpu_core_map[cpu]);
107			cpumask_set_cpu(cpu, &cpu_core_map[i]);
108		}
109	}
110}
111
112static inline void set_cpu_llc_shared_map(int cpu)
113{
114	int i;
115
116	cpumask_set_cpu(cpu, &cpu_llc_shared_setup_map);
117
118	for_each_cpu(i, &cpu_llc_shared_setup_map) {
119		if (cpu_to_node(cpu) == cpu_to_node(i)) {
120			cpumask_set_cpu(i, &cpu_llc_shared_map[cpu]);
121			cpumask_set_cpu(cpu, &cpu_llc_shared_map[i]);
122		}
123	}
124}
125
126static inline void clear_cpu_llc_shared_map(int cpu)
127{
128	int i;
129
130	for_each_cpu(i, &cpu_llc_shared_setup_map) {
131		if (cpu_to_node(cpu) == cpu_to_node(i)) {
132			cpumask_clear_cpu(i, &cpu_llc_shared_map[cpu]);
133			cpumask_clear_cpu(cpu, &cpu_llc_shared_map[i]);
134		}
135	}
136
137	cpumask_clear_cpu(cpu, &cpu_llc_shared_setup_map);
138}
139
140static inline void set_cpu_sibling_map(int cpu)
141{
142	int i;
143
144	cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
145
146	for_each_cpu(i, &cpu_sibling_setup_map) {
147		if (cpus_are_siblings(cpu, i)) {
148			cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
149			cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
150		}
151	}
152}
153
154static inline void clear_cpu_sibling_map(int cpu)
155{
156	int i;
157
158	for_each_cpu(i, &cpu_sibling_setup_map) {
159		if (cpus_are_siblings(cpu, i)) {
160			cpumask_clear_cpu(i, &cpu_sibling_map[cpu]);
161			cpumask_clear_cpu(cpu, &cpu_sibling_map[i]);
162		}
163	}
164
165	cpumask_clear_cpu(cpu, &cpu_sibling_setup_map);
166}
167
168/*
169 * Calculate a new cpu_foreign_map mask whenever a
170 * new cpu appears or disappears.
171 */
172void calculate_cpu_foreign_map(void)
173{
174	int i, k, core_present;
175	cpumask_t temp_foreign_map;
176
177	/* Re-calculate the mask */
178	cpumask_clear(&temp_foreign_map);
179	for_each_online_cpu(i) {
180		core_present = 0;
181		for_each_cpu(k, &temp_foreign_map)
182			if (cpus_are_siblings(i, k))
183				core_present = 1;
184		if (!core_present)
185			cpumask_set_cpu(i, &temp_foreign_map);
186	}
187
188	for_each_online_cpu(i)
189		cpumask_andnot(&cpu_foreign_map[i],
190			       &temp_foreign_map, &cpu_sibling_map[i]);
191}
192
193/* Send mailbox buffer via Mail_Send */
194static void csr_mail_send(uint64_t data, int cpu, int mailbox)
195{
196	uint64_t val;
197
198	/* Send high 32 bits */
199	val = IOCSR_MBUF_SEND_BLOCKING;
200	val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
201	val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
202	val |= (data & IOCSR_MBUF_SEND_H32_MASK);
203	iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND);
204
205	/* Send low 32 bits */
206	val = IOCSR_MBUF_SEND_BLOCKING;
207	val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
208	val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
209	val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT);
210	iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND);
211};
212
213static u32 ipi_read_clear(int cpu)
214{
215	u32 action;
216
217	/* Load the ipi register to figure out what we're supposed to do */
218	action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS);
219	/* Clear the ipi register to clear the interrupt */
220	iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR);
221	wbflush();
222
223	return action;
224}
225
226static void ipi_write_action(int cpu, u32 action)
227{
228	uint32_t val;
229
230	val = IOCSR_IPI_SEND_BLOCKING | action;
231	val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT);
232	iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND);
233}
234
235static void loongson_send_ipi_single(int cpu, unsigned int action)
236{
237	ipi_write_action(cpu_logical_map(cpu), (u32)action);
238}
239
240static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action)
241{
242	unsigned int i;
243
244	for_each_cpu(i, mask)
245		ipi_write_action(cpu_logical_map(i), (u32)action);
246}
247
248/*
249 * This function sends a 'reschedule' IPI to another CPU.
250 * it goes straight through and wastes no time serializing
251 * anything. Worst case is that we lose a reschedule ...
252 */
253void arch_smp_send_reschedule(int cpu)
254{
255	mp_ops.send_ipi_single(cpu, ACTION_RESCHEDULE);
256}
257EXPORT_SYMBOL_GPL(arch_smp_send_reschedule);
258
259#ifdef CONFIG_IRQ_WORK
260void arch_irq_work_raise(void)
261{
262	mp_ops.send_ipi_single(smp_processor_id(), ACTION_IRQ_WORK);
263}
264#endif
265
266static irqreturn_t loongson_ipi_interrupt(int irq, void *dev)
267{
268	unsigned int action;
269	unsigned int cpu = smp_processor_id();
270
271	action = ipi_read_clear(cpu_logical_map(cpu));
272
273	if (action & SMP_RESCHEDULE) {
274		scheduler_ipi();
275		per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++;
276	}
277
278	if (action & SMP_CALL_FUNCTION) {
279		generic_smp_call_function_interrupt();
280		per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++;
281	}
282
283	if (action & SMP_IRQ_WORK) {
284		irq_work_run();
285		per_cpu(irq_stat, cpu).ipi_irqs[IPI_IRQ_WORK]++;
286	}
287
288	if (action & SMP_CLEAR_VECTOR) {
289		complete_irq_moving();
290		per_cpu(irq_stat, cpu).ipi_irqs[IPI_CLEAR_VECTOR]++;
291	}
292
293	return IRQ_HANDLED;
294}
295
296static void loongson_init_ipi(void)
297{
298	int r, ipi_irq;
299
300	ipi_irq = get_percpu_irq(INT_IPI);
301	if (ipi_irq < 0)
302		panic("IPI IRQ mapping failed\n");
303
304	irq_set_percpu_devid(ipi_irq);
305	r = request_percpu_irq(ipi_irq, loongson_ipi_interrupt, "IPI", &irq_stat);
306	if (r < 0)
307		panic("IPI IRQ request failed\n");
308}
309
310struct smp_ops mp_ops = {
311	.init_ipi		= loongson_init_ipi,
312	.send_ipi_single	= loongson_send_ipi_single,
313	.send_ipi_mask		= loongson_send_ipi_mask,
314};
315
316static void __init fdt_smp_setup(void)
317{
318#ifdef CONFIG_OF
319	unsigned int cpu, cpuid;
320	struct device_node *node = NULL;
321
322	for_each_of_cpu_node(node) {
323		if (!of_device_is_available(node))
324			continue;
325
326		cpuid = of_get_cpu_hwid(node, 0);
327		if (cpuid >= nr_cpu_ids)
328			continue;
329
330		if (cpuid == loongson_sysconf.boot_cpu_id)
331			cpu = 0;
332		else
333			cpu = find_first_zero_bit(cpumask_bits(cpu_present_mask), NR_CPUS);
334
335		num_processors++;
336		set_cpu_possible(cpu, true);
337		set_cpu_present(cpu, true);
338		__cpu_number_map[cpuid] = cpu;
339		__cpu_logical_map[cpu] = cpuid;
340
341		early_numa_add_cpu(cpuid, 0);
342		set_cpuid_to_node(cpuid, 0);
343	}
344
345	loongson_sysconf.nr_cpus = num_processors;
346	set_bit(0, loongson_sysconf.cores_io_master);
347#endif
348}
349
350void __init loongson_smp_setup(void)
351{
352	fdt_smp_setup();
353
354	if (loongson_sysconf.cores_per_package == 0)
355		loongson_sysconf.cores_per_package = num_processors;
356
357	cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package;
358	cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package;
359
360	pv_ipi_init();
361	iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
362	pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus);
363}
364
365void __init loongson_prepare_cpus(unsigned int max_cpus)
366{
367	int i = 0;
368
369	parse_acpi_topology();
370	cpu_data[0].global_id = cpu_logical_map(0);
371
372	for (i = 0; i < loongson_sysconf.nr_cpus; i++) {
373		set_cpu_present(i, true);
374		csr_mail_send(0, __cpu_logical_map[i], 0);
375	}
376
377	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
378}
379
380/*
381 * Setup the PC, SP, and TP of a secondary processor and start it running!
382 */
383void loongson_boot_secondary(int cpu, struct task_struct *idle)
384{
385	unsigned long entry;
386
387	pr_info("Booting CPU#%d...\n", cpu);
388
389	entry = __pa_symbol((unsigned long)&smpboot_entry);
390	cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle);
391	cpuboot_data.thread_info = (unsigned long)task_thread_info(idle);
392
393	csr_mail_send(entry, cpu_logical_map(cpu), 0);
394
395	loongson_send_ipi_single(cpu, ACTION_BOOT_CPU);
396}
397
398/*
399 * SMP init and finish on secondary CPUs
400 */
401void loongson_init_secondary(void)
402{
403	unsigned int cpu = smp_processor_id();
404	unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 |
405			     ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER | ECFGF_SIP0;
406
407	change_csr_ecfg(ECFG0_IM, imask);
408
409	iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
410
411#ifdef CONFIG_NUMA
412	numa_add_cpu(cpu);
413#endif
414	per_cpu(cpu_state, cpu) = CPU_ONLINE;
415	cpu_data[cpu].package =
416		     cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
417	cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core :
418		     cpu_logical_map(cpu) % loongson_sysconf.cores_per_package;
419	cpu_data[cpu].global_id = cpu_logical_map(cpu);
420}
421
422void loongson_smp_finish(void)
423{
424	local_irq_enable();
425	iocsr_write64(0, LOONGARCH_IOCSR_MBUF0);
426	pr_info("CPU#%d finished\n", smp_processor_id());
427}
428
429#ifdef CONFIG_HOTPLUG_CPU
430
431int loongson_cpu_disable(void)
432{
433	unsigned long flags;
434	unsigned int cpu = smp_processor_id();
435
436	if (io_master(cpu))
437		return -EBUSY;
438
439#ifdef CONFIG_NUMA
440	numa_remove_cpu(cpu);
441#endif
442	set_cpu_online(cpu, false);
443	clear_cpu_sibling_map(cpu);
444	clear_cpu_llc_shared_map(cpu);
445	calculate_cpu_foreign_map();
446	local_irq_save(flags);
447	irq_migrate_all_off_this_cpu();
448	clear_csr_ecfg(ECFG0_IM);
449	local_irq_restore(flags);
450	local_flush_tlb_all();
451
452	return 0;
453}
454
455void loongson_cpu_die(unsigned int cpu)
456{
457	while (per_cpu(cpu_state, cpu) != CPU_DEAD)
458		cpu_relax();
459
460	mb();
461}
462
463static void __noreturn idle_play_dead(void)
464{
465	register uint64_t addr;
466	register void (*init_fn)(void);
467
468	idle_task_exit();
469	local_irq_enable();
470	set_csr_ecfg(ECFGF_IPI);
471	__this_cpu_write(cpu_state, CPU_DEAD);
472
473	__smp_mb();
474	do {
475		__asm__ __volatile__("idle 0\n\t");
476		addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0);
477	} while (addr == 0);
478
479	local_irq_disable();
480	init_fn = (void *)TO_CACHE(addr);
481	iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR);
482
483	init_fn();
484	BUG();
485}
486
487#ifdef CONFIG_HIBERNATION
488static void __noreturn poll_play_dead(void)
489{
490	register uint64_t addr;
491	register void (*init_fn)(void);
492
493	idle_task_exit();
494	__this_cpu_write(cpu_state, CPU_DEAD);
495
496	__smp_mb();
497	do {
498		__asm__ __volatile__("nop\n\t");
499		addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0);
500	} while (addr == 0);
501
502	init_fn = (void *)TO_CACHE(addr);
503	iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR);
504
505	init_fn();
506	BUG();
507}
508#endif
509
510static void (*play_dead)(void) = idle_play_dead;
511
512void __noreturn arch_cpu_idle_dead(void)
513{
514	play_dead();
515	BUG(); /* play_dead() doesn't return */
516}
517
518#ifdef CONFIG_HIBERNATION
519int hibernate_resume_nonboot_cpu_disable(void)
520{
521	int ret;
522
523	play_dead = poll_play_dead;
524	ret = suspend_disable_secondary_cpus();
525	play_dead = idle_play_dead;
526
527	return ret;
528}
529#endif
530
531#endif
532
533/*
534 * Power management
535 */
536#ifdef CONFIG_PM
537
538static int loongson_ipi_suspend(void *data)
539{
540	return 0;
541}
542
543static void loongson_ipi_resume(void *data)
544{
545	iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
546}
547
548static const struct syscore_ops loongson_ipi_syscore_ops = {
549	.resume         = loongson_ipi_resume,
550	.suspend        = loongson_ipi_suspend,
551};
552
553static struct syscore loongson_ipi_syscore = {
554	.ops = &loongson_ipi_syscore_ops,
555};
556
557/*
558 * Enable boot cpu ipi before enabling nonboot cpus
559 * during syscore_resume.
560 */
561static int __init ipi_pm_init(void)
562{
563	register_syscore(&loongson_ipi_syscore);
564	return 0;
565}
566
567core_initcall(ipi_pm_init);
568#endif
569
570/* Preload SMP state for boot cpu */
571void __init smp_prepare_boot_cpu(void)
572{
573	unsigned int cpu, node, rr_node;
574
575	set_cpu_possible(0, true);
576	set_cpu_online(0, true);
577	set_my_cpu_offset(per_cpu_offset(0));
578	numa_add_cpu(0);
579
580	rr_node = first_node(node_online_map);
581	for_each_possible_cpu(cpu) {
582		node = early_cpu_to_node(cpu);
583
584		/*
585		 * The mapping between present cpus and nodes has been
586		 * built during MADT and SRAT parsing.
587		 *
588		 * If possible cpus = present cpus here, early_cpu_to_node
589		 * will return valid node.
590		 *
591		 * If possible cpus > present cpus here (e.g. some possible
592		 * cpus will be added by cpu-hotplug later), for possible but
593		 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE,
594		 * and we just map them to online nodes in round-robin way.
595		 * Once hotplugged, new correct mapping will be built for them.
596		 */
597		if (node != NUMA_NO_NODE)
598			set_cpu_numa_node(cpu, node);
599		else {
600			set_cpu_numa_node(cpu, rr_node);
601			rr_node = next_node_in(rr_node, node_online_map);
602		}
603	}
604
605	pv_spinlock_init();
606}
607
608/* called from main before smp_init() */
609void __init smp_prepare_cpus(unsigned int max_cpus)
610{
611	init_new_context(current, &init_mm);
612	current_thread_info()->cpu = 0;
613	loongson_prepare_cpus(max_cpus);
614	set_cpu_sibling_map(0);
615	set_cpu_llc_shared_map(0);
616	set_cpu_core_map(0);
617	calculate_cpu_foreign_map();
618#ifndef CONFIG_HOTPLUG_CPU
619	init_cpu_present(cpu_possible_mask);
620#endif
621}
622
623int __cpu_up(unsigned int cpu, struct task_struct *tidle)
624{
625	loongson_boot_secondary(cpu, tidle);
626
627	/* Wait for CPU to start and be ready to sync counters */
628	if (!wait_for_completion_timeout(&cpu_starting,
629					 msecs_to_jiffies(5000))) {
630		pr_crit("CPU%u: failed to start\n", cpu);
631		return -EIO;
632	}
633
634	/* Wait for CPU to finish startup & mark itself online before return */
635	wait_for_completion(&cpu_running);
636
637	return 0;
638}
639
640/*
641 * First C code run on the secondary CPUs after being started up by
642 * the master.
643 */
644asmlinkage void start_secondary(void)
645{
646	unsigned int cpu;
647
648	sync_counter();
649	cpu = raw_smp_processor_id();
650	set_my_cpu_offset(per_cpu_offset(cpu));
651
652	cpu_probe();
653	constant_clockevent_init();
654	loongson_init_secondary();
655
656	set_cpu_sibling_map(cpu);
657	set_cpu_llc_shared_map(cpu);
658	set_cpu_core_map(cpu);
659
660	notify_cpu_starting(cpu);
661
662	/* Notify boot CPU that we're starting */
663	complete(&cpu_starting);
664
665	/* The CPU is running, now mark it online */
666	set_cpu_online(cpu, true);
667
668	calculate_cpu_foreign_map();
669
670	/*
671	 * Notify boot CPU that we're up & online and it can safely return
672	 * from __cpu_up()
673	 */
674	complete(&cpu_running);
675
676	/*
677	 * irq will be enabled in loongson_smp_finish(), enabling it too
678	 * early is dangerous.
679	 */
680	WARN_ON_ONCE(!irqs_disabled());
681	loongson_smp_finish();
682
683	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
684}
685
686void __init smp_cpus_done(unsigned int max_cpus)
687{
688}
689
690static void stop_this_cpu(void *dummy)
691{
692	set_cpu_online(smp_processor_id(), false);
693	calculate_cpu_foreign_map();
694	local_irq_disable();
695	while (true);
696}
697
698void smp_send_stop(void)
699{
700	smp_call_function(stop_this_cpu, NULL, 0);
701}
702
703#ifdef CONFIG_PROFILING
704int setup_profiling_timer(unsigned int multiplier)
705{
706	return 0;
707}
708#endif
709
710static void flush_tlb_all_ipi(void *info)
711{
712	local_flush_tlb_all();
713}
714
715void flush_tlb_all(void)
716{
717	on_each_cpu(flush_tlb_all_ipi, NULL, 1);
718}
719
720static void flush_tlb_mm_ipi(void *mm)
721{
722	local_flush_tlb_mm((struct mm_struct *)mm);
723}
724
725void flush_tlb_mm(struct mm_struct *mm)
726{
727	if (atomic_read(&mm->mm_users) == 0)
728		return;		/* happens as a result of exit_mmap() */
729
730	preempt_disable();
731
732	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
733		on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1);
734	} else {
735		unsigned int cpu;
736
737		for_each_online_cpu(cpu) {
738			if (cpu != smp_processor_id() && cpu_context(cpu, mm))
739				cpu_context(cpu, mm) = 0;
740		}
741		local_flush_tlb_mm(mm);
742	}
743
744	preempt_enable();
745}
746
747struct flush_tlb_data {
748	struct vm_area_struct *vma;
749	unsigned long addr1;
750	unsigned long addr2;
751};
752
753static void flush_tlb_range_ipi(void *info)
754{
755	struct flush_tlb_data *fd = info;
756
757	local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
758}
759
760void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
761{
762	struct mm_struct *mm = vma->vm_mm;
763
764	preempt_disable();
765	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
766		struct flush_tlb_data fd = {
767			.vma = vma,
768			.addr1 = start,
769			.addr2 = end,
770		};
771
772		on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1);
773	} else {
774		unsigned int cpu;
775
776		for_each_online_cpu(cpu) {
777			if (cpu != smp_processor_id() && cpu_context(cpu, mm))
778				cpu_context(cpu, mm) = 0;
779		}
780		local_flush_tlb_range(vma, start, end);
781	}
782	preempt_enable();
783}
784
785static void flush_tlb_kernel_range_ipi(void *info)
786{
787	struct flush_tlb_data *fd = info;
788
789	local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
790}
791
792void flush_tlb_kernel_range(unsigned long start, unsigned long end)
793{
794	struct flush_tlb_data fd = {
795		.addr1 = start,
796		.addr2 = end,
797	};
798
799	on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1);
800}
801
802static void flush_tlb_page_ipi(void *info)
803{
804	struct flush_tlb_data *fd = info;
805
806	local_flush_tlb_page(fd->vma, fd->addr1);
807}
808
809void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
810{
811	preempt_disable();
812	if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
813		struct flush_tlb_data fd = {
814			.vma = vma,
815			.addr1 = page,
816		};
817
818		on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1);
819	} else {
820		unsigned int cpu;
821
822		for_each_online_cpu(cpu) {
823			if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm))
824				cpu_context(cpu, vma->vm_mm) = 0;
825		}
826		local_flush_tlb_page(vma, page);
827	}
828	preempt_enable();
829}
830EXPORT_SYMBOL(flush_tlb_page);
831
832static void flush_tlb_one_ipi(void *info)
833{
834	unsigned long vaddr = (unsigned long) info;
835
836	local_flush_tlb_one(vaddr);
837}
838
839void flush_tlb_one(unsigned long vaddr)
840{
841	on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1);
842}
843EXPORT_SYMBOL(flush_tlb_one);