smp.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *	linux/arch/alpha/kernel/smp.c
  4 *
  5 *      2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com)
  6 *            Renamed modified smp_call_function to smp_call_function_on_cpu()
  7 *            Created an function that conforms to the old calling convention
  8 *            of smp_call_function().
  9 *
 10 *            This is helpful for DCPI.
 11 *
 12 */
 13
 14#include <linux/errno.h>
 15#include <linux/kernel.h>
 16#include <linux/kernel_stat.h>
 17#include <linux/module.h>
 18#include <linux/sched/mm.h>
 19#include <linux/mm.h>
 20#include <linux/err.h>
 21#include <linux/threads.h>
 22#include <linux/smp.h>
 23#include <linux/interrupt.h>
 24#include <linux/init.h>
 25#include <linux/delay.h>
 26#include <linux/spinlock.h>
 27#include <linux/irq.h>
 28#include <linux/cache.h>
 29#include <linux/profile.h>
 30#include <linux/bitops.h>
 31#include <linux/cpu.h>
 32
 33#include <asm/hwrpb.h>
 34#include <asm/ptrace.h>
 35#include <linux/atomic.h>
 36
 37#include <asm/io.h>
 38#include <asm/irq.h>
 39#include <asm/mmu_context.h>
 40#include <asm/tlbflush.h>
 41#include <asm/cacheflush.h>
 42
 43#include "proto.h"
 44#include "irq_impl.h"
 45
 46
 47#define DEBUG_SMP 0
 48#if DEBUG_SMP
 49#define DBGS(args)	printk args
 50#else
 51#define DBGS(args)
 52#endif
 53
 54/* A collection of per-processor data.  */
 55struct cpuinfo_alpha cpu_data[NR_CPUS];
 56EXPORT_SYMBOL(cpu_data);
 57
 58/* A collection of single bit ipi messages.  */
 59static struct {
 60	unsigned long bits ____cacheline_aligned;
 61} ipi_data[NR_CPUS] __cacheline_aligned;
 62
 63enum ipi_message_type {
 64	IPI_RESCHEDULE,
 65	IPI_CALL_FUNC,
 66	IPI_CPU_STOP,
 67};
 68
 69/* Set to a secondary's cpuid when it comes online.  */
 70static int smp_secondary_alive = 0;
 71
 72int smp_num_probed;		/* Internal processor count */
 73int smp_num_cpus = 1;		/* Number that came online.  */
 74EXPORT_SYMBOL(smp_num_cpus);
 75
 76/*
 77 * Called by both boot and secondaries to move global data into
 78 *  per-processor storage.
 79 */
 80static inline void __init
 81smp_store_cpu_info(int cpuid)
 82{
 83	cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
 84	cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
 85	cpu_data[cpuid].need_new_asn = 0;
 86	cpu_data[cpuid].asn_lock = 0;
 87}
 88
 89/*
 90 * Ideally sets up per-cpu profiling hooks.  Doesn't do much now...
 91 */
 92static inline void __init
 93smp_setup_percpu_timer(int cpuid)
 94{
 95	cpu_data[cpuid].prof_counter = 1;
 96	cpu_data[cpuid].prof_multiplier = 1;
 97}
 98
 99static void __init
100wait_boot_cpu_to_stop(int cpuid)
101{
102	unsigned long stop = jiffies + 10*HZ;
103
104	while (time_before(jiffies, stop)) {
105	        if (!smp_secondary_alive)
106			return;
107		barrier();
108	}
109
110	printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
111	for (;;)
112		barrier();
113}
114
115/*
116 * Where secondaries begin a life of C.
117 */
118void __init
119smp_callin(void)
120{
121	int cpuid = hard_smp_processor_id();
122
123	if (cpu_online(cpuid)) {
124		printk("??, cpu 0x%x already present??\n", cpuid);
125		BUG();
126	}
127	set_cpu_online(cpuid, true);
128
129	/* Turn on machine checks.  */
130	wrmces(7);
131
132	/* Set trap vectors.  */
133	trap_init();
134
135	/* Set interrupt vector.  */
136	wrent(entInt, 0);
137
138	/* Get our local ticker going. */
139	smp_setup_percpu_timer(cpuid);
140	init_clockevent();
141
142	/* Call platform-specific callin, if specified */
143	if (alpha_mv.smp_callin)
144		alpha_mv.smp_callin();
145
146	/* All kernel threads share the same mm context.  */
147	mmgrab(&init_mm);
148	current->active_mm = &init_mm;
149
150	/* inform the notifiers about the new cpu */
151	notify_cpu_starting(cpuid);
152
153	/* Must have completely accurate bogos.  */
154	local_irq_enable();
155
156	/* Wait boot CPU to stop with irq enabled before running
157	   calibrate_delay. */
158	wait_boot_cpu_to_stop(cpuid);
159	mb();
160	calibrate_delay();
161
162	smp_store_cpu_info(cpuid);
163	/* Allow master to continue only after we written loops_per_jiffy.  */
164	wmb();
165	smp_secondary_alive = 1;
166
167	DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
168	      cpuid, current, current->active_mm));
169
170	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
171}
172
173/* Wait until hwrpb->txrdy is clear for cpu.  Return -1 on timeout.  */
174static int
175wait_for_txrdy (unsigned long cpumask)
176{
177	unsigned long timeout;
178
179	if (!(hwrpb->txrdy & cpumask))
180		return 0;
181
182	timeout = jiffies + 10*HZ;
183	while (time_before(jiffies, timeout)) {
184		if (!(hwrpb->txrdy & cpumask))
185			return 0;
186		udelay(10);
187		barrier();
188	}
189
190	return -1;
191}
192
193/*
194 * Send a message to a secondary's console.  "START" is one such
195 * interesting message.  ;-)
196 */
197static void
198send_secondary_console_msg(char *str, int cpuid)
199{
200	struct percpu_struct *cpu;
201	register char *cp1, *cp2;
202	unsigned long cpumask;
203	size_t len;
204
205	cpu = (struct percpu_struct *)
206		((char*)hwrpb
207		 + hwrpb->processor_offset
208		 + cpuid * hwrpb->processor_size);
209
210	cpumask = (1UL << cpuid);
211	if (wait_for_txrdy(cpumask))
212		goto timeout;
213
214	cp2 = str;
215	len = strlen(cp2);
216	*(unsigned int *)&cpu->ipc_buffer[0] = len;
217	cp1 = (char *) &cpu->ipc_buffer[1];
218	memcpy(cp1, cp2, len);
219
220	/* atomic test and set */
221	wmb();
222	set_bit(cpuid, &hwrpb->rxrdy);
223
224	if (wait_for_txrdy(cpumask))
225		goto timeout;
226	return;
227
228 timeout:
229	printk("Processor %x not ready\n", cpuid);
230}
231
232/*
233 * A secondary console wants to send a message.  Receive it.
234 */
235static void
236recv_secondary_console_msg(void)
237{
238	int mycpu, i, cnt;
239	unsigned long txrdy = hwrpb->txrdy;
240	char *cp1, *cp2, buf[80];
241	struct percpu_struct *cpu;
242
243	DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));
244
245	mycpu = hard_smp_processor_id();
246
247	for (i = 0; i < NR_CPUS; i++) {
248		if (!(txrdy & (1UL << i)))
249			continue;
250
251		DBGS(("recv_secondary_console_msg: "
252		      "TXRDY contains CPU %d.\n", i));
253
254		cpu = (struct percpu_struct *)
255		  ((char*)hwrpb
256		   + hwrpb->processor_offset
257		   + i * hwrpb->processor_size);
258
259 		DBGS(("recv_secondary_console_msg: on %d from %d"
260		      " HALT_REASON 0x%lx FLAGS 0x%lx\n",
261		      mycpu, i, cpu->halt_reason, cpu->flags));
262
263		cnt = cpu->ipc_buffer[0] >> 32;
264		if (cnt <= 0 || cnt >= 80)
265			strcpy(buf, "<<< BOGUS MSG >>>");
266		else {
267			cp1 = (char *) &cpu->ipc_buffer[1];
268			cp2 = buf;
269			memcpy(cp2, cp1, cnt);
270			cp2[cnt] = '\0';
271			
272			while ((cp2 = strchr(cp2, '\r')) != 0) {
273				*cp2 = ' ';
274				if (cp2[1] == '\n')
275					cp2[1] = ' ';
276			}
277		}
278
279		DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
280		      "message is '%s'\n", mycpu, buf));
281	}
282
283	hwrpb->txrdy = 0;
284}
285
286/*
287 * Convince the console to have a secondary cpu begin execution.
288 */
289static int
290secondary_cpu_start(int cpuid, struct task_struct *idle)
291{
292	struct percpu_struct *cpu;
293	struct pcb_struct *hwpcb, *ipcb;
294	unsigned long timeout;
295	  
296	cpu = (struct percpu_struct *)
297		((char*)hwrpb
298		 + hwrpb->processor_offset
299		 + cpuid * hwrpb->processor_size);
300	hwpcb = (struct pcb_struct *) cpu->hwpcb;
301	ipcb = &task_thread_info(idle)->pcb;
302
303	/* Initialize the CPU's HWPCB to something just good enough for
304	   us to get started.  Immediately after starting, we'll swpctx
305	   to the target idle task's pcb.  Reuse the stack in the mean
306	   time.  Precalculate the target PCBB.  */
307	hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
308	hwpcb->usp = 0;
309	hwpcb->ptbr = ipcb->ptbr;
310	hwpcb->pcc = 0;
311	hwpcb->asn = 0;
312	hwpcb->unique = virt_to_phys(ipcb);
313	hwpcb->flags = ipcb->flags;
314	hwpcb->res1 = hwpcb->res2 = 0;
315
316#if 0
317	DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
318	      hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
319#endif
320	DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
321	      cpuid, idle->state, ipcb->flags));
322
323	/* Setup HWRPB fields that SRM uses to activate secondary CPU */
324	hwrpb->CPU_restart = __smp_callin;
325	hwrpb->CPU_restart_data = (unsigned long) __smp_callin;
326
327	/* Recalculate and update the HWRPB checksum */
328	hwrpb_update_checksum(hwrpb);
329
330	/*
331	 * Send a "start" command to the specified processor.
332	 */
333
334	/* SRM III 3.4.1.3 */
335	cpu->flags |= 0x22;	/* turn on Context Valid and Restart Capable */
336	cpu->flags &= ~1;	/* turn off Bootstrap In Progress */
337	wmb();
338
339	send_secondary_console_msg("START\r\n", cpuid);
340
341	/* Wait 10 seconds for an ACK from the console.  */
342	timeout = jiffies + 10*HZ;
343	while (time_before(jiffies, timeout)) {
344		if (cpu->flags & 1)
345			goto started;
346		udelay(10);
347		barrier();
348	}
349	printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
350	return -1;
351
352 started:
353	DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
354	return 0;
355}
356
357/*
358 * Bring one cpu online.
359 */
360static int
361smp_boot_one_cpu(int cpuid, struct task_struct *idle)
362{
363	unsigned long timeout;
364
365	/* Signal the secondary to wait a moment.  */
366	smp_secondary_alive = -1;
367
368	/* Whirrr, whirrr, whirrrrrrrrr... */
369	if (secondary_cpu_start(cpuid, idle))
370		return -1;
371
372	/* Notify the secondary CPU it can run calibrate_delay.  */
373	mb();
374	smp_secondary_alive = 0;
375
376	/* We've been acked by the console; wait one second for
377	   the task to start up for real.  */
378	timeout = jiffies + 1*HZ;
379	while (time_before(jiffies, timeout)) {
380		if (smp_secondary_alive == 1)
381			goto alive;
382		udelay(10);
383		barrier();
384	}
385
386	/* We failed to boot the CPU.  */
387
388	printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
389	return -1;
390
391 alive:
392	/* Another "Red Snapper". */
393	return 0;
394}
395
396/*
397 * Called from setup_arch.  Detect an SMP system and which processors
398 * are present.
399 */
400void __init
401setup_smp(void)
402{
403	struct percpu_struct *cpubase, *cpu;
404	unsigned long i;
405
406	if (boot_cpuid != 0) {
407		printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
408		       boot_cpuid);
409	}
410
411	if (hwrpb->nr_processors > 1) {
412		int boot_cpu_palrev;
413
414		DBGS(("setup_smp: nr_processors %ld\n",
415		      hwrpb->nr_processors));
416
417		cpubase = (struct percpu_struct *)
418			((char*)hwrpb + hwrpb->processor_offset);
419		boot_cpu_palrev = cpubase->pal_revision;
420
421		for (i = 0; i < hwrpb->nr_processors; i++) {
422			cpu = (struct percpu_struct *)
423				((char *)cpubase + i*hwrpb->processor_size);
424			if ((cpu->flags & 0x1cc) == 0x1cc) {
425				smp_num_probed++;
426				set_cpu_possible(i, true);
427				set_cpu_present(i, true);
428				cpu->pal_revision = boot_cpu_palrev;
429			}
430
431			DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
432			      i, cpu->flags, cpu->type));
433			DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
434			      i, cpu->pal_revision));
435		}
436	} else {
437		smp_num_probed = 1;
438	}
439
440	printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
441	       smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
442}
443
444/*
445 * Called by smp_init prepare the secondaries
446 */
447void __init
448smp_prepare_cpus(unsigned int max_cpus)
449{
450	/* Take care of some initial bookkeeping.  */
451	memset(ipi_data, 0, sizeof(ipi_data));
452
453	current_thread_info()->cpu = boot_cpuid;
454
455	smp_store_cpu_info(boot_cpuid);
456	smp_setup_percpu_timer(boot_cpuid);
457
458	/* Nothing to do on a UP box, or when told not to.  */
459	if (smp_num_probed == 1 || max_cpus == 0) {
460		init_cpu_possible(cpumask_of(boot_cpuid));
461		init_cpu_present(cpumask_of(boot_cpuid));
462		printk(KERN_INFO "SMP mode deactivated.\n");
463		return;
464	}
465
466	printk(KERN_INFO "SMP starting up secondaries.\n");
467
468	smp_num_cpus = smp_num_probed;
469}
470
471int
472__cpu_up(unsigned int cpu, struct task_struct *tidle)
473{
474	smp_boot_one_cpu(cpu, tidle);
475
476	return cpu_online(cpu) ? 0 : -ENOSYS;
477}
478
479void __init
480smp_cpus_done(unsigned int max_cpus)
481{
482	int cpu;
483	unsigned long bogosum = 0;
484
485	for(cpu = 0; cpu < NR_CPUS; cpu++) 
486		if (cpu_online(cpu))
487			bogosum += cpu_data[cpu].loops_per_jiffy;
488	
489	printk(KERN_INFO "SMP: Total of %d processors activated "
490	       "(%lu.%02lu BogoMIPS).\n",
491	       num_online_cpus(), 
492	       (bogosum + 2500) / (500000/HZ),
493	       ((bogosum + 2500) / (5000/HZ)) % 100);
494}
495
496static void
497send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
498{
499	int i;
500
501	mb();
502	for_each_cpu(i, to_whom)
503		set_bit(operation, &ipi_data[i].bits);
504
505	mb();
506	for_each_cpu(i, to_whom)
507		wripir(i);
508}
509
510void
511handle_ipi(struct pt_regs *regs)
512{
513	int this_cpu = smp_processor_id();
514	unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
515	unsigned long ops;
516
517#if 0
518	DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
519	      this_cpu, *pending_ipis, regs->pc));
520#endif
521
522	mb();	/* Order interrupt and bit testing. */
523	while ((ops = xchg(pending_ipis, 0)) != 0) {
524	  mb();	/* Order bit clearing and data access. */
525	  do {
526		unsigned long which;
527
528		which = ops & -ops;
529		ops &= ~which;
530		which = __ffs(which);
531
532		switch (which) {
533		case IPI_RESCHEDULE:
534			scheduler_ipi();
535			break;
536
537		case IPI_CALL_FUNC:
538			generic_smp_call_function_interrupt();
539			break;
540
541		case IPI_CPU_STOP:
542			halt();
543
544		default:
545			printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
546			       this_cpu, which);
547			break;
548		}
549	  } while (ops);
550
551	  mb();	/* Order data access and bit testing. */
552	}
553
554	cpu_data[this_cpu].ipi_count++;
555
556	if (hwrpb->txrdy)
557		recv_secondary_console_msg();
558}
559
560void
561arch_smp_send_reschedule(int cpu)
562{
563#ifdef DEBUG_IPI_MSG
564	if (cpu == hard_smp_processor_id())
565		printk(KERN_WARNING
566		       "smp_send_reschedule: Sending IPI to self.\n");
567#endif
568	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
569}
570
571void
572smp_send_stop(void)
573{
574	cpumask_t to_whom;
575	cpumask_copy(&to_whom, cpu_online_mask);
576	cpumask_clear_cpu(smp_processor_id(), &to_whom);
577#ifdef DEBUG_IPI_MSG
578	if (hard_smp_processor_id() != boot_cpu_id)
579		printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
580#endif
581	send_ipi_message(&to_whom, IPI_CPU_STOP);
582}
583
584void arch_send_call_function_ipi_mask(const struct cpumask *mask)
585{
586	send_ipi_message(mask, IPI_CALL_FUNC);
587}
588
589void arch_send_call_function_single_ipi(int cpu)
590{
591	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
592}
593
594static void
595ipi_imb(void *ignored)
596{
597	imb();
598}
599
600void
601smp_imb(void)
602{
603	/* Must wait other processors to flush their icache before continue. */
604	on_each_cpu(ipi_imb, NULL, 1);
605}
606EXPORT_SYMBOL(smp_imb);
607
608static void
609ipi_flush_tlb_all(void *ignored)
610{
611	tbia();
612}
613
614void
615flush_tlb_all(void)
616{
617	/* Although we don't have any data to pass, we do want to
618	   synchronize with the other processors.  */
619	on_each_cpu(ipi_flush_tlb_all, NULL, 1);
620}
621
622#define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
623
624static void
625ipi_flush_tlb_mm(void *x)
626{
627	struct mm_struct *mm = x;
628	if (mm == current->active_mm && !asn_locked())
629		flush_tlb_current(mm);
630	else
631		flush_tlb_other(mm);
632}
633
634void
635flush_tlb_mm(struct mm_struct *mm)
636{
637	preempt_disable();
638
639	if (mm == current->active_mm) {
640		flush_tlb_current(mm);
641		if (atomic_read(&mm->mm_users) <= 1) {
642			int cpu, this_cpu = smp_processor_id();
643			for (cpu = 0; cpu < NR_CPUS; cpu++) {
644				if (!cpu_online(cpu) || cpu == this_cpu)
645					continue;
646				if (mm->context[cpu])
647					mm->context[cpu] = 0;
648			}
649			preempt_enable();
650			return;
651		}
652	}
653
654	smp_call_function(ipi_flush_tlb_mm, mm, 1);
655
656	preempt_enable();
657}
658EXPORT_SYMBOL(flush_tlb_mm);
659
660struct flush_tlb_page_struct {
661	struct vm_area_struct *vma;
662	struct mm_struct *mm;
663	unsigned long addr;
664};
665
666static void
667ipi_flush_tlb_page(void *x)
668{
669	struct flush_tlb_page_struct *data = x;
670	struct mm_struct * mm = data->mm;
671
672	if (mm == current->active_mm && !asn_locked())
673		flush_tlb_current_page(mm, data->vma, data->addr);
674	else
675		flush_tlb_other(mm);
676}
677
678void
679flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
680{
681	struct flush_tlb_page_struct data;
682	struct mm_struct *mm = vma->vm_mm;
683
684	preempt_disable();
685
686	if (mm == current->active_mm) {
687		flush_tlb_current_page(mm, vma, addr);
688		if (atomic_read(&mm->mm_users) <= 1) {
689			int cpu, this_cpu = smp_processor_id();
690			for (cpu = 0; cpu < NR_CPUS; cpu++) {
691				if (!cpu_online(cpu) || cpu == this_cpu)
692					continue;
693				if (mm->context[cpu])
694					mm->context[cpu] = 0;
695			}
696			preempt_enable();
697			return;
698		}
699	}
700
701	data.vma = vma;
702	data.mm = mm;
703	data.addr = addr;
704
705	smp_call_function(ipi_flush_tlb_page, &data, 1);
706
707	preempt_enable();
708}
709EXPORT_SYMBOL(flush_tlb_page);
710
711void
712flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
713{
714	/* On the Alpha we always flush the whole user tlb.  */
715	flush_tlb_mm(vma->vm_mm);
716}
717EXPORT_SYMBOL(flush_tlb_range);
718
719static void
720ipi_flush_icache_page(void *x)
721{
722	struct mm_struct *mm = (struct mm_struct *) x;
723	if (mm == current->active_mm && !asn_locked())
724		__load_new_mm_context(mm);
725	else
726		flush_tlb_other(mm);
727}
728
729void
730flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
731			unsigned long addr, int len)
732{
733	struct mm_struct *mm = vma->vm_mm;
734
735	if ((vma->vm_flags & VM_EXEC) == 0)
736		return;
737
738	preempt_disable();
739
740	if (mm == current->active_mm) {
741		__load_new_mm_context(mm);
742		if (atomic_read(&mm->mm_users) <= 1) {
743			int cpu, this_cpu = smp_processor_id();
744			for (cpu = 0; cpu < NR_CPUS; cpu++) {
745				if (!cpu_online(cpu) || cpu == this_cpu)
746					continue;
747				if (mm->context[cpu])
748					mm->context[cpu] = 0;
749			}
750			preempt_enable();
751			return;
752		}
753	}
754
755	smp_call_function(ipi_flush_icache_page, mm, 1);
756
757	preempt_enable();
758}