smp.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *  SMP related functions
   4 *
   5 *    Copyright IBM Corp. 1999, 2012
   6 *    Author(s): Denis Joseph Barrow,
   7 *		 Martin Schwidefsky <schwidefsky@de.ibm.com>,
   8 *
   9 *  based on other smp stuff by
  10 *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
  11 *    (c) 1998 Ingo Molnar
  12 *
  13 * The code outside of smp.c uses logical cpu numbers, only smp.c does
  14 * the translation of logical to physical cpu ids. All new code that
  15 * operates on physical cpu numbers needs to go into smp.c.
  16 */
  17
  18#define pr_fmt(fmt) "cpu: " fmt
  19
  20#include <linux/cpufeature.h>
  21#include <linux/workqueue.h>
  22#include <linux/memblock.h>
  23#include <linux/export.h>
  24#include <linux/init.h>
  25#include <linux/mm.h>
  26#include <linux/err.h>
  27#include <linux/spinlock.h>
  28#include <linux/kernel_stat.h>
  29#include <linux/delay.h>
  30#include <linux/interrupt.h>
  31#include <linux/irqflags.h>
  32#include <linux/irq_work.h>
  33#include <linux/cpu.h>
  34#include <linux/slab.h>
  35#include <linux/sched/hotplug.h>
  36#include <linux/sched/task_stack.h>
  37#include <linux/crash_dump.h>
  38#include <linux/kprobes.h>
  39#include <asm/access-regs.h>
  40#include <asm/asm-offsets.h>
  41#include <asm/machine.h>
  42#include <asm/ctlreg.h>
  43#include <asm/pfault.h>
  44#include <asm/diag.h>
  45#include <asm/facility.h>
  46#include <asm/fpu.h>
  47#include <asm/ipl.h>
  48#include <asm/setup.h>
  49#include <asm/irq.h>
  50#include <asm/tlbflush.h>
  51#include <asm/vtimer.h>
  52#include <asm/abs_lowcore.h>
  53#include <asm/sclp.h>
  54#include <asm/debug.h>
  55#include <asm/os_info.h>
  56#include <asm/sigp.h>
  57#include <asm/idle.h>
  58#include <asm/nmi.h>
  59#include <asm/stacktrace.h>
  60#include <asm/topology.h>
  61#include <asm/vdso.h>
  62#include <asm/maccess.h>
  63#include "entry.h"
  64
  65enum {
  66	ec_schedule = 0,
  67	ec_call_function_single,
  68	ec_stop_cpu,
  69	ec_mcck_pending,
  70	ec_irq_work,
  71};
  72
  73enum {
  74	CPU_STATE_STANDBY,
  75	CPU_STATE_CONFIGURED,
  76};
  77
  78static u8 boot_core_type;
  79DEFINE_PER_CPU(struct pcpu, pcpu_devices);
  80/*
  81 * Pointer to the pcpu area of the boot CPU. This is required when a restart
  82 * interrupt is triggered on an offline CPU. For that case accessing percpu
  83 * data with the common primitives does not work, since the percpu offset is
  84 * stored in a non existent lowcore.
  85 */
  86static struct pcpu *ipl_pcpu;
  87
  88unsigned int smp_cpu_mt_shift;
  89EXPORT_SYMBOL(smp_cpu_mt_shift);
  90
  91unsigned int smp_cpu_mtid;
  92EXPORT_SYMBOL(smp_cpu_mtid);
  93
  94#ifdef CONFIG_CRASH_DUMP
  95__vector128 __initdata boot_cpu_vector_save_area[__NUM_VXRS];
  96#endif
  97
  98static unsigned int smp_max_threads __initdata = -1U;
  99cpumask_t cpu_setup_mask;
 100
 101static int __init early_smt(char *s)
 102{
 103	get_option(&s, &smp_max_threads);
 104	return 0;
 105}
 106early_param("smt", early_smt);
 107
 108/*
 109 * The smp_cpu_state_mutex must be held when changing the state or polarization
 110 * member of a pcpu data structure within the pcpu_devices array.
 111 */
 112DEFINE_MUTEX(smp_cpu_state_mutex);
 113
 114/*
 115 * Signal processor helper functions.
 116 */
 117static inline int __pcpu_sigp_relax(u16 addr, u8 order, unsigned long parm)
 118{
 119	int cc;
 120
 121	while (1) {
 122		cc = __pcpu_sigp(addr, order, parm, NULL);
 123		if (cc != SIGP_CC_BUSY)
 124			return cc;
 125		cpu_relax();
 126	}
 127}
 128
 129static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
 130{
 131	int cc, retry;
 132
 133	for (retry = 0; ; retry++) {
 134		cc = __pcpu_sigp(pcpu->address, order, parm, NULL);
 135		if (cc != SIGP_CC_BUSY)
 136			break;
 137		if (retry >= 3)
 138			udelay(10);
 139	}
 140	return cc;
 141}
 142
 143static inline int pcpu_stopped(struct pcpu *pcpu)
 144{
 145	u32 status;
 146
 147	if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
 148			0, &status) != SIGP_CC_STATUS_STORED)
 149		return 0;
 150	return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
 151}
 152
 153static inline int pcpu_running(struct pcpu *pcpu)
 154{
 155	if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
 156			0, NULL) != SIGP_CC_STATUS_STORED)
 157		return 1;
 158	/* Status stored condition code is equivalent to cpu not running. */
 159	return 0;
 160}
 161
 162/*
 163 * Find struct pcpu by cpu address.
 164 */
 165static struct pcpu *pcpu_find_address(const struct cpumask *mask, u16 address)
 166{
 167	int cpu;
 168
 169	for_each_cpu(cpu, mask)
 170		if (per_cpu(pcpu_devices, cpu).address == address)
 171			return &per_cpu(pcpu_devices, cpu);
 172	return NULL;
 173}
 174
 175static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
 176{
 177	if (test_and_set_bit(ec_bit, &pcpu->ec_mask))
 178		return;
 179	pcpu->ec_clk = get_tod_clock_fast();
 180	pcpu_sigp_retry(pcpu, SIGP_EXTERNAL_CALL, 0);
 181}
 182
 183static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
 184{
 185	unsigned long async_stack, nodat_stack, mcck_stack;
 186	struct lowcore *lc;
 187
 188	lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
 189	nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
 190	async_stack = stack_alloc();
 191	mcck_stack = stack_alloc();
 192	if (!lc || !nodat_stack || !async_stack || !mcck_stack)
 193		goto out;
 194	memcpy(lc, get_lowcore(), 512);
 195	memset((char *) lc + 512, 0, sizeof(*lc) - 512);
 196	lc->async_stack = async_stack + STACK_INIT_OFFSET;
 197	lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET;
 198	lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET;
 199	lc->cpu_nr = cpu;
 200	lc->spinlock_lockval = arch_spin_lockval(cpu);
 201	lc->spinlock_index = 0;
 202	lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW);
 203	lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW);
 204	lc->preempt_count = PREEMPT_DISABLED;
 205	if (nmi_alloc_mcesa(&lc->mcesad))
 206		goto out;
 207	if (abs_lowcore_map(cpu, lc, true))
 208		goto out_mcesa;
 209	lowcore_ptr[cpu] = lc;
 210	pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc));
 211	return 0;
 212
 213out_mcesa:
 214	nmi_free_mcesa(&lc->mcesad);
 215out:
 216	stack_free(mcck_stack);
 217	stack_free(async_stack);
 218	free_pages(nodat_stack, THREAD_SIZE_ORDER);
 219	free_pages((unsigned long) lc, LC_ORDER);
 220	return -ENOMEM;
 221}
 222
 223static void pcpu_free_lowcore(struct pcpu *pcpu, int cpu)
 224{
 225	unsigned long async_stack, nodat_stack, mcck_stack;
 226	struct lowcore *lc;
 227
 228	lc = lowcore_ptr[cpu];
 229	nodat_stack = lc->nodat_stack - STACK_INIT_OFFSET;
 230	async_stack = lc->async_stack - STACK_INIT_OFFSET;
 231	mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET;
 232	pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
 233	lowcore_ptr[cpu] = NULL;
 234	abs_lowcore_unmap(cpu);
 235	nmi_free_mcesa(&lc->mcesad);
 236	stack_free(async_stack);
 237	stack_free(mcck_stack);
 238	free_pages(nodat_stack, THREAD_SIZE_ORDER);
 239	free_pages((unsigned long) lc, LC_ORDER);
 240}
 241
 242static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
 243{
 244	struct lowcore *lc, *abs_lc;
 245
 246	lc = lowcore_ptr[cpu];
 247	cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask);
 248	cpumask_set_cpu(cpu, mm_cpumask(&init_mm));
 249	lc->cpu_nr = cpu;
 250	lc->pcpu = (unsigned long)pcpu;
 251	lc->restart_flags = RESTART_FLAG_CTLREGS;
 252	lc->spinlock_lockval = arch_spin_lockval(cpu);
 253	lc->spinlock_index = 0;
 254	lc->percpu_offset = __per_cpu_offset[cpu];
 255	lc->kernel_asce = get_lowcore()->kernel_asce;
 256	lc->user_asce = s390_invalid_asce;
 257	lc->user_timer = lc->system_timer =
 258		lc->steal_timer = lc->avg_steal_timer = 0;
 259	abs_lc = get_abs_lowcore();
 260	memcpy(lc->cregs_save_area, abs_lc->cregs_save_area, sizeof(lc->cregs_save_area));
 261	put_abs_lowcore(abs_lc);
 262	lc->cregs_save_area[1] = lc->user_asce;
 263	lc->cregs_save_area[7] = lc->user_asce;
 264	save_access_regs((unsigned int *) lc->access_regs_save_area);
 265	arch_spin_lock_setup(cpu);
 266}
 267
 268static void pcpu_attach_task(int cpu, struct task_struct *tsk)
 269{
 270	struct lowcore *lc;
 271
 272	lc = lowcore_ptr[cpu];
 273	lc->kernel_stack = (unsigned long)task_stack_page(tsk) + STACK_INIT_OFFSET;
 274	lc->current_task = (unsigned long)tsk;
 275	lc->lpp = LPP_MAGIC;
 276	lc->current_pid = tsk->pid;
 277	lc->user_timer = tsk->thread.user_timer;
 278	lc->guest_timer = tsk->thread.guest_timer;
 279	lc->system_timer = tsk->thread.system_timer;
 280	lc->hardirq_timer = tsk->thread.hardirq_timer;
 281	lc->softirq_timer = tsk->thread.softirq_timer;
 282	lc->steal_timer = 0;
 283#ifdef CONFIG_STACKPROTECTOR
 284	lc->stack_canary = tsk->stack_canary;
 285#endif
 286}
 287
 288static void pcpu_start_fn(int cpu, void (*func)(void *), void *data)
 289{
 290	struct lowcore *lc;
 291
 292	lc = lowcore_ptr[cpu];
 293	lc->restart_stack = lc->kernel_stack;
 294	lc->restart_fn = (unsigned long) func;
 295	lc->restart_data = (unsigned long) data;
 296	lc->restart_source = -1U;
 297	pcpu_sigp_retry(per_cpu_ptr(&pcpu_devices, cpu), SIGP_RESTART, 0);
 298}
 299
 300typedef void (pcpu_delegate_fn)(void *);
 301
 302/*
 303 * Call function via PSW restart on pcpu and stop the current cpu.
 304 */
 305static void __pcpu_delegate(pcpu_delegate_fn *func, void *data)
 306{
 307	func(data);	/* should not return */
 308}
 309
 310static void __noreturn pcpu_delegate(struct pcpu *pcpu, int cpu,
 311				     pcpu_delegate_fn *func,
 312				     void *data, unsigned long stack)
 313{
 314	struct lowcore *lc, *abs_lc;
 315	unsigned int source_cpu;
 316
 317	lc = lowcore_ptr[cpu];
 318	source_cpu = stap();
 319
 320	if (pcpu->address == source_cpu) {
 321		call_on_stack(2, stack, void, __pcpu_delegate,
 322			      pcpu_delegate_fn *, func, void *, data);
 323	}
 324	/* Stop target cpu (if func returns this stops the current cpu). */
 325	pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
 326	pcpu_sigp_retry(pcpu, SIGP_CPU_RESET, 0);
 327	/* Restart func on the target cpu and stop the current cpu. */
 328	if (lc) {
 329		lc->restart_stack = stack;
 330		lc->restart_fn = (unsigned long)func;
 331		lc->restart_data = (unsigned long)data;
 332		lc->restart_source = source_cpu;
 333	} else {
 334		abs_lc = get_abs_lowcore();
 335		abs_lc->restart_stack = stack;
 336		abs_lc->restart_fn = (unsigned long)func;
 337		abs_lc->restart_data = (unsigned long)data;
 338		abs_lc->restart_source = source_cpu;
 339		put_abs_lowcore(abs_lc);
 340	}
 341	asm volatile(
 342		"0:	sigp	0,%0,%2	# sigp restart to target cpu\n"
 343		"	brc	2,0b	# busy, try again\n"
 344		"1:	sigp	0,%1,%3	# sigp stop to current cpu\n"
 345		"	brc	2,1b	# busy, try again"
 346		: : "d" (pcpu->address), "d" (source_cpu),
 347		    "K" (SIGP_RESTART), "K" (SIGP_STOP)
 348		: "0", "1", "cc");
 349	for (;;) ;
 350}
 351
 352/*
 353 * Enable additional logical cpus for multi-threading.
 354 */
 355static int pcpu_set_smt(unsigned int mtid)
 356{
 357	int cc;
 358
 359	if (smp_cpu_mtid == mtid)
 360		return 0;
 361	cc = __pcpu_sigp(0, SIGP_SET_MULTI_THREADING, mtid, NULL);
 362	if (cc == 0) {
 363		smp_cpu_mtid = mtid;
 364		smp_cpu_mt_shift = 0;
 365		while (smp_cpu_mtid >= (1U << smp_cpu_mt_shift))
 366			smp_cpu_mt_shift++;
 367		per_cpu(pcpu_devices, 0).address = stap();
 368	}
 369	return cc;
 370}
 371
 372/*
 373 * Call function on the ipl CPU.
 374 */
 375void __noreturn smp_call_ipl_cpu(void (*func)(void *), void *data)
 376{
 377	struct lowcore *lc = lowcore_ptr[0];
 378
 379	if (ipl_pcpu->address == stap())
 380		lc = get_lowcore();
 381
 382	pcpu_delegate(ipl_pcpu, 0, func, data, lc->nodat_stack);
 383}
 384
 385int smp_find_processor_id(u16 address)
 386{
 387	int cpu;
 388
 389	for_each_present_cpu(cpu)
 390		if (per_cpu(pcpu_devices, cpu).address == address)
 391			return cpu;
 392	return -1;
 393}
 394
 395void schedule_mcck_handler(void)
 396{
 397	pcpu_ec_call(this_cpu_ptr(&pcpu_devices), ec_mcck_pending);
 398}
 399
 400bool notrace arch_vcpu_is_preempted(int cpu)
 401{
 402	if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu))
 403		return false;
 404	if (pcpu_running(per_cpu_ptr(&pcpu_devices, cpu)))
 405		return false;
 406	return true;
 407}
 408EXPORT_SYMBOL(arch_vcpu_is_preempted);
 409
 410void notrace smp_yield_cpu(int cpu)
 411{
 412	if (!machine_has_diag9c())
 413		return;
 414	diag_stat_inc_norecursion(DIAG_STAT_X09C);
 415	asm volatile("diag %0,0,0x9c"
 416		     : : "d" (per_cpu(pcpu_devices, cpu).address));
 417}
 418EXPORT_SYMBOL_GPL(smp_yield_cpu);
 419
 420/*
 421 * Send cpus emergency shutdown signal. This gives the cpus the
 422 * opportunity to complete outstanding interrupts.
 423 */
 424void notrace smp_emergency_stop(void)
 425{
 426	static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
 427	static cpumask_t cpumask;
 428	u64 end;
 429	int cpu;
 430
 431	arch_spin_lock(&lock);
 432	cpumask_copy(&cpumask, cpu_online_mask);
 433	cpumask_clear_cpu(smp_processor_id(), &cpumask);
 434
 435	end = get_tod_clock_monotonic() + (1000000UL << 12);
 436	for_each_cpu(cpu, &cpumask) {
 437		struct pcpu *pcpu = per_cpu_ptr(&pcpu_devices, cpu);
 438		set_bit(ec_stop_cpu, &pcpu->ec_mask);
 439		while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
 440				   0, NULL) == SIGP_CC_BUSY &&
 441		       get_tod_clock_monotonic() < end)
 442			cpu_relax();
 443	}
 444	while (get_tod_clock_monotonic() < end) {
 445		for_each_cpu(cpu, &cpumask)
 446			if (pcpu_stopped(per_cpu_ptr(&pcpu_devices, cpu)))
 447				cpumask_clear_cpu(cpu, &cpumask);
 448		if (cpumask_empty(&cpumask))
 449			break;
 450		cpu_relax();
 451	}
 452	arch_spin_unlock(&lock);
 453}
 454NOKPROBE_SYMBOL(smp_emergency_stop);
 455
 456/*
 457 * Stop all cpus but the current one.
 458 */
 459void smp_send_stop(void)
 460{
 461	struct pcpu *pcpu;
 462	int cpu;
 463
 464	/* Disable all interrupts/machine checks */
 465	__load_psw_mask(PSW_KERNEL_BITS);
 466	trace_hardirqs_off();
 467
 468	debug_set_critical();
 469
 470	if (oops_in_progress)
 471		smp_emergency_stop();
 472
 473	/* stop all processors */
 474	for_each_online_cpu(cpu) {
 475		if (cpu == smp_processor_id())
 476			continue;
 477		pcpu = per_cpu_ptr(&pcpu_devices, cpu);
 478		pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
 479		while (!pcpu_stopped(pcpu))
 480			cpu_relax();
 481	}
 482}
 483
 484/*
 485 * This is the main routine where commands issued by other
 486 * cpus are handled.
 487 */
 488static void smp_handle_ext_call(void)
 489{
 490	unsigned long bits;
 491
 492	/* handle bit signal external calls */
 493	bits = this_cpu_xchg(pcpu_devices.ec_mask, 0);
 494	if (test_bit(ec_stop_cpu, &bits))
 495		smp_stop_cpu();
 496	if (test_bit(ec_schedule, &bits))
 497		scheduler_ipi();
 498	if (test_bit(ec_call_function_single, &bits))
 499		generic_smp_call_function_single_interrupt();
 500	if (test_bit(ec_mcck_pending, &bits))
 501		s390_handle_mcck();
 502	if (test_bit(ec_irq_work, &bits))
 503		irq_work_run();
 504}
 505
 506static void do_ext_call_interrupt(struct ext_code ext_code,
 507				  unsigned int param32, unsigned long param64)
 508{
 509	inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS);
 510	smp_handle_ext_call();
 511}
 512
 513void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 514{
 515	int cpu;
 516
 517	for_each_cpu(cpu, mask)
 518		pcpu_ec_call(per_cpu_ptr(&pcpu_devices, cpu), ec_call_function_single);
 519}
 520
 521void arch_send_call_function_single_ipi(int cpu)
 522{
 523	pcpu_ec_call(per_cpu_ptr(&pcpu_devices, cpu), ec_call_function_single);
 524}
 525
 526/*
 527 * this function sends a 'reschedule' IPI to another CPU.
 528 * it goes straight through and wastes no time serializing
 529 * anything. Worst case is that we lose a reschedule ...
 530 */
 531void arch_smp_send_reschedule(int cpu)
 532{
 533	pcpu_ec_call(per_cpu_ptr(&pcpu_devices, cpu), ec_schedule);
 534}
 535
 536#ifdef CONFIG_IRQ_WORK
 537void arch_irq_work_raise(void)
 538{
 539	pcpu_ec_call(this_cpu_ptr(&pcpu_devices), ec_irq_work);
 540}
 541#endif
 542
 543#ifdef CONFIG_CRASH_DUMP
 544
 545int smp_store_status(int cpu)
 546{
 547	struct lowcore *lc;
 548	struct pcpu *pcpu;
 549	unsigned long pa;
 550
 551	pcpu = per_cpu_ptr(&pcpu_devices, cpu);
 552	lc = lowcore_ptr[cpu];
 553	pa = __pa(&lc->floating_pt_save_area);
 554	if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_STATUS_AT_ADDRESS,
 555			      pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
 556		return -EIO;
 557	if (!cpu_has_vx() && !cpu_has_gs())
 558		return 0;
 559	pa = lc->mcesad & MCESA_ORIGIN_MASK;
 560	if (cpu_has_gs())
 561		pa |= lc->mcesad & MCESA_LC_MASK;
 562	if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS,
 563			      pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
 564		return -EIO;
 565	return 0;
 566}
 567
 568/*
 569 * Collect CPU state of the previous, crashed system.
 570 * There are three cases:
 571 * 1) standard zfcp/nvme dump
 572 *    condition: OLDMEM_BASE == NULL && is_ipl_type_dump() == true
 573 *    The state for all CPUs except the boot CPU needs to be collected
 574 *    with sigp stop-and-store-status. The boot CPU state is located in
 575 *    the absolute lowcore of the memory stored in the HSA. The zcore code
 576 *    will copy the boot CPU state from the HSA.
 577 * 2) stand-alone kdump for SCSI/NVMe (zfcp/nvme dump with swapped memory)
 578 *    condition: OLDMEM_BASE != NULL && is_ipl_type_dump() == true
 579 *    The state for all CPUs except the boot CPU needs to be collected
 580 *    with sigp stop-and-store-status. The firmware or the boot-loader
 581 *    stored the registers of the boot CPU in the absolute lowcore in the
 582 *    memory of the old system.
 583 * 3) kdump or stand-alone kdump for DASD
 584 *    condition: OLDMEM_BASE != NULL && is_ipl_type_dump() == false
 585 *    The state for all CPUs except the boot CPU needs to be collected
 586 *    with sigp stop-and-store-status. The kexec code or the boot-loader
 587 *    stored the registers of the boot CPU in the memory of the old system.
 588 *
 589 * Note that the legacy kdump mode where the old kernel stored the CPU states
 590 * does no longer exist: setup_arch() explicitly deactivates the elfcorehdr=
 591 * kernel parameter. The is_kdump_kernel() implementation on s390 is independent
 592 * of the elfcorehdr= parameter.
 593 */
 594static bool dump_available(void)
 595{
 596	return oldmem_data.start || is_ipl_type_dump();
 597}
 598
 599void __init smp_save_dump_ipl_cpu(void)
 600{
 601	struct save_area *sa;
 602	void *regs;
 603
 604	if (!dump_available())
 605		return;
 606	sa = save_area_alloc(true);
 607	regs = memblock_alloc_or_panic(512, 8);
 608	copy_oldmem_kernel(regs, __LC_FPREGS_SAVE_AREA, 512);
 609	save_area_add_regs(sa, regs);
 610	memblock_free(regs, 512);
 611	if (cpu_has_vx())
 612		save_area_add_vxrs(sa, boot_cpu_vector_save_area);
 613}
 614
 615void __init smp_save_dump_secondary_cpus(void)
 616{
 617	int addr, boot_cpu_addr, max_cpu_addr;
 618	struct save_area *sa;
 619	void *page;
 620
 621	if (!dump_available())
 622		return;
 623	/* Allocate a page as dumping area for the store status sigps */
 624	page = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
 625	if (!page)
 626		panic("ERROR: Failed to allocate %lx bytes below %lx\n",
 627		      PAGE_SIZE, 1UL << 31);
 628
 629	/* Set multi-threading state to the previous system. */
 630	pcpu_set_smt(sclp.mtid_prev);
 631	boot_cpu_addr = stap();
 632	max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev;
 633	for (addr = 0; addr <= max_cpu_addr; addr++) {
 634		if (addr == boot_cpu_addr)
 635			continue;
 636		if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0) ==
 637		    SIGP_CC_NOT_OPERATIONAL)
 638			continue;
 639		sa = save_area_alloc(false);
 640		__pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, __pa(page));
 641		save_area_add_regs(sa, page);
 642		if (cpu_has_vx()) {
 643			__pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, __pa(page));
 644			save_area_add_vxrs(sa, page);
 645		}
 646	}
 647	memblock_free(page, PAGE_SIZE);
 648	diag_amode31_ops.diag308_reset();
 649	pcpu_set_smt(0);
 650}
 651#endif /* CONFIG_CRASH_DUMP */
 652
 653void smp_cpu_set_polarization(int cpu, int val)
 654{
 655	per_cpu(pcpu_devices, cpu).polarization = val;
 656}
 657
 658int smp_cpu_get_polarization(int cpu)
 659{
 660	return per_cpu(pcpu_devices, cpu).polarization;
 661}
 662
 663void smp_cpu_set_capacity(int cpu, unsigned long val)
 664{
 665	per_cpu(pcpu_devices, cpu).capacity = val;
 666}
 667
 668unsigned long smp_cpu_get_capacity(int cpu)
 669{
 670	return per_cpu(pcpu_devices, cpu).capacity;
 671}
 672
 673void smp_set_core_capacity(int cpu, unsigned long val)
 674{
 675	int i;
 676
 677	cpu = smp_get_base_cpu(cpu);
 678	for (i = cpu; (i <= cpu + smp_cpu_mtid) && (i < nr_cpu_ids); i++)
 679		smp_cpu_set_capacity(i, val);
 680}
 681
 682int smp_cpu_get_cpu_address(int cpu)
 683{
 684	return per_cpu(pcpu_devices, cpu).address;
 685}
 686
 687static void __ref smp_get_core_info(struct sclp_core_info *info, int early)
 688{
 689	static int use_sigp_detection;
 690	int address;
 691
 692	if (use_sigp_detection || sclp_get_core_info(info, early)) {
 693		use_sigp_detection = 1;
 694		for (address = 0;
 695		     address < (SCLP_MAX_CORES << smp_cpu_mt_shift);
 696		     address += (1U << smp_cpu_mt_shift)) {
 697			if (__pcpu_sigp_relax(address, SIGP_SENSE, 0) ==
 698			    SIGP_CC_NOT_OPERATIONAL)
 699				continue;
 700			info->core[info->configured].core_id =
 701				address >> smp_cpu_mt_shift;
 702			info->core[info->configured].type = boot_core_type;
 703			info->configured++;
 704		}
 705		info->combined = info->configured;
 706	}
 707}
 708
 709static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail,
 710			bool configured, bool early)
 711{
 712	struct pcpu *pcpu;
 713	int cpu, nr, i;
 714	u16 address;
 715
 716	nr = 0;
 717	if (sclp.has_core_type && core->type != boot_core_type)
 718		return nr;
 719	cpu = cpumask_first(avail);
 720	address = core->core_id << smp_cpu_mt_shift;
 721	for (i = 0; (i <= smp_cpu_mtid) && (cpu < nr_cpu_ids); i++) {
 722		if (pcpu_find_address(cpu_present_mask, address + i))
 723			continue;
 724		pcpu = per_cpu_ptr(&pcpu_devices, cpu);
 725		pcpu->address = address + i;
 726		if (configured)
 727			pcpu->state = CPU_STATE_CONFIGURED;
 728		else
 729			pcpu->state = CPU_STATE_STANDBY;
 730		smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
 731		smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH);
 732		set_cpu_present(cpu, true);
 733		if (!early && arch_register_cpu(cpu))
 734			set_cpu_present(cpu, false);
 735		else
 736			nr++;
 737		cpumask_clear_cpu(cpu, avail);
 738		cpu = cpumask_next(cpu, avail);
 739	}
 740	return nr;
 741}
 742
 743static int __smp_rescan_cpus(struct sclp_core_info *info, bool early)
 744{
 745	struct sclp_core_entry *core;
 746	static cpumask_t avail;
 747	bool configured;
 748	u16 core_id;
 749	int nr, i;
 750
 751	cpus_read_lock();
 752	mutex_lock(&smp_cpu_state_mutex);
 753	nr = 0;
 754	cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
 755	/*
 756	 * Add IPL core first (which got logical CPU number 0) to make sure
 757	 * that all SMT threads get subsequent logical CPU numbers.
 758	 */
 759	if (early) {
 760		core_id = per_cpu(pcpu_devices, 0).address >> smp_cpu_mt_shift;
 761		for (i = 0; i < info->configured; i++) {
 762			core = &info->core[i];
 763			if (core->core_id == core_id) {
 764				nr += smp_add_core(core, &avail, true, early);
 765				break;
 766			}
 767		}
 768	}
 769	for (i = 0; i < info->combined; i++) {
 770		configured = i < info->configured;
 771		nr += smp_add_core(&info->core[i], &avail, configured, early);
 772	}
 773	mutex_unlock(&smp_cpu_state_mutex);
 774	cpus_read_unlock();
 775	return nr;
 776}
 777
 778void __init smp_detect_cpus(void)
 779{
 780	unsigned int cpu, mtid, c_cpus, s_cpus;
 781	struct sclp_core_info *info;
 782	u16 address;
 783
 784	/* Get CPU information */
 785	info = memblock_alloc_or_panic(sizeof(*info), 8);
 786	smp_get_core_info(info, 1);
 787	/* Find boot CPU type */
 788	if (sclp.has_core_type) {
 789		address = stap();
 790		for (cpu = 0; cpu < info->combined; cpu++)
 791			if (info->core[cpu].core_id == address) {
 792				/* The boot cpu dictates the cpu type. */
 793				boot_core_type = info->core[cpu].type;
 794				break;
 795			}
 796		if (cpu >= info->combined)
 797			panic("Could not find boot CPU type");
 798	}
 799
 800	/* Set multi-threading state for the current system */
 801	mtid = boot_core_type ? sclp.mtid : sclp.mtid_cp;
 802	mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1;
 803	pcpu_set_smt(mtid);
 804	cpu_smt_set_num_threads(smp_cpu_mtid + 1, smp_cpu_mtid + 1);
 805
 806	/* Print number of CPUs */
 807	c_cpus = s_cpus = 0;
 808	for (cpu = 0; cpu < info->combined; cpu++) {
 809		if (sclp.has_core_type &&
 810		    info->core[cpu].type != boot_core_type)
 811			continue;
 812		if (cpu < info->configured)
 813			c_cpus += smp_cpu_mtid + 1;
 814		else
 815			s_cpus += smp_cpu_mtid + 1;
 816	}
 817	pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
 818	memblock_free(info, sizeof(*info));
 819}
 820
 821/*
 822 *	Activate a secondary processor.
 823 */
 824static void smp_start_secondary(void *cpuvoid)
 825{
 826	struct lowcore *lc = get_lowcore();
 827	int cpu = raw_smp_processor_id();
 828
 829	lc->last_update_clock = get_tod_clock();
 830	lc->restart_stack = (unsigned long)restart_stack;
 831	lc->restart_fn = (unsigned long)do_restart;
 832	lc->restart_data = 0;
 833	lc->restart_source = -1U;
 834	lc->restart_flags = 0;
 835	restore_access_regs(lc->access_regs_save_area);
 836	cpu_init();
 837	rcutree_report_cpu_starting(cpu);
 838	init_cpu_timer();
 839	vtime_init();
 840	vdso_getcpu_init();
 841	pfault_init();
 842	cpumask_set_cpu(cpu, &cpu_setup_mask);
 843	update_cpu_masks();
 844	notify_cpu_starting(cpu);
 845	if (topology_cpu_dedicated(cpu))
 846		set_cpu_flag(CIF_DEDICATED_CPU);
 847	else
 848		clear_cpu_flag(CIF_DEDICATED_CPU);
 849	set_cpu_online(cpu, true);
 850	inc_irq_stat(CPU_RST);
 851	local_irq_enable();
 852	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 853}
 854
 855/* Upping and downing of CPUs */
 856int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 857{
 858	struct pcpu *pcpu = per_cpu_ptr(&pcpu_devices, cpu);
 859	int rc;
 860
 861	if (pcpu->state != CPU_STATE_CONFIGURED)
 862		return -EIO;
 863	if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) !=
 864	    SIGP_CC_ORDER_CODE_ACCEPTED)
 865		return -EIO;
 866
 867	rc = pcpu_alloc_lowcore(pcpu, cpu);
 868	if (rc)
 869		return rc;
 870	/*
 871	 * Make sure global control register contents do not change
 872	 * until new CPU has initialized control registers.
 873	 */
 874	system_ctlreg_lock();
 875	pcpu_prepare_secondary(pcpu, cpu);
 876	pcpu_attach_task(cpu, tidle);
 877	pcpu_start_fn(cpu, smp_start_secondary, NULL);
 878	/* Wait until cpu puts itself in the online & active maps */
 879	while (!cpu_online(cpu))
 880		cpu_relax();
 881	system_ctlreg_unlock();
 882	return 0;
 883}
 884
 885static unsigned int setup_possible_cpus __initdata;
 886
 887static int __init _setup_possible_cpus(char *s)
 888{
 889	get_option(&s, &setup_possible_cpus);
 890	return 0;
 891}
 892early_param("possible_cpus", _setup_possible_cpus);
 893
 894int __cpu_disable(void)
 895{
 896	struct ctlreg cregs[16];
 897	int cpu;
 898
 899	/* Handle possible pending IPIs */
 900	smp_handle_ext_call();
 901	cpu = smp_processor_id();
 902	set_cpu_online(cpu, false);
 903	cpumask_clear_cpu(cpu, &cpu_setup_mask);
 904	update_cpu_masks();
 905	/* Disable pseudo page faults on this cpu. */
 906	pfault_fini();
 907	/* Disable interrupt sources via control register. */
 908	__local_ctl_store(0, 15, cregs);
 909	cregs[0].val  &= ~0x0000ee70UL;	/* disable all external interrupts */
 910	cregs[6].val  &= ~0xff000000UL;	/* disable all I/O interrupts */
 911	cregs[14].val &= ~0x1f000000UL;	/* disable most machine checks */
 912	__local_ctl_load(0, 15, cregs);
 913	clear_cpu_flag(CIF_NOHZ_DELAY);
 914	return 0;
 915}
 916
 917void __cpu_die(unsigned int cpu)
 918{
 919	struct pcpu *pcpu;
 920
 921	/* Wait until target cpu is down */
 922	pcpu = per_cpu_ptr(&pcpu_devices, cpu);
 923	while (!pcpu_stopped(pcpu))
 924		cpu_relax();
 925	pcpu_free_lowcore(pcpu, cpu);
 926	cpumask_clear_cpu(cpu, mm_cpumask(&init_mm));
 927	cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask);
 928	pcpu->flags = 0;
 929}
 930
 931void __noreturn cpu_die(void)
 932{
 933	idle_task_exit();
 934	pcpu_sigp_retry(this_cpu_ptr(&pcpu_devices), SIGP_STOP, 0);
 935	for (;;) ;
 936}
 937
 938void __init smp_fill_possible_mask(void)
 939{
 940	unsigned int possible, sclp_max, cpu;
 941
 942	sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1;
 943	sclp_max = min(smp_max_threads, sclp_max);
 944	sclp_max = (sclp.max_cores * sclp_max) ?: nr_cpu_ids;
 945	possible = setup_possible_cpus ?: nr_cpu_ids;
 946	possible = min(possible, sclp_max);
 947	for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++)
 948		set_cpu_possible(cpu, true);
 949}
 950
 951void __init smp_prepare_cpus(unsigned int max_cpus)
 952{
 953	if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt))
 954		panic("Couldn't request external interrupt 0x1201");
 955	system_ctl_set_bit(0, 14);
 956	if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
 957		panic("Couldn't request external interrupt 0x1202");
 958	system_ctl_set_bit(0, 13);
 959	smp_rescan_cpus(true);
 960}
 961
 962void __init smp_prepare_boot_cpu(void)
 963{
 964	struct lowcore *lc = get_lowcore();
 965
 966	WARN_ON(!cpu_present(0) || !cpu_online(0));
 967	lc->percpu_offset = __per_cpu_offset[0];
 968	ipl_pcpu = per_cpu_ptr(&pcpu_devices, 0);
 969	ipl_pcpu->state = CPU_STATE_CONFIGURED;
 970	lc->pcpu = (unsigned long)ipl_pcpu;
 971	smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
 972	smp_cpu_set_capacity(0, CPU_CAPACITY_HIGH);
 973}
 974
 975void __init smp_setup_processor_id(void)
 976{
 977	struct lowcore *lc = get_lowcore();
 978
 979	lc->cpu_nr = 0;
 980	per_cpu(pcpu_devices, 0).address = stap();
 981	lc->spinlock_lockval = arch_spin_lockval(0);
 982	lc->spinlock_index = 0;
 983}
 984
 985/*
 986 * the frequency of the profiling timer can be changed
 987 * by writing a multiplier value into /proc/profile.
 988 *
 989 * usually you want to run this on all CPUs ;)
 990 */
 991int setup_profiling_timer(unsigned int multiplier)
 992{
 993	return 0;
 994}
 995
 996static ssize_t cpu_configure_show(struct device *dev,
 997				  struct device_attribute *attr, char *buf)
 998{
 999	ssize_t count;
1000
1001	mutex_lock(&smp_cpu_state_mutex);
1002	count = sysfs_emit(buf, "%d\n", per_cpu(pcpu_devices, dev->id).state);
1003	mutex_unlock(&smp_cpu_state_mutex);
1004	return count;
1005}
1006
1007static ssize_t cpu_configure_store(struct device *dev,
1008				   struct device_attribute *attr,
1009				   const char *buf, size_t count)
1010{
1011	struct pcpu *pcpu;
1012	int cpu, val, rc, i;
1013	char delim;
1014
1015	if (sscanf(buf, "%d %c", &val, &delim) != 1)
1016		return -EINVAL;
1017	if (val != 0 && val != 1)
1018		return -EINVAL;
1019	cpus_read_lock();
1020	mutex_lock(&smp_cpu_state_mutex);
1021	rc = -EBUSY;
1022	/* disallow configuration changes of online cpus */
1023	cpu = dev->id;
1024	cpu = smp_get_base_cpu(cpu);
1025	for (i = 0; i <= smp_cpu_mtid; i++)
1026		if (cpu_online(cpu + i))
1027			goto out;
1028	pcpu = per_cpu_ptr(&pcpu_devices, cpu);
1029	rc = 0;
1030	switch (val) {
1031	case 0:
1032		if (pcpu->state != CPU_STATE_CONFIGURED)
1033			break;
1034		rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift);
1035		if (rc)
1036			break;
1037		for (i = 0; i <= smp_cpu_mtid; i++) {
1038			if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1039				continue;
1040			per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_STANDBY;
1041			smp_cpu_set_polarization(cpu + i,
1042						 POLARIZATION_UNKNOWN);
1043		}
1044		topology_expect_change();
1045		break;
1046	case 1:
1047		if (pcpu->state != CPU_STATE_STANDBY)
1048			break;
1049		rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift);
1050		if (rc)
1051			break;
1052		for (i = 0; i <= smp_cpu_mtid; i++) {
1053			if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1054				continue;
1055			per_cpu(pcpu_devices, cpu + i).state = CPU_STATE_CONFIGURED;
1056			smp_cpu_set_polarization(cpu + i,
1057						 POLARIZATION_UNKNOWN);
1058		}
1059		topology_expect_change();
1060		break;
1061	default:
1062		break;
1063	}
1064out:
1065	mutex_unlock(&smp_cpu_state_mutex);
1066	cpus_read_unlock();
1067	return rc ? rc : count;
1068}
1069static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
1070
1071static ssize_t show_cpu_address(struct device *dev,
1072				struct device_attribute *attr, char *buf)
1073{
1074	return sysfs_emit(buf, "%d\n", per_cpu(pcpu_devices, dev->id).address);
1075}
1076static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
1077
1078static struct attribute *cpu_common_attrs[] = {
1079	&dev_attr_configure.attr,
1080	&dev_attr_address.attr,
1081	NULL,
1082};
1083
1084static struct attribute_group cpu_common_attr_group = {
1085	.attrs = cpu_common_attrs,
1086};
1087
1088static struct attribute *cpu_online_attrs[] = {
1089	&dev_attr_idle_count.attr,
1090	&dev_attr_idle_time_us.attr,
1091	NULL,
1092};
1093
1094static struct attribute_group cpu_online_attr_group = {
1095	.attrs = cpu_online_attrs,
1096};
1097
1098static int smp_cpu_online(unsigned int cpu)
1099{
1100	struct cpu *c = per_cpu_ptr(&cpu_devices, cpu);
1101
1102	return sysfs_create_group(&c->dev.kobj, &cpu_online_attr_group);
1103}
1104
1105static int smp_cpu_pre_down(unsigned int cpu)
1106{
1107	struct cpu *c = per_cpu_ptr(&cpu_devices, cpu);
1108
1109	sysfs_remove_group(&c->dev.kobj, &cpu_online_attr_group);
1110	return 0;
1111}
1112
1113bool arch_cpu_is_hotpluggable(int cpu)
1114{
1115	return !!cpu;
1116}
1117
1118int arch_register_cpu(int cpu)
1119{
1120	struct cpu *c = per_cpu_ptr(&cpu_devices, cpu);
1121	int rc;
1122
1123	c->hotpluggable = arch_cpu_is_hotpluggable(cpu);
1124	rc = register_cpu(c, cpu);
1125	if (rc)
1126		goto out;
1127	rc = sysfs_create_group(&c->dev.kobj, &cpu_common_attr_group);
1128	if (rc)
1129		goto out_cpu;
1130	rc = topology_cpu_init(c);
1131	if (rc)
1132		goto out_topology;
1133	return 0;
1134
1135out_topology:
1136	sysfs_remove_group(&c->dev.kobj, &cpu_common_attr_group);
1137out_cpu:
1138	unregister_cpu(c);
1139out:
1140	return rc;
1141}
1142
1143int __ref smp_rescan_cpus(bool early)
1144{
1145	struct sclp_core_info *info;
1146	int nr;
1147
1148	info = kzalloc(sizeof(*info), GFP_KERNEL);
1149	if (!info)
1150		return -ENOMEM;
1151	smp_get_core_info(info, 0);
1152	nr = __smp_rescan_cpus(info, early);
1153	kfree(info);
1154	if (nr)
1155		topology_schedule_update();
1156	return 0;
1157}
1158
1159static ssize_t __ref rescan_store(struct device *dev,
1160				  struct device_attribute *attr,
1161				  const char *buf,
1162				  size_t count)
1163{
1164	int rc;
1165
1166	rc = lock_device_hotplug_sysfs();
1167	if (rc)
1168		return rc;
1169	rc = smp_rescan_cpus(false);
1170	unlock_device_hotplug();
1171	return rc ? rc : count;
1172}
1173static DEVICE_ATTR_WO(rescan);
1174
1175static int __init s390_smp_init(void)
1176{
1177	struct device *dev_root;
1178	int rc;
1179
1180	dev_root = bus_get_dev_root(&cpu_subsys);
1181	if (dev_root) {
1182		rc = device_create_file(dev_root, &dev_attr_rescan);
1183		put_device(dev_root);
1184		if (rc)
1185			return rc;
1186	}
1187	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
1188			       smp_cpu_online, smp_cpu_pre_down);
1189	rc = rc <= 0 ? rc : 0;
1190	return rc;
1191}
1192subsys_initcall(s390_smp_init);