cpu.c

   1/* CPU control.
   2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3 *
   4 * This code is licenced under the GPL.
   5 */
   6#include <linux/sched/mm.h>
   7#include <linux/proc_fs.h>
   8#include <linux/smp.h>
   9#include <linux/init.h>
  10#include <linux/notifier.h>
  11#include <linux/sched/signal.h>
  12#include <linux/sched/hotplug.h>
  13#include <linux/sched/isolation.h>
  14#include <linux/sched/task.h>
  15#include <linux/sched/smt.h>
  16#include <linux/unistd.h>
  17#include <linux/cpu.h>
  18#include <linux/oom.h>
  19#include <linux/rcupdate.h>
  20#include <linux/delay.h>
  21#include <linux/export.h>
  22#include <linux/bug.h>
  23#include <linux/kthread.h>
  24#include <linux/stop_machine.h>
  25#include <linux/mutex.h>
  26#include <linux/gfp.h>
  27#include <linux/suspend.h>
  28#include <linux/lockdep.h>
  29#include <linux/tick.h>
  30#include <linux/irq.h>
  31#include <linux/nmi.h>
  32#include <linux/smpboot.h>
  33#include <linux/relay.h>
  34#include <linux/slab.h>
  35#include <linux/scs.h>
  36#include <linux/percpu-rwsem.h>
  37#include <linux/cpuset.h>
  38#include <linux/random.h>
  39#include <linux/cc_platform.h>
  40#include <linux/parser.h>
  41
  42#include <trace/events/power.h>
  43#define CREATE_TRACE_POINTS
  44#include <trace/events/cpuhp.h>
  45
  46#include "smpboot.h"
  47
  48/**
  49 * struct cpuhp_cpu_state - Per cpu hotplug state storage
  50 * @state:	The current cpu state
  51 * @target:	The target state
  52 * @fail:	Current CPU hotplug callback state
  53 * @thread:	Pointer to the hotplug thread
  54 * @should_run:	Thread should execute
  55 * @rollback:	Perform a rollback
  56 * @single:	Single callback invocation
  57 * @bringup:	Single callback bringup or teardown selector
  58 * @node:	Remote CPU node; for multi-instance, do a
  59 *		single entry callback for install/remove
  60 * @last:	For multi-instance rollback, remember how far we got
  61 * @cb_state:	The state for a single callback (install/uninstall)
  62 * @result:	Result of the operation
  63 * @ap_sync_state:	State for AP synchronization
  64 * @done_up:	Signal completion to the issuer of the task for cpu-up
  65 * @done_down:	Signal completion to the issuer of the task for cpu-down
  66 */
  67struct cpuhp_cpu_state {
  68	enum cpuhp_state	state;
  69	enum cpuhp_state	target;
  70	enum cpuhp_state	fail;
  71#ifdef CONFIG_SMP
  72	struct task_struct	*thread;
  73	bool			should_run;
  74	bool			rollback;
  75	bool			single;
  76	bool			bringup;
  77	struct hlist_node	*node;
  78	struct hlist_node	*last;
  79	enum cpuhp_state	cb_state;
  80	int			result;
  81	atomic_t		ap_sync_state;
  82	struct completion	done_up;
  83	struct completion	done_down;
  84#endif
  85};
  86
  87static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  88	.fail = CPUHP_INVALID,
  89};
  90
  91#ifdef CONFIG_SMP
  92cpumask_t cpus_booted_once_mask;
  93#endif
  94
  95#if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  96static struct lockdep_map cpuhp_state_up_map =
  97	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  98static struct lockdep_map cpuhp_state_down_map =
  99	STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
 100
 101
 102static inline void cpuhp_lock_acquire(bool bringup)
 103{
 104	lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
 105}
 106
 107static inline void cpuhp_lock_release(bool bringup)
 108{
 109	lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
 110}
 111#else
 112
 113static inline void cpuhp_lock_acquire(bool bringup) { }
 114static inline void cpuhp_lock_release(bool bringup) { }
 115
 116#endif
 117
 118/**
 119 * struct cpuhp_step - Hotplug state machine step
 120 * @name:	Name of the step
 121 * @startup:	Startup function of the step
 122 * @teardown:	Teardown function of the step
 123 * @cant_stop:	Bringup/teardown can't be stopped at this step
 124 * @multi_instance:	State has multiple instances which get added afterwards
 125 */
 126struct cpuhp_step {
 127	const char		*name;
 128	union {
 129		int		(*single)(unsigned int cpu);
 130		int		(*multi)(unsigned int cpu,
 131					 struct hlist_node *node);
 132	} startup;
 133	union {
 134		int		(*single)(unsigned int cpu);
 135		int		(*multi)(unsigned int cpu,
 136					 struct hlist_node *node);
 137	} teardown;
 138	/* private: */
 139	struct hlist_head	list;
 140	/* public: */
 141	bool			cant_stop;
 142	bool			multi_instance;
 143};
 144
 145static DEFINE_MUTEX(cpuhp_state_mutex);
 146static struct cpuhp_step cpuhp_hp_states[];
 147
 148static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 149{
 150	return cpuhp_hp_states + state;
 151}
 152
 153static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
 154{
 155	return bringup ? !step->startup.single : !step->teardown.single;
 156}
 157
 158/**
 159 * cpuhp_invoke_callback - Invoke the callbacks for a given state
 160 * @cpu:	The cpu for which the callback should be invoked
 161 * @state:	The state to do callbacks for
 162 * @bringup:	True if the bringup callback should be invoked
 163 * @node:	For multi-instance, do a single entry callback for install/remove
 164 * @lastp:	For multi-instance rollback, remember how far we got
 165 *
 166 * Called from cpu hotplug and from the state register machinery.
 167 *
 168 * Return: %0 on success or a negative errno code
 169 */
 170static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 171				 bool bringup, struct hlist_node *node,
 172				 struct hlist_node **lastp)
 173{
 174	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 175	struct cpuhp_step *step = cpuhp_get_step(state);
 176	int (*cbm)(unsigned int cpu, struct hlist_node *node);
 177	int (*cb)(unsigned int cpu);
 178	int ret, cnt;
 179
 180	if (st->fail == state) {
 181		st->fail = CPUHP_INVALID;
 182		return -EAGAIN;
 183	}
 184
 185	if (cpuhp_step_empty(bringup, step)) {
 186		WARN_ON_ONCE(1);
 187		return 0;
 188	}
 189
 190	if (!step->multi_instance) {
 191		WARN_ON_ONCE(lastp && *lastp);
 192		cb = bringup ? step->startup.single : step->teardown.single;
 193
 194		trace_cpuhp_enter(cpu, st->target, state, cb);
 195		ret = cb(cpu);
 196		trace_cpuhp_exit(cpu, st->state, state, ret);
 197		return ret;
 198	}
 199	cbm = bringup ? step->startup.multi : step->teardown.multi;
 200
 201	/* Single invocation for instance add/remove */
 202	if (node) {
 203		WARN_ON_ONCE(lastp && *lastp);
 204		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 205		ret = cbm(cpu, node);
 206		trace_cpuhp_exit(cpu, st->state, state, ret);
 207		return ret;
 208	}
 209
 210	/* State transition. Invoke on all instances */
 211	cnt = 0;
 212	hlist_for_each(node, &step->list) {
 213		if (lastp && node == *lastp)
 214			break;
 215
 216		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 217		ret = cbm(cpu, node);
 218		trace_cpuhp_exit(cpu, st->state, state, ret);
 219		if (ret) {
 220			if (!lastp)
 221				goto err;
 222
 223			*lastp = node;
 224			return ret;
 225		}
 226		cnt++;
 227	}
 228	if (lastp)
 229		*lastp = NULL;
 230	return 0;
 231err:
 232	/* Rollback the instances if one failed */
 233	cbm = !bringup ? step->startup.multi : step->teardown.multi;
 234	if (!cbm)
 235		return ret;
 236
 237	hlist_for_each(node, &step->list) {
 238		if (!cnt--)
 239			break;
 240
 241		trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 242		ret = cbm(cpu, node);
 243		trace_cpuhp_exit(cpu, st->state, state, ret);
 244		/*
 245		 * Rollback must not fail,
 246		 */
 247		WARN_ON_ONCE(ret);
 248	}
 249	return ret;
 250}
 251
 252/*
 253 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 254 */
 255static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 256{
 257	return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 258}
 259
 260#ifdef CONFIG_SMP
 261static bool cpuhp_is_ap_state(enum cpuhp_state state)
 262{
 263	/*
 264	 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 265	 * purposes as that state is handled explicitly in cpu_down.
 266	 */
 267	return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 268}
 269
 270static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 271{
 272	struct completion *done = bringup ? &st->done_up : &st->done_down;
 273	wait_for_completion(done);
 274}
 275
 276static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 277{
 278	struct completion *done = bringup ? &st->done_up : &st->done_down;
 279	complete(done);
 280}
 281
 282/* Synchronization state management */
 283enum cpuhp_sync_state {
 284	SYNC_STATE_DEAD,
 285	SYNC_STATE_KICKED,
 286	SYNC_STATE_SHOULD_DIE,
 287	SYNC_STATE_ALIVE,
 288	SYNC_STATE_SHOULD_ONLINE,
 289	SYNC_STATE_ONLINE,
 290};
 291
 292#ifdef CONFIG_HOTPLUG_CORE_SYNC
 293/**
 294 * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
 295 * @state:	The synchronization state to set
 296 *
 297 * No synchronization point. Just update of the synchronization state, but implies
 298 * a full barrier so that the AP changes are visible before the control CPU proceeds.
 299 */
 300static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state)
 301{
 302	atomic_t *st = this_cpu_ptr(&cpuhp_state.ap_sync_state);
 303
 304	(void)atomic_xchg(st, state);
 305}
 306
 307void __weak arch_cpuhp_sync_state_poll(void) { cpu_relax(); }
 308
 309static bool cpuhp_wait_for_sync_state(unsigned int cpu, enum cpuhp_sync_state state,
 310				      enum cpuhp_sync_state next_state)
 311{
 312	atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
 313	ktime_t now, end, start = ktime_get();
 314	int sync;
 315
 316	end = start + 10ULL * NSEC_PER_SEC;
 317
 318	sync = atomic_read(st);
 319	while (1) {
 320		if (sync == state) {
 321			if (!atomic_try_cmpxchg(st, &sync, next_state))
 322				continue;
 323			return true;
 324		}
 325
 326		now = ktime_get();
 327		if (now > end) {
 328			/* Timeout. Leave the state unchanged */
 329			return false;
 330		} else if (now - start < NSEC_PER_MSEC) {
 331			/* Poll for one millisecond */
 332			arch_cpuhp_sync_state_poll();
 333		} else {
 334			usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
 335		}
 336		sync = atomic_read(st);
 337	}
 338	return true;
 339}
 340#else  /* CONFIG_HOTPLUG_CORE_SYNC */
 341static inline void cpuhp_ap_update_sync_state(enum cpuhp_sync_state state) { }
 342#endif /* !CONFIG_HOTPLUG_CORE_SYNC */
 343
 344#ifdef CONFIG_HOTPLUG_CORE_SYNC_DEAD
 345/**
 346 * cpuhp_ap_report_dead - Update synchronization state to DEAD
 347 *
 348 * No synchronization point. Just update of the synchronization state.
 349 */
 350void cpuhp_ap_report_dead(void)
 351{
 352	cpuhp_ap_update_sync_state(SYNC_STATE_DEAD);
 353}
 354
 355void __weak arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { }
 356
 357/*
 358 * Late CPU shutdown synchronization point. Cannot use cpuhp_state::done_down
 359 * because the AP cannot issue complete() at this stage.
 360 */
 361static void cpuhp_bp_sync_dead(unsigned int cpu)
 362{
 363	atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
 364	int sync = atomic_read(st);
 365
 366	do {
 367		/* CPU can have reported dead already. Don't overwrite that! */
 368		if (sync == SYNC_STATE_DEAD)
 369			break;
 370	} while (!atomic_try_cmpxchg(st, &sync, SYNC_STATE_SHOULD_DIE));
 371
 372	if (cpuhp_wait_for_sync_state(cpu, SYNC_STATE_DEAD, SYNC_STATE_DEAD)) {
 373		/* CPU reached dead state. Invoke the cleanup function */
 374		arch_cpuhp_cleanup_dead_cpu(cpu);
 375		return;
 376	}
 377
 378	/* No further action possible. Emit message and give up. */
 379	pr_err("CPU%u failed to report dead state\n", cpu);
 380}
 381#else /* CONFIG_HOTPLUG_CORE_SYNC_DEAD */
 382static inline void cpuhp_bp_sync_dead(unsigned int cpu) { }
 383#endif /* !CONFIG_HOTPLUG_CORE_SYNC_DEAD */
 384
 385#ifdef CONFIG_HOTPLUG_CORE_SYNC_FULL
 386/**
 387 * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
 388 *
 389 * Updates the AP synchronization state to SYNC_STATE_ALIVE and waits
 390 * for the BP to release it.
 391 */
 392void cpuhp_ap_sync_alive(void)
 393{
 394	atomic_t *st = this_cpu_ptr(&cpuhp_state.ap_sync_state);
 395
 396	cpuhp_ap_update_sync_state(SYNC_STATE_ALIVE);
 397
 398	/* Wait for the control CPU to release it. */
 399	while (atomic_read(st) != SYNC_STATE_SHOULD_ONLINE)
 400		cpu_relax();
 401}
 402
 403static bool cpuhp_can_boot_ap(unsigned int cpu)
 404{
 405	atomic_t *st = per_cpu_ptr(&cpuhp_state.ap_sync_state, cpu);
 406	int sync = atomic_read(st);
 407
 408again:
 409	switch (sync) {
 410	case SYNC_STATE_DEAD:
 411		/* CPU is properly dead */
 412		break;
 413	case SYNC_STATE_KICKED:
 414		/* CPU did not come up in previous attempt */
 415		break;
 416	case SYNC_STATE_ALIVE:
 417		/* CPU is stuck cpuhp_ap_sync_alive(). */
 418		break;
 419	default:
 420		/* CPU failed to report online or dead and is in limbo state. */
 421		return false;
 422	}
 423
 424	/* Prepare for booting */
 425	if (!atomic_try_cmpxchg(st, &sync, SYNC_STATE_KICKED))
 426		goto again;
 427
 428	return true;
 429}
 430
 431void __weak arch_cpuhp_cleanup_kick_cpu(unsigned int cpu) { }
 432
 433/*
 434 * Early CPU bringup synchronization point. Cannot use cpuhp_state::done_up
 435 * because the AP cannot issue complete() so early in the bringup.
 436 */
 437static int cpuhp_bp_sync_alive(unsigned int cpu)
 438{
 439	int ret = 0;
 440
 441	if (!IS_ENABLED(CONFIG_HOTPLUG_CORE_SYNC_FULL))
 442		return 0;
 443
 444	if (!cpuhp_wait_for_sync_state(cpu, SYNC_STATE_ALIVE, SYNC_STATE_SHOULD_ONLINE)) {
 445		pr_err("CPU%u failed to report alive state\n", cpu);
 446		ret = -EIO;
 447	}
 448
 449	/* Let the architecture cleanup the kick alive mechanics. */
 450	arch_cpuhp_cleanup_kick_cpu(cpu);
 451	return ret;
 452}
 453#else /* CONFIG_HOTPLUG_CORE_SYNC_FULL */
 454static inline int cpuhp_bp_sync_alive(unsigned int cpu) { return 0; }
 455static inline bool cpuhp_can_boot_ap(unsigned int cpu) { return true; }
 456#endif /* !CONFIG_HOTPLUG_CORE_SYNC_FULL */
 457
 458/* Serializes the updates to cpu_online_mask, cpu_present_mask */
 459static DEFINE_MUTEX(cpu_add_remove_lock);
 460bool cpuhp_tasks_frozen;
 461EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 462
 463/*
 464 * The following two APIs (cpu_maps_update_begin/done) must be used when
 465 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 466 */
 467void cpu_maps_update_begin(void)
 468{
 469	mutex_lock(&cpu_add_remove_lock);
 470}
 471
 472void cpu_maps_update_done(void)
 473{
 474	mutex_unlock(&cpu_add_remove_lock);
 475}
 476
 477/*
 478 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 479 * Should always be manipulated under cpu_add_remove_lock
 480 */
 481static int cpu_hotplug_disabled;
 482
 483#ifdef CONFIG_HOTPLUG_CPU
 484
 485DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 486
 487static bool cpu_hotplug_offline_disabled __ro_after_init;
 488
 489void cpus_read_lock(void)
 490{
 491	percpu_down_read(&cpu_hotplug_lock);
 492}
 493EXPORT_SYMBOL_GPL(cpus_read_lock);
 494
 495int cpus_read_trylock(void)
 496{
 497	return percpu_down_read_trylock(&cpu_hotplug_lock);
 498}
 499EXPORT_SYMBOL_GPL(cpus_read_trylock);
 500
 501void cpus_read_unlock(void)
 502{
 503	percpu_up_read(&cpu_hotplug_lock);
 504}
 505EXPORT_SYMBOL_GPL(cpus_read_unlock);
 506
 507void cpus_write_lock(void)
 508{
 509	percpu_down_write(&cpu_hotplug_lock);
 510}
 511
 512void cpus_write_unlock(void)
 513{
 514	percpu_up_write(&cpu_hotplug_lock);
 515}
 516
 517void lockdep_assert_cpus_held(void)
 518{
 519	/*
 520	 * We can't have hotplug operations before userspace starts running,
 521	 * and some init codepaths will knowingly not take the hotplug lock.
 522	 * This is all valid, so mute lockdep until it makes sense to report
 523	 * unheld locks.
 524	 */
 525	if (system_state < SYSTEM_RUNNING)
 526		return;
 527
 528	percpu_rwsem_assert_held(&cpu_hotplug_lock);
 529}
 530EXPORT_SYMBOL_GPL(lockdep_assert_cpus_held);
 531
 532#ifdef CONFIG_LOCKDEP
 533int lockdep_is_cpus_held(void)
 534{
 535	return percpu_rwsem_is_held(&cpu_hotplug_lock);
 536}
 537#endif
 538
 539static void lockdep_acquire_cpus_lock(void)
 540{
 541	rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
 542}
 543
 544static void lockdep_release_cpus_lock(void)
 545{
 546	rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
 547}
 548
 549/* Declare CPU offlining not supported */
 550void cpu_hotplug_disable_offlining(void)
 551{
 552	cpu_maps_update_begin();
 553	cpu_hotplug_offline_disabled = true;
 554	cpu_maps_update_done();
 555}
 556
 557/*
 558 * Wait for currently running CPU hotplug operations to complete (if any) and
 559 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 560 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 561 * hotplug path before performing hotplug operations. So acquiring that lock
 562 * guarantees mutual exclusion from any currently running hotplug operations.
 563 */
 564void cpu_hotplug_disable(void)
 565{
 566	cpu_maps_update_begin();
 567	cpu_hotplug_disabled++;
 568	cpu_maps_update_done();
 569}
 570EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 571
 572static void __cpu_hotplug_enable(void)
 573{
 574	if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 575		return;
 576	cpu_hotplug_disabled--;
 577}
 578
 579void cpu_hotplug_enable(void)
 580{
 581	cpu_maps_update_begin();
 582	__cpu_hotplug_enable();
 583	cpu_maps_update_done();
 584}
 585EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 586
 587#else
 588
 589static void lockdep_acquire_cpus_lock(void)
 590{
 591}
 592
 593static void lockdep_release_cpus_lock(void)
 594{
 595}
 596
 597#endif	/* CONFIG_HOTPLUG_CPU */
 598
 599/*
 600 * Architectures that need SMT-specific errata handling during SMT hotplug
 601 * should override this.
 602 */
 603void __weak arch_smt_update(void) { }
 604
 605#ifdef CONFIG_HOTPLUG_SMT
 606
 607enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
 608static unsigned int cpu_smt_max_threads __ro_after_init;
 609unsigned int cpu_smt_num_threads __read_mostly = UINT_MAX;
 610
 611void __init cpu_smt_disable(bool force)
 612{
 613	if (!cpu_smt_possible())
 614		return;
 615
 616	if (force) {
 617		pr_info("SMT: Force disabled\n");
 618		cpu_smt_control = CPU_SMT_FORCE_DISABLED;
 619	} else {
 620		pr_info("SMT: disabled\n");
 621		cpu_smt_control = CPU_SMT_DISABLED;
 622	}
 623	cpu_smt_num_threads = 1;
 624}
 625
 626/*
 627 * The decision whether SMT is supported can only be done after the full
 628 * CPU identification. Called from architecture code.
 629 */
 630void __init cpu_smt_set_num_threads(unsigned int num_threads,
 631				    unsigned int max_threads)
 632{
 633	WARN_ON(!num_threads || (num_threads > max_threads));
 634
 635	if (max_threads == 1)
 636		cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
 637
 638	cpu_smt_max_threads = max_threads;
 639
 640	/*
 641	 * If SMT has been disabled via the kernel command line or SMT is
 642	 * not supported, set cpu_smt_num_threads to 1 for consistency.
 643	 * If enabled, take the architecture requested number of threads
 644	 * to bring up into account.
 645	 */
 646	if (cpu_smt_control != CPU_SMT_ENABLED)
 647		cpu_smt_num_threads = 1;
 648	else if (num_threads < cpu_smt_num_threads)
 649		cpu_smt_num_threads = num_threads;
 650}
 651
 652static int __init smt_cmdline_disable(char *str)
 653{
 654	cpu_smt_disable(str && !strcmp(str, "force"));
 655	return 0;
 656}
 657early_param("nosmt", smt_cmdline_disable);
 658
 659/*
 660 * For Archicture supporting partial SMT states check if the thread is allowed.
 661 * Otherwise this has already been checked through cpu_smt_max_threads when
 662 * setting the SMT level.
 663 */
 664static inline bool cpu_smt_thread_allowed(unsigned int cpu)
 665{
 666#ifdef CONFIG_SMT_NUM_THREADS_DYNAMIC
 667	return topology_smt_thread_allowed(cpu);
 668#else
 669	return true;
 670#endif
 671}
 672
 673static inline bool cpu_bootable(unsigned int cpu)
 674{
 675	if (cpu_smt_control == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
 676		return true;
 677
 678	/* All CPUs are bootable if controls are not configured */
 679	if (cpu_smt_control == CPU_SMT_NOT_IMPLEMENTED)
 680		return true;
 681
 682	/* All CPUs are bootable if CPU is not SMT capable */
 683	if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 684		return true;
 685
 686	if (topology_is_primary_thread(cpu))
 687		return true;
 688
 689	/*
 690	 * On x86 it's required to boot all logical CPUs at least once so
 691	 * that the init code can get a chance to set CR4.MCE on each
 692	 * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
 693	 * core will shutdown the machine.
 694	 */
 695	return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
 696}
 697
 698/* Returns true if SMT is supported and not forcefully (irreversibly) disabled */
 699bool cpu_smt_possible(void)
 700{
 701	return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
 702		cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
 703}
 704EXPORT_SYMBOL_GPL(cpu_smt_possible);
 705
 706#else
 707static inline bool cpu_bootable(unsigned int cpu) { return true; }
 708#endif
 709
 710static inline enum cpuhp_state
 711cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
 712{
 713	enum cpuhp_state prev_state = st->state;
 714	bool bringup = st->state < target;
 715
 716	st->rollback = false;
 717	st->last = NULL;
 718
 719	st->target = target;
 720	st->single = false;
 721	st->bringup = bringup;
 722	if (cpu_dying(cpu) != !bringup)
 723		set_cpu_dying(cpu, !bringup);
 724
 725	return prev_state;
 726}
 727
 728static inline void
 729cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
 730		  enum cpuhp_state prev_state)
 731{
 732	bool bringup = !st->bringup;
 733
 734	st->target = prev_state;
 735
 736	/*
 737	 * Already rolling back. No need invert the bringup value or to change
 738	 * the current state.
 739	 */
 740	if (st->rollback)
 741		return;
 742
 743	st->rollback = true;
 744
 745	/*
 746	 * If we have st->last we need to undo partial multi_instance of this
 747	 * state first. Otherwise start undo at the previous state.
 748	 */
 749	if (!st->last) {
 750		if (st->bringup)
 751			st->state--;
 752		else
 753			st->state++;
 754	}
 755
 756	st->bringup = bringup;
 757	if (cpu_dying(cpu) != !bringup)
 758		set_cpu_dying(cpu, !bringup);
 759}
 760
 761/* Regular hotplug invocation of the AP hotplug thread */
 762static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 763{
 764	if (!st->single && st->state == st->target)
 765		return;
 766
 767	st->result = 0;
 768	/*
 769	 * Make sure the above stores are visible before should_run becomes
 770	 * true. Paired with the mb() above in cpuhp_thread_fun()
 771	 */
 772	smp_mb();
 773	st->should_run = true;
 774	wake_up_process(st->thread);
 775	wait_for_ap_thread(st, st->bringup);
 776}
 777
 778static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
 779			 enum cpuhp_state target)
 780{
 781	enum cpuhp_state prev_state;
 782	int ret;
 783
 784	prev_state = cpuhp_set_state(cpu, st, target);
 785	__cpuhp_kick_ap(st);
 786	if ((ret = st->result)) {
 787		cpuhp_reset_state(cpu, st, prev_state);
 788		__cpuhp_kick_ap(st);
 789	}
 790
 791	return ret;
 792}
 793
 794static int bringup_wait_for_ap_online(unsigned int cpu)
 795{
 796	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 797
 798	/* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 799	wait_for_ap_thread(st, true);
 800	if (WARN_ON_ONCE((!cpu_online(cpu))))
 801		return -ECANCELED;
 802
 803	/* Unpark the hotplug thread of the target cpu */
 804	kthread_unpark(st->thread);
 805
 806	/*
 807	 * SMT soft disabling on X86 requires to bring the CPU out of the
 808	 * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
 809	 * CPU marked itself as booted_once in notify_cpu_starting() so the
 810	 * cpu_bootable() check will now return false if this is not the
 811	 * primary sibling.
 812	 */
 813	if (!cpu_bootable(cpu))
 814		return -ECANCELED;
 815	return 0;
 816}
 817
 818#ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
 819static int cpuhp_kick_ap_alive(unsigned int cpu)
 820{
 821	if (!cpuhp_can_boot_ap(cpu))
 822		return -EAGAIN;
 823
 824	return arch_cpuhp_kick_ap_alive(cpu, idle_thread_get(cpu));
 825}
 826
 827static int cpuhp_bringup_ap(unsigned int cpu)
 828{
 829	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 830	int ret;
 831
 832	/*
 833	 * Some architectures have to walk the irq descriptors to
 834	 * setup the vector space for the cpu which comes online.
 835	 * Prevent irq alloc/free across the bringup.
 836	 */
 837	irq_lock_sparse();
 838
 839	ret = cpuhp_bp_sync_alive(cpu);
 840	if (ret)
 841		goto out_unlock;
 842
 843	ret = bringup_wait_for_ap_online(cpu);
 844	if (ret)
 845		goto out_unlock;
 846
 847	irq_unlock_sparse();
 848
 849	if (st->target <= CPUHP_AP_ONLINE_IDLE)
 850		return 0;
 851
 852	return cpuhp_kick_ap(cpu, st, st->target);
 853
 854out_unlock:
 855	irq_unlock_sparse();
 856	return ret;
 857}
 858#else
 859static int bringup_cpu(unsigned int cpu)
 860{
 861	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 862	struct task_struct *idle = idle_thread_get(cpu);
 863	int ret;
 864
 865	if (!cpuhp_can_boot_ap(cpu))
 866		return -EAGAIN;
 867
 868	/*
 869	 * Some architectures have to walk the irq descriptors to
 870	 * setup the vector space for the cpu which comes online.
 871	 *
 872	 * Prevent irq alloc/free across the bringup by acquiring the
 873	 * sparse irq lock. Hold it until the upcoming CPU completes the
 874	 * startup in cpuhp_online_idle() which allows to avoid
 875	 * intermediate synchronization points in the architecture code.
 876	 */
 877	irq_lock_sparse();
 878
 879	ret = __cpu_up(cpu, idle);
 880	if (ret)
 881		goto out_unlock;
 882
 883	ret = cpuhp_bp_sync_alive(cpu);
 884	if (ret)
 885		goto out_unlock;
 886
 887	ret = bringup_wait_for_ap_online(cpu);
 888	if (ret)
 889		goto out_unlock;
 890
 891	irq_unlock_sparse();
 892
 893	if (st->target <= CPUHP_AP_ONLINE_IDLE)
 894		return 0;
 895
 896	return cpuhp_kick_ap(cpu, st, st->target);
 897
 898out_unlock:
 899	irq_unlock_sparse();
 900	return ret;
 901}
 902#endif
 903
 904static int finish_cpu(unsigned int cpu)
 905{
 906	struct task_struct *idle = idle_thread_get(cpu);
 907	struct mm_struct *mm = idle->active_mm;
 908
 909	/*
 910	 * sched_force_init_mm() ensured the use of &init_mm,
 911	 * drop that refcount now that the CPU has stopped.
 912	 */
 913	WARN_ON(mm != &init_mm);
 914	idle->active_mm = NULL;
 915	mmdrop_lazy_tlb(mm);
 916
 917	return 0;
 918}
 919
 920/*
 921 * Hotplug state machine related functions
 922 */
 923
 924/*
 925 * Get the next state to run. Empty ones will be skipped. Returns true if a
 926 * state must be run.
 927 *
 928 * st->state will be modified ahead of time, to match state_to_run, as if it
 929 * has already ran.
 930 */
 931static bool cpuhp_next_state(bool bringup,
 932			     enum cpuhp_state *state_to_run,
 933			     struct cpuhp_cpu_state *st,
 934			     enum cpuhp_state target)
 935{
 936	do {
 937		if (bringup) {
 938			if (st->state >= target)
 939				return false;
 940
 941			*state_to_run = ++st->state;
 942		} else {
 943			if (st->state <= target)
 944				return false;
 945
 946			*state_to_run = st->state--;
 947		}
 948
 949		if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
 950			break;
 951	} while (true);
 952
 953	return true;
 954}
 955
 956static int __cpuhp_invoke_callback_range(bool bringup,
 957					 unsigned int cpu,
 958					 struct cpuhp_cpu_state *st,
 959					 enum cpuhp_state target,
 960					 bool nofail)
 961{
 962	enum cpuhp_state state;
 963	int ret = 0;
 964
 965	while (cpuhp_next_state(bringup, &state, st, target)) {
 966		int err;
 967
 968		err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
 969		if (!err)
 970			continue;
 971
 972		if (nofail) {
 973			pr_warn("CPU %u %s state %s (%d) failed (%d)\n",
 974				cpu, bringup ? "UP" : "DOWN",
 975				cpuhp_get_step(st->state)->name,
 976				st->state, err);
 977			ret = -1;
 978		} else {
 979			ret = err;
 980			break;
 981		}
 982	}
 983
 984	return ret;
 985}
 986
 987static inline int cpuhp_invoke_callback_range(bool bringup,
 988					      unsigned int cpu,
 989					      struct cpuhp_cpu_state *st,
 990					      enum cpuhp_state target)
 991{
 992	return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false);
 993}
 994
 995static inline void cpuhp_invoke_callback_range_nofail(bool bringup,
 996						      unsigned int cpu,
 997						      struct cpuhp_cpu_state *st,
 998						      enum cpuhp_state target)
 999{
1000	__cpuhp_invoke_callback_range(bringup, cpu, st, target, true);
1001}
1002
1003static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
1004{
1005	if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
1006		return true;
1007	/*
1008	 * When CPU hotplug is disabled, then taking the CPU down is not
1009	 * possible because takedown_cpu() and the architecture and
1010	 * subsystem specific mechanisms are not available. So the CPU
1011	 * which would be completely unplugged again needs to stay around
1012	 * in the current state.
1013	 */
1014	return st->state <= CPUHP_BRINGUP_CPU;
1015}
1016
1017static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
1018			      enum cpuhp_state target)
1019{
1020	enum cpuhp_state prev_state = st->state;
1021	int ret = 0;
1022
1023	ret = cpuhp_invoke_callback_range(true, cpu, st, target);
1024	if (ret) {
1025		pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
1026			 ret, cpu, cpuhp_get_step(st->state)->name,
1027			 st->state);
1028
1029		cpuhp_reset_state(cpu, st, prev_state);
1030		if (can_rollback_cpu(st))
1031			WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
1032							    prev_state));
1033	}
1034	return ret;
1035}
1036
1037/*
1038 * The cpu hotplug threads manage the bringup and teardown of the cpus
1039 */
1040static int cpuhp_should_run(unsigned int cpu)
1041{
1042	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1043
1044	return st->should_run;
1045}
1046
1047/*
1048 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
1049 * callbacks when a state gets [un]installed at runtime.
1050 *
1051 * Each invocation of this function by the smpboot thread does a single AP
1052 * state callback.
1053 *
1054 * It has 3 modes of operation:
1055 *  - single: runs st->cb_state
1056 *  - up:     runs ++st->state, while st->state < st->target
1057 *  - down:   runs st->state--, while st->state > st->target
1058 *
1059 * When complete or on error, should_run is cleared and the completion is fired.
1060 */
1061static void cpuhp_thread_fun(unsigned int cpu)
1062{
1063	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1064	bool bringup = st->bringup;
1065	enum cpuhp_state state;
1066
1067	if (WARN_ON_ONCE(!st->should_run))
1068		return;
1069
1070	/*
1071	 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
1072	 * that if we see ->should_run we also see the rest of the state.
1073	 */
1074	smp_mb();
1075
1076	/*
1077	 * The BP holds the hotplug lock, but we're now running on the AP,
1078	 * ensure that anybody asserting the lock is held, will actually find
1079	 * it so.
1080	 */
1081	lockdep_acquire_cpus_lock();
1082	cpuhp_lock_acquire(bringup);
1083
1084	if (st->single) {
1085		state = st->cb_state;
1086		st->should_run = false;
1087	} else {
1088		st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
1089		if (!st->should_run)
1090			goto end;
1091	}
1092
1093	WARN_ON_ONCE(!cpuhp_is_ap_state(state));
1094
1095	if (cpuhp_is_atomic_state(state)) {
1096		local_irq_disable();
1097		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
1098		local_irq_enable();
1099
1100		/*
1101		 * STARTING/DYING must not fail!
1102		 */
1103		WARN_ON_ONCE(st->result);
1104	} else {
1105		st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
1106	}
1107
1108	if (st->result) {
1109		/*
1110		 * If we fail on a rollback, we're up a creek without no
1111		 * paddle, no way forward, no way back. We loose, thanks for
1112		 * playing.
1113		 */
1114		WARN_ON_ONCE(st->rollback);
1115		st->should_run = false;
1116	}
1117
1118end:
1119	cpuhp_lock_release(bringup);
1120	lockdep_release_cpus_lock();
1121
1122	if (!st->should_run)
1123		complete_ap_thread(st, bringup);
1124}
1125
1126/* Invoke a single callback on a remote cpu */
1127static int
1128cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
1129			 struct hlist_node *node)
1130{
1131	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1132	int ret;
1133
1134	if (!cpu_online(cpu))
1135		return 0;
1136
1137	cpuhp_lock_acquire(false);
1138	cpuhp_lock_release(false);
1139
1140	cpuhp_lock_acquire(true);
1141	cpuhp_lock_release(true);
1142
1143	/*
1144	 * If we are up and running, use the hotplug thread. For early calls
1145	 * we invoke the thread function directly.
1146	 */
1147	if (!st->thread)
1148		return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1149
1150	st->rollback = false;
1151	st->last = NULL;
1152
1153	st->node = node;
1154	st->bringup = bringup;
1155	st->cb_state = state;
1156	st->single = true;
1157
1158	__cpuhp_kick_ap(st);
1159
1160	/*
1161	 * If we failed and did a partial, do a rollback.
1162	 */
1163	if ((ret = st->result) && st->last) {
1164		st->rollback = true;
1165		st->bringup = !bringup;
1166
1167		__cpuhp_kick_ap(st);
1168	}
1169
1170	/*
1171	 * Clean up the leftovers so the next hotplug operation wont use stale
1172	 * data.
1173	 */
1174	st->node = st->last = NULL;
1175	return ret;
1176}
1177
1178static int cpuhp_kick_ap_work(unsigned int cpu)
1179{
1180	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1181	enum cpuhp_state prev_state = st->state;
1182	int ret;
1183
1184	cpuhp_lock_acquire(false);
1185	cpuhp_lock_release(false);
1186
1187	cpuhp_lock_acquire(true);
1188	cpuhp_lock_release(true);
1189
1190	trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
1191	ret = cpuhp_kick_ap(cpu, st, st->target);
1192	trace_cpuhp_exit(cpu, st->state, prev_state, ret);
1193
1194	return ret;
1195}
1196
1197static struct smp_hotplug_thread cpuhp_threads = {
1198	.store			= &cpuhp_state.thread,
1199	.thread_should_run	= cpuhp_should_run,
1200	.thread_fn		= cpuhp_thread_fun,
1201	.thread_comm		= "cpuhp/%u",
1202	.selfparking		= true,
1203};
1204
1205static __init void cpuhp_init_state(void)
1206{
1207	struct cpuhp_cpu_state *st;
1208	int cpu;
1209
1210	for_each_possible_cpu(cpu) {
1211		st = per_cpu_ptr(&cpuhp_state, cpu);
1212		init_completion(&st->done_up);
1213		init_completion(&st->done_down);
1214	}
1215}
1216
1217void __init cpuhp_threads_init(void)
1218{
1219	cpuhp_init_state();
1220	BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
1221	kthread_unpark(this_cpu_read(cpuhp_state.thread));
1222}
1223
1224#ifdef CONFIG_HOTPLUG_CPU
1225#ifndef arch_clear_mm_cpumask_cpu
1226#define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
1227#endif
1228
1229/**
1230 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
1231 * @cpu: a CPU id
1232 *
1233 * This function walks all processes, finds a valid mm struct for each one and
1234 * then clears a corresponding bit in mm's cpumask.  While this all sounds
1235 * trivial, there are various non-obvious corner cases, which this function
1236 * tries to solve in a safe manner.
1237 *
1238 * Also note that the function uses a somewhat relaxed locking scheme, so it may
1239 * be called only for an already offlined CPU.
1240 */
1241void clear_tasks_mm_cpumask(int cpu)
1242{
1243	struct task_struct *p;
1244
1245	/*
1246	 * This function is called after the cpu is taken down and marked
1247	 * offline, so its not like new tasks will ever get this cpu set in
1248	 * their mm mask. -- Peter Zijlstra
1249	 * Thus, we may use rcu_read_lock() here, instead of grabbing
1250	 * full-fledged tasklist_lock.
1251	 */
1252	WARN_ON(cpu_online(cpu));
1253	rcu_read_lock();
1254	for_each_process(p) {
1255		struct task_struct *t;
1256
1257		/*
1258		 * Main thread might exit, but other threads may still have
1259		 * a valid mm. Find one.
1260		 */
1261		t = find_lock_task_mm(p);
1262		if (!t)
1263			continue;
1264		arch_clear_mm_cpumask_cpu(cpu, t->mm);
1265		task_unlock(t);
1266	}
1267	rcu_read_unlock();
1268}
1269
1270/* Take this CPU down. */
1271static int take_cpu_down(void *_param)
1272{
1273	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1274	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
1275	int err, cpu = smp_processor_id();
1276
1277	/* Ensure this CPU doesn't handle any more interrupts. */
1278	err = __cpu_disable();
1279	if (err < 0)
1280		return err;
1281
1282	/*
1283	 * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
1284	 * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
1285	 */
1286	WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));
1287
1288	/*
1289	 * Invoke the former CPU_DYING callbacks. DYING must not fail!
1290	 */
1291	cpuhp_invoke_callback_range_nofail(false, cpu, st, target);
1292
1293	/* Park the stopper thread */
1294	stop_machine_park(cpu);
1295	return 0;
1296}
1297
1298static int takedown_cpu(unsigned int cpu)
1299{
1300	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1301	int err;
1302
1303	/* Park the smpboot threads */
1304	kthread_park(st->thread);
1305
1306	/*
1307	 * Prevent irq alloc/free while the dying cpu reorganizes the
1308	 * interrupt affinities.
1309	 */
1310	irq_lock_sparse();
1311
1312	err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
1313	if (err) {
1314		/* CPU refused to die */
1315		irq_unlock_sparse();
1316		/* Unpark the hotplug thread so we can rollback there */
1317		kthread_unpark(st->thread);
1318		return err;
1319	}
1320	BUG_ON(cpu_online(cpu));
1321
1322	/*
1323	 * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
1324	 * all runnable tasks from the CPU, there's only the idle task left now
1325	 * that the migration thread is done doing the stop_machine thing.
1326	 *
1327	 * Wait for the stop thread to go away.
1328	 */
1329	wait_for_ap_thread(st, false);
1330	BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
1331
1332	/* Interrupts are moved away from the dying cpu, reenable alloc/free */
1333	irq_unlock_sparse();
1334
1335	hotplug_cpu__broadcast_tick_pull(cpu);
1336	/* This actually kills the CPU. */
1337	__cpu_die(cpu);
1338
1339	cpuhp_bp_sync_dead(cpu);
1340
1341	lockdep_cleanup_dead_cpu(cpu, idle_thread_get(cpu));
1342
1343	/*
1344	 * Callbacks must be re-integrated right away to the RCU state machine.
1345	 * Otherwise an RCU callback could block a further teardown function
1346	 * waiting for its completion.
1347	 */
1348	rcutree_migrate_callbacks(cpu);
1349
1350	return 0;
1351}
1352
1353static void cpuhp_complete_idle_dead(void *arg)
1354{
1355	struct cpuhp_cpu_state *st = arg;
1356
1357	complete_ap_thread(st, false);
1358}
1359
1360void cpuhp_report_idle_dead(void)
1361{
1362	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1363
1364	BUG_ON(st->state != CPUHP_AP_OFFLINE);
1365	tick_assert_timekeeping_handover();
1366	rcutree_report_cpu_dead();
1367	st->state = CPUHP_AP_IDLE_DEAD;
1368	/*
1369	 * We cannot call complete after rcutree_report_cpu_dead() so we delegate it
1370	 * to an online cpu.
1371	 */
1372	smp_call_function_single(cpumask_first(cpu_online_mask),
1373				 cpuhp_complete_idle_dead, st, 0);
1374}
1375
1376static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
1377				enum cpuhp_state target)
1378{
1379	enum cpuhp_state prev_state = st->state;
1380	int ret = 0;
1381
1382	ret = cpuhp_invoke_callback_range(false, cpu, st, target);
1383	if (ret) {
1384		pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
1385			 ret, cpu, cpuhp_get_step(st->state)->name,
1386			 st->state);
1387
1388		cpuhp_reset_state(cpu, st, prev_state);
1389
1390		if (st->state < prev_state)
1391			WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
1392							    prev_state));
1393	}
1394
1395	return ret;
1396}
1397
1398/* Requires cpu_add_remove_lock to be held */
1399static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1400			   enum cpuhp_state target)
1401{
1402	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1403	int prev_state, ret = 0;
1404
1405	if (num_online_cpus() == 1)
1406		return -EBUSY;
1407
1408	if (!cpu_present(cpu))
1409		return -EINVAL;
1410
1411	cpus_write_lock();
1412
1413	cpuhp_tasks_frozen = tasks_frozen;
1414
1415	prev_state = cpuhp_set_state(cpu, st, target);
1416	/*
1417	 * If the current CPU state is in the range of the AP hotplug thread,
1418	 * then we need to kick the thread.
1419	 */
1420	if (st->state > CPUHP_TEARDOWN_CPU) {
1421		st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1422		ret = cpuhp_kick_ap_work(cpu);
1423		/*
1424		 * The AP side has done the error rollback already. Just
1425		 * return the error code..
1426		 */
1427		if (ret)
1428			goto out;
1429
1430		/*
1431		 * We might have stopped still in the range of the AP hotplug
1432		 * thread. Nothing to do anymore.
1433		 */
1434		if (st->state > CPUHP_TEARDOWN_CPU)
1435			goto out;
1436
1437		st->target = target;
1438	}
1439	/*
1440	 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1441	 * to do the further cleanups.
1442	 */
1443	ret = cpuhp_down_callbacks(cpu, st, target);
1444	if (ret && st->state < prev_state) {
1445		if (st->state == CPUHP_TEARDOWN_CPU) {
1446			cpuhp_reset_state(cpu, st, prev_state);
1447			__cpuhp_kick_ap(st);
1448		} else {
1449			WARN(1, "DEAD callback error for CPU%d", cpu);
1450		}
1451	}
1452
1453out:
1454	cpus_write_unlock();
1455	arch_smt_update();
1456	return ret;
1457}
1458
1459struct cpu_down_work {
1460	unsigned int		cpu;
1461	enum cpuhp_state	target;
1462};
1463
1464static long __cpu_down_maps_locked(void *arg)
1465{
1466	struct cpu_down_work *work = arg;
1467
1468	return _cpu_down(work->cpu, 0, work->target);
1469}
1470
1471static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1472{
1473	struct cpu_down_work work = { .cpu = cpu, .target = target, };
1474
1475	/*
1476	 * If the platform does not support hotplug, report it explicitly to
1477	 * differentiate it from a transient offlining failure.
1478	 */
1479	if (cpu_hotplug_offline_disabled)
1480		return -EOPNOTSUPP;
1481	if (cpu_hotplug_disabled)
1482		return -EBUSY;
1483
1484	/*
1485	 * Ensure that the control task does not run on the to be offlined
1486	 * CPU to prevent a deadlock against cfs_b->period_timer.
1487	 * Also keep at least one housekeeping cpu onlined to avoid generating
1488	 * an empty sched_domain span.
1489	 */
1490	for_each_cpu_and(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)) {
1491		if (cpu != work.cpu)
1492			return work_on_cpu(cpu, __cpu_down_maps_locked, &work);
1493	}
1494	return -EBUSY;
1495}
1496
1497static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1498{
1499	int err;
1500
1501	cpu_maps_update_begin();
1502	err = cpu_down_maps_locked(cpu, target);
1503	cpu_maps_update_done();
1504	return err;
1505}
1506
1507/**
1508 * cpu_device_down - Bring down a cpu device
1509 * @dev: Pointer to the cpu device to offline
1510 *
1511 * This function is meant to be used by device core cpu subsystem only.
1512 *
1513 * Other subsystems should use remove_cpu() instead.
1514 *
1515 * Return: %0 on success or a negative errno code
1516 */
1517int cpu_device_down(struct device *dev)
1518{
1519	return cpu_down(dev->id, CPUHP_OFFLINE);
1520}
1521
1522int remove_cpu(unsigned int cpu)
1523{
1524	int ret;
1525
1526	lock_device_hotplug();
1527	ret = device_offline(get_cpu_device(cpu));
1528	unlock_device_hotplug();
1529
1530	return ret;
1531}
1532EXPORT_SYMBOL_GPL(remove_cpu);
1533
1534void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1535{
1536	unsigned int cpu;
1537	int error;
1538
1539	cpu_maps_update_begin();
1540
1541	/*
1542	 * Make certain the cpu I'm about to reboot on is online.
1543	 *
1544	 * This is inline to what migrate_to_reboot_cpu() already do.
1545	 */
1546	if (!cpu_online(primary_cpu))
1547		primary_cpu = cpumask_first(cpu_online_mask);
1548
1549	for_each_online_cpu(cpu) {
1550		if (cpu == primary_cpu)
1551			continue;
1552
1553		error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1554		if (error) {
1555			pr_err("Failed to offline CPU%d - error=%d",
1556				cpu, error);
1557			break;
1558		}
1559	}
1560
1561	/*
1562	 * Ensure all but the reboot CPU are offline.
1563	 */
1564	BUG_ON(num_online_cpus() > 1);
1565
1566	/*
1567	 * Make sure the CPUs won't be enabled by someone else after this
1568	 * point. Kexec will reboot to a new kernel shortly resetting
1569	 * everything along the way.
1570	 */
1571	cpu_hotplug_disabled++;
1572
1573	cpu_maps_update_done();
1574}
1575
1576#else
1577#define takedown_cpu		NULL
1578#endif /*CONFIG_HOTPLUG_CPU*/
1579
1580/**
1581 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1582 * @cpu: cpu that just started
1583 *
1584 * It must be called by the arch code on the new cpu, before the new cpu
1585 * enables interrupts and before the "boot" cpu returns from __cpu_up().
1586 */
1587void notify_cpu_starting(unsigned int cpu)
1588{
1589	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1590	enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1591
1592	rcutree_report_cpu_starting(cpu);	/* Enables RCU usage on this CPU. */
1593	cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1594
1595	/*
1596	 * STARTING must not fail!
1597	 */
1598	cpuhp_invoke_callback_range_nofail(true, cpu, st, target);
1599}
1600
1601/*
1602 * Called from the idle task. Wake up the controlling task which brings the
1603 * hotplug thread of the upcoming CPU up and then delegates the rest of the
1604 * online bringup to the hotplug thread.
1605 */
1606void cpuhp_online_idle(enum cpuhp_state state)
1607{
1608	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1609
1610	/* Happens for the boot cpu */
1611	if (state != CPUHP_AP_ONLINE_IDLE)
1612		return;
1613
1614	cpuhp_ap_update_sync_state(SYNC_STATE_ONLINE);
1615
1616	/*
1617	 * Unpark the stopper thread before we start the idle loop (and start
1618	 * scheduling); this ensures the stopper task is always available.
1619	 */
1620	stop_machine_unpark(smp_processor_id());
1621
1622	st->state = CPUHP_AP_ONLINE_IDLE;
1623	complete_ap_thread(st, true);
1624}
1625
1626/* Requires cpu_add_remove_lock to be held */
1627static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1628{
1629	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1630	struct task_struct *idle;
1631	int ret = 0;
1632
1633	cpus_write_lock();
1634
1635	if (!cpu_present(cpu)) {
1636		ret = -EINVAL;
1637		goto out;
1638	}
1639
1640	/*
1641	 * The caller of cpu_up() might have raced with another
1642	 * caller. Nothing to do.
1643	 */
1644	if (st->state >= target)
1645		goto out;
1646
1647	if (st->state == CPUHP_OFFLINE) {
1648		/* Let it fail before we try to bring the cpu up */
1649		idle = idle_thread_get(cpu);
1650		if (IS_ERR(idle)) {
1651			ret = PTR_ERR(idle);
1652			goto out;
1653		}
1654
1655		/*
1656		 * Reset stale stack state from the last time this CPU was online.
1657		 */
1658		scs_task_reset(idle);
1659		kasan_unpoison_task_stack(idle);
1660	}
1661
1662	cpuhp_tasks_frozen = tasks_frozen;
1663
1664	cpuhp_set_state(cpu, st, target);
1665	/*
1666	 * If the current CPU state is in the range of the AP hotplug thread,
1667	 * then we need to kick the thread once more.
1668	 */
1669	if (st->state > CPUHP_BRINGUP_CPU) {
1670		ret = cpuhp_kick_ap_work(cpu);
1671		/*
1672		 * The AP side has done the error rollback already. Just
1673		 * return the error code..
1674		 */
1675		if (ret)
1676			goto out;
1677	}
1678
1679	/*
1680	 * Try to reach the target state. We max out on the BP at
1681	 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1682	 * responsible for bringing it up to the target state.
1683	 */
1684	target = min((int)target, CPUHP_BRINGUP_CPU);
1685	ret = cpuhp_up_callbacks(cpu, st, target);
1686out:
1687	cpus_write_unlock();
1688	arch_smt_update();
1689	return ret;
1690}
1691
1692static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1693{
1694	int err = 0;
1695
1696	if (!cpu_possible(cpu)) {
1697		pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1698		       cpu);
1699		return -EINVAL;
1700	}
1701
1702	err = try_online_node(cpu_to_node(cpu));
1703	if (err)
1704		return err;
1705
1706	cpu_maps_update_begin();
1707
1708	if (cpu_hotplug_disabled) {
1709		err = -EBUSY;
1710		goto out;
1711	}
1712	if (!cpu_bootable(cpu)) {
1713		err = -EPERM;
1714		goto out;
1715	}
1716
1717	err = _cpu_up(cpu, 0, target);
1718out:
1719	cpu_maps_update_done();
1720	return err;
1721}
1722
1723/**
1724 * cpu_device_up - Bring up a cpu device
1725 * @dev: Pointer to the cpu device to online
1726 *
1727 * This function is meant to be used by device core cpu subsystem only.
1728 *
1729 * Other subsystems should use add_cpu() instead.
1730 *
1731 * Return: %0 on success or a negative errno code
1732 */
1733int cpu_device_up(struct device *dev)
1734{
1735	return cpu_up(dev->id, CPUHP_ONLINE);
1736}
1737
1738int add_cpu(unsigned int cpu)
1739{
1740	int ret;
1741
1742	lock_device_hotplug();
1743	ret = device_online(get_cpu_device(cpu));
1744	unlock_device_hotplug();
1745
1746	return ret;
1747}
1748EXPORT_SYMBOL_GPL(add_cpu);
1749
1750/**
1751 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1752 * @sleep_cpu: The cpu we hibernated on and should be brought up.
1753 *
1754 * On some architectures like arm64, we can hibernate on any CPU, but on
1755 * wake up the CPU we hibernated on might be offline as a side effect of
1756 * using maxcpus= for example.
1757 *
1758 * Return: %0 on success or a negative errno code
1759 */
1760int bringup_hibernate_cpu(unsigned int sleep_cpu)
1761{
1762	int ret;
1763
1764	if (!cpu_online(sleep_cpu)) {
1765		pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1766		ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1767		if (ret) {
1768			pr_err("Failed to bring hibernate-CPU up!\n");
1769			return ret;
1770		}
1771	}
1772	return 0;
1773}
1774
1775static void __init cpuhp_bringup_mask(const struct cpumask *mask, unsigned int ncpus,
1776				      enum cpuhp_state target)
1777{
1778	unsigned int cpu;
1779
1780	for_each_cpu(cpu, mask) {
1781		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1782
1783		if (cpu_up(cpu, target) && can_rollback_cpu(st)) {
1784			/*
1785			 * If this failed then cpu_up() might have only
1786			 * rolled back to CPUHP_BP_KICK_AP for the final
1787			 * online. Clean it up. NOOP if already rolled back.
1788			 */
1789			WARN_ON(cpuhp_invoke_callback_range(false, cpu, st, CPUHP_OFFLINE));
1790		}
1791
1792		if (!--ncpus)
1793			break;
1794	}
1795}
1796
1797#ifdef CONFIG_HOTPLUG_PARALLEL
1798static bool __cpuhp_parallel_bringup __ro_after_init = true;
1799
1800static int __init parallel_bringup_parse_param(char *arg)
1801{
1802	return kstrtobool(arg, &__cpuhp_parallel_bringup);
1803}
1804early_param("cpuhp.parallel", parallel_bringup_parse_param);
1805
1806#ifdef CONFIG_HOTPLUG_SMT
1807static inline bool cpuhp_smt_aware(void)
1808{
1809	return cpu_smt_max_threads > 1;
1810}
1811
1812static inline const struct cpumask *cpuhp_get_primary_thread_mask(void)
1813{
1814	return cpu_primary_thread_mask;
1815}
1816#else
1817static inline bool cpuhp_smt_aware(void)
1818{
1819	return false;
1820}
1821static inline const struct cpumask *cpuhp_get_primary_thread_mask(void)
1822{
1823	return cpu_none_mask;
1824}
1825#endif
1826
1827bool __weak arch_cpuhp_init_parallel_bringup(void)
1828{
1829	return true;
1830}
1831
1832/*
1833 * On architectures which have enabled parallel bringup this invokes all BP
1834 * prepare states for each of the to be onlined APs first. The last state
1835 * sends the startup IPI to the APs. The APs proceed through the low level
1836 * bringup code in parallel and then wait for the control CPU to release
1837 * them one by one for the final onlining procedure.
1838 *
1839 * This avoids waiting for each AP to respond to the startup IPI in
1840 * CPUHP_BRINGUP_CPU.
1841 */
1842static bool __init cpuhp_bringup_cpus_parallel(unsigned int ncpus)
1843{
1844	const struct cpumask *mask = cpu_present_mask;
1845
1846	if (__cpuhp_parallel_bringup)
1847		__cpuhp_parallel_bringup = arch_cpuhp_init_parallel_bringup();
1848	if (!__cpuhp_parallel_bringup)
1849		return false;
1850
1851	if (cpuhp_smt_aware()) {
1852		const struct cpumask *pmask = cpuhp_get_primary_thread_mask();
1853		static struct cpumask tmp_mask __initdata;
1854
1855		/*
1856		 * X86 requires to prevent that SMT siblings stopped while
1857		 * the primary thread does a microcode update for various
1858		 * reasons. Bring the primary threads up first.
1859		 */
1860		cpumask_and(&tmp_mask, mask, pmask);
1861		cpuhp_bringup_mask(&tmp_mask, ncpus, CPUHP_BP_KICK_AP);
1862		cpuhp_bringup_mask(&tmp_mask, ncpus, CPUHP_ONLINE);
1863		/* Account for the online CPUs */
1864		ncpus -= num_online_cpus();
1865		if (!ncpus)
1866			return true;
1867		/* Create the mask for secondary CPUs */
1868		cpumask_andnot(&tmp_mask, mask, pmask);
1869		mask = &tmp_mask;
1870	}
1871
1872	/* Bring the not-yet started CPUs up */
1873	cpuhp_bringup_mask(mask, ncpus, CPUHP_BP_KICK_AP);
1874	cpuhp_bringup_mask(mask, ncpus, CPUHP_ONLINE);
1875	return true;
1876}
1877#else
1878static inline bool cpuhp_bringup_cpus_parallel(unsigned int ncpus) { return false; }
1879#endif /* CONFIG_HOTPLUG_PARALLEL */
1880
1881void __init bringup_nonboot_cpus(unsigned int max_cpus)
1882{
1883	if (!max_cpus)
1884		return;
1885
1886	/* Try parallel bringup optimization if enabled */
1887	if (cpuhp_bringup_cpus_parallel(max_cpus))
1888		return;
1889
1890	/* Full per CPU serialized bringup */
1891	cpuhp_bringup_mask(cpu_present_mask, max_cpus, CPUHP_ONLINE);
1892}
1893
1894#ifdef CONFIG_PM_SLEEP_SMP
1895static cpumask_var_t frozen_cpus;
1896
1897int freeze_secondary_cpus(int primary)
1898{
1899	int cpu, error = 0;
1900
1901	cpu_maps_update_begin();
1902	if (primary == -1) {
1903		primary = cpumask_first(cpu_online_mask);
1904		if (!housekeeping_cpu(primary, HK_TYPE_TIMER))
1905			primary = housekeeping_any_cpu(HK_TYPE_TIMER);
1906	} else {
1907		if (!cpu_online(primary))
1908			primary = cpumask_first(cpu_online_mask);
1909	}
1910
1911	/*
1912	 * We take down all of the non-boot CPUs in one shot to avoid races
1913	 * with the userspace trying to use the CPU hotplug at the same time
1914	 */
1915	cpumask_clear(frozen_cpus);
1916
1917	pr_info("Disabling non-boot CPUs ...\n");
1918	for (cpu = nr_cpu_ids - 1; cpu >= 0; cpu--) {
1919		if (!cpu_online(cpu) || cpu == primary)
1920			continue;
1921
1922		if (pm_wakeup_pending()) {
1923			pr_info("Wakeup pending. Abort CPU freeze\n");
1924			error = -EBUSY;
1925			break;
1926		}
1927
1928		trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1929		error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1930		trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1931		if (!error)
1932			cpumask_set_cpu(cpu, frozen_cpus);
1933		else {
1934			pr_err("Error taking CPU%d down: %d\n", cpu, error);
1935			break;
1936		}
1937	}
1938
1939	if (!error)
1940		BUG_ON(num_online_cpus() > 1);
1941	else
1942		pr_err("Non-boot CPUs are not disabled\n");
1943
1944	/*
1945	 * Make sure the CPUs won't be enabled by someone else. We need to do
1946	 * this even in case of failure as all freeze_secondary_cpus() users are
1947	 * supposed to do thaw_secondary_cpus() on the failure path.
1948	 */
1949	cpu_hotplug_disabled++;
1950
1951	cpu_maps_update_done();
1952	return error;
1953}
1954
1955void __weak arch_thaw_secondary_cpus_begin(void)
1956{
1957}
1958
1959void __weak arch_thaw_secondary_cpus_end(void)
1960{
1961}
1962
1963void thaw_secondary_cpus(void)
1964{
1965	int cpu, error;
1966
1967	/* Allow everyone to use the CPU hotplug again */
1968	cpu_maps_update_begin();
1969	__cpu_hotplug_enable();
1970	if (cpumask_empty(frozen_cpus))
1971		goto out;
1972
1973	pr_info("Enabling non-boot CPUs ...\n");
1974
1975	arch_thaw_secondary_cpus_begin();
1976
1977	for_each_cpu(cpu, frozen_cpus) {
1978		trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1979		error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1980		trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1981		if (!error) {
1982			pr_info("CPU%d is up\n", cpu);
1983			continue;
1984		}
1985		pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1986	}
1987
1988	arch_thaw_secondary_cpus_end();
1989
1990	cpumask_clear(frozen_cpus);
1991out:
1992	cpu_maps_update_done();
1993}
1994
1995static int __init alloc_frozen_cpus(void)
1996{
1997	if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1998		return -ENOMEM;
1999	return 0;
2000}
2001core_initcall(alloc_frozen_cpus);
2002
2003/*
2004 * When callbacks for CPU hotplug notifications are being executed, we must
2005 * ensure that the state of the system with respect to the tasks being frozen
2006 * or not, as reported by the notification, remains unchanged *throughout the
2007 * duration* of the execution of the callbacks.
2008 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
2009 *
2010 * This synchronization is implemented by mutually excluding regular CPU
2011 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
2012 * Hibernate notifications.
2013 */
2014static int
2015cpu_hotplug_pm_callback(struct notifier_block *nb,
2016			unsigned long action, void *ptr)
2017{
2018	switch (action) {
2019
2020	case PM_SUSPEND_PREPARE:
2021	case PM_HIBERNATION_PREPARE:
2022		cpu_hotplug_disable();
2023		break;
2024
2025	case PM_POST_SUSPEND:
2026	case PM_POST_HIBERNATION:
2027		cpu_hotplug_enable();
2028		break;
2029
2030	default:
2031		return NOTIFY_DONE;
2032	}
2033
2034	return NOTIFY_OK;
2035}
2036
2037
2038static int __init cpu_hotplug_pm_sync_init(void)
2039{
2040	/*
2041	 * cpu_hotplug_pm_callback has higher priority than x86
2042	 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
2043	 * to disable cpu hotplug to avoid cpu hotplug race.
2044	 */
2045	pm_notifier(cpu_hotplug_pm_callback, 0);
2046	return 0;
2047}
2048core_initcall(cpu_hotplug_pm_sync_init);
2049
2050#endif /* CONFIG_PM_SLEEP_SMP */
2051
2052int __boot_cpu_id;
2053
2054#endif /* CONFIG_SMP */
2055
2056/* Boot processor state steps */
2057static struct cpuhp_step cpuhp_hp_states[] = {
2058	[CPUHP_OFFLINE] = {
2059		.name			= "offline",
2060		.startup.single		= NULL,
2061		.teardown.single	= NULL,
2062	},
2063#ifdef CONFIG_SMP
2064	[CPUHP_CREATE_THREADS]= {
2065		.name			= "threads:prepare",
2066		.startup.single		= smpboot_create_threads,
2067		.teardown.single	= NULL,
2068		.cant_stop		= true,
2069	},
2070	[CPUHP_RANDOM_PREPARE] = {
2071		.name			= "random:prepare",
2072		.startup.single		= random_prepare_cpu,
2073		.teardown.single	= NULL,
2074	},
2075	[CPUHP_WORKQUEUE_PREP] = {
2076		.name			= "workqueue:prepare",
2077		.startup.single		= workqueue_prepare_cpu,
2078		.teardown.single	= NULL,
2079	},
2080	[CPUHP_HRTIMERS_PREPARE] = {
2081		.name			= "hrtimers:prepare",
2082		.startup.single		= hrtimers_prepare_cpu,
2083		.teardown.single	= NULL,
2084	},
2085	[CPUHP_SMPCFD_PREPARE] = {
2086		.name			= "smpcfd:prepare",
2087		.startup.single		= smpcfd_prepare_cpu,
2088		.teardown.single	= smpcfd_dead_cpu,
2089	},
2090	[CPUHP_RELAY_PREPARE] = {
2091		.name			= "relay:prepare",
2092		.startup.single		= relay_prepare_cpu,
2093		.teardown.single	= NULL,
2094	},
2095	[CPUHP_RCUTREE_PREP] = {
2096		.name			= "RCU/tree:prepare",
2097		.startup.single		= rcutree_prepare_cpu,
2098		.teardown.single	= rcutree_dead_cpu,
2099	},
2100	/*
2101	 * On the tear-down path, timers_dead_cpu() must be invoked
2102	 * before blk_mq_queue_reinit_notify() from notify_dead(),
2103	 * otherwise a RCU stall occurs.
2104	 */
2105	[CPUHP_TIMERS_PREPARE] = {
2106		.name			= "timers:prepare",
2107		.startup.single		= timers_prepare_cpu,
2108		.teardown.single	= timers_dead_cpu,
2109	},
2110
2111#ifdef CONFIG_HOTPLUG_SPLIT_STARTUP
2112	/*
2113	 * Kicks the AP alive. AP will wait in cpuhp_ap_sync_alive() until
2114	 * the next step will release it.
2115	 */
2116	[CPUHP_BP_KICK_AP] = {
2117		.name			= "cpu:kick_ap",
2118		.startup.single		= cpuhp_kick_ap_alive,
2119	},
2120
2121	/*
2122	 * Waits for the AP to reach cpuhp_ap_sync_alive() and then
2123	 * releases it for the complete bringup.
2124	 */
2125	[CPUHP_BRINGUP_CPU] = {
2126		.name			= "cpu:bringup",
2127		.startup.single		= cpuhp_bringup_ap,
2128		.teardown.single	= finish_cpu,
2129		.cant_stop		= true,
2130	},
2131#else
2132	/*
2133	 * All-in-one CPU bringup state which includes the kick alive.
2134	 */
2135	[CPUHP_BRINGUP_CPU] = {
2136		.name			= "cpu:bringup",
2137		.startup.single		= bringup_cpu,
2138		.teardown.single	= finish_cpu,
2139		.cant_stop		= true,
2140	},
2141#endif
2142	/* Final state before CPU kills itself */
2143	[CPUHP_AP_IDLE_DEAD] = {
2144		.name			= "idle:dead",
2145	},
2146	/*
2147	 * Last state before CPU enters the idle loop to die. Transient state
2148	 * for synchronization.
2149	 */
2150	[CPUHP_AP_OFFLINE] = {
2151		.name			= "ap:offline",
2152		.cant_stop		= true,
2153	},
2154	/* First state is scheduler control. Interrupts are disabled */
2155	[CPUHP_AP_SCHED_STARTING] = {
2156		.name			= "sched:starting",
2157		.startup.single		= sched_cpu_starting,
2158		.teardown.single	= sched_cpu_dying,
2159	},
2160	[CPUHP_AP_RCUTREE_DYING] = {
2161		.name			= "RCU/tree:dying",
2162		.startup.single		= NULL,
2163		.teardown.single	= rcutree_dying_cpu,
2164	},
2165	[CPUHP_AP_SMPCFD_DYING] = {
2166		.name			= "smpcfd:dying",
2167		.startup.single		= NULL,
2168		.teardown.single	= smpcfd_dying_cpu,
2169	},
2170	[CPUHP_AP_HRTIMERS_DYING] = {
2171		.name			= "hrtimers:dying",
2172		.startup.single		= hrtimers_cpu_starting,
2173		.teardown.single	= hrtimers_cpu_dying,
2174	},
2175	[CPUHP_AP_TICK_DYING] = {
2176		.name			= "tick:dying",
2177		.startup.single		= NULL,
2178		.teardown.single	= tick_cpu_dying,
2179	},
2180	/* Entry state on starting. Interrupts enabled from here on. Transient
2181	 * state for synchronsization */
2182	[CPUHP_AP_ONLINE] = {
2183		.name			= "ap:online",
2184	},
2185	/*
2186	 * Handled on control processor until the plugged processor manages
2187	 * this itself.
2188	 */
2189	[CPUHP_TEARDOWN_CPU] = {
2190		.name			= "cpu:teardown",
2191		.startup.single		= NULL,
2192		.teardown.single	= takedown_cpu,
2193		.cant_stop		= true,
2194	},
2195
2196	[CPUHP_AP_SCHED_WAIT_EMPTY] = {
2197		.name			= "sched:waitempty",
2198		.startup.single		= NULL,
2199		.teardown.single	= sched_cpu_wait_empty,
2200	},
2201
2202	/* Handle smpboot threads park/unpark */
2203	[CPUHP_AP_SMPBOOT_THREADS] = {
2204		.name			= "smpboot/threads:online",
2205		.startup.single		= smpboot_unpark_threads,
2206		.teardown.single	= smpboot_park_threads,
2207	},
2208	[CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
2209		.name			= "irq/affinity:online",
2210		.startup.single		= irq_affinity_online_cpu,
2211		.teardown.single	= NULL,
2212	},
2213	[CPUHP_AP_PERF_ONLINE] = {
2214		.name			= "perf:online",
2215		.startup.single		= perf_event_init_cpu,
2216		.teardown.single	= perf_event_exit_cpu,
2217	},
2218	[CPUHP_AP_WATCHDOG_ONLINE] = {
2219		.name			= "lockup_detector:online",
2220		.startup.single		= lockup_detector_online_cpu,
2221		.teardown.single	= lockup_detector_offline_cpu,
2222	},
2223	[CPUHP_AP_WORKQUEUE_ONLINE] = {
2224		.name			= "workqueue:online",
2225		.startup.single		= workqueue_online_cpu,
2226		.teardown.single	= workqueue_offline_cpu,
2227	},
2228	[CPUHP_AP_RANDOM_ONLINE] = {
2229		.name			= "random:online",
2230		.startup.single		= random_online_cpu,
2231		.teardown.single	= NULL,
2232	},
2233	[CPUHP_AP_RCUTREE_ONLINE] = {
2234		.name			= "RCU/tree:online",
2235		.startup.single		= rcutree_online_cpu,
2236		.teardown.single	= rcutree_offline_cpu,
2237	},
2238#endif
2239	/*
2240	 * The dynamically registered state space is here
2241	 */
2242
2243#ifdef CONFIG_SMP
2244	/* Last state is scheduler control setting the cpu active */
2245	[CPUHP_AP_ACTIVE] = {
2246		.name			= "sched:active",
2247		.startup.single		= sched_cpu_activate,
2248		.teardown.single	= sched_cpu_deactivate,
2249	},
2250#endif
2251
2252	/* CPU is fully up and running. */
2253	[CPUHP_ONLINE] = {
2254		.name			= "online",
2255		.startup.single		= NULL,
2256		.teardown.single	= NULL,
2257	},
2258};
2259
2260/* Sanity check for callbacks */
2261static int cpuhp_cb_check(enum cpuhp_state state)
2262{
2263	if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
2264		return -EINVAL;
2265	return 0;
2266}
2267
2268/*
2269 * Returns a free for dynamic slot assignment of the Online state. The states
2270 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
2271 * by having no name assigned.
2272 */
2273static int cpuhp_reserve_state(enum cpuhp_state state)
2274{
2275	enum cpuhp_state i, end;
2276	struct cpuhp_step *step;
2277
2278	switch (state) {
2279	case CPUHP_AP_ONLINE_DYN:
2280		step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
2281		end = CPUHP_AP_ONLINE_DYN_END;
2282		break;
2283	case CPUHP_BP_PREPARE_DYN:
2284		step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
2285		end = CPUHP_BP_PREPARE_DYN_END;
2286		break;
2287	default:
2288		return -EINVAL;
2289	}
2290
2291	for (i = state; i <= end; i++, step++) {
2292		if (!step->name)
2293			return i;
2294	}
2295	WARN(1, "No more dynamic states available for CPU hotplug\n");
2296	return -ENOSPC;
2297}
2298
2299static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
2300				 int (*startup)(unsigned int cpu),
2301				 int (*teardown)(unsigned int cpu),
2302				 bool multi_instance)
2303{
2304	/* (Un)Install the callbacks for further cpu hotplug operations */
2305	struct cpuhp_step *sp;
2306	int ret = 0;
2307
2308	/*
2309	 * If name is NULL, then the state gets removed.
2310	 *
2311	 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
2312	 * the first allocation from these dynamic ranges, so the removal
2313	 * would trigger a new allocation and clear the wrong (already
2314	 * empty) state, leaving the callbacks of the to be cleared state
2315	 * dangling, which causes wreckage on the next hotplug operation.
2316	 */
2317	if (name && (state == CPUHP_AP_ONLINE_DYN ||
2318		     state == CPUHP_BP_PREPARE_DYN)) {
2319		ret = cpuhp_reserve_state(state);
2320		if (ret < 0)
2321			return ret;
2322		state = ret;
2323	}
2324	sp = cpuhp_get_step(state);
2325	if (name && sp->name)
2326		return -EBUSY;
2327
2328	sp->startup.single = startup;
2329	sp->teardown.single = teardown;
2330	sp->name = name;
2331	sp->multi_instance = multi_instance;
2332	INIT_HLIST_HEAD(&sp->list);
2333	return ret;
2334}
2335
2336static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
2337{
2338	return cpuhp_get_step(state)->teardown.single;
2339}
2340
2341/*
2342 * Call the startup/teardown function for a step either on the AP or
2343 * on the current CPU.
2344 */
2345static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
2346			    struct hlist_node *node)
2347{
2348	struct cpuhp_step *sp = cpuhp_get_step(state);
2349	int ret;
2350
2351	/*
2352	 * If there's nothing to do, we done.
2353	 * Relies on the union for multi_instance.
2354	 */
2355	if (cpuhp_step_empty(bringup, sp))
2356		return 0;
2357	/*
2358	 * The non AP bound callbacks can fail on bringup. On teardown
2359	 * e.g. module removal we crash for now.
2360	 */
2361#ifdef CONFIG_SMP
2362	if (cpuhp_is_ap_state(state))
2363		ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
2364	else
2365		ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
2366#else
2367	if (cpuhp_is_atomic_state(state)) {
2368		guard(irqsave)();
2369		ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
2370		/* STARTING/DYING must not fail! */
2371		WARN_ON_ONCE(ret);
2372	} else {
2373		ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
2374	}
2375#endif
2376	BUG_ON(ret && !bringup);
2377	return ret;
2378}
2379
2380/*
2381 * Called from __cpuhp_setup_state on a recoverable failure.
2382 *
2383 * Note: The teardown callbacks for rollback are not allowed to fail!
2384 */
2385static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
2386				   struct hlist_node *node)
2387{
2388	int cpu;
2389
2390	/* Roll back the already executed steps on the other cpus */
2391	for_each_present_cpu(cpu) {
2392		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2393		int cpustate = st->state;
2394
2395		if (cpu >= failedcpu)
2396			break;
2397
2398		/* Did we invoke the startup call on that cpu ? */
2399		if (cpustate >= state)
2400			cpuhp_issue_call(cpu, state, false, node);
2401	}
2402}
2403
2404int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
2405					  struct hlist_node *node,
2406					  bool invoke)
2407{
2408	struct cpuhp_step *sp;
2409	int cpu;
2410	int ret;
2411
2412	lockdep_assert_cpus_held();
2413
2414	sp = cpuhp_get_step(state);
2415	if (sp->multi_instance == false)
2416		return -EINVAL;
2417
2418	mutex_lock(&cpuhp_state_mutex);
2419
2420	if (!invoke || !sp->startup.multi)
2421		goto add_node;
2422
2423	/*
2424	 * Try to call the startup callback for each present cpu
2425	 * depending on the hotplug state of the cpu.
2426	 */
2427	for_each_present_cpu(cpu) {
2428		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2429		int cpustate = st->state;
2430
2431		if (cpustate < state)
2432			continue;
2433
2434		ret = cpuhp_issue_call(cpu, state, true, node);
2435		if (ret) {
2436			if (sp->teardown.multi)
2437				cpuhp_rollback_install(cpu, state, node);
2438			goto unlock;
2439		}
2440	}
2441add_node:
2442	ret = 0;
2443	hlist_add_head(node, &sp->list);
2444unlock:
2445	mutex_unlock(&cpuhp_state_mutex);
2446	return ret;
2447}
2448
2449int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
2450			       bool invoke)
2451{
2452	int ret;
2453
2454	cpus_read_lock();
2455	ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
2456	cpus_read_unlock();
2457	return ret;
2458}
2459EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
2460
2461/**
2462 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2463 * @state:		The state to setup
2464 * @name:		Name of the step
2465 * @invoke:		If true, the startup function is invoked for cpus where
2466 *			cpu state >= @state
2467 * @startup:		startup callback function
2468 * @teardown:		teardown callback function
2469 * @multi_instance:	State is set up for multiple instances which get
2470 *			added afterwards.
2471 *
2472 * The caller needs to hold cpus read locked while calling this function.
2473 * Return:
2474 *   On success:
2475 *      Positive state number if @state is CPUHP_AP_ONLINE_DYN or CPUHP_BP_PREPARE_DYN;
2476 *      0 for all other states
2477 *   On failure: proper (negative) error code
2478 */
2479int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
2480				   const char *name, bool invoke,
2481				   int (*startup)(unsigned int cpu),
2482				   int (*teardown)(unsigned int cpu),
2483				   bool multi_instance)
2484{
2485	int cpu, ret = 0;
2486	bool dynstate;
2487
2488	lockdep_assert_cpus_held();
2489
2490	if (cpuhp_cb_check(state) || !name)
2491		return -EINVAL;
2492
2493	mutex_lock(&cpuhp_state_mutex);
2494
2495	ret = cpuhp_store_callbacks(state, name, startup, teardown,
2496				    multi_instance);
2497
2498	dynstate = state == CPUHP_AP_ONLINE_DYN || state == CPUHP_BP_PREPARE_DYN;
2499	if (ret > 0 && dynstate) {
2500		state = ret;
2501		ret = 0;
2502	}
2503
2504	if (ret || !invoke || !startup)
2505		goto out;
2506
2507	/*
2508	 * Try to call the startup callback for each present cpu
2509	 * depending on the hotplug state of the cpu.
2510	 */
2511	for_each_present_cpu(cpu) {
2512		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2513		int cpustate = st->state;
2514
2515		if (cpustate < state)
2516			continue;
2517
2518		ret = cpuhp_issue_call(cpu, state, true, NULL);
2519		if (ret) {
2520			if (teardown)
2521				cpuhp_rollback_install(cpu, state, NULL);
2522			cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2523			goto out;
2524		}
2525	}
2526out:
2527	mutex_unlock(&cpuhp_state_mutex);
2528	/*
2529	 * If the requested state is CPUHP_AP_ONLINE_DYN or CPUHP_BP_PREPARE_DYN,
2530	 * return the dynamically allocated state in case of success.
2531	 */
2532	if (!ret && dynstate)
2533		return state;
2534	return ret;
2535}
2536EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
2537
2538int __cpuhp_setup_state(enum cpuhp_state state,
2539			const char *name, bool invoke,
2540			int (*startup)(unsigned int cpu),
2541			int (*teardown)(unsigned int cpu),
2542			bool multi_instance)
2543{
2544	int ret;
2545
2546	cpus_read_lock();
2547	ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
2548					     teardown, multi_instance);
2549	cpus_read_unlock();
2550	return ret;
2551}
2552EXPORT_SYMBOL(__cpuhp_setup_state);
2553
2554int __cpuhp_state_remove_instance(enum cpuhp_state state,
2555				  struct hlist_node *node, bool invoke)
2556{
2557	struct cpuhp_step *sp = cpuhp_get_step(state);
2558	int cpu;
2559
2560	BUG_ON(cpuhp_cb_check(state));
2561
2562	if (!sp->multi_instance)
2563		return -EINVAL;
2564
2565	cpus_read_lock();
2566	mutex_lock(&cpuhp_state_mutex);
2567
2568	if (!invoke || !cpuhp_get_teardown_cb(state))
2569		goto remove;
2570	/*
2571	 * Call the teardown callback for each present cpu depending
2572	 * on the hotplug state of the cpu. This function is not
2573	 * allowed to fail currently!
2574	 */
2575	for_each_present_cpu(cpu) {
2576		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2577		int cpustate = st->state;
2578
2579		if (cpustate >= state)
2580			cpuhp_issue_call(cpu, state, false, node);
2581	}
2582
2583remove:
2584	hlist_del(node);
2585	mutex_unlock(&cpuhp_state_mutex);
2586	cpus_read_unlock();
2587
2588	return 0;
2589}
2590EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
2591
2592/**
2593 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2594 * @state:	The state to remove
2595 * @invoke:	If true, the teardown function is invoked for cpus where
2596 *		cpu state >= @state
2597 *
2598 * The caller needs to hold cpus read locked while calling this function.
2599 * The teardown callback is currently not allowed to fail. Think
2600 * about module removal!
2601 */
2602void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2603{
2604	struct cpuhp_step *sp = cpuhp_get_step(state);
2605	int cpu;
2606
2607	BUG_ON(cpuhp_cb_check(state));
2608
2609	lockdep_assert_cpus_held();
2610
2611	mutex_lock(&cpuhp_state_mutex);
2612	if (sp->multi_instance) {
2613		WARN(!hlist_empty(&sp->list),
2614		     "Error: Removing state %d which has instances left.\n",
2615		     state);
2616		goto remove;
2617	}
2618
2619	if (!invoke || !cpuhp_get_teardown_cb(state))
2620		goto remove;
2621
2622	/*
2623	 * Call the teardown callback for each present cpu depending
2624	 * on the hotplug state of the cpu. This function is not
2625	 * allowed to fail currently!
2626	 */
2627	for_each_present_cpu(cpu) {
2628		struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2629		int cpustate = st->state;
2630
2631		if (cpustate >= state)
2632			cpuhp_issue_call(cpu, state, false, NULL);
2633	}
2634remove:
2635	cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2636	mutex_unlock(&cpuhp_state_mutex);
2637}
2638EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2639
2640void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2641{
2642	cpus_read_lock();
2643	__cpuhp_remove_state_cpuslocked(state, invoke);
2644	cpus_read_unlock();
2645}
2646EXPORT_SYMBOL(__cpuhp_remove_state);
2647
2648#ifdef CONFIG_HOTPLUG_SMT
2649static void cpuhp_offline_cpu_device(unsigned int cpu)
2650{
2651	struct device *dev = get_cpu_device(cpu);
2652
2653	dev->offline = true;
2654	/* Tell user space about the state change */
2655	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2656}
2657
2658static void cpuhp_online_cpu_device(unsigned int cpu)
2659{
2660	struct device *dev = get_cpu_device(cpu);
2661
2662	dev->offline = false;
2663	/* Tell user space about the state change */
2664	kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2665}
2666
2667int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2668{
2669	int cpu, ret = 0;
2670
2671	cpu_maps_update_begin();
2672	for_each_online_cpu(cpu) {
2673		if (topology_is_primary_thread(cpu))
2674			continue;
2675		/*
2676		 * Disable can be called with CPU_SMT_ENABLED when changing
2677		 * from a higher to lower number of SMT threads per core.
2678		 */
2679		if (ctrlval == CPU_SMT_ENABLED && cpu_smt_thread_allowed(cpu))
2680			continue;
2681		ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2682		if (ret)
2683			break;
2684		/*
2685		 * As this needs to hold the cpu maps lock it's impossible
2686		 * to call device_offline() because that ends up calling
2687		 * cpu_down() which takes cpu maps lock. cpu maps lock
2688		 * needs to be held as this might race against in kernel
2689		 * abusers of the hotplug machinery (thermal management).
2690		 *
2691		 * So nothing would update device:offline state. That would
2692		 * leave the sysfs entry stale and prevent onlining after
2693		 * smt control has been changed to 'off' again. This is
2694		 * called under the sysfs hotplug lock, so it is properly
2695		 * serialized against the regular offline usage.
2696		 */
2697		cpuhp_offline_cpu_device(cpu);
2698	}
2699	if (!ret)
2700		cpu_smt_control = ctrlval;
2701	cpu_maps_update_done();
2702	return ret;
2703}
2704
2705/* Check if the core a CPU belongs to is online */
2706#if !defined(topology_is_core_online)
2707static inline bool topology_is_core_online(unsigned int cpu)
2708{
2709	return true;
2710}
2711#endif
2712
2713int cpuhp_smt_enable(void)
2714{
2715	int cpu, ret = 0;
2716
2717	cpu_maps_update_begin();
2718	cpu_smt_control = CPU_SMT_ENABLED;
2719	for_each_present_cpu(cpu) {
2720		/* Skip online CPUs and CPUs on offline nodes */
2721		if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2722			continue;
2723		if (!cpu_smt_thread_allowed(cpu) || !topology_is_core_online(cpu))
2724			continue;
2725		ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2726		if (ret)
2727			break;
2728		/* See comment in cpuhp_smt_disable() */
2729		cpuhp_online_cpu_device(cpu);
2730	}
2731	cpu_maps_update_done();
2732	return ret;
2733}
2734#endif
2735
2736#if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2737static ssize_t state_show(struct device *dev,
2738			  struct device_attribute *attr, char *buf)
2739{
2740	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2741
2742	return sprintf(buf, "%d\n", st->state);
2743}
2744static DEVICE_ATTR_RO(state);
2745
2746static ssize_t target_store(struct device *dev, struct device_attribute *attr,
2747			    const char *buf, size_t count)
2748{
2749	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2750	struct cpuhp_step *sp;
2751	int target, ret;
2752
2753	ret = kstrtoint(buf, 10, &target);
2754	if (ret)
2755		return ret;
2756
2757#ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2758	if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2759		return -EINVAL;
2760#else
2761	if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2762		return -EINVAL;
2763#endif
2764
2765	ret = lock_device_hotplug_sysfs();
2766	if (ret)
2767		return ret;
2768
2769	mutex_lock(&cpuhp_state_mutex);
2770	sp = cpuhp_get_step(target);
2771	ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2772	mutex_unlock(&cpuhp_state_mutex);
2773	if (ret)
2774		goto out;
2775
2776	if (st->state < target)
2777		ret = cpu_up(dev->id, target);
2778	else if (st->state > target)
2779		ret = cpu_down(dev->id, target);
2780	else if (WARN_ON(st->target != target))
2781		st->target = target;
2782out:
2783	unlock_device_hotplug();
2784	return ret ? ret : count;
2785}
2786
2787static ssize_t target_show(struct device *dev,
2788			   struct device_attribute *attr, char *buf)
2789{
2790	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2791
2792	return sprintf(buf, "%d\n", st->target);
2793}
2794static DEVICE_ATTR_RW(target);
2795
2796static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
2797			  const char *buf, size_t count)
2798{
2799	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2800	struct cpuhp_step *sp;
2801	int fail, ret;
2802
2803	ret = kstrtoint(buf, 10, &fail);
2804	if (ret)
2805		return ret;
2806
2807	if (fail == CPUHP_INVALID) {
2808		st->fail = fail;
2809		return count;
2810	}
2811
2812	if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2813		return -EINVAL;
2814
2815	/*
2816	 * Cannot fail STARTING/DYING callbacks.
2817	 */
2818	if (cpuhp_is_atomic_state(fail))
2819		return -EINVAL;
2820
2821	/*
2822	 * DEAD callbacks cannot fail...
2823	 * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
2824	 * triggering STARTING callbacks, a failure in this state would
2825	 * hinder rollback.
2826	 */
2827	if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
2828		return -EINVAL;
2829
2830	/*
2831	 * Cannot fail anything that doesn't have callbacks.
2832	 */
2833	mutex_lock(&cpuhp_state_mutex);
2834	sp = cpuhp_get_step(fail);
2835	if (!sp->startup.single && !sp->teardown.single)
2836		ret = -EINVAL;
2837	mutex_unlock(&cpuhp_state_mutex);
2838	if (ret)
2839		return ret;
2840
2841	st->fail = fail;
2842
2843	return count;
2844}
2845
2846static ssize_t fail_show(struct device *dev,
2847			 struct device_attribute *attr, char *buf)
2848{
2849	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2850
2851	return sprintf(buf, "%d\n", st->fail);
2852}
2853
2854static DEVICE_ATTR_RW(fail);
2855
2856static struct attribute *cpuhp_cpu_attrs[] = {
2857	&dev_attr_state.attr,
2858	&dev_attr_target.attr,
2859	&dev_attr_fail.attr,
2860	NULL
2861};
2862
2863static const struct attribute_group cpuhp_cpu_attr_group = {
2864	.attrs = cpuhp_cpu_attrs,
2865	.name = "hotplug",
2866};
2867
2868static ssize_t states_show(struct device *dev,
2869				 struct device_attribute *attr, char *buf)
2870{
2871	ssize_t cur, res = 0;
2872	int i;
2873
2874	mutex_lock(&cpuhp_state_mutex);
2875	for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2876		struct cpuhp_step *sp = cpuhp_get_step(i);
2877
2878		if (sp->name) {
2879			cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2880			buf += cur;
2881			res += cur;
2882		}
2883	}
2884	mutex_unlock(&cpuhp_state_mutex);
2885	return res;
2886}
2887static DEVICE_ATTR_RO(states);
2888
2889static struct attribute *cpuhp_cpu_root_attrs[] = {
2890	&dev_attr_states.attr,
2891	NULL
2892};
2893
2894static const struct attribute_group cpuhp_cpu_root_attr_group = {
2895	.attrs = cpuhp_cpu_root_attrs,
2896	.name = "hotplug",
2897};
2898
2899#ifdef CONFIG_HOTPLUG_SMT
2900
2901static bool cpu_smt_num_threads_valid(unsigned int threads)
2902{
2903	if (IS_ENABLED(CONFIG_SMT_NUM_THREADS_DYNAMIC))
2904		return threads >= 1 && threads <= cpu_smt_max_threads;
2905	return threads == 1 || threads == cpu_smt_max_threads;
2906}
2907
2908static ssize_t
2909__store_smt_control(struct device *dev, struct device_attribute *attr,
2910		    const char *buf, size_t count)
2911{
2912	int ctrlval, ret, num_threads, orig_threads;
2913	bool force_off;
2914
2915	if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2916		return -EPERM;
2917
2918	if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2919		return -ENODEV;
2920
2921	if (sysfs_streq(buf, "on")) {
2922		ctrlval = CPU_SMT_ENABLED;
2923		num_threads = cpu_smt_max_threads;
2924	} else if (sysfs_streq(buf, "off")) {
2925		ctrlval = CPU_SMT_DISABLED;
2926		num_threads = 1;
2927	} else if (sysfs_streq(buf, "forceoff")) {
2928		ctrlval = CPU_SMT_FORCE_DISABLED;
2929		num_threads = 1;
2930	} else if (kstrtoint(buf, 10, &num_threads) == 0) {
2931		if (num_threads == 1)
2932			ctrlval = CPU_SMT_DISABLED;
2933		else if (cpu_smt_num_threads_valid(num_threads))
2934			ctrlval = CPU_SMT_ENABLED;
2935		else
2936			return -EINVAL;
2937	} else {
2938		return -EINVAL;
2939	}
2940
2941	ret = lock_device_hotplug_sysfs();
2942	if (ret)
2943		return ret;
2944
2945	orig_threads = cpu_smt_num_threads;
2946	cpu_smt_num_threads = num_threads;
2947
2948	force_off = ctrlval != cpu_smt_control && ctrlval == CPU_SMT_FORCE_DISABLED;
2949
2950	if (num_threads > orig_threads)
2951		ret = cpuhp_smt_enable();
2952	else if (num_threads < orig_threads || force_off)
2953		ret = cpuhp_smt_disable(ctrlval);
2954
2955	unlock_device_hotplug();
2956	return ret ? ret : count;
2957}
2958
2959#else /* !CONFIG_HOTPLUG_SMT */
2960static ssize_t
2961__store_smt_control(struct device *dev, struct device_attribute *attr,
2962		    const char *buf, size_t count)
2963{
2964	return -ENODEV;
2965}
2966#endif /* CONFIG_HOTPLUG_SMT */
2967
2968static const char *smt_states[] = {
2969	[CPU_SMT_ENABLED]		= "on",
2970	[CPU_SMT_DISABLED]		= "off",
2971	[CPU_SMT_FORCE_DISABLED]	= "forceoff",
2972	[CPU_SMT_NOT_SUPPORTED]		= "notsupported",
2973	[CPU_SMT_NOT_IMPLEMENTED]	= "notimplemented",
2974};
2975
2976static ssize_t control_show(struct device *dev,
2977			    struct device_attribute *attr, char *buf)
2978{
2979	const char *state = smt_states[cpu_smt_control];
2980
2981#ifdef CONFIG_HOTPLUG_SMT
2982	/*
2983	 * If SMT is enabled but not all threads are enabled then show the
2984	 * number of threads. If all threads are enabled show "on". Otherwise
2985	 * show the state name.
2986	 */
2987	if (cpu_smt_control == CPU_SMT_ENABLED &&
2988	    cpu_smt_num_threads != cpu_smt_max_threads)
2989		return sysfs_emit(buf, "%d\n", cpu_smt_num_threads);
2990#endif
2991
2992	return sysfs_emit(buf, "%s\n", state);
2993}
2994
2995static ssize_t control_store(struct device *dev, struct device_attribute *attr,
2996			     const char *buf, size_t count)
2997{
2998	return __store_smt_control(dev, attr, buf, count);
2999}
3000static DEVICE_ATTR_RW(control);
3001
3002static ssize_t active_show(struct device *dev,
3003			   struct device_attribute *attr, char *buf)
3004{
3005	return sysfs_emit(buf, "%d\n", sched_smt_active());
3006}
3007static DEVICE_ATTR_RO(active);
3008
3009static struct attribute *cpuhp_smt_attrs[] = {
3010	&dev_attr_control.attr,
3011	&dev_attr_active.attr,
3012	NULL
3013};
3014
3015static const struct attribute_group cpuhp_smt_attr_group = {
3016	.attrs = cpuhp_smt_attrs,
3017	.name = "smt",
3018};
3019
3020static int __init cpu_smt_sysfs_init(void)
3021{
3022	struct device *dev_root;
3023	int ret = -ENODEV;
3024
3025	dev_root = bus_get_dev_root(&cpu_subsys);
3026	if (dev_root) {
3027		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group);
3028		put_device(dev_root);
3029	}
3030	return ret;
3031}
3032
3033static int __init cpuhp_sysfs_init(void)
3034{
3035	struct device *dev_root;
3036	int cpu, ret;
3037
3038	ret = cpu_smt_sysfs_init();
3039	if (ret)
3040		return ret;
3041
3042	dev_root = bus_get_dev_root(&cpu_subsys);
3043	if (dev_root) {
3044		ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group);
3045		put_device(dev_root);
3046		if (ret)
3047			return ret;
3048	}
3049
3050	for_each_possible_cpu(cpu) {
3051		struct device *dev = get_cpu_device(cpu);
3052
3053		if (!dev)
3054			continue;
3055		ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
3056		if (ret)
3057			return ret;
3058	}
3059	return 0;
3060}
3061device_initcall(cpuhp_sysfs_init);
3062#endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
3063
3064/*
3065 * cpu_bit_bitmap[] is a special, "compressed" data structure that
3066 * represents all NR_CPUS bits binary values of 1<<nr.
3067 *
3068 * It is used by cpumask_of() to get a constant address to a CPU
3069 * mask value that has a single bit set only.
3070 */
3071
3072/* cpu_bit_bitmap[0] is empty - so we can back into it */
3073#define MASK_DECLARE_1(x)	[x+1][0] = (1UL << (x))
3074#define MASK_DECLARE_2(x)	MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
3075#define MASK_DECLARE_4(x)	MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
3076#define MASK_DECLARE_8(x)	MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
3077
3078const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
3079
3080	MASK_DECLARE_8(0),	MASK_DECLARE_8(8),
3081	MASK_DECLARE_8(16),	MASK_DECLARE_8(24),
3082#if BITS_PER_LONG > 32
3083	MASK_DECLARE_8(32),	MASK_DECLARE_8(40),
3084	MASK_DECLARE_8(48),	MASK_DECLARE_8(56),
3085#endif
3086};
3087EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
3088
3089const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
3090EXPORT_SYMBOL(cpu_all_bits);
3091
3092#ifdef CONFIG_INIT_ALL_POSSIBLE
3093struct cpumask __cpu_possible_mask __ro_after_init
3094	= {CPU_BITS_ALL};
3095unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
3096#else
3097struct cpumask __cpu_possible_mask __ro_after_init;
3098unsigned int __num_possible_cpus __ro_after_init;
3099#endif
3100EXPORT_SYMBOL(__cpu_possible_mask);
3101EXPORT_SYMBOL(__num_possible_cpus);
3102
3103struct cpumask __cpu_online_mask __read_mostly;
3104EXPORT_SYMBOL(__cpu_online_mask);
3105
3106struct cpumask __cpu_enabled_mask __read_mostly;
3107EXPORT_SYMBOL(__cpu_enabled_mask);
3108
3109struct cpumask __cpu_present_mask __read_mostly;
3110EXPORT_SYMBOL(__cpu_present_mask);
3111
3112struct cpumask __cpu_active_mask __read_mostly;
3113EXPORT_SYMBOL(__cpu_active_mask);
3114
3115struct cpumask __cpu_dying_mask __read_mostly;
3116EXPORT_SYMBOL(__cpu_dying_mask);
3117
3118atomic_t __num_online_cpus __read_mostly;
3119EXPORT_SYMBOL(__num_online_cpus);
3120
3121void init_cpu_present(const struct cpumask *src)
3122{
3123	cpumask_copy(&__cpu_present_mask, src);
3124}
3125
3126void init_cpu_possible(const struct cpumask *src)
3127{
3128	cpumask_copy(&__cpu_possible_mask, src);
3129	__num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
3130}
3131
3132void set_cpu_online(unsigned int cpu, bool online)
3133{
3134	/*
3135	 * atomic_inc/dec() is required to handle the horrid abuse of this
3136	 * function by the reboot and kexec code which invoke it from
3137	 * IPI/NMI broadcasts when shutting down CPUs. Invocation from
3138	 * regular CPU hotplug is properly serialized.
3139	 *
3140	 * Note, that the fact that __num_online_cpus is of type atomic_t
3141	 * does not protect readers which are not serialized against
3142	 * concurrent hotplug operations.
3143	 */
3144	if (online) {
3145		if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
3146			atomic_inc(&__num_online_cpus);
3147	} else {
3148		if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
3149			atomic_dec(&__num_online_cpus);
3150	}
3151}
3152
3153/*
3154 * This should be marked __init, but there is a boatload of call sites
3155 * which need to be fixed up to do so. Sigh...
3156 */
3157void set_cpu_possible(unsigned int cpu, bool possible)
3158{
3159	if (possible) {
3160		if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
3161			__num_possible_cpus++;
3162	} else {
3163		if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
3164			__num_possible_cpus--;
3165	}
3166}
3167
3168/*
3169 * Activate the first processor.
3170 */
3171void __init boot_cpu_init(void)
3172{
3173	int cpu = smp_processor_id();
3174
3175	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
3176	set_cpu_online(cpu, true);
3177	set_cpu_active(cpu, true);
3178	set_cpu_present(cpu, true);
3179	set_cpu_possible(cpu, true);
3180
3181#ifdef CONFIG_SMP
3182	__boot_cpu_id = cpu;
3183#endif
3184}
3185
3186/*
3187 * Must be called _AFTER_ setting up the per_cpu areas
3188 */
3189void __init boot_cpu_hotplug_init(void)
3190{
3191#ifdef CONFIG_SMP
3192	cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
3193	atomic_set(this_cpu_ptr(&cpuhp_state.ap_sync_state), SYNC_STATE_ONLINE);
3194#endif
3195	this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
3196	this_cpu_write(cpuhp_state.target, CPUHP_ONLINE);
3197}
3198
3199#ifdef CONFIG_CPU_MITIGATIONS
3200/*
3201 * All except the cross-thread attack vector are mitigated by default.
3202 * Cross-thread mitigation often requires disabling SMT which is expensive
3203 * so cross-thread mitigations are only partially enabled by default.
3204 *
3205 * Guest-to-Host and Guest-to-Guest vectors are only needed if KVM support is
3206 * present.
3207 */
3208static bool attack_vectors[NR_CPU_ATTACK_VECTORS] __ro_after_init = {
3209	[CPU_MITIGATE_USER_KERNEL] = true,
3210	[CPU_MITIGATE_USER_USER] = true,
3211	[CPU_MITIGATE_GUEST_HOST] = IS_ENABLED(CONFIG_KVM),
3212	[CPU_MITIGATE_GUEST_GUEST] = IS_ENABLED(CONFIG_KVM),
3213};
3214
3215bool cpu_attack_vector_mitigated(enum cpu_attack_vectors v)
3216{
3217	if (v < NR_CPU_ATTACK_VECTORS)
3218		return attack_vectors[v];
3219
3220	WARN_ONCE(1, "Invalid attack vector %d\n", v);
3221	return false;
3222}
3223
3224/*
3225 * There are 3 global options, 'off', 'auto', 'auto,nosmt'. These may optionally
3226 * be combined with attack-vector disables which follow them.
3227 *
3228 * Examples:
3229 *   mitigations=auto,no_user_kernel,no_user_user,no_cross_thread
3230 *   mitigations=auto,nosmt,no_guest_host,no_guest_guest
3231 *
3232 * mitigations=off is equivalent to disabling all attack vectors.
3233 */
3234enum cpu_mitigations {
3235	CPU_MITIGATIONS_OFF,
3236	CPU_MITIGATIONS_AUTO,
3237	CPU_MITIGATIONS_AUTO_NOSMT,
3238};
3239
3240enum {
3241	NO_USER_KERNEL,
3242	NO_USER_USER,
3243	NO_GUEST_HOST,
3244	NO_GUEST_GUEST,
3245	NO_CROSS_THREAD,
3246	NR_VECTOR_PARAMS,
3247};
3248
3249enum smt_mitigations smt_mitigations __ro_after_init = SMT_MITIGATIONS_AUTO;
3250static enum cpu_mitigations cpu_mitigations __ro_after_init = CPU_MITIGATIONS_AUTO;
3251
3252static const match_table_t global_mitigations = {
3253	{ CPU_MITIGATIONS_AUTO_NOSMT,	"auto,nosmt"},
3254	{ CPU_MITIGATIONS_AUTO,		"auto"},
3255	{ CPU_MITIGATIONS_OFF,		"off"},
3256};
3257
3258static const match_table_t vector_mitigations = {
3259	{ NO_USER_KERNEL,	"no_user_kernel"},
3260	{ NO_USER_USER,		"no_user_user"},
3261	{ NO_GUEST_HOST,	"no_guest_host"},
3262	{ NO_GUEST_GUEST,	"no_guest_guest"},
3263	{ NO_CROSS_THREAD,	"no_cross_thread"},
3264	{ NR_VECTOR_PARAMS,	NULL},
3265};
3266
3267static int __init mitigations_parse_global_opt(char *arg)
3268{
3269	int i;
3270
3271	for (i = 0; i < ARRAY_SIZE(global_mitigations); i++) {
3272		const char *pattern = global_mitigations[i].pattern;
3273
3274		if (!strncmp(arg, pattern, strlen(pattern))) {
3275			cpu_mitigations = global_mitigations[i].token;
3276			return strlen(pattern);
3277		}
3278	}
3279
3280	return 0;
3281}
3282
3283static int __init mitigations_parse_cmdline(char *arg)
3284{
3285	char *s, *p;
3286	int len;
3287
3288	len = mitigations_parse_global_opt(arg);
3289
3290	if (cpu_mitigations_off()) {
3291		memset(attack_vectors, 0, sizeof(attack_vectors));
3292		smt_mitigations = SMT_MITIGATIONS_OFF;
3293	} else if (cpu_mitigations_auto_nosmt()) {
3294		smt_mitigations = SMT_MITIGATIONS_ON;
3295	}
3296
3297	p = arg + len;
3298
3299	if (!*p)
3300		return 0;
3301
3302	/* Attack vector controls may come after the ',' */
3303	if (*p++ != ',' || !IS_ENABLED(CONFIG_ARCH_HAS_CPU_ATTACK_VECTORS)) {
3304		pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",	arg);
3305		return 0;
3306	}
3307
3308	while ((s = strsep(&p, ",")) != NULL) {
3309		switch (match_token(s, vector_mitigations, NULL)) {
3310		case NO_USER_KERNEL:
3311			attack_vectors[CPU_MITIGATE_USER_KERNEL] = false;
3312			break;
3313		case NO_USER_USER:
3314			attack_vectors[CPU_MITIGATE_USER_USER] = false;
3315			break;
3316		case NO_GUEST_HOST:
3317			attack_vectors[CPU_MITIGATE_GUEST_HOST] = false;
3318			break;
3319		case NO_GUEST_GUEST:
3320			attack_vectors[CPU_MITIGATE_GUEST_GUEST] = false;
3321			break;
3322		case NO_CROSS_THREAD:
3323			smt_mitigations = SMT_MITIGATIONS_OFF;
3324			break;
3325		default:
3326			pr_crit("Unsupported mitigations options %s\n",	s);
3327			return 0;
3328		}
3329	}
3330
3331	return 0;
3332}
3333
3334/* mitigations=off */
3335bool cpu_mitigations_off(void)
3336{
3337	return cpu_mitigations == CPU_MITIGATIONS_OFF;
3338}
3339EXPORT_SYMBOL_GPL(cpu_mitigations_off);
3340
3341/* mitigations=auto,nosmt */
3342bool cpu_mitigations_auto_nosmt(void)
3343{
3344	return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
3345}
3346EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
3347#else
3348static int __init mitigations_parse_cmdline(char *arg)
3349{
3350	pr_crit("Kernel compiled without mitigations, ignoring 'mitigations'; system may still be vulnerable\n");
3351	return 0;
3352}
3353#endif
3354early_param("mitigations", mitigations_parse_cmdline);