FILE
smp.c
File preview with syntax highlighting.
linux-6.19.14/arch/powerpc/kernel/smp.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * SMP support for ppc. 4 * 5 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great 6 * deal of code from the sparc and intel versions. 7 * 8 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> 9 * 10 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and 11 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com 12 */ 13 14#undef DEBUG 15 16#include <linux/kernel.h> 17#include <linux/export.h> 18#include <linux/sched/mm.h> 19#include <linux/sched/task_stack.h> 20#include <linux/sched/topology.h> 21#include <linux/smp.h> 22#include <linux/interrupt.h> 23#include <linux/delay.h> 24#include <linux/init.h> 25#include <linux/spinlock.h> 26#include <linux/cache.h> 27#include <linux/err.h> 28#include <linux/device.h> 29#include <linux/cpu.h> 30#include <linux/notifier.h> 31#include <linux/topology.h> 32#include <linux/profile.h> 33#include <linux/processor.h> 34#include <linux/random.h> 35#include <linux/stackprotector.h> 36#include <linux/pgtable.h> 37#include <linux/clockchips.h> 38#include <linux/kexec.h> 39 40#include <asm/ptrace.h> 41#include <linux/atomic.h> 42#include <asm/irq.h> 43#include <asm/hw_irq.h> 44#include <asm/kvm_ppc.h> 45#include <asm/dbell.h> 46#include <asm/page.h> 47#include <asm/smp.h> 48#include <asm/time.h> 49#include <asm/machdep.h> 50#include <asm/mmu_context.h> 51#include <asm/cputhreads.h> 52#include <asm/cputable.h> 53#include <asm/mpic.h> 54#include <asm/vdso_datapage.h> 55#ifdef CONFIG_PPC64 56#include <asm/paca.h> 57#endif 58#include <asm/vdso.h> 59#include <asm/debug.h> 60#include <asm/cpu_has_feature.h> 61#include <asm/ftrace.h> 62#include <asm/kup.h> 63#include <asm/fadump.h> 64#include <asm/systemcfg.h> 65 66#include <trace/events/ipi.h> 67 68#ifdef DEBUG 69#include <asm/udbg.h> 70#define DBG(fmt...) udbg_printf(fmt) 71#else 72#define DBG(fmt...) 73#endif 74 75#ifdef CONFIG_HOTPLUG_CPU 76/* State of each CPU during hotplug phases */ 77static DEFINE_PER_CPU(int, cpu_state) = { 0 }; 78#endif 79 80struct task_struct *secondary_current; 81bool has_big_cores __ro_after_init; 82bool coregroup_enabled __ro_after_init; 83bool thread_group_shares_l2 __ro_after_init; 84bool thread_group_shares_l3 __ro_after_init; 85 86DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map); 87DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map); 88DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map); 89DEFINE_PER_CPU(cpumask_var_t, cpu_core_map); 90static DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map); 91 92EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); 93EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map); 94EXPORT_PER_CPU_SYMBOL(cpu_core_map); 95EXPORT_SYMBOL_GPL(has_big_cores); 96 97#define MAX_THREAD_LIST_SIZE 8 98#define THREAD_GROUP_SHARE_L1 1 99#define THREAD_GROUP_SHARE_L2_L3 2 100struct thread_groups { 101 unsigned int property; 102 unsigned int nr_groups; 103 unsigned int threads_per_group; 104 unsigned int thread_list[MAX_THREAD_LIST_SIZE]; 105}; 106 107/* Maximum number of properties that groups of threads within a core can share */ 108#define MAX_THREAD_GROUP_PROPERTIES 2 109 110struct thread_groups_list { 111 unsigned int nr_properties; 112 struct thread_groups property_tgs[MAX_THREAD_GROUP_PROPERTIES]; 113}; 114 115static struct thread_groups_list tgl[NR_CPUS] __initdata; 116/* 117 * On big-cores system, thread_group_l1_cache_map for each CPU corresponds to 118 * the set its siblings that share the L1-cache. 119 */ 120DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map); 121 122/* 123 * On some big-cores system, thread_group_l2_cache_map for each CPU 124 * corresponds to the set its siblings within the core that share the 125 * L2-cache. 126 */ 127DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map); 128 129/* 130 * On P10, thread_group_l3_cache_map for each CPU is equal to the 131 * thread_group_l2_cache_map 132 */ 133DEFINE_PER_CPU(cpumask_var_t, thread_group_l3_cache_map); 134 135/* SMP operations for this machine */ 136struct smp_ops_t *smp_ops; 137 138/* Can't be static due to PowerMac hackery */ 139volatile unsigned int cpu_callin_map[NR_CPUS]; 140 141int smt_enabled_at_boot = 1; 142 143/* 144 * Returns 1 if the specified cpu should be brought up during boot. 145 * Used to inhibit booting threads if they've been disabled or 146 * limited on the command line 147 */ 148int smp_generic_cpu_bootable(unsigned int nr) 149{ 150 /* Special case - we inhibit secondary thread startup 151 * during boot if the user requests it. 152 */ 153 if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) { 154 if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0) 155 return 0; 156 if (smt_enabled_at_boot 157 && cpu_thread_in_core(nr) >= smt_enabled_at_boot) 158 return 0; 159 } 160 161 return 1; 162} 163 164 165#ifdef CONFIG_PPC64 166int smp_generic_kick_cpu(int nr) 167{ 168 if (nr < 0 || nr >= nr_cpu_ids) 169 return -EINVAL; 170 171 /* 172 * The processor is currently spinning, waiting for the 173 * cpu_start field to become non-zero After we set cpu_start, 174 * the processor will continue on to secondary_start 175 */ 176 if (!paca_ptrs[nr]->cpu_start) { 177 paca_ptrs[nr]->cpu_start = 1; 178 smp_mb(); 179 return 0; 180 } 181 182#ifdef CONFIG_HOTPLUG_CPU 183 /* 184 * Ok it's not there, so it might be soft-unplugged, let's 185 * try to bring it back 186 */ 187 generic_set_cpu_up(nr); 188 smp_wmb(); 189 smp_send_reschedule(nr); 190#endif /* CONFIG_HOTPLUG_CPU */ 191 192 return 0; 193} 194#endif /* CONFIG_PPC64 */ 195 196static irqreturn_t call_function_action(int irq, void *data) 197{ 198 generic_smp_call_function_interrupt(); 199 return IRQ_HANDLED; 200} 201 202static irqreturn_t reschedule_action(int irq, void *data) 203{ 204 scheduler_ipi(); 205 return IRQ_HANDLED; 206} 207 208#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 209static irqreturn_t tick_broadcast_ipi_action(int irq, void *data) 210{ 211 timer_broadcast_interrupt(); 212 return IRQ_HANDLED; 213} 214#endif 215 216#ifdef CONFIG_NMI_IPI 217static irqreturn_t nmi_ipi_action(int irq, void *data) 218{ 219 smp_handle_nmi_ipi(get_irq_regs()); 220 return IRQ_HANDLED; 221} 222#endif 223 224static irq_handler_t smp_ipi_action[] = { 225 [PPC_MSG_CALL_FUNCTION] = call_function_action, 226 [PPC_MSG_RESCHEDULE] = reschedule_action, 227#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 228 [PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action, 229#endif 230#ifdef CONFIG_NMI_IPI 231 [PPC_MSG_NMI_IPI] = nmi_ipi_action, 232#endif 233}; 234 235/* 236 * The NMI IPI is a fallback and not truly non-maskable. It is simpler 237 * than going through the call function infrastructure, and strongly 238 * serialized, so it is more appropriate for debugging. 239 */ 240const char *smp_ipi_name[] = { 241 [PPC_MSG_CALL_FUNCTION] = "ipi call function", 242 [PPC_MSG_RESCHEDULE] = "ipi reschedule", 243#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 244 [PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast", 245#endif 246#ifdef CONFIG_NMI_IPI 247 [PPC_MSG_NMI_IPI] = "nmi ipi", 248#endif 249}; 250 251/* optional function to request ipi, for controllers with >= 4 ipis */ 252int smp_request_message_ipi(int virq, int msg) 253{ 254 int err; 255 256 if (msg < 0 || msg > PPC_MSG_NMI_IPI) 257 return -EINVAL; 258#ifndef CONFIG_NMI_IPI 259 if (msg == PPC_MSG_NMI_IPI) 260 return 1; 261#endif 262 263 err = request_irq(virq, smp_ipi_action[msg], 264 IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND, 265 smp_ipi_name[msg], NULL); 266 WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n", 267 virq, smp_ipi_name[msg], err); 268 269 return err; 270} 271 272#ifdef CONFIG_PPC_SMP_MUXED_IPI 273struct cpu_messages { 274 long messages; /* current messages */ 275}; 276static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message); 277 278void smp_muxed_ipi_set_message(int cpu, int msg) 279{ 280 struct cpu_messages *info = &per_cpu(ipi_message, cpu); 281 char *message = (char *)&info->messages; 282 283 /* 284 * Order previous accesses before accesses in the IPI handler. 285 */ 286 smp_mb(); 287 WRITE_ONCE(message[msg], 1); 288} 289 290void smp_muxed_ipi_message_pass(int cpu, int msg) 291{ 292 smp_muxed_ipi_set_message(cpu, msg); 293 294 /* 295 * cause_ipi functions are required to include a full barrier 296 * before doing whatever causes the IPI. 297 */ 298 smp_ops->cause_ipi(cpu); 299} 300 301#ifdef __BIG_ENDIAN__ 302#define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A))) 303#else 304#define IPI_MESSAGE(A) (1uL << (8 * (A))) 305#endif 306 307irqreturn_t smp_ipi_demux(void) 308{ 309 mb(); /* order any irq clear */ 310 311 return smp_ipi_demux_relaxed(); 312} 313 314/* sync-free variant. Callers should ensure synchronization */ 315irqreturn_t smp_ipi_demux_relaxed(void) 316{ 317 struct cpu_messages *info; 318 unsigned long all; 319 320 info = this_cpu_ptr(&ipi_message); 321 do { 322 all = xchg(&info->messages, 0); 323#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) 324 /* 325 * Must check for PPC_MSG_RM_HOST_ACTION messages 326 * before PPC_MSG_CALL_FUNCTION messages because when 327 * a VM is destroyed, we call kick_all_cpus_sync() 328 * to ensure that any pending PPC_MSG_RM_HOST_ACTION 329 * messages have completed before we free any VCPUs. 330 */ 331 if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION)) 332 kvmppc_xics_ipi_action(); 333#endif 334 if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION)) 335 generic_smp_call_function_interrupt(); 336 if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE)) 337 scheduler_ipi(); 338#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 339 if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST)) 340 timer_broadcast_interrupt(); 341#endif 342#ifdef CONFIG_NMI_IPI 343 if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI)) 344 nmi_ipi_action(0, NULL); 345#endif 346 } while (READ_ONCE(info->messages)); 347 348 return IRQ_HANDLED; 349} 350#endif /* CONFIG_PPC_SMP_MUXED_IPI */ 351 352static inline void do_message_pass(int cpu, int msg) 353{ 354 if (smp_ops->message_pass) 355 smp_ops->message_pass(cpu, msg); 356#ifdef CONFIG_PPC_SMP_MUXED_IPI 357 else 358 smp_muxed_ipi_message_pass(cpu, msg); 359#endif 360} 361 362void arch_smp_send_reschedule(int cpu) 363{ 364 if (likely(smp_ops)) 365 do_message_pass(cpu, PPC_MSG_RESCHEDULE); 366} 367EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); 368 369void arch_send_call_function_single_ipi(int cpu) 370{ 371 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION); 372} 373 374void arch_send_call_function_ipi_mask(const struct cpumask *mask) 375{ 376 unsigned int cpu; 377 378 for_each_cpu(cpu, mask) 379 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION); 380} 381 382#ifdef CONFIG_NMI_IPI 383 384/* 385 * "NMI IPI" system. 386 * 387 * NMI IPIs may not be recoverable, so should not be used as ongoing part of 388 * a running system. They can be used for crash, debug, halt/reboot, etc. 389 * 390 * The IPI call waits with interrupts disabled until all targets enter the 391 * NMI handler, then returns. Subsequent IPIs can be issued before targets 392 * have returned from their handlers, so there is no guarantee about 393 * concurrency or re-entrancy. 394 * 395 * A new NMI can be issued before all targets exit the handler. 396 * 397 * The IPI call may time out without all targets entering the NMI handler. 398 * In that case, there is some logic to recover (and ignore subsequent 399 * NMI interrupts that may eventually be raised), but the platform interrupt 400 * handler may not be able to distinguish this from other exception causes, 401 * which may cause a crash. 402 */ 403 404static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0); 405static struct cpumask nmi_ipi_pending_mask; 406static bool nmi_ipi_busy = false; 407static void (*nmi_ipi_function)(struct pt_regs *) = NULL; 408 409noinstr static void nmi_ipi_lock_start(unsigned long *flags) 410{ 411 raw_local_irq_save(*flags); 412 hard_irq_disable(); 413 while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) { 414 raw_local_irq_restore(*flags); 415 spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0); 416 raw_local_irq_save(*flags); 417 hard_irq_disable(); 418 } 419} 420 421noinstr static void nmi_ipi_lock(void) 422{ 423 while (raw_atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) 424 spin_until_cond(raw_atomic_read(&__nmi_ipi_lock) == 0); 425} 426 427noinstr static void nmi_ipi_unlock(void) 428{ 429 smp_mb(); 430 WARN_ON(raw_atomic_read(&__nmi_ipi_lock) != 1); 431 raw_atomic_set(&__nmi_ipi_lock, 0); 432} 433 434noinstr static void nmi_ipi_unlock_end(unsigned long *flags) 435{ 436 nmi_ipi_unlock(); 437 raw_local_irq_restore(*flags); 438} 439 440/* 441 * Platform NMI handler calls this to ack 442 */ 443noinstr int smp_handle_nmi_ipi(struct pt_regs *regs) 444{ 445 void (*fn)(struct pt_regs *) = NULL; 446 unsigned long flags; 447 int me = raw_smp_processor_id(); 448 int ret = 0; 449 450 /* 451 * Unexpected NMIs are possible here because the interrupt may not 452 * be able to distinguish NMI IPIs from other types of NMIs, or 453 * because the caller may have timed out. 454 */ 455 nmi_ipi_lock_start(&flags); 456 if (cpumask_test_cpu(me, &nmi_ipi_pending_mask)) { 457 cpumask_clear_cpu(me, &nmi_ipi_pending_mask); 458 fn = READ_ONCE(nmi_ipi_function); 459 WARN_ON_ONCE(!fn); 460 ret = 1; 461 } 462 nmi_ipi_unlock_end(&flags); 463 464 if (fn) 465 fn(regs); 466 467 return ret; 468} 469 470static void do_smp_send_nmi_ipi(int cpu, bool safe) 471{ 472 if (!safe && smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu)) 473 return; 474 475 if (cpu >= 0) { 476 do_message_pass(cpu, PPC_MSG_NMI_IPI); 477 } else { 478 int c; 479 480 for_each_online_cpu(c) { 481 if (c == raw_smp_processor_id()) 482 continue; 483 do_message_pass(c, PPC_MSG_NMI_IPI); 484 } 485 } 486} 487 488/* 489 * - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS. 490 * - fn is the target callback function. 491 * - delay_us > 0 is the delay before giving up waiting for targets to 492 * begin executing the handler, == 0 specifies indefinite delay. 493 */ 494static int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), 495 u64 delay_us, bool safe) 496{ 497 unsigned long flags; 498 int me = raw_smp_processor_id(); 499 int ret = 1; 500 501 BUG_ON(cpu == me); 502 BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS); 503 504 if (unlikely(!smp_ops)) 505 return 0; 506 507 nmi_ipi_lock_start(&flags); 508 while (nmi_ipi_busy) { 509 nmi_ipi_unlock_end(&flags); 510 spin_until_cond(!nmi_ipi_busy); 511 nmi_ipi_lock_start(&flags); 512 } 513 nmi_ipi_busy = true; 514 nmi_ipi_function = fn; 515 516 WARN_ON_ONCE(!cpumask_empty(&nmi_ipi_pending_mask)); 517 518 if (cpu < 0) { 519 /* ALL_OTHERS */ 520 cpumask_copy(&nmi_ipi_pending_mask, cpu_online_mask); 521 cpumask_clear_cpu(me, &nmi_ipi_pending_mask); 522 } else { 523 cpumask_set_cpu(cpu, &nmi_ipi_pending_mask); 524 } 525 526 nmi_ipi_unlock(); 527 528 /* Interrupts remain hard disabled */ 529 530 do_smp_send_nmi_ipi(cpu, safe); 531 532 nmi_ipi_lock(); 533 /* nmi_ipi_busy is set here, so unlock/lock is okay */ 534 while (!cpumask_empty(&nmi_ipi_pending_mask)) { 535 nmi_ipi_unlock(); 536 udelay(1); 537 nmi_ipi_lock(); 538 if (delay_us) { 539 delay_us--; 540 if (!delay_us) 541 break; 542 } 543 } 544 545 if (!cpumask_empty(&nmi_ipi_pending_mask)) { 546 /* Timeout waiting for CPUs to call smp_handle_nmi_ipi */ 547 ret = 0; 548 cpumask_clear(&nmi_ipi_pending_mask); 549 } 550 551 nmi_ipi_function = NULL; 552 nmi_ipi_busy = false; 553 554 nmi_ipi_unlock_end(&flags); 555 556 return ret; 557} 558 559int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us) 560{ 561 return __smp_send_nmi_ipi(cpu, fn, delay_us, false); 562} 563 564int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us) 565{ 566 return __smp_send_nmi_ipi(cpu, fn, delay_us, true); 567} 568#endif /* CONFIG_NMI_IPI */ 569 570#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 571void tick_broadcast(const struct cpumask *mask) 572{ 573 unsigned int cpu; 574 575 for_each_cpu(cpu, mask) 576 do_message_pass(cpu, PPC_MSG_TICK_BROADCAST); 577} 578#endif 579 580#ifdef CONFIG_DEBUGGER 581static void debugger_ipi_callback(struct pt_regs *regs) 582{ 583 debugger_ipi(regs); 584} 585 586void smp_send_debugger_break(void) 587{ 588 smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, debugger_ipi_callback, 1000000); 589} 590#endif 591 592#ifdef CONFIG_CRASH_DUMP 593void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) 594{ 595 int cpu; 596 597 smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000); 598 if (kdump_in_progress() && crash_wake_offline) { 599 for_each_present_cpu(cpu) { 600 if (cpu_online(cpu)) 601 continue; 602 /* 603 * crash_ipi_callback will wait for 604 * all cpus, including offline CPUs. 605 * We don't care about nmi_ipi_function. 606 * Offline cpus will jump straight into 607 * crash_ipi_callback, we can skip the 608 * entire NMI dance and waiting for 609 * cpus to clear pending mask, etc. 610 */ 611 do_smp_send_nmi_ipi(cpu, false); 612 } 613 } 614} 615#endif 616 617void crash_smp_send_stop(void) 618{ 619 static bool stopped = false; 620 621 /* 622 * In case of fadump, register data for all CPUs is captured by f/w 623 * on ibm,os-term rtas call. Skip IPI callbacks to other CPUs before 624 * this rtas call to avoid tricky post processing of those CPUs' 625 * backtraces. 626 */ 627 if (should_fadump_crash()) 628 return; 629 630 if (stopped) 631 return; 632 633 stopped = true; 634 635#ifdef CONFIG_CRASH_DUMP 636 if (kexec_crash_image) { 637 crash_kexec_prepare(); 638 return; 639 } 640#endif 641 642 smp_send_stop(); 643} 644 645#ifdef CONFIG_NMI_IPI 646static void nmi_stop_this_cpu(struct pt_regs *regs) 647{ 648 /* 649 * IRQs are already hard disabled by the smp_handle_nmi_ipi. 650 */ 651 set_cpu_online(smp_processor_id(), false); 652 653 spin_begin(); 654 while (1) 655 spin_cpu_relax(); 656} 657 658void smp_send_stop(void) 659{ 660 smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, nmi_stop_this_cpu, 1000000); 661} 662 663#else /* CONFIG_NMI_IPI */ 664 665static void stop_this_cpu(void *dummy) 666{ 667 hard_irq_disable(); 668 669 /* 670 * Offlining CPUs in stop_this_cpu can result in scheduler warnings, 671 * (see commit de6e5d38417e), but printk_safe_flush_on_panic() wants 672 * to know other CPUs are offline before it breaks locks to flush 673 * printk buffers, in case we panic()ed while holding the lock. 674 */ 675 set_cpu_online(smp_processor_id(), false); 676 677 spin_begin(); 678 while (1) 679 spin_cpu_relax(); 680} 681 682void smp_send_stop(void) 683{ 684 static bool stopped = false; 685 686 /* 687 * Prevent waiting on csd lock from a previous smp_send_stop. 688 * This is racy, but in general callers try to do the right 689 * thing and only fire off one smp_send_stop (e.g., see 690 * kernel/panic.c) 691 */ 692 if (stopped) 693 return; 694 695 stopped = true; 696 697 smp_call_function(stop_this_cpu, NULL, 0); 698} 699#endif /* CONFIG_NMI_IPI */ 700 701static struct task_struct *current_set[NR_CPUS]; 702 703static void smp_store_cpu_info(int id) 704{ 705 per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR); 706#ifdef CONFIG_PPC_E500 707 per_cpu(next_tlbcam_idx, id) 708 = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1; 709#endif 710} 711 712/* 713 * Relationships between CPUs are maintained in a set of per-cpu cpumasks so 714 * rather than just passing around the cpumask we pass around a function that 715 * returns the that cpumask for the given CPU. 716 */ 717static void set_cpus_related(int i, int j, struct cpumask *(*get_cpumask)(int)) 718{ 719 cpumask_set_cpu(i, get_cpumask(j)); 720 cpumask_set_cpu(j, get_cpumask(i)); 721} 722 723#ifdef CONFIG_HOTPLUG_CPU 724static void set_cpus_unrelated(int i, int j, 725 struct cpumask *(*get_cpumask)(int)) 726{ 727 cpumask_clear_cpu(i, get_cpumask(j)); 728 cpumask_clear_cpu(j, get_cpumask(i)); 729} 730#endif 731 732/* 733 * Extends set_cpus_related. Instead of setting one CPU at a time in 734 * dstmask, set srcmask at oneshot. dstmask should be super set of srcmask. 735 */ 736static void or_cpumasks_related(int i, int j, struct cpumask *(*srcmask)(int), 737 struct cpumask *(*dstmask)(int)) 738{ 739 struct cpumask *mask; 740 int k; 741 742 mask = srcmask(j); 743 for_each_cpu(k, srcmask(i)) 744 cpumask_or(dstmask(k), dstmask(k), mask); 745 746 if (i == j) 747 return; 748 749 mask = srcmask(i); 750 for_each_cpu(k, srcmask(j)) 751 cpumask_or(dstmask(k), dstmask(k), mask); 752} 753 754/* 755 * parse_thread_groups: Parses the "ibm,thread-groups" device tree 756 * property for the CPU device node @dn and stores 757 * the parsed output in the thread_groups_list 758 * structure @tglp. 759 * 760 * @dn: The device node of the CPU device. 761 * @tglp: Pointer to a thread group list structure into which the parsed 762 * output of "ibm,thread-groups" is stored. 763 * 764 * ibm,thread-groups[0..N-1] array defines which group of threads in 765 * the CPU-device node can be grouped together based on the property. 766 * 767 * This array can represent thread groupings for multiple properties. 768 * 769 * ibm,thread-groups[i + 0] tells us the property based on which the 770 * threads are being grouped together. If this value is 1, it implies 771 * that the threads in the same group share L1, translation cache. If 772 * the value is 2, it implies that the threads in the same group share 773 * the same L2 cache. 774 * 775 * ibm,thread-groups[i+1] tells us how many such thread groups exist for the 776 * property ibm,thread-groups[i] 777 * 778 * ibm,thread-groups[i+2] tells us the number of threads in each such 779 * group. 780 * Suppose k = (ibm,thread-groups[i+1] * ibm,thread-groups[i+2]), then, 781 * 782 * ibm,thread-groups[i+3..i+k+2] (is the list of threads identified by 783 * "ibm,ppc-interrupt-server#s" arranged as per their membership in 784 * the grouping. 785 * 786 * Example: 787 * If "ibm,thread-groups" = [1,2,4,8,10,12,14,9,11,13,15,2,2,4,8,10,12,14,9,11,13,15] 788 * This can be decomposed up into two consecutive arrays: 789 * a) [1,2,4,8,10,12,14,9,11,13,15] 790 * b) [2,2,4,8,10,12,14,9,11,13,15] 791 * 792 * where in, 793 * 794 * a) provides information of Property "1" being shared by "2" groups, 795 * each with "4" threads each. The "ibm,ppc-interrupt-server#s" of 796 * the first group is {8,10,12,14} and the 797 * "ibm,ppc-interrupt-server#s" of the second group is 798 * {9,11,13,15}. Property "1" is indicative of the thread in the 799 * group sharing L1 cache, translation cache and Instruction Data 800 * flow. 801 * 802 * b) provides information of Property "2" being shared by "2" groups, 803 * each group with "4" threads. The "ibm,ppc-interrupt-server#s" of 804 * the first group is {8,10,12,14} and the 805 * "ibm,ppc-interrupt-server#s" of the second group is 806 * {9,11,13,15}. Property "2" indicates that the threads in each 807 * group share the L2-cache. 808 * 809 * Returns 0 on success, -EINVAL if the property does not exist, 810 * -ENODATA if property does not have a value, and -EOVERFLOW if the 811 * property data isn't large enough. 812 */ 813static int parse_thread_groups(struct device_node *dn, 814 struct thread_groups_list *tglp) 815{ 816 unsigned int property_idx = 0; 817 u32 *thread_group_array; 818 size_t total_threads; 819 int ret = 0, count; 820 u32 *thread_list; 821 int i = 0; 822 823 count = of_property_count_u32_elems(dn, "ibm,thread-groups"); 824 thread_group_array = kcalloc(count, sizeof(u32), GFP_KERNEL); 825 if (!thread_group_array) 826 return -ENOMEM; 827 ret = of_property_read_u32_array(dn, "ibm,thread-groups", 828 thread_group_array, count); 829 if (ret) 830 goto out_free; 831 832 while (i < count && property_idx < MAX_THREAD_GROUP_PROPERTIES) { 833 int j; 834 struct thread_groups *tg = &tglp->property_tgs[property_idx++]; 835 836 tg->property = thread_group_array[i]; 837 tg->nr_groups = thread_group_array[i + 1]; 838 tg->threads_per_group = thread_group_array[i + 2]; 839 total_threads = tg->nr_groups * tg->threads_per_group; 840 841 thread_list = &thread_group_array[i + 3]; 842 843 for (j = 0; j < total_threads; j++) 844 tg->thread_list[j] = thread_list[j]; 845 i = i + 3 + total_threads; 846 } 847 848 tglp->nr_properties = property_idx; 849 850out_free: 851 kfree(thread_group_array); 852 return ret; 853} 854 855/* 856 * get_cpu_thread_group_start : Searches the thread group in tg->thread_list 857 * that @cpu belongs to. 858 * 859 * @cpu : The logical CPU whose thread group is being searched. 860 * @tg : The thread-group structure of the CPU node which @cpu belongs 861 * to. 862 * 863 * Returns the index to tg->thread_list that points to the start 864 * of the thread_group that @cpu belongs to. 865 * 866 * Returns -1 if cpu doesn't belong to any of the groups pointed to by 867 * tg->thread_list. 868 */ 869static int get_cpu_thread_group_start(int cpu, struct thread_groups *tg) 870{ 871 int hw_cpu_id = get_hard_smp_processor_id(cpu); 872 int i, j; 873 874 for (i = 0; i < tg->nr_groups; i++) { 875 int group_start = i * tg->threads_per_group; 876 877 for (j = 0; j < tg->threads_per_group; j++) { 878 int idx = group_start + j; 879 880 if (tg->thread_list[idx] == hw_cpu_id) 881 return group_start; 882 } 883 } 884 885 return -1; 886} 887 888static struct thread_groups *__init get_thread_groups(int cpu, 889 int group_property, 890 int *err) 891{ 892 struct device_node *dn = of_get_cpu_node(cpu, NULL); 893 struct thread_groups_list *cpu_tgl = &tgl[cpu]; 894 struct thread_groups *tg = NULL; 895 int i; 896 *err = 0; 897 898 if (!dn) { 899 *err = -ENODATA; 900 return NULL; 901 } 902 903 if (!cpu_tgl->nr_properties) { 904 *err = parse_thread_groups(dn, cpu_tgl); 905 if (*err) 906 goto out; 907 } 908 909 for (i = 0; i < cpu_tgl->nr_properties; i++) { 910 if (cpu_tgl->property_tgs[i].property == group_property) { 911 tg = &cpu_tgl->property_tgs[i]; 912 break; 913 } 914 } 915 916 if (!tg) 917 *err = -EINVAL; 918out: 919 of_node_put(dn); 920 return tg; 921} 922 923static int __init update_mask_from_threadgroup(cpumask_var_t *mask, struct thread_groups *tg, 924 int cpu, int cpu_group_start) 925{ 926 int first_thread = cpu_first_thread_sibling(cpu); 927 int i; 928 929 zalloc_cpumask_var_node(mask, GFP_KERNEL, cpu_to_node(cpu)); 930 931 for (i = first_thread; i < first_thread + threads_per_core; i++) { 932 int i_group_start = get_cpu_thread_group_start(i, tg); 933 934 if (unlikely(i_group_start == -1)) { 935 WARN_ON_ONCE(1); 936 return -ENODATA; 937 } 938 939 if (i_group_start == cpu_group_start) 940 cpumask_set_cpu(i, *mask); 941 } 942 943 return 0; 944} 945 946static int __init init_thread_group_cache_map(int cpu, int cache_property) 947 948{ 949 int cpu_group_start = -1, err = 0; 950 struct thread_groups *tg = NULL; 951 cpumask_var_t *mask = NULL; 952 953 if (cache_property != THREAD_GROUP_SHARE_L1 && 954 cache_property != THREAD_GROUP_SHARE_L2_L3) 955 return -EINVAL; 956 957 tg = get_thread_groups(cpu, cache_property, &err); 958 959 if (!tg) 960 return err; 961 962 cpu_group_start = get_cpu_thread_group_start(cpu, tg); 963 964 if (unlikely(cpu_group_start == -1)) { 965 WARN_ON_ONCE(1); 966 return -ENODATA; 967 } 968 969 if (cache_property == THREAD_GROUP_SHARE_L1) { 970 mask = &per_cpu(thread_group_l1_cache_map, cpu); 971 update_mask_from_threadgroup(mask, tg, cpu, cpu_group_start); 972 } 973 else if (cache_property == THREAD_GROUP_SHARE_L2_L3) { 974 mask = &per_cpu(thread_group_l2_cache_map, cpu); 975 update_mask_from_threadgroup(mask, tg, cpu, cpu_group_start); 976 mask = &per_cpu(thread_group_l3_cache_map, cpu); 977 update_mask_from_threadgroup(mask, tg, cpu, cpu_group_start); 978 } 979 980 981 return 0; 982} 983 984static bool shared_caches __ro_after_init; 985 986#ifdef CONFIG_SCHED_SMT 987/* cpumask of CPUs with asymmetric SMT dependency */ 988static int powerpc_smt_flags(void) 989{ 990 int flags = SD_SHARE_CPUCAPACITY | SD_SHARE_LLC; 991 992 if (cpu_has_feature(CPU_FTR_ASYM_SMT)) { 993 printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n"); 994 flags |= SD_ASYM_PACKING; 995 } 996 return flags; 997} 998#endif 999 1000/* 1001 * On shared processor LPARs scheduled on a big core (which has two or more 1002 * independent thread groups per core), prefer lower numbered CPUs, so 1003 * that workload consolidates to lesser number of cores. 1004 */ 1005static __ro_after_init DEFINE_STATIC_KEY_FALSE(splpar_asym_pack); 1006 1007/* 1008 * P9 has a slightly odd architecture where pairs of cores share an L2 cache. 1009 * This topology makes it *much* cheaper to migrate tasks between adjacent cores 1010 * since the migrated task remains cache hot. We want to take advantage of this 1011 * at the scheduler level so an extra topology level is required. 1012 */ 1013static int powerpc_shared_cache_flags(void) 1014{ 1015 if (static_branch_unlikely(&splpar_asym_pack)) 1016 return SD_SHARE_LLC | SD_ASYM_PACKING; 1017 1018 return SD_SHARE_LLC; 1019} 1020 1021static int powerpc_shared_proc_flags(void) 1022{ 1023 if (static_branch_unlikely(&splpar_asym_pack)) 1024 return SD_ASYM_PACKING; 1025 1026 return 0; 1027} 1028 1029/* 1030 * We can't just pass cpu_l2_cache_mask() directly because 1031 * returns a non-const pointer and the compiler barfs on that. 1032 */ 1033static const struct cpumask *tl_cache_mask(struct sched_domain_topology_level *tl, int cpu) 1034{ 1035 return per_cpu(cpu_l2_cache_map, cpu); 1036} 1037 1038#ifdef CONFIG_SCHED_SMT 1039static const struct cpumask *tl_smallcore_smt_mask(struct sched_domain_topology_level *tl, int cpu) 1040{ 1041 return cpu_smallcore_mask(cpu); 1042} 1043#endif 1044 1045struct cpumask *cpu_coregroup_mask(int cpu) 1046{ 1047 return per_cpu(cpu_coregroup_map, cpu); 1048} 1049 1050static bool has_coregroup_support(void) 1051{ 1052 /* Coregroup identification not available on shared systems */ 1053 if (is_shared_processor()) 1054 return 0; 1055 1056 return coregroup_enabled; 1057} 1058 1059static int __init init_big_cores(void) 1060{ 1061 int cpu; 1062 1063 for_each_possible_cpu(cpu) { 1064 int err = init_thread_group_cache_map(cpu, THREAD_GROUP_SHARE_L1); 1065 1066 if (err) 1067 return err; 1068 1069 zalloc_cpumask_var_node(&per_cpu(cpu_smallcore_map, cpu), 1070 GFP_KERNEL, 1071 cpu_to_node(cpu)); 1072 } 1073 1074 has_big_cores = true; 1075 1076 for_each_possible_cpu(cpu) { 1077 int err = init_thread_group_cache_map(cpu, THREAD_GROUP_SHARE_L2_L3); 1078 1079 if (err) 1080 return err; 1081 } 1082 1083 thread_group_shares_l2 = true; 1084 thread_group_shares_l3 = true; 1085 pr_debug("L2/L3 cache only shared by the threads in the small core\n"); 1086 1087 return 0; 1088} 1089 1090/* 1091 * die_mask and die_id are only available on systems which support 1092 * multiple coregroups within a same package. On all other systems, die_mask 1093 * would be same as package mask and die_id would be set to -1. 1094 */ 1095const struct cpumask *cpu_die_mask(int cpu) 1096{ 1097 if (has_coregroup_support()) 1098 return per_cpu(cpu_coregroup_map, cpu); 1099 else 1100 return cpu_node_mask(cpu); 1101} 1102EXPORT_SYMBOL_GPL(cpu_die_mask); 1103 1104int cpu_die_id(int cpu) 1105{ 1106 if (has_coregroup_support()) 1107 return cpu_to_coregroup_id(cpu); 1108 else 1109 return -1; 1110} 1111EXPORT_SYMBOL_GPL(cpu_die_id); 1112 1113void __init smp_prepare_cpus(unsigned int max_cpus) 1114{ 1115 unsigned int cpu, num_threads; 1116 1117 DBG("smp_prepare_cpus\n"); 1118 1119 /* 1120 * setup_cpu may need to be called on the boot cpu. We haven't 1121 * spun any cpus up but lets be paranoid. 1122 */ 1123 BUG_ON(boot_cpuid != smp_processor_id()); 1124 1125 /* Fixup boot cpu */ 1126 smp_store_cpu_info(boot_cpuid); 1127 cpu_callin_map[boot_cpuid] = 1; 1128 1129 for_each_possible_cpu(cpu) { 1130 zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu), 1131 GFP_KERNEL, cpu_to_node(cpu)); 1132 zalloc_cpumask_var_node(&per_cpu(cpu_l2_cache_map, cpu), 1133 GFP_KERNEL, cpu_to_node(cpu)); 1134 zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu), 1135 GFP_KERNEL, cpu_to_node(cpu)); 1136 if (has_coregroup_support()) 1137 zalloc_cpumask_var_node(&per_cpu(cpu_coregroup_map, cpu), 1138 GFP_KERNEL, cpu_to_node(cpu)); 1139 1140#ifdef CONFIG_NUMA 1141 /* 1142 * numa_node_id() works after this. 1143 */ 1144 if (cpu_present(cpu)) { 1145 set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]); 1146 set_cpu_numa_mem(cpu, 1147 local_memory_node(numa_cpu_lookup_table[cpu])); 1148 } 1149#endif 1150 } 1151 1152 /* Init the cpumasks so the boot CPU is related to itself */ 1153 cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid)); 1154 cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid)); 1155 cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid)); 1156 1157 if (has_coregroup_support()) 1158 cpumask_set_cpu(boot_cpuid, cpu_coregroup_mask(boot_cpuid)); 1159 1160 init_big_cores(); 1161 if (has_big_cores) { 1162 cpumask_set_cpu(boot_cpuid, 1163 cpu_smallcore_mask(boot_cpuid)); 1164 } 1165 1166 if (cpu_to_chip_id(boot_cpuid) != -1) { 1167 int idx = DIV_ROUND_UP(num_possible_cpus(), threads_per_core); 1168 1169 /* 1170 * All threads of a core will all belong to the same core, 1171 * chip_id_lookup_table will have one entry per core. 1172 * Assumption: if boot_cpuid doesn't have a chip-id, then no 1173 * other CPUs, will also not have chip-id. 1174 */ 1175 chip_id_lookup_table = kcalloc(idx, sizeof(int), GFP_KERNEL); 1176 if (chip_id_lookup_table) 1177 memset(chip_id_lookup_table, -1, sizeof(int) * idx); 1178 } 1179 1180 if (smp_ops && smp_ops->probe) 1181 smp_ops->probe(); 1182 1183 // Initalise the generic SMT topology support 1184 num_threads = 1; 1185 if (smt_enabled_at_boot) 1186 num_threads = smt_enabled_at_boot; 1187 cpu_smt_set_num_threads(num_threads, threads_per_core); 1188} 1189 1190void __init smp_prepare_boot_cpu(void) 1191{ 1192 BUG_ON(smp_processor_id() != boot_cpuid); 1193#ifdef CONFIG_PPC64 1194 paca_ptrs[boot_cpuid]->__current = current; 1195#endif 1196 set_numa_node(numa_cpu_lookup_table[boot_cpuid]); 1197 current_set[boot_cpuid] = current; 1198} 1199 1200#ifdef CONFIG_HOTPLUG_CPU 1201 1202int generic_cpu_disable(void) 1203{ 1204 unsigned int cpu = smp_processor_id(); 1205 1206 if (cpu == boot_cpuid) 1207 return -EBUSY; 1208 1209 set_cpu_online(cpu, false); 1210#ifdef CONFIG_PPC64_PROC_SYSTEMCFG 1211 systemcfg->processorCount--; 1212#endif 1213 /* Update affinity of all IRQs previously aimed at this CPU */ 1214 irq_migrate_all_off_this_cpu(); 1215 1216 /* 1217 * Depending on the details of the interrupt controller, it's possible 1218 * that one of the interrupts we just migrated away from this CPU is 1219 * actually already pending on this CPU. If we leave it in that state 1220 * the interrupt will never be EOI'ed, and will never fire again. So 1221 * temporarily enable interrupts here, to allow any pending interrupt to 1222 * be received (and EOI'ed), before we take this CPU offline. 1223 */ 1224 local_irq_enable(); 1225 mdelay(1); 1226 local_irq_disable(); 1227 1228 return 0; 1229} 1230 1231void generic_cpu_die(unsigned int cpu) 1232{ 1233 int i; 1234 1235 for (i = 0; i < 100; i++) { 1236 smp_rmb(); 1237 if (is_cpu_dead(cpu)) 1238 return; 1239 msleep(100); 1240 } 1241 printk(KERN_ERR "CPU%d didn't die...\n", cpu); 1242} 1243 1244void generic_set_cpu_dead(unsigned int cpu) 1245{ 1246 per_cpu(cpu_state, cpu) = CPU_DEAD; 1247} 1248 1249/* 1250 * The cpu_state should be set to CPU_UP_PREPARE in kick_cpu(), otherwise 1251 * the cpu_state is always CPU_DEAD after calling generic_set_cpu_dead(), 1252 * which makes the delay in generic_cpu_die() not happen. 1253 */ 1254void generic_set_cpu_up(unsigned int cpu) 1255{ 1256 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE; 1257} 1258 1259int generic_check_cpu_restart(unsigned int cpu) 1260{ 1261 return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE; 1262} 1263 1264int is_cpu_dead(unsigned int cpu) 1265{ 1266 return per_cpu(cpu_state, cpu) == CPU_DEAD; 1267} 1268 1269static bool secondaries_inhibited(void) 1270{ 1271 return kvm_hv_mode_active(); 1272} 1273 1274#else /* HOTPLUG_CPU */ 1275 1276#define secondaries_inhibited() 0 1277 1278#endif 1279 1280static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle) 1281{ 1282#ifdef CONFIG_PPC64 1283 paca_ptrs[cpu]->__current = idle; 1284 paca_ptrs[cpu]->kstack = (unsigned long)task_stack_page(idle) + 1285 THREAD_SIZE - STACK_FRAME_MIN_SIZE; 1286#endif 1287 task_thread_info(idle)->cpu = cpu; 1288 secondary_current = current_set[cpu] = idle; 1289} 1290 1291int __cpu_up(unsigned int cpu, struct task_struct *tidle) 1292{ 1293 const unsigned long boot_spin_ms = 5 * MSEC_PER_SEC; 1294 const bool booting = system_state < SYSTEM_RUNNING; 1295 const unsigned long hp_spin_ms = 1; 1296 unsigned long deadline; 1297 int rc; 1298 const unsigned long spin_wait_ms = booting ? boot_spin_ms : hp_spin_ms; 1299 1300 /* 1301 * Don't allow secondary threads to come online if inhibited 1302 */ 1303 if (threads_per_core > 1 && secondaries_inhibited() && 1304 cpu_thread_in_subcore(cpu)) 1305 return -EBUSY; 1306 1307 if (smp_ops == NULL || 1308 (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu))) 1309 return -EINVAL; 1310 1311 cpu_idle_thread_init(cpu, tidle); 1312 1313 /* 1314 * The platform might need to allocate resources prior to bringing 1315 * up the CPU 1316 */ 1317 if (smp_ops->prepare_cpu) { 1318 rc = smp_ops->prepare_cpu(cpu); 1319 if (rc) 1320 return rc; 1321 } 1322 1323 /* Make sure callin-map entry is 0 (can be leftover a CPU 1324 * hotplug 1325 */ 1326 cpu_callin_map[cpu] = 0; 1327 1328 /* The information for processor bringup must 1329 * be written out to main store before we release 1330 * the processor. 1331 */ 1332 smp_mb(); 1333 1334 /* wake up cpus */ 1335 DBG("smp: kicking cpu %d\n", cpu); 1336 rc = smp_ops->kick_cpu(cpu); 1337 if (rc) { 1338 pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc); 1339 return rc; 1340 } 1341 1342 /* 1343 * At boot time, simply spin on the callin word until the 1344 * deadline passes. 1345 * 1346 * At run time, spin for an optimistic amount of time to avoid 1347 * sleeping in the common case. 1348 */ 1349 deadline = jiffies + msecs_to_jiffies(spin_wait_ms); 1350 spin_until_cond(cpu_callin_map[cpu] || time_is_before_jiffies(deadline)); 1351 1352 if (!cpu_callin_map[cpu] && system_state >= SYSTEM_RUNNING) { 1353 const unsigned long sleep_interval_us = 10 * USEC_PER_MSEC; 1354 const unsigned long sleep_wait_ms = 100 * MSEC_PER_SEC; 1355 1356 deadline = jiffies + msecs_to_jiffies(sleep_wait_ms); 1357 while (!cpu_callin_map[cpu] && time_is_after_jiffies(deadline)) 1358 fsleep(sleep_interval_us); 1359 } 1360 1361 if (!cpu_callin_map[cpu]) { 1362 printk(KERN_ERR "Processor %u is stuck.\n", cpu); 1363 return -ENOENT; 1364 } 1365 1366 DBG("Processor %u found.\n", cpu); 1367 1368 if (smp_ops->give_timebase) 1369 smp_ops->give_timebase(); 1370 1371 /* Wait until cpu puts itself in the online & active maps */ 1372 spin_until_cond(cpu_online(cpu)); 1373 1374 return 0; 1375} 1376 1377/* Return the value of the reg property corresponding to the given 1378 * logical cpu. 1379 */ 1380int cpu_to_core_id(int cpu) 1381{ 1382 struct device_node *np; 1383 int id = -1; 1384 1385 np = of_get_cpu_node(cpu, NULL); 1386 if (!np) 1387 goto out; 1388 1389 id = of_get_cpu_hwid(np, 0); 1390out: 1391 of_node_put(np); 1392 return id; 1393} 1394EXPORT_SYMBOL_GPL(cpu_to_core_id); 1395 1396/* Helper routines for cpu to core mapping */ 1397int cpu_core_index_of_thread(int cpu) 1398{ 1399 return cpu >> threads_shift; 1400} 1401EXPORT_SYMBOL_GPL(cpu_core_index_of_thread); 1402 1403int cpu_first_thread_of_core(int core) 1404{ 1405 return core << threads_shift; 1406} 1407EXPORT_SYMBOL_GPL(cpu_first_thread_of_core); 1408 1409/* Must be called when no change can occur to cpu_present_mask, 1410 * i.e. during cpu online or offline. 1411 */ 1412static struct device_node *cpu_to_l2cache(int cpu) 1413{ 1414 struct device_node *np; 1415 struct device_node *cache; 1416 1417 if (!cpu_present(cpu)) 1418 return NULL; 1419 1420 np = of_get_cpu_node(cpu, NULL); 1421 if (np == NULL) 1422 return NULL; 1423 1424 cache = of_find_next_cache_node(np); 1425 1426 of_node_put(np); 1427 1428 return cache; 1429} 1430 1431static bool update_mask_by_l2(int cpu, cpumask_var_t *mask) 1432{ 1433 struct cpumask *(*submask_fn)(int) = cpu_sibling_mask; 1434 struct device_node *l2_cache, *np; 1435 int i; 1436 1437 if (has_big_cores) 1438 submask_fn = cpu_smallcore_mask; 1439 1440 /* 1441 * If the threads in a thread-group share L2 cache, then the 1442 * L2-mask can be obtained from thread_group_l2_cache_map. 1443 */ 1444 if (thread_group_shares_l2) { 1445 cpumask_set_cpu(cpu, cpu_l2_cache_mask(cpu)); 1446 1447 for_each_cpu(i, per_cpu(thread_group_l2_cache_map, cpu)) { 1448 if (cpu_online(i)) 1449 set_cpus_related(i, cpu, cpu_l2_cache_mask); 1450 } 1451 1452 /* Verify that L1-cache siblings are a subset of L2 cache-siblings */ 1453 if (!cpumask_equal(submask_fn(cpu), cpu_l2_cache_mask(cpu)) && 1454 !cpumask_subset(submask_fn(cpu), cpu_l2_cache_mask(cpu))) { 1455 pr_warn_once("CPU %d : Inconsistent L1 and L2 cache siblings\n", 1456 cpu); 1457 } 1458 1459 return true; 1460 } 1461 1462 l2_cache = cpu_to_l2cache(cpu); 1463 if (!l2_cache || !*mask) { 1464 /* Assume only core siblings share cache with this CPU */ 1465 for_each_cpu(i, cpu_sibling_mask(cpu)) 1466 set_cpus_related(cpu, i, cpu_l2_cache_mask); 1467 1468 return false; 1469 } 1470 1471 cpumask_and(*mask, cpu_online_mask, cpu_node_mask(cpu)); 1472 1473 /* Update l2-cache mask with all the CPUs that are part of submask */ 1474 or_cpumasks_related(cpu, cpu, submask_fn, cpu_l2_cache_mask); 1475 1476 /* Skip all CPUs already part of current CPU l2-cache mask */ 1477 cpumask_andnot(*mask, *mask, cpu_l2_cache_mask(cpu)); 1478 1479 for_each_cpu(i, *mask) { 1480 /* 1481 * when updating the marks the current CPU has not been marked 1482 * online, but we need to update the cache masks 1483 */ 1484 np = cpu_to_l2cache(i); 1485 1486 /* Skip all CPUs already part of current CPU l2-cache */ 1487 if (np == l2_cache) { 1488 or_cpumasks_related(cpu, i, submask_fn, cpu_l2_cache_mask); 1489 cpumask_andnot(*mask, *mask, submask_fn(i)); 1490 } else { 1491 cpumask_andnot(*mask, *mask, cpu_l2_cache_mask(i)); 1492 } 1493 1494 of_node_put(np); 1495 } 1496 of_node_put(l2_cache); 1497 1498 return true; 1499} 1500 1501#ifdef CONFIG_HOTPLUG_CPU 1502static void remove_cpu_from_masks(int cpu) 1503{ 1504 struct cpumask *(*mask_fn)(int) = cpu_sibling_mask; 1505 int i; 1506 1507 unmap_cpu_from_node(cpu); 1508 1509 if (shared_caches) 1510 mask_fn = cpu_l2_cache_mask; 1511 1512 for_each_cpu(i, mask_fn(cpu)) { 1513 set_cpus_unrelated(cpu, i, cpu_l2_cache_mask); 1514 set_cpus_unrelated(cpu, i, cpu_sibling_mask); 1515 if (has_big_cores) 1516 set_cpus_unrelated(cpu, i, cpu_smallcore_mask); 1517 } 1518 1519 for_each_cpu(i, cpu_core_mask(cpu)) 1520 set_cpus_unrelated(cpu, i, cpu_core_mask); 1521 1522 if (has_coregroup_support()) { 1523 for_each_cpu(i, cpu_coregroup_mask(cpu)) 1524 set_cpus_unrelated(cpu, i, cpu_coregroup_mask); 1525 } 1526} 1527#endif 1528 1529static inline void add_cpu_to_smallcore_masks(int cpu) 1530{ 1531 int i; 1532 1533 if (!has_big_cores) 1534 return; 1535 1536 cpumask_set_cpu(cpu, cpu_smallcore_mask(cpu)); 1537 1538 for_each_cpu(i, per_cpu(thread_group_l1_cache_map, cpu)) { 1539 if (cpu_online(i)) 1540 set_cpus_related(i, cpu, cpu_smallcore_mask); 1541 } 1542} 1543 1544static void update_coregroup_mask(int cpu, cpumask_var_t *mask) 1545{ 1546 struct cpumask *(*submask_fn)(int) = cpu_sibling_mask; 1547 int coregroup_id = cpu_to_coregroup_id(cpu); 1548 int i; 1549 1550 if (shared_caches) 1551 submask_fn = cpu_l2_cache_mask; 1552 1553 if (!*mask) { 1554 /* Assume only siblings are part of this CPU's coregroup */ 1555 for_each_cpu(i, submask_fn(cpu)) 1556 set_cpus_related(cpu, i, cpu_coregroup_mask); 1557 1558 return; 1559 } 1560 1561 cpumask_and(*mask, cpu_online_mask, cpu_node_mask(cpu)); 1562 1563 /* Update coregroup mask with all the CPUs that are part of submask */ 1564 or_cpumasks_related(cpu, cpu, submask_fn, cpu_coregroup_mask); 1565 1566 /* Skip all CPUs already part of coregroup mask */ 1567 cpumask_andnot(*mask, *mask, cpu_coregroup_mask(cpu)); 1568 1569 for_each_cpu(i, *mask) { 1570 /* Skip all CPUs not part of this coregroup */ 1571 if (coregroup_id == cpu_to_coregroup_id(i)) { 1572 or_cpumasks_related(cpu, i, submask_fn, cpu_coregroup_mask); 1573 cpumask_andnot(*mask, *mask, submask_fn(i)); 1574 } else { 1575 cpumask_andnot(*mask, *mask, cpu_coregroup_mask(i)); 1576 } 1577 } 1578} 1579 1580static void add_cpu_to_masks(int cpu) 1581{ 1582 struct cpumask *(*submask_fn)(int) = cpu_sibling_mask; 1583 int first_thread = cpu_first_thread_sibling(cpu); 1584 cpumask_var_t mask; 1585 int chip_id = -1; 1586 bool ret; 1587 int i; 1588 1589 /* 1590 * This CPU will not be in the online mask yet so we need to manually 1591 * add it to its own thread sibling mask. 1592 */ 1593 map_cpu_to_node(cpu, cpu_to_node(cpu)); 1594 cpumask_set_cpu(cpu, cpu_sibling_mask(cpu)); 1595 cpumask_set_cpu(cpu, cpu_core_mask(cpu)); 1596 1597 for (i = first_thread; i < first_thread + threads_per_core; i++) 1598 if (cpu_online(i)) 1599 set_cpus_related(i, cpu, cpu_sibling_mask); 1600 1601 add_cpu_to_smallcore_masks(cpu); 1602 1603 /* In CPU-hotplug path, hence use GFP_ATOMIC */ 1604 ret = alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu)); 1605 update_mask_by_l2(cpu, &mask); 1606 1607 if (has_coregroup_support()) 1608 update_coregroup_mask(cpu, &mask); 1609 1610 if (chip_id_lookup_table && ret) 1611 chip_id = cpu_to_chip_id(cpu); 1612 1613 if (shared_caches) 1614 submask_fn = cpu_l2_cache_mask; 1615 1616 /* Update core_mask with all the CPUs that are part of submask */ 1617 or_cpumasks_related(cpu, cpu, submask_fn, cpu_core_mask); 1618 1619 /* Skip all CPUs already part of current CPU core mask */ 1620 cpumask_andnot(mask, cpu_online_mask, cpu_core_mask(cpu)); 1621 1622 /* If chip_id is -1; limit the cpu_core_mask to within PKG */ 1623 if (chip_id == -1) 1624 cpumask_and(mask, mask, cpu_node_mask(cpu)); 1625 1626 for_each_cpu(i, mask) { 1627 if (chip_id == cpu_to_chip_id(i)) { 1628 or_cpumasks_related(cpu, i, submask_fn, cpu_core_mask); 1629 cpumask_andnot(mask, mask, submask_fn(i)); 1630 } else { 1631 cpumask_andnot(mask, mask, cpu_core_mask(i)); 1632 } 1633 } 1634 1635 free_cpumask_var(mask); 1636} 1637 1638/* Activate a secondary processor. */ 1639__no_stack_protector 1640void start_secondary(void *unused) 1641{ 1642 unsigned int cpu = raw_smp_processor_id(); 1643 1644 /* PPC64 calls setup_kup() in early_setup_secondary() */ 1645 if (IS_ENABLED(CONFIG_PPC32)) 1646 setup_kup(); 1647 1648 mmgrab_lazy_tlb(&init_mm); 1649 current->active_mm = &init_mm; 1650 VM_WARN_ON(cpumask_test_cpu(smp_processor_id(), mm_cpumask(&init_mm))); 1651 cpumask_set_cpu(cpu, mm_cpumask(&init_mm)); 1652 inc_mm_active_cpus(&init_mm); 1653 1654 smp_store_cpu_info(cpu); 1655 set_dec(tb_ticks_per_jiffy); 1656 rcutree_report_cpu_starting(cpu); 1657 cpu_callin_map[cpu] = 1; 1658 1659 if (smp_ops->setup_cpu) 1660 smp_ops->setup_cpu(cpu); 1661 if (smp_ops->take_timebase) 1662 smp_ops->take_timebase(); 1663 1664 secondary_cpu_time_init(); 1665 1666#ifdef CONFIG_PPC64_PROC_SYSTEMCFG 1667 if (system_state == SYSTEM_RUNNING) 1668 systemcfg->processorCount++; 1669#endif 1670 1671#ifdef CONFIG_PPC64 1672 vdso_getcpu_init(); 1673#endif 1674 set_numa_node(numa_cpu_lookup_table[cpu]); 1675 set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu])); 1676 1677 /* Update topology CPU masks */ 1678 add_cpu_to_masks(cpu); 1679 1680 /* 1681 * Check for any shared caches. Note that this must be done on a 1682 * per-core basis because one core in the pair might be disabled. 1683 */ 1684 if (!shared_caches) { 1685 struct cpumask *(*sibling_mask)(int) = cpu_sibling_mask; 1686 struct cpumask *mask = cpu_l2_cache_mask(cpu); 1687 1688 if (has_big_cores) 1689 sibling_mask = cpu_smallcore_mask; 1690 1691 if (cpumask_weight(mask) > cpumask_weight(sibling_mask(cpu))) 1692 shared_caches = true; 1693 } 1694 1695 smp_wmb(); 1696 notify_cpu_starting(cpu); 1697 set_cpu_online(cpu, true); 1698 1699 boot_init_stack_canary(); 1700 1701 local_irq_enable(); 1702 1703 /* We can enable ftrace for secondary cpus now */ 1704 this_cpu_enable_ftrace(); 1705 1706 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 1707 1708 BUG(); 1709} 1710 1711static struct sched_domain_topology_level powerpc_topology[6]; 1712 1713static void __init build_sched_topology(void) 1714{ 1715 int i = 0; 1716 1717 if (is_shared_processor() && has_big_cores) 1718 static_branch_enable(&splpar_asym_pack); 1719 1720#ifdef CONFIG_SCHED_SMT 1721 if (has_big_cores) { 1722 pr_info("Big cores detected but using small core scheduling\n"); 1723 powerpc_topology[i++] = 1724 SDTL_INIT(tl_smallcore_smt_mask, powerpc_smt_flags, SMT); 1725 } else { 1726 powerpc_topology[i++] = SDTL_INIT(tl_smt_mask, powerpc_smt_flags, SMT); 1727 } 1728#endif 1729 if (shared_caches) { 1730 powerpc_topology[i++] = 1731 SDTL_INIT(tl_cache_mask, powerpc_shared_cache_flags, CACHE); 1732 } 1733 1734 if (has_coregroup_support()) { 1735 powerpc_topology[i++] = 1736 SDTL_INIT(tl_mc_mask, powerpc_shared_proc_flags, MC); 1737 } 1738 1739 powerpc_topology[i++] = SDTL_INIT(tl_pkg_mask, powerpc_shared_proc_flags, PKG); 1740 1741 /* There must be one trailing NULL entry left. */ 1742 BUG_ON(i >= ARRAY_SIZE(powerpc_topology) - 1); 1743 1744 set_sched_topology(powerpc_topology); 1745} 1746 1747void __init smp_cpus_done(unsigned int max_cpus) 1748{ 1749 /* 1750 * We are running pinned to the boot CPU, see rest_init(). 1751 */ 1752 if (smp_ops && smp_ops->setup_cpu) 1753 smp_ops->setup_cpu(boot_cpuid); 1754 1755 if (smp_ops && smp_ops->bringup_done) 1756 smp_ops->bringup_done(); 1757 1758 dump_numa_cpu_topology(); 1759 build_sched_topology(); 1760} 1761 1762/* 1763 * For asym packing, by default lower numbered CPU has higher priority. 1764 * On shared processors, pack to lower numbered core. However avoid moving 1765 * between thread_groups within the same core. 1766 */ 1767int arch_asym_cpu_priority(int cpu) 1768{ 1769 if (static_branch_unlikely(&splpar_asym_pack)) 1770 return -cpu / threads_per_core; 1771 1772 return -cpu; 1773} 1774 1775#ifdef CONFIG_HOTPLUG_CPU 1776int __cpu_disable(void) 1777{ 1778 int cpu = smp_processor_id(); 1779 int err; 1780 1781 if (!smp_ops->cpu_disable) 1782 return -ENOSYS; 1783 1784 this_cpu_disable_ftrace(); 1785 1786 err = smp_ops->cpu_disable(); 1787 if (err) 1788 return err; 1789 1790 /* Update sibling maps */ 1791 remove_cpu_from_masks(cpu); 1792 1793 return 0; 1794} 1795 1796void __cpu_die(unsigned int cpu) 1797{ 1798 /* 1799 * This could perhaps be a generic call in idlea_task_dead(), but 1800 * that requires testing from all archs, so first put it here to 1801 */ 1802 VM_WARN_ON_ONCE(!cpumask_test_cpu(cpu, mm_cpumask(&init_mm))); 1803 dec_mm_active_cpus(&init_mm); 1804 cpumask_clear_cpu(cpu, mm_cpumask(&init_mm)); 1805 1806 if (smp_ops->cpu_die) 1807 smp_ops->cpu_die(cpu); 1808} 1809 1810void __noreturn arch_cpu_idle_dead(void) 1811{ 1812 /* 1813 * Disable on the down path. This will be re-enabled by 1814 * start_secondary() via start_secondary_resume() below 1815 */ 1816 this_cpu_disable_ftrace(); 1817 1818 if (smp_ops->cpu_offline_self) 1819 smp_ops->cpu_offline_self(); 1820 1821 /* If we return, we re-enter start_secondary */ 1822 start_secondary_resume(); 1823} 1824 1825#endif
~/
Read-only file preview.