smpboot.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * SMP initialisation and IPI support
  4 * Based on arch/arm64/kernel/smp.c
  5 *
  6 * Copyright (C) 2012 ARM Ltd.
  7 * Copyright (C) 2015 Regents of the University of California
  8 * Copyright (C) 2017 SiFive
  9 */
 10
 11#include <linux/acpi.h>
 12#include <linux/arch_topology.h>
 13#include <linux/module.h>
 14#include <linux/init.h>
 15#include <linux/kernel.h>
 16#include <linux/mm.h>
 17#include <linux/sched.h>
 18#include <linux/kernel_stat.h>
 19#include <linux/notifier.h>
 20#include <linux/cpu.h>
 21#include <linux/percpu.h>
 22#include <linux/delay.h>
 23#include <linux/err.h>
 24#include <linux/irq.h>
 25#include <linux/of.h>
 26#include <linux/sched/task_stack.h>
 27#include <linux/sched/mm.h>
 28
 29#include <asm/cacheflush.h>
 30#include <asm/cpu_ops.h>
 31#include <asm/irq.h>
 32#include <asm/mmu_context.h>
 33#include <asm/numa.h>
 34#include <asm/tlbflush.h>
 35#include <asm/sections.h>
 36#include <asm/smp.h>
 37#include <uapi/asm/hwcap.h>
 38#include <asm/vector.h>
 39
 40#include "head.h"
 41
 42#ifndef CONFIG_HOTPLUG_PARALLEL
 43static DECLARE_COMPLETION(cpu_running);
 44#endif
 45
 46void __init smp_prepare_cpus(unsigned int max_cpus)
 47{
 48	int cpuid;
 49	unsigned int curr_cpuid;
 50
 51	init_cpu_topology();
 52
 53	curr_cpuid = smp_processor_id();
 54	store_cpu_topology(curr_cpuid);
 55	numa_store_cpu_info(curr_cpuid);
 56	numa_add_cpu(curr_cpuid);
 57
 58	/* This covers non-smp usecase mandated by "nosmp" option */
 59	if (max_cpus == 0)
 60		return;
 61
 62	for_each_possible_cpu(cpuid) {
 63		if (cpuid == curr_cpuid)
 64			continue;
 65		set_cpu_present(cpuid, true);
 66		numa_store_cpu_info(cpuid);
 67	}
 68}
 69
 70#ifdef CONFIG_ACPI
 71static unsigned int cpu_count = 1;
 72
 73static int __init acpi_parse_rintc(union acpi_subtable_headers *header, const unsigned long end)
 74{
 75	unsigned long hart;
 76	static bool found_boot_cpu;
 77	struct acpi_madt_rintc *processor = (struct acpi_madt_rintc *)header;
 78
 79	/*
 80	 * Each RINTC structure in MADT will have a flag. If ACPI_MADT_ENABLED
 81	 * bit in the flag is not enabled, it means OS should not try to enable
 82	 * the cpu to which RINTC belongs.
 83	 */
 84	if (!(processor->flags & ACPI_MADT_ENABLED))
 85		return 0;
 86
 87	if (BAD_MADT_ENTRY(processor, end))
 88		return -EINVAL;
 89
 90	acpi_table_print_madt_entry(&header->common);
 91
 92	hart = processor->hart_id;
 93	if (hart == INVALID_HARTID) {
 94		pr_warn("Invalid hartid\n");
 95		return 0;
 96	}
 97
 98	if (hart == cpuid_to_hartid_map(0)) {
 99		BUG_ON(found_boot_cpu);
100		found_boot_cpu = true;
101		return 0;
102	}
103
104	if (cpu_count >= NR_CPUS) {
105		pr_warn("NR_CPUS is too small for the number of ACPI tables.\n");
106		return 0;
107	}
108
109	cpuid_to_hartid_map(cpu_count) = hart;
110	cpu_count++;
111
112	return 0;
113}
114
115static void __init acpi_parse_and_init_cpus(void)
116{
117	acpi_table_parse_madt(ACPI_MADT_TYPE_RINTC, acpi_parse_rintc, 0);
118}
119#else
120#define acpi_parse_and_init_cpus(...)	do { } while (0)
121#endif
122
123static void __init of_parse_and_init_cpus(void)
124{
125	struct device_node *dn;
126	unsigned long hart;
127	bool found_boot_cpu = false;
128	int cpuid = 1;
129	int rc;
130
131	for_each_of_cpu_node(dn) {
132		rc = riscv_early_of_processor_hartid(dn, &hart);
133		if (rc < 0)
134			continue;
135
136		if (hart == cpuid_to_hartid_map(0)) {
137			BUG_ON(found_boot_cpu);
138			found_boot_cpu = 1;
139			early_map_cpu_to_node(0, of_node_to_nid(dn));
140			continue;
141		}
142		if (cpuid >= NR_CPUS) {
143			pr_warn("Invalid cpuid [%d] for hartid [%lu]\n",
144				cpuid, hart);
145			continue;
146		}
147
148		cpuid_to_hartid_map(cpuid) = hart;
149		early_map_cpu_to_node(cpuid, of_node_to_nid(dn));
150		cpuid++;
151	}
152
153	BUG_ON(!found_boot_cpu);
154
155	if (cpuid > nr_cpu_ids)
156		pr_warn("Total number of cpus [%d] is greater than nr_cpus option value [%d]\n",
157			cpuid, nr_cpu_ids);
158}
159
160void __init setup_smp(void)
161{
162	int cpuid;
163
164	cpu_set_ops();
165
166	if (acpi_disabled)
167		of_parse_and_init_cpus();
168	else
169		acpi_parse_and_init_cpus();
170
171	for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++)
172		if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID)
173			set_cpu_possible(cpuid, true);
174}
175
176static int start_secondary_cpu(int cpu, struct task_struct *tidle)
177{
178	if (cpu_ops->cpu_start)
179		return cpu_ops->cpu_start(cpu, tidle);
180
181	return -EOPNOTSUPP;
182}
183
184#ifdef CONFIG_HOTPLUG_PARALLEL
185int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle)
186{
187	return start_secondary_cpu(cpu, tidle);
188}
189#else
190int __cpu_up(unsigned int cpu, struct task_struct *tidle)
191{
192	int ret = 0;
193	tidle->thread_info.cpu = cpu;
194
195	ret = start_secondary_cpu(cpu, tidle);
196	if (!ret) {
197		wait_for_completion_timeout(&cpu_running,
198					    msecs_to_jiffies(1000));
199
200		if (!cpu_online(cpu)) {
201			pr_crit("CPU%u: failed to come online\n", cpu);
202			ret = -EIO;
203		}
204	} else {
205		pr_crit("CPU%u: failed to start\n", cpu);
206	}
207
208	return ret;
209}
210#endif
211
212void __init smp_cpus_done(unsigned int max_cpus)
213{
214}
215
216/*
217 * C entry point for a secondary processor.
218 */
219asmlinkage __visible void smp_callin(void)
220{
221	struct mm_struct *mm = &init_mm;
222	unsigned int curr_cpuid = smp_processor_id();
223
224	if (has_vector()) {
225		/*
226		 * Return as early as possible so the hart with a mismatching
227		 * vlen won't boot.
228		 */
229		if (riscv_v_setup_vsize())
230			return;
231	}
232
233	/* All kernel threads share the same mm context.  */
234	mmgrab(mm);
235	current->active_mm = mm;
236
237#ifdef CONFIG_HOTPLUG_PARALLEL
238	cpuhp_ap_sync_alive();
239#endif
240
241	store_cpu_topology(curr_cpuid);
242	notify_cpu_starting(curr_cpuid);
243
244	riscv_ipi_enable();
245
246	numa_add_cpu(curr_cpuid);
247
248	pr_debug("CPU%u: Booted secondary hartid %lu\n", curr_cpuid,
249		cpuid_to_hartid_map(curr_cpuid));
250
251	set_cpu_online(curr_cpuid, true);
252
253	/*
254	 * Remote cache and TLB flushes are ignored while the CPU is offline,
255	 * so flush them both right now just in case.
256	 */
257	local_flush_icache_all();
258	local_flush_tlb_all();
259#ifndef CONFIG_HOTPLUG_PARALLEL
260	complete(&cpu_running);
261#endif
262	/*
263	 * Disable preemption before enabling interrupts, so we don't try to
264	 * schedule a CPU that hasn't actually started yet.
265	 */
266	local_irq_enable();
267	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
268}