smpboot.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Common SMP CPU bringup/teardown functions
  4 */
  5#include <linux/cpu.h>
  6#include <linux/err.h>
  7#include <linux/smp.h>
  8#include <linux/delay.h>
  9#include <linux/init.h>
 10#include <linux/list.h>
 11#include <linux/slab.h>
 12#include <linux/sched.h>
 13#include <linux/sched/task.h>
 14#include <linux/export.h>
 15#include <linux/percpu.h>
 16#include <linux/kthread.h>
 17#include <linux/smpboot.h>
 18
 19#include "smpboot.h"
 20
 21#ifdef CONFIG_GENERIC_SMP_IDLE_THREAD
 22/*
 23 * For the hotplug case we keep the task structs around and reuse
 24 * them.
 25 */
 26static DEFINE_PER_CPU(struct task_struct *, idle_threads);
 27
 28struct task_struct *idle_thread_get(unsigned int cpu)
 29{
 30	struct task_struct *tsk = per_cpu(idle_threads, cpu);
 31
 32	if (!tsk)
 33		return ERR_PTR(-ENOMEM);
 34	return tsk;
 35}
 36
 37void __init idle_thread_set_boot_cpu(void)
 38{
 39	per_cpu(idle_threads, smp_processor_id()) = current;
 40}
 41
 42/**
 43 * idle_init - Initialize the idle thread for a cpu
 44 * @cpu:	The cpu for which the idle thread should be initialized
 45 *
 46 * Creates the thread if it does not exist.
 47 */
 48static __always_inline void idle_init(unsigned int cpu)
 49{
 50	struct task_struct *tsk = per_cpu(idle_threads, cpu);
 51
 52	if (!tsk) {
 53		tsk = fork_idle(cpu);
 54		if (IS_ERR(tsk))
 55			pr_err("SMP: fork_idle() failed for CPU %u\n", cpu);
 56		else
 57			per_cpu(idle_threads, cpu) = tsk;
 58	}
 59}
 60
 61/**
 62 * idle_threads_init - Initialize idle threads for all cpus
 63 */
 64void __init idle_threads_init(void)
 65{
 66	unsigned int cpu, boot_cpu;
 67
 68	boot_cpu = smp_processor_id();
 69
 70	for_each_possible_cpu(cpu) {
 71		if (cpu != boot_cpu)
 72			idle_init(cpu);
 73	}
 74}
 75#endif
 76
 77static LIST_HEAD(hotplug_threads);
 78static DEFINE_MUTEX(smpboot_threads_lock);
 79
 80struct smpboot_thread_data {
 81	unsigned int			cpu;
 82	unsigned int			status;
 83	struct smp_hotplug_thread	*ht;
 84};
 85
 86enum {
 87	HP_THREAD_NONE = 0,
 88	HP_THREAD_ACTIVE,
 89	HP_THREAD_PARKED,
 90};
 91
 92/**
 93 * smpboot_thread_fn - percpu hotplug thread loop function
 94 * @data:	thread data pointer
 95 *
 96 * Checks for thread stop and park conditions. Calls the necessary
 97 * setup, cleanup, park and unpark functions for the registered
 98 * thread.
 99 *
100 * Returns 1 when the thread should exit, 0 otherwise.
101 */
102static int smpboot_thread_fn(void *data)
103{
104	struct smpboot_thread_data *td = data;
105	struct smp_hotplug_thread *ht = td->ht;
106
107	while (1) {
108		set_current_state(TASK_INTERRUPTIBLE);
109		preempt_disable();
110		if (kthread_should_stop()) {
111			__set_current_state(TASK_RUNNING);
112			preempt_enable();
113			/* cleanup must mirror setup */
114			if (ht->cleanup && td->status != HP_THREAD_NONE)
115				ht->cleanup(td->cpu, cpu_online(td->cpu));
116			kfree(td);
117			return 0;
118		}
119
120		if (kthread_should_park()) {
121			__set_current_state(TASK_RUNNING);
122			preempt_enable();
123			if (ht->park && td->status == HP_THREAD_ACTIVE) {
124				BUG_ON(td->cpu != smp_processor_id());
125				ht->park(td->cpu);
126				td->status = HP_THREAD_PARKED;
127			}
128			kthread_parkme();
129			/* We might have been woken for stop */
130			continue;
131		}
132
133		BUG_ON(td->cpu != smp_processor_id());
134
135		/* Check for state change setup */
136		switch (td->status) {
137		case HP_THREAD_NONE:
138			__set_current_state(TASK_RUNNING);
139			preempt_enable();
140			if (ht->setup)
141				ht->setup(td->cpu);
142			td->status = HP_THREAD_ACTIVE;
143			continue;
144
145		case HP_THREAD_PARKED:
146			__set_current_state(TASK_RUNNING);
147			preempt_enable();
148			if (ht->unpark)
149				ht->unpark(td->cpu);
150			td->status = HP_THREAD_ACTIVE;
151			continue;
152		}
153
154		if (!ht->thread_should_run(td->cpu)) {
155			preempt_enable_no_resched();
156			schedule();
157		} else {
158			__set_current_state(TASK_RUNNING);
159			preempt_enable();
160			ht->thread_fn(td->cpu);
161		}
162	}
163}
164
165static int
166__smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
167{
168	struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
169	struct smpboot_thread_data *td;
170
171	if (tsk)
172		return 0;
173
174	td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu));
175	if (!td)
176		return -ENOMEM;
177	td->cpu = cpu;
178	td->ht = ht;
179
180	tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu,
181				    ht->thread_comm);
182	if (IS_ERR(tsk)) {
183		kfree(td);
184		return PTR_ERR(tsk);
185	}
186	kthread_set_per_cpu(tsk, cpu);
187	/*
188	 * Park the thread so that it could start right on the CPU
189	 * when it is available.
190	 */
191	kthread_park(tsk);
192	get_task_struct(tsk);
193	*per_cpu_ptr(ht->store, cpu) = tsk;
194	if (ht->create) {
195		/*
196		 * Make sure that the task has actually scheduled out
197		 * into park position, before calling the create
198		 * callback. At least the migration thread callback
199		 * requires that the task is off the runqueue.
200		 */
201		if (!wait_task_inactive(tsk, TASK_PARKED))
202			WARN_ON(1);
203		else
204			ht->create(cpu);
205	}
206	return 0;
207}
208
209int smpboot_create_threads(unsigned int cpu)
210{
211	struct smp_hotplug_thread *cur;
212	int ret = 0;
213
214	mutex_lock(&smpboot_threads_lock);
215	list_for_each_entry(cur, &hotplug_threads, list) {
216		ret = __smpboot_create_thread(cur, cpu);
217		if (ret)
218			break;
219	}
220	mutex_unlock(&smpboot_threads_lock);
221	return ret;
222}
223
224static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
225{
226	struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
227
228	if (!ht->selfparking)
229		kthread_unpark(tsk);
230}
231
232int smpboot_unpark_threads(unsigned int cpu)
233{
234	struct smp_hotplug_thread *cur;
235
236	mutex_lock(&smpboot_threads_lock);
237	list_for_each_entry(cur, &hotplug_threads, list)
238		smpboot_unpark_thread(cur, cpu);
239	mutex_unlock(&smpboot_threads_lock);
240	return 0;
241}
242
243static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu)
244{
245	struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
246
247	if (tsk && !ht->selfparking)
248		kthread_park(tsk);
249}
250
251int smpboot_park_threads(unsigned int cpu)
252{
253	struct smp_hotplug_thread *cur;
254
255	mutex_lock(&smpboot_threads_lock);
256	list_for_each_entry_reverse(cur, &hotplug_threads, list)
257		smpboot_park_thread(cur, cpu);
258	mutex_unlock(&smpboot_threads_lock);
259	return 0;
260}
261
262static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
263{
264	unsigned int cpu;
265
266	/* We need to destroy also the parked threads of offline cpus */
267	for_each_possible_cpu(cpu) {
268		struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
269
270		if (tsk) {
271			kthread_stop_put(tsk);
272			*per_cpu_ptr(ht->store, cpu) = NULL;
273		}
274	}
275}
276
277/**
278 * smpboot_register_percpu_thread - Register a per_cpu thread related
279 * 					    to hotplug
280 * @plug_thread:	Hotplug thread descriptor
281 *
282 * Creates and starts the threads on all online cpus.
283 */
284int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread)
285{
286	unsigned int cpu;
287	int ret = 0;
288
289	cpus_read_lock();
290	mutex_lock(&smpboot_threads_lock);
291	for_each_online_cpu(cpu) {
292		ret = __smpboot_create_thread(plug_thread, cpu);
293		if (ret) {
294			smpboot_destroy_threads(plug_thread);
295			goto out;
296		}
297		smpboot_unpark_thread(plug_thread, cpu);
298	}
299	list_add(&plug_thread->list, &hotplug_threads);
300out:
301	mutex_unlock(&smpboot_threads_lock);
302	cpus_read_unlock();
303	return ret;
304}
305EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread);
306
307/**
308 * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug
309 * @plug_thread:	Hotplug thread descriptor
310 *
311 * Stops all threads on all possible cpus.
312 */
313void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)
314{
315	cpus_read_lock();
316	mutex_lock(&smpboot_threads_lock);
317	list_del(&plug_thread->list);
318	smpboot_destroy_threads(plug_thread);
319	mutex_unlock(&smpboot_threads_lock);
320	cpus_read_unlock();
321}
322EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread);