kthread.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Kernel thread helper functions.
   3 *   Copyright (C) 2004 IBM Corporation, Rusty Russell.
   4 *   Copyright (C) 2009 Red Hat, Inc.
   5 *
   6 * Creation is done via kthreadd, so that we get a clean environment
   7 * even if we're invoked from userspace (think modprobe, hotplug cpu,
   8 * etc.).
   9 */
  10#include <uapi/linux/sched/types.h>
  11#include <linux/mm.h>
  12#include <linux/mmu_context.h>
  13#include <linux/sched.h>
  14#include <linux/sched/mm.h>
  15#include <linux/sched/task.h>
  16#include <linux/kthread.h>
  17#include <linux/completion.h>
  18#include <linux/err.h>
  19#include <linux/cgroup.h>
  20#include <linux/cpuset.h>
  21#include <linux/unistd.h>
  22#include <linux/file.h>
  23#include <linux/export.h>
  24#include <linux/mutex.h>
  25#include <linux/slab.h>
  26#include <linux/freezer.h>
  27#include <linux/ptrace.h>
  28#include <linux/uaccess.h>
  29#include <linux/numa.h>
  30#include <linux/sched/isolation.h>
  31#include <trace/events/sched.h>
  32
  33
  34static DEFINE_SPINLOCK(kthread_create_lock);
  35static LIST_HEAD(kthread_create_list);
  36struct task_struct *kthreadd_task;
  37
  38static LIST_HEAD(kthreads_hotplug);
  39static DEFINE_MUTEX(kthreads_hotplug_lock);
  40
  41struct kthread_create_info
  42{
  43	/* Information passed to kthread() from kthreadd. */
  44	char *full_name;
  45	int (*threadfn)(void *data);
  46	void *data;
  47	int node;
  48
  49	/* Result passed back to kthread_create() from kthreadd. */
  50	struct task_struct *result;
  51	struct completion *done;
  52
  53	struct list_head list;
  54};
  55
  56struct kthread {
  57	unsigned long flags;
  58	unsigned int cpu;
  59	unsigned int node;
  60	int started;
  61	int result;
  62	int (*threadfn)(void *);
  63	void *data;
  64	struct completion parked;
  65	struct completion exited;
  66#ifdef CONFIG_BLK_CGROUP
  67	struct cgroup_subsys_state *blkcg_css;
  68#endif
  69	/* To store the full name if task comm is truncated. */
  70	char *full_name;
  71	struct task_struct *task;
  72	struct list_head hotplug_node;
  73	struct cpumask *preferred_affinity;
  74};
  75
  76enum KTHREAD_BITS {
  77	KTHREAD_IS_PER_CPU = 0,
  78	KTHREAD_SHOULD_STOP,
  79	KTHREAD_SHOULD_PARK,
  80};
  81
  82static inline struct kthread *to_kthread(struct task_struct *k)
  83{
  84	WARN_ON(!(k->flags & PF_KTHREAD));
  85	return k->worker_private;
  86}
  87
  88void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk)
  89{
  90	struct kthread *kthread = to_kthread(tsk);
  91
  92	if (!kthread || !kthread->full_name) {
  93		strscpy(buf, tsk->comm, buf_size);
  94		return;
  95	}
  96
  97	strscpy_pad(buf, kthread->full_name, buf_size);
  98}
  99
 100bool set_kthread_struct(struct task_struct *p)
 101{
 102	struct kthread *kthread;
 103
 104	if (WARN_ON_ONCE(to_kthread(p)))
 105		return false;
 106
 107	kthread = kzalloc(sizeof(*kthread), GFP_KERNEL);
 108	if (!kthread)
 109		return false;
 110
 111	init_completion(&kthread->exited);
 112	init_completion(&kthread->parked);
 113	INIT_LIST_HEAD(&kthread->hotplug_node);
 114	p->vfork_done = &kthread->exited;
 115
 116	kthread->task = p;
 117	kthread->node = tsk_fork_get_node(current);
 118	p->worker_private = kthread;
 119	return true;
 120}
 121
 122void free_kthread_struct(struct task_struct *k)
 123{
 124	struct kthread *kthread;
 125
 126	/*
 127	 * Can be NULL if kmalloc() in set_kthread_struct() failed.
 128	 */
 129	kthread = to_kthread(k);
 130	if (!kthread)
 131		return;
 132
 133#ifdef CONFIG_BLK_CGROUP
 134	WARN_ON_ONCE(kthread->blkcg_css);
 135#endif
 136	k->worker_private = NULL;
 137	kfree(kthread->full_name);
 138	kfree(kthread);
 139}
 140
 141/**
 142 * kthread_should_stop - should this kthread return now?
 143 *
 144 * When someone calls kthread_stop() on your kthread, it will be woken
 145 * and this will return true.  You should then return, and your return
 146 * value will be passed through to kthread_stop().
 147 */
 148bool kthread_should_stop(void)
 149{
 150	return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
 151}
 152EXPORT_SYMBOL(kthread_should_stop);
 153
 154static bool __kthread_should_park(struct task_struct *k)
 155{
 156	return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);
 157}
 158
 159/**
 160 * kthread_should_park - should this kthread park now?
 161 *
 162 * When someone calls kthread_park() on your kthread, it will be woken
 163 * and this will return true.  You should then do the necessary
 164 * cleanup and call kthread_parkme()
 165 *
 166 * Similar to kthread_should_stop(), but this keeps the thread alive
 167 * and in a park position. kthread_unpark() "restarts" the thread and
 168 * calls the thread function again.
 169 */
 170bool kthread_should_park(void)
 171{
 172	return __kthread_should_park(current);
 173}
 174EXPORT_SYMBOL_GPL(kthread_should_park);
 175
 176bool kthread_should_stop_or_park(void)
 177{
 178	struct kthread *kthread = tsk_is_kthread(current);
 179
 180	if (!kthread)
 181		return false;
 182
 183	return kthread->flags & (BIT(KTHREAD_SHOULD_STOP) | BIT(KTHREAD_SHOULD_PARK));
 184}
 185
 186/**
 187 * kthread_freezable_should_stop - should this freezable kthread return now?
 188 * @was_frozen: optional out parameter, indicates whether %current was frozen
 189 *
 190 * kthread_should_stop() for freezable kthreads, which will enter
 191 * refrigerator if necessary.  This function is safe from kthread_stop() /
 192 * freezer deadlock and freezable kthreads should use this function instead
 193 * of calling try_to_freeze() directly.
 194 */
 195bool kthread_freezable_should_stop(bool *was_frozen)
 196{
 197	bool frozen = false;
 198
 199	might_sleep();
 200
 201	if (unlikely(freezing(current)))
 202		frozen = __refrigerator(true);
 203
 204	if (was_frozen)
 205		*was_frozen = frozen;
 206
 207	return kthread_should_stop();
 208}
 209EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
 210
 211/**
 212 * kthread_func - return the function specified on kthread creation
 213 * @task: kthread task in question
 214 *
 215 * Returns NULL if the task is not a kthread.
 216 */
 217void *kthread_func(struct task_struct *task)
 218{
 219	struct kthread *kthread = tsk_is_kthread(task);
 220	if (kthread)
 221		return kthread->threadfn;
 222	return NULL;
 223}
 224EXPORT_SYMBOL_GPL(kthread_func);
 225
 226/**
 227 * kthread_data - return data value specified on kthread creation
 228 * @task: kthread task in question
 229 *
 230 * Return the data value specified when kthread @task was created.
 231 * The caller is responsible for ensuring the validity of @task when
 232 * calling this function.
 233 */
 234void *kthread_data(struct task_struct *task)
 235{
 236	return to_kthread(task)->data;
 237}
 238EXPORT_SYMBOL_GPL(kthread_data);
 239
 240/**
 241 * kthread_probe_data - speculative version of kthread_data()
 242 * @task: possible kthread task in question
 243 *
 244 * @task could be a kthread task.  Return the data value specified when it
 245 * was created if accessible.  If @task isn't a kthread task or its data is
 246 * inaccessible for any reason, %NULL is returned.  This function requires
 247 * that @task itself is safe to dereference.
 248 */
 249void *kthread_probe_data(struct task_struct *task)
 250{
 251	struct kthread *kthread = tsk_is_kthread(task);
 252	void *data = NULL;
 253
 254	if (kthread)
 255		copy_from_kernel_nofault(&data, &kthread->data, sizeof(data));
 256	return data;
 257}
 258
 259static void __kthread_parkme(struct kthread *self)
 260{
 261	for (;;) {
 262		/*
 263		 * TASK_PARKED is a special state; we must serialize against
 264		 * possible pending wakeups to avoid store-store collisions on
 265		 * task->state.
 266		 *
 267		 * Such a collision might possibly result in the task state
 268		 * changin from TASK_PARKED and us failing the
 269		 * wait_task_inactive() in kthread_park().
 270		 */
 271		set_special_state(TASK_PARKED);
 272		if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))
 273			break;
 274
 275		/*
 276		 * Thread is going to call schedule(), do not preempt it,
 277		 * or the caller of kthread_park() may spend more time in
 278		 * wait_task_inactive().
 279		 */
 280		preempt_disable();
 281		complete(&self->parked);
 282		schedule_preempt_disabled();
 283		preempt_enable();
 284	}
 285	__set_current_state(TASK_RUNNING);
 286}
 287
 288void kthread_parkme(void)
 289{
 290	__kthread_parkme(to_kthread(current));
 291}
 292EXPORT_SYMBOL_GPL(kthread_parkme);
 293
 294void kthread_do_exit(struct kthread *kthread, long result)
 295{
 296	kthread->result = result;
 297	if (!list_empty(&kthread->hotplug_node)) {
 298		mutex_lock(&kthreads_hotplug_lock);
 299		list_del(&kthread->hotplug_node);
 300		mutex_unlock(&kthreads_hotplug_lock);
 301
 302		if (kthread->preferred_affinity) {
 303			kfree(kthread->preferred_affinity);
 304			kthread->preferred_affinity = NULL;
 305		}
 306	}
 307}
 308
 309/**
 310 * kthread_complete_and_exit - Exit the current kthread.
 311 * @comp: Completion to complete
 312 * @code: The integer value to return to kthread_stop().
 313 *
 314 * If present, complete @comp and then return code to kthread_stop().
 315 *
 316 * A kernel thread whose module may be removed after the completion of
 317 * @comp can use this function to exit safely.
 318 *
 319 * Does not return.
 320 */
 321void __noreturn kthread_complete_and_exit(struct completion *comp, long code)
 322{
 323	if (comp)
 324		complete(comp);
 325
 326	kthread_exit(code);
 327}
 328EXPORT_SYMBOL(kthread_complete_and_exit);
 329
 330static void kthread_fetch_affinity(struct kthread *kthread, struct cpumask *cpumask)
 331{
 332	const struct cpumask *pref;
 333
 334	if (kthread->preferred_affinity) {
 335		pref = kthread->preferred_affinity;
 336	} else {
 337		if (WARN_ON_ONCE(kthread->node == NUMA_NO_NODE))
 338			return;
 339		pref = cpumask_of_node(kthread->node);
 340	}
 341
 342	cpumask_and(cpumask, pref, housekeeping_cpumask(HK_TYPE_KTHREAD));
 343	if (cpumask_empty(cpumask))
 344		cpumask_copy(cpumask, housekeeping_cpumask(HK_TYPE_KTHREAD));
 345}
 346
 347static void kthread_affine_node(void)
 348{
 349	struct kthread *kthread = to_kthread(current);
 350	cpumask_var_t affinity;
 351
 352	WARN_ON_ONCE(kthread_is_per_cpu(current));
 353
 354	if (kthread->node == NUMA_NO_NODE) {
 355		housekeeping_affine(current, HK_TYPE_KTHREAD);
 356	} else {
 357		if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) {
 358			WARN_ON_ONCE(1);
 359			return;
 360		}
 361
 362		mutex_lock(&kthreads_hotplug_lock);
 363		WARN_ON_ONCE(!list_empty(&kthread->hotplug_node));
 364		list_add_tail(&kthread->hotplug_node, &kthreads_hotplug);
 365		/*
 366		 * The node cpumask is racy when read from kthread() but:
 367		 * - a racing CPU going down will either fail on the subsequent
 368		 *   call to set_cpus_allowed_ptr() or be migrated to housekeepers
 369		 *   afterwards by the scheduler.
 370		 * - a racing CPU going up will be handled by kthreads_online_cpu()
 371		 */
 372		kthread_fetch_affinity(kthread, affinity);
 373		set_cpus_allowed_ptr(current, affinity);
 374		mutex_unlock(&kthreads_hotplug_lock);
 375
 376		free_cpumask_var(affinity);
 377	}
 378}
 379
 380static int kthread(void *_create)
 381{
 382	static const struct sched_param param = { .sched_priority = 0 };
 383	/* Copy data: it's on kthread's stack */
 384	struct kthread_create_info *create = _create;
 385	int (*threadfn)(void *data) = create->threadfn;
 386	void *data = create->data;
 387	struct completion *done;
 388	struct kthread *self;
 389	int ret;
 390
 391	self = to_kthread(current);
 392
 393	/* Release the structure when caller killed by a fatal signal. */
 394	done = xchg(&create->done, NULL);
 395	if (!done) {
 396		kfree(create->full_name);
 397		kfree(create);
 398		kthread_exit(-EINTR);
 399	}
 400
 401	self->full_name = create->full_name;
 402	self->threadfn = threadfn;
 403	self->data = data;
 404
 405	/*
 406	 * The new thread inherited kthreadd's priority and CPU mask. Reset
 407	 * back to default in case they have been changed.
 408	 */
 409	sched_setscheduler_nocheck(current, SCHED_NORMAL, &param);
 410
 411	/* OK, tell user we're spawned, wait for stop or wakeup */
 412	__set_current_state(TASK_UNINTERRUPTIBLE);
 413	create->result = current;
 414	/*
 415	 * Thread is going to call schedule(), do not preempt it,
 416	 * or the creator may spend more time in wait_task_inactive().
 417	 */
 418	preempt_disable();
 419	complete(done);
 420	schedule_preempt_disabled();
 421	preempt_enable();
 422
 423	self->started = 1;
 424
 425	if (!(current->flags & PF_NO_SETAFFINITY) && !self->preferred_affinity)
 426		kthread_affine_node();
 427
 428	ret = -EINTR;
 429	if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {
 430		cgroup_kthread_ready();
 431		__kthread_parkme(self);
 432		ret = threadfn(data);
 433	}
 434	kthread_exit(ret);
 435}
 436
 437/* called from kernel_clone() to get node information for about to be created task */
 438int tsk_fork_get_node(struct task_struct *tsk)
 439{
 440#ifdef CONFIG_NUMA
 441	if (tsk == kthreadd_task)
 442		return tsk->pref_node_fork;
 443#endif
 444	return NUMA_NO_NODE;
 445}
 446
 447static void create_kthread(struct kthread_create_info *create)
 448{
 449	int pid;
 450
 451#ifdef CONFIG_NUMA
 452	current->pref_node_fork = create->node;
 453#endif
 454	/* We want our own signal handler (we take no signals by default). */
 455	pid = kernel_thread(kthread, create, create->full_name,
 456			    CLONE_FS | CLONE_FILES | SIGCHLD);
 457	if (pid < 0) {
 458		/* Release the structure when caller killed by a fatal signal. */
 459		struct completion *done = xchg(&create->done, NULL);
 460
 461		kfree(create->full_name);
 462		if (!done) {
 463			kfree(create);
 464			return;
 465		}
 466		create->result = ERR_PTR(pid);
 467		complete(done);
 468	}
 469}
 470
 471static __printf(4, 0)
 472struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
 473						    void *data, int node,
 474						    const char namefmt[],
 475						    va_list args)
 476{
 477	DECLARE_COMPLETION_ONSTACK(done);
 478	struct task_struct *task;
 479	struct kthread_create_info *create = kmalloc(sizeof(*create),
 480						     GFP_KERNEL);
 481
 482	if (!create)
 483		return ERR_PTR(-ENOMEM);
 484	create->threadfn = threadfn;
 485	create->data = data;
 486	create->node = node;
 487	create->done = &done;
 488	create->full_name = kvasprintf(GFP_KERNEL, namefmt, args);
 489	if (!create->full_name) {
 490		task = ERR_PTR(-ENOMEM);
 491		goto free_create;
 492	}
 493
 494	spin_lock(&kthread_create_lock);
 495	list_add_tail(&create->list, &kthread_create_list);
 496	spin_unlock(&kthread_create_lock);
 497
 498	wake_up_process(kthreadd_task);
 499	/*
 500	 * Wait for completion in killable state, for I might be chosen by
 501	 * the OOM killer while kthreadd is trying to allocate memory for
 502	 * new kernel thread.
 503	 */
 504	if (unlikely(wait_for_completion_killable(&done))) {
 505		/*
 506		 * If I was killed by a fatal signal before kthreadd (or new
 507		 * kernel thread) calls complete(), leave the cleanup of this
 508		 * structure to that thread.
 509		 */
 510		if (xchg(&create->done, NULL))
 511			return ERR_PTR(-EINTR);
 512		/*
 513		 * kthreadd (or new kernel thread) will call complete()
 514		 * shortly.
 515		 */
 516		wait_for_completion(&done);
 517	}
 518	task = create->result;
 519free_create:
 520	kfree(create);
 521	return task;
 522}
 523
 524/**
 525 * kthread_create_on_node - create a kthread.
 526 * @threadfn: the function to run until signal_pending(current).
 527 * @data: data ptr for @threadfn.
 528 * @node: task and thread structures for the thread are allocated on this node
 529 * @namefmt: printf-style name for the thread.
 530 *
 531 * Description: This helper function creates and names a kernel
 532 * thread.  The thread will be stopped: use wake_up_process() to start
 533 * it.  See also kthread_run().  The new thread has SCHED_NORMAL policy and
 534 * is affine to all CPUs.
 535 *
 536 * If thread is going to be bound on a particular cpu, give its node
 537 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
 538 * When woken, the thread will run @threadfn() with @data as its
 539 * argument. @threadfn() can either return directly if it is a
 540 * standalone thread for which no one will call kthread_stop(), or
 541 * return when 'kthread_should_stop()' is true (which means
 542 * kthread_stop() has been called).  The return value should be zero
 543 * or a negative error number; it will be passed to kthread_stop().
 544 *
 545 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
 546 */
 547struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
 548					   void *data, int node,
 549					   const char namefmt[],
 550					   ...)
 551{
 552	struct task_struct *task;
 553	va_list args;
 554
 555	va_start(args, namefmt);
 556	task = __kthread_create_on_node(threadfn, data, node, namefmt, args);
 557	va_end(args);
 558
 559	return task;
 560}
 561EXPORT_SYMBOL(kthread_create_on_node);
 562
 563static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, unsigned int state)
 564{
 565	if (!wait_task_inactive(p, state)) {
 566		WARN_ON(1);
 567		return;
 568	}
 569
 570	scoped_guard (raw_spinlock_irqsave, &p->pi_lock)
 571		set_cpus_allowed_force(p, mask);
 572
 573	/* It's safe because the task is inactive. */
 574	p->flags |= PF_NO_SETAFFINITY;
 575}
 576
 577static void __kthread_bind(struct task_struct *p, unsigned int cpu, unsigned int state)
 578{
 579	__kthread_bind_mask(p, cpumask_of(cpu), state);
 580}
 581
 582void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
 583{
 584	struct kthread *kthread = to_kthread(p);
 585	__kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
 586	WARN_ON_ONCE(kthread->started);
 587}
 588
 589/**
 590 * kthread_bind - bind a just-created kthread to a cpu.
 591 * @p: thread created by kthread_create().
 592 * @cpu: cpu (might not be online, must be possible) for @k to run on.
 593 *
 594 * Description: This function is equivalent to set_cpus_allowed(),
 595 * except that @cpu doesn't need to be online, and the thread must be
 596 * stopped (i.e., just returned from kthread_create()).
 597 */
 598void kthread_bind(struct task_struct *p, unsigned int cpu)
 599{
 600	struct kthread *kthread = to_kthread(p);
 601	__kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
 602	WARN_ON_ONCE(kthread->started);
 603}
 604EXPORT_SYMBOL(kthread_bind);
 605
 606/**
 607 * kthread_create_on_cpu - Create a cpu bound kthread
 608 * @threadfn: the function to run until signal_pending(current).
 609 * @data: data ptr for @threadfn.
 610 * @cpu: The cpu on which the thread should be bound,
 611 * @namefmt: printf-style name for the thread. Format is restricted
 612 *	     to "name.*%u". Code fills in cpu number.
 613 *
 614 * Description: This helper function creates and names a kernel thread
 615 */
 616struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
 617					  void *data, unsigned int cpu,
 618					  const char *namefmt)
 619{
 620	struct task_struct *p;
 621
 622	p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
 623				   cpu);
 624	if (IS_ERR(p))
 625		return p;
 626	kthread_bind(p, cpu);
 627	/* CPU hotplug need to bind once again when unparking the thread. */
 628	to_kthread(p)->cpu = cpu;
 629	return p;
 630}
 631EXPORT_SYMBOL(kthread_create_on_cpu);
 632
 633void kthread_set_per_cpu(struct task_struct *k, int cpu)
 634{
 635	struct kthread *kthread = to_kthread(k);
 636	if (!kthread)
 637		return;
 638
 639	WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY));
 640
 641	if (cpu < 0) {
 642		clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
 643		return;
 644	}
 645
 646	kthread->cpu = cpu;
 647	set_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
 648}
 649
 650bool kthread_is_per_cpu(struct task_struct *p)
 651{
 652	struct kthread *kthread = tsk_is_kthread(p);
 653	if (!kthread)
 654		return false;
 655
 656	return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
 657}
 658
 659/**
 660 * kthread_unpark - unpark a thread created by kthread_create().
 661 * @k:		thread created by kthread_create().
 662 *
 663 * Sets kthread_should_park() for @k to return false, wakes it, and
 664 * waits for it to return. If the thread is marked percpu then its
 665 * bound to the cpu again.
 666 */
 667void kthread_unpark(struct task_struct *k)
 668{
 669	struct kthread *kthread = to_kthread(k);
 670
 671	if (!test_bit(KTHREAD_SHOULD_PARK, &kthread->flags))
 672		return;
 673	/*
 674	 * Newly created kthread was parked when the CPU was offline.
 675	 * The binding was lost and we need to set it again.
 676	 */
 677	if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
 678		__kthread_bind(k, kthread->cpu, TASK_PARKED);
 679
 680	clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
 681	/*
 682	 * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup.
 683	 */
 684	wake_up_state(k, TASK_PARKED);
 685}
 686EXPORT_SYMBOL_GPL(kthread_unpark);
 687
 688/**
 689 * kthread_park - park a thread created by kthread_create().
 690 * @k: thread created by kthread_create().
 691 *
 692 * Sets kthread_should_park() for @k to return true, wakes it, and
 693 * waits for it to return. This can also be called after kthread_create()
 694 * instead of calling wake_up_process(): the thread will park without
 695 * calling threadfn().
 696 *
 697 * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
 698 * If called by the kthread itself just the park bit is set.
 699 */
 700int kthread_park(struct task_struct *k)
 701{
 702	struct kthread *kthread = to_kthread(k);
 703
 704	if (WARN_ON(k->flags & PF_EXITING))
 705		return -ENOSYS;
 706
 707	if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags)))
 708		return -EBUSY;
 709
 710	set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
 711	if (k != current) {
 712		wake_up_process(k);
 713		/*
 714		 * Wait for __kthread_parkme() to complete(), this means we
 715		 * _will_ have TASK_PARKED and are about to call schedule().
 716		 */
 717		wait_for_completion(&kthread->parked);
 718		/*
 719		 * Now wait for that schedule() to complete and the task to
 720		 * get scheduled out.
 721		 */
 722		WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED));
 723	}
 724
 725	return 0;
 726}
 727EXPORT_SYMBOL_GPL(kthread_park);
 728
 729/**
 730 * kthread_stop - stop a thread created by kthread_create().
 731 * @k: thread created by kthread_create().
 732 *
 733 * Sets kthread_should_stop() for @k to return true, wakes it, and
 734 * waits for it to exit. This can also be called after kthread_create()
 735 * instead of calling wake_up_process(): the thread will exit without
 736 * calling threadfn().
 737 *
 738 * If threadfn() may call kthread_exit() itself, the caller must ensure
 739 * task_struct can't go away.
 740 *
 741 * Returns the result of threadfn(), or %-EINTR if wake_up_process()
 742 * was never called.
 743 */
 744int kthread_stop(struct task_struct *k)
 745{
 746	struct kthread *kthread;
 747	int ret;
 748
 749	trace_sched_kthread_stop(k);
 750
 751	get_task_struct(k);
 752	kthread = to_kthread(k);
 753	set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
 754	kthread_unpark(k);
 755	set_tsk_thread_flag(k, TIF_NOTIFY_SIGNAL);
 756	wake_up_process(k);
 757	wait_for_completion(&kthread->exited);
 758	ret = kthread->result;
 759	put_task_struct(k);
 760
 761	trace_sched_kthread_stop_ret(ret);
 762	return ret;
 763}
 764EXPORT_SYMBOL(kthread_stop);
 765
 766/**
 767 * kthread_stop_put - stop a thread and put its task struct
 768 * @k: thread created by kthread_create().
 769 *
 770 * Stops a thread created by kthread_create() and put its task_struct.
 771 * Only use when holding an extra task struct reference obtained by
 772 * calling get_task_struct().
 773 */
 774int kthread_stop_put(struct task_struct *k)
 775{
 776	int ret;
 777
 778	ret = kthread_stop(k);
 779	put_task_struct(k);
 780	return ret;
 781}
 782EXPORT_SYMBOL(kthread_stop_put);
 783
 784int kthreadd(void *unused)
 785{
 786	static const char comm[TASK_COMM_LEN] = "kthreadd";
 787	struct task_struct *tsk = current;
 788
 789	/* Setup a clean context for our children to inherit. */
 790	set_task_comm(tsk, comm);
 791	ignore_signals(tsk);
 792	set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_TYPE_KTHREAD));
 793	set_mems_allowed(node_states[N_MEMORY]);
 794
 795	current->flags |= PF_NOFREEZE;
 796	cgroup_init_kthreadd();
 797
 798	for (;;) {
 799		set_current_state(TASK_INTERRUPTIBLE);
 800		if (list_empty(&kthread_create_list))
 801			schedule();
 802		__set_current_state(TASK_RUNNING);
 803
 804		spin_lock(&kthread_create_lock);
 805		while (!list_empty(&kthread_create_list)) {
 806			struct kthread_create_info *create;
 807
 808			create = list_entry(kthread_create_list.next,
 809					    struct kthread_create_info, list);
 810			list_del_init(&create->list);
 811			spin_unlock(&kthread_create_lock);
 812
 813			create_kthread(create);
 814
 815			spin_lock(&kthread_create_lock);
 816		}
 817		spin_unlock(&kthread_create_lock);
 818	}
 819
 820	return 0;
 821}
 822
 823int kthread_affine_preferred(struct task_struct *p, const struct cpumask *mask)
 824{
 825	struct kthread *kthread = to_kthread(p);
 826	cpumask_var_t affinity;
 827	int ret = 0;
 828
 829	if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE) || kthread->started) {
 830		WARN_ON(1);
 831		return -EINVAL;
 832	}
 833
 834	WARN_ON_ONCE(kthread->preferred_affinity);
 835
 836	if (!zalloc_cpumask_var(&affinity, GFP_KERNEL))
 837		return -ENOMEM;
 838
 839	kthread->preferred_affinity = kzalloc(sizeof(struct cpumask), GFP_KERNEL);
 840	if (!kthread->preferred_affinity) {
 841		ret = -ENOMEM;
 842		goto out;
 843	}
 844
 845	mutex_lock(&kthreads_hotplug_lock);
 846	cpumask_copy(kthread->preferred_affinity, mask);
 847	WARN_ON_ONCE(!list_empty(&kthread->hotplug_node));
 848	list_add_tail(&kthread->hotplug_node, &kthreads_hotplug);
 849	kthread_fetch_affinity(kthread, affinity);
 850
 851	scoped_guard (raw_spinlock_irqsave, &p->pi_lock)
 852		set_cpus_allowed_force(p, affinity);
 853
 854	mutex_unlock(&kthreads_hotplug_lock);
 855out:
 856	free_cpumask_var(affinity);
 857
 858	return ret;
 859}
 860EXPORT_SYMBOL_GPL(kthread_affine_preferred);
 861
 862/*
 863 * Re-affine kthreads according to their preferences
 864 * and the newly online CPU. The CPU down part is handled
 865 * by select_fallback_rq() which default re-affines to
 866 * housekeepers from other nodes in case the preferred
 867 * affinity doesn't apply anymore.
 868 */
 869static int kthreads_online_cpu(unsigned int cpu)
 870{
 871	cpumask_var_t affinity;
 872	struct kthread *k;
 873	int ret;
 874
 875	guard(mutex)(&kthreads_hotplug_lock);
 876
 877	if (list_empty(&kthreads_hotplug))
 878		return 0;
 879
 880	if (!zalloc_cpumask_var(&affinity, GFP_KERNEL))
 881		return -ENOMEM;
 882
 883	ret = 0;
 884
 885	list_for_each_entry(k, &kthreads_hotplug, hotplug_node) {
 886		if (WARN_ON_ONCE((k->task->flags & PF_NO_SETAFFINITY) ||
 887				 kthread_is_per_cpu(k->task))) {
 888			ret = -EINVAL;
 889			continue;
 890		}
 891		kthread_fetch_affinity(k, affinity);
 892		set_cpus_allowed_ptr(k->task, affinity);
 893	}
 894
 895	free_cpumask_var(affinity);
 896
 897	return ret;
 898}
 899
 900static int kthreads_init(void)
 901{
 902	return cpuhp_setup_state(CPUHP_AP_KTHREADS_ONLINE, "kthreads:online",
 903				kthreads_online_cpu, NULL);
 904}
 905early_initcall(kthreads_init);
 906
 907void __kthread_init_worker(struct kthread_worker *worker,
 908				const char *name,
 909				struct lock_class_key *key)
 910{
 911	memset(worker, 0, sizeof(struct kthread_worker));
 912	raw_spin_lock_init(&worker->lock);
 913	lockdep_set_class_and_name(&worker->lock, key, name);
 914	INIT_LIST_HEAD(&worker->work_list);
 915	INIT_LIST_HEAD(&worker->delayed_work_list);
 916}
 917EXPORT_SYMBOL_GPL(__kthread_init_worker);
 918
 919/**
 920 * kthread_worker_fn - kthread function to process kthread_worker
 921 * @worker_ptr: pointer to initialized kthread_worker
 922 *
 923 * This function implements the main cycle of kthread worker. It processes
 924 * work_list until it is stopped with kthread_stop(). It sleeps when the queue
 925 * is empty.
 926 *
 927 * The works are not allowed to keep any locks, disable preemption or interrupts
 928 * when they finish. There is defined a safe point for freezing when one work
 929 * finishes and before a new one is started.
 930 *
 931 * Also the works must not be handled by more than one worker at the same time,
 932 * see also kthread_queue_work().
 933 */
 934int kthread_worker_fn(void *worker_ptr)
 935{
 936	struct kthread_worker *worker = worker_ptr;
 937	struct kthread_work *work;
 938
 939	/*
 940	 * FIXME: Update the check and remove the assignment when all kthread
 941	 * worker users are created using kthread_create_worker*() functions.
 942	 */
 943	WARN_ON(worker->task && worker->task != current);
 944	worker->task = current;
 945
 946	if (worker->flags & KTW_FREEZABLE)
 947		set_freezable();
 948
 949repeat:
 950	set_current_state(TASK_INTERRUPTIBLE);	/* mb paired w/ kthread_stop */
 951
 952	if (kthread_should_stop()) {
 953		__set_current_state(TASK_RUNNING);
 954		raw_spin_lock_irq(&worker->lock);
 955		worker->task = NULL;
 956		raw_spin_unlock_irq(&worker->lock);
 957		return 0;
 958	}
 959
 960	work = NULL;
 961	raw_spin_lock_irq(&worker->lock);
 962	if (!list_empty(&worker->work_list)) {
 963		work = list_first_entry(&worker->work_list,
 964					struct kthread_work, node);
 965		list_del_init(&work->node);
 966	}
 967	worker->current_work = work;
 968	raw_spin_unlock_irq(&worker->lock);
 969
 970	if (work) {
 971		kthread_work_func_t func = work->func;
 972		__set_current_state(TASK_RUNNING);
 973		trace_sched_kthread_work_execute_start(work);
 974		work->func(work);
 975		/*
 976		 * Avoid dereferencing work after this point.  The trace
 977		 * event only cares about the address.
 978		 */
 979		trace_sched_kthread_work_execute_end(work, func);
 980	} else if (!freezing(current)) {
 981		schedule();
 982	} else {
 983		/*
 984		 * Handle the case where the current remains
 985		 * TASK_INTERRUPTIBLE. try_to_freeze() expects
 986		 * the current to be TASK_RUNNING.
 987		 */
 988		__set_current_state(TASK_RUNNING);
 989	}
 990
 991	try_to_freeze();
 992	cond_resched();
 993	goto repeat;
 994}
 995EXPORT_SYMBOL_GPL(kthread_worker_fn);
 996
 997static __printf(3, 0) struct kthread_worker *
 998__kthread_create_worker_on_node(unsigned int flags, int node,
 999				const char namefmt[], va_list args)
1000{
1001	struct kthread_worker *worker;
1002	struct task_struct *task;
1003
1004	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1005	if (!worker)
1006		return ERR_PTR(-ENOMEM);
1007
1008	kthread_init_worker(worker);
1009
1010	task = __kthread_create_on_node(kthread_worker_fn, worker,
1011					node, namefmt, args);
1012	if (IS_ERR(task))
1013		goto fail_task;
1014
1015	worker->flags = flags;
1016	worker->task = task;
1017
1018	return worker;
1019
1020fail_task:
1021	kfree(worker);
1022	return ERR_CAST(task);
1023}
1024
1025/**
1026 * kthread_create_worker_on_node - create a kthread worker
1027 * @flags: flags modifying the default behavior of the worker
1028 * @node: task structure for the thread is allocated on this node
1029 * @namefmt: printf-style name for the kthread worker (task).
1030 *
1031 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
1032 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
1033 * when the caller was killed by a fatal signal.
1034 */
1035struct kthread_worker *
1036kthread_create_worker_on_node(unsigned int flags, int node, const char namefmt[], ...)
1037{
1038	struct kthread_worker *worker;
1039	va_list args;
1040
1041	va_start(args, namefmt);
1042	worker = __kthread_create_worker_on_node(flags, node, namefmt, args);
1043	va_end(args);
1044
1045	return worker;
1046}
1047EXPORT_SYMBOL(kthread_create_worker_on_node);
1048
1049/**
1050 * kthread_create_worker_on_cpu - create a kthread worker and bind it
1051 *	to a given CPU and the associated NUMA node.
1052 * @cpu: CPU number
1053 * @flags: flags modifying the default behavior of the worker
1054 * @namefmt: printf-style name for the thread. Format is restricted
1055 *	     to "name.*%u". Code fills in cpu number.
1056 *
1057 * Use a valid CPU number if you want to bind the kthread worker
1058 * to the given CPU and the associated NUMA node.
1059 *
1060 * A good practice is to add the cpu number also into the worker name.
1061 * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu).
1062 *
1063 * CPU hotplug:
1064 * The kthread worker API is simple and generic. It just provides a way
1065 * to create, use, and destroy workers.
1066 *
1067 * It is up to the API user how to handle CPU hotplug. They have to decide
1068 * how to handle pending work items, prevent queuing new ones, and
1069 * restore the functionality when the CPU goes off and on. There are a
1070 * few catches:
1071 *
1072 *    - CPU affinity gets lost when it is scheduled on an offline CPU.
1073 *
1074 *    - The worker might not exist when the CPU was off when the user
1075 *      created the workers.
1076 *
1077 * Good practice is to implement two CPU hotplug callbacks and to
1078 * destroy/create the worker when the CPU goes down/up.
1079 *
1080 * Return:
1081 * The pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
1082 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
1083 * when the caller was killed by a fatal signal.
1084 */
1085struct kthread_worker *
1086kthread_create_worker_on_cpu(int cpu, unsigned int flags,
1087			     const char namefmt[])
1088{
1089	struct kthread_worker *worker;
1090
1091	worker = kthread_create_worker_on_node(flags, cpu_to_node(cpu), namefmt, cpu);
1092	if (!IS_ERR(worker))
1093		kthread_bind(worker->task, cpu);
1094
1095	return worker;
1096}
1097EXPORT_SYMBOL(kthread_create_worker_on_cpu);
1098
1099/*
1100 * Returns true when the work could not be queued at the moment.
1101 * It happens when it is already pending in a worker list
1102 * or when it is being cancelled.
1103 */
1104static inline bool queuing_blocked(struct kthread_worker *worker,
1105				   struct kthread_work *work)
1106{
1107	lockdep_assert_held(&worker->lock);
1108
1109	return !list_empty(&work->node) || work->canceling;
1110}
1111
1112static void kthread_insert_work_sanity_check(struct kthread_worker *worker,
1113					     struct kthread_work *work)
1114{
1115	lockdep_assert_held(&worker->lock);
1116	WARN_ON_ONCE(!list_empty(&work->node));
1117	/* Do not use a work with >1 worker, see kthread_queue_work() */
1118	WARN_ON_ONCE(work->worker && work->worker != worker);
1119}
1120
1121/* insert @work before @pos in @worker */
1122static void kthread_insert_work(struct kthread_worker *worker,
1123				struct kthread_work *work,
1124				struct list_head *pos)
1125{
1126	kthread_insert_work_sanity_check(worker, work);
1127
1128	trace_sched_kthread_work_queue_work(worker, work);
1129
1130	list_add_tail(&work->node, pos);
1131	work->worker = worker;
1132	if (!worker->current_work && likely(worker->task))
1133		wake_up_process(worker->task);
1134}
1135
1136/**
1137 * kthread_queue_work - queue a kthread_work
1138 * @worker: target kthread_worker
1139 * @work: kthread_work to queue
1140 *
1141 * Queue @work to work processor @task for async execution.  @task
1142 * must have been created with kthread_create_worker().  Returns %true
1143 * if @work was successfully queued, %false if it was already pending.
1144 *
1145 * Reinitialize the work if it needs to be used by another worker.
1146 * For example, when the worker was stopped and started again.
1147 */
1148bool kthread_queue_work(struct kthread_worker *worker,
1149			struct kthread_work *work)
1150{
1151	bool ret = false;
1152	unsigned long flags;
1153
1154	raw_spin_lock_irqsave(&worker->lock, flags);
1155	if (!queuing_blocked(worker, work)) {
1156		kthread_insert_work(worker, work, &worker->work_list);
1157		ret = true;
1158	}
1159	raw_spin_unlock_irqrestore(&worker->lock, flags);
1160	return ret;
1161}
1162EXPORT_SYMBOL_GPL(kthread_queue_work);
1163
1164/**
1165 * kthread_delayed_work_timer_fn - callback that queues the associated kthread
1166 *	delayed work when the timer expires.
1167 * @t: pointer to the expired timer
1168 *
1169 * The format of the function is defined by struct timer_list.
1170 * It should have been called from irqsafe timer with irq already off.
1171 */
1172void kthread_delayed_work_timer_fn(struct timer_list *t)
1173{
1174	struct kthread_delayed_work *dwork = timer_container_of(dwork, t,
1175								timer);
1176	struct kthread_work *work = &dwork->work;
1177	struct kthread_worker *worker = work->worker;
1178	unsigned long flags;
1179
1180	/*
1181	 * This might happen when a pending work is reinitialized.
1182	 * It means that it is used a wrong way.
1183	 */
1184	if (WARN_ON_ONCE(!worker))
1185		return;
1186
1187	raw_spin_lock_irqsave(&worker->lock, flags);
1188	/* Work must not be used with >1 worker, see kthread_queue_work(). */
1189	WARN_ON_ONCE(work->worker != worker);
1190
1191	/* Move the work from worker->delayed_work_list. */
1192	WARN_ON_ONCE(list_empty(&work->node));
1193	list_del_init(&work->node);
1194	if (!work->canceling)
1195		kthread_insert_work(worker, work, &worker->work_list);
1196
1197	raw_spin_unlock_irqrestore(&worker->lock, flags);
1198}
1199EXPORT_SYMBOL(kthread_delayed_work_timer_fn);
1200
1201static void __kthread_queue_delayed_work(struct kthread_worker *worker,
1202					 struct kthread_delayed_work *dwork,
1203					 unsigned long delay)
1204{
1205	struct timer_list *timer = &dwork->timer;
1206	struct kthread_work *work = &dwork->work;
1207
1208	WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
1209
1210	/*
1211	 * If @delay is 0, queue @dwork->work immediately.  This is for
1212	 * both optimization and correctness.  The earliest @timer can
1213	 * expire is on the closest next tick and delayed_work users depend
1214	 * on that there's no such delay when @delay is 0.
1215	 */
1216	if (!delay) {
1217		kthread_insert_work(worker, work, &worker->work_list);
1218		return;
1219	}
1220
1221	/* Be paranoid and try to detect possible races already now. */
1222	kthread_insert_work_sanity_check(worker, work);
1223
1224	list_add(&work->node, &worker->delayed_work_list);
1225	work->worker = worker;
1226	timer->expires = jiffies + delay;
1227	add_timer(timer);
1228}
1229
1230/**
1231 * kthread_queue_delayed_work - queue the associated kthread work
1232 *	after a delay.
1233 * @worker: target kthread_worker
1234 * @dwork: kthread_delayed_work to queue
1235 * @delay: number of jiffies to wait before queuing
1236 *
1237 * If the work has not been pending it starts a timer that will queue
1238 * the work after the given @delay. If @delay is zero, it queues the
1239 * work immediately.
1240 *
1241 * Return: %false if the @work has already been pending. It means that
1242 * either the timer was running or the work was queued. It returns %true
1243 * otherwise.
1244 */
1245bool kthread_queue_delayed_work(struct kthread_worker *worker,
1246				struct kthread_delayed_work *dwork,
1247				unsigned long delay)
1248{
1249	struct kthread_work *work = &dwork->work;
1250	unsigned long flags;
1251	bool ret = false;
1252
1253	raw_spin_lock_irqsave(&worker->lock, flags);
1254
1255	if (!queuing_blocked(worker, work)) {
1256		__kthread_queue_delayed_work(worker, dwork, delay);
1257		ret = true;
1258	}
1259
1260	raw_spin_unlock_irqrestore(&worker->lock, flags);
1261	return ret;
1262}
1263EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);
1264
1265struct kthread_flush_work {
1266	struct kthread_work	work;
1267	struct completion	done;
1268};
1269
1270static void kthread_flush_work_fn(struct kthread_work *work)
1271{
1272	struct kthread_flush_work *fwork =
1273		container_of(work, struct kthread_flush_work, work);
1274	complete(&fwork->done);
1275}
1276
1277/**
1278 * kthread_flush_work - flush a kthread_work
1279 * @work: work to flush
1280 *
1281 * If @work is queued or executing, wait for it to finish execution.
1282 */
1283void kthread_flush_work(struct kthread_work *work)
1284{
1285	struct kthread_flush_work fwork = {
1286		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
1287		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
1288	};
1289	struct kthread_worker *worker;
1290	bool noop = false;
1291
1292	worker = work->worker;
1293	if (!worker)
1294		return;
1295
1296	raw_spin_lock_irq(&worker->lock);
1297	/* Work must not be used with >1 worker, see kthread_queue_work(). */
1298	WARN_ON_ONCE(work->worker != worker);
1299
1300	if (!list_empty(&work->node))
1301		kthread_insert_work(worker, &fwork.work, work->node.next);
1302	else if (worker->current_work == work)
1303		kthread_insert_work(worker, &fwork.work,
1304				    worker->work_list.next);
1305	else
1306		noop = true;
1307
1308	raw_spin_unlock_irq(&worker->lock);
1309
1310	if (!noop)
1311		wait_for_completion(&fwork.done);
1312}
1313EXPORT_SYMBOL_GPL(kthread_flush_work);
1314
1315/*
1316 * Make sure that the timer is neither set nor running and could
1317 * not manipulate the work list_head any longer.
1318 *
1319 * The function is called under worker->lock. The lock is temporary
1320 * released but the timer can't be set again in the meantime.
1321 */
1322static void kthread_cancel_delayed_work_timer(struct kthread_work *work,
1323					      unsigned long *flags)
1324{
1325	struct kthread_delayed_work *dwork =
1326		container_of(work, struct kthread_delayed_work, work);
1327	struct kthread_worker *worker = work->worker;
1328
1329	/*
1330	 * timer_delete_sync() must be called to make sure that the timer
1331	 * callback is not running. The lock must be temporary released
1332	 * to avoid a deadlock with the callback. In the meantime,
1333	 * any queuing is blocked by setting the canceling counter.
1334	 */
1335	work->canceling++;
1336	raw_spin_unlock_irqrestore(&worker->lock, *flags);
1337	timer_delete_sync(&dwork->timer);
1338	raw_spin_lock_irqsave(&worker->lock, *flags);
1339	work->canceling--;
1340}
1341
1342/*
1343 * This function removes the work from the worker queue.
1344 *
1345 * It is called under worker->lock. The caller must make sure that
1346 * the timer used by delayed work is not running, e.g. by calling
1347 * kthread_cancel_delayed_work_timer().
1348 *
1349 * The work might still be in use when this function finishes. See the
1350 * current_work proceed by the worker.
1351 *
1352 * Return: %true if @work was pending and successfully canceled,
1353 *	%false if @work was not pending
1354 */
1355static bool __kthread_cancel_work(struct kthread_work *work)
1356{
1357	/*
1358	 * Try to remove the work from a worker list. It might either
1359	 * be from worker->work_list or from worker->delayed_work_list.
1360	 */
1361	if (!list_empty(&work->node)) {
1362		list_del_init(&work->node);
1363		return true;
1364	}
1365
1366	return false;
1367}
1368
1369/**
1370 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
1371 * @worker: kthread worker to use
1372 * @dwork: kthread delayed work to queue
1373 * @delay: number of jiffies to wait before queuing
1374 *
1375 * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise,
1376 * modify @dwork's timer so that it expires after @delay. If @delay is zero,
1377 * @work is guaranteed to be queued immediately.
1378 *
1379 * Return: %false if @dwork was idle and queued, %true otherwise.
1380 *
1381 * A special case is when the work is being canceled in parallel.
1382 * It might be caused either by the real kthread_cancel_delayed_work_sync()
1383 * or yet another kthread_mod_delayed_work() call. We let the other command
1384 * win and return %true here. The return value can be used for reference
1385 * counting and the number of queued works stays the same. Anyway, the caller
1386 * is supposed to synchronize these operations a reasonable way.
1387 *
1388 * This function is safe to call from any context including IRQ handler.
1389 * See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
1390 * for details.
1391 */
1392bool kthread_mod_delayed_work(struct kthread_worker *worker,
1393			      struct kthread_delayed_work *dwork,
1394			      unsigned long delay)
1395{
1396	struct kthread_work *work = &dwork->work;
1397	unsigned long flags;
1398	int ret;
1399
1400	raw_spin_lock_irqsave(&worker->lock, flags);
1401
1402	/* Do not bother with canceling when never queued. */
1403	if (!work->worker) {
1404		ret = false;
1405		goto fast_queue;
1406	}
1407
1408	/* Work must not be used with >1 worker, see kthread_queue_work() */
1409	WARN_ON_ONCE(work->worker != worker);
1410
1411	/*
1412	 * Temporary cancel the work but do not fight with another command
1413	 * that is canceling the work as well.
1414	 *
1415	 * It is a bit tricky because of possible races with another
1416	 * mod_delayed_work() and cancel_delayed_work() callers.
1417	 *
1418	 * The timer must be canceled first because worker->lock is released
1419	 * when doing so. But the work can be removed from the queue (list)
1420	 * only when it can be queued again so that the return value can
1421	 * be used for reference counting.
1422	 */
1423	kthread_cancel_delayed_work_timer(work, &flags);
1424	if (work->canceling) {
1425		/* The number of works in the queue does not change. */
1426		ret = true;
1427		goto out;
1428	}
1429	ret = __kthread_cancel_work(work);
1430
1431fast_queue:
1432	__kthread_queue_delayed_work(worker, dwork, delay);
1433out:
1434	raw_spin_unlock_irqrestore(&worker->lock, flags);
1435	return ret;
1436}
1437EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);
1438
1439static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork)
1440{
1441	struct kthread_worker *worker = work->worker;
1442	unsigned long flags;
1443	int ret = false;
1444
1445	if (!worker)
1446		goto out;
1447
1448	raw_spin_lock_irqsave(&worker->lock, flags);
1449	/* Work must not be used with >1 worker, see kthread_queue_work(). */
1450	WARN_ON_ONCE(work->worker != worker);
1451
1452	if (is_dwork)
1453		kthread_cancel_delayed_work_timer(work, &flags);
1454
1455	ret = __kthread_cancel_work(work);
1456
1457	if (worker->current_work != work)
1458		goto out_fast;
1459
1460	/*
1461	 * The work is in progress and we need to wait with the lock released.
1462	 * In the meantime, block any queuing by setting the canceling counter.
1463	 */
1464	work->canceling++;
1465	raw_spin_unlock_irqrestore(&worker->lock, flags);
1466	kthread_flush_work(work);
1467	raw_spin_lock_irqsave(&worker->lock, flags);
1468	work->canceling--;
1469
1470out_fast:
1471	raw_spin_unlock_irqrestore(&worker->lock, flags);
1472out:
1473	return ret;
1474}
1475
1476/**
1477 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
1478 * @work: the kthread work to cancel
1479 *
1480 * Cancel @work and wait for its execution to finish.  This function
1481 * can be used even if the work re-queues itself. On return from this
1482 * function, @work is guaranteed to be not pending or executing on any CPU.
1483 *
1484 * kthread_cancel_work_sync(&delayed_work->work) must not be used for
1485 * delayed_work's. Use kthread_cancel_delayed_work_sync() instead.
1486 *
1487 * The caller must ensure that the worker on which @work was last
1488 * queued can't be destroyed before this function returns.
1489 *
1490 * Return: %true if @work was pending, %false otherwise.
1491 */
1492bool kthread_cancel_work_sync(struct kthread_work *work)
1493{
1494	return __kthread_cancel_work_sync(work, false);
1495}
1496EXPORT_SYMBOL_GPL(kthread_cancel_work_sync);
1497
1498/**
1499 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
1500 *	wait for it to finish.
1501 * @dwork: the kthread delayed work to cancel
1502 *
1503 * This is kthread_cancel_work_sync() for delayed works.
1504 *
1505 * Return: %true if @dwork was pending, %false otherwise.
1506 */
1507bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork)
1508{
1509	return __kthread_cancel_work_sync(&dwork->work, true);
1510}
1511EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync);
1512
1513/**
1514 * kthread_flush_worker - flush all current works on a kthread_worker
1515 * @worker: worker to flush
1516 *
1517 * Wait until all currently executing or pending works on @worker are
1518 * finished.
1519 */
1520void kthread_flush_worker(struct kthread_worker *worker)
1521{
1522	struct kthread_flush_work fwork = {
1523		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
1524		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
1525	};
1526
1527	kthread_queue_work(worker, &fwork.work);
1528	wait_for_completion(&fwork.done);
1529}
1530EXPORT_SYMBOL_GPL(kthread_flush_worker);
1531
1532/**
1533 * kthread_destroy_worker - destroy a kthread worker
1534 * @worker: worker to be destroyed
1535 *
1536 * Flush and destroy @worker.  The simple flush is enough because the kthread
1537 * worker API is used only in trivial scenarios.  There are no multi-step state
1538 * machines needed.
1539 *
1540 * Note that this function is not responsible for handling delayed work, so
1541 * caller should be responsible for queuing or canceling all delayed work items
1542 * before invoke this function.
1543 */
1544void kthread_destroy_worker(struct kthread_worker *worker)
1545{
1546	struct task_struct *task;
1547
1548	task = worker->task;
1549	if (WARN_ON(!task))
1550		return;
1551
1552	kthread_flush_worker(worker);
1553	kthread_stop(task);
1554	WARN_ON(!list_empty(&worker->delayed_work_list));
1555	WARN_ON(!list_empty(&worker->work_list));
1556	kfree(worker);
1557}
1558EXPORT_SYMBOL(kthread_destroy_worker);
1559
1560/**
1561 * kthread_use_mm - make the calling kthread operate on an address space
1562 * @mm: address space to operate on
1563 */
1564void kthread_use_mm(struct mm_struct *mm)
1565{
1566	struct mm_struct *active_mm;
1567	struct task_struct *tsk = current;
1568
1569	WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1570	WARN_ON_ONCE(tsk->mm);
1571	WARN_ON_ONCE(!mm->user_ns);
1572
1573	/*
1574	 * It is possible for mm to be the same as tsk->active_mm, but
1575	 * we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm),
1576	 * because these references are not equivalent.
1577	 */
1578	mmgrab(mm);
1579
1580	task_lock(tsk);
1581	/* Hold off tlb flush IPIs while switching mm's */
1582	local_irq_disable();
1583	active_mm = tsk->active_mm;
1584	tsk->active_mm = mm;
1585	tsk->mm = mm;
1586	membarrier_update_current_mm(mm);
1587	switch_mm_irqs_off(active_mm, mm, tsk);
1588	local_irq_enable();
1589	task_unlock(tsk);
1590#ifdef finish_arch_post_lock_switch
1591	finish_arch_post_lock_switch();
1592#endif
1593
1594	/*
1595	 * When a kthread starts operating on an address space, the loop
1596	 * in membarrier_{private,global}_expedited() may not observe
1597	 * that tsk->mm, and not issue an IPI. Membarrier requires a
1598	 * memory barrier after storing to tsk->mm, before accessing
1599	 * user-space memory. A full memory barrier for membarrier
1600	 * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by
1601	 * mmdrop_lazy_tlb().
1602	 */
1603	mmdrop_lazy_tlb(active_mm);
1604}
1605EXPORT_SYMBOL_GPL(kthread_use_mm);
1606
1607/**
1608 * kthread_unuse_mm - reverse the effect of kthread_use_mm()
1609 * @mm: address space to operate on
1610 */
1611void kthread_unuse_mm(struct mm_struct *mm)
1612{
1613	struct task_struct *tsk = current;
1614
1615	WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
1616	WARN_ON_ONCE(!tsk->mm);
1617
1618	task_lock(tsk);
1619	/*
1620	 * When a kthread stops operating on an address space, the loop
1621	 * in membarrier_{private,global}_expedited() may not observe
1622	 * that tsk->mm, and not issue an IPI. Membarrier requires a
1623	 * memory barrier after accessing user-space memory, before
1624	 * clearing tsk->mm.
1625	 */
1626	smp_mb__after_spinlock();
1627	local_irq_disable();
1628	tsk->mm = NULL;
1629	membarrier_update_current_mm(NULL);
1630	mmgrab_lazy_tlb(mm);
1631	/* active_mm is still 'mm' */
1632	enter_lazy_tlb(mm, tsk);
1633	local_irq_enable();
1634	task_unlock(tsk);
1635
1636	mmdrop(mm);
1637}
1638EXPORT_SYMBOL_GPL(kthread_unuse_mm);
1639
1640#ifdef CONFIG_BLK_CGROUP
1641/**
1642 * kthread_associate_blkcg - associate blkcg to current kthread
1643 * @css: the cgroup info
1644 *
1645 * Current thread must be a kthread. The thread is running jobs on behalf of
1646 * other threads. In some cases, we expect the jobs attach cgroup info of
1647 * original threads instead of that of current thread. This function stores
1648 * original thread's cgroup info in current kthread context for later
1649 * retrieval.
1650 */
1651void kthread_associate_blkcg(struct cgroup_subsys_state *css)
1652{
1653	struct kthread *kthread;
1654
1655	if (!(current->flags & PF_KTHREAD))
1656		return;
1657	kthread = to_kthread(current);
1658	if (!kthread)
1659		return;
1660
1661	if (kthread->blkcg_css) {
1662		css_put(kthread->blkcg_css);
1663		kthread->blkcg_css = NULL;
1664	}
1665	if (css) {
1666		css_get(css);
1667		kthread->blkcg_css = css;
1668	}
1669}
1670EXPORT_SYMBOL(kthread_associate_blkcg);
1671
1672/**
1673 * kthread_blkcg - get associated blkcg css of current kthread
1674 *
1675 * Current thread must be a kthread.
1676 */
1677struct cgroup_subsys_state *kthread_blkcg(void)
1678{
1679	struct kthread *kthread;
1680
1681	if (current->flags & PF_KTHREAD) {
1682		kthread = to_kthread(current);
1683		if (kthread)
1684			return kthread->blkcg_css;
1685	}
1686	return NULL;
1687}
1688#endif