isolation.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 *  Housekeeping management. Manage the targets for routine code that can run on
  4 *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
  5 *
  6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
  7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
  8 *
  9 */
 10#include <linux/sched/isolation.h>
 11#include "sched.h"
 12
 13enum hk_flags {
 14	HK_FLAG_DOMAIN		= BIT(HK_TYPE_DOMAIN),
 15	HK_FLAG_MANAGED_IRQ	= BIT(HK_TYPE_MANAGED_IRQ),
 16	HK_FLAG_KERNEL_NOISE	= BIT(HK_TYPE_KERNEL_NOISE),
 17};
 18
 19DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
 20EXPORT_SYMBOL_GPL(housekeeping_overridden);
 21
 22struct housekeeping {
 23	cpumask_var_t cpumasks[HK_TYPE_MAX];
 24	unsigned long flags;
 25};
 26
 27static struct housekeeping housekeeping;
 28
 29bool housekeeping_enabled(enum hk_type type)
 30{
 31	return !!(housekeeping.flags & BIT(type));
 32}
 33EXPORT_SYMBOL_GPL(housekeeping_enabled);
 34
 35int housekeeping_any_cpu(enum hk_type type)
 36{
 37	int cpu;
 38
 39	if (static_branch_unlikely(&housekeeping_overridden)) {
 40		if (housekeeping.flags & BIT(type)) {
 41			cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
 42			if (cpu < nr_cpu_ids)
 43				return cpu;
 44
 45			cpu = cpumask_any_and_distribute(housekeeping.cpumasks[type], cpu_online_mask);
 46			if (likely(cpu < nr_cpu_ids))
 47				return cpu;
 48			/*
 49			 * Unless we have another problem this can only happen
 50			 * at boot time before start_secondary() brings the 1st
 51			 * housekeeping CPU up.
 52			 */
 53			WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
 54				     type != HK_TYPE_TIMER);
 55		}
 56	}
 57	return smp_processor_id();
 58}
 59EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
 60
 61const struct cpumask *housekeeping_cpumask(enum hk_type type)
 62{
 63	if (static_branch_unlikely(&housekeeping_overridden))
 64		if (housekeeping.flags & BIT(type))
 65			return housekeeping.cpumasks[type];
 66	return cpu_possible_mask;
 67}
 68EXPORT_SYMBOL_GPL(housekeeping_cpumask);
 69
 70void housekeeping_affine(struct task_struct *t, enum hk_type type)
 71{
 72	if (static_branch_unlikely(&housekeeping_overridden))
 73		if (housekeeping.flags & BIT(type))
 74			set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
 75}
 76EXPORT_SYMBOL_GPL(housekeeping_affine);
 77
 78bool housekeeping_test_cpu(int cpu, enum hk_type type)
 79{
 80	if (static_branch_unlikely(&housekeeping_overridden))
 81		if (housekeeping.flags & BIT(type))
 82			return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
 83	return true;
 84}
 85EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
 86
 87void __init housekeeping_init(void)
 88{
 89	enum hk_type type;
 90
 91	if (!housekeeping.flags)
 92		return;
 93
 94	static_branch_enable(&housekeeping_overridden);
 95
 96	if (housekeeping.flags & HK_FLAG_KERNEL_NOISE)
 97		sched_tick_offload_init();
 98
 99	for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
100		/* We need at least one CPU to handle housekeeping work */
101		WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
102	}
103}
104
105static void __init housekeeping_setup_type(enum hk_type type,
106					   cpumask_var_t housekeeping_staging)
107{
108
109	alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
110	cpumask_copy(housekeeping.cpumasks[type],
111		     housekeeping_staging);
112}
113
114static int __init housekeeping_setup(char *str, unsigned long flags)
115{
116	cpumask_var_t non_housekeeping_mask, housekeeping_staging;
117	unsigned int first_cpu;
118	int err = 0;
119
120	if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) {
121		if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
122			pr_warn("Housekeeping: nohz unsupported."
123				" Build with CONFIG_NO_HZ_FULL\n");
124			return 0;
125		}
126	}
127
128	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
129	if (cpulist_parse(str, non_housekeeping_mask) < 0) {
130		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
131		goto free_non_housekeeping_mask;
132	}
133
134	alloc_bootmem_cpumask_var(&housekeeping_staging);
135	cpumask_andnot(housekeeping_staging,
136		       cpu_possible_mask, non_housekeeping_mask);
137
138	first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
139	if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
140		__cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
141		__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
142		if (!housekeeping.flags) {
143			pr_warn("Housekeeping: must include one present CPU, "
144				"using boot CPU:%d\n", smp_processor_id());
145		}
146	}
147
148	if (cpumask_empty(non_housekeeping_mask))
149		goto free_housekeeping_staging;
150
151	if (!housekeeping.flags) {
152		/* First setup call ("nohz_full=" or "isolcpus=") */
153		enum hk_type type;
154
155		for_each_set_bit(type, &flags, HK_TYPE_MAX)
156			housekeeping_setup_type(type, housekeeping_staging);
157	} else {
158		/* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
159		enum hk_type type;
160		unsigned long iter_flags = flags & housekeeping.flags;
161
162		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
163			if (!cpumask_equal(housekeeping_staging,
164					   housekeeping.cpumasks[type])) {
165				pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
166				goto free_housekeeping_staging;
167			}
168		}
169
170		/*
171		 * Check the combination of nohz_full and isolcpus=domain,
172		 * necessary to avoid problems with the timer migration
173		 * hierarchy. managed_irq is ignored by this check since it
174		 * isn't considered in the timer migration logic.
175		 */
176		iter_flags = housekeeping.flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN);
177		type = find_first_bit(&iter_flags, HK_TYPE_MAX);
178		/*
179		 * Pass the check if none of these flags were previously set or
180		 * are not in the current selection.
181		 */
182		iter_flags = flags & (HK_FLAG_KERNEL_NOISE | HK_FLAG_DOMAIN);
183		first_cpu = (type == HK_TYPE_MAX || !iter_flags) ? 0 :
184			    cpumask_first_and_and(cpu_present_mask,
185				    housekeeping_staging, housekeeping.cpumasks[type]);
186		if (first_cpu >= min(nr_cpu_ids, setup_max_cpus)) {
187			pr_warn("Housekeeping: must include one present CPU "
188				"neither in nohz_full= nor in isolcpus=domain, "
189				"ignoring setting %s\n", str);
190			goto free_housekeeping_staging;
191		}
192
193		iter_flags = flags & ~housekeeping.flags;
194
195		for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
196			housekeeping_setup_type(type, housekeeping_staging);
197	}
198
199	if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE))
200		tick_nohz_full_setup(non_housekeeping_mask);
201
202	housekeeping.flags |= flags;
203	err = 1;
204
205free_housekeeping_staging:
206	free_bootmem_cpumask_var(housekeeping_staging);
207free_non_housekeeping_mask:
208	free_bootmem_cpumask_var(non_housekeeping_mask);
209
210	return err;
211}
212
213static int __init housekeeping_nohz_full_setup(char *str)
214{
215	unsigned long flags;
216
217	flags = HK_FLAG_KERNEL_NOISE;
218
219	return housekeeping_setup(str, flags);
220}
221__setup("nohz_full=", housekeeping_nohz_full_setup);
222
223static int __init housekeeping_isolcpus_setup(char *str)
224{
225	unsigned long flags = 0;
226	bool illegal = false;
227	char *par;
228	int len;
229
230	while (isalpha(*str)) {
231		/*
232		 * isolcpus=nohz is equivalent to nohz_full.
233		 */
234		if (!strncmp(str, "nohz,", 5)) {
235			str += 5;
236			flags |= HK_FLAG_KERNEL_NOISE;
237			continue;
238		}
239
240		if (!strncmp(str, "domain,", 7)) {
241			str += 7;
242			flags |= HK_FLAG_DOMAIN;
243			continue;
244		}
245
246		if (!strncmp(str, "managed_irq,", 12)) {
247			str += 12;
248			flags |= HK_FLAG_MANAGED_IRQ;
249			continue;
250		}
251
252		/*
253		 * Skip unknown sub-parameter and validate that it is not
254		 * containing an invalid character.
255		 */
256		for (par = str, len = 0; *str && *str != ','; str++, len++) {
257			if (!isalpha(*str) && *str != '_')
258				illegal = true;
259		}
260
261		if (illegal) {
262			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
263			return 0;
264		}
265
266		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
267		str++;
268	}
269
270	/* Default behaviour for isolcpus without flags */
271	if (!flags)
272		flags |= HK_FLAG_DOMAIN;
273
274	return housekeeping_setup(str, flags);
275}
276__setup("isolcpus=", housekeeping_isolcpus_setup);