topology.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *    Copyright IBM Corp. 2007, 2011
  4 */
  5
  6#define pr_fmt(fmt) "cpu: " fmt
  7
  8#include <linux/cpufeature.h>
  9#include <linux/workqueue.h>
 10#include <linux/memblock.h>
 11#include <linux/uaccess.h>
 12#include <linux/sysctl.h>
 13#include <linux/cpuset.h>
 14#include <linux/device.h>
 15#include <linux/export.h>
 16#include <linux/kernel.h>
 17#include <linux/sched.h>
 18#include <linux/sched/topology.h>
 19#include <linux/delay.h>
 20#include <linux/init.h>
 21#include <linux/slab.h>
 22#include <linux/cpu.h>
 23#include <linux/smp.h>
 24#include <linux/mm.h>
 25#include <linux/nodemask.h>
 26#include <linux/node.h>
 27#include <asm/hiperdispatch.h>
 28#include <asm/sysinfo.h>
 29#include <asm/asm.h>
 30
 31#define PTF_HORIZONTAL	(0UL)
 32#define PTF_VERTICAL	(1UL)
 33#define PTF_CHECK	(2UL)
 34
 35enum {
 36	TOPOLOGY_MODE_HW,
 37	TOPOLOGY_MODE_SINGLE,
 38	TOPOLOGY_MODE_PACKAGE,
 39	TOPOLOGY_MODE_UNINITIALIZED
 40};
 41
 42struct mask_info {
 43	struct mask_info *next;
 44	unsigned char id;
 45	cpumask_t mask;
 46};
 47
 48static int topology_mode = TOPOLOGY_MODE_UNINITIALIZED;
 49static void set_topology_timer(void);
 50static void topology_work_fn(struct work_struct *work);
 51static struct sysinfo_15_1_x *tl_info;
 52static int cpu_management;
 53
 54static DECLARE_WORK(topology_work, topology_work_fn);
 55
 56/*
 57 * Socket/Book linked lists and cpu_topology updates are
 58 * protected by "sched_domains_mutex".
 59 */
 60static struct mask_info socket_info;
 61static struct mask_info book_info;
 62static struct mask_info drawer_info;
 63
 64struct cpu_topology_s390 cpu_topology[NR_CPUS];
 65EXPORT_SYMBOL_GPL(cpu_topology);
 66
 67static void cpu_group_map(cpumask_t *dst, struct mask_info *info, unsigned int cpu)
 68{
 69	static cpumask_t mask;
 70
 71	cpumask_clear(&mask);
 72	if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
 73		goto out;
 74	cpumask_set_cpu(cpu, &mask);
 75	switch (topology_mode) {
 76	case TOPOLOGY_MODE_HW:
 77		while (info) {
 78			if (cpumask_test_cpu(cpu, &info->mask)) {
 79				cpumask_copy(&mask, &info->mask);
 80				break;
 81			}
 82			info = info->next;
 83		}
 84		break;
 85	case TOPOLOGY_MODE_PACKAGE:
 86		cpumask_copy(&mask, cpu_present_mask);
 87		break;
 88	default:
 89		fallthrough;
 90	case TOPOLOGY_MODE_SINGLE:
 91		break;
 92	}
 93	cpumask_and(&mask, &mask, &cpu_setup_mask);
 94out:
 95	cpumask_copy(dst, &mask);
 96}
 97
 98static void cpu_thread_map(cpumask_t *dst, unsigned int cpu)
 99{
100	static cpumask_t mask;
101	unsigned int max_cpu;
102
103	cpumask_clear(&mask);
104	if (!cpumask_test_cpu(cpu, &cpu_setup_mask))
105		goto out;
106	cpumask_set_cpu(cpu, &mask);
107	if (topology_mode != TOPOLOGY_MODE_HW)
108		goto out;
109	cpu -= cpu % (smp_cpu_mtid + 1);
110	max_cpu = min(cpu + smp_cpu_mtid, nr_cpu_ids - 1);
111	for (; cpu <= max_cpu; cpu++) {
112		if (cpumask_test_cpu(cpu, &cpu_setup_mask))
113			cpumask_set_cpu(cpu, &mask);
114	}
115out:
116	cpumask_copy(dst, &mask);
117}
118
119#define TOPOLOGY_CORE_BITS	64
120
121static void add_cpus_to_mask(struct topology_core *tl_core,
122			     struct mask_info *drawer,
123			     struct mask_info *book,
124			     struct mask_info *socket)
125{
126	struct cpu_topology_s390 *topo;
127	unsigned int core;
128
129	for_each_set_bit(core, &tl_core->mask, TOPOLOGY_CORE_BITS) {
130		unsigned int max_cpu, rcore;
131		int cpu;
132
133		rcore = TOPOLOGY_CORE_BITS - 1 - core + tl_core->origin;
134		cpu = smp_find_processor_id(rcore << smp_cpu_mt_shift);
135		if (cpu < 0)
136			continue;
137		max_cpu = min(cpu + smp_cpu_mtid, nr_cpu_ids - 1);
138		for (; cpu <= max_cpu; cpu++) {
139			topo = &cpu_topology[cpu];
140			topo->drawer_id = drawer->id;
141			topo->book_id = book->id;
142			topo->socket_id = socket->id;
143			topo->core_id = rcore;
144			topo->thread_id = cpu;
145			topo->dedicated = tl_core->d;
146			cpumask_set_cpu(cpu, &drawer->mask);
147			cpumask_set_cpu(cpu, &book->mask);
148			cpumask_set_cpu(cpu, &socket->mask);
149			smp_cpu_set_polarization(cpu, tl_core->pp);
150			smp_cpu_set_capacity(cpu, CPU_CAPACITY_HIGH);
151		}
152	}
153}
154
155static void clear_masks(void)
156{
157	struct mask_info *info;
158
159	info = &socket_info;
160	while (info) {
161		cpumask_clear(&info->mask);
162		info = info->next;
163	}
164	info = &book_info;
165	while (info) {
166		cpumask_clear(&info->mask);
167		info = info->next;
168	}
169	info = &drawer_info;
170	while (info) {
171		cpumask_clear(&info->mask);
172		info = info->next;
173	}
174}
175
176static union topology_entry *next_tle(union topology_entry *tle)
177{
178	if (!tle->nl)
179		return (union topology_entry *)((struct topology_core *)tle + 1);
180	return (union topology_entry *)((struct topology_container *)tle + 1);
181}
182
183static void tl_to_masks(struct sysinfo_15_1_x *info)
184{
185	struct mask_info *socket = &socket_info;
186	struct mask_info *book = &book_info;
187	struct mask_info *drawer = &drawer_info;
188	union topology_entry *tle, *end;
189
190	clear_masks();
191	tle = info->tle;
192	end = (union topology_entry *)((unsigned long)info + info->length);
193	while (tle < end) {
194		switch (tle->nl) {
195		case 3:
196			drawer = drawer->next;
197			drawer->id = tle->container.id;
198			break;
199		case 2:
200			book = book->next;
201			book->id = tle->container.id;
202			break;
203		case 1:
204			socket = socket->next;
205			socket->id = tle->container.id;
206			break;
207		case 0:
208			add_cpus_to_mask(&tle->cpu, drawer, book, socket);
209			break;
210		default:
211			clear_masks();
212			return;
213		}
214		tle = next_tle(tle);
215	}
216}
217
218static void topology_update_polarization_simple(void)
219{
220	int cpu;
221
222	for_each_possible_cpu(cpu)
223		smp_cpu_set_polarization(cpu, POLARIZATION_HRZ);
224}
225
226static int ptf(unsigned long fc)
227{
228	int cc;
229
230	asm volatile(
231		"	.insn	rre,0xb9a20000,%[fc],%[fc]\n"
232		CC_IPM(cc)
233		: CC_OUT(cc, cc)
234		: [fc] "d" (fc)
235		: CC_CLOBBER);
236	return CC_TRANSFORM(cc);
237}
238
239int topology_set_cpu_management(int fc)
240{
241	int cpu, rc;
242
243	if (!cpu_has_topology())
244		return -EOPNOTSUPP;
245	if (fc)
246		rc = ptf(PTF_VERTICAL);
247	else
248		rc = ptf(PTF_HORIZONTAL);
249	if (rc)
250		return -EBUSY;
251	for_each_possible_cpu(cpu)
252		smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
253	return rc;
254}
255
256void update_cpu_masks(void)
257{
258	struct cpu_topology_s390 *topo, *topo_package, *topo_sibling;
259	int cpu, sibling, pkg_first, smt_first, id;
260
261	for_each_possible_cpu(cpu) {
262		topo = &cpu_topology[cpu];
263		cpu_thread_map(&topo->thread_mask, cpu);
264		cpu_group_map(&topo->core_mask, &socket_info, cpu);
265		cpu_group_map(&topo->book_mask, &book_info, cpu);
266		cpu_group_map(&topo->drawer_mask, &drawer_info, cpu);
267		topo->booted_cores = 0;
268		if (topology_mode != TOPOLOGY_MODE_HW) {
269			id = topology_mode == TOPOLOGY_MODE_PACKAGE ? 0 : cpu;
270			topo->thread_id = cpu;
271			topo->core_id = cpu;
272			topo->socket_id = id;
273			topo->book_id = id;
274			topo->drawer_id = id;
275		}
276	}
277	hd_reset_state();
278	for_each_online_cpu(cpu) {
279		topo = &cpu_topology[cpu];
280		pkg_first = cpumask_first(&topo->core_mask);
281		topo_package = &cpu_topology[pkg_first];
282		if (cpu == pkg_first) {
283			for_each_cpu(sibling, &topo->core_mask) {
284				topo_sibling = &cpu_topology[sibling];
285				smt_first = cpumask_first(&topo_sibling->thread_mask);
286				if (sibling == smt_first) {
287					topo_package->booted_cores++;
288					hd_add_core(sibling);
289				}
290			}
291		} else {
292			topo->booted_cores = topo_package->booted_cores;
293		}
294	}
295}
296
297void store_topology(struct sysinfo_15_1_x *info)
298{
299	stsi(info, 15, 1, topology_mnest_limit());
300}
301
302static void __arch_update_dedicated_flag(void *arg)
303{
304	if (topology_cpu_dedicated(smp_processor_id()))
305		set_cpu_flag(CIF_DEDICATED_CPU);
306	else
307		clear_cpu_flag(CIF_DEDICATED_CPU);
308}
309
310static int __arch_update_cpu_topology(void)
311{
312	struct sysinfo_15_1_x *info = tl_info;
313	int rc, hd_status;
314
315	hd_status = 0;
316	rc = 0;
317	mutex_lock(&smp_cpu_state_mutex);
318	if (cpu_has_topology()) {
319		rc = 1;
320		store_topology(info);
321		tl_to_masks(info);
322	}
323	update_cpu_masks();
324	if (!cpu_has_topology())
325		topology_update_polarization_simple();
326	if (cpu_management == 1)
327		hd_status = hd_enable_hiperdispatch();
328	mutex_unlock(&smp_cpu_state_mutex);
329	if (hd_status == 0)
330		hd_disable_hiperdispatch();
331	return rc;
332}
333
334int arch_update_cpu_topology(void)
335{
336	int rc;
337
338	rc = __arch_update_cpu_topology();
339	on_each_cpu(__arch_update_dedicated_flag, NULL, 0);
340	return rc;
341}
342
343static void topology_work_fn(struct work_struct *work)
344{
345	rebuild_sched_domains();
346}
347
348void topology_schedule_update(void)
349{
350	schedule_work(&topology_work);
351}
352
353static void topology_flush_work(void)
354{
355	flush_work(&topology_work);
356}
357
358static void topology_timer_fn(struct timer_list *unused)
359{
360	if (ptf(PTF_CHECK))
361		topology_schedule_update();
362	set_topology_timer();
363}
364
365static struct timer_list topology_timer;
366
367static atomic_t topology_poll = ATOMIC_INIT(0);
368
369static void set_topology_timer(void)
370{
371	if (atomic_add_unless(&topology_poll, -1, 0))
372		mod_timer(&topology_timer, jiffies + msecs_to_jiffies(100));
373	else
374		mod_timer(&topology_timer, jiffies + secs_to_jiffies(60));
375}
376
377void topology_expect_change(void)
378{
379	if (!cpu_has_topology())
380		return;
381	/* This is racy, but it doesn't matter since it is just a heuristic.
382	 * Worst case is that we poll in a higher frequency for a bit longer.
383	 */
384	if (atomic_read(&topology_poll) > 60)
385		return;
386	atomic_add(60, &topology_poll);
387	set_topology_timer();
388}
389
390static int set_polarization(int polarization)
391{
392	int rc = 0;
393
394	cpus_read_lock();
395	mutex_lock(&smp_cpu_state_mutex);
396	if (cpu_management == polarization)
397		goto out;
398	rc = topology_set_cpu_management(polarization);
399	if (rc)
400		goto out;
401	cpu_management = polarization;
402	topology_expect_change();
403out:
404	mutex_unlock(&smp_cpu_state_mutex);
405	cpus_read_unlock();
406	return rc;
407}
408
409static ssize_t dispatching_show(struct device *dev,
410				struct device_attribute *attr,
411				char *buf)
412{
413	ssize_t count;
414
415	mutex_lock(&smp_cpu_state_mutex);
416	count = sysfs_emit(buf, "%d\n", cpu_management);
417	mutex_unlock(&smp_cpu_state_mutex);
418	return count;
419}
420
421static ssize_t dispatching_store(struct device *dev,
422				 struct device_attribute *attr,
423				 const char *buf,
424				 size_t count)
425{
426	int val, rc;
427	char delim;
428
429	if (sscanf(buf, "%d %c", &val, &delim) != 1)
430		return -EINVAL;
431	if (val != 0 && val != 1)
432		return -EINVAL;
433	rc = set_polarization(val);
434	return rc ? rc : count;
435}
436static DEVICE_ATTR_RW(dispatching);
437
438static ssize_t cpu_polarization_show(struct device *dev,
439				     struct device_attribute *attr, char *buf)
440{
441	int cpu = dev->id;
442	ssize_t count;
443
444	mutex_lock(&smp_cpu_state_mutex);
445	switch (smp_cpu_get_polarization(cpu)) {
446	case POLARIZATION_HRZ:
447		count = sysfs_emit(buf, "horizontal\n");
448		break;
449	case POLARIZATION_VL:
450		count = sysfs_emit(buf, "vertical:low\n");
451		break;
452	case POLARIZATION_VM:
453		count = sysfs_emit(buf, "vertical:medium\n");
454		break;
455	case POLARIZATION_VH:
456		count = sysfs_emit(buf, "vertical:high\n");
457		break;
458	default:
459		count = sysfs_emit(buf, "unknown\n");
460		break;
461	}
462	mutex_unlock(&smp_cpu_state_mutex);
463	return count;
464}
465static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);
466
467static struct attribute *topology_cpu_attrs[] = {
468	&dev_attr_polarization.attr,
469	NULL,
470};
471
472static struct attribute_group topology_cpu_attr_group = {
473	.attrs = topology_cpu_attrs,
474};
475
476static ssize_t cpu_dedicated_show(struct device *dev,
477				  struct device_attribute *attr, char *buf)
478{
479	int cpu = dev->id;
480	ssize_t count;
481
482	mutex_lock(&smp_cpu_state_mutex);
483	count = sysfs_emit(buf, "%d\n", topology_cpu_dedicated(cpu));
484	mutex_unlock(&smp_cpu_state_mutex);
485	return count;
486}
487static DEVICE_ATTR(dedicated, 0444, cpu_dedicated_show, NULL);
488
489static struct attribute *topology_extra_cpu_attrs[] = {
490	&dev_attr_dedicated.attr,
491	NULL,
492};
493
494static struct attribute_group topology_extra_cpu_attr_group = {
495	.attrs = topology_extra_cpu_attrs,
496};
497
498int topology_cpu_init(struct cpu *cpu)
499{
500	int rc;
501
502	rc = sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group);
503	if (rc || !cpu_has_topology())
504		return rc;
505	rc = sysfs_create_group(&cpu->dev.kobj, &topology_extra_cpu_attr_group);
506	if (rc)
507		sysfs_remove_group(&cpu->dev.kobj, &topology_cpu_attr_group);
508	return rc;
509}
510
511const struct cpumask *cpu_coregroup_mask(int cpu)
512{
513	return &cpu_topology[cpu].core_mask;
514}
515
516static const struct cpumask *tl_book_mask(struct sched_domain_topology_level *tl, int cpu)
517{
518	return &cpu_topology[cpu].book_mask;
519}
520
521static const struct cpumask *tl_drawer_mask(struct sched_domain_topology_level *tl, int cpu)
522{
523	return &cpu_topology[cpu].drawer_mask;
524}
525
526static struct sched_domain_topology_level s390_topology[] = {
527	SDTL_INIT(tl_smt_mask, cpu_smt_flags, SMT),
528	SDTL_INIT(tl_mc_mask, cpu_core_flags, MC),
529	SDTL_INIT(tl_book_mask, NULL, BOOK),
530	SDTL_INIT(tl_drawer_mask, NULL, DRAWER),
531	SDTL_INIT(tl_pkg_mask, NULL, PKG),
532	{ NULL, },
533};
534
535static void __init alloc_masks(struct sysinfo_15_1_x *info,
536			       struct mask_info *mask, int offset)
537{
538	int i, nr_masks;
539
540	nr_masks = info->mag[TOPOLOGY_NR_MAG - offset];
541	for (i = 0; i < info->mnest - offset; i++)
542		nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
543	nr_masks = max(nr_masks, 1);
544	for (i = 0; i < nr_masks; i++) {
545		mask->next = memblock_alloc_or_panic(sizeof(*mask->next), 8);
546		mask = mask->next;
547	}
548}
549
550static int __init detect_polarization(union topology_entry *tle)
551{
552	struct topology_core *tl_core;
553
554	while (tle->nl)
555		tle = next_tle(tle);
556	tl_core = (struct topology_core *)tle;
557	return tl_core->pp != POLARIZATION_HRZ;
558}
559
560void __init topology_init_early(void)
561{
562	struct sysinfo_15_1_x *info;
563
564	set_sched_topology(s390_topology);
565	if (topology_mode == TOPOLOGY_MODE_UNINITIALIZED) {
566		if (cpu_has_topology())
567			topology_mode = TOPOLOGY_MODE_HW;
568		else
569			topology_mode = TOPOLOGY_MODE_SINGLE;
570	}
571	if (!cpu_has_topology())
572		goto out;
573	tl_info = memblock_alloc_or_panic(PAGE_SIZE, PAGE_SIZE);
574	info = tl_info;
575	store_topology(info);
576	cpu_management = detect_polarization(info->tle);
577	pr_info("The CPU configuration topology of the machine is: %d %d %d %d %d %d / %d\n",
578		info->mag[0], info->mag[1], info->mag[2], info->mag[3],
579		info->mag[4], info->mag[5], info->mnest);
580	alloc_masks(info, &socket_info, 1);
581	alloc_masks(info, &book_info, 2);
582	alloc_masks(info, &drawer_info, 3);
583out:
584	cpumask_set_cpu(0, &cpu_setup_mask);
585	__arch_update_cpu_topology();
586	__arch_update_dedicated_flag(NULL);
587}
588
589static inline int topology_get_mode(int enabled)
590{
591	if (!enabled)
592		return TOPOLOGY_MODE_SINGLE;
593	return cpu_has_topology() ? TOPOLOGY_MODE_HW : TOPOLOGY_MODE_PACKAGE;
594}
595
596static inline int topology_is_enabled(void)
597{
598	return topology_mode != TOPOLOGY_MODE_SINGLE;
599}
600
601static int __init topology_setup(char *str)
602{
603	bool enabled;
604	int rc;
605
606	rc = kstrtobool(str, &enabled);
607	if (rc)
608		return rc;
609	topology_mode = topology_get_mode(enabled);
610	return 0;
611}
612early_param("topology", topology_setup);
613
614static int topology_ctl_handler(const struct ctl_table *ctl, int write,
615				void *buffer, size_t *lenp, loff_t *ppos)
616{
617	int enabled = topology_is_enabled();
618	int new_mode;
619	int rc;
620	struct ctl_table ctl_entry = {
621		.procname	= ctl->procname,
622		.data		= &enabled,
623		.maxlen		= sizeof(int),
624		.extra1		= SYSCTL_ZERO,
625		.extra2		= SYSCTL_ONE,
626	};
627
628	rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos);
629	if (rc < 0 || !write)
630		return rc;
631
632	mutex_lock(&smp_cpu_state_mutex);
633	new_mode = topology_get_mode(enabled);
634	if (topology_mode != new_mode) {
635		topology_mode = new_mode;
636		topology_schedule_update();
637	}
638	mutex_unlock(&smp_cpu_state_mutex);
639	topology_flush_work();
640
641	return rc;
642}
643
644static int polarization_ctl_handler(const struct ctl_table *ctl, int write,
645				    void *buffer, size_t *lenp, loff_t *ppos)
646{
647	int polarization;
648	int rc;
649	struct ctl_table ctl_entry = {
650		.procname	= ctl->procname,
651		.data		= &polarization,
652		.maxlen		= sizeof(int),
653		.extra1		= SYSCTL_ZERO,
654		.extra2		= SYSCTL_ONE,
655	};
656
657	polarization = cpu_management;
658	rc = proc_douintvec_minmax(&ctl_entry, write, buffer, lenp, ppos);
659	if (rc < 0 || !write)
660		return rc;
661	return set_polarization(polarization);
662}
663
664static const struct ctl_table topology_ctl_table[] = {
665	{
666		.procname	= "topology",
667		.mode		= 0644,
668		.proc_handler	= topology_ctl_handler,
669	},
670	{
671		.procname	= "polarization",
672		.mode		= 0644,
673		.proc_handler	= polarization_ctl_handler,
674	},
675};
676
677static int __init topology_init(void)
678{
679	struct device *dev_root;
680	int rc = 0;
681
682	timer_setup(&topology_timer, topology_timer_fn, TIMER_DEFERRABLE);
683	if (cpu_has_topology())
684		set_topology_timer();
685	else
686		topology_update_polarization_simple();
687	if (IS_ENABLED(CONFIG_SCHED_TOPOLOGY_VERTICAL))
688		set_polarization(1);
689	register_sysctl("s390", topology_ctl_table);
690
691	dev_root = bus_get_dev_root(&cpu_subsys);
692	if (dev_root) {
693		rc = device_create_file(dev_root, &dev_attr_dispatching);
694		put_device(dev_root);
695	}
696	return rc;
697}
698device_initcall(topology_init);