affinity.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Manage affinity to optimize IPIs inside the kernel perf API. */
  3#define _GNU_SOURCE 1
  4#include <sched.h>
  5#include <stdlib.h>
  6#include <linux/bitmap.h>
  7#include <linux/zalloc.h>
  8#include <perf/cpumap.h>
  9#include "perf.h"
 10#include "cpumap.h"
 11#include "affinity.h"
 12
 13static int get_cpu_set_size(void)
 14{
 15	int sz = cpu__max_cpu().cpu + 8 - 1;
 16	/*
 17	 * sched_getaffinity doesn't like masks smaller than the kernel.
 18	 * Hopefully that's big enough.
 19	 */
 20	if (sz < 4096)
 21		sz = 4096;
 22	return sz / 8;
 23}
 24
 25int affinity__setup(struct affinity *a)
 26{
 27	int cpu_set_size = get_cpu_set_size();
 28
 29	a->orig_cpus = bitmap_zalloc(cpu_set_size * 8);
 30	if (!a->orig_cpus)
 31		return -1;
 32	sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
 33	a->sched_cpus = bitmap_zalloc(cpu_set_size * 8);
 34	if (!a->sched_cpus) {
 35		zfree(&a->orig_cpus);
 36		return -1;
 37	}
 38	bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size);
 39	a->changed = false;
 40	return 0;
 41}
 42
 43/*
 44 * perf_event_open does an IPI internally to the target CPU.
 45 * It is more efficient to change perf's affinity to the target
 46 * CPU and then set up all events on that CPU, so we amortize
 47 * CPU communication.
 48 */
 49void affinity__set(struct affinity *a, int cpu)
 50{
 51	int cpu_set_size = get_cpu_set_size();
 52
 53	/*
 54	 * Return:
 55	 * - if cpu is -1
 56	 * - restrict out of bound access to sched_cpus
 57	 */
 58	if (cpu == -1 || ((cpu >= (cpu_set_size * 8))))
 59		return;
 60
 61	a->changed = true;
 62	__set_bit(cpu, a->sched_cpus);
 63	/*
 64	 * We ignore errors because affinity is just an optimization.
 65	 * This could happen for example with isolated CPUs or cpusets.
 66	 * In this case the IPIs inside the kernel's perf API still work.
 67	 */
 68	sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus);
 69	__clear_bit(cpu, a->sched_cpus);
 70}
 71
 72static void __affinity__cleanup(struct affinity *a)
 73{
 74	int cpu_set_size = get_cpu_set_size();
 75
 76	if (a->changed)
 77		sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
 78	zfree(&a->sched_cpus);
 79	zfree(&a->orig_cpus);
 80}
 81
 82void affinity__cleanup(struct affinity *a)
 83{
 84	if (a != NULL)
 85		__affinity__cleanup(a);
 86}
 87
 88void cpu_map__set_affinity(const struct perf_cpu_map *cpumap)
 89{
 90	int cpu_set_size = get_cpu_set_size();
 91	unsigned long *cpuset = bitmap_zalloc(cpu_set_size * 8);
 92	struct perf_cpu cpu;
 93	int idx;
 94
 95	if (!cpuset)
 96		return;
 97
 98	perf_cpu_map__for_each_cpu_skip_any(cpu, idx, cpumap)
 99		__set_bit(cpu.cpu, cpuset);
100
101	sched_setaffinity(0, cpu_set_size, (cpu_set_t *)cpuset);
102	zfree(&cpuset);
103}