percpu-stats.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 *
  4 * Copyright (C) 2017		Facebook Inc.
  5 * Copyright (C) 2017		Dennis Zhou <dennis@kernel.org>
  6 *
  7 * Prints statistics about the percpu allocator and backing chunks.
  8 */
  9#include <linux/debugfs.h>
 10#include <linux/list.h>
 11#include <linux/percpu.h>
 12#include <linux/seq_file.h>
 13#include <linux/sort.h>
 14#include <linux/vmalloc.h>
 15
 16#include "percpu-internal.h"
 17
 18#define P(X, Y) \
 19	seq_printf(m, "  %-20s: %12lld\n", X, (long long int)Y)
 20
 21struct percpu_stats pcpu_stats;
 22struct pcpu_alloc_info pcpu_stats_ai;
 23
 24static int cmpint(const void *a, const void *b)
 25{
 26	return *(int *)a - *(int *)b;
 27}
 28
 29/*
 30 * Iterates over all chunks to find the max nr_alloc entries.
 31 */
 32static int find_max_nr_alloc(void)
 33{
 34	struct pcpu_chunk *chunk;
 35	int slot, max_nr_alloc;
 36
 37	max_nr_alloc = 0;
 38	for (slot = 0; slot < pcpu_nr_slots; slot++)
 39		list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list)
 40			max_nr_alloc = max(max_nr_alloc, chunk->nr_alloc);
 41
 42	return max_nr_alloc;
 43}
 44
 45/*
 46 * Prints out chunk state. Fragmentation is considered between
 47 * the beginning of the chunk to the last allocation.
 48 *
 49 * All statistics are in bytes unless stated otherwise.
 50 */
 51static void chunk_map_stats(struct seq_file *m, struct pcpu_chunk *chunk,
 52			    int *buffer)
 53{
 54	struct pcpu_block_md *chunk_md = &chunk->chunk_md;
 55	int i, last_alloc, as_len, start, end;
 56	int *alloc_sizes, *p;
 57	/* statistics */
 58	int sum_frag = 0, max_frag = 0;
 59	int cur_min_alloc = 0, cur_med_alloc = 0, cur_max_alloc = 0;
 60
 61	alloc_sizes = buffer;
 62
 63	/*
 64	 * find_last_bit returns the start value if nothing found.
 65	 * Therefore, we must determine if it is a failure of find_last_bit
 66	 * and set the appropriate value.
 67	 */
 68	last_alloc = find_last_bit(chunk->alloc_map,
 69				   pcpu_chunk_map_bits(chunk) -
 70				   chunk->end_offset / PCPU_MIN_ALLOC_SIZE - 1);
 71	last_alloc = test_bit(last_alloc, chunk->alloc_map) ?
 72		     last_alloc + 1 : 0;
 73
 74	as_len = 0;
 75	start = chunk->start_offset / PCPU_MIN_ALLOC_SIZE;
 76
 77	/*
 78	 * If a bit is set in the allocation map, the bound_map identifies
 79	 * where the allocation ends.  If the allocation is not set, the
 80	 * bound_map does not identify free areas as it is only kept accurate
 81	 * on allocation, not free.
 82	 *
 83	 * Positive values are allocations and negative values are free
 84	 * fragments.
 85	 */
 86	while (start < last_alloc) {
 87		if (test_bit(start, chunk->alloc_map)) {
 88			end = find_next_bit(chunk->bound_map, last_alloc,
 89					    start + 1);
 90			alloc_sizes[as_len] = 1;
 91		} else {
 92			end = find_next_bit(chunk->alloc_map, last_alloc,
 93					    start + 1);
 94			alloc_sizes[as_len] = -1;
 95		}
 96
 97		alloc_sizes[as_len++] *= (end - start) * PCPU_MIN_ALLOC_SIZE;
 98
 99		start = end;
100	}
101
102	/*
103	 * The negative values are free fragments and thus sorting gives the
104	 * free fragments at the beginning in largest first order.
105	 */
106	if (as_len > 0) {
107		sort(alloc_sizes, as_len, sizeof(int), cmpint, NULL);
108
109		/* iterate through the unallocated fragments */
110		for (i = 0, p = alloc_sizes; *p < 0 && i < as_len; i++, p++) {
111			sum_frag -= *p;
112			max_frag = max(max_frag, -1 * (*p));
113		}
114
115		cur_min_alloc = alloc_sizes[i];
116		cur_med_alloc = alloc_sizes[(i + as_len - 1) / 2];
117		cur_max_alloc = alloc_sizes[as_len - 1];
118	}
119
120	P("nr_alloc", chunk->nr_alloc);
121	P("max_alloc_size", chunk->max_alloc_size);
122	P("empty_pop_pages", chunk->nr_empty_pop_pages);
123	P("first_bit", chunk_md->first_free);
124	P("free_bytes", chunk->free_bytes);
125	P("contig_bytes", chunk_md->contig_hint * PCPU_MIN_ALLOC_SIZE);
126	P("sum_frag", sum_frag);
127	P("max_frag", max_frag);
128	P("cur_min_alloc", cur_min_alloc);
129	P("cur_med_alloc", cur_med_alloc);
130	P("cur_max_alloc", cur_max_alloc);
131	seq_putc(m, '\n');
132}
133
134static int percpu_stats_show(struct seq_file *m, void *v)
135{
136	struct pcpu_chunk *chunk;
137	int slot, max_nr_alloc;
138	int *buffer;
139
140alloc_buffer:
141	spin_lock_irq(&pcpu_lock);
142	max_nr_alloc = find_max_nr_alloc();
143	spin_unlock_irq(&pcpu_lock);
144
145	/* there can be at most this many free and allocated fragments */
146	buffer = vmalloc_array(2 * max_nr_alloc + 1, sizeof(int));
147	if (!buffer)
148		return -ENOMEM;
149
150	spin_lock_irq(&pcpu_lock);
151
152	/* if the buffer allocated earlier is too small */
153	if (max_nr_alloc < find_max_nr_alloc()) {
154		spin_unlock_irq(&pcpu_lock);
155		vfree(buffer);
156		goto alloc_buffer;
157	}
158
159#define PL(X)								\
160	seq_printf(m, "  %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X)
161
162	seq_printf(m,
163			"Percpu Memory Statistics\n"
164			"Allocation Info:\n"
165			"----------------------------------------\n");
166	PL(unit_size);
167	PL(static_size);
168	PL(reserved_size);
169	PL(dyn_size);
170	PL(atom_size);
171	PL(alloc_size);
172	seq_putc(m, '\n');
173
174#undef PL
175
176#define PU(X) \
177	seq_printf(m, "  %-20s: %12llu\n", #X, (unsigned long long)pcpu_stats.X)
178
179	seq_printf(m,
180			"Global Stats:\n"
181			"----------------------------------------\n");
182	PU(nr_alloc);
183	PU(nr_dealloc);
184	PU(nr_cur_alloc);
185	PU(nr_max_alloc);
186	PU(nr_chunks);
187	PU(nr_max_chunks);
188	PU(min_alloc_size);
189	PU(max_alloc_size);
190	P("empty_pop_pages", pcpu_nr_empty_pop_pages);
191	seq_putc(m, '\n');
192
193#undef PU
194
195	seq_printf(m,
196			"Per Chunk Stats:\n"
197			"----------------------------------------\n");
198
199	if (pcpu_reserved_chunk) {
200		seq_puts(m, "Chunk: <- Reserved Chunk\n");
201		chunk_map_stats(m, pcpu_reserved_chunk, buffer);
202	}
203
204	for (slot = 0; slot < pcpu_nr_slots; slot++) {
205		list_for_each_entry(chunk, &pcpu_chunk_lists[slot], list) {
206			if (chunk == pcpu_first_chunk)
207				seq_puts(m, "Chunk: <- First Chunk\n");
208			else if (slot == pcpu_to_depopulate_slot)
209				seq_puts(m, "Chunk (to_depopulate)\n");
210			else if (slot == pcpu_sidelined_slot)
211				seq_puts(m, "Chunk (sidelined):\n");
212			else
213				seq_puts(m, "Chunk:\n");
214			chunk_map_stats(m, chunk, buffer);
215		}
216	}
217
218	spin_unlock_irq(&pcpu_lock);
219
220	vfree(buffer);
221
222	return 0;
223}
224DEFINE_SHOW_ATTRIBUTE(percpu_stats);
225
226static int __init init_percpu_stats_debugfs(void)
227{
228	debugfs_create_file("percpu_stats", 0444, NULL, NULL,
229			&percpu_stats_fops);
230
231	return 0;
232}
233
234late_initcall(init_percpu_stats_debugfs);