lru_gen_util.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2025, Google LLC.
  4 */
  5
  6#include <time.h>
  7
  8#include "lru_gen_util.h"
  9
 10/*
 11 * Tracks state while we parse memcg lru_gen stats. The file we're parsing is
 12 * structured like this (some extra whitespace elided):
 13 *
 14 * memcg (id) (path)
 15 * node (id)
 16 * (gen_nr) (age_in_ms) (nr_anon_pages) (nr_file_pages)
 17 */
 18struct memcg_stats_parse_context {
 19	bool consumed; /* Whether or not this line was consumed */
 20	/* Next parse handler to invoke */
 21	void (*next_handler)(struct memcg_stats *stats,
 22			     struct memcg_stats_parse_context *ctx,
 23			     char *line);
 24	int current_node_idx; /* Current index in nodes array */
 25	const char *name; /* The name of the memcg we're looking for */
 26};
 27
 28static void memcg_stats_handle_searching(struct memcg_stats *stats,
 29					 struct memcg_stats_parse_context *ctx,
 30					 char *line);
 31static void memcg_stats_handle_in_memcg(struct memcg_stats *stats,
 32					struct memcg_stats_parse_context *ctx,
 33					char *line);
 34static void memcg_stats_handle_in_node(struct memcg_stats *stats,
 35				       struct memcg_stats_parse_context *ctx,
 36				       char *line);
 37
 38struct split_iterator {
 39	char *str;
 40	char *save;
 41};
 42
 43static char *split_next(struct split_iterator *it)
 44{
 45	char *ret = strtok_r(it->str, " \t\n\r", &it->save);
 46
 47	it->str = NULL;
 48	return ret;
 49}
 50
 51static void memcg_stats_handle_searching(struct memcg_stats *stats,
 52					 struct memcg_stats_parse_context *ctx,
 53					 char *line)
 54{
 55	struct split_iterator it = { .str = line };
 56	char *prefix = split_next(&it);
 57	char *memcg_id = split_next(&it);
 58	char *memcg_name = split_next(&it);
 59	char *end;
 60
 61	ctx->consumed = true;
 62
 63	if (!prefix || strcmp("memcg", prefix))
 64		return; /* Not a memcg line (maybe empty), skip */
 65
 66	TEST_ASSERT(memcg_id && memcg_name,
 67		    "malformed memcg line; no memcg id or memcg_name");
 68
 69	if (strcmp(memcg_name + 1, ctx->name))
 70		return; /* Wrong memcg, skip */
 71
 72	/* Found it! */
 73
 74	stats->memcg_id = strtoul(memcg_id, &end, 10);
 75	TEST_ASSERT(*end == '\0', "malformed memcg id '%s'", memcg_id);
 76	if (!stats->memcg_id)
 77		return; /* Removed memcg? */
 78
 79	ctx->next_handler = memcg_stats_handle_in_memcg;
 80}
 81
 82static void memcg_stats_handle_in_memcg(struct memcg_stats *stats,
 83					struct memcg_stats_parse_context *ctx,
 84					char *line)
 85{
 86	struct split_iterator it = { .str = line };
 87	char *prefix = split_next(&it);
 88	char *id = split_next(&it);
 89	long found_node_id;
 90	char *end;
 91
 92	ctx->consumed = true;
 93	ctx->current_node_idx = -1;
 94
 95	if (!prefix)
 96		return; /* Skip empty lines */
 97
 98	if (!strcmp("memcg", prefix)) {
 99		/* Memcg done, found next one; stop. */
100		ctx->next_handler = NULL;
101		return;
102	} else if (strcmp("node", prefix))
103		TEST_ASSERT(false, "found malformed line after 'memcg ...',"
104				   "token: '%s'", prefix);
105
106	/* At this point we know we have a node line. Parse the ID. */
107
108	TEST_ASSERT(id, "malformed node line; no node id");
109
110	found_node_id = strtol(id, &end, 10);
111	TEST_ASSERT(*end == '\0', "malformed node id '%s'", id);
112
113	ctx->current_node_idx = stats->nr_nodes++;
114	TEST_ASSERT(ctx->current_node_idx < MAX_NR_NODES,
115		    "memcg has stats for too many nodes, max is %d",
116		    MAX_NR_NODES);
117	stats->nodes[ctx->current_node_idx].node = found_node_id;
118
119	ctx->next_handler = memcg_stats_handle_in_node;
120}
121
122static void memcg_stats_handle_in_node(struct memcg_stats *stats,
123				       struct memcg_stats_parse_context *ctx,
124				       char *line)
125{
126	char *my_line = strdup(line);
127	struct split_iterator it = { .str = my_line };
128	char *gen, *age, *nr_anon, *nr_file;
129	struct node_stats *node_stats;
130	struct generation_stats *gen_stats;
131	char *end;
132
133	TEST_ASSERT(it.str, "failed to copy input line");
134
135	gen = split_next(&it);
136
137	if (!gen)
138		goto out_consume; /* Skip empty lines */
139
140	if (!strcmp("memcg", gen) || !strcmp("node", gen)) {
141		/*
142		 * Reached next memcg or node section. Don't consume, let the
143		 * other handler deal with this.
144		 */
145		ctx->next_handler = memcg_stats_handle_in_memcg;
146		goto out;
147	}
148
149	node_stats = &stats->nodes[ctx->current_node_idx];
150	TEST_ASSERT(node_stats->nr_gens < MAX_NR_GENS,
151		    "found too many generation lines; max is %d",
152		    MAX_NR_GENS);
153	gen_stats = &node_stats->gens[node_stats->nr_gens++];
154
155	age = split_next(&it);
156	nr_anon = split_next(&it);
157	nr_file = split_next(&it);
158
159	TEST_ASSERT(age && nr_anon && nr_file,
160		    "malformed generation line; not enough tokens");
161
162	gen_stats->gen = (int)strtol(gen, &end, 10);
163	TEST_ASSERT(*end == '\0', "malformed generation number '%s'", gen);
164
165	gen_stats->age_ms = strtol(age, &end, 10);
166	TEST_ASSERT(*end == '\0', "malformed generation age '%s'", age);
167
168	gen_stats->nr_anon = strtol(nr_anon, &end, 10);
169	TEST_ASSERT(*end == '\0', "malformed anonymous page count '%s'",
170		    nr_anon);
171
172	gen_stats->nr_file = strtol(nr_file, &end, 10);
173	TEST_ASSERT(*end == '\0', "malformed file page count '%s'", nr_file);
174
175out_consume:
176	ctx->consumed = true;
177out:
178	free(my_line);
179}
180
181static void print_memcg_stats(const struct memcg_stats *stats, const char *name)
182{
183	int node, gen;
184
185	pr_debug("stats for memcg %s (id %lu):\n", name, stats->memcg_id);
186	for (node = 0; node < stats->nr_nodes; ++node) {
187		pr_debug("\tnode %d\n", stats->nodes[node].node);
188		for (gen = 0; gen < stats->nodes[node].nr_gens; ++gen) {
189			const struct generation_stats *gstats =
190				&stats->nodes[node].gens[gen];
191
192			pr_debug("\t\tgen %d\tage_ms %ld"
193				 "\tnr_anon %ld\tnr_file %ld\n",
194				 gstats->gen, gstats->age_ms, gstats->nr_anon,
195				 gstats->nr_file);
196		}
197	}
198}
199
200/* Re-read lru_gen debugfs information for @memcg into @stats. */
201void lru_gen_read_memcg_stats(struct memcg_stats *stats, const char *memcg)
202{
203	FILE *f;
204	ssize_t read = 0;
205	char *line = NULL;
206	size_t bufsz;
207	struct memcg_stats_parse_context ctx = {
208		.next_handler = memcg_stats_handle_searching,
209		.name = memcg,
210	};
211
212	memset(stats, 0, sizeof(struct memcg_stats));
213
214	f = fopen(LRU_GEN_DEBUGFS, "r");
215	TEST_ASSERT(f, "fopen(%s) failed", LRU_GEN_DEBUGFS);
216
217	while (ctx.next_handler && (read = getline(&line, &bufsz, f)) > 0) {
218		ctx.consumed = false;
219
220		do {
221			ctx.next_handler(stats, &ctx, line);
222			if (!ctx.next_handler)
223				break;
224		} while (!ctx.consumed);
225	}
226
227	if (read < 0 && !feof(f))
228		TEST_ASSERT(false, "getline(%s) failed", LRU_GEN_DEBUGFS);
229
230	TEST_ASSERT(stats->memcg_id > 0, "Couldn't find memcg: %s\n"
231		    "Did the memcg get created in the proper mount?",
232		    memcg);
233	if (line)
234		free(line);
235	TEST_ASSERT(!fclose(f), "fclose(%s) failed", LRU_GEN_DEBUGFS);
236
237	print_memcg_stats(stats, memcg);
238}
239
240/*
241 * Find all pages tracked by lru_gen for this memcg in generation @target_gen.
242 *
243 * If @target_gen is negative, look for all generations.
244 */
245long lru_gen_sum_memcg_stats_for_gen(int target_gen,
246				     const struct memcg_stats *stats)
247{
248	int node, gen;
249	long total_nr = 0;
250
251	for (node = 0; node < stats->nr_nodes; ++node) {
252		const struct node_stats *node_stats = &stats->nodes[node];
253
254		for (gen = 0; gen < node_stats->nr_gens; ++gen) {
255			const struct generation_stats *gen_stats =
256				&node_stats->gens[gen];
257
258			if (target_gen >= 0 && gen_stats->gen != target_gen)
259				continue;
260
261			total_nr += gen_stats->nr_anon + gen_stats->nr_file;
262		}
263	}
264
265	return total_nr;
266}
267
268/* Find all pages tracked by lru_gen for this memcg. */
269long lru_gen_sum_memcg_stats(const struct memcg_stats *stats)
270{
271	return lru_gen_sum_memcg_stats_for_gen(-1, stats);
272}
273
274/*
275 * If lru_gen aging should force page table scanning.
276 *
277 * If you want to set this to false, you will need to do eviction
278 * before doing extra aging passes.
279 */
280static const bool force_scan = true;
281
282static void run_aging_impl(unsigned long memcg_id, int node_id, int max_gen)
283{
284	FILE *f = fopen(LRU_GEN_DEBUGFS, "w");
285	char *command;
286	size_t sz;
287
288	TEST_ASSERT(f, "fopen(%s) failed", LRU_GEN_DEBUGFS);
289	sz = asprintf(&command, "+ %lu %d %d 1 %d\n",
290		      memcg_id, node_id, max_gen, force_scan);
291	TEST_ASSERT(sz > 0, "creating aging command failed");
292
293	pr_debug("Running aging command: %s", command);
294	if (fwrite(command, sizeof(char), sz, f) < sz) {
295		TEST_ASSERT(false, "writing aging command %s to %s failed",
296			    command, LRU_GEN_DEBUGFS);
297	}
298
299	TEST_ASSERT(!fclose(f), "fclose(%s) failed", LRU_GEN_DEBUGFS);
300}
301
302void lru_gen_do_aging(struct memcg_stats *stats, const char *memcg)
303{
304	int node, gen;
305
306	pr_debug("lru_gen: invoking aging...\n");
307
308	/* Must read memcg stats to construct the proper aging command. */
309	lru_gen_read_memcg_stats(stats, memcg);
310
311	for (node = 0; node < stats->nr_nodes; ++node) {
312		int max_gen = 0;
313
314		for (gen = 0; gen < stats->nodes[node].nr_gens; ++gen) {
315			int this_gen = stats->nodes[node].gens[gen].gen;
316
317			max_gen = max_gen > this_gen ? max_gen : this_gen;
318		}
319
320		run_aging_impl(stats->memcg_id, stats->nodes[node].node,
321			       max_gen);
322	}
323
324	/* Re-read so callers get updated information */
325	lru_gen_read_memcg_stats(stats, memcg);
326}
327
328/*
329 * Find which generation contains at least @pages pages, assuming that
330 * such a generation exists.
331 */
332int lru_gen_find_generation(const struct memcg_stats *stats,
333			    unsigned long pages)
334{
335	int node, gen, gen_idx, min_gen = INT_MAX, max_gen = -1;
336
337	for (node = 0; node < stats->nr_nodes; ++node)
338		for (gen_idx = 0; gen_idx < stats->nodes[node].nr_gens;
339		     ++gen_idx) {
340			gen = stats->nodes[node].gens[gen_idx].gen;
341			max_gen = gen > max_gen ? gen : max_gen;
342			min_gen = gen < min_gen ? gen : min_gen;
343		}
344
345	for (gen = min_gen; gen <= max_gen; ++gen)
346		/* See if this generation has enough pages. */
347		if (lru_gen_sum_memcg_stats_for_gen(gen, stats) > pages)
348			return gen;
349
350	return -1;
351}
352
353bool lru_gen_usable(void)
354{
355	long required_features = LRU_GEN_ENABLED | LRU_GEN_MM_WALK;
356	int lru_gen_fd, lru_gen_debug_fd;
357	char mglru_feature_str[8] = {};
358	long mglru_features;
359
360	lru_gen_fd = open(LRU_GEN_ENABLED_PATH, O_RDONLY);
361	if (lru_gen_fd < 0) {
362		puts("lru_gen: Could not open " LRU_GEN_ENABLED_PATH);
363		return false;
364	}
365	if (read(lru_gen_fd, &mglru_feature_str, 7) < 7) {
366		puts("lru_gen: Could not read from " LRU_GEN_ENABLED_PATH);
367		close(lru_gen_fd);
368		return false;
369	}
370	close(lru_gen_fd);
371
372	mglru_features = strtol(mglru_feature_str, NULL, 16);
373	if ((mglru_features & required_features) != required_features) {
374		printf("lru_gen: missing features, got: 0x%lx, expected: 0x%lx\n",
375		       mglru_features, required_features);
376		printf("lru_gen: Try 'echo 0x%lx > /sys/kernel/mm/lru_gen/enabled'\n",
377		       required_features);
378		return false;
379	}
380
381	lru_gen_debug_fd = open(LRU_GEN_DEBUGFS, O_RDWR);
382	__TEST_REQUIRE(lru_gen_debug_fd >= 0,
383		       "lru_gen: Could not open " LRU_GEN_DEBUGFS ", "
384		       "but lru_gen is enabled, so cannot use page_idle.");
385	close(lru_gen_debug_fd);
386	return true;
387}