thp_settings.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <fcntl.h>
  3#include <limits.h>
  4#include <stdio.h>
  5#include <stdlib.h>
  6#include <string.h>
  7#include <unistd.h>
  8
  9#include "thp_settings.h"
 10
 11#define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/"
 12#define MAX_SETTINGS_DEPTH 4
 13static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH];
 14static int settings_index;
 15static struct thp_settings saved_settings;
 16static char dev_queue_read_ahead_path[PATH_MAX];
 17
 18static const char * const thp_enabled_strings[] = {
 19	"never",
 20	"always",
 21	"inherit",
 22	"madvise",
 23	NULL
 24};
 25
 26static const char * const thp_defrag_strings[] = {
 27	"always",
 28	"defer",
 29	"defer+madvise",
 30	"madvise",
 31	"never",
 32	NULL
 33};
 34
 35static const char * const shmem_enabled_strings[] = {
 36	"never",
 37	"always",
 38	"within_size",
 39	"advise",
 40	"inherit",
 41	"deny",
 42	"force",
 43	NULL
 44};
 45
 46int read_file(const char *path, char *buf, size_t buflen)
 47{
 48	int fd;
 49	ssize_t numread;
 50
 51	fd = open(path, O_RDONLY);
 52	if (fd == -1)
 53		return 0;
 54
 55	numread = read(fd, buf, buflen - 1);
 56	if (numread < 1) {
 57		close(fd);
 58		return 0;
 59	}
 60
 61	buf[numread] = '\0';
 62	close(fd);
 63
 64	return (unsigned int) numread;
 65}
 66
 67int write_file(const char *path, const char *buf, size_t buflen)
 68{
 69	int fd;
 70	ssize_t numwritten;
 71
 72	fd = open(path, O_WRONLY);
 73	if (fd == -1) {
 74		printf("open(%s)\n", path);
 75		exit(EXIT_FAILURE);
 76		return 0;
 77	}
 78
 79	numwritten = write(fd, buf, buflen - 1);
 80	close(fd);
 81	if (numwritten < 1) {
 82		printf("write(%s)\n", buf);
 83		exit(EXIT_FAILURE);
 84		return 0;
 85	}
 86
 87	return (unsigned int) numwritten;
 88}
 89
 90unsigned long read_num(const char *path)
 91{
 92	char buf[21];
 93
 94	if (read_file(path, buf, sizeof(buf)) < 0) {
 95		perror("read_file()");
 96		exit(EXIT_FAILURE);
 97	}
 98
 99	return strtoul(buf, NULL, 10);
100}
101
102void write_num(const char *path, unsigned long num)
103{
104	char buf[21];
105
106	sprintf(buf, "%ld", num);
107	if (!write_file(path, buf, strlen(buf) + 1)) {
108		perror(path);
109		exit(EXIT_FAILURE);
110	}
111}
112
113int thp_read_string(const char *name, const char * const strings[])
114{
115	char path[PATH_MAX];
116	char buf[256];
117	char *c;
118	int ret;
119
120	ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
121	if (ret >= PATH_MAX) {
122		printf("%s: Pathname is too long\n", __func__);
123		exit(EXIT_FAILURE);
124	}
125
126	if (!read_file(path, buf, sizeof(buf))) {
127		perror(path);
128		exit(EXIT_FAILURE);
129	}
130
131	c = strchr(buf, '[');
132	if (!c) {
133		printf("%s: Parse failure\n", __func__);
134		exit(EXIT_FAILURE);
135	}
136
137	c++;
138	memmove(buf, c, sizeof(buf) - (c - buf));
139
140	c = strchr(buf, ']');
141	if (!c) {
142		printf("%s: Parse failure\n", __func__);
143		exit(EXIT_FAILURE);
144	}
145	*c = '\0';
146
147	ret = 0;
148	while (strings[ret]) {
149		if (!strcmp(strings[ret], buf))
150			return ret;
151		ret++;
152	}
153
154	printf("Failed to parse %s\n", name);
155	exit(EXIT_FAILURE);
156}
157
158void thp_write_string(const char *name, const char *val)
159{
160	char path[PATH_MAX];
161	int ret;
162
163	ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
164	if (ret >= PATH_MAX) {
165		printf("%s: Pathname is too long\n", __func__);
166		exit(EXIT_FAILURE);
167	}
168
169	if (!write_file(path, val, strlen(val) + 1)) {
170		perror(path);
171		exit(EXIT_FAILURE);
172	}
173}
174
175unsigned long thp_read_num(const char *name)
176{
177	char path[PATH_MAX];
178	int ret;
179
180	ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
181	if (ret >= PATH_MAX) {
182		printf("%s: Pathname is too long\n", __func__);
183		exit(EXIT_FAILURE);
184	}
185	return read_num(path);
186}
187
188void thp_write_num(const char *name, unsigned long num)
189{
190	char path[PATH_MAX];
191	int ret;
192
193	ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
194	if (ret >= PATH_MAX) {
195		printf("%s: Pathname is too long\n", __func__);
196		exit(EXIT_FAILURE);
197	}
198	write_num(path, num);
199}
200
201void thp_read_settings(struct thp_settings *settings)
202{
203	unsigned long orders = thp_supported_orders();
204	unsigned long shmem_orders = thp_shmem_supported_orders();
205	char path[PATH_MAX];
206	int i;
207
208	*settings = (struct thp_settings) {
209		.thp_enabled = thp_read_string("enabled", thp_enabled_strings),
210		.thp_defrag = thp_read_string("defrag", thp_defrag_strings),
211		.shmem_enabled =
212			thp_read_string("shmem_enabled", shmem_enabled_strings),
213		.use_zero_page = thp_read_num("use_zero_page"),
214	};
215	settings->khugepaged = (struct khugepaged_settings) {
216		.defrag = thp_read_num("khugepaged/defrag"),
217		.alloc_sleep_millisecs =
218			thp_read_num("khugepaged/alloc_sleep_millisecs"),
219		.scan_sleep_millisecs =
220			thp_read_num("khugepaged/scan_sleep_millisecs"),
221		.max_ptes_none = thp_read_num("khugepaged/max_ptes_none"),
222		.max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"),
223		.max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"),
224		.pages_to_scan = thp_read_num("khugepaged/pages_to_scan"),
225	};
226	if (dev_queue_read_ahead_path[0])
227		settings->read_ahead_kb = read_num(dev_queue_read_ahead_path);
228
229	for (i = 0; i < NR_ORDERS; i++) {
230		if (!((1 << i) & orders)) {
231			settings->hugepages[i].enabled = THP_NEVER;
232			continue;
233		}
234		snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",
235			(getpagesize() >> 10) << i);
236		settings->hugepages[i].enabled =
237			thp_read_string(path, thp_enabled_strings);
238	}
239
240	for (i = 0; i < NR_ORDERS; i++) {
241		if (!((1 << i) & shmem_orders)) {
242			settings->shmem_hugepages[i].enabled = SHMEM_NEVER;
243			continue;
244		}
245		snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled",
246			(getpagesize() >> 10) << i);
247		settings->shmem_hugepages[i].enabled =
248			thp_read_string(path, shmem_enabled_strings);
249	}
250}
251
252void thp_write_settings(struct thp_settings *settings)
253{
254	struct khugepaged_settings *khugepaged = &settings->khugepaged;
255	unsigned long orders = thp_supported_orders();
256	unsigned long shmem_orders = thp_shmem_supported_orders();
257	char path[PATH_MAX];
258	int enabled;
259	int i;
260
261	thp_write_string("enabled", thp_enabled_strings[settings->thp_enabled]);
262	thp_write_string("defrag", thp_defrag_strings[settings->thp_defrag]);
263	thp_write_string("shmem_enabled",
264			shmem_enabled_strings[settings->shmem_enabled]);
265	thp_write_num("use_zero_page", settings->use_zero_page);
266
267	thp_write_num("khugepaged/defrag", khugepaged->defrag);
268	thp_write_num("khugepaged/alloc_sleep_millisecs",
269			khugepaged->alloc_sleep_millisecs);
270	thp_write_num("khugepaged/scan_sleep_millisecs",
271			khugepaged->scan_sleep_millisecs);
272	thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none);
273	thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap);
274	thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared);
275	thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan);
276
277	if (dev_queue_read_ahead_path[0])
278		write_num(dev_queue_read_ahead_path, settings->read_ahead_kb);
279
280	for (i = 0; i < NR_ORDERS; i++) {
281		if (!((1 << i) & orders))
282			continue;
283		snprintf(path, PATH_MAX, "hugepages-%ukB/enabled",
284			(getpagesize() >> 10) << i);
285		enabled = settings->hugepages[i].enabled;
286		thp_write_string(path, thp_enabled_strings[enabled]);
287	}
288
289	for (i = 0; i < NR_ORDERS; i++) {
290		if (!((1 << i) & shmem_orders))
291			continue;
292		snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled",
293			(getpagesize() >> 10) << i);
294		enabled = settings->shmem_hugepages[i].enabled;
295		thp_write_string(path, shmem_enabled_strings[enabled]);
296	}
297}
298
299struct thp_settings *thp_current_settings(void)
300{
301	if (!settings_index) {
302		printf("Fail: No settings set");
303		exit(EXIT_FAILURE);
304	}
305	return settings_stack + settings_index - 1;
306}
307
308void thp_push_settings(struct thp_settings *settings)
309{
310	if (settings_index >= MAX_SETTINGS_DEPTH) {
311		printf("Fail: Settings stack exceeded");
312		exit(EXIT_FAILURE);
313	}
314	settings_stack[settings_index++] = *settings;
315	thp_write_settings(thp_current_settings());
316}
317
318void thp_pop_settings(void)
319{
320	if (settings_index <= 0) {
321		printf("Fail: Settings stack empty");
322		exit(EXIT_FAILURE);
323	}
324	--settings_index;
325	thp_write_settings(thp_current_settings());
326}
327
328void thp_restore_settings(void)
329{
330	thp_write_settings(&saved_settings);
331}
332
333void thp_save_settings(void)
334{
335	thp_read_settings(&saved_settings);
336}
337
338void thp_set_read_ahead_path(char *path)
339{
340	if (!path) {
341		dev_queue_read_ahead_path[0] = '\0';
342		return;
343	}
344
345	strncpy(dev_queue_read_ahead_path, path,
346		sizeof(dev_queue_read_ahead_path));
347	dev_queue_read_ahead_path[sizeof(dev_queue_read_ahead_path) - 1] = '\0';
348}
349
350static unsigned long __thp_supported_orders(bool is_shmem)
351{
352	unsigned long orders = 0;
353	char path[PATH_MAX];
354	char buf[256];
355	int ret, i;
356	char anon_dir[] = "enabled";
357	char shmem_dir[] = "shmem_enabled";
358
359	for (i = 0; i < NR_ORDERS; i++) {
360		ret = snprintf(path, PATH_MAX, THP_SYSFS "hugepages-%ukB/%s",
361			       (getpagesize() >> 10) << i, is_shmem ? shmem_dir : anon_dir);
362		if (ret >= PATH_MAX) {
363			printf("%s: Pathname is too long\n", __func__);
364			exit(EXIT_FAILURE);
365		}
366
367		ret = read_file(path, buf, sizeof(buf));
368		if (ret)
369			orders |= 1UL << i;
370	}
371
372	return orders;
373}
374
375unsigned long thp_supported_orders(void)
376{
377	return __thp_supported_orders(false);
378}
379
380unsigned long thp_shmem_supported_orders(void)
381{
382	return __thp_supported_orders(true);
383}
384
385bool thp_available(void)
386{
387	if (access(THP_SYSFS, F_OK) != 0)
388		return false;
389	return true;
390}
391
392bool thp_is_enabled(void)
393{
394	if (!thp_available())
395		return false;
396
397	int mode = thp_read_string("enabled", thp_enabled_strings);
398
399	/* THP is considered enabled if it's either "always" or "madvise" */
400	return mode == 1 || mode == 3;
401}