hugetlb_sysfs.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * HugeTLB sysfs interfaces.
  4 * (C) Nadia Yvette Chambers, April 2004
  5 */
  6
  7#include <linux/swap.h>
  8#include <linux/page_owner.h>
  9#include <linux/page-isolation.h>
 10
 11#include "hugetlb_vmemmap.h"
 12#include "hugetlb_internal.h"
 13
 14#define HSTATE_ATTR_RO(_name) \
 15	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 16
 17#define HSTATE_ATTR_WO(_name) \
 18	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)
 19
 20#define HSTATE_ATTR(_name) \
 21	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)
 22
 23static struct kobject *hugepages_kobj;
 24static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
 25
 26static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
 27
 28static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
 29{
 30	int i;
 31
 32	for (i = 0; i < HUGE_MAX_HSTATE; i++)
 33		if (hstate_kobjs[i] == kobj) {
 34			if (nidp)
 35				*nidp = NUMA_NO_NODE;
 36			return &hstates[i];
 37		}
 38
 39	return kobj_to_node_hstate(kobj, nidp);
 40}
 41
 42static ssize_t nr_hugepages_show_common(struct kobject *kobj,
 43					struct kobj_attribute *attr, char *buf)
 44{
 45	struct hstate *h;
 46	unsigned long nr_huge_pages;
 47	int nid;
 48
 49	h = kobj_to_hstate(kobj, &nid);
 50	if (nid == NUMA_NO_NODE)
 51		nr_huge_pages = h->nr_huge_pages;
 52	else
 53		nr_huge_pages = h->nr_huge_pages_node[nid];
 54
 55	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
 56}
 57
 58static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
 59					 struct kobject *kobj, const char *buf,
 60					 size_t len)
 61{
 62	struct hstate *h;
 63	unsigned long count;
 64	int nid;
 65	int err;
 66
 67	err = kstrtoul(buf, 10, &count);
 68	if (err)
 69		return err;
 70
 71	h = kobj_to_hstate(kobj, &nid);
 72	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
 73}
 74
 75static ssize_t nr_hugepages_show(struct kobject *kobj,
 76				       struct kobj_attribute *attr, char *buf)
 77{
 78	return nr_hugepages_show_common(kobj, attr, buf);
 79}
 80
 81static ssize_t nr_hugepages_store(struct kobject *kobj,
 82	       struct kobj_attribute *attr, const char *buf, size_t len)
 83{
 84	return nr_hugepages_store_common(false, kobj, buf, len);
 85}
 86HSTATE_ATTR(nr_hugepages);
 87
 88#ifdef CONFIG_NUMA
 89
 90/*
 91 * hstate attribute for optionally mempolicy-based constraint on persistent
 92 * huge page alloc/free.
 93 */
 94static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
 95					   struct kobj_attribute *attr,
 96					   char *buf)
 97{
 98	return nr_hugepages_show_common(kobj, attr, buf);
 99}
100
101static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
102	       struct kobj_attribute *attr, const char *buf, size_t len)
103{
104	return nr_hugepages_store_common(true, kobj, buf, len);
105}
106HSTATE_ATTR(nr_hugepages_mempolicy);
107#endif
108
109
110static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
111					struct kobj_attribute *attr, char *buf)
112{
113	struct hstate *h = kobj_to_hstate(kobj, NULL);
114	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
115}
116
117static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
118		struct kobj_attribute *attr, const char *buf, size_t count)
119{
120	int err;
121	unsigned long input;
122	struct hstate *h = kobj_to_hstate(kobj, NULL);
123
124	if (hstate_is_gigantic_no_runtime(h))
125		return -EINVAL;
126
127	err = kstrtoul(buf, 10, &input);
128	if (err)
129		return err;
130
131	spin_lock_irq(&hugetlb_lock);
132	h->nr_overcommit_huge_pages = input;
133	spin_unlock_irq(&hugetlb_lock);
134
135	return count;
136}
137HSTATE_ATTR(nr_overcommit_hugepages);
138
139static ssize_t free_hugepages_show(struct kobject *kobj,
140					struct kobj_attribute *attr, char *buf)
141{
142	struct hstate *h;
143	unsigned long free_huge_pages;
144	int nid;
145
146	h = kobj_to_hstate(kobj, &nid);
147	if (nid == NUMA_NO_NODE)
148		free_huge_pages = h->free_huge_pages;
149	else
150		free_huge_pages = h->free_huge_pages_node[nid];
151
152	return sysfs_emit(buf, "%lu\n", free_huge_pages);
153}
154HSTATE_ATTR_RO(free_hugepages);
155
156static ssize_t resv_hugepages_show(struct kobject *kobj,
157					struct kobj_attribute *attr, char *buf)
158{
159	struct hstate *h = kobj_to_hstate(kobj, NULL);
160	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
161}
162HSTATE_ATTR_RO(resv_hugepages);
163
164static ssize_t surplus_hugepages_show(struct kobject *kobj,
165					struct kobj_attribute *attr, char *buf)
166{
167	struct hstate *h;
168	unsigned long surplus_huge_pages;
169	int nid;
170
171	h = kobj_to_hstate(kobj, &nid);
172	if (nid == NUMA_NO_NODE)
173		surplus_huge_pages = h->surplus_huge_pages;
174	else
175		surplus_huge_pages = h->surplus_huge_pages_node[nid];
176
177	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
178}
179HSTATE_ATTR_RO(surplus_hugepages);
180
181static ssize_t demote_store(struct kobject *kobj,
182	       struct kobj_attribute *attr, const char *buf, size_t len)
183{
184	unsigned long nr_demote;
185	unsigned long nr_available;
186	nodemask_t nodes_allowed, *n_mask;
187	struct hstate *h;
188	int err;
189	int nid;
190
191	err = kstrtoul(buf, 10, &nr_demote);
192	if (err)
193		return err;
194	h = kobj_to_hstate(kobj, &nid);
195
196	if (nid != NUMA_NO_NODE) {
197		init_nodemask_of_node(&nodes_allowed, nid);
198		n_mask = &nodes_allowed;
199	} else {
200		n_mask = &node_states[N_MEMORY];
201	}
202
203	/* Synchronize with other sysfs operations modifying huge pages */
204	mutex_lock(&h->resize_lock);
205	spin_lock_irq(&hugetlb_lock);
206
207	while (nr_demote) {
208		long rc;
209
210		/*
211		 * Check for available pages to demote each time thorough the
212		 * loop as demote_pool_huge_page will drop hugetlb_lock.
213		 */
214		if (nid != NUMA_NO_NODE)
215			nr_available = h->free_huge_pages_node[nid];
216		else
217			nr_available = h->free_huge_pages;
218		nr_available -= h->resv_huge_pages;
219		if (!nr_available)
220			break;
221
222		rc = demote_pool_huge_page(h, n_mask, nr_demote);
223		if (rc < 0) {
224			err = rc;
225			break;
226		}
227
228		nr_demote -= rc;
229	}
230
231	spin_unlock_irq(&hugetlb_lock);
232	mutex_unlock(&h->resize_lock);
233
234	if (err)
235		return err;
236	return len;
237}
238HSTATE_ATTR_WO(demote);
239
240static ssize_t demote_size_show(struct kobject *kobj,
241					struct kobj_attribute *attr, char *buf)
242{
243	struct hstate *h = kobj_to_hstate(kobj, NULL);
244	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;
245
246	return sysfs_emit(buf, "%lukB\n", demote_size);
247}
248
249static ssize_t demote_size_store(struct kobject *kobj,
250					struct kobj_attribute *attr,
251					const char *buf, size_t count)
252{
253	struct hstate *h, *demote_hstate;
254	unsigned long demote_size;
255	unsigned int demote_order;
256
257	demote_size = (unsigned long)memparse(buf, NULL);
258
259	demote_hstate = size_to_hstate(demote_size);
260	if (!demote_hstate)
261		return -EINVAL;
262	demote_order = demote_hstate->order;
263	if (demote_order < HUGETLB_PAGE_ORDER)
264		return -EINVAL;
265
266	/* demote order must be smaller than hstate order */
267	h = kobj_to_hstate(kobj, NULL);
268	if (demote_order >= h->order)
269		return -EINVAL;
270
271	/* resize_lock synchronizes access to demote size and writes */
272	mutex_lock(&h->resize_lock);
273	h->demote_order = demote_order;
274	mutex_unlock(&h->resize_lock);
275
276	return count;
277}
278HSTATE_ATTR(demote_size);
279
280static struct attribute *hstate_attrs[] = {
281	&nr_hugepages_attr.attr,
282	&nr_overcommit_hugepages_attr.attr,
283	&free_hugepages_attr.attr,
284	&resv_hugepages_attr.attr,
285	&surplus_hugepages_attr.attr,
286#ifdef CONFIG_NUMA
287	&nr_hugepages_mempolicy_attr.attr,
288#endif
289	NULL,
290};
291
292static const struct attribute_group hstate_attr_group = {
293	.attrs = hstate_attrs,
294};
295
296static struct attribute *hstate_demote_attrs[] = {
297	&demote_size_attr.attr,
298	&demote_attr.attr,
299	NULL,
300};
301
302static const struct attribute_group hstate_demote_attr_group = {
303	.attrs = hstate_demote_attrs,
304};
305
306static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
307				    struct kobject **hstate_kobjs,
308				    const struct attribute_group *hstate_attr_group)
309{
310	int retval;
311	int hi = hstate_index(h);
312
313	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
314	if (!hstate_kobjs[hi])
315		return -ENOMEM;
316
317	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
318	if (retval) {
319		kobject_put(hstate_kobjs[hi]);
320		hstate_kobjs[hi] = NULL;
321		return retval;
322	}
323
324	if (h->demote_order) {
325		retval = sysfs_create_group(hstate_kobjs[hi],
326					    &hstate_demote_attr_group);
327		if (retval) {
328			pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
329			sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
330			kobject_put(hstate_kobjs[hi]);
331			hstate_kobjs[hi] = NULL;
332			return retval;
333		}
334	}
335
336	return 0;
337}
338
339#ifdef CONFIG_NUMA
340static bool hugetlb_sysfs_initialized __ro_after_init;
341
342/*
343 * node_hstate/s - associate per node hstate attributes, via their kobjects,
344 * with node devices in node_devices[] using a parallel array.  The array
345 * index of a node device or _hstate == node id.
346 * This is here to avoid any static dependency of the node device driver, in
347 * the base kernel, on the hugetlb module.
348 */
349struct node_hstate {
350	struct kobject		*hugepages_kobj;
351	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
352};
353static struct node_hstate node_hstates[MAX_NUMNODES];
354
355/*
356 * A subset of global hstate attributes for node devices
357 */
358static struct attribute *per_node_hstate_attrs[] = {
359	&nr_hugepages_attr.attr,
360	&free_hugepages_attr.attr,
361	&surplus_hugepages_attr.attr,
362	NULL,
363};
364
365static const struct attribute_group per_node_hstate_attr_group = {
366	.attrs = per_node_hstate_attrs,
367};
368
369/*
370 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
371 * Returns node id via non-NULL nidp.
372 */
373static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
374{
375	int nid;
376
377	for (nid = 0; nid < nr_node_ids; nid++) {
378		struct node_hstate *nhs = &node_hstates[nid];
379		int i;
380		for (i = 0; i < HUGE_MAX_HSTATE; i++)
381			if (nhs->hstate_kobjs[i] == kobj) {
382				if (nidp)
383					*nidp = nid;
384				return &hstates[i];
385			}
386	}
387
388	BUG();
389	return NULL;
390}
391
392/*
393 * Unregister hstate attributes from a single node device.
394 * No-op if no hstate attributes attached.
395 */
396void hugetlb_unregister_node(struct node *node)
397{
398	struct hstate *h;
399	struct node_hstate *nhs = &node_hstates[node->dev.id];
400
401	if (!nhs->hugepages_kobj)
402		return;		/* no hstate attributes */
403
404	for_each_hstate(h) {
405		int idx = hstate_index(h);
406		struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];
407
408		if (!hstate_kobj)
409			continue;
410		if (h->demote_order)
411			sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
412		sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
413		kobject_put(hstate_kobj);
414		nhs->hstate_kobjs[idx] = NULL;
415	}
416
417	kobject_put(nhs->hugepages_kobj);
418	nhs->hugepages_kobj = NULL;
419}
420
421
422/*
423 * Register hstate attributes for a single node device.
424 * No-op if attributes already registered.
425 */
426void hugetlb_register_node(struct node *node)
427{
428	struct hstate *h;
429	struct node_hstate *nhs = &node_hstates[node->dev.id];
430	int err;
431
432	if (!hugetlb_sysfs_initialized)
433		return;
434
435	if (nhs->hugepages_kobj)
436		return;		/* already allocated */
437
438	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
439							&node->dev.kobj);
440	if (!nhs->hugepages_kobj)
441		return;
442
443	for_each_hstate(h) {
444		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
445						nhs->hstate_kobjs,
446						&per_node_hstate_attr_group);
447		if (err) {
448			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
449				h->name, node->dev.id);
450			hugetlb_unregister_node(node);
451			break;
452		}
453	}
454}
455
456/*
457 * hugetlb init time:  register hstate attributes for all registered node
458 * devices of nodes that have memory.  All on-line nodes should have
459 * registered their associated device by this time.
460 */
461static void __init hugetlb_register_all_nodes(void)
462{
463	int nid;
464
465	for_each_online_node(nid)
466		hugetlb_register_node(node_devices[nid]);
467}
468#else	/* !CONFIG_NUMA */
469
470static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
471{
472	BUG();
473	if (nidp)
474		*nidp = -1;
475	return NULL;
476}
477
478static void hugetlb_register_all_nodes(void) { }
479
480#endif
481
482void __init hugetlb_sysfs_init(void)
483{
484	struct hstate *h;
485	int err;
486
487	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
488	if (!hugepages_kobj)
489		return;
490
491	for_each_hstate(h) {
492		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
493					 hstate_kobjs, &hstate_attr_group);
494		if (err)
495			pr_err("HugeTLB: Unable to add hstate %s\n", h->name);
496	}
497
498#ifdef CONFIG_NUMA
499	hugetlb_sysfs_initialized = true;
500#endif
501	hugetlb_register_all_nodes();
502}