hugetlb_internal.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * Internal HugeTLB definitions.
  4 * (C) Nadia Yvette Chambers, April 2004
  5 */
  6
  7#ifndef _LINUX_HUGETLB_INTERNAL_H
  8#define _LINUX_HUGETLB_INTERNAL_H
  9
 10#include <linux/hugetlb.h>
 11#include <linux/hugetlb_cgroup.h>
 12
 13/*
 14 * Check if the hstate represents gigantic pages but gigantic page
 15 * runtime support is not available. This is a common condition used to
 16 * skip operations that cannot be performed on gigantic pages when runtime
 17 * support is disabled.
 18 */
 19static inline bool hstate_is_gigantic_no_runtime(struct hstate *h)
 20{
 21	return hstate_is_gigantic(h) && !gigantic_page_runtime_supported();
 22}
 23
 24/*
 25 * common helper functions for hstate_next_node_to_{alloc|free}.
 26 * We may have allocated or freed a huge page based on a different
 27 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
 28 * be outside of *nodes_allowed.  Ensure that we use an allowed
 29 * node for alloc or free.
 30 */
 31static inline int next_node_allowed(int nid, nodemask_t *nodes_allowed)
 32{
 33	nid = next_node_in(nid, *nodes_allowed);
 34	VM_BUG_ON(nid >= MAX_NUMNODES);
 35
 36	return nid;
 37}
 38
 39static inline int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
 40{
 41	if (!node_isset(nid, *nodes_allowed))
 42		nid = next_node_allowed(nid, nodes_allowed);
 43	return nid;
 44}
 45
 46/*
 47 * returns the previously saved node ["this node"] from which to
 48 * allocate a persistent huge page for the pool and advance the
 49 * next node from which to allocate, handling wrap at end of node
 50 * mask.
 51 */
 52static inline int hstate_next_node_to_alloc(int *next_node,
 53					    nodemask_t *nodes_allowed)
 54{
 55	int nid;
 56
 57	VM_BUG_ON(!nodes_allowed);
 58
 59	nid = get_valid_node_allowed(*next_node, nodes_allowed);
 60	*next_node = next_node_allowed(nid, nodes_allowed);
 61
 62	return nid;
 63}
 64
 65/*
 66 * helper for remove_pool_hugetlb_folio() - return the previously saved
 67 * node ["this node"] from which to free a huge page.  Advance the
 68 * next node id whether or not we find a free huge page to free so
 69 * that the next attempt to free addresses the next node.
 70 */
 71static inline int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
 72{
 73	int nid;
 74
 75	VM_BUG_ON(!nodes_allowed);
 76
 77	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
 78	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
 79
 80	return nid;
 81}
 82
 83#define for_each_node_mask_to_alloc(next_node, nr_nodes, node, mask)		\
 84	for (nr_nodes = nodes_weight(*mask);				\
 85		nr_nodes > 0 &&						\
 86		((node = hstate_next_node_to_alloc(next_node, mask)) || 1);	\
 87		nr_nodes--)
 88
 89#define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
 90	for (nr_nodes = nodes_weight(*mask);				\
 91		nr_nodes > 0 &&						\
 92		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
 93		nr_nodes--)
 94
 95extern void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
 96				 bool adjust_surplus);
 97extern void add_hugetlb_folio(struct hstate *h, struct folio *folio,
 98			      bool adjust_surplus);
 99extern void init_new_hugetlb_folio(struct folio *folio);
100extern void prep_and_add_allocated_folios(struct hstate *h,
101					  struct list_head *folio_list);
102extern long demote_pool_huge_page(struct hstate *src,
103				  nodemask_t *nodes_allowed,
104				  unsigned long nr_to_demote);
105extern ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
106					   struct hstate *h, int nid,
107					   unsigned long count, size_t len);
108
109extern void hugetlb_sysfs_init(void) __init;
110
111#ifdef CONFIG_SYSCTL
112extern void hugetlb_sysctl_init(void);
113#else
114static inline void hugetlb_sysctl_init(void) { }
115#endif
116
117#endif /* _LINUX_HUGETLB_INTERNAL_H */