hugetlb_cma.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2
  3#include <linux/mm.h>
  4#include <linux/cma.h>
  5#include <linux/compiler.h>
  6#include <linux/mm_inline.h>
  7
  8#include <asm/page.h>
  9#include <asm/setup.h>
 10
 11#include <linux/hugetlb.h>
 12#include "internal.h"
 13#include "hugetlb_cma.h"
 14
 15
 16static struct cma *hugetlb_cma[MAX_NUMNODES];
 17static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
 18static bool hugetlb_cma_only;
 19static unsigned long hugetlb_cma_size __initdata;
 20
 21void hugetlb_cma_free_folio(struct folio *folio)
 22{
 23	int nid = folio_nid(folio);
 24
 25	WARN_ON_ONCE(!cma_free_folio(hugetlb_cma[nid], folio));
 26}
 27
 28
 29struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask,
 30				      int nid, nodemask_t *nodemask)
 31{
 32	int node;
 33	struct folio *folio = NULL;
 34
 35	if (hugetlb_cma[nid])
 36		folio = cma_alloc_folio(hugetlb_cma[nid], order, gfp_mask);
 37
 38	if (!folio && !(gfp_mask & __GFP_THISNODE)) {
 39		for_each_node_mask(node, *nodemask) {
 40			if (node == nid || !hugetlb_cma[node])
 41				continue;
 42
 43			folio = cma_alloc_folio(hugetlb_cma[node], order, gfp_mask);
 44			if (folio)
 45				break;
 46		}
 47	}
 48
 49	if (folio)
 50		folio_set_hugetlb_cma(folio);
 51
 52	return folio;
 53}
 54
 55struct huge_bootmem_page * __init
 56hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid, bool node_exact)
 57{
 58	struct cma *cma;
 59	struct huge_bootmem_page *m;
 60	int node = *nid;
 61
 62	cma = hugetlb_cma[*nid];
 63	m = cma_reserve_early(cma, huge_page_size(h));
 64	if (!m) {
 65		if (node_exact)
 66			return NULL;
 67
 68		for_each_node_mask(node, hugetlb_bootmem_nodes) {
 69			cma = hugetlb_cma[node];
 70			if (!cma || node == *nid)
 71				continue;
 72			m = cma_reserve_early(cma, huge_page_size(h));
 73			if (m) {
 74				*nid = node;
 75				break;
 76			}
 77		}
 78	}
 79
 80	if (m) {
 81		m->flags = HUGE_BOOTMEM_CMA;
 82		m->cma = cma;
 83	}
 84
 85	return m;
 86}
 87
 88
 89static bool cma_reserve_called __initdata;
 90
 91static int __init cmdline_parse_hugetlb_cma(char *p)
 92{
 93	int nid, count = 0;
 94	unsigned long tmp;
 95	char *s = p;
 96
 97	while (*s) {
 98		if (sscanf(s, "%lu%n", &tmp, &count) != 1)
 99			break;
100
101		if (s[count] == ':') {
102			if (tmp >= MAX_NUMNODES)
103				break;
104			nid = array_index_nospec(tmp, MAX_NUMNODES);
105
106			s += count + 1;
107			tmp = memparse(s, &s);
108			hugetlb_cma_size_in_node[nid] = tmp;
109			hugetlb_cma_size += tmp;
110
111			/*
112			 * Skip the separator if have one, otherwise
113			 * break the parsing.
114			 */
115			if (*s == ',')
116				s++;
117			else
118				break;
119		} else {
120			hugetlb_cma_size = memparse(p, &p);
121			break;
122		}
123	}
124
125	return 0;
126}
127
128early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
129
130static int __init cmdline_parse_hugetlb_cma_only(char *p)
131{
132	return kstrtobool(p, &hugetlb_cma_only);
133}
134
135early_param("hugetlb_cma_only", cmdline_parse_hugetlb_cma_only);
136
137void __init hugetlb_cma_reserve(int order)
138{
139	unsigned long size, reserved, per_node;
140	bool node_specific_cma_alloc = false;
141	int nid;
142
143	/*
144	 * HugeTLB CMA reservation is required for gigantic
145	 * huge pages which could not be allocated via the
146	 * page allocator. Just warn if there is any change
147	 * breaking this assumption.
148	 */
149	VM_WARN_ON(order <= MAX_PAGE_ORDER);
150	cma_reserve_called = true;
151
152	if (!hugetlb_cma_size)
153		return;
154
155	hugetlb_bootmem_set_nodes();
156
157	for (nid = 0; nid < MAX_NUMNODES; nid++) {
158		if (hugetlb_cma_size_in_node[nid] == 0)
159			continue;
160
161		if (!node_isset(nid, hugetlb_bootmem_nodes)) {
162			pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
163			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
164			hugetlb_cma_size_in_node[nid] = 0;
165			continue;
166		}
167
168		if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
169			pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
170				nid, (PAGE_SIZE << order) / SZ_1M);
171			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
172			hugetlb_cma_size_in_node[nid] = 0;
173		} else {
174			node_specific_cma_alloc = true;
175		}
176	}
177
178	/* Validate the CMA size again in case some invalid nodes specified. */
179	if (!hugetlb_cma_size)
180		return;
181
182	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
183		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
184			(PAGE_SIZE << order) / SZ_1M);
185		hugetlb_cma_size = 0;
186		return;
187	}
188
189	if (!node_specific_cma_alloc) {
190		/*
191		 * If 3 GB area is requested on a machine with 4 numa nodes,
192		 * let's allocate 1 GB on first three nodes and ignore the last one.
193		 */
194		per_node = DIV_ROUND_UP(hugetlb_cma_size,
195					nodes_weight(hugetlb_bootmem_nodes));
196		pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
197			hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
198	}
199
200	reserved = 0;
201	for_each_node_mask(nid, hugetlb_bootmem_nodes) {
202		int res;
203		char name[CMA_MAX_NAME];
204
205		if (node_specific_cma_alloc) {
206			if (hugetlb_cma_size_in_node[nid] == 0)
207				continue;
208
209			size = hugetlb_cma_size_in_node[nid];
210		} else {
211			size = min(per_node, hugetlb_cma_size - reserved);
212		}
213
214		size = round_up(size, PAGE_SIZE << order);
215
216		snprintf(name, sizeof(name), "hugetlb%d", nid);
217		/*
218		 * Note that 'order per bit' is based on smallest size that
219		 * may be returned to CMA allocator in the case of
220		 * huge page demotion.
221		 */
222		res = cma_declare_contiguous_multi(size, PAGE_SIZE << order,
223					HUGETLB_PAGE_ORDER, name,
224					&hugetlb_cma[nid], nid);
225		if (res) {
226			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
227				res, nid);
228			continue;
229		}
230
231		reserved += size;
232		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
233			size / SZ_1M, nid);
234
235		if (reserved >= hugetlb_cma_size)
236			break;
237	}
238
239	if (!reserved)
240		/*
241		 * hugetlb_cma_size is used to determine if allocations from
242		 * cma are possible.  Set to zero if no cma regions are set up.
243		 */
244		hugetlb_cma_size = 0;
245}
246
247void __init hugetlb_cma_check(void)
248{
249	if (!hugetlb_cma_size || cma_reserve_called)
250		return;
251
252	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
253}
254
255bool hugetlb_cma_exclusive_alloc(void)
256{
257	return hugetlb_cma_only;
258}
259
260unsigned long __init hugetlb_cma_total_size(void)
261{
262	return hugetlb_cma_size;
263}
264
265void __init hugetlb_cma_validate_params(void)
266{
267	if (!hugetlb_cma_size)
268		hugetlb_cma_only = false;
269}
270
271bool __init hugetlb_early_cma(struct hstate *h)
272{
273	if (arch_has_huge_bootmem_alloc())
274		return false;
275
276	return hstate_is_gigantic(h) && hugetlb_cma_only;
277}