swiotlb.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Dynamic DMA mapping support.
   4 *
   5 * This implementation is a fallback for platforms that do not support
   6 * I/O TLBs (aka DMA address translation hardware).
   7 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
   8 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
   9 * Copyright (C) 2000, 2003 Hewlett-Packard Co
  10 *	David Mosberger-Tang <davidm@hpl.hp.com>
  11 *
  12 * 03/05/07 davidm	Switch from PCI-DMA to generic device DMA API.
  13 * 00/12/13 davidm	Rename to swiotlb.c and add mark_clean() to avoid
  14 *			unnecessary i-cache flushing.
  15 * 04/07/.. ak		Better overflow handling. Assorted fixes.
  16 * 05/09/10 linville	Add support for syncing ranges, support syncing for
  17 *			DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
  18 * 08/12/11 beckyb	Add highmem support
  19 */
  20
  21#define pr_fmt(fmt) "software IO TLB: " fmt
  22
  23#include <linux/cache.h>
  24#include <linux/cc_platform.h>
  25#include <linux/ctype.h>
  26#include <linux/debugfs.h>
  27#include <linux/dma-direct.h>
  28#include <linux/dma-map-ops.h>
  29#include <linux/export.h>
  30#include <linux/gfp.h>
  31#include <linux/highmem.h>
  32#include <linux/io.h>
  33#include <linux/kmsan-checks.h>
  34#include <linux/iommu-helper.h>
  35#include <linux/init.h>
  36#include <linux/memblock.h>
  37#include <linux/mm.h>
  38#include <linux/pfn.h>
  39#include <linux/rculist.h>
  40#include <linux/scatterlist.h>
  41#include <linux/set_memory.h>
  42#include <linux/spinlock.h>
  43#include <linux/string.h>
  44#include <linux/swiotlb.h>
  45#include <linux/types.h>
  46#ifdef CONFIG_DMA_RESTRICTED_POOL
  47#include <linux/of.h>
  48#include <linux/of_fdt.h>
  49#include <linux/of_reserved_mem.h>
  50#include <linux/slab.h>
  51#endif
  52
  53#define CREATE_TRACE_POINTS
  54#include <trace/events/swiotlb.h>
  55
  56#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
  57
  58/*
  59 * Minimum IO TLB size to bother booting with.  Systems with mainly
  60 * 64bit capable cards will only lightly use the swiotlb.  If we can't
  61 * allocate a contiguous 1MB, we're probably in trouble anyway.
  62 */
  63#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
  64
  65/**
  66 * struct io_tlb_slot - IO TLB slot descriptor
  67 * @orig_addr:	The original address corresponding to a mapped entry.
  68 * @alloc_size:	Size of the allocated buffer.
  69 * @list:	The free list describing the number of free entries available
  70 *		from each index.
  71 * @pad_slots:	Number of preceding padding slots. Valid only in the first
  72 *		allocated non-padding slot.
  73 */
  74struct io_tlb_slot {
  75	phys_addr_t orig_addr;
  76	size_t alloc_size;
  77	unsigned short list;
  78	unsigned short pad_slots;
  79};
  80
  81static bool swiotlb_force_bounce;
  82static bool swiotlb_force_disable;
  83
  84#ifdef CONFIG_SWIOTLB_DYNAMIC
  85
  86static void swiotlb_dyn_alloc(struct work_struct *work);
  87
  88static struct io_tlb_mem io_tlb_default_mem = {
  89	.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
  90	.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
  91	.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
  92					swiotlb_dyn_alloc),
  93};
  94
  95#else  /* !CONFIG_SWIOTLB_DYNAMIC */
  96
  97static struct io_tlb_mem io_tlb_default_mem;
  98
  99#endif	/* CONFIG_SWIOTLB_DYNAMIC */
 100
 101static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
 102static unsigned long default_nareas;
 103
 104/**
 105 * struct io_tlb_area - IO TLB memory area descriptor
 106 *
 107 * This is a single area with a single lock.
 108 *
 109 * @used:	The number of used IO TLB block.
 110 * @index:	The slot index to start searching in this area for next round.
 111 * @lock:	The lock to protect the above data structures in the map and
 112 *		unmap calls.
 113 */
 114struct io_tlb_area {
 115	unsigned long used;
 116	unsigned int index;
 117	spinlock_t lock;
 118};
 119
 120/*
 121 * Round up number of slabs to the next power of 2. The last area is going
 122 * be smaller than the rest if default_nslabs is not power of two.
 123 * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
 124 * otherwise a segment may span two or more areas. It conflicts with free
 125 * contiguous slots tracking: free slots are treated contiguous no matter
 126 * whether they cross an area boundary.
 127 *
 128 * Return true if default_nslabs is rounded up.
 129 */
 130static bool round_up_default_nslabs(void)
 131{
 132	if (!default_nareas)
 133		return false;
 134
 135	if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
 136		default_nslabs = IO_TLB_SEGSIZE * default_nareas;
 137	else if (is_power_of_2(default_nslabs))
 138		return false;
 139	default_nslabs = roundup_pow_of_two(default_nslabs);
 140	return true;
 141}
 142
 143/**
 144 * swiotlb_adjust_nareas() - adjust the number of areas and slots
 145 * @nareas:	Desired number of areas. Zero is treated as 1.
 146 *
 147 * Adjust the default number of areas in a memory pool.
 148 * The default size of the memory pool may also change to meet minimum area
 149 * size requirements.
 150 */
 151static void swiotlb_adjust_nareas(unsigned int nareas)
 152{
 153	if (!nareas)
 154		nareas = 1;
 155	else if (!is_power_of_2(nareas))
 156		nareas = roundup_pow_of_two(nareas);
 157
 158	default_nareas = nareas;
 159
 160	pr_info("area num %d.\n", nareas);
 161	if (round_up_default_nslabs())
 162		pr_info("SWIOTLB bounce buffer size roundup to %luMB",
 163			(default_nslabs << IO_TLB_SHIFT) >> 20);
 164}
 165
 166/**
 167 * limit_nareas() - get the maximum number of areas for a given memory pool size
 168 * @nareas:	Desired number of areas.
 169 * @nslots:	Total number of slots in the memory pool.
 170 *
 171 * Limit the number of areas to the maximum possible number of areas in
 172 * a memory pool of the given size.
 173 *
 174 * Return: Maximum possible number of areas.
 175 */
 176static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
 177{
 178	if (nslots < nareas * IO_TLB_SEGSIZE)
 179		return nslots / IO_TLB_SEGSIZE;
 180	return nareas;
 181}
 182
 183static int __init
 184setup_io_tlb_npages(char *str)
 185{
 186	if (isdigit(*str)) {
 187		/* avoid tail segment of size < IO_TLB_SEGSIZE */
 188		default_nslabs =
 189			ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
 190	}
 191	if (*str == ',')
 192		++str;
 193	if (isdigit(*str))
 194		swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
 195	if (*str == ',')
 196		++str;
 197	if (!strcmp(str, "force"))
 198		swiotlb_force_bounce = true;
 199	else if (!strcmp(str, "noforce"))
 200		swiotlb_force_disable = true;
 201
 202	return 0;
 203}
 204early_param("swiotlb", setup_io_tlb_npages);
 205
 206unsigned long swiotlb_size_or_default(void)
 207{
 208	return default_nslabs << IO_TLB_SHIFT;
 209}
 210
 211void __init swiotlb_adjust_size(unsigned long size)
 212{
 213	/*
 214	 * If swiotlb parameter has not been specified, give a chance to
 215	 * architectures such as those supporting memory encryption to
 216	 * adjust/expand SWIOTLB size for their use.
 217	 */
 218	if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
 219		return;
 220
 221	size = ALIGN(size, IO_TLB_SIZE);
 222	default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
 223	if (round_up_default_nslabs())
 224		size = default_nslabs << IO_TLB_SHIFT;
 225	pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
 226}
 227
 228void swiotlb_print_info(void)
 229{
 230	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 231
 232	if (!mem->nslabs) {
 233		pr_warn("No low mem\n");
 234		return;
 235	}
 236
 237	pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
 238	       (mem->nslabs << IO_TLB_SHIFT) >> 20);
 239}
 240
 241static inline unsigned long io_tlb_offset(unsigned long val)
 242{
 243	return val & (IO_TLB_SEGSIZE - 1);
 244}
 245
 246static inline unsigned long nr_slots(u64 val)
 247{
 248	return DIV_ROUND_UP(val, IO_TLB_SIZE);
 249}
 250
 251/*
 252 * Early SWIOTLB allocation may be too early to allow an architecture to
 253 * perform the desired operations.  This function allows the architecture to
 254 * call SWIOTLB when the operations are possible.  It needs to be called
 255 * before the SWIOTLB memory is used.
 256 */
 257void __init swiotlb_update_mem_attributes(void)
 258{
 259	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 260	unsigned long bytes;
 261
 262	if (!mem->nslabs || mem->late_alloc)
 263		return;
 264	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
 265	set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
 266}
 267
 268static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
 269		unsigned long nslabs, bool late_alloc, unsigned int nareas)
 270{
 271	void *vaddr = phys_to_virt(start);
 272	unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
 273
 274	mem->nslabs = nslabs;
 275	mem->start = start;
 276	mem->end = mem->start + bytes;
 277	mem->late_alloc = late_alloc;
 278	mem->nareas = nareas;
 279	mem->area_nslabs = nslabs / mem->nareas;
 280
 281	for (i = 0; i < mem->nareas; i++) {
 282		spin_lock_init(&mem->areas[i].lock);
 283		mem->areas[i].index = 0;
 284		mem->areas[i].used = 0;
 285	}
 286
 287	for (i = 0; i < mem->nslabs; i++) {
 288		mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
 289					 mem->nslabs - i);
 290		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
 291		mem->slots[i].alloc_size = 0;
 292		mem->slots[i].pad_slots = 0;
 293	}
 294
 295	memset(vaddr, 0, bytes);
 296	mem->vaddr = vaddr;
 297	return;
 298}
 299
 300/**
 301 * add_mem_pool() - add a memory pool to the allocator
 302 * @mem:	Software IO TLB allocator.
 303 * @pool:	Memory pool to be added.
 304 */
 305static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
 306{
 307#ifdef CONFIG_SWIOTLB_DYNAMIC
 308	spin_lock(&mem->lock);
 309	list_add_rcu(&pool->node, &mem->pools);
 310	mem->nslabs += pool->nslabs;
 311	spin_unlock(&mem->lock);
 312#else
 313	mem->nslabs = pool->nslabs;
 314#endif
 315}
 316
 317static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
 318		unsigned int flags,
 319		int (*remap)(void *tlb, unsigned long nslabs))
 320{
 321	size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
 322	void *tlb;
 323
 324	/*
 325	 * By default allocate the bounce buffer memory from low memory, but
 326	 * allow to pick a location everywhere for hypervisors with guest
 327	 * memory encryption.
 328	 */
 329	if (flags & SWIOTLB_ANY)
 330		tlb = memblock_alloc(bytes, PAGE_SIZE);
 331	else
 332		tlb = memblock_alloc_low(bytes, PAGE_SIZE);
 333
 334	if (!tlb) {
 335		pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
 336			__func__, bytes);
 337		return NULL;
 338	}
 339
 340	if (remap && remap(tlb, nslabs) < 0) {
 341		memblock_free(tlb, PAGE_ALIGN(bytes));
 342		pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
 343		return NULL;
 344	}
 345
 346	return tlb;
 347}
 348
 349/*
 350 * Statically reserve bounce buffer space and initialize bounce buffer data
 351 * structures for the software IO TLB used to implement the DMA API.
 352 */
 353void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
 354		int (*remap)(void *tlb, unsigned long nslabs))
 355{
 356	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 357	unsigned long nslabs;
 358	unsigned int nareas;
 359	size_t alloc_size;
 360	void *tlb;
 361
 362	if (!addressing_limit && !swiotlb_force_bounce)
 363		return;
 364	if (swiotlb_force_disable)
 365		return;
 366
 367	io_tlb_default_mem.force_bounce =
 368		swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
 369
 370#ifdef CONFIG_SWIOTLB_DYNAMIC
 371	if (!remap)
 372		io_tlb_default_mem.can_grow = true;
 373	if (flags & SWIOTLB_ANY)
 374		io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
 375	else
 376		io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
 377#endif
 378
 379	if (!default_nareas)
 380		swiotlb_adjust_nareas(num_possible_cpus());
 381
 382	nslabs = default_nslabs;
 383	nareas = limit_nareas(default_nareas, nslabs);
 384	while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
 385		if (nslabs <= IO_TLB_MIN_SLABS)
 386			return;
 387		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
 388		nareas = limit_nareas(nareas, nslabs);
 389	}
 390
 391	if (default_nslabs != nslabs) {
 392		pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
 393			default_nslabs, nslabs);
 394		default_nslabs = nslabs;
 395	}
 396
 397	alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
 398	mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
 399	if (!mem->slots) {
 400		pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
 401			__func__, alloc_size, PAGE_SIZE);
 402		return;
 403	}
 404
 405	mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
 406		nareas), SMP_CACHE_BYTES);
 407	if (!mem->areas) {
 408		pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
 409		return;
 410	}
 411
 412	swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas);
 413	add_mem_pool(&io_tlb_default_mem, mem);
 414
 415	if (flags & SWIOTLB_VERBOSE)
 416		swiotlb_print_info();
 417}
 418
 419void __init swiotlb_init(bool addressing_limit, unsigned int flags)
 420{
 421	swiotlb_init_remap(addressing_limit, flags, NULL);
 422}
 423
 424/*
 425 * Systems with larger DMA zones (those that don't support ISA) can
 426 * initialize the swiotlb later using the slab allocator if needed.
 427 * This should be just like above, but with some error catching.
 428 */
 429int swiotlb_init_late(size_t size, gfp_t gfp_mask,
 430		int (*remap)(void *tlb, unsigned long nslabs))
 431{
 432	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 433	unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
 434	unsigned int nareas;
 435	unsigned char *vstart = NULL;
 436	unsigned int order, area_order;
 437	bool retried = false;
 438	int rc = 0;
 439
 440	if (io_tlb_default_mem.nslabs)
 441		return 0;
 442
 443	if (swiotlb_force_disable)
 444		return 0;
 445
 446	io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
 447
 448#ifdef CONFIG_SWIOTLB_DYNAMIC
 449	if (!remap)
 450		io_tlb_default_mem.can_grow = true;
 451	if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
 452		io_tlb_default_mem.phys_limit = zone_dma_limit;
 453	else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
 454		io_tlb_default_mem.phys_limit = max(DMA_BIT_MASK(32), zone_dma_limit);
 455	else
 456		io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
 457#endif
 458
 459	if (!default_nareas)
 460		swiotlb_adjust_nareas(num_possible_cpus());
 461
 462retry:
 463	order = get_order(nslabs << IO_TLB_SHIFT);
 464	nslabs = SLABS_PER_PAGE << order;
 465
 466	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
 467		vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
 468						  order);
 469		if (vstart)
 470			break;
 471		order--;
 472		nslabs = SLABS_PER_PAGE << order;
 473		retried = true;
 474	}
 475
 476	if (!vstart)
 477		return -ENOMEM;
 478
 479	if (remap)
 480		rc = remap(vstart, nslabs);
 481	if (rc) {
 482		free_pages((unsigned long)vstart, order);
 483
 484		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
 485		if (nslabs < IO_TLB_MIN_SLABS)
 486			return rc;
 487		retried = true;
 488		goto retry;
 489	}
 490
 491	if (retried) {
 492		pr_warn("only able to allocate %ld MB\n",
 493			(PAGE_SIZE << order) >> 20);
 494	}
 495
 496	nareas = limit_nareas(default_nareas, nslabs);
 497	area_order = get_order(array_size(sizeof(*mem->areas), nareas));
 498	mem->areas = (struct io_tlb_area *)
 499		__get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
 500	if (!mem->areas)
 501		goto error_area;
 502
 503	mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 504		get_order(array_size(sizeof(*mem->slots), nslabs)));
 505	if (!mem->slots)
 506		goto error_slots;
 507
 508	set_memory_decrypted((unsigned long)vstart,
 509			     (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
 510	swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
 511				 nareas);
 512	add_mem_pool(&io_tlb_default_mem, mem);
 513
 514	swiotlb_print_info();
 515	return 0;
 516
 517error_slots:
 518	free_pages((unsigned long)mem->areas, area_order);
 519error_area:
 520	free_pages((unsigned long)vstart, order);
 521	return -ENOMEM;
 522}
 523
 524void __init swiotlb_exit(void)
 525{
 526	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 527	unsigned long tbl_vaddr;
 528	size_t tbl_size, slots_size;
 529	unsigned int area_order;
 530
 531	if (swiotlb_force_bounce)
 532		return;
 533
 534	if (!mem->nslabs)
 535		return;
 536
 537	pr_info("tearing down default memory pool\n");
 538	tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
 539	tbl_size = PAGE_ALIGN(mem->end - mem->start);
 540	slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
 541
 542	set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
 543	if (mem->late_alloc) {
 544		area_order = get_order(array_size(sizeof(*mem->areas),
 545			mem->nareas));
 546		free_pages((unsigned long)mem->areas, area_order);
 547		free_pages(tbl_vaddr, get_order(tbl_size));
 548		free_pages((unsigned long)mem->slots, get_order(slots_size));
 549	} else {
 550		memblock_free_late(__pa(mem->areas),
 551			array_size(sizeof(*mem->areas), mem->nareas));
 552		memblock_free_late(mem->start, tbl_size);
 553		memblock_free_late(__pa(mem->slots), slots_size);
 554	}
 555
 556	memset(mem, 0, sizeof(*mem));
 557}
 558
 559#ifdef CONFIG_SWIOTLB_DYNAMIC
 560
 561/**
 562 * alloc_dma_pages() - allocate pages to be used for DMA
 563 * @gfp:	GFP flags for the allocation.
 564 * @bytes:	Size of the buffer.
 565 * @phys_limit:	Maximum allowed physical address of the buffer.
 566 *
 567 * Allocate pages from the buddy allocator. If successful, make the allocated
 568 * pages decrypted that they can be used for DMA.
 569 *
 570 * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
 571 * if the allocated physical address was above @phys_limit.
 572 */
 573static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
 574{
 575	unsigned int order = get_order(bytes);
 576	struct page *page;
 577	phys_addr_t paddr;
 578	void *vaddr;
 579
 580	page = alloc_pages(gfp, order);
 581	if (!page)
 582		return NULL;
 583
 584	paddr = page_to_phys(page);
 585	if (paddr + bytes - 1 > phys_limit) {
 586		__free_pages(page, order);
 587		return ERR_PTR(-EAGAIN);
 588	}
 589
 590	vaddr = phys_to_virt(paddr);
 591	if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
 592		goto error;
 593	return page;
 594
 595error:
 596	/* Intentional leak if pages cannot be encrypted again. */
 597	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 598		__free_pages(page, order);
 599	return NULL;
 600}
 601
 602/**
 603 * swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
 604 * @dev:	Device for which a memory pool is allocated.
 605 * @bytes:	Size of the buffer.
 606 * @phys_limit:	Maximum allowed physical address of the buffer.
 607 * @gfp:	GFP flags for the allocation.
 608 *
 609 * Return: Allocated pages, or %NULL on allocation failure.
 610 */
 611static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
 612		u64 phys_limit, gfp_t gfp)
 613{
 614	struct page *page;
 615
 616	/*
 617	 * Allocate from the atomic pools if memory is encrypted and
 618	 * the allocation is atomic, because decrypting may block.
 619	 */
 620	if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
 621		void *vaddr;
 622
 623		if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
 624			return NULL;
 625
 626		return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
 627					   dma_coherent_ok);
 628	}
 629
 630	gfp &= ~GFP_ZONEMASK;
 631	if (phys_limit <= zone_dma_limit)
 632		gfp |= __GFP_DMA;
 633	else if (phys_limit <= DMA_BIT_MASK(32))
 634		gfp |= __GFP_DMA32;
 635
 636	while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
 637		if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
 638		    phys_limit < DMA_BIT_MASK(64) &&
 639		    !(gfp & (__GFP_DMA32 | __GFP_DMA)))
 640			gfp |= __GFP_DMA32;
 641		else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
 642			 !(gfp & __GFP_DMA))
 643			gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
 644		else
 645			return NULL;
 646	}
 647
 648	return page;
 649}
 650
 651/**
 652 * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
 653 * @vaddr:	Virtual address of the buffer.
 654 * @bytes:	Size of the buffer.
 655 */
 656static void swiotlb_free_tlb(void *vaddr, size_t bytes)
 657{
 658	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
 659	    dma_free_from_pool(NULL, vaddr, bytes))
 660		return;
 661
 662	/* Intentional leak if pages cannot be encrypted again. */
 663	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 664		__free_pages(virt_to_page(vaddr), get_order(bytes));
 665}
 666
 667/**
 668 * swiotlb_alloc_pool() - allocate a new IO TLB memory pool
 669 * @dev:	Device for which a memory pool is allocated.
 670 * @minslabs:	Minimum number of slabs.
 671 * @nslabs:	Desired (maximum) number of slabs.
 672 * @nareas:	Number of areas.
 673 * @phys_limit:	Maximum DMA buffer physical address.
 674 * @gfp:	GFP flags for the allocations.
 675 *
 676 * Allocate and initialize a new IO TLB memory pool. The actual number of
 677 * slabs may be reduced if allocation of @nslabs fails. If even
 678 * @minslabs cannot be allocated, this function fails.
 679 *
 680 * Return: New memory pool, or %NULL on allocation failure.
 681 */
 682static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 683		unsigned long minslabs, unsigned long nslabs,
 684		unsigned int nareas, u64 phys_limit, gfp_t gfp)
 685{
 686	struct io_tlb_pool *pool;
 687	unsigned int slot_order;
 688	struct page *tlb;
 689	size_t pool_size;
 690	size_t tlb_size;
 691
 692	if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
 693		nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
 694		nareas = limit_nareas(nareas, nslabs);
 695	}
 696
 697	pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
 698	pool = kzalloc(pool_size, gfp);
 699	if (!pool)
 700		goto error;
 701	pool->areas = (void *)pool + sizeof(*pool);
 702
 703	tlb_size = nslabs << IO_TLB_SHIFT;
 704	while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
 705		if (nslabs <= minslabs)
 706			goto error_tlb;
 707		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
 708		nareas = limit_nareas(nareas, nslabs);
 709		tlb_size = nslabs << IO_TLB_SHIFT;
 710	}
 711
 712	slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
 713	pool->slots = (struct io_tlb_slot *)
 714		__get_free_pages(gfp, slot_order);
 715	if (!pool->slots)
 716		goto error_slots;
 717
 718	swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);
 719	return pool;
 720
 721error_slots:
 722	swiotlb_free_tlb(page_address(tlb), tlb_size);
 723error_tlb:
 724	kfree(pool);
 725error:
 726	return NULL;
 727}
 728
 729/**
 730 * swiotlb_dyn_alloc() - dynamic memory pool allocation worker
 731 * @work:	Pointer to dyn_alloc in struct io_tlb_mem.
 732 */
 733static void swiotlb_dyn_alloc(struct work_struct *work)
 734{
 735	struct io_tlb_mem *mem =
 736		container_of(work, struct io_tlb_mem, dyn_alloc);
 737	struct io_tlb_pool *pool;
 738
 739	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
 740				  default_nareas, mem->phys_limit, GFP_KERNEL);
 741	if (!pool) {
 742		pr_warn_ratelimited("Failed to allocate new pool");
 743		return;
 744	}
 745
 746	add_mem_pool(mem, pool);
 747}
 748
 749/**
 750 * swiotlb_dyn_free() - RCU callback to free a memory pool
 751 * @rcu:	RCU head in the corresponding struct io_tlb_pool.
 752 */
 753static void swiotlb_dyn_free(struct rcu_head *rcu)
 754{
 755	struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
 756	size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
 757	size_t tlb_size = pool->end - pool->start;
 758
 759	free_pages((unsigned long)pool->slots, get_order(slots_size));
 760	swiotlb_free_tlb(pool->vaddr, tlb_size);
 761	kfree(pool);
 762}
 763
 764/**
 765 * __swiotlb_find_pool() - find the IO TLB pool for a physical address
 766 * @dev:        Device which has mapped the DMA buffer.
 767 * @paddr:      Physical address within the DMA buffer.
 768 *
 769 * Find the IO TLB memory pool descriptor which contains the given physical
 770 * address, if any. This function is for use only when the dev is known to
 771 * be using swiotlb. Use swiotlb_find_pool() for the more general case
 772 * when this condition is not met.
 773 *
 774 * Return: Memory pool which contains @paddr, or %NULL if none.
 775 */
 776struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
 777{
 778	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
 779	struct io_tlb_pool *pool;
 780
 781	rcu_read_lock();
 782	list_for_each_entry_rcu(pool, &mem->pools, node) {
 783		if (paddr >= pool->start && paddr < pool->end)
 784			goto out;
 785	}
 786
 787	list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
 788		if (paddr >= pool->start && paddr < pool->end)
 789			goto out;
 790	}
 791	pool = NULL;
 792out:
 793	rcu_read_unlock();
 794	return pool;
 795}
 796
 797/**
 798 * swiotlb_del_pool() - remove an IO TLB pool from a device
 799 * @dev:	Owning device.
 800 * @pool:	Memory pool to be removed.
 801 */
 802static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
 803{
 804	unsigned long flags;
 805
 806	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
 807	list_del_rcu(&pool->node);
 808	spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
 809
 810	call_rcu(&pool->rcu, swiotlb_dyn_free);
 811}
 812
 813#endif	/* CONFIG_SWIOTLB_DYNAMIC */
 814
 815/**
 816 * swiotlb_dev_init() - initialize swiotlb fields in &struct device
 817 * @dev:	Device to be initialized.
 818 */
 819void swiotlb_dev_init(struct device *dev)
 820{
 821	dev->dma_io_tlb_mem = &io_tlb_default_mem;
 822#ifdef CONFIG_SWIOTLB_DYNAMIC
 823	INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
 824	spin_lock_init(&dev->dma_io_tlb_lock);
 825	dev->dma_uses_io_tlb = false;
 826#endif
 827}
 828
 829/**
 830 * swiotlb_align_offset() - Get required offset into an IO TLB allocation.
 831 * @dev:         Owning device.
 832 * @align_mask:  Allocation alignment mask.
 833 * @addr:        DMA address.
 834 *
 835 * Return the minimum offset from the start of an IO TLB allocation which is
 836 * required for a given buffer address and allocation alignment to keep the
 837 * device happy.
 838 *
 839 * First, the address bits covered by min_align_mask must be identical in the
 840 * original address and the bounce buffer address. High bits are preserved by
 841 * choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
 842 * padding bytes before the bounce buffer.
 843 *
 844 * Second, @align_mask specifies which bits of the first allocated slot must
 845 * be zero. This may require allocating additional padding slots, and then the
 846 * offset (in bytes) from the first such padding slot is returned.
 847 */
 848static unsigned int swiotlb_align_offset(struct device *dev,
 849					 unsigned int align_mask, u64 addr)
 850{
 851	return addr & dma_get_min_align_mask(dev) &
 852		(align_mask | (IO_TLB_SIZE - 1));
 853}
 854
 855/*
 856 * Bounce: copy the swiotlb buffer from or back to the original dma location
 857 */
 858static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
 859			   enum dma_data_direction dir, struct io_tlb_pool *mem)
 860{
 861	int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
 862	phys_addr_t orig_addr = mem->slots[index].orig_addr;
 863	size_t alloc_size = mem->slots[index].alloc_size;
 864	unsigned long pfn = PFN_DOWN(orig_addr);
 865	unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
 866	int tlb_offset;
 867
 868	if (orig_addr == INVALID_PHYS_ADDR)
 869		return;
 870
 871	/*
 872	 * It's valid for tlb_offset to be negative. This can happen when the
 873	 * "offset" returned by swiotlb_align_offset() is non-zero, and the
 874	 * tlb_addr is pointing within the first "offset" bytes of the second
 875	 * or subsequent slots of the allocated swiotlb area. While it's not
 876	 * valid for tlb_addr to be pointing within the first "offset" bytes
 877	 * of the first slot, there's no way to check for such an error since
 878	 * this function can't distinguish the first slot from the second and
 879	 * subsequent slots.
 880	 */
 881	tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
 882		     swiotlb_align_offset(dev, 0, orig_addr);
 883
 884	orig_addr += tlb_offset;
 885	alloc_size -= tlb_offset;
 886
 887	if (size > alloc_size) {
 888		dev_WARN_ONCE(dev, 1,
 889			"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
 890			alloc_size, size);
 891		size = alloc_size;
 892	}
 893
 894	if (PageHighMem(pfn_to_page(pfn))) {
 895		unsigned int offset = orig_addr & ~PAGE_MASK;
 896		struct page *page;
 897		unsigned int sz = 0;
 898		unsigned long flags;
 899
 900		while (size) {
 901			sz = min_t(size_t, PAGE_SIZE - offset, size);
 902
 903			local_irq_save(flags);
 904			page = pfn_to_page(pfn);
 905			if (dir == DMA_TO_DEVICE) {
 906				/*
 907				 * Ideally, kmsan_check_highmem_page()
 908				 * could be used here to detect infoleaks,
 909				 * but callers may map uninitialized buffers
 910				 * that will be written by the device,
 911				 * causing false positives.
 912				 */
 913				memcpy_from_page(vaddr, page, offset, sz);
 914			} else {
 915				kmsan_unpoison_memory(vaddr, sz);
 916				memcpy_to_page(page, offset, vaddr, sz);
 917			}
 918			local_irq_restore(flags);
 919
 920			size -= sz;
 921			pfn++;
 922			vaddr += sz;
 923			offset = 0;
 924		}
 925	} else if (dir == DMA_TO_DEVICE) {
 926		/*
 927		 * Ideally, kmsan_check_memory() could be used here to detect
 928		 * infoleaks (uninitialized data being sent to device), but
 929		 * callers may map uninitialized buffers that will be written
 930		 * by the device, causing false positives.
 931		 */
 932		memcpy(vaddr, phys_to_virt(orig_addr), size);
 933	} else {
 934		kmsan_unpoison_memory(vaddr, size);
 935		memcpy(phys_to_virt(orig_addr), vaddr, size);
 936	}
 937}
 938
 939static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
 940{
 941	return start + (idx << IO_TLB_SHIFT);
 942}
 943
 944/*
 945 * Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
 946 */
 947static inline unsigned long get_max_slots(unsigned long boundary_mask)
 948{
 949	return (boundary_mask >> IO_TLB_SHIFT) + 1;
 950}
 951
 952static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
 953{
 954	if (index >= mem->area_nslabs)
 955		return 0;
 956	return index;
 957}
 958
 959/*
 960 * Track the total used slots with a global atomic value in order to have
 961 * correct information to determine the high water mark. The mem_used()
 962 * function gives imprecise results because there's no locking across
 963 * multiple areas.
 964 */
 965#ifdef CONFIG_DEBUG_FS
 966static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
 967{
 968	unsigned long old_hiwater, new_used;
 969
 970	new_used = atomic_long_add_return(nslots, &mem->total_used);
 971	old_hiwater = atomic_long_read(&mem->used_hiwater);
 972	do {
 973		if (new_used <= old_hiwater)
 974			break;
 975	} while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
 976					  &old_hiwater, new_used));
 977}
 978
 979static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
 980{
 981	atomic_long_sub(nslots, &mem->total_used);
 982}
 983
 984#else /* !CONFIG_DEBUG_FS */
 985static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
 986{
 987}
 988static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
 989{
 990}
 991#endif /* CONFIG_DEBUG_FS */
 992
 993#ifdef CONFIG_SWIOTLB_DYNAMIC
 994#ifdef CONFIG_DEBUG_FS
 995static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
 996{
 997	atomic_long_add(nslots, &mem->transient_nslabs);
 998}
 999
1000static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
1001{
1002	atomic_long_sub(nslots, &mem->transient_nslabs);
1003}
1004
1005#else /* !CONFIG_DEBUG_FS */
1006static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
1007{
1008}
1009static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
1010{
1011}
1012#endif /* CONFIG_DEBUG_FS */
1013#endif /* CONFIG_SWIOTLB_DYNAMIC */
1014
1015/**
1016 * swiotlb_search_pool_area() - search one memory area in one pool
1017 * @dev:	Device which maps the buffer.
1018 * @pool:	Memory pool to be searched.
1019 * @area_index:	Index of the IO TLB memory area to be searched.
1020 * @orig_addr:	Original (non-bounced) IO buffer address.
1021 * @alloc_size: Total requested size of the bounce buffer,
1022 *		including initial alignment padding.
1023 * @alloc_align_mask:	Required alignment of the allocated buffer.
1024 *
1025 * Find a suitable sequence of IO TLB entries for the request and allocate
1026 * a buffer from the given IO TLB memory area.
1027 * This function takes care of locking.
1028 *
1029 * Return: Index of the first allocated slot, or -1 on error.
1030 */
1031static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
1032		int area_index, phys_addr_t orig_addr, size_t alloc_size,
1033		unsigned int alloc_align_mask)
1034{
1035	struct io_tlb_area *area = pool->areas + area_index;
1036	unsigned long boundary_mask = dma_get_seg_boundary(dev);
1037	dma_addr_t tbl_dma_addr =
1038		phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
1039	unsigned long max_slots = get_max_slots(boundary_mask);
1040	unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1041	unsigned int nslots = nr_slots(alloc_size), stride;
1042	unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
1043	unsigned int index, slots_checked, count = 0, i;
1044	unsigned long flags;
1045	unsigned int slot_base;
1046	unsigned int slot_index;
1047
1048	BUG_ON(!nslots);
1049	BUG_ON(area_index >= pool->nareas);
1050
1051	/*
1052	 * Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1053	 * page-aligned in the absence of any other alignment requirements.
1054	 * 'alloc_align_mask' was later introduced to specify the alignment
1055	 * explicitly, however this is passed as zero for streaming mappings
1056	 * and so we preserve the old behaviour there in case any drivers are
1057	 * relying on it.
1058	 */
1059	if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1060		alloc_align_mask = PAGE_SIZE - 1;
1061
1062	/*
1063	 * Ensure that the allocation is at least slot-aligned and update
1064	 * 'iotlb_align_mask' to ignore bits that will be preserved when
1065	 * offsetting into the allocation.
1066	 */
1067	alloc_align_mask |= (IO_TLB_SIZE - 1);
1068	iotlb_align_mask &= ~alloc_align_mask;
1069
1070	/*
1071	 * For mappings with an alignment requirement don't bother looping to
1072	 * unaligned slots once we found an aligned one.
1073	 */
1074	stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1075
1076	spin_lock_irqsave(&area->lock, flags);
1077	if (unlikely(nslots > pool->area_nslabs - area->used))
1078		goto not_found;
1079
1080	slot_base = area_index * pool->area_nslabs;
1081	index = area->index;
1082
1083	for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1084		phys_addr_t tlb_addr;
1085
1086		slot_index = slot_base + index;
1087		tlb_addr = slot_addr(tbl_dma_addr, slot_index);
1088
1089		if ((tlb_addr & alloc_align_mask) ||
1090		    (orig_addr && (tlb_addr & iotlb_align_mask) !=
1091				  (orig_addr & iotlb_align_mask))) {
1092			index = wrap_area_index(pool, index + 1);
1093			slots_checked++;
1094			continue;
1095		}
1096
1097		if (!iommu_is_span_boundary(slot_index, nslots,
1098					    nr_slots(tbl_dma_addr),
1099					    max_slots)) {
1100			if (pool->slots[slot_index].list >= nslots)
1101				goto found;
1102		}
1103		index = wrap_area_index(pool, index + stride);
1104		slots_checked += stride;
1105	}
1106
1107not_found:
1108	spin_unlock_irqrestore(&area->lock, flags);
1109	return -1;
1110
1111found:
1112	/*
1113	 * If we find a slot that indicates we have 'nslots' number of
1114	 * contiguous buffers, we allocate the buffers from that slot onwards
1115	 * and set the list of free entries to '0' indicating unavailable.
1116	 */
1117	for (i = slot_index; i < slot_index + nslots; i++) {
1118		pool->slots[i].list = 0;
1119		pool->slots[i].alloc_size = alloc_size - (offset +
1120				((i - slot_index) << IO_TLB_SHIFT));
1121	}
1122	for (i = slot_index - 1;
1123	     io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
1124	     pool->slots[i].list; i--)
1125		pool->slots[i].list = ++count;
1126
1127	/*
1128	 * Update the indices to avoid searching in the next round.
1129	 */
1130	area->index = wrap_area_index(pool, index + nslots);
1131	area->used += nslots;
1132	spin_unlock_irqrestore(&area->lock, flags);
1133
1134	inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);
1135	return slot_index;
1136}
1137
1138#ifdef CONFIG_SWIOTLB_DYNAMIC
1139
1140/**
1141 * swiotlb_search_area() - search one memory area in all pools
1142 * @dev:	Device which maps the buffer.
1143 * @start_cpu:	Start CPU number.
1144 * @cpu_offset:	Offset from @start_cpu.
1145 * @orig_addr:	Original (non-bounced) IO buffer address.
1146 * @alloc_size: Total requested size of the bounce buffer,
1147 *		including initial alignment padding.
1148 * @alloc_align_mask:	Required alignment of the allocated buffer.
1149 * @retpool:	Used memory pool, updated on return.
1150 *
1151 * Search one memory area in all pools for a sequence of slots that match the
1152 * allocation constraints.
1153 *
1154 * Return: Index of the first allocated slot, or -1 on error.
1155 */
1156static int swiotlb_search_area(struct device *dev, int start_cpu,
1157		int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1158		unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1159{
1160	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1161	struct io_tlb_pool *pool;
1162	int area_index;
1163	int index = -1;
1164
1165	rcu_read_lock();
1166	list_for_each_entry_rcu(pool, &mem->pools, node) {
1167		if (cpu_offset >= pool->nareas)
1168			continue;
1169		area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1170		index = swiotlb_search_pool_area(dev, pool, area_index,
1171						 orig_addr, alloc_size,
1172						 alloc_align_mask);
1173		if (index >= 0) {
1174			*retpool = pool;
1175			break;
1176		}
1177	}
1178	rcu_read_unlock();
1179	return index;
1180}
1181
1182/**
1183 * swiotlb_find_slots() - search for slots in the whole swiotlb
1184 * @dev:	Device which maps the buffer.
1185 * @orig_addr:	Original (non-bounced) IO buffer address.
1186 * @alloc_size: Total requested size of the bounce buffer,
1187 *		including initial alignment padding.
1188 * @alloc_align_mask:	Required alignment of the allocated buffer.
1189 * @retpool:	Used memory pool, updated on return.
1190 *
1191 * Search through the whole software IO TLB to find a sequence of slots that
1192 * match the allocation constraints.
1193 *
1194 * Return: Index of the first allocated slot, or -1 on error.
1195 */
1196static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1197		size_t alloc_size, unsigned int alloc_align_mask,
1198		struct io_tlb_pool **retpool)
1199{
1200	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1201	struct io_tlb_pool *pool;
1202	unsigned long nslabs;
1203	unsigned long flags;
1204	u64 phys_limit;
1205	int cpu, i;
1206	int index;
1207
1208	if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
1209		return -1;
1210
1211	cpu = raw_smp_processor_id();
1212	for (i = 0; i < default_nareas; ++i) {
1213		index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
1214					    alloc_align_mask, &pool);
1215		if (index >= 0)
1216			goto found;
1217	}
1218
1219	if (!mem->can_grow)
1220		return -1;
1221
1222	schedule_work(&mem->dyn_alloc);
1223
1224	nslabs = nr_slots(alloc_size);
1225	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1226	pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
1227				  GFP_NOWAIT);
1228	if (!pool)
1229		return -1;
1230
1231	index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
1232					 alloc_size, alloc_align_mask);
1233	if (index < 0) {
1234		swiotlb_dyn_free(&pool->rcu);
1235		return -1;
1236	}
1237
1238	pool->transient = true;
1239	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1240	list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
1241	spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
1242	inc_transient_used(mem, pool->nslabs);
1243
1244found:
1245	WRITE_ONCE(dev->dma_uses_io_tlb, true);
1246
1247	/*
1248	 * The general barrier orders reads and writes against a presumed store
1249	 * of the SWIOTLB buffer address by a device driver (to a driver private
1250	 * data structure). It serves two purposes.
1251	 *
1252	 * First, the store to dev->dma_uses_io_tlb must be ordered before the
1253	 * presumed store. This guarantees that the returned buffer address
1254	 * cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1255	 *
1256	 * Second, the load from mem->pools must be ordered before the same
1257	 * presumed store. This guarantees that the returned buffer address
1258	 * cannot be observed by another CPU before an update of the RCU list
1259	 * that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1260	 * atomicity).
1261	 *
1262	 * See also the comment in swiotlb_find_pool().
1263	 */
1264	smp_mb();
1265
1266	*retpool = pool;
1267	return index;
1268}
1269
1270#else  /* !CONFIG_SWIOTLB_DYNAMIC */
1271
1272static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1273		size_t alloc_size, unsigned int alloc_align_mask,
1274		struct io_tlb_pool **retpool)
1275{
1276	struct io_tlb_pool *pool;
1277	int start, i;
1278	int index;
1279
1280	*retpool = pool = &dev->dma_io_tlb_mem->defpool;
1281	i = start = raw_smp_processor_id() & (pool->nareas - 1);
1282	do {
1283		index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1284						 alloc_size, alloc_align_mask);
1285		if (index >= 0)
1286			return index;
1287		if (++i >= pool->nareas)
1288			i = 0;
1289	} while (i != start);
1290	return -1;
1291}
1292
1293#endif /* CONFIG_SWIOTLB_DYNAMIC */
1294
1295#ifdef CONFIG_DEBUG_FS
1296
1297/**
1298 * mem_used() - get number of used slots in an allocator
1299 * @mem:	Software IO TLB allocator.
1300 *
1301 * The result is accurate in this version of the function, because an atomic
1302 * counter is available if CONFIG_DEBUG_FS is set.
1303 *
1304 * Return: Number of used slots.
1305 */
1306static unsigned long mem_used(struct io_tlb_mem *mem)
1307{
1308	return atomic_long_read(&mem->total_used);
1309}
1310
1311#else /* !CONFIG_DEBUG_FS */
1312
1313/**
1314 * mem_pool_used() - get number of used slots in a memory pool
1315 * @pool:	Software IO TLB memory pool.
1316 *
1317 * The result is not accurate, see mem_used().
1318 *
1319 * Return: Approximate number of used slots.
1320 */
1321static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1322{
1323	int i;
1324	unsigned long used = 0;
1325
1326	for (i = 0; i < pool->nareas; i++)
1327		used += pool->areas[i].used;
1328	return used;
1329}
1330
1331/**
1332 * mem_used() - get number of used slots in an allocator
1333 * @mem:	Software IO TLB allocator.
1334 *
1335 * The result is not accurate, because there is no locking of individual
1336 * areas.
1337 *
1338 * Return: Approximate number of used slots.
1339 */
1340static unsigned long mem_used(struct io_tlb_mem *mem)
1341{
1342#ifdef CONFIG_SWIOTLB_DYNAMIC
1343	struct io_tlb_pool *pool;
1344	unsigned long used = 0;
1345
1346	rcu_read_lock();
1347	list_for_each_entry_rcu(pool, &mem->pools, node)
1348		used += mem_pool_used(pool);
1349	rcu_read_unlock();
1350
1351	return used;
1352#else
1353	return mem_pool_used(&mem->defpool);
1354#endif
1355}
1356
1357#endif /* CONFIG_DEBUG_FS */
1358
1359/**
1360 * swiotlb_tbl_map_single() - bounce buffer map a single contiguous physical area
1361 * @dev:		Device which maps the buffer.
1362 * @orig_addr:		Original (non-bounced) physical IO buffer address
1363 * @mapping_size:	Requested size of the actual bounce buffer, excluding
1364 *			any pre- or post-padding for alignment
1365 * @alloc_align_mask:	Required start and end alignment of the allocated buffer
1366 * @dir:		DMA direction
1367 * @attrs:		Optional DMA attributes for the map operation
1368 *
1369 * Find and allocate a suitable sequence of IO TLB slots for the request.
1370 * The allocated space starts at an alignment specified by alloc_align_mask,
1371 * and the size of the allocated space is rounded up so that the total amount
1372 * of allocated space is a multiple of (alloc_align_mask + 1). If
1373 * alloc_align_mask is zero, the allocated space may be at any alignment and
1374 * the size is not rounded up.
1375 *
1376 * The returned address is within the allocated space and matches the bits
1377 * of orig_addr that are specified in the DMA min_align_mask for the device. As
1378 * such, this returned address may be offset from the beginning of the allocated
1379 * space. The bounce buffer space starting at the returned address for
1380 * mapping_size bytes is initialized to the contents of the original IO buffer
1381 * area. Any pre-padding (due to an offset) and any post-padding (due to
1382 * rounding-up the size) is not initialized.
1383 */
1384phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1385		size_t mapping_size, unsigned int alloc_align_mask,
1386		enum dma_data_direction dir, unsigned long attrs)
1387{
1388	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1389	unsigned int offset;
1390	struct io_tlb_pool *pool;
1391	unsigned int i;
1392	size_t size;
1393	int index;
1394	phys_addr_t tlb_addr;
1395	unsigned short pad_slots;
1396
1397	if (!mem || !mem->nslabs) {
1398		dev_warn_ratelimited(dev,
1399			"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1400		return (phys_addr_t)DMA_MAPPING_ERROR;
1401	}
1402
1403	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
1404		pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1405
1406	/*
1407	 * The default swiotlb memory pool is allocated with PAGE_SIZE
1408	 * alignment. If a mapping is requested with larger alignment,
1409	 * the mapping may be unable to use the initial slot(s) in all
1410	 * sets of IO_TLB_SEGSIZE slots. In such case, a mapping request
1411	 * of or near the maximum mapping size would always fail.
1412	 */
1413	dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK,
1414		"Alloc alignment may prevent fulfilling requests with max mapping_size\n");
1415
1416	offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);
1417	size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
1418	index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);
1419	if (index == -1) {
1420		if (!(attrs & DMA_ATTR_NO_WARN))
1421			dev_warn_ratelimited(dev,
1422	"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1423				 size, mem->nslabs, mem_used(mem));
1424		return (phys_addr_t)DMA_MAPPING_ERROR;
1425	}
1426
1427	/*
1428	 * If dma_skip_sync was set, reset it on first SWIOTLB buffer
1429	 * mapping to always sync SWIOTLB buffers.
1430	 */
1431	dma_reset_need_sync(dev);
1432
1433	/*
1434	 * Save away the mapping from the original address to the DMA address.
1435	 * This is needed when we sync the memory.  Then we sync the buffer if
1436	 * needed.
1437	 */
1438	pad_slots = offset >> IO_TLB_SHIFT;
1439	offset &= (IO_TLB_SIZE - 1);
1440	index += pad_slots;
1441	pool->slots[index].pad_slots = pad_slots;
1442	for (i = 0; i < (nr_slots(size) - pad_slots); i++)
1443		pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
1444	tlb_addr = slot_addr(pool->start, index) + offset;
1445	/*
1446	 * When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1447	 * the original buffer to the TLB buffer before initiating DMA in order
1448	 * to preserve the original's data if the device does a partial write,
1449	 * i.e. if the device doesn't overwrite the entire buffer.  Preserving
1450	 * the original data, even if it's garbage, is necessary to match
1451	 * hardware behavior.  Use of swiotlb is supposed to be transparent,
1452	 * i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1453	 */
1454	swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool);
1455	return tlb_addr;
1456}
1457
1458static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
1459				  struct io_tlb_pool *mem)
1460{
1461	unsigned long flags;
1462	unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
1463	int index, nslots, aindex;
1464	struct io_tlb_area *area;
1465	int count, i;
1466
1467	index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1468	index -= mem->slots[index].pad_slots;
1469	nslots = nr_slots(mem->slots[index].alloc_size + offset);
1470	aindex = index / mem->area_nslabs;
1471	area = &mem->areas[aindex];
1472
1473	/*
1474	 * Return the buffer to the free list by setting the corresponding
1475	 * entries to indicate the number of contiguous entries available.
1476	 * While returning the entries to the free list, we merge the entries
1477	 * with slots below and above the pool being returned.
1478	 */
1479	BUG_ON(aindex >= mem->nareas);
1480
1481	spin_lock_irqsave(&area->lock, flags);
1482	if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1483		count = mem->slots[index + nslots].list;
1484	else
1485		count = 0;
1486
1487	/*
1488	 * Step 1: return the slots to the free list, merging the slots with
1489	 * superceeding slots
1490	 */
1491	for (i = index + nslots - 1; i >= index; i--) {
1492		mem->slots[i].list = ++count;
1493		mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1494		mem->slots[i].alloc_size = 0;
1495		mem->slots[i].pad_slots = 0;
1496	}
1497
1498	/*
1499	 * Step 2: merge the returned slots with the preceding slots, if
1500	 * available (non zero)
1501	 */
1502	for (i = index - 1;
1503	     io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1504	     i--)
1505		mem->slots[i].list = ++count;
1506	area->used -= nslots;
1507	spin_unlock_irqrestore(&area->lock, flags);
1508
1509	dec_used(dev->dma_io_tlb_mem, nslots);
1510}
1511
1512#ifdef CONFIG_SWIOTLB_DYNAMIC
1513
1514/**
1515 * swiotlb_del_transient() - delete a transient memory pool
1516 * @dev:	Device which mapped the buffer.
1517 * @tlb_addr:	Physical address within a bounce buffer.
1518 * @pool:       Pointer to the transient memory pool to be checked and deleted.
1519 *
1520 * Check whether the address belongs to a transient SWIOTLB memory pool.
1521 * If yes, then delete the pool.
1522 *
1523 * Return: %true if @tlb_addr belonged to a transient pool that was released.
1524 */
1525static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr,
1526		struct io_tlb_pool *pool)
1527{
1528	if (!pool->transient)
1529		return false;
1530
1531	dec_used(dev->dma_io_tlb_mem, pool->nslabs);
1532	swiotlb_del_pool(dev, pool);
1533	dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);
1534	return true;
1535}
1536
1537#else  /* !CONFIG_SWIOTLB_DYNAMIC */
1538
1539static inline bool swiotlb_del_transient(struct device *dev,
1540		phys_addr_t tlb_addr, struct io_tlb_pool *pool)
1541{
1542	return false;
1543}
1544
1545#endif	/* CONFIG_SWIOTLB_DYNAMIC */
1546
1547/*
1548 * tlb_addr is the physical address of the bounce buffer to unmap.
1549 */
1550void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1551		size_t mapping_size, enum dma_data_direction dir,
1552		unsigned long attrs, struct io_tlb_pool *pool)
1553{
1554	/*
1555	 * First, sync the memory before unmapping the entry
1556	 */
1557	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1558	    (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1559		swiotlb_bounce(dev, tlb_addr, mapping_size,
1560						DMA_FROM_DEVICE, pool);
1561
1562	if (swiotlb_del_transient(dev, tlb_addr, pool))
1563		return;
1564	swiotlb_release_slots(dev, tlb_addr, pool);
1565}
1566
1567void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1568		size_t size, enum dma_data_direction dir,
1569		struct io_tlb_pool *pool)
1570{
1571	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1572		swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool);
1573	else
1574		BUG_ON(dir != DMA_FROM_DEVICE);
1575}
1576
1577void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1578		size_t size, enum dma_data_direction dir,
1579		struct io_tlb_pool *pool)
1580{
1581	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1582		swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool);
1583	else
1584		BUG_ON(dir != DMA_TO_DEVICE);
1585}
1586
1587/*
1588 * Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1589 * to the device copy the data into it as well.
1590 */
1591dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1592		enum dma_data_direction dir, unsigned long attrs)
1593{
1594	phys_addr_t swiotlb_addr;
1595	dma_addr_t dma_addr;
1596
1597	trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
1598
1599	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
1600	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1601		return DMA_MAPPING_ERROR;
1602
1603	/* Ensure that the address returned is DMA'ble */
1604	dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
1605	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1606		__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
1607			attrs | DMA_ATTR_SKIP_CPU_SYNC,
1608			swiotlb_find_pool(dev, swiotlb_addr));
1609		dev_WARN_ONCE(dev, 1,
1610			"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1611			&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1612		return DMA_MAPPING_ERROR;
1613	}
1614
1615	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1616		arch_sync_dma_for_device(swiotlb_addr, size, dir);
1617	return dma_addr;
1618}
1619
1620size_t swiotlb_max_mapping_size(struct device *dev)
1621{
1622	int min_align_mask = dma_get_min_align_mask(dev);
1623	int min_align = 0;
1624
1625	/*
1626	 * swiotlb_find_slots() skips slots according to
1627	 * min align mask. This affects max mapping size.
1628	 * Take it into acount here.
1629	 */
1630	if (min_align_mask)
1631		min_align = roundup(min_align_mask, IO_TLB_SIZE);
1632
1633	return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1634}
1635
1636/**
1637 * is_swiotlb_allocated() - check if the default software IO TLB is initialized
1638 */
1639bool is_swiotlb_allocated(void)
1640{
1641	return io_tlb_default_mem.nslabs;
1642}
1643
1644bool is_swiotlb_active(struct device *dev)
1645{
1646	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1647
1648	return mem && mem->nslabs;
1649}
1650
1651/**
1652 * default_swiotlb_base() - get the base address of the default SWIOTLB
1653 *
1654 * Get the lowest physical address used by the default software IO TLB pool.
1655 */
1656phys_addr_t default_swiotlb_base(void)
1657{
1658#ifdef CONFIG_SWIOTLB_DYNAMIC
1659	io_tlb_default_mem.can_grow = false;
1660#endif
1661	return io_tlb_default_mem.defpool.start;
1662}
1663
1664/**
1665 * default_swiotlb_limit() - get the address limit of the default SWIOTLB
1666 *
1667 * Get the highest physical address used by the default software IO TLB pool.
1668 */
1669phys_addr_t default_swiotlb_limit(void)
1670{
1671#ifdef CONFIG_SWIOTLB_DYNAMIC
1672	return io_tlb_default_mem.phys_limit;
1673#else
1674	return io_tlb_default_mem.defpool.end - 1;
1675#endif
1676}
1677
1678#ifdef CONFIG_DEBUG_FS
1679#ifdef CONFIG_SWIOTLB_DYNAMIC
1680static unsigned long mem_transient_used(struct io_tlb_mem *mem)
1681{
1682	return atomic_long_read(&mem->transient_nslabs);
1683}
1684
1685static int io_tlb_transient_used_get(void *data, u64 *val)
1686{
1687	struct io_tlb_mem *mem = data;
1688
1689	*val = mem_transient_used(mem);
1690	return 0;
1691}
1692
1693DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
1694			 NULL, "%llu\n");
1695#endif /* CONFIG_SWIOTLB_DYNAMIC */
1696
1697static int io_tlb_used_get(void *data, u64 *val)
1698{
1699	struct io_tlb_mem *mem = data;
1700
1701	*val = mem_used(mem);
1702	return 0;
1703}
1704
1705static int io_tlb_hiwater_get(void *data, u64 *val)
1706{
1707	struct io_tlb_mem *mem = data;
1708
1709	*val = atomic_long_read(&mem->used_hiwater);
1710	return 0;
1711}
1712
1713static int io_tlb_hiwater_set(void *data, u64 val)
1714{
1715	struct io_tlb_mem *mem = data;
1716
1717	/* Only allow setting to zero */
1718	if (val != 0)
1719		return -EINVAL;
1720
1721	atomic_long_set(&mem->used_hiwater, val);
1722	return 0;
1723}
1724
1725DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1726DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1727				io_tlb_hiwater_set, "%llu\n");
1728
1729static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1730					 const char *dirname)
1731{
1732	mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
1733	if (!mem->nslabs)
1734		return;
1735
1736	debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
1737	debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
1738			&fops_io_tlb_used);
1739	debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
1740			&fops_io_tlb_hiwater);
1741#ifdef CONFIG_SWIOTLB_DYNAMIC
1742	debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
1743			    mem, &fops_io_tlb_transient_used);
1744#endif
1745}
1746
1747static int __init swiotlb_create_default_debugfs(void)
1748{
1749	swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
1750	return 0;
1751}
1752
1753late_initcall(swiotlb_create_default_debugfs);
1754
1755#else  /* !CONFIG_DEBUG_FS */
1756
1757static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1758						const char *dirname)
1759{
1760}
1761
1762#endif	/* CONFIG_DEBUG_FS */
1763
1764#ifdef CONFIG_DMA_RESTRICTED_POOL
1765
1766struct page *swiotlb_alloc(struct device *dev, size_t size)
1767{
1768	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1769	struct io_tlb_pool *pool;
1770	phys_addr_t tlb_addr;
1771	unsigned int align;
1772	int index;
1773
1774	if (!mem)
1775		return NULL;
1776
1777	align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1778	index = swiotlb_find_slots(dev, 0, size, align, &pool);
1779	if (index == -1)
1780		return NULL;
1781
1782	tlb_addr = slot_addr(pool->start, index);
1783	if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1784		dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1785			      &tlb_addr);
1786		swiotlb_release_slots(dev, tlb_addr, pool);
1787		return NULL;
1788	}
1789
1790	return pfn_to_page(PFN_DOWN(tlb_addr));
1791}
1792
1793bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1794{
1795	phys_addr_t tlb_addr = page_to_phys(page);
1796	struct io_tlb_pool *pool;
1797
1798	pool = swiotlb_find_pool(dev, tlb_addr);
1799	if (!pool)
1800		return false;
1801
1802	swiotlb_release_slots(dev, tlb_addr, pool);
1803
1804	return true;
1805}
1806
1807static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1808				    struct device *dev)
1809{
1810	struct io_tlb_mem *mem = rmem->priv;
1811	unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1812
1813	/* Set Per-device io tlb area to one */
1814	unsigned int nareas = 1;
1815
1816	if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
1817		dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
1818		return -EINVAL;
1819	}
1820
1821	/*
1822	 * Since multiple devices can share the same pool, the private data,
1823	 * io_tlb_mem struct, will be initialized by the first device attached
1824	 * to it.
1825	 */
1826	if (!mem) {
1827		struct io_tlb_pool *pool;
1828
1829		mem = kzalloc(sizeof(*mem), GFP_KERNEL);
1830		if (!mem)
1831			return -ENOMEM;
1832		pool = &mem->defpool;
1833
1834		pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);
1835		if (!pool->slots) {
1836			kfree(mem);
1837			return -ENOMEM;
1838		}
1839
1840		pool->areas = kcalloc(nareas, sizeof(*pool->areas),
1841				GFP_KERNEL);
1842		if (!pool->areas) {
1843			kfree(pool->slots);
1844			kfree(mem);
1845			return -ENOMEM;
1846		}
1847
1848		set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
1849				     rmem->size >> PAGE_SHIFT);
1850		swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
1851					 false, nareas);
1852		mem->force_bounce = true;
1853		mem->for_alloc = true;
1854#ifdef CONFIG_SWIOTLB_DYNAMIC
1855		spin_lock_init(&mem->lock);
1856		INIT_LIST_HEAD_RCU(&mem->pools);
1857#endif
1858		add_mem_pool(mem, pool);
1859
1860		rmem->priv = mem;
1861
1862		swiotlb_create_debugfs_files(mem, rmem->name);
1863	}
1864
1865	dev->dma_io_tlb_mem = mem;
1866
1867	return 0;
1868}
1869
1870static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
1871					struct device *dev)
1872{
1873	dev->dma_io_tlb_mem = &io_tlb_default_mem;
1874}
1875
1876static const struct reserved_mem_ops rmem_swiotlb_ops = {
1877	.device_init = rmem_swiotlb_device_init,
1878	.device_release = rmem_swiotlb_device_release,
1879};
1880
1881static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
1882{
1883	unsigned long node = rmem->fdt_node;
1884
1885	if (of_get_flat_dt_prop(node, "reusable", NULL) ||
1886	    of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
1887	    of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
1888	    of_get_flat_dt_prop(node, "no-map", NULL))
1889		return -EINVAL;
1890
1891	rmem->ops = &rmem_swiotlb_ops;
1892	pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
1893		&rmem->base, (unsigned long)rmem->size / SZ_1M);
1894	return 0;
1895}
1896
1897RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
1898#endif /* CONFIG_DMA_RESTRICTED_POOL */