FILE
mapping.c
File preview with syntax highlighting.
linux-6.19.14/kernel/dma/mapping.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * arch-independent dma-mapping routines 4 * 5 * Copyright (c) 2006 SUSE Linux Products GmbH 6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de> 7 */ 8#include <linux/memblock.h> /* for max_pfn */ 9#include <linux/acpi.h> 10#include <linux/dma-map-ops.h> 11#include <linux/export.h> 12#include <linux/gfp.h> 13#include <linux/iommu-dma.h> 14#include <linux/kmsan.h> 15#include <linux/of_device.h> 16#include <linux/slab.h> 17#include <linux/vmalloc.h> 18#include "debug.h" 19#include "direct.h" 20 21#define CREATE_TRACE_POINTS 22#include <trace/events/dma.h> 23 24#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ 25 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ 26 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) 27bool dma_default_coherent = IS_ENABLED(CONFIG_ARCH_DMA_DEFAULT_COHERENT); 28#endif 29 30/* 31 * Managed DMA API 32 */ 33struct dma_devres { 34 size_t size; 35 void *vaddr; 36 dma_addr_t dma_handle; 37 unsigned long attrs; 38}; 39 40static void dmam_release(struct device *dev, void *res) 41{ 42 struct dma_devres *this = res; 43 44 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, 45 this->attrs); 46} 47 48static int dmam_match(struct device *dev, void *res, void *match_data) 49{ 50 struct dma_devres *this = res, *match = match_data; 51 52 if (this->vaddr == match->vaddr) { 53 WARN_ON(this->size != match->size || 54 this->dma_handle != match->dma_handle); 55 return 1; 56 } 57 return 0; 58} 59 60/** 61 * dmam_free_coherent - Managed dma_free_coherent() 62 * @dev: Device to free coherent memory for 63 * @size: Size of allocation 64 * @vaddr: Virtual address of the memory to free 65 * @dma_handle: DMA handle of the memory to free 66 * 67 * Managed dma_free_coherent(). 68 */ 69void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, 70 dma_addr_t dma_handle) 71{ 72 struct dma_devres match_data = { size, vaddr, dma_handle }; 73 74 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); 75 dma_free_coherent(dev, size, vaddr, dma_handle); 76} 77EXPORT_SYMBOL(dmam_free_coherent); 78 79/** 80 * dmam_alloc_attrs - Managed dma_alloc_attrs() 81 * @dev: Device to allocate non_coherent memory for 82 * @size: Size of allocation 83 * @dma_handle: Out argument for allocated DMA handle 84 * @gfp: Allocation flags 85 * @attrs: Flags in the DMA_ATTR_* namespace. 86 * 87 * Managed dma_alloc_attrs(). Memory allocated using this function will be 88 * automatically released on driver detach. 89 * 90 * RETURNS: 91 * Pointer to allocated memory on success, NULL on failure. 92 */ 93void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 94 gfp_t gfp, unsigned long attrs) 95{ 96 struct dma_devres *dr; 97 void *vaddr; 98 99 dr = devres_alloc(dmam_release, sizeof(*dr), gfp); 100 if (!dr) 101 return NULL; 102 103 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); 104 if (!vaddr) { 105 devres_free(dr); 106 return NULL; 107 } 108 109 dr->vaddr = vaddr; 110 dr->dma_handle = *dma_handle; 111 dr->size = size; 112 dr->attrs = attrs; 113 114 devres_add(dev, dr); 115 116 return vaddr; 117} 118EXPORT_SYMBOL(dmam_alloc_attrs); 119 120static bool dma_go_direct(struct device *dev, dma_addr_t mask, 121 const struct dma_map_ops *ops) 122{ 123 if (use_dma_iommu(dev)) 124 return false; 125 126 if (likely(!ops)) 127 return true; 128 129#ifdef CONFIG_DMA_OPS_BYPASS 130 if (dev->dma_ops_bypass) 131 return min_not_zero(mask, dev->bus_dma_limit) >= 132 dma_direct_get_required_mask(dev); 133#endif 134 return false; 135} 136 137 138/* 139 * Check if the devices uses a direct mapping for streaming DMA operations. 140 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large 141 * enough. 142 */ 143static inline bool dma_alloc_direct(struct device *dev, 144 const struct dma_map_ops *ops) 145{ 146 return dma_go_direct(dev, dev->coherent_dma_mask, ops); 147} 148 149static inline bool dma_map_direct(struct device *dev, 150 const struct dma_map_ops *ops) 151{ 152 return dma_go_direct(dev, *dev->dma_mask, ops); 153} 154 155dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 156 enum dma_data_direction dir, unsigned long attrs) 157{ 158 const struct dma_map_ops *ops = get_dma_ops(dev); 159 bool is_mmio = attrs & DMA_ATTR_MMIO; 160 dma_addr_t addr = DMA_MAPPING_ERROR; 161 162 BUG_ON(!valid_dma_direction(dir)); 163 164 if (WARN_ON_ONCE(!dev->dma_mask)) 165 return DMA_MAPPING_ERROR; 166 167 if (dma_map_direct(dev, ops) || 168 (!is_mmio && arch_dma_map_phys_direct(dev, phys + size))) 169 addr = dma_direct_map_phys(dev, phys, size, dir, attrs); 170 else if (use_dma_iommu(dev)) 171 addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); 172 else if (ops->map_phys) 173 addr = ops->map_phys(dev, phys, size, dir, attrs); 174 175 if (!is_mmio) 176 kmsan_handle_dma(phys, size, dir); 177 trace_dma_map_phys(dev, phys, addr, size, dir, attrs); 178 debug_dma_map_phys(dev, phys, size, dir, addr, attrs); 179 180 return addr; 181} 182EXPORT_SYMBOL_GPL(dma_map_phys); 183 184dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 185 size_t offset, size_t size, enum dma_data_direction dir, 186 unsigned long attrs) 187{ 188 phys_addr_t phys = page_to_phys(page) + offset; 189 190 if (unlikely(attrs & DMA_ATTR_MMIO)) 191 return DMA_MAPPING_ERROR; 192 193 if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && 194 WARN_ON_ONCE(is_zone_device_page(page))) 195 return DMA_MAPPING_ERROR; 196 197 return dma_map_phys(dev, phys, size, dir, attrs); 198} 199EXPORT_SYMBOL(dma_map_page_attrs); 200 201void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, 202 enum dma_data_direction dir, unsigned long attrs) 203{ 204 const struct dma_map_ops *ops = get_dma_ops(dev); 205 bool is_mmio = attrs & DMA_ATTR_MMIO; 206 207 BUG_ON(!valid_dma_direction(dir)); 208 if (dma_map_direct(dev, ops) || 209 (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size))) 210 dma_direct_unmap_phys(dev, addr, size, dir, attrs); 211 else if (use_dma_iommu(dev)) 212 iommu_dma_unmap_phys(dev, addr, size, dir, attrs); 213 else if (ops->unmap_phys) 214 ops->unmap_phys(dev, addr, size, dir, attrs); 215 trace_dma_unmap_phys(dev, addr, size, dir, attrs); 216 debug_dma_unmap_phys(dev, addr, size, dir); 217} 218EXPORT_SYMBOL_GPL(dma_unmap_phys); 219 220void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 221 enum dma_data_direction dir, unsigned long attrs) 222{ 223 if (unlikely(attrs & DMA_ATTR_MMIO)) 224 return; 225 226 dma_unmap_phys(dev, addr, size, dir, attrs); 227} 228EXPORT_SYMBOL(dma_unmap_page_attrs); 229 230static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 231 int nents, enum dma_data_direction dir, unsigned long attrs) 232{ 233 const struct dma_map_ops *ops = get_dma_ops(dev); 234 int ents; 235 236 BUG_ON(!valid_dma_direction(dir)); 237 238 if (WARN_ON_ONCE(!dev->dma_mask)) 239 return 0; 240 241 if (dma_map_direct(dev, ops) || 242 arch_dma_map_sg_direct(dev, sg, nents)) 243 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); 244 else if (use_dma_iommu(dev)) 245 ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); 246 else 247 ents = ops->map_sg(dev, sg, nents, dir, attrs); 248 249 if (ents > 0) { 250 kmsan_handle_dma_sg(sg, nents, dir); 251 trace_dma_map_sg(dev, sg, nents, ents, dir, attrs); 252 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); 253 } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && 254 ents != -EIO && ents != -EREMOTEIO)) { 255 trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs); 256 return -EIO; 257 } 258 259 return ents; 260} 261 262/** 263 * dma_map_sg_attrs - Map the given buffer for DMA 264 * @dev: The device for which to perform the DMA operation 265 * @sg: The sg_table object describing the buffer 266 * @nents: Number of entries to map 267 * @dir: DMA direction 268 * @attrs: Optional DMA attributes for the map operation 269 * 270 * Maps a buffer described by a scatterlist passed in the sg argument with 271 * nents segments for the @dir DMA operation by the @dev device. 272 * 273 * Returns the number of mapped entries (which can be less than nents) 274 * on success. Zero is returned for any error. 275 * 276 * dma_unmap_sg_attrs() should be used to unmap the buffer with the 277 * original sg and original nents (not the value returned by this funciton). 278 */ 279unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 280 int nents, enum dma_data_direction dir, unsigned long attrs) 281{ 282 int ret; 283 284 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); 285 if (ret < 0) 286 return 0; 287 return ret; 288} 289EXPORT_SYMBOL(dma_map_sg_attrs); 290 291/** 292 * dma_map_sgtable - Map the given buffer for DMA 293 * @dev: The device for which to perform the DMA operation 294 * @sgt: The sg_table object describing the buffer 295 * @dir: DMA direction 296 * @attrs: Optional DMA attributes for the map operation 297 * 298 * Maps a buffer described by a scatterlist stored in the given sg_table 299 * object for the @dir DMA operation by the @dev device. After success, the 300 * ownership for the buffer is transferred to the DMA domain. One has to 301 * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the 302 * ownership of the buffer back to the CPU domain before touching the 303 * buffer by the CPU. 304 * 305 * Returns 0 on success or a negative error code on error. The following 306 * error codes are supported with the given meaning: 307 * 308 * -EINVAL An invalid argument, unaligned access or other error 309 * in usage. Will not succeed if retried. 310 * -ENOMEM Insufficient resources (like memory or IOVA space) to 311 * complete the mapping. Should succeed if retried later. 312 * -EIO Legacy error code with an unknown meaning. eg. this is 313 * returned if a lower level call returned 314 * DMA_MAPPING_ERROR. 315 * -EREMOTEIO The DMA device cannot access P2PDMA memory specified 316 * in the sg_table. This will not succeed if retried. 317 */ 318int dma_map_sgtable(struct device *dev, struct sg_table *sgt, 319 enum dma_data_direction dir, unsigned long attrs) 320{ 321 int nents; 322 323 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); 324 if (nents < 0) 325 return nents; 326 sgt->nents = nents; 327 return 0; 328} 329EXPORT_SYMBOL_GPL(dma_map_sgtable); 330 331void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, 332 int nents, enum dma_data_direction dir, 333 unsigned long attrs) 334{ 335 const struct dma_map_ops *ops = get_dma_ops(dev); 336 337 BUG_ON(!valid_dma_direction(dir)); 338 trace_dma_unmap_sg(dev, sg, nents, dir, attrs); 339 debug_dma_unmap_sg(dev, sg, nents, dir); 340 if (dma_map_direct(dev, ops) || 341 arch_dma_unmap_sg_direct(dev, sg, nents)) 342 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); 343 else if (use_dma_iommu(dev)) 344 iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); 345 else if (ops->unmap_sg) 346 ops->unmap_sg(dev, sg, nents, dir, attrs); 347} 348EXPORT_SYMBOL(dma_unmap_sg_attrs); 349 350dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 351 size_t size, enum dma_data_direction dir, unsigned long attrs) 352{ 353 if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && 354 WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) 355 return DMA_MAPPING_ERROR; 356 357 return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO); 358} 359EXPORT_SYMBOL(dma_map_resource); 360 361void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 362 enum dma_data_direction dir, unsigned long attrs) 363{ 364 dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO); 365} 366EXPORT_SYMBOL(dma_unmap_resource); 367 368#ifdef CONFIG_DMA_NEED_SYNC 369void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 370 enum dma_data_direction dir) 371{ 372 const struct dma_map_ops *ops = get_dma_ops(dev); 373 374 BUG_ON(!valid_dma_direction(dir)); 375 if (dma_map_direct(dev, ops)) 376 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 377 else if (use_dma_iommu(dev)) 378 iommu_dma_sync_single_for_cpu(dev, addr, size, dir); 379 else if (ops->sync_single_for_cpu) 380 ops->sync_single_for_cpu(dev, addr, size, dir); 381 trace_dma_sync_single_for_cpu(dev, addr, size, dir); 382 debug_dma_sync_single_for_cpu(dev, addr, size, dir); 383} 384EXPORT_SYMBOL(__dma_sync_single_for_cpu); 385 386void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, 387 size_t size, enum dma_data_direction dir) 388{ 389 const struct dma_map_ops *ops = get_dma_ops(dev); 390 391 BUG_ON(!valid_dma_direction(dir)); 392 if (dma_map_direct(dev, ops)) 393 dma_direct_sync_single_for_device(dev, addr, size, dir); 394 else if (use_dma_iommu(dev)) 395 iommu_dma_sync_single_for_device(dev, addr, size, dir); 396 else if (ops->sync_single_for_device) 397 ops->sync_single_for_device(dev, addr, size, dir); 398 trace_dma_sync_single_for_device(dev, addr, size, dir); 399 debug_dma_sync_single_for_device(dev, addr, size, dir); 400} 401EXPORT_SYMBOL(__dma_sync_single_for_device); 402 403void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 404 int nelems, enum dma_data_direction dir) 405{ 406 const struct dma_map_ops *ops = get_dma_ops(dev); 407 408 BUG_ON(!valid_dma_direction(dir)); 409 if (dma_map_direct(dev, ops)) 410 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); 411 else if (use_dma_iommu(dev)) 412 iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 413 else if (ops->sync_sg_for_cpu) 414 ops->sync_sg_for_cpu(dev, sg, nelems, dir); 415 trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 416 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 417} 418EXPORT_SYMBOL(__dma_sync_sg_for_cpu); 419 420void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 421 int nelems, enum dma_data_direction dir) 422{ 423 const struct dma_map_ops *ops = get_dma_ops(dev); 424 425 BUG_ON(!valid_dma_direction(dir)); 426 if (dma_map_direct(dev, ops)) 427 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); 428 else if (use_dma_iommu(dev)) 429 iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); 430 else if (ops->sync_sg_for_device) 431 ops->sync_sg_for_device(dev, sg, nelems, dir); 432 trace_dma_sync_sg_for_device(dev, sg, nelems, dir); 433 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); 434} 435EXPORT_SYMBOL(__dma_sync_sg_for_device); 436 437bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr) 438{ 439 const struct dma_map_ops *ops = get_dma_ops(dev); 440 441 if (dma_map_direct(dev, ops)) 442 /* 443 * dma_skip_sync could've been reset on first SWIOTLB buffer 444 * mapping, but @dma_addr is not necessary an SWIOTLB buffer. 445 * In this case, fall back to more granular check. 446 */ 447 return dma_direct_need_sync(dev, dma_addr); 448 return true; 449} 450EXPORT_SYMBOL_GPL(__dma_need_sync); 451 452/** 453 * dma_need_unmap - does this device need dma_unmap_* operations 454 * @dev: device to check 455 * 456 * If this function returns %false, drivers can skip calling dma_unmap_* after 457 * finishing an I/O. This function must be called after all mappings that might 458 * need to be unmapped have been performed. 459 */ 460bool dma_need_unmap(struct device *dev) 461{ 462 if (!dma_map_direct(dev, get_dma_ops(dev))) 463 return true; 464 if (!dev->dma_skip_sync) 465 return true; 466 return IS_ENABLED(CONFIG_DMA_API_DEBUG); 467} 468EXPORT_SYMBOL_GPL(dma_need_unmap); 469 470static void dma_setup_need_sync(struct device *dev) 471{ 472 const struct dma_map_ops *ops = get_dma_ops(dev); 473 474 if (dma_map_direct(dev, ops) || use_dma_iommu(dev)) 475 /* 476 * dma_skip_sync will be reset to %false on first SWIOTLB buffer 477 * mapping, if any. During the device initialization, it's 478 * enough to check only for the DMA coherence. 479 */ 480 dev->dma_skip_sync = dev_is_dma_coherent(dev); 481 else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && 482 !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) 483 /* 484 * Synchronization is not possible when none of DMA sync ops 485 * is set. 486 */ 487 dev->dma_skip_sync = true; 488 else 489 dev->dma_skip_sync = false; 490} 491#else /* !CONFIG_DMA_NEED_SYNC */ 492static inline void dma_setup_need_sync(struct device *dev) { } 493#endif /* !CONFIG_DMA_NEED_SYNC */ 494 495/* 496 * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems 497 * that the intention is to allow exporting memory allocated via the 498 * coherent DMA APIs through the dma_buf API, which only accepts a 499 * scattertable. This presents a couple of problems: 500 * 1. Not all memory allocated via the coherent DMA APIs is backed by 501 * a struct page 502 * 2. Passing coherent DMA memory into the streaming APIs is not allowed 503 * as we will try to flush the memory through a different alias to that 504 * actually being used (and the flushes are redundant.) 505 */ 506int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, 507 void *cpu_addr, dma_addr_t dma_addr, size_t size, 508 unsigned long attrs) 509{ 510 const struct dma_map_ops *ops = get_dma_ops(dev); 511 512 if (dma_alloc_direct(dev, ops)) 513 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, 514 size, attrs); 515 if (use_dma_iommu(dev)) 516 return iommu_dma_get_sgtable(dev, sgt, cpu_addr, dma_addr, 517 size, attrs); 518 if (!ops->get_sgtable) 519 return -ENXIO; 520 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); 521} 522EXPORT_SYMBOL(dma_get_sgtable_attrs); 523 524#ifdef CONFIG_MMU 525/* 526 * Return the page attributes used for mapping dma_alloc_* memory, either in 527 * kernel space if remapping is needed, or to userspace through dma_mmap_*. 528 */ 529pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) 530{ 531 if (dev_is_dma_coherent(dev)) 532 return prot; 533#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE 534 if (attrs & DMA_ATTR_WRITE_COMBINE) 535 return pgprot_writecombine(prot); 536#endif 537 return pgprot_dmacoherent(prot); 538} 539#endif /* CONFIG_MMU */ 540 541/** 542 * dma_can_mmap - check if a given device supports dma_mmap_* 543 * @dev: device to check 544 * 545 * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to 546 * map DMA allocations to userspace. 547 */ 548bool dma_can_mmap(struct device *dev) 549{ 550 const struct dma_map_ops *ops = get_dma_ops(dev); 551 552 if (dma_alloc_direct(dev, ops)) 553 return dma_direct_can_mmap(dev); 554 if (use_dma_iommu(dev)) 555 return true; 556 return ops->mmap != NULL; 557} 558EXPORT_SYMBOL_GPL(dma_can_mmap); 559 560/** 561 * dma_mmap_attrs - map a coherent DMA allocation into user space 562 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 563 * @vma: vm_area_struct describing requested user mapping 564 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs 565 * @dma_addr: device-view address returned from dma_alloc_attrs 566 * @size: size of memory originally requested in dma_alloc_attrs 567 * @attrs: attributes of mapping properties requested in dma_alloc_attrs 568 * 569 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user 570 * space. The coherent DMA buffer must not be freed by the driver until the 571 * user space mapping has been released. 572 */ 573int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 574 void *cpu_addr, dma_addr_t dma_addr, size_t size, 575 unsigned long attrs) 576{ 577 const struct dma_map_ops *ops = get_dma_ops(dev); 578 579 if (dma_alloc_direct(dev, ops)) 580 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, 581 attrs); 582 if (use_dma_iommu(dev)) 583 return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size, 584 attrs); 585 if (!ops->mmap) 586 return -ENXIO; 587 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 588} 589EXPORT_SYMBOL(dma_mmap_attrs); 590 591u64 dma_get_required_mask(struct device *dev) 592{ 593 const struct dma_map_ops *ops = get_dma_ops(dev); 594 595 if (dma_alloc_direct(dev, ops)) 596 return dma_direct_get_required_mask(dev); 597 598 if (use_dma_iommu(dev)) 599 return DMA_BIT_MASK(32); 600 601 if (ops->get_required_mask) 602 return ops->get_required_mask(dev); 603 604 /* 605 * We require every DMA ops implementation to at least support a 32-bit 606 * DMA mask (and use bounce buffering if that isn't supported in 607 * hardware). As the direct mapping code has its own routine to 608 * actually report an optimal mask we default to 32-bit here as that 609 * is the right thing for most IOMMUs, and at least not actively 610 * harmful in general. 611 */ 612 return DMA_BIT_MASK(32); 613} 614EXPORT_SYMBOL_GPL(dma_get_required_mask); 615 616void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 617 gfp_t flag, unsigned long attrs) 618{ 619 const struct dma_map_ops *ops = get_dma_ops(dev); 620 void *cpu_addr; 621 622 WARN_ON_ONCE(!dev->coherent_dma_mask); 623 624 /* 625 * DMA allocations can never be turned back into a page pointer, so 626 * requesting compound pages doesn't make sense (and can't even be 627 * supported at all by various backends). 628 */ 629 if (WARN_ON_ONCE(flag & __GFP_COMP)) 630 return NULL; 631 632 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) { 633 trace_dma_alloc(dev, cpu_addr, *dma_handle, size, 634 DMA_BIDIRECTIONAL, flag, attrs); 635 return cpu_addr; 636 } 637 638 /* let the implementation decide on the zone to allocate from: */ 639 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); 640 641 if (dma_alloc_direct(dev, ops)) { 642 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); 643 } else if (use_dma_iommu(dev)) { 644 cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs); 645 } else if (ops->alloc) { 646 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); 647 } else { 648 trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag, 649 attrs); 650 return NULL; 651 } 652 653 trace_dma_alloc(dev, cpu_addr, *dma_handle, size, DMA_BIDIRECTIONAL, 654 flag, attrs); 655 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs); 656 return cpu_addr; 657} 658EXPORT_SYMBOL(dma_alloc_attrs); 659 660void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, 661 dma_addr_t dma_handle, unsigned long attrs) 662{ 663 const struct dma_map_ops *ops = get_dma_ops(dev); 664 665 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr)) 666 return; 667 /* 668 * On non-coherent platforms which implement DMA-coherent buffers via 669 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting 670 * this far in IRQ context is a) at risk of a BUG_ON() or trying to 671 * sleep on some machines, and b) an indication that the driver is 672 * probably misusing the coherent API anyway. 673 */ 674 WARN_ON(irqs_disabled()); 675 676 trace_dma_free(dev, cpu_addr, dma_handle, size, DMA_BIDIRECTIONAL, 677 attrs); 678 if (!cpu_addr) 679 return; 680 681 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); 682 if (dma_alloc_direct(dev, ops)) 683 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); 684 else if (use_dma_iommu(dev)) 685 iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs); 686 else if (ops->free) 687 ops->free(dev, size, cpu_addr, dma_handle, attrs); 688} 689EXPORT_SYMBOL(dma_free_attrs); 690 691static struct page *__dma_alloc_pages(struct device *dev, size_t size, 692 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 693{ 694 const struct dma_map_ops *ops = get_dma_ops(dev); 695 696 if (WARN_ON_ONCE(!dev->coherent_dma_mask)) 697 return NULL; 698 if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM))) 699 return NULL; 700 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 701 return NULL; 702 703 size = PAGE_ALIGN(size); 704 if (dma_alloc_direct(dev, ops)) 705 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); 706 if (use_dma_iommu(dev)) 707 return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp); 708 if (!ops->alloc_pages_op) 709 return NULL; 710 return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); 711} 712 713struct page *dma_alloc_pages(struct device *dev, size_t size, 714 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 715{ 716 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); 717 718 if (page) { 719 trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle, 720 size, dir, gfp, 0); 721 debug_dma_alloc_pages(dev, page, size, dir, *dma_handle, 0); 722 } else { 723 trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0); 724 } 725 return page; 726} 727EXPORT_SYMBOL_GPL(dma_alloc_pages); 728 729static void __dma_free_pages(struct device *dev, size_t size, struct page *page, 730 dma_addr_t dma_handle, enum dma_data_direction dir) 731{ 732 const struct dma_map_ops *ops = get_dma_ops(dev); 733 734 size = PAGE_ALIGN(size); 735 if (dma_alloc_direct(dev, ops)) 736 dma_direct_free_pages(dev, size, page, dma_handle, dir); 737 else if (use_dma_iommu(dev)) 738 dma_common_free_pages(dev, size, page, dma_handle, dir); 739 else if (ops->free_pages) 740 ops->free_pages(dev, size, page, dma_handle, dir); 741} 742 743void dma_free_pages(struct device *dev, size_t size, struct page *page, 744 dma_addr_t dma_handle, enum dma_data_direction dir) 745{ 746 trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0); 747 debug_dma_free_pages(dev, page, size, dir, dma_handle); 748 __dma_free_pages(dev, size, page, dma_handle, dir); 749} 750EXPORT_SYMBOL_GPL(dma_free_pages); 751 752int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, 753 size_t size, struct page *page) 754{ 755 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 756 757 if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff) 758 return -ENXIO; 759 return remap_pfn_range(vma, vma->vm_start, 760 page_to_pfn(page) + vma->vm_pgoff, 761 vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); 762} 763EXPORT_SYMBOL_GPL(dma_mmap_pages); 764 765static struct sg_table *alloc_single_sgt(struct device *dev, size_t size, 766 enum dma_data_direction dir, gfp_t gfp) 767{ 768 struct sg_table *sgt; 769 struct page *page; 770 771 sgt = kmalloc(sizeof(*sgt), gfp); 772 if (!sgt) 773 return NULL; 774 if (sg_alloc_table(sgt, 1, gfp)) 775 goto out_free_sgt; 776 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp); 777 if (!page) 778 goto out_free_table; 779 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 780 sg_dma_len(sgt->sgl) = sgt->sgl->length; 781 return sgt; 782out_free_table: 783 sg_free_table(sgt); 784out_free_sgt: 785 kfree(sgt); 786 return NULL; 787} 788 789struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, 790 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) 791{ 792 struct sg_table *sgt; 793 794 if (WARN_ON_ONCE(attrs & ~DMA_ATTR_ALLOC_SINGLE_PAGES)) 795 return NULL; 796 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 797 return NULL; 798 799 if (use_dma_iommu(dev)) 800 sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs); 801 else 802 sgt = alloc_single_sgt(dev, size, dir, gfp); 803 804 if (sgt) { 805 sgt->nents = 1; 806 trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs); 807 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); 808 } else { 809 trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs); 810 } 811 return sgt; 812} 813EXPORT_SYMBOL_GPL(dma_alloc_noncontiguous); 814 815static void free_single_sgt(struct device *dev, size_t size, 816 struct sg_table *sgt, enum dma_data_direction dir) 817{ 818 __dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address, 819 dir); 820 sg_free_table(sgt); 821 kfree(sgt); 822} 823 824void dma_free_noncontiguous(struct device *dev, size_t size, 825 struct sg_table *sgt, enum dma_data_direction dir) 826{ 827 trace_dma_free_sgt(dev, sgt, size, dir); 828 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); 829 830 if (use_dma_iommu(dev)) 831 iommu_dma_free_noncontiguous(dev, size, sgt, dir); 832 else 833 free_single_sgt(dev, size, sgt, dir); 834} 835EXPORT_SYMBOL_GPL(dma_free_noncontiguous); 836 837void *dma_vmap_noncontiguous(struct device *dev, size_t size, 838 struct sg_table *sgt) 839{ 840 841 if (use_dma_iommu(dev)) 842 return iommu_dma_vmap_noncontiguous(dev, size, sgt); 843 844 return page_address(sg_page(sgt->sgl)); 845} 846EXPORT_SYMBOL_GPL(dma_vmap_noncontiguous); 847 848void dma_vunmap_noncontiguous(struct device *dev, void *vaddr) 849{ 850 if (use_dma_iommu(dev)) 851 iommu_dma_vunmap_noncontiguous(dev, vaddr); 852} 853EXPORT_SYMBOL_GPL(dma_vunmap_noncontiguous); 854 855int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma, 856 size_t size, struct sg_table *sgt) 857{ 858 if (use_dma_iommu(dev)) 859 return iommu_dma_mmap_noncontiguous(dev, vma, size, sgt); 860 return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl)); 861} 862EXPORT_SYMBOL_GPL(dma_mmap_noncontiguous); 863 864static int dma_supported(struct device *dev, u64 mask) 865{ 866 const struct dma_map_ops *ops = get_dma_ops(dev); 867 868 if (use_dma_iommu(dev)) { 869 if (WARN_ON(ops)) 870 return false; 871 return true; 872 } 873 874 /* 875 * ->dma_supported sets and clears the bypass flag, so ignore it here 876 * and always call into the method if there is one. 877 */ 878 if (ops) { 879 if (!ops->dma_supported) 880 return true; 881 return ops->dma_supported(dev, mask); 882 } 883 884 return dma_direct_supported(dev, mask); 885} 886 887bool dma_pci_p2pdma_supported(struct device *dev) 888{ 889 const struct dma_map_ops *ops = get_dma_ops(dev); 890 891 /* 892 * Note: dma_ops_bypass is not checked here because P2PDMA should 893 * not be used with dma mapping ops that do not have support even 894 * if the specific device is bypassing them. 895 */ 896 897 /* if ops is not set, dma direct and default IOMMU support P2PDMA */ 898 return !ops; 899} 900EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported); 901 902int dma_set_mask(struct device *dev, u64 mask) 903{ 904 /* 905 * Truncate the mask to the actually supported dma_addr_t width to 906 * avoid generating unsupportable addresses. 907 */ 908 mask = (dma_addr_t)mask; 909 910 if (!dev->dma_mask || !dma_supported(dev, mask)) 911 return -EIO; 912 913 arch_dma_set_mask(dev, mask); 914 *dev->dma_mask = mask; 915 dma_setup_need_sync(dev); 916 917 return 0; 918} 919EXPORT_SYMBOL(dma_set_mask); 920 921int dma_set_coherent_mask(struct device *dev, u64 mask) 922{ 923 /* 924 * Truncate the mask to the actually supported dma_addr_t width to 925 * avoid generating unsupportable addresses. 926 */ 927 mask = (dma_addr_t)mask; 928 929 if (!dma_supported(dev, mask)) 930 return -EIO; 931 932 dev->coherent_dma_mask = mask; 933 return 0; 934} 935EXPORT_SYMBOL(dma_set_coherent_mask); 936 937static bool __dma_addressing_limited(struct device *dev) 938{ 939 const struct dma_map_ops *ops = get_dma_ops(dev); 940 941 if (min_not_zero(dma_get_mask(dev), dev->bus_dma_limit) < 942 dma_get_required_mask(dev)) 943 return true; 944 945 if (unlikely(ops) || use_dma_iommu(dev)) 946 return false; 947 return !dma_direct_all_ram_mapped(dev); 948} 949 950/** 951 * dma_addressing_limited - return if the device is addressing limited 952 * @dev: device to check 953 * 954 * Return %true if the devices DMA mask is too small to address all memory in 955 * the system, else %false. Lack of addressing bits is the prime reason for 956 * bounce buffering, but might not be the only one. 957 */ 958bool dma_addressing_limited(struct device *dev) 959{ 960 if (!__dma_addressing_limited(dev)) 961 return false; 962 963 dev_dbg(dev, "device is DMA addressing limited\n"); 964 return true; 965} 966EXPORT_SYMBOL_GPL(dma_addressing_limited); 967 968size_t dma_max_mapping_size(struct device *dev) 969{ 970 const struct dma_map_ops *ops = get_dma_ops(dev); 971 size_t size = SIZE_MAX; 972 973 if (dma_map_direct(dev, ops)) 974 size = dma_direct_max_mapping_size(dev); 975 else if (use_dma_iommu(dev)) 976 size = iommu_dma_max_mapping_size(dev); 977 else if (ops && ops->max_mapping_size) 978 size = ops->max_mapping_size(dev); 979 980 return size; 981} 982EXPORT_SYMBOL_GPL(dma_max_mapping_size); 983 984size_t dma_opt_mapping_size(struct device *dev) 985{ 986 const struct dma_map_ops *ops = get_dma_ops(dev); 987 size_t size = SIZE_MAX; 988 989 if (use_dma_iommu(dev)) 990 size = iommu_dma_opt_mapping_size(); 991 else if (ops && ops->opt_mapping_size) 992 size = ops->opt_mapping_size(); 993 994 return min(dma_max_mapping_size(dev), size); 995} 996EXPORT_SYMBOL_GPL(dma_opt_mapping_size); 997 998unsigned long dma_get_merge_boundary(struct device *dev) 999{ 1000 const struct dma_map_ops *ops = get_dma_ops(dev); 1001 1002 if (use_dma_iommu(dev)) 1003 return iommu_dma_get_merge_boundary(dev); 1004 1005 if (!ops || !ops->get_merge_boundary) 1006 return 0; /* can't merge */ 1007 1008 return ops->get_merge_boundary(dev); 1009} 1010EXPORT_SYMBOL_GPL(dma_get_merge_boundary);
~/
Read-only file preview.