FILE
debug.c
File preview with syntax highlighting.
linux-6.19.14/kernel/dma/debug.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2008 Advanced Micro Devices, Inc. 4 * 5 * Author: Joerg Roedel <joerg.roedel@amd.com> 6 */ 7 8#define pr_fmt(fmt) "DMA-API: " fmt 9 10#include <linux/sched/task_stack.h> 11#include <linux/scatterlist.h> 12#include <linux/dma-map-ops.h> 13#include <linux/sched/task.h> 14#include <linux/stacktrace.h> 15#include <linux/spinlock.h> 16#include <linux/vmalloc.h> 17#include <linux/debugfs.h> 18#include <linux/uaccess.h> 19#include <linux/export.h> 20#include <linux/device.h> 21#include <linux/types.h> 22#include <linux/sched.h> 23#include <linux/ctype.h> 24#include <linux/list.h> 25#include <linux/slab.h> 26#include <linux/swiotlb.h> 27#include <asm/sections.h> 28#include "debug.h" 29 30#define HASH_SIZE 16384ULL 31#define HASH_FN_SHIFT 13 32#define HASH_FN_MASK (HASH_SIZE - 1) 33 34#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) 35/* If the pool runs out, add this many new entries at once */ 36#define DMA_DEBUG_DYNAMIC_ENTRIES (PAGE_SIZE / sizeof(struct dma_debug_entry)) 37 38enum { 39 dma_debug_single, 40 dma_debug_sg, 41 dma_debug_coherent, 42 dma_debug_noncoherent, 43 dma_debug_phy, 44}; 45 46enum map_err_types { 47 MAP_ERR_CHECK_NOT_APPLICABLE, 48 MAP_ERR_NOT_CHECKED, 49 MAP_ERR_CHECKED, 50}; 51 52#define DMA_DEBUG_STACKTRACE_ENTRIES 5 53 54/** 55 * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping 56 * @list: node on pre-allocated free_entries list 57 * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent 58 * @dev_addr: dma address 59 * @size: length of the mapping 60 * @type: single, page, sg, coherent 61 * @direction: enum dma_data_direction 62 * @sg_call_ents: 'nents' from dma_map_sg 63 * @sg_mapped_ents: 'mapped_ents' from dma_map_sg 64 * @paddr: physical start address of the mapping 65 * @map_err_type: track whether dma_mapping_error() was checked 66 * @is_cache_clean: driver promises not to write to buffer while mapped 67 * @stack_len: number of backtrace entries in @stack_entries 68 * @stack_entries: stack of backtrace history 69 */ 70struct dma_debug_entry { 71 struct list_head list; 72 struct device *dev; 73 u64 dev_addr; 74 u64 size; 75 int type; 76 int direction; 77 int sg_call_ents; 78 int sg_mapped_ents; 79 phys_addr_t paddr; 80 enum map_err_types map_err_type; 81 bool is_cache_clean; 82#ifdef CONFIG_STACKTRACE 83 unsigned int stack_len; 84 unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; 85#endif 86} ____cacheline_aligned_in_smp; 87 88typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *); 89 90struct hash_bucket { 91 struct list_head list; 92 spinlock_t lock; 93}; 94 95/* Hash list to save the allocated dma addresses */ 96static struct hash_bucket dma_entry_hash[HASH_SIZE]; 97/* List of pre-allocated dma_debug_entry's */ 98static LIST_HEAD(free_entries); 99/* Lock for the list above */ 100static DEFINE_SPINLOCK(free_entries_lock); 101 102/* Global disable flag - will be set in case of an error */ 103static bool global_disable __read_mostly; 104 105/* Early initialization disable flag, set at the end of dma_debug_init */ 106static bool dma_debug_initialized __read_mostly; 107 108static inline bool dma_debug_disabled(void) 109{ 110 return global_disable || !dma_debug_initialized; 111} 112 113/* Global error count */ 114static u32 error_count; 115 116/* Global error show enable*/ 117static u32 show_all_errors __read_mostly; 118/* Number of errors to show */ 119static u32 show_num_errors = 1; 120 121static u32 num_free_entries; 122static u32 min_free_entries; 123static u32 nr_total_entries; 124 125/* number of preallocated entries requested by kernel cmdline */ 126static u32 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES; 127 128/* per-driver filter related state */ 129 130#define NAME_MAX_LEN 64 131 132static char current_driver_name[NAME_MAX_LEN] __read_mostly; 133static struct device_driver *current_driver __read_mostly; 134 135static DEFINE_RWLOCK(driver_name_lock); 136 137static const char *const maperr2str[] = { 138 [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable", 139 [MAP_ERR_NOT_CHECKED] = "dma map error not checked", 140 [MAP_ERR_CHECKED] = "dma map error checked", 141}; 142 143static const char *type2name[] = { 144 [dma_debug_single] = "single", 145 [dma_debug_sg] = "scatter-gather", 146 [dma_debug_coherent] = "coherent", 147 [dma_debug_noncoherent] = "noncoherent", 148 [dma_debug_phy] = "phy", 149}; 150 151static const char *dir2name[] = { 152 [DMA_BIDIRECTIONAL] = "DMA_BIDIRECTIONAL", 153 [DMA_TO_DEVICE] = "DMA_TO_DEVICE", 154 [DMA_FROM_DEVICE] = "DMA_FROM_DEVICE", 155 [DMA_NONE] = "DMA_NONE", 156}; 157 158/* 159 * The access to some variables in this macro is racy. We can't use atomic_t 160 * here because all these variables are exported to debugfs. Some of them even 161 * writeable. This is also the reason why a lock won't help much. But anyway, 162 * the races are no big deal. Here is why: 163 * 164 * error_count: the addition is racy, but the worst thing that can happen is 165 * that we don't count some errors 166 * show_num_errors: the subtraction is racy. Also no big deal because in 167 * worst case this will result in one warning more in the 168 * system log than the user configured. This variable is 169 * writeable via debugfs. 170 */ 171static inline void dump_entry_trace(struct dma_debug_entry *entry) 172{ 173#ifdef CONFIG_STACKTRACE 174 if (entry) { 175 pr_warn("Mapped at:\n"); 176 stack_trace_print(entry->stack_entries, entry->stack_len, 0); 177 } 178#endif 179} 180 181static bool driver_filter(struct device *dev) 182{ 183 struct device_driver *drv; 184 unsigned long flags; 185 bool ret; 186 187 /* driver filter off */ 188 if (likely(!current_driver_name[0])) 189 return true; 190 191 /* driver filter on and initialized */ 192 if (current_driver && dev && dev->driver == current_driver) 193 return true; 194 195 /* driver filter on, but we can't filter on a NULL device... */ 196 if (!dev) 197 return false; 198 199 if (current_driver || !current_driver_name[0]) 200 return false; 201 202 /* driver filter on but not yet initialized */ 203 drv = dev->driver; 204 if (!drv) 205 return false; 206 207 /* lock to protect against change of current_driver_name */ 208 read_lock_irqsave(&driver_name_lock, flags); 209 210 ret = false; 211 if (drv->name && 212 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) { 213 current_driver = drv; 214 ret = true; 215 } 216 217 read_unlock_irqrestore(&driver_name_lock, flags); 218 219 return ret; 220} 221 222#define err_printk(dev, entry, format, arg...) do { \ 223 error_count += 1; \ 224 if (driver_filter(dev) && \ 225 (show_all_errors || show_num_errors > 0)) { \ 226 WARN(1, pr_fmt("%s %s: ") format, \ 227 dev ? dev_driver_string(dev) : "NULL", \ 228 dev ? dev_name(dev) : "NULL", ## arg); \ 229 dump_entry_trace(entry); \ 230 } \ 231 if (!show_all_errors && show_num_errors > 0) \ 232 show_num_errors -= 1; \ 233 } while (0); 234 235/* 236 * Hash related functions 237 * 238 * Every DMA-API request is saved into a struct dma_debug_entry. To 239 * have quick access to these structs they are stored into a hash. 240 */ 241static int hash_fn(struct dma_debug_entry *entry) 242{ 243 /* 244 * Hash function is based on the dma address. 245 * We use bits 20-27 here as the index into the hash 246 */ 247 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK; 248} 249 250/* 251 * Request exclusive access to a hash bucket for a given dma_debug_entry. 252 */ 253static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry, 254 unsigned long *flags) 255 __acquires(&dma_entry_hash[idx].lock) 256{ 257 int idx = hash_fn(entry); 258 unsigned long __flags; 259 260 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags); 261 *flags = __flags; 262 return &dma_entry_hash[idx]; 263} 264 265/* 266 * Give up exclusive access to the hash bucket 267 */ 268static void put_hash_bucket(struct hash_bucket *bucket, 269 unsigned long flags) 270 __releases(&bucket->lock) 271{ 272 spin_unlock_irqrestore(&bucket->lock, flags); 273} 274 275static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b) 276{ 277 return ((a->dev_addr == b->dev_addr) && 278 (a->dev == b->dev)) ? true : false; 279} 280 281static bool containing_match(struct dma_debug_entry *a, 282 struct dma_debug_entry *b) 283{ 284 if (a->dev != b->dev) 285 return false; 286 287 if ((b->dev_addr <= a->dev_addr) && 288 ((b->dev_addr + b->size) >= (a->dev_addr + a->size))) 289 return true; 290 291 return false; 292} 293 294/* 295 * Search a given entry in the hash bucket list 296 */ 297static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket, 298 struct dma_debug_entry *ref, 299 match_fn match) 300{ 301 struct dma_debug_entry *entry, *ret = NULL; 302 int matches = 0, match_lvl, last_lvl = -1; 303 304 list_for_each_entry(entry, &bucket->list, list) { 305 if (!match(ref, entry)) 306 continue; 307 308 /* 309 * Some drivers map the same physical address multiple 310 * times. Without a hardware IOMMU this results in the 311 * same device addresses being put into the dma-debug 312 * hash multiple times too. This can result in false 313 * positives being reported. Therefore we implement a 314 * best-fit algorithm here which returns the entry from 315 * the hash which fits best to the reference value 316 * instead of the first-fit. 317 */ 318 matches += 1; 319 match_lvl = 0; 320 entry->size == ref->size ? ++match_lvl : 0; 321 entry->type == ref->type ? ++match_lvl : 0; 322 entry->direction == ref->direction ? ++match_lvl : 0; 323 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; 324 325 if (match_lvl == 4) { 326 /* perfect-fit - return the result */ 327 return entry; 328 } else if (match_lvl > last_lvl) { 329 /* 330 * We found an entry that fits better then the 331 * previous one or it is the 1st match. 332 */ 333 last_lvl = match_lvl; 334 ret = entry; 335 } 336 } 337 338 /* 339 * If we have multiple matches but no perfect-fit, just return 340 * NULL. 341 */ 342 ret = (matches == 1) ? ret : NULL; 343 344 return ret; 345} 346 347static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket, 348 struct dma_debug_entry *ref) 349{ 350 return __hash_bucket_find(bucket, ref, exact_match); 351} 352 353static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket, 354 struct dma_debug_entry *ref, 355 unsigned long *flags) 356{ 357 358 struct dma_debug_entry *entry, index = *ref; 359 int limit = min(HASH_SIZE, (index.dev_addr >> HASH_FN_SHIFT) + 1); 360 361 for (int i = 0; i < limit; i++) { 362 entry = __hash_bucket_find(*bucket, ref, containing_match); 363 364 if (entry) 365 return entry; 366 367 /* 368 * Nothing found, go back a hash bucket 369 */ 370 put_hash_bucket(*bucket, *flags); 371 index.dev_addr -= (1 << HASH_FN_SHIFT); 372 *bucket = get_hash_bucket(&index, flags); 373 } 374 375 return NULL; 376} 377 378/* 379 * Add an entry to a hash bucket 380 */ 381static void hash_bucket_add(struct hash_bucket *bucket, 382 struct dma_debug_entry *entry) 383{ 384 list_add_tail(&entry->list, &bucket->list); 385} 386 387/* 388 * Remove entry from a hash bucket list 389 */ 390static void hash_bucket_del(struct dma_debug_entry *entry) 391{ 392 list_del(&entry->list); 393} 394 395/* 396 * For each mapping (initial cacheline in the case of 397 * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a 398 * scatterlist, or the cacheline specified in dma_map_single) insert 399 * into this tree using the cacheline as the key. At 400 * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If 401 * the entry already exists at insertion time add a tag as a reference 402 * count for the overlapping mappings. For now, the overlap tracking 403 * just ensures that 'unmaps' balance 'maps' before marking the 404 * cacheline idle, but we should also be flagging overlaps as an API 405 * violation. 406 * 407 * Memory usage is mostly constrained by the maximum number of available 408 * dma-debug entries in that we need a free dma_debug_entry before 409 * inserting into the tree. In the case of dma_map_page and 410 * dma_alloc_coherent there is only one dma_debug_entry and one 411 * dma_active_cacheline entry to track per event. dma_map_sg(), on the 412 * other hand, consumes a single dma_debug_entry, but inserts 'nents' 413 * entries into the tree. 414 * 415 * Use __GFP_NOWARN because the printk from an OOM, to netconsole, could end 416 * up right back in the DMA debugging code, leading to a deadlock. 417 */ 418static RADIX_TREE(dma_active_cacheline, GFP_ATOMIC | __GFP_NOWARN); 419static DEFINE_SPINLOCK(radix_lock); 420#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1) 421#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT) 422#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT) 423 424static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry) 425{ 426 return ((entry->paddr >> PAGE_SHIFT) << CACHELINE_PER_PAGE_SHIFT) + 427 (offset_in_page(entry->paddr) >> L1_CACHE_SHIFT); 428} 429 430static int active_cacheline_read_overlap(phys_addr_t cln) 431{ 432 int overlap = 0, i; 433 434 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--) 435 if (radix_tree_tag_get(&dma_active_cacheline, cln, i)) 436 overlap |= 1 << i; 437 return overlap; 438} 439 440static int active_cacheline_set_overlap(phys_addr_t cln, int overlap) 441{ 442 int i; 443 444 if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0) 445 return overlap; 446 447 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--) 448 if (overlap & 1 << i) 449 radix_tree_tag_set(&dma_active_cacheline, cln, i); 450 else 451 radix_tree_tag_clear(&dma_active_cacheline, cln, i); 452 453 return overlap; 454} 455 456static void active_cacheline_inc_overlap(phys_addr_t cln, bool is_cache_clean) 457{ 458 int overlap = active_cacheline_read_overlap(cln); 459 460 overlap = active_cacheline_set_overlap(cln, ++overlap); 461 462 /* If we overflowed the overlap counter then we're potentially 463 * leaking dma-mappings. 464 */ 465 WARN_ONCE(!is_cache_clean && overlap > ACTIVE_CACHELINE_MAX_OVERLAP, 466 pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"), 467 ACTIVE_CACHELINE_MAX_OVERLAP, &cln); 468} 469 470static int active_cacheline_dec_overlap(phys_addr_t cln) 471{ 472 int overlap = active_cacheline_read_overlap(cln); 473 474 return active_cacheline_set_overlap(cln, --overlap); 475} 476 477static int active_cacheline_insert(struct dma_debug_entry *entry, 478 bool *overlap_cache_clean) 479{ 480 phys_addr_t cln = to_cacheline_number(entry); 481 unsigned long flags; 482 int rc; 483 484 *overlap_cache_clean = false; 485 486 /* If the device is not writing memory then we don't have any 487 * concerns about the cpu consuming stale data. This mitigates 488 * legitimate usages of overlapping mappings. 489 */ 490 if (entry->direction == DMA_TO_DEVICE) 491 return 0; 492 493 spin_lock_irqsave(&radix_lock, flags); 494 rc = radix_tree_insert(&dma_active_cacheline, cln, entry); 495 if (rc == -EEXIST) { 496 struct dma_debug_entry *existing; 497 498 active_cacheline_inc_overlap(cln, entry->is_cache_clean); 499 existing = radix_tree_lookup(&dma_active_cacheline, cln); 500 /* A lookup failure here after we got -EEXIST is unexpected. */ 501 WARN_ON(!existing); 502 if (existing) 503 *overlap_cache_clean = existing->is_cache_clean; 504 } 505 spin_unlock_irqrestore(&radix_lock, flags); 506 507 return rc; 508} 509 510static void active_cacheline_remove(struct dma_debug_entry *entry) 511{ 512 phys_addr_t cln = to_cacheline_number(entry); 513 unsigned long flags; 514 515 /* ...mirror the insert case */ 516 if (entry->direction == DMA_TO_DEVICE) 517 return; 518 519 spin_lock_irqsave(&radix_lock, flags); 520 /* since we are counting overlaps the final put of the 521 * cacheline will occur when the overlap count is 0. 522 * active_cacheline_dec_overlap() returns -1 in that case 523 */ 524 if (active_cacheline_dec_overlap(cln) < 0) 525 radix_tree_delete(&dma_active_cacheline, cln); 526 spin_unlock_irqrestore(&radix_lock, flags); 527} 528 529/* 530 * Dump mappings entries on kernel space for debugging purposes 531 */ 532void debug_dma_dump_mappings(struct device *dev) 533{ 534 int idx; 535 phys_addr_t cln; 536 537 for (idx = 0; idx < HASH_SIZE; idx++) { 538 struct hash_bucket *bucket = &dma_entry_hash[idx]; 539 struct dma_debug_entry *entry; 540 unsigned long flags; 541 542 spin_lock_irqsave(&bucket->lock, flags); 543 list_for_each_entry(entry, &bucket->list, list) { 544 if (!dev || dev == entry->dev) { 545 cln = to_cacheline_number(entry); 546 dev_info(entry->dev, 547 "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n", 548 type2name[entry->type], idx, 549 &entry->paddr, entry->dev_addr, 550 entry->size, &cln, 551 dir2name[entry->direction], 552 maperr2str[entry->map_err_type]); 553 } 554 } 555 spin_unlock_irqrestore(&bucket->lock, flags); 556 557 cond_resched(); 558 } 559} 560 561/* 562 * Dump mappings entries on user space via debugfs 563 */ 564static int dump_show(struct seq_file *seq, void *v) 565{ 566 int idx; 567 phys_addr_t cln; 568 569 for (idx = 0; idx < HASH_SIZE; idx++) { 570 struct hash_bucket *bucket = &dma_entry_hash[idx]; 571 struct dma_debug_entry *entry; 572 unsigned long flags; 573 574 spin_lock_irqsave(&bucket->lock, flags); 575 list_for_each_entry(entry, &bucket->list, list) { 576 cln = to_cacheline_number(entry); 577 seq_printf(seq, 578 "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n", 579 dev_driver_string(entry->dev), 580 dev_name(entry->dev), 581 type2name[entry->type], idx, 582 &entry->paddr, entry->dev_addr, 583 entry->size, &cln, 584 dir2name[entry->direction], 585 maperr2str[entry->map_err_type]); 586 } 587 spin_unlock_irqrestore(&bucket->lock, flags); 588 } 589 return 0; 590} 591DEFINE_SHOW_ATTRIBUTE(dump); 592 593/* 594 * Wrapper function for adding an entry to the hash. 595 * This function takes care of locking itself. 596 */ 597static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) 598{ 599 bool overlap_cache_clean; 600 struct hash_bucket *bucket; 601 unsigned long flags; 602 int rc; 603 604 entry->is_cache_clean = !!(attrs & DMA_ATTR_CPU_CACHE_CLEAN); 605 606 bucket = get_hash_bucket(entry, &flags); 607 hash_bucket_add(bucket, entry); 608 put_hash_bucket(bucket, flags); 609 610 rc = active_cacheline_insert(entry, &overlap_cache_clean); 611 if (rc == -ENOMEM) { 612 pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n"); 613 global_disable = true; 614 } else if (rc == -EEXIST && 615 !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && 616 !(entry->is_cache_clean && overlap_cache_clean) && 617 dma_get_cache_alignment() >= L1_CACHE_BYTES && 618 !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && 619 is_swiotlb_active(entry->dev))) { 620 err_printk(entry->dev, entry, 621 "cacheline tracking EEXIST, overlapping mappings aren't supported\n"); 622 } 623} 624 625static int dma_debug_create_entries(gfp_t gfp) 626{ 627 struct dma_debug_entry *entry; 628 int i; 629 630 entry = (void *)get_zeroed_page(gfp); 631 if (!entry) 632 return -ENOMEM; 633 634 for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++) 635 list_add_tail(&entry[i].list, &free_entries); 636 637 num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES; 638 nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES; 639 640 return 0; 641} 642 643static struct dma_debug_entry *__dma_entry_alloc(void) 644{ 645 struct dma_debug_entry *entry; 646 647 entry = list_entry(free_entries.next, struct dma_debug_entry, list); 648 list_del(&entry->list); 649 memset(entry, 0, sizeof(*entry)); 650 651 num_free_entries -= 1; 652 if (num_free_entries < min_free_entries) 653 min_free_entries = num_free_entries; 654 655 return entry; 656} 657 658/* 659 * This should be called outside of free_entries_lock scope to avoid potential 660 * deadlocks with serial consoles that use DMA. 661 */ 662static void __dma_entry_alloc_check_leak(u32 nr_entries) 663{ 664 u32 tmp = nr_entries % nr_prealloc_entries; 665 666 /* Shout each time we tick over some multiple of the initial pool */ 667 if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) { 668 pr_info("dma_debug_entry pool grown to %u (%u00%%)\n", 669 nr_entries, 670 (nr_entries / nr_prealloc_entries)); 671 } 672} 673 674/* struct dma_entry allocator 675 * 676 * The next two functions implement the allocator for 677 * struct dma_debug_entries. 678 */ 679static struct dma_debug_entry *dma_entry_alloc(void) 680{ 681 bool alloc_check_leak = false; 682 struct dma_debug_entry *entry; 683 unsigned long flags; 684 u32 nr_entries; 685 686 spin_lock_irqsave(&free_entries_lock, flags); 687 if (num_free_entries == 0) { 688 if (dma_debug_create_entries(GFP_ATOMIC)) { 689 global_disable = true; 690 spin_unlock_irqrestore(&free_entries_lock, flags); 691 pr_err("debugging out of memory - disabling\n"); 692 return NULL; 693 } 694 alloc_check_leak = true; 695 nr_entries = nr_total_entries; 696 } 697 698 entry = __dma_entry_alloc(); 699 700 spin_unlock_irqrestore(&free_entries_lock, flags); 701 702 if (alloc_check_leak) 703 __dma_entry_alloc_check_leak(nr_entries); 704 705#ifdef CONFIG_STACKTRACE 706 entry->stack_len = stack_trace_save(entry->stack_entries, 707 ARRAY_SIZE(entry->stack_entries), 708 1); 709#endif 710 return entry; 711} 712 713static void dma_entry_free(struct dma_debug_entry *entry) 714{ 715 unsigned long flags; 716 717 active_cacheline_remove(entry); 718 719 /* 720 * add to beginning of the list - this way the entries are 721 * more likely cache hot when they are reallocated. 722 */ 723 spin_lock_irqsave(&free_entries_lock, flags); 724 list_add(&entry->list, &free_entries); 725 num_free_entries += 1; 726 spin_unlock_irqrestore(&free_entries_lock, flags); 727} 728 729/* 730 * DMA-API debugging init code 731 * 732 * The init code does two things: 733 * 1. Initialize core data structures 734 * 2. Preallocate a given number of dma_debug_entry structs 735 */ 736 737static ssize_t filter_read(struct file *file, char __user *user_buf, 738 size_t count, loff_t *ppos) 739{ 740 char buf[NAME_MAX_LEN + 1]; 741 unsigned long flags; 742 int len; 743 744 if (!current_driver_name[0]) 745 return 0; 746 747 /* 748 * We can't copy to userspace directly because current_driver_name can 749 * only be read under the driver_name_lock with irqs disabled. So 750 * create a temporary copy first. 751 */ 752 read_lock_irqsave(&driver_name_lock, flags); 753 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name); 754 read_unlock_irqrestore(&driver_name_lock, flags); 755 756 return simple_read_from_buffer(user_buf, count, ppos, buf, len); 757} 758 759static ssize_t filter_write(struct file *file, const char __user *userbuf, 760 size_t count, loff_t *ppos) 761{ 762 char buf[NAME_MAX_LEN]; 763 unsigned long flags; 764 size_t len; 765 int i; 766 767 /* 768 * We can't copy from userspace directly. Access to 769 * current_driver_name is protected with a write_lock with irqs 770 * disabled. Since copy_from_user can fault and may sleep we 771 * need to copy to temporary buffer first 772 */ 773 len = min(count, (size_t)(NAME_MAX_LEN - 1)); 774 if (copy_from_user(buf, userbuf, len)) 775 return -EFAULT; 776 777 buf[len] = 0; 778 779 write_lock_irqsave(&driver_name_lock, flags); 780 781 /* 782 * Now handle the string we got from userspace very carefully. 783 * The rules are: 784 * - only use the first token we got 785 * - token delimiter is everything looking like a space 786 * character (' ', '\n', '\t' ...) 787 * 788 */ 789 if (!isalnum(buf[0])) { 790 /* 791 * If the first character userspace gave us is not 792 * alphanumerical then assume the filter should be 793 * switched off. 794 */ 795 if (current_driver_name[0]) 796 pr_info("switching off dma-debug driver filter\n"); 797 current_driver_name[0] = 0; 798 current_driver = NULL; 799 goto out_unlock; 800 } 801 802 /* 803 * Now parse out the first token and use it as the name for the 804 * driver to filter for. 805 */ 806 for (i = 0; i < NAME_MAX_LEN - 1; ++i) { 807 current_driver_name[i] = buf[i]; 808 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0) 809 break; 810 } 811 current_driver_name[i] = 0; 812 current_driver = NULL; 813 814 pr_info("enable driver filter for driver [%s]\n", 815 current_driver_name); 816 817out_unlock: 818 write_unlock_irqrestore(&driver_name_lock, flags); 819 820 return count; 821} 822 823static const struct file_operations filter_fops = { 824 .read = filter_read, 825 .write = filter_write, 826 .llseek = default_llseek, 827}; 828 829static int __init dma_debug_fs_init(void) 830{ 831 struct dentry *dentry = debugfs_create_dir("dma-api", NULL); 832 833 debugfs_create_bool("disabled", 0444, dentry, &global_disable); 834 debugfs_create_u32("error_count", 0444, dentry, &error_count); 835 debugfs_create_u32("all_errors", 0644, dentry, &show_all_errors); 836 debugfs_create_u32("num_errors", 0644, dentry, &show_num_errors); 837 debugfs_create_u32("num_free_entries", 0444, dentry, &num_free_entries); 838 debugfs_create_u32("min_free_entries", 0444, dentry, &min_free_entries); 839 debugfs_create_u32("nr_total_entries", 0444, dentry, &nr_total_entries); 840 debugfs_create_file("driver_filter", 0644, dentry, NULL, &filter_fops); 841 debugfs_create_file("dump", 0444, dentry, NULL, &dump_fops); 842 843 return 0; 844} 845core_initcall_sync(dma_debug_fs_init); 846 847static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry) 848{ 849 struct dma_debug_entry *entry; 850 unsigned long flags; 851 int count = 0, i; 852 853 for (i = 0; i < HASH_SIZE; ++i) { 854 spin_lock_irqsave(&dma_entry_hash[i].lock, flags); 855 list_for_each_entry(entry, &dma_entry_hash[i].list, list) { 856 if (entry->dev == dev) { 857 count += 1; 858 *out_entry = entry; 859 } 860 } 861 spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags); 862 } 863 864 return count; 865} 866 867static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data) 868{ 869 struct device *dev = data; 870 struct dma_debug_entry *entry; 871 int count; 872 873 if (dma_debug_disabled()) 874 return 0; 875 876 switch (action) { 877 case BUS_NOTIFY_UNBOUND_DRIVER: 878 count = device_dma_allocations(dev, &entry); 879 if (count == 0) 880 break; 881 err_printk(dev, entry, "device driver has pending " 882 "DMA allocations while released from device " 883 "[count=%d]\n" 884 "One of leaked entries details: " 885 "[device address=0x%016llx] [size=%llu bytes] " 886 "[mapped with %s] [mapped as %s]\n", 887 count, entry->dev_addr, entry->size, 888 dir2name[entry->direction], type2name[entry->type]); 889 break; 890 default: 891 break; 892 } 893 894 return 0; 895} 896 897void dma_debug_add_bus(const struct bus_type *bus) 898{ 899 struct notifier_block *nb; 900 901 if (dma_debug_disabled()) 902 return; 903 904 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 905 if (nb == NULL) { 906 pr_err("dma_debug_add_bus: out of memory\n"); 907 return; 908 } 909 910 nb->notifier_call = dma_debug_device_change; 911 912 bus_register_notifier(bus, nb); 913} 914 915static int dma_debug_init(void) 916{ 917 int i, nr_pages; 918 919 /* Do not use dma_debug_initialized here, since we really want to be 920 * called to set dma_debug_initialized 921 */ 922 if (global_disable) 923 return 0; 924 925 for (i = 0; i < HASH_SIZE; ++i) { 926 INIT_LIST_HEAD(&dma_entry_hash[i].list); 927 spin_lock_init(&dma_entry_hash[i].lock); 928 } 929 930 nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES); 931 for (i = 0; i < nr_pages; ++i) 932 dma_debug_create_entries(GFP_KERNEL); 933 if (num_free_entries >= nr_prealloc_entries) { 934 pr_info("preallocated %d debug entries\n", nr_total_entries); 935 } else if (num_free_entries > 0) { 936 pr_warn("%d debug entries requested but only %d allocated\n", 937 nr_prealloc_entries, nr_total_entries); 938 } else { 939 pr_err("debugging out of memory error - disabled\n"); 940 global_disable = true; 941 942 return 0; 943 } 944 min_free_entries = num_free_entries; 945 946 dma_debug_initialized = true; 947 948 pr_info("debugging enabled by kernel config\n"); 949 return 0; 950} 951core_initcall(dma_debug_init); 952 953static __init int dma_debug_cmdline(char *str) 954{ 955 if (!str) 956 return -EINVAL; 957 958 if (strncmp(str, "off", 3) == 0) { 959 pr_info("debugging disabled on kernel command line\n"); 960 global_disable = true; 961 } 962 963 return 1; 964} 965 966static __init int dma_debug_entries_cmdline(char *str) 967{ 968 if (!str) 969 return -EINVAL; 970 if (!get_option(&str, &nr_prealloc_entries)) 971 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES; 972 return 1; 973} 974 975__setup("dma_debug=", dma_debug_cmdline); 976__setup("dma_debug_entries=", dma_debug_entries_cmdline); 977 978static void check_unmap(struct dma_debug_entry *ref) 979{ 980 struct dma_debug_entry *entry; 981 struct hash_bucket *bucket; 982 unsigned long flags; 983 984 bucket = get_hash_bucket(ref, &flags); 985 entry = bucket_find_exact(bucket, ref); 986 987 if (!entry) { 988 /* must drop lock before calling dma_mapping_error */ 989 put_hash_bucket(bucket, flags); 990 991 if (dma_mapping_error(ref->dev, ref->dev_addr)) { 992 err_printk(ref->dev, NULL, 993 "device driver tries to free an " 994 "invalid DMA memory address\n"); 995 } else { 996 err_printk(ref->dev, NULL, 997 "device driver tries to free DMA " 998 "memory it has not allocated [device " 999 "address=0x%016llx] [size=%llu bytes]\n", 1000 ref->dev_addr, ref->size); 1001 } 1002 return; 1003 } 1004 1005 if (ref->size != entry->size) { 1006 err_printk(ref->dev, entry, "device driver frees " 1007 "DMA memory with different size " 1008 "[device address=0x%016llx] [map size=%llu bytes] " 1009 "[unmap size=%llu bytes]\n", 1010 ref->dev_addr, entry->size, ref->size); 1011 } 1012 1013 if (ref->type != entry->type) { 1014 err_printk(ref->dev, entry, "device driver frees " 1015 "DMA memory with wrong function " 1016 "[device address=0x%016llx] [size=%llu bytes] " 1017 "[mapped as %s] [unmapped as %s]\n", 1018 ref->dev_addr, ref->size, 1019 type2name[entry->type], type2name[ref->type]); 1020 } else if ((entry->type == dma_debug_coherent || 1021 entry->type == dma_debug_noncoherent) && 1022 ref->paddr != entry->paddr) { 1023 err_printk(ref->dev, entry, "device driver frees " 1024 "DMA memory with different CPU address " 1025 "[device address=0x%016llx] [size=%llu bytes] " 1026 "[cpu alloc address=0x%pa] " 1027 "[cpu free address=0x%pa]", 1028 ref->dev_addr, ref->size, 1029 &entry->paddr, 1030 &ref->paddr); 1031 } 1032 1033 if (ref->sg_call_ents && ref->type == dma_debug_sg && 1034 ref->sg_call_ents != entry->sg_call_ents) { 1035 err_printk(ref->dev, entry, "device driver frees " 1036 "DMA sg list with different entry count " 1037 "[map count=%d] [unmap count=%d]\n", 1038 entry->sg_call_ents, ref->sg_call_ents); 1039 } 1040 1041 /* 1042 * This may be no bug in reality - but most implementations of the 1043 * DMA API don't handle this properly, so check for it here 1044 */ 1045 if (ref->direction != entry->direction) { 1046 err_printk(ref->dev, entry, "device driver frees " 1047 "DMA memory with different direction " 1048 "[device address=0x%016llx] [size=%llu bytes] " 1049 "[mapped with %s] [unmapped with %s]\n", 1050 ref->dev_addr, ref->size, 1051 dir2name[entry->direction], 1052 dir2name[ref->direction]); 1053 } 1054 1055 /* 1056 * Drivers should use dma_mapping_error() to check the returned 1057 * addresses of dma_map_single() and dma_map_page(). 1058 * If not, print this warning message. See Documentation/core-api/dma-api.rst. 1059 */ 1060 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { 1061 err_printk(ref->dev, entry, 1062 "device driver failed to check map error" 1063 "[device address=0x%016llx] [size=%llu bytes] " 1064 "[mapped as %s]", 1065 ref->dev_addr, ref->size, 1066 type2name[entry->type]); 1067 } 1068 1069 hash_bucket_del(entry); 1070 put_hash_bucket(bucket, flags); 1071 1072 /* 1073 * Free the entry outside of bucket_lock to avoid ABBA deadlocks 1074 * between that and radix_lock. 1075 */ 1076 dma_entry_free(entry); 1077} 1078 1079static void check_for_stack(struct device *dev, phys_addr_t phys) 1080{ 1081 void *addr; 1082 struct vm_struct *stack_vm_area = task_stack_vm_area(current); 1083 1084 if (!stack_vm_area) { 1085 /* Stack is direct-mapped. */ 1086 if (PhysHighMem(phys)) 1087 return; 1088 addr = phys_to_virt(phys); 1089 if (object_is_on_stack(addr)) 1090 err_printk(dev, NULL, "device driver maps memory from stack [addr=%p]\n", addr); 1091 } else { 1092 /* Stack is vmalloced. */ 1093 int i; 1094 1095 for (i = 0; i < stack_vm_area->nr_pages; i++) { 1096 if (__phys_to_pfn(phys) != 1097 page_to_pfn(stack_vm_area->pages[i])) 1098 continue; 1099 1100 addr = (u8 *)current->stack + i * PAGE_SIZE + 1101 (phys % PAGE_SIZE); 1102 err_printk(dev, NULL, "device driver maps memory from stack [probable addr=%p]\n", addr); 1103 break; 1104 } 1105 } 1106} 1107 1108static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len) 1109{ 1110 if (memory_intersects(_stext, _etext, addr, len) || 1111 memory_intersects(__start_rodata, __end_rodata, addr, len)) 1112 err_printk(dev, NULL, "device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len); 1113} 1114 1115static void check_sync(struct device *dev, 1116 struct dma_debug_entry *ref, 1117 bool to_cpu) 1118{ 1119 struct dma_debug_entry *entry; 1120 struct hash_bucket *bucket; 1121 unsigned long flags; 1122 1123 bucket = get_hash_bucket(ref, &flags); 1124 1125 entry = bucket_find_contain(&bucket, ref, &flags); 1126 1127 if (!entry) { 1128 err_printk(dev, NULL, "device driver tries " 1129 "to sync DMA memory it has not allocated " 1130 "[device address=0x%016llx] [size=%llu bytes]\n", 1131 (unsigned long long)ref->dev_addr, ref->size); 1132 goto out; 1133 } 1134 1135 if (ref->size > entry->size) { 1136 err_printk(dev, entry, "device driver syncs" 1137 " DMA memory outside allocated range " 1138 "[device address=0x%016llx] " 1139 "[allocation size=%llu bytes] " 1140 "[sync offset+size=%llu]\n", 1141 entry->dev_addr, entry->size, 1142 ref->size); 1143 } 1144 1145 if (entry->direction == DMA_BIDIRECTIONAL) 1146 goto out; 1147 1148 if (ref->direction != entry->direction) { 1149 err_printk(dev, entry, "device driver syncs " 1150 "DMA memory with different direction " 1151 "[device address=0x%016llx] [size=%llu bytes] " 1152 "[mapped with %s] [synced with %s]\n", 1153 (unsigned long long)ref->dev_addr, entry->size, 1154 dir2name[entry->direction], 1155 dir2name[ref->direction]); 1156 } 1157 1158 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && 1159 !(ref->direction == DMA_TO_DEVICE)) 1160 err_printk(dev, entry, "device driver syncs " 1161 "device read-only DMA memory for cpu " 1162 "[device address=0x%016llx] [size=%llu bytes] " 1163 "[mapped with %s] [synced with %s]\n", 1164 (unsigned long long)ref->dev_addr, entry->size, 1165 dir2name[entry->direction], 1166 dir2name[ref->direction]); 1167 1168 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && 1169 !(ref->direction == DMA_FROM_DEVICE)) 1170 err_printk(dev, entry, "device driver syncs " 1171 "device write-only DMA memory to device " 1172 "[device address=0x%016llx] [size=%llu bytes] " 1173 "[mapped with %s] [synced with %s]\n", 1174 (unsigned long long)ref->dev_addr, entry->size, 1175 dir2name[entry->direction], 1176 dir2name[ref->direction]); 1177 1178 if (ref->sg_call_ents && ref->type == dma_debug_sg && 1179 ref->sg_call_ents != entry->sg_call_ents) { 1180 err_printk(ref->dev, entry, "device driver syncs " 1181 "DMA sg list with different entry count " 1182 "[map count=%d] [sync count=%d]\n", 1183 entry->sg_call_ents, ref->sg_call_ents); 1184 } 1185 1186out: 1187 put_hash_bucket(bucket, flags); 1188} 1189 1190static void check_sg_segment(struct device *dev, struct scatterlist *sg) 1191{ 1192 unsigned int max_seg = dma_get_max_seg_size(dev); 1193 u64 start, end, boundary = dma_get_seg_boundary(dev); 1194 1195 /* 1196 * Either the driver forgot to set dma_parms appropriately, or 1197 * whoever generated the list forgot to check them. 1198 */ 1199 if (sg->length > max_seg) 1200 err_printk(dev, NULL, "mapping sg segment longer than device claims to support [len=%u] [max=%u]\n", 1201 sg->length, max_seg); 1202 /* 1203 * In some cases this could potentially be the DMA API 1204 * implementation's fault, but it would usually imply that 1205 * the scatterlist was built inappropriately to begin with. 1206 */ 1207 start = sg_dma_address(sg); 1208 end = start + sg_dma_len(sg) - 1; 1209 if ((start ^ end) & ~boundary) 1210 err_printk(dev, NULL, "mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n", 1211 start, end, boundary); 1212} 1213 1214void debug_dma_map_single(struct device *dev, const void *addr, 1215 unsigned long len) 1216{ 1217 if (unlikely(dma_debug_disabled())) 1218 return; 1219 1220 if (!virt_addr_valid(addr)) 1221 err_printk(dev, NULL, "device driver maps memory from invalid area [addr=%p] [len=%lu]\n", 1222 addr, len); 1223 1224 if (is_vmalloc_addr(addr)) 1225 err_printk(dev, NULL, "device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n", 1226 addr, len); 1227} 1228EXPORT_SYMBOL(debug_dma_map_single); 1229 1230void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 1231 int direction, dma_addr_t dma_addr, unsigned long attrs) 1232{ 1233 struct dma_debug_entry *entry; 1234 1235 if (unlikely(dma_debug_disabled())) 1236 return; 1237 1238 if (dma_mapping_error(dev, dma_addr)) 1239 return; 1240 1241 entry = dma_entry_alloc(); 1242 if (!entry) 1243 return; 1244 1245 entry->dev = dev; 1246 entry->type = dma_debug_phy; 1247 entry->paddr = phys; 1248 entry->dev_addr = dma_addr; 1249 entry->size = size; 1250 entry->direction = direction; 1251 entry->map_err_type = MAP_ERR_NOT_CHECKED; 1252 1253 if (!(attrs & DMA_ATTR_MMIO)) { 1254 check_for_stack(dev, phys); 1255 1256 if (!PhysHighMem(phys)) 1257 check_for_illegal_area(dev, phys_to_virt(phys), size); 1258 } 1259 1260 add_dma_entry(entry, attrs); 1261} 1262 1263void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 1264{ 1265 struct dma_debug_entry ref; 1266 struct dma_debug_entry *entry; 1267 struct hash_bucket *bucket; 1268 unsigned long flags; 1269 1270 if (unlikely(dma_debug_disabled())) 1271 return; 1272 1273 ref.dev = dev; 1274 ref.dev_addr = dma_addr; 1275 bucket = get_hash_bucket(&ref, &flags); 1276 1277 list_for_each_entry(entry, &bucket->list, list) { 1278 if (!exact_match(&ref, entry)) 1279 continue; 1280 1281 /* 1282 * The same physical address can be mapped multiple 1283 * times. Without a hardware IOMMU this results in the 1284 * same device addresses being put into the dma-debug 1285 * hash multiple times too. This can result in false 1286 * positives being reported. Therefore we implement a 1287 * best-fit algorithm here which updates the first entry 1288 * from the hash which fits the reference value and is 1289 * not currently listed as being checked. 1290 */ 1291 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { 1292 entry->map_err_type = MAP_ERR_CHECKED; 1293 break; 1294 } 1295 } 1296 1297 put_hash_bucket(bucket, flags); 1298} 1299EXPORT_SYMBOL(debug_dma_mapping_error); 1300 1301void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, 1302 size_t size, int direction) 1303{ 1304 struct dma_debug_entry ref = { 1305 .type = dma_debug_phy, 1306 .dev = dev, 1307 .dev_addr = dma_addr, 1308 .size = size, 1309 .direction = direction, 1310 }; 1311 1312 if (unlikely(dma_debug_disabled())) 1313 return; 1314 check_unmap(&ref); 1315} 1316 1317void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, 1318 int nents, int mapped_ents, int direction, 1319 unsigned long attrs) 1320{ 1321 struct dma_debug_entry *entry; 1322 struct scatterlist *s; 1323 int i; 1324 1325 if (unlikely(dma_debug_disabled())) 1326 return; 1327 1328 for_each_sg(sg, s, nents, i) { 1329 check_for_stack(dev, sg_phys(s)); 1330 if (!PageHighMem(sg_page(s))) 1331 check_for_illegal_area(dev, sg_virt(s), s->length); 1332 } 1333 1334 for_each_sg(sg, s, mapped_ents, i) { 1335 entry = dma_entry_alloc(); 1336 if (!entry) 1337 return; 1338 1339 entry->type = dma_debug_sg; 1340 entry->dev = dev; 1341 entry->paddr = sg_phys(s); 1342 entry->size = sg_dma_len(s); 1343 entry->dev_addr = sg_dma_address(s); 1344 entry->direction = direction; 1345 entry->sg_call_ents = nents; 1346 entry->sg_mapped_ents = mapped_ents; 1347 1348 check_sg_segment(dev, s); 1349 1350 add_dma_entry(entry, attrs); 1351 } 1352} 1353 1354static int get_nr_mapped_entries(struct device *dev, 1355 struct dma_debug_entry *ref) 1356{ 1357 struct dma_debug_entry *entry; 1358 struct hash_bucket *bucket; 1359 unsigned long flags; 1360 int mapped_ents; 1361 1362 bucket = get_hash_bucket(ref, &flags); 1363 entry = bucket_find_exact(bucket, ref); 1364 mapped_ents = 0; 1365 1366 if (entry) 1367 mapped_ents = entry->sg_mapped_ents; 1368 put_hash_bucket(bucket, flags); 1369 1370 return mapped_ents; 1371} 1372 1373void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, 1374 int nelems, int dir) 1375{ 1376 struct scatterlist *s; 1377 int mapped_ents = 0, i; 1378 1379 if (unlikely(dma_debug_disabled())) 1380 return; 1381 1382 for_each_sg(sglist, s, nelems, i) { 1383 1384 struct dma_debug_entry ref = { 1385 .type = dma_debug_sg, 1386 .dev = dev, 1387 .paddr = sg_phys(s), 1388 .dev_addr = sg_dma_address(s), 1389 .size = sg_dma_len(s), 1390 .direction = dir, 1391 .sg_call_ents = nelems, 1392 }; 1393 1394 if (mapped_ents && i >= mapped_ents) 1395 break; 1396 1397 if (!i) 1398 mapped_ents = get_nr_mapped_entries(dev, &ref); 1399 1400 check_unmap(&ref); 1401 } 1402} 1403 1404static phys_addr_t virt_to_paddr(void *virt) 1405{ 1406 struct page *page; 1407 1408 if (is_vmalloc_addr(virt)) 1409 page = vmalloc_to_page(virt); 1410 else 1411 page = virt_to_page(virt); 1412 1413 return page_to_phys(page) + offset_in_page(virt); 1414} 1415 1416void debug_dma_alloc_coherent(struct device *dev, size_t size, 1417 dma_addr_t dma_addr, void *virt, 1418 unsigned long attrs) 1419{ 1420 struct dma_debug_entry *entry; 1421 1422 if (unlikely(dma_debug_disabled())) 1423 return; 1424 1425 if (unlikely(virt == NULL)) 1426 return; 1427 1428 /* handle vmalloc and linear addresses */ 1429 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt)) 1430 return; 1431 1432 entry = dma_entry_alloc(); 1433 if (!entry) 1434 return; 1435 1436 entry->type = dma_debug_coherent; 1437 entry->dev = dev; 1438 entry->paddr = virt_to_paddr(virt); 1439 entry->size = size; 1440 entry->dev_addr = dma_addr; 1441 entry->direction = DMA_BIDIRECTIONAL; 1442 1443 add_dma_entry(entry, attrs); 1444} 1445 1446void debug_dma_free_coherent(struct device *dev, size_t size, 1447 void *virt, dma_addr_t dma_addr) 1448{ 1449 struct dma_debug_entry ref = { 1450 .type = dma_debug_coherent, 1451 .dev = dev, 1452 .dev_addr = dma_addr, 1453 .size = size, 1454 .direction = DMA_BIDIRECTIONAL, 1455 }; 1456 1457 /* handle vmalloc and linear addresses */ 1458 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt)) 1459 return; 1460 1461 ref.paddr = virt_to_paddr(virt); 1462 1463 if (unlikely(dma_debug_disabled())) 1464 return; 1465 1466 check_unmap(&ref); 1467} 1468 1469void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, 1470 size_t size, int direction) 1471{ 1472 struct dma_debug_entry ref; 1473 1474 if (unlikely(dma_debug_disabled())) 1475 return; 1476 1477 ref.type = dma_debug_single; 1478 ref.dev = dev; 1479 ref.dev_addr = dma_handle; 1480 ref.size = size; 1481 ref.direction = direction; 1482 ref.sg_call_ents = 0; 1483 1484 check_sync(dev, &ref, true); 1485} 1486 1487void debug_dma_sync_single_for_device(struct device *dev, 1488 dma_addr_t dma_handle, size_t size, 1489 int direction) 1490{ 1491 struct dma_debug_entry ref; 1492 1493 if (unlikely(dma_debug_disabled())) 1494 return; 1495 1496 ref.type = dma_debug_single; 1497 ref.dev = dev; 1498 ref.dev_addr = dma_handle; 1499 ref.size = size; 1500 ref.direction = direction; 1501 ref.sg_call_ents = 0; 1502 1503 check_sync(dev, &ref, false); 1504} 1505 1506void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1507 int nelems, int direction) 1508{ 1509 struct scatterlist *s; 1510 int mapped_ents = 0, i; 1511 1512 if (unlikely(dma_debug_disabled())) 1513 return; 1514 1515 for_each_sg(sg, s, nelems, i) { 1516 1517 struct dma_debug_entry ref = { 1518 .type = dma_debug_sg, 1519 .dev = dev, 1520 .paddr = sg_phys(s), 1521 .dev_addr = sg_dma_address(s), 1522 .size = sg_dma_len(s), 1523 .direction = direction, 1524 .sg_call_ents = nelems, 1525 }; 1526 1527 if (!i) 1528 mapped_ents = get_nr_mapped_entries(dev, &ref); 1529 1530 if (i >= mapped_ents) 1531 break; 1532 1533 check_sync(dev, &ref, true); 1534 } 1535} 1536 1537void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1538 int nelems, int direction) 1539{ 1540 struct scatterlist *s; 1541 int mapped_ents = 0, i; 1542 1543 if (unlikely(dma_debug_disabled())) 1544 return; 1545 1546 for_each_sg(sg, s, nelems, i) { 1547 1548 struct dma_debug_entry ref = { 1549 .type = dma_debug_sg, 1550 .dev = dev, 1551 .paddr = sg_phys(sg), 1552 .dev_addr = sg_dma_address(s), 1553 .size = sg_dma_len(s), 1554 .direction = direction, 1555 .sg_call_ents = nelems, 1556 }; 1557 if (!i) 1558 mapped_ents = get_nr_mapped_entries(dev, &ref); 1559 1560 if (i >= mapped_ents) 1561 break; 1562 1563 check_sync(dev, &ref, false); 1564 } 1565} 1566 1567void debug_dma_alloc_pages(struct device *dev, struct page *page, 1568 size_t size, int direction, 1569 dma_addr_t dma_addr, 1570 unsigned long attrs) 1571{ 1572 struct dma_debug_entry *entry; 1573 1574 if (unlikely(dma_debug_disabled())) 1575 return; 1576 1577 entry = dma_entry_alloc(); 1578 if (!entry) 1579 return; 1580 1581 entry->type = dma_debug_noncoherent; 1582 entry->dev = dev; 1583 entry->paddr = page_to_phys(page); 1584 entry->size = size; 1585 entry->dev_addr = dma_addr; 1586 entry->direction = direction; 1587 1588 add_dma_entry(entry, attrs); 1589} 1590 1591void debug_dma_free_pages(struct device *dev, struct page *page, 1592 size_t size, int direction, 1593 dma_addr_t dma_addr) 1594{ 1595 struct dma_debug_entry ref = { 1596 .type = dma_debug_noncoherent, 1597 .dev = dev, 1598 .paddr = page_to_phys(page), 1599 .dev_addr = dma_addr, 1600 .size = size, 1601 .direction = direction, 1602 }; 1603 1604 if (unlikely(dma_debug_disabled())) 1605 return; 1606 1607 check_unmap(&ref); 1608} 1609 1610static int __init dma_debug_driver_setup(char *str) 1611{ 1612 int i; 1613 1614 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) { 1615 current_driver_name[i] = *str; 1616 if (*str == 0) 1617 break; 1618 } 1619 1620 if (current_driver_name[0]) 1621 pr_info("enable driver filter for driver [%s]\n", 1622 current_driver_name); 1623 1624 1625 return 1; 1626} 1627__setup("dma_debug_driver=", dma_debug_driver_setup);
~/
Read-only file preview.