FILE
iommu.c
File preview with syntax highlighting.
linux-6.19.14/drivers/iommu/amd/iommu.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. 4 * Author: Joerg Roedel <jroedel@suse.de> 5 * Leo Duran <leo.duran@amd.com> 6 */ 7 8#define pr_fmt(fmt) "AMD-Vi: " fmt 9#define dev_fmt(fmt) pr_fmt(fmt) 10 11#include <linux/ratelimit.h> 12#include <linux/pci.h> 13#include <linux/acpi.h> 14#include <linux/pci-ats.h> 15#include <linux/bitmap.h> 16#include <linux/slab.h> 17#include <linux/string_choices.h> 18#include <linux/debugfs.h> 19#include <linux/scatterlist.h> 20#include <linux/dma-map-ops.h> 21#include <linux/dma-direct.h> 22#include <linux/idr.h> 23#include <linux/iommu-helper.h> 24#include <linux/delay.h> 25#include <linux/amd-iommu.h> 26#include <linux/notifier.h> 27#include <linux/export.h> 28#include <linux/irq.h> 29#include <linux/irqchip/irq-msi-lib.h> 30#include <linux/msi.h> 31#include <linux/irqdomain.h> 32#include <linux/percpu.h> 33#include <linux/cc_platform.h> 34#include <asm/irq_remapping.h> 35#include <asm/io_apic.h> 36#include <asm/apic.h> 37#include <asm/hw_irq.h> 38#include <asm/proto.h> 39#include <asm/iommu.h> 40#include <asm/gart.h> 41#include <asm/dma.h> 42#include <uapi/linux/iommufd.h> 43#include <linux/generic_pt/iommu.h> 44 45#include "amd_iommu.h" 46#include "../irq_remapping.h" 47#include "../iommu-pages.h" 48 49#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) 50 51/* Reserved IOVA ranges */ 52#define MSI_RANGE_START (0xfee00000) 53#define MSI_RANGE_END (0xfeefffff) 54#define HT_RANGE_START (0xfd00000000ULL) 55#define HT_RANGE_END (0xffffffffffULL) 56 57LIST_HEAD(ioapic_map); 58LIST_HEAD(hpet_map); 59LIST_HEAD(acpihid_map); 60 61const struct iommu_ops amd_iommu_ops; 62 63int amd_iommu_max_glx_val = -1; 64 65/* 66 * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap 67 * to know which ones are already in use. 68 */ 69DEFINE_IDA(pdom_ids); 70 71static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, 72 struct iommu_domain *old); 73 74static void set_dte_entry(struct amd_iommu *iommu, 75 struct iommu_dev_data *dev_data, 76 phys_addr_t top_paddr, unsigned int top_level); 77 78static void amd_iommu_change_top(struct pt_iommu *iommu_table, 79 phys_addr_t top_paddr, unsigned int top_level); 80 81static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid); 82 83static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid); 84static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain); 85static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, 86 bool enable); 87 88/**************************************************************************** 89 * 90 * Helper functions 91 * 92 ****************************************************************************/ 93 94static __always_inline void amd_iommu_atomic128_set(__int128 *ptr, __int128 val) 95{ 96 /* 97 * Note: 98 * We use arch_cmpxchg128_local() because: 99 * - Need cmpxchg16b instruction mainly for 128-bit store to DTE 100 * (not necessary for cmpxchg since this function is already 101 * protected by a spin_lock for this DTE). 102 * - Neither need LOCK_PREFIX nor try loop because of the spin_lock. 103 */ 104 arch_cmpxchg128_local(ptr, *ptr, val); 105} 106 107static void write_dte_upper128(struct dev_table_entry *ptr, struct dev_table_entry *new) 108{ 109 struct dev_table_entry old; 110 111 old.data128[1] = ptr->data128[1]; 112 /* 113 * Preserve DTE_DATA2_INTR_MASK. This needs to be 114 * done here since it requires to be inside 115 * spin_lock(&dev_data->dte_lock) context. 116 */ 117 new->data[2] &= ~DTE_DATA2_INTR_MASK; 118 new->data[2] |= old.data[2] & DTE_DATA2_INTR_MASK; 119 120 amd_iommu_atomic128_set(&ptr->data128[1], new->data128[1]); 121} 122 123static void write_dte_lower128(struct dev_table_entry *ptr, struct dev_table_entry *new) 124{ 125 amd_iommu_atomic128_set(&ptr->data128[0], new->data128[0]); 126} 127 128/* 129 * Note: 130 * IOMMU reads the entire Device Table entry in a single 256-bit transaction 131 * but the driver is programming DTE using 2 128-bit cmpxchg. So, the driver 132 * need to ensure the following: 133 * - DTE[V|GV] bit is being written last when setting. 134 * - DTE[V|GV] bit is being written first when clearing. 135 * 136 * This function is used only by code, which updates DMA translation part of the DTE. 137 * So, only consider control bits related to DMA when updating the entry. 138 */ 139static void update_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data, 140 struct dev_table_entry *new) 141{ 142 unsigned long flags; 143 struct dev_table_entry *dev_table = get_dev_table(iommu); 144 struct dev_table_entry *ptr = &dev_table[dev_data->devid]; 145 146 spin_lock_irqsave(&dev_data->dte_lock, flags); 147 148 if (!(ptr->data[0] & DTE_FLAG_V)) { 149 /* Existing DTE is not valid. */ 150 write_dte_upper128(ptr, new); 151 write_dte_lower128(ptr, new); 152 iommu_flush_dte_sync(iommu, dev_data->devid); 153 } else if (!(new->data[0] & DTE_FLAG_V)) { 154 /* Existing DTE is valid. New DTE is not valid. */ 155 write_dte_lower128(ptr, new); 156 write_dte_upper128(ptr, new); 157 iommu_flush_dte_sync(iommu, dev_data->devid); 158 } else if (!FIELD_GET(DTE_FLAG_GV, ptr->data[0])) { 159 /* 160 * Both DTEs are valid. 161 * Existing DTE has no guest page table. 162 */ 163 write_dte_upper128(ptr, new); 164 write_dte_lower128(ptr, new); 165 iommu_flush_dte_sync(iommu, dev_data->devid); 166 } else if (!FIELD_GET(DTE_FLAG_GV, new->data[0])) { 167 /* 168 * Both DTEs are valid. 169 * Existing DTE has guest page table, 170 * new DTE has no guest page table, 171 */ 172 write_dte_lower128(ptr, new); 173 write_dte_upper128(ptr, new); 174 iommu_flush_dte_sync(iommu, dev_data->devid); 175 } else if (FIELD_GET(DTE_GPT_LEVEL_MASK, ptr->data[2]) != 176 FIELD_GET(DTE_GPT_LEVEL_MASK, new->data[2])) { 177 /* 178 * Both DTEs are valid and have guest page table, 179 * but have different number of levels. So, we need 180 * to upadte both upper and lower 128-bit value, which 181 * require disabling and flushing. 182 */ 183 struct dev_table_entry clear = {}; 184 185 /* First disable DTE */ 186 write_dte_lower128(ptr, &clear); 187 iommu_flush_dte_sync(iommu, dev_data->devid); 188 189 /* Then update DTE */ 190 write_dte_upper128(ptr, new); 191 write_dte_lower128(ptr, new); 192 iommu_flush_dte_sync(iommu, dev_data->devid); 193 } else { 194 /* 195 * Both DTEs are valid and have guest page table, 196 * and same number of levels. We just need to only 197 * update the lower 128-bit. So no need to disable DTE. 198 */ 199 write_dte_lower128(ptr, new); 200 } 201 202 spin_unlock_irqrestore(&dev_data->dte_lock, flags); 203} 204 205static void get_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data, 206 struct dev_table_entry *dte) 207{ 208 unsigned long flags; 209 struct dev_table_entry *ptr; 210 struct dev_table_entry *dev_table = get_dev_table(iommu); 211 212 ptr = &dev_table[dev_data->devid]; 213 214 spin_lock_irqsave(&dev_data->dte_lock, flags); 215 dte->data128[0] = ptr->data128[0]; 216 dte->data128[1] = ptr->data128[1]; 217 spin_unlock_irqrestore(&dev_data->dte_lock, flags); 218} 219 220static inline bool pdom_is_v2_pgtbl_mode(struct protection_domain *pdom) 221{ 222 return (pdom && (pdom->pd_mode == PD_MODE_V2)); 223} 224 225static inline bool pdom_is_in_pt_mode(struct protection_domain *pdom) 226{ 227 return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY); 228} 229 230/* 231 * We cannot support PASID w/ existing v1 page table in the same domain 232 * since it will be nested. However, existing domain w/ v2 page table 233 * or passthrough mode can be used for PASID. 234 */ 235static inline bool pdom_is_sva_capable(struct protection_domain *pdom) 236{ 237 return pdom_is_v2_pgtbl_mode(pdom) || pdom_is_in_pt_mode(pdom); 238} 239 240static inline int get_acpihid_device_id(struct device *dev, 241 struct acpihid_map_entry **entry) 242{ 243 struct acpi_device *adev = ACPI_COMPANION(dev); 244 struct acpihid_map_entry *p, *p1 = NULL; 245 int hid_count = 0; 246 bool fw_bug; 247 248 if (!adev) 249 return -ENODEV; 250 251 list_for_each_entry(p, &acpihid_map, list) { 252 if (acpi_dev_hid_uid_match(adev, p->hid, 253 p->uid[0] ? p->uid : NULL)) { 254 p1 = p; 255 fw_bug = false; 256 hid_count = 1; 257 break; 258 } 259 260 /* 261 * Count HID matches w/o UID, raise FW_BUG but allow exactly one match 262 */ 263 if (acpi_dev_hid_match(adev, p->hid)) { 264 p1 = p; 265 hid_count++; 266 fw_bug = true; 267 } 268 } 269 270 if (!p1) 271 return -EINVAL; 272 if (fw_bug) 273 dev_err_once(dev, FW_BUG "No ACPI device matched UID, but %d device%s matched HID.\n", 274 hid_count, str_plural(hid_count)); 275 if (hid_count > 1) 276 return -EINVAL; 277 if (entry) 278 *entry = p1; 279 280 return p1->devid; 281} 282 283static inline int get_device_sbdf_id(struct device *dev) 284{ 285 int sbdf; 286 287 if (dev_is_pci(dev)) 288 sbdf = get_pci_sbdf_id(to_pci_dev(dev)); 289 else 290 sbdf = get_acpihid_device_id(dev, NULL); 291 292 return sbdf; 293} 294 295struct dev_table_entry *get_dev_table(struct amd_iommu *iommu) 296{ 297 struct dev_table_entry *dev_table; 298 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 299 300 BUG_ON(pci_seg == NULL); 301 dev_table = pci_seg->dev_table; 302 BUG_ON(dev_table == NULL); 303 304 return dev_table; 305} 306 307static inline u16 get_device_segment(struct device *dev) 308{ 309 u16 seg; 310 311 if (dev_is_pci(dev)) { 312 struct pci_dev *pdev = to_pci_dev(dev); 313 314 seg = pci_domain_nr(pdev->bus); 315 } else { 316 u32 devid = get_acpihid_device_id(dev, NULL); 317 318 seg = PCI_SBDF_TO_SEGID(devid); 319 } 320 321 return seg; 322} 323 324/* Writes the specific IOMMU for a device into the PCI segment rlookup table */ 325void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid) 326{ 327 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 328 329 pci_seg->rlookup_table[devid] = iommu; 330} 331 332static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid) 333{ 334 struct amd_iommu_pci_seg *pci_seg; 335 336 for_each_pci_segment(pci_seg) { 337 if (pci_seg->id == seg) 338 return pci_seg->rlookup_table[devid]; 339 } 340 return NULL; 341} 342 343static struct amd_iommu *rlookup_amd_iommu(struct device *dev) 344{ 345 u16 seg = get_device_segment(dev); 346 int devid = get_device_sbdf_id(dev); 347 348 if (devid < 0) 349 return NULL; 350 return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid)); 351} 352 353static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid) 354{ 355 struct iommu_dev_data *dev_data; 356 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 357 358 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); 359 if (!dev_data) 360 return NULL; 361 362 mutex_init(&dev_data->mutex); 363 spin_lock_init(&dev_data->dte_lock); 364 dev_data->devid = devid; 365 ratelimit_default_init(&dev_data->rs); 366 367 llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list); 368 return dev_data; 369} 370 371struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid) 372{ 373 struct iommu_dev_data *dev_data; 374 struct llist_node *node; 375 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 376 377 if (llist_empty(&pci_seg->dev_data_list)) 378 return NULL; 379 380 node = pci_seg->dev_data_list.first; 381 llist_for_each_entry(dev_data, node, dev_data_list) { 382 if (dev_data->devid == devid) 383 return dev_data; 384 } 385 386 return NULL; 387} 388 389static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) 390{ 391 struct dev_table_entry new; 392 struct amd_iommu *iommu; 393 struct iommu_dev_data *dev_data, *alias_data; 394 u16 devid = pci_dev_id(pdev); 395 int ret = 0; 396 397 if (devid == alias) 398 return 0; 399 400 iommu = rlookup_amd_iommu(&pdev->dev); 401 if (!iommu) 402 return 0; 403 404 /* Copy the data from pdev */ 405 dev_data = dev_iommu_priv_get(&pdev->dev); 406 if (!dev_data) { 407 pr_err("%s : Failed to get dev_data for 0x%x\n", __func__, devid); 408 ret = -EINVAL; 409 goto out; 410 } 411 get_dte256(iommu, dev_data, &new); 412 413 /* Setup alias */ 414 alias_data = find_dev_data(iommu, alias); 415 if (!alias_data) { 416 pr_err("%s : Failed to get alias dev_data for 0x%x\n", __func__, alias); 417 ret = -EINVAL; 418 goto out; 419 } 420 update_dte256(iommu, alias_data, &new); 421 422 amd_iommu_set_rlookup_table(iommu, alias); 423out: 424 return ret; 425} 426 427static void clone_aliases(struct amd_iommu *iommu, struct device *dev) 428{ 429 struct pci_dev *pdev; 430 431 if (!dev_is_pci(dev)) 432 return; 433 pdev = to_pci_dev(dev); 434 435 /* 436 * The IVRS alias stored in the alias table may not be 437 * part of the PCI DMA aliases if it's bus differs 438 * from the original device. 439 */ 440 clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL); 441 442 pci_for_each_dma_alias(pdev, clone_alias, NULL); 443} 444 445static void setup_aliases(struct amd_iommu *iommu, struct device *dev) 446{ 447 struct pci_dev *pdev = to_pci_dev(dev); 448 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 449 u16 ivrs_alias; 450 451 /* For ACPI HID devices, there are no aliases */ 452 if (!dev_is_pci(dev)) 453 return; 454 455 /* 456 * Add the IVRS alias to the pci aliases if it is on the same 457 * bus. The IVRS table may know about a quirk that we don't. 458 */ 459 ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)]; 460 if (ivrs_alias != pci_dev_id(pdev) && 461 PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) 462 pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1); 463 464 clone_aliases(iommu, dev); 465} 466 467static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid) 468{ 469 struct iommu_dev_data *dev_data; 470 471 dev_data = search_dev_data(iommu, devid); 472 473 if (dev_data == NULL) { 474 dev_data = alloc_dev_data(iommu, devid); 475 if (!dev_data) 476 return NULL; 477 478 if (translation_pre_enabled(iommu)) 479 dev_data->defer_attach = true; 480 } 481 482 return dev_data; 483} 484 485/* 486* Find or create an IOMMU group for a acpihid device. 487*/ 488static struct iommu_group *acpihid_device_group(struct device *dev) 489{ 490 struct acpihid_map_entry *p, *entry = NULL; 491 int devid; 492 493 devid = get_acpihid_device_id(dev, &entry); 494 if (devid < 0) 495 return ERR_PTR(devid); 496 497 list_for_each_entry(p, &acpihid_map, list) { 498 if ((devid == p->devid) && p->group) 499 entry->group = p->group; 500 } 501 502 if (!entry->group) 503 entry->group = generic_device_group(dev); 504 else 505 iommu_group_ref_get(entry->group); 506 507 return entry->group; 508} 509 510static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data) 511{ 512 return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP); 513} 514 515static u32 pdev_get_caps(struct pci_dev *pdev) 516{ 517 int features; 518 u32 flags = 0; 519 520 if (pci_ats_supported(pdev)) 521 flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP; 522 523 if (pci_pri_supported(pdev)) 524 flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP; 525 526 features = pci_pasid_features(pdev); 527 if (features >= 0) { 528 flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP; 529 530 if (features & PCI_PASID_CAP_EXEC) 531 flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP; 532 533 if (features & PCI_PASID_CAP_PRIV) 534 flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP; 535 } 536 537 return flags; 538} 539 540static inline int pdev_enable_cap_ats(struct pci_dev *pdev) 541{ 542 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 543 int ret = -EINVAL; 544 545 if (dev_data->ats_enabled) 546 return 0; 547 548 if (amd_iommu_iotlb_sup && 549 (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) { 550 ret = pci_enable_ats(pdev, PAGE_SHIFT); 551 if (!ret) { 552 dev_data->ats_enabled = 1; 553 dev_data->ats_qdep = pci_ats_queue_depth(pdev); 554 } 555 } 556 557 return ret; 558} 559 560static inline void pdev_disable_cap_ats(struct pci_dev *pdev) 561{ 562 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 563 564 if (dev_data->ats_enabled) { 565 pci_disable_ats(pdev); 566 dev_data->ats_enabled = 0; 567 } 568} 569 570static inline int pdev_enable_cap_pri(struct pci_dev *pdev) 571{ 572 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 573 int ret = -EINVAL; 574 575 if (dev_data->pri_enabled) 576 return 0; 577 578 if (!dev_data->ats_enabled) 579 return 0; 580 581 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) { 582 /* 583 * First reset the PRI state of the device. 584 * FIXME: Hardcode number of outstanding requests for now 585 */ 586 if (!pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32)) { 587 dev_data->pri_enabled = 1; 588 dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev); 589 590 ret = 0; 591 } 592 } 593 594 return ret; 595} 596 597static inline void pdev_disable_cap_pri(struct pci_dev *pdev) 598{ 599 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 600 601 if (dev_data->pri_enabled) { 602 pci_disable_pri(pdev); 603 dev_data->pri_enabled = 0; 604 } 605} 606 607static inline int pdev_enable_cap_pasid(struct pci_dev *pdev) 608{ 609 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 610 int ret = -EINVAL; 611 612 if (dev_data->pasid_enabled) 613 return 0; 614 615 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) { 616 /* Only allow access to user-accessible pages */ 617 ret = pci_enable_pasid(pdev, 0); 618 if (!ret) 619 dev_data->pasid_enabled = 1; 620 } 621 622 return ret; 623} 624 625static inline void pdev_disable_cap_pasid(struct pci_dev *pdev) 626{ 627 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); 628 629 if (dev_data->pasid_enabled) { 630 pci_disable_pasid(pdev); 631 dev_data->pasid_enabled = 0; 632 } 633} 634 635static void pdev_enable_caps(struct pci_dev *pdev) 636{ 637 pdev_enable_cap_pasid(pdev); 638 pdev_enable_cap_ats(pdev); 639 pdev_enable_cap_pri(pdev); 640} 641 642static void pdev_disable_caps(struct pci_dev *pdev) 643{ 644 pdev_disable_cap_ats(pdev); 645 pdev_disable_cap_pasid(pdev); 646 pdev_disable_cap_pri(pdev); 647} 648 649/* 650 * This function checks if the driver got a valid device from the caller to 651 * avoid dereferencing invalid pointers. 652 */ 653static bool check_device(struct device *dev) 654{ 655 struct amd_iommu_pci_seg *pci_seg; 656 struct amd_iommu *iommu; 657 int devid, sbdf; 658 659 if (!dev) 660 return false; 661 662 sbdf = get_device_sbdf_id(dev); 663 if (sbdf < 0) 664 return false; 665 devid = PCI_SBDF_TO_DEVID(sbdf); 666 667 iommu = rlookup_amd_iommu(dev); 668 if (!iommu) 669 return false; 670 671 /* Out of our scope? */ 672 pci_seg = iommu->pci_seg; 673 if (devid > pci_seg->last_bdf) 674 return false; 675 676 return true; 677} 678 679static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) 680{ 681 struct iommu_dev_data *dev_data; 682 int devid, sbdf; 683 684 if (dev_iommu_priv_get(dev)) 685 return 0; 686 687 sbdf = get_device_sbdf_id(dev); 688 if (sbdf < 0) 689 return sbdf; 690 691 devid = PCI_SBDF_TO_DEVID(sbdf); 692 dev_data = find_dev_data(iommu, devid); 693 if (!dev_data) 694 return -ENOMEM; 695 696 dev_data->dev = dev; 697 698 /* 699 * The dev_iommu_priv_set() needes to be called before setup_aliases. 700 * Otherwise, subsequent call to dev_iommu_priv_get() will fail. 701 */ 702 dev_iommu_priv_set(dev, dev_data); 703 setup_aliases(iommu, dev); 704 705 /* 706 * By default we use passthrough mode for IOMMUv2 capable device. 707 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to 708 * invalid address), we ignore the capability for the device so 709 * it'll be forced to go into translation mode. 710 */ 711 if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && 712 dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) { 713 dev_data->flags = pdev_get_caps(to_pci_dev(dev)); 714 } 715 716 return 0; 717} 718 719static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) 720{ 721 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 722 struct dev_table_entry *dev_table = get_dev_table(iommu); 723 int devid, sbdf; 724 725 sbdf = get_device_sbdf_id(dev); 726 if (sbdf < 0) 727 return; 728 729 devid = PCI_SBDF_TO_DEVID(sbdf); 730 pci_seg->rlookup_table[devid] = NULL; 731 memset(&dev_table[devid], 0, sizeof(struct dev_table_entry)); 732 733 setup_aliases(iommu, dev); 734} 735 736 737/**************************************************************************** 738 * 739 * Interrupt handling functions 740 * 741 ****************************************************************************/ 742 743static void dump_dte_entry(struct amd_iommu *iommu, u16 devid) 744{ 745 int i; 746 struct dev_table_entry dte; 747 struct iommu_dev_data *dev_data = find_dev_data(iommu, devid); 748 749 get_dte256(iommu, dev_data, &dte); 750 751 for (i = 0; i < 4; ++i) 752 pr_err("DTE[%d]: %016llx\n", i, dte.data[i]); 753} 754 755static void dump_command(unsigned long phys_addr) 756{ 757 struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr); 758 int i; 759 760 for (i = 0; i < 4; ++i) 761 pr_err("CMD[%d]: %08x\n", i, cmd->data[i]); 762} 763 764static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event) 765{ 766 struct iommu_dev_data *dev_data = NULL; 767 int devid, vmg_tag, flags; 768 struct pci_dev *pdev; 769 u64 spa; 770 771 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 772 vmg_tag = (event[1]) & 0xFFFF; 773 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 774 spa = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8); 775 776 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 777 devid & 0xff); 778 if (pdev) 779 dev_data = dev_iommu_priv_get(&pdev->dev); 780 781 if (dev_data) { 782 if (__ratelimit(&dev_data->rs)) { 783 pci_err(pdev, "Event logged [RMP_HW_ERROR vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", 784 vmg_tag, spa, flags); 785 } 786 } else { 787 pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", 788 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 789 vmg_tag, spa, flags); 790 } 791 792 if (pdev) 793 pci_dev_put(pdev); 794} 795 796static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event) 797{ 798 struct iommu_dev_data *dev_data = NULL; 799 int devid, flags_rmp, vmg_tag, flags; 800 struct pci_dev *pdev; 801 u64 gpa; 802 803 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 804 flags_rmp = (event[0] >> EVENT_FLAGS_SHIFT) & 0xFF; 805 vmg_tag = (event[1]) & 0xFFFF; 806 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 807 gpa = ((u64)event[3] << 32) | event[2]; 808 809 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 810 devid & 0xff); 811 if (pdev) 812 dev_data = dev_iommu_priv_get(&pdev->dev); 813 814 if (dev_data) { 815 if (__ratelimit(&dev_data->rs)) { 816 pci_err(pdev, "Event logged [RMP_PAGE_FAULT vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", 817 vmg_tag, gpa, flags_rmp, flags); 818 } 819 } else { 820 pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", 821 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 822 vmg_tag, gpa, flags_rmp, flags); 823 } 824 825 if (pdev) 826 pci_dev_put(pdev); 827} 828 829#define IS_IOMMU_MEM_TRANSACTION(flags) \ 830 (((flags) & EVENT_FLAG_I) == 0) 831 832#define IS_WRITE_REQUEST(flags) \ 833 ((flags) & EVENT_FLAG_RW) 834 835static void amd_iommu_report_page_fault(struct amd_iommu *iommu, 836 u16 devid, u16 domain_id, 837 u64 address, int flags) 838{ 839 struct iommu_dev_data *dev_data = NULL; 840 struct pci_dev *pdev; 841 842 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), 843 devid & 0xff); 844 if (pdev) 845 dev_data = dev_iommu_priv_get(&pdev->dev); 846 847 if (dev_data) { 848 /* 849 * If this is a DMA fault (for which the I(nterrupt) 850 * bit will be unset), allow report_iommu_fault() to 851 * prevent logging it. 852 */ 853 if (IS_IOMMU_MEM_TRANSACTION(flags)) { 854 /* Device not attached to domain properly */ 855 if (dev_data->domain == NULL) { 856 pr_err_ratelimited("Event logged [Device not attached to domain properly]\n"); 857 pr_err_ratelimited(" device=%04x:%02x:%02x.%x domain=0x%04x\n", 858 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), 859 PCI_FUNC(devid), domain_id); 860 goto out; 861 } 862 863 if (!report_iommu_fault(&dev_data->domain->domain, 864 &pdev->dev, address, 865 IS_WRITE_REQUEST(flags) ? 866 IOMMU_FAULT_WRITE : 867 IOMMU_FAULT_READ)) 868 goto out; 869 } 870 871 if (__ratelimit(&dev_data->rs)) { 872 pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", 873 domain_id, address, flags); 874 } 875 } else { 876 pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", 877 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 878 domain_id, address, flags); 879 } 880 881out: 882 if (pdev) 883 pci_dev_put(pdev); 884} 885 886static void iommu_print_event(struct amd_iommu *iommu, void *__evt) 887{ 888 struct device *dev = iommu->iommu.dev; 889 int type, devid, flags, tag; 890 volatile u32 *event = __evt; 891 int count = 0; 892 u64 address, ctrl; 893 u32 pasid; 894 895retry: 896 type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK; 897 devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; 898 pasid = (event[0] & EVENT_DOMID_MASK_HI) | 899 (event[1] & EVENT_DOMID_MASK_LO); 900 flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; 901 address = (u64)(((u64)event[3]) << 32) | event[2]; 902 ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); 903 904 if (type == 0) { 905 /* Did we hit the erratum? */ 906 if (++count == LOOP_TIMEOUT) { 907 pr_err("No event written to event log\n"); 908 return; 909 } 910 udelay(1); 911 goto retry; 912 } 913 914 if (type == EVENT_TYPE_IO_FAULT) { 915 amd_iommu_report_page_fault(iommu, devid, pasid, address, flags); 916 return; 917 } 918 919 switch (type) { 920 case EVENT_TYPE_ILL_DEV: 921 dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", 922 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 923 pasid, address, flags); 924 dev_err(dev, "Control Reg : 0x%llx\n", ctrl); 925 dump_dte_entry(iommu, devid); 926 break; 927 case EVENT_TYPE_DEV_TAB_ERR: 928 dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x " 929 "address=0x%llx flags=0x%04x]\n", 930 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 931 address, flags); 932 break; 933 case EVENT_TYPE_PAGE_TAB_ERR: 934 dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n", 935 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 936 pasid, address, flags); 937 break; 938 case EVENT_TYPE_ILL_CMD: 939 dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address); 940 dump_command(address); 941 break; 942 case EVENT_TYPE_CMD_HARD_ERR: 943 dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n", 944 address, flags); 945 break; 946 case EVENT_TYPE_IOTLB_INV_TO: 947 dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n", 948 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 949 address); 950 break; 951 case EVENT_TYPE_INV_DEV_REQ: 952 dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", 953 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 954 pasid, address, flags); 955 break; 956 case EVENT_TYPE_RMP_FAULT: 957 amd_iommu_report_rmp_fault(iommu, event); 958 break; 959 case EVENT_TYPE_RMP_HW_ERR: 960 amd_iommu_report_rmp_hw_error(iommu, event); 961 break; 962 case EVENT_TYPE_INV_PPR_REQ: 963 pasid = PPR_PASID(*((u64 *)__evt)); 964 tag = event[1] & 0x03FF; 965 dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n", 966 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), 967 pasid, address, flags, tag); 968 break; 969 default: 970 dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n", 971 event[0], event[1], event[2], event[3]); 972 } 973 974 /* 975 * To detect the hardware errata 732 we need to clear the 976 * entry back to zero. This issue does not exist on SNP 977 * enabled system. Also this buffer is not writeable on 978 * SNP enabled system. 979 */ 980 if (!amd_iommu_snp_en) 981 memset(__evt, 0, 4 * sizeof(u32)); 982} 983 984static void iommu_poll_events(struct amd_iommu *iommu) 985{ 986 u32 head, tail; 987 988 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 989 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); 990 991 while (head != tail) { 992 iommu_print_event(iommu, iommu->evt_buf + head); 993 994 /* Update head pointer of hardware ring-buffer */ 995 head = (head + EVENT_ENTRY_SIZE) % EVT_BUFFER_SIZE; 996 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); 997 } 998 999} 1000 1001#ifdef CONFIG_IRQ_REMAP 1002static int (*iommu_ga_log_notifier)(u32); 1003 1004int amd_iommu_register_ga_log_notifier(int (*notifier)(u32)) 1005{ 1006 iommu_ga_log_notifier = notifier; 1007 1008 /* 1009 * Ensure all in-flight IRQ handlers run to completion before returning 1010 * to the caller, e.g. to ensure module code isn't unloaded while it's 1011 * being executed in the IRQ handler. 1012 */ 1013 if (!notifier) 1014 synchronize_rcu(); 1015 1016 return 0; 1017} 1018EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier); 1019 1020static void iommu_poll_ga_log(struct amd_iommu *iommu) 1021{ 1022 u32 head, tail; 1023 1024 if (iommu->ga_log == NULL) 1025 return; 1026 1027 head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 1028 tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET); 1029 1030 while (head != tail) { 1031 volatile u64 *raw; 1032 u64 log_entry; 1033 1034 raw = (u64 *)(iommu->ga_log + head); 1035 1036 /* Avoid memcpy function-call overhead */ 1037 log_entry = *raw; 1038 1039 /* Update head pointer of hardware ring-buffer */ 1040 head = (head + GA_ENTRY_SIZE) % GA_LOG_SIZE; 1041 writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); 1042 1043 /* Handle GA entry */ 1044 switch (GA_REQ_TYPE(log_entry)) { 1045 case GA_GUEST_NR: 1046 if (!iommu_ga_log_notifier) 1047 break; 1048 1049 pr_debug("%s: devid=%#x, ga_tag=%#x\n", 1050 __func__, GA_DEVID(log_entry), 1051 GA_TAG(log_entry)); 1052 1053 if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0) 1054 pr_err("GA log notifier failed.\n"); 1055 break; 1056 default: 1057 break; 1058 } 1059 } 1060} 1061 1062static void 1063amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) 1064{ 1065 if (!irq_remapping_enabled || !dev_is_pci(dev) || 1066 !pci_dev_has_default_msi_parent_domain(to_pci_dev(dev))) 1067 return; 1068 1069 dev_set_msi_domain(dev, iommu->ir_domain); 1070} 1071 1072#else /* CONFIG_IRQ_REMAP */ 1073static inline void 1074amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { } 1075#endif /* !CONFIG_IRQ_REMAP */ 1076 1077static void amd_iommu_handle_irq(void *data, const char *evt_type, 1078 u32 int_mask, u32 overflow_mask, 1079 void (*int_handler)(struct amd_iommu *), 1080 void (*overflow_handler)(struct amd_iommu *)) 1081{ 1082 struct amd_iommu *iommu = (struct amd_iommu *) data; 1083 u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 1084 u32 mask = int_mask | overflow_mask; 1085 1086 while (status & mask) { 1087 /* Enable interrupt sources again */ 1088 writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET); 1089 1090 if (int_handler) { 1091 pr_devel("Processing IOMMU (ivhd%d) %s Log\n", 1092 iommu->index, evt_type); 1093 int_handler(iommu); 1094 } 1095 1096 if ((status & overflow_mask) && overflow_handler) 1097 overflow_handler(iommu); 1098 1099 /* 1100 * Hardware bug: ERBT1312 1101 * When re-enabling interrupt (by writing 1 1102 * to clear the bit), the hardware might also try to set 1103 * the interrupt bit in the event status register. 1104 * In this scenario, the bit will be set, and disable 1105 * subsequent interrupts. 1106 * 1107 * Workaround: The IOMMU driver should read back the 1108 * status register and check if the interrupt bits are cleared. 1109 * If not, driver will need to go through the interrupt handler 1110 * again and re-clear the bits 1111 */ 1112 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); 1113 } 1114} 1115 1116irqreturn_t amd_iommu_int_thread_evtlog(int irq, void *data) 1117{ 1118 amd_iommu_handle_irq(data, "Evt", MMIO_STATUS_EVT_INT_MASK, 1119 MMIO_STATUS_EVT_OVERFLOW_MASK, 1120 iommu_poll_events, amd_iommu_restart_event_logging); 1121 1122 return IRQ_HANDLED; 1123} 1124 1125irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data) 1126{ 1127 amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK, 1128 MMIO_STATUS_PPR_OVERFLOW_MASK, 1129 amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log); 1130 1131 return IRQ_HANDLED; 1132} 1133 1134irqreturn_t amd_iommu_int_thread_galog(int irq, void *data) 1135{ 1136#ifdef CONFIG_IRQ_REMAP 1137 amd_iommu_handle_irq(data, "GA", MMIO_STATUS_GALOG_INT_MASK, 1138 MMIO_STATUS_GALOG_OVERFLOW_MASK, 1139 iommu_poll_ga_log, amd_iommu_restart_ga_log); 1140#endif 1141 1142 return IRQ_HANDLED; 1143} 1144 1145irqreturn_t amd_iommu_int_thread(int irq, void *data) 1146{ 1147 amd_iommu_int_thread_evtlog(irq, data); 1148 amd_iommu_int_thread_pprlog(irq, data); 1149 amd_iommu_int_thread_galog(irq, data); 1150 1151 return IRQ_HANDLED; 1152} 1153 1154/**************************************************************************** 1155 * 1156 * IOMMU command queuing functions 1157 * 1158 ****************************************************************************/ 1159 1160static void dump_command_buffer(struct amd_iommu *iommu) 1161{ 1162 struct iommu_cmd *cmd; 1163 u32 head, tail; 1164 int i; 1165 1166 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); 1167 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 1168 1169 pr_err("CMD Buffer head=%llu tail=%llu\n", MMIO_CMD_BUFFER_HEAD(head), 1170 MMIO_CMD_BUFFER_TAIL(tail)); 1171 1172 for (i = 0; i < CMD_BUFFER_ENTRIES; i++) { 1173 cmd = (struct iommu_cmd *)(iommu->cmd_buf + i * sizeof(*cmd)); 1174 pr_err("%3d: %08x %08x %08x %08x\n", i, cmd->data[0], cmd->data[1], cmd->data[2], 1175 cmd->data[3]); 1176 } 1177} 1178 1179static int wait_on_sem(struct amd_iommu *iommu, u64 data) 1180{ 1181 int i = 0; 1182 1183 /* 1184 * cmd_sem holds a monotonically non-decreasing completion sequence 1185 * number. 1186 */ 1187 while ((__s64)(READ_ONCE(*iommu->cmd_sem) - data) < 0 && 1188 i < LOOP_TIMEOUT) { 1189 udelay(1); 1190 i += 1; 1191 } 1192 1193 if (i == LOOP_TIMEOUT) { 1194 1195 pr_alert("IOMMU %04x:%02x:%02x.%01x: Completion-Wait loop timed out\n", 1196 iommu->pci_seg->id, PCI_BUS_NUM(iommu->devid), 1197 PCI_SLOT(iommu->devid), PCI_FUNC(iommu->devid)); 1198 1199 if (amd_iommu_dump) 1200 DO_ONCE_LITE(dump_command_buffer, iommu); 1201 1202 return -EIO; 1203 } 1204 1205 return 0; 1206} 1207 1208static void copy_cmd_to_buffer(struct amd_iommu *iommu, 1209 struct iommu_cmd *cmd) 1210{ 1211 u8 *target; 1212 u32 tail; 1213 1214 /* Copy command to buffer */ 1215 tail = iommu->cmd_buf_tail; 1216 target = iommu->cmd_buf + tail; 1217 memcpy(target, cmd, sizeof(*cmd)); 1218 1219 tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; 1220 iommu->cmd_buf_tail = tail; 1221 1222 /* Tell the IOMMU about it */ 1223 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); 1224} 1225 1226static void build_completion_wait(struct iommu_cmd *cmd, 1227 struct amd_iommu *iommu, 1228 u64 data) 1229{ 1230 u64 paddr = iommu->cmd_sem_paddr; 1231 1232 memset(cmd, 0, sizeof(*cmd)); 1233 cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK; 1234 cmd->data[1] = upper_32_bits(paddr); 1235 cmd->data[2] = lower_32_bits(data); 1236 cmd->data[3] = upper_32_bits(data); 1237 CMD_SET_TYPE(cmd, CMD_COMPL_WAIT); 1238} 1239 1240static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) 1241{ 1242 memset(cmd, 0, sizeof(*cmd)); 1243 cmd->data[0] = devid; 1244 CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY); 1245} 1246 1247/* 1248 * Builds an invalidation address which is suitable for one page or multiple 1249 * pages. Sets the size bit (S) as needed is more than one page is flushed. 1250 */ 1251static inline u64 build_inv_address(u64 address, size_t size) 1252{ 1253 u64 pages, end, msb_diff; 1254 1255 pages = iommu_num_pages(address, size, PAGE_SIZE); 1256 1257 if (pages == 1) 1258 return address & PAGE_MASK; 1259 1260 end = address + size - 1; 1261 1262 /* 1263 * msb_diff would hold the index of the most significant bit that 1264 * flipped between the start and end. 1265 */ 1266 msb_diff = fls64(end ^ address) - 1; 1267 1268 /* 1269 * Bits 63:52 are sign extended. If for some reason bit 51 is different 1270 * between the start and the end, invalidate everything. 1271 */ 1272 if (unlikely(msb_diff > 51)) { 1273 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; 1274 } else { 1275 /* 1276 * The msb-bit must be clear on the address. Just set all the 1277 * lower bits. 1278 */ 1279 address |= (1ull << msb_diff) - 1; 1280 } 1281 1282 /* Clear bits 11:0 */ 1283 address &= PAGE_MASK; 1284 1285 /* Set the size bit - we flush more than one 4kb page */ 1286 return address | CMD_INV_IOMMU_PAGES_SIZE_MASK; 1287} 1288 1289static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, 1290 size_t size, u16 domid, 1291 ioasid_t pasid, bool gn) 1292{ 1293 u64 inv_address = build_inv_address(address, size); 1294 1295 memset(cmd, 0, sizeof(*cmd)); 1296 1297 cmd->data[1] |= domid; 1298 cmd->data[2] = lower_32_bits(inv_address); 1299 cmd->data[3] = upper_32_bits(inv_address); 1300 /* PDE bit - we want to flush everything, not only the PTEs */ 1301 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; 1302 if (gn) { 1303 cmd->data[0] |= pasid; 1304 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; 1305 } 1306 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); 1307} 1308 1309static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, 1310 u64 address, size_t size, 1311 ioasid_t pasid, bool gn) 1312{ 1313 u64 inv_address = build_inv_address(address, size); 1314 1315 memset(cmd, 0, sizeof(*cmd)); 1316 1317 cmd->data[0] = devid; 1318 cmd->data[0] |= (qdep & 0xff) << 24; 1319 cmd->data[1] = devid; 1320 cmd->data[2] = lower_32_bits(inv_address); 1321 cmd->data[3] = upper_32_bits(inv_address); 1322 if (gn) { 1323 cmd->data[0] |= ((pasid >> 8) & 0xff) << 16; 1324 cmd->data[1] |= (pasid & 0xff) << 16; 1325 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; 1326 } 1327 1328 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES); 1329} 1330 1331static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid, 1332 int status, int tag, u8 gn) 1333{ 1334 memset(cmd, 0, sizeof(*cmd)); 1335 1336 cmd->data[0] = devid; 1337 if (gn) { 1338 cmd->data[1] = pasid; 1339 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK; 1340 } 1341 cmd->data[3] = tag & 0x1ff; 1342 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT; 1343 1344 CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR); 1345} 1346 1347static void build_inv_all(struct iommu_cmd *cmd) 1348{ 1349 memset(cmd, 0, sizeof(*cmd)); 1350 CMD_SET_TYPE(cmd, CMD_INV_ALL); 1351} 1352 1353static void build_inv_irt(struct iommu_cmd *cmd, u16 devid) 1354{ 1355 memset(cmd, 0, sizeof(*cmd)); 1356 cmd->data[0] = devid; 1357 CMD_SET_TYPE(cmd, CMD_INV_IRT); 1358} 1359 1360/* 1361 * Writes the command to the IOMMUs command buffer and informs the 1362 * hardware about the new command. 1363 */ 1364static int __iommu_queue_command_sync(struct amd_iommu *iommu, 1365 struct iommu_cmd *cmd, 1366 bool sync) 1367{ 1368 unsigned int count = 0; 1369 u32 left, next_tail; 1370 1371 next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; 1372again: 1373 left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE; 1374 1375 if (left <= 0x20) { 1376 /* Skip udelay() the first time around */ 1377 if (count++) { 1378 if (count == LOOP_TIMEOUT) { 1379 pr_err("Command buffer timeout\n"); 1380 return -EIO; 1381 } 1382 1383 udelay(1); 1384 } 1385 1386 /* Update head and recheck remaining space */ 1387 iommu->cmd_buf_head = readl(iommu->mmio_base + 1388 MMIO_CMD_HEAD_OFFSET); 1389 1390 goto again; 1391 } 1392 1393 copy_cmd_to_buffer(iommu, cmd); 1394 1395 /* Do we need to make sure all commands are processed? */ 1396 iommu->need_sync = sync; 1397 1398 return 0; 1399} 1400 1401static int iommu_queue_command_sync(struct amd_iommu *iommu, 1402 struct iommu_cmd *cmd, 1403 bool sync) 1404{ 1405 unsigned long flags; 1406 int ret; 1407 1408 raw_spin_lock_irqsave(&iommu->lock, flags); 1409 ret = __iommu_queue_command_sync(iommu, cmd, sync); 1410 raw_spin_unlock_irqrestore(&iommu->lock, flags); 1411 1412 return ret; 1413} 1414 1415static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) 1416{ 1417 return iommu_queue_command_sync(iommu, cmd, true); 1418} 1419 1420static u64 get_cmdsem_val(struct amd_iommu *iommu) 1421{ 1422 lockdep_assert_held(&iommu->lock); 1423 return ++iommu->cmd_sem_val; 1424} 1425 1426/* 1427 * This function queues a completion wait command into the command 1428 * buffer of an IOMMU 1429 */ 1430static int iommu_completion_wait(struct amd_iommu *iommu) 1431{ 1432 struct iommu_cmd cmd; 1433 unsigned long flags; 1434 int ret; 1435 u64 data; 1436 1437 if (!iommu->need_sync) 1438 return 0; 1439 1440 raw_spin_lock_irqsave(&iommu->lock, flags); 1441 1442 data = get_cmdsem_val(iommu); 1443 build_completion_wait(&cmd, iommu, data); 1444 1445 ret = __iommu_queue_command_sync(iommu, &cmd, false); 1446 raw_spin_unlock_irqrestore(&iommu->lock, flags); 1447 1448 if (ret) 1449 return ret; 1450 1451 ret = wait_on_sem(iommu, data); 1452 1453 return ret; 1454} 1455 1456static void domain_flush_complete(struct protection_domain *domain) 1457{ 1458 struct pdom_iommu_info *pdom_iommu_info; 1459 unsigned long i; 1460 1461 lockdep_assert_held(&domain->lock); 1462 1463 /* 1464 * Devices of this domain are behind this IOMMU 1465 * We need to wait for completion of all commands. 1466 */ 1467 xa_for_each(&domain->iommu_array, i, pdom_iommu_info) 1468 iommu_completion_wait(pdom_iommu_info->iommu); 1469} 1470 1471static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid) 1472{ 1473 struct iommu_cmd cmd; 1474 1475 build_inv_dte(&cmd, devid); 1476 1477 return iommu_queue_command(iommu, &cmd); 1478} 1479 1480static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid) 1481{ 1482 int ret; 1483 1484 ret = iommu_flush_dte(iommu, devid); 1485 if (!ret) 1486 iommu_completion_wait(iommu); 1487} 1488 1489static void amd_iommu_flush_dte_all(struct amd_iommu *iommu) 1490{ 1491 u32 devid; 1492 u16 last_bdf = iommu->pci_seg->last_bdf; 1493 1494 for (devid = 0; devid <= last_bdf; ++devid) 1495 iommu_flush_dte(iommu, devid); 1496 1497 iommu_completion_wait(iommu); 1498} 1499 1500/* 1501 * This function uses heavy locking and may disable irqs for some time. But 1502 * this is no issue because it is only called during resume. 1503 */ 1504static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu) 1505{ 1506 u32 dom_id; 1507 u16 last_bdf = iommu->pci_seg->last_bdf; 1508 1509 for (dom_id = 0; dom_id <= last_bdf; ++dom_id) { 1510 struct iommu_cmd cmd; 1511 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1512 dom_id, IOMMU_NO_PASID, false); 1513 iommu_queue_command(iommu, &cmd); 1514 } 1515 1516 iommu_completion_wait(iommu); 1517} 1518 1519static void amd_iommu_flush_tlb_domid(struct amd_iommu *iommu, u32 dom_id) 1520{ 1521 struct iommu_cmd cmd; 1522 1523 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1524 dom_id, IOMMU_NO_PASID, false); 1525 iommu_queue_command(iommu, &cmd); 1526 1527 iommu_completion_wait(iommu); 1528} 1529 1530static void amd_iommu_flush_all(struct amd_iommu *iommu) 1531{ 1532 struct iommu_cmd cmd; 1533 1534 build_inv_all(&cmd); 1535 1536 iommu_queue_command(iommu, &cmd); 1537 iommu_completion_wait(iommu); 1538} 1539 1540static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid) 1541{ 1542 struct iommu_cmd cmd; 1543 1544 build_inv_irt(&cmd, devid); 1545 1546 iommu_queue_command(iommu, &cmd); 1547} 1548 1549static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) 1550{ 1551 u32 devid; 1552 u16 last_bdf = iommu->pci_seg->last_bdf; 1553 1554 if (iommu->irtcachedis_enabled) 1555 return; 1556 1557 for (devid = 0; devid <= last_bdf; devid++) 1558 iommu_flush_irt(iommu, devid); 1559 1560 iommu_completion_wait(iommu); 1561} 1562 1563void amd_iommu_flush_all_caches(struct amd_iommu *iommu) 1564{ 1565 if (check_feature(FEATURE_IA)) { 1566 amd_iommu_flush_all(iommu); 1567 } else { 1568 amd_iommu_flush_dte_all(iommu); 1569 amd_iommu_flush_irt_all(iommu); 1570 amd_iommu_flush_tlb_all(iommu); 1571 } 1572} 1573 1574/* 1575 * Command send function for flushing on-device TLB 1576 */ 1577static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address, 1578 size_t size, ioasid_t pasid, bool gn) 1579{ 1580 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 1581 struct iommu_cmd cmd; 1582 int qdep = dev_data->ats_qdep; 1583 1584 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, 1585 size, pasid, gn); 1586 1587 return iommu_queue_command(iommu, &cmd); 1588} 1589 1590static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data) 1591{ 1592 struct amd_iommu *iommu = data; 1593 1594 return iommu_flush_dte(iommu, alias); 1595} 1596 1597/* 1598 * Command send function for invalidating a device table entry 1599 */ 1600static int device_flush_dte(struct iommu_dev_data *dev_data) 1601{ 1602 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 1603 struct pci_dev *pdev = NULL; 1604 struct amd_iommu_pci_seg *pci_seg; 1605 u16 alias; 1606 int ret; 1607 1608 if (dev_is_pci(dev_data->dev)) 1609 pdev = to_pci_dev(dev_data->dev); 1610 1611 if (pdev) 1612 ret = pci_for_each_dma_alias(pdev, 1613 device_flush_dte_alias, iommu); 1614 else 1615 ret = iommu_flush_dte(iommu, dev_data->devid); 1616 if (ret) 1617 return ret; 1618 1619 pci_seg = iommu->pci_seg; 1620 alias = pci_seg->alias_table[dev_data->devid]; 1621 if (alias != dev_data->devid) { 1622 ret = iommu_flush_dte(iommu, alias); 1623 if (ret) 1624 return ret; 1625 } 1626 1627 if (dev_data->ats_enabled) { 1628 /* Invalidate the entire contents of an IOTLB */ 1629 ret = device_flush_iotlb(dev_data, 0, ~0UL, 1630 IOMMU_NO_PASID, false); 1631 } 1632 1633 return ret; 1634} 1635 1636static int domain_flush_pages_v2(struct protection_domain *pdom, 1637 u64 address, size_t size) 1638{ 1639 struct iommu_dev_data *dev_data; 1640 struct iommu_cmd cmd; 1641 int ret = 0; 1642 1643 lockdep_assert_held(&pdom->lock); 1644 list_for_each_entry(dev_data, &pdom->dev_list, list) { 1645 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 1646 u16 domid = dev_data->gcr3_info.domid; 1647 1648 build_inv_iommu_pages(&cmd, address, size, 1649 domid, IOMMU_NO_PASID, true); 1650 1651 ret |= iommu_queue_command(iommu, &cmd); 1652 } 1653 1654 return ret; 1655} 1656 1657static int domain_flush_pages_v1(struct protection_domain *pdom, 1658 u64 address, size_t size) 1659{ 1660 struct pdom_iommu_info *pdom_iommu_info; 1661 struct iommu_cmd cmd; 1662 int ret = 0; 1663 unsigned long i; 1664 1665 lockdep_assert_held(&pdom->lock); 1666 1667 build_inv_iommu_pages(&cmd, address, size, 1668 pdom->id, IOMMU_NO_PASID, false); 1669 1670 xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) { 1671 /* 1672 * Devices of this domain are behind this IOMMU 1673 * We need a TLB flush 1674 */ 1675 ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd); 1676 } 1677 1678 return ret; 1679} 1680 1681/* 1682 * TLB invalidation function which is called from the mapping functions. 1683 * It flushes range of PTEs of the domain. 1684 */ 1685static void __domain_flush_pages(struct protection_domain *domain, 1686 u64 address, size_t size) 1687{ 1688 struct iommu_dev_data *dev_data; 1689 int ret = 0; 1690 ioasid_t pasid = IOMMU_NO_PASID; 1691 bool gn = false; 1692 1693 lockdep_assert_held(&domain->lock); 1694 1695 if (pdom_is_v2_pgtbl_mode(domain)) { 1696 gn = true; 1697 ret = domain_flush_pages_v2(domain, address, size); 1698 } else { 1699 ret = domain_flush_pages_v1(domain, address, size); 1700 } 1701 1702 list_for_each_entry(dev_data, &domain->dev_list, list) { 1703 1704 if (!dev_data->ats_enabled) 1705 continue; 1706 1707 ret |= device_flush_iotlb(dev_data, address, size, pasid, gn); 1708 } 1709 1710 WARN_ON(ret); 1711} 1712 1713void amd_iommu_domain_flush_pages(struct protection_domain *domain, 1714 u64 address, size_t size) 1715{ 1716 lockdep_assert_held(&domain->lock); 1717 1718 if (likely(!amd_iommu_np_cache)) { 1719 __domain_flush_pages(domain, address, size); 1720 1721 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ 1722 domain_flush_complete(domain); 1723 1724 return; 1725 } 1726 1727 /* 1728 * When NpCache is on, we infer that we run in a VM and use a vIOMMU. 1729 * In such setups it is best to avoid flushes of ranges which are not 1730 * naturally aligned, since it would lead to flushes of unmodified 1731 * PTEs. Such flushes would require the hypervisor to do more work than 1732 * necessary. Therefore, perform repeated flushes of aligned ranges 1733 * until you cover the range. Each iteration flushes the smaller 1734 * between the natural alignment of the address that we flush and the 1735 * greatest naturally aligned region that fits in the range. 1736 */ 1737 while (size != 0) { 1738 int addr_alignment = __ffs(address); 1739 int size_alignment = __fls(size); 1740 int min_alignment; 1741 size_t flush_size; 1742 1743 /* 1744 * size is always non-zero, but address might be zero, causing 1745 * addr_alignment to be negative. As the casting of the 1746 * argument in __ffs(address) to long might trim the high bits 1747 * of the address on x86-32, cast to long when doing the check. 1748 */ 1749 if (likely((unsigned long)address != 0)) 1750 min_alignment = min(addr_alignment, size_alignment); 1751 else 1752 min_alignment = size_alignment; 1753 1754 flush_size = 1ul << min_alignment; 1755 1756 __domain_flush_pages(domain, address, flush_size); 1757 address += flush_size; 1758 size -= flush_size; 1759 } 1760 1761 /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ 1762 domain_flush_complete(domain); 1763} 1764 1765/* Flush the whole IO/TLB for a given protection domain - including PDE */ 1766static void amd_iommu_domain_flush_all(struct protection_domain *domain) 1767{ 1768 amd_iommu_domain_flush_pages(domain, 0, 1769 CMD_INV_IOMMU_ALL_PAGES_ADDRESS); 1770} 1771 1772void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data, 1773 ioasid_t pasid, u64 address, size_t size) 1774{ 1775 struct iommu_cmd cmd; 1776 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 1777 1778 build_inv_iommu_pages(&cmd, address, size, 1779 dev_data->gcr3_info.domid, pasid, true); 1780 iommu_queue_command(iommu, &cmd); 1781 1782 if (dev_data->ats_enabled) 1783 device_flush_iotlb(dev_data, address, size, pasid, true); 1784 1785 iommu_completion_wait(iommu); 1786} 1787 1788static void dev_flush_pasid_all(struct iommu_dev_data *dev_data, 1789 ioasid_t pasid) 1790{ 1791 amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0, 1792 CMD_INV_IOMMU_ALL_PAGES_ADDRESS); 1793} 1794 1795int amd_iommu_complete_ppr(struct device *dev, u32 pasid, int status, int tag) 1796{ 1797 struct iommu_dev_data *dev_data; 1798 struct amd_iommu *iommu; 1799 struct iommu_cmd cmd; 1800 1801 dev_data = dev_iommu_priv_get(dev); 1802 iommu = get_amd_iommu_from_dev(dev); 1803 1804 build_complete_ppr(&cmd, dev_data->devid, pasid, status, 1805 tag, dev_data->pri_tlp); 1806 1807 return iommu_queue_command(iommu, &cmd); 1808} 1809 1810/**************************************************************************** 1811 * 1812 * The next functions belong to the domain allocation. A domain is 1813 * allocated for every IOMMU as the default domain. If device isolation 1814 * is enabled, every device get its own domain. The most important thing 1815 * about domains is the page table mapping the DMA address space they 1816 * contain. 1817 * 1818 ****************************************************************************/ 1819int amd_iommu_pdom_id_alloc(void) 1820{ 1821 return ida_alloc_range(&pdom_ids, 1, MAX_DOMAIN_ID - 1, GFP_ATOMIC); 1822} 1823 1824int amd_iommu_pdom_id_reserve(u16 id, gfp_t gfp) 1825{ 1826 return ida_alloc_range(&pdom_ids, id, id, gfp); 1827} 1828 1829void amd_iommu_pdom_id_free(int id) 1830{ 1831 ida_free(&pdom_ids, id); 1832} 1833 1834void amd_iommu_pdom_id_destroy(void) 1835{ 1836 ida_destroy(&pdom_ids); 1837} 1838 1839static void free_gcr3_tbl_level1(u64 *tbl) 1840{ 1841 u64 *ptr; 1842 int i; 1843 1844 for (i = 0; i < 512; ++i) { 1845 if (!(tbl[i] & GCR3_VALID)) 1846 continue; 1847 1848 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); 1849 1850 iommu_free_pages(ptr); 1851 } 1852} 1853 1854static void free_gcr3_tbl_level2(u64 *tbl) 1855{ 1856 u64 *ptr; 1857 int i; 1858 1859 for (i = 0; i < 512; ++i) { 1860 if (!(tbl[i] & GCR3_VALID)) 1861 continue; 1862 1863 ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK); 1864 1865 free_gcr3_tbl_level1(ptr); 1866 } 1867} 1868 1869static void free_gcr3_table(struct gcr3_tbl_info *gcr3_info) 1870{ 1871 if (gcr3_info->glx == 2) 1872 free_gcr3_tbl_level2(gcr3_info->gcr3_tbl); 1873 else if (gcr3_info->glx == 1) 1874 free_gcr3_tbl_level1(gcr3_info->gcr3_tbl); 1875 else 1876 WARN_ON_ONCE(gcr3_info->glx != 0); 1877 1878 gcr3_info->glx = 0; 1879 1880 /* Free per device domain ID */ 1881 amd_iommu_pdom_id_free(gcr3_info->domid); 1882 1883 iommu_free_pages(gcr3_info->gcr3_tbl); 1884 gcr3_info->gcr3_tbl = NULL; 1885} 1886 1887/* 1888 * Number of GCR3 table levels required. Level must be 4-Kbyte 1889 * page and can contain up to 512 entries. 1890 */ 1891static int get_gcr3_levels(int pasids) 1892{ 1893 int levels; 1894 1895 if (pasids == -1) 1896 return amd_iommu_max_glx_val; 1897 1898 levels = get_count_order(pasids); 1899 1900 return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels; 1901} 1902 1903static int setup_gcr3_table(struct gcr3_tbl_info *gcr3_info, 1904 struct amd_iommu *iommu, int pasids) 1905{ 1906 int levels = get_gcr3_levels(pasids); 1907 int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; 1908 int domid; 1909 1910 if (levels > amd_iommu_max_glx_val) 1911 return -EINVAL; 1912 1913 if (gcr3_info->gcr3_tbl) 1914 return -EBUSY; 1915 1916 /* Allocate per device domain ID */ 1917 domid = amd_iommu_pdom_id_alloc(); 1918 if (domid <= 0) 1919 return -ENOSPC; 1920 gcr3_info->domid = domid; 1921 1922 gcr3_info->gcr3_tbl = iommu_alloc_pages_node_sz(nid, GFP_ATOMIC, SZ_4K); 1923 if (gcr3_info->gcr3_tbl == NULL) { 1924 amd_iommu_pdom_id_free(domid); 1925 return -ENOMEM; 1926 } 1927 1928 gcr3_info->glx = levels; 1929 1930 return 0; 1931} 1932 1933static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info, 1934 ioasid_t pasid, bool alloc) 1935{ 1936 int index; 1937 u64 *pte; 1938 u64 *root = gcr3_info->gcr3_tbl; 1939 int level = gcr3_info->glx; 1940 1941 while (true) { 1942 1943 index = (pasid >> (9 * level)) & 0x1ff; 1944 pte = &root[index]; 1945 1946 if (level == 0) 1947 break; 1948 1949 if (!(*pte & GCR3_VALID)) { 1950 if (!alloc) 1951 return NULL; 1952 1953 root = (void *)get_zeroed_page(GFP_ATOMIC); 1954 if (root == NULL) 1955 return NULL; 1956 1957 *pte = iommu_virt_to_phys(root) | GCR3_VALID; 1958 } 1959 1960 root = iommu_phys_to_virt(*pte & PAGE_MASK); 1961 1962 level -= 1; 1963 } 1964 1965 return pte; 1966} 1967 1968static int update_gcr3(struct iommu_dev_data *dev_data, 1969 ioasid_t pasid, unsigned long gcr3, bool set) 1970{ 1971 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 1972 u64 *pte; 1973 1974 pte = __get_gcr3_pte(gcr3_info, pasid, true); 1975 if (pte == NULL) 1976 return -ENOMEM; 1977 1978 if (set) 1979 *pte = (gcr3 & PAGE_MASK) | GCR3_VALID; 1980 else 1981 *pte = 0; 1982 1983 dev_flush_pasid_all(dev_data, pasid); 1984 return 0; 1985} 1986 1987int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid, 1988 unsigned long gcr3) 1989{ 1990 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 1991 int ret; 1992 1993 iommu_group_mutex_assert(dev_data->dev); 1994 1995 ret = update_gcr3(dev_data, pasid, gcr3, true); 1996 if (ret) 1997 return ret; 1998 1999 gcr3_info->pasid_cnt++; 2000 return ret; 2001} 2002 2003int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid) 2004{ 2005 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2006 int ret; 2007 2008 iommu_group_mutex_assert(dev_data->dev); 2009 2010 ret = update_gcr3(dev_data, pasid, 0, false); 2011 if (ret) 2012 return ret; 2013 2014 gcr3_info->pasid_cnt--; 2015 return ret; 2016} 2017 2018static void make_clear_dte(struct iommu_dev_data *dev_data, struct dev_table_entry *ptr, 2019 struct dev_table_entry *new) 2020{ 2021 /* All existing DTE must have V bit set */ 2022 new->data128[0] = DTE_FLAG_V; 2023 new->data128[1] = 0; 2024} 2025 2026/* 2027 * Note: 2028 * The old value for GCR3 table and GPT have been cleared from caller. 2029 */ 2030static void set_dte_gcr3_table(struct amd_iommu *iommu, 2031 struct iommu_dev_data *dev_data, 2032 struct dev_table_entry *target) 2033{ 2034 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2035 u64 gcr3; 2036 2037 if (!gcr3_info->gcr3_tbl) 2038 return; 2039 2040 pr_debug("%s: devid=%#x, glx=%#x, gcr3_tbl=%#llx\n", 2041 __func__, dev_data->devid, gcr3_info->glx, 2042 (unsigned long long)gcr3_info->gcr3_tbl); 2043 2044 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl); 2045 2046 target->data[0] |= DTE_FLAG_GV | 2047 FIELD_PREP(DTE_GLX, gcr3_info->glx) | 2048 FIELD_PREP(DTE_GCR3_14_12, gcr3 >> 12); 2049 if (pdom_is_v2_pgtbl_mode(dev_data->domain)) 2050 target->data[0] |= DTE_FLAG_GIOV; 2051 2052 target->data[1] |= FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) | 2053 FIELD_PREP(DTE_GCR3_51_31, gcr3 >> 31); 2054 2055 /* Guest page table can only support 4 and 5 levels */ 2056 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) 2057 target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL); 2058 else 2059 target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL); 2060} 2061 2062static void set_dte_entry(struct amd_iommu *iommu, 2063 struct iommu_dev_data *dev_data, 2064 phys_addr_t top_paddr, unsigned int top_level) 2065{ 2066 u16 domid; 2067 u32 old_domid; 2068 struct dev_table_entry *initial_dte; 2069 struct dev_table_entry new = {}; 2070 struct protection_domain *domain = dev_data->domain; 2071 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2072 struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid]; 2073 struct pt_iommu_amdv1_hw_info pt_info; 2074 2075 make_clear_dte(dev_data, dte, &new); 2076 2077 if (gcr3_info && gcr3_info->gcr3_tbl) 2078 domid = dev_data->gcr3_info.domid; 2079 else { 2080 domid = domain->id; 2081 2082 if (domain->domain.type & __IOMMU_DOMAIN_PAGING) { 2083 /* 2084 * When updating the IO pagetable, the new top and level 2085 * are provided as parameters. For other operations i.e. 2086 * device attach, retrieve the current pagetable info 2087 * via the IOMMU PT API. 2088 */ 2089 if (top_paddr) { 2090 pt_info.host_pt_root = top_paddr; 2091 pt_info.mode = top_level + 1; 2092 } else { 2093 WARN_ON(top_paddr || top_level); 2094 pt_iommu_amdv1_hw_info(&domain->amdv1, 2095 &pt_info); 2096 } 2097 2098 new.data[0] |= __sme_set(pt_info.host_pt_root) | 2099 (pt_info.mode & DEV_ENTRY_MODE_MASK) 2100 << DEV_ENTRY_MODE_SHIFT; 2101 } 2102 } 2103 2104 new.data[0] |= DTE_FLAG_IR | DTE_FLAG_IW; 2105 2106 /* 2107 * When SNP is enabled, we can only support TV=1 with non-zero domain ID. 2108 * This is prevented by the SNP-enable and IOMMU_DOMAIN_IDENTITY check in 2109 * do_iommu_domain_alloc(). 2110 */ 2111 WARN_ON(amd_iommu_snp_en && (domid == 0)); 2112 new.data[0] |= DTE_FLAG_TV; 2113 2114 if (dev_data->ppr) 2115 new.data[0] |= 1ULL << DEV_ENTRY_PPR; 2116 2117 if (domain->dirty_tracking) 2118 new.data[0] |= DTE_FLAG_HAD; 2119 2120 if (dev_data->ats_enabled) 2121 new.data[1] |= DTE_FLAG_IOTLB; 2122 2123 old_domid = READ_ONCE(dte->data[1]) & DEV_DOMID_MASK; 2124 new.data[1] |= domid; 2125 2126 /* 2127 * Restore cached persistent DTE bits, which can be set by information 2128 * in IVRS table. See set_dev_entry_from_acpi(). 2129 */ 2130 initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid); 2131 if (initial_dte) { 2132 new.data128[0] |= initial_dte->data128[0]; 2133 new.data128[1] |= initial_dte->data128[1]; 2134 } 2135 2136 set_dte_gcr3_table(iommu, dev_data, &new); 2137 2138 update_dte256(iommu, dev_data, &new); 2139 2140 /* 2141 * A kdump kernel might be replacing a domain ID that was copied from 2142 * the previous kernel--if so, it needs to flush the translation cache 2143 * entries for the old domain ID that is being overwritten 2144 */ 2145 if (old_domid) { 2146 amd_iommu_flush_tlb_domid(iommu, old_domid); 2147 } 2148} 2149 2150/* 2151 * Clear DMA-remap related flags to block all DMA (blockeded domain) 2152 */ 2153static void clear_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data) 2154{ 2155 struct dev_table_entry new = {}; 2156 struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid]; 2157 2158 make_clear_dte(dev_data, dte, &new); 2159 update_dte256(iommu, dev_data, &new); 2160} 2161 2162/* Update and flush DTE for the given device */ 2163static void dev_update_dte(struct iommu_dev_data *dev_data, bool set) 2164{ 2165 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); 2166 2167 if (set) 2168 set_dte_entry(iommu, dev_data, 0, 0); 2169 else 2170 clear_dte_entry(iommu, dev_data); 2171 2172 clone_aliases(iommu, dev_data->dev); 2173 device_flush_dte(dev_data); 2174 iommu_completion_wait(iommu); 2175} 2176 2177/* 2178 * If domain is SVA capable then initialize GCR3 table. Also if domain is 2179 * in v2 page table mode then update GCR3[0]. 2180 */ 2181static int init_gcr3_table(struct iommu_dev_data *dev_data, 2182 struct protection_domain *pdom) 2183{ 2184 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2185 int max_pasids = dev_data->max_pasids; 2186 struct pt_iommu_x86_64_hw_info pt_info; 2187 int ret = 0; 2188 2189 /* 2190 * If domain is in pt mode then setup GCR3 table only if device 2191 * is PASID capable 2192 */ 2193 if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data)) 2194 return ret; 2195 2196 /* 2197 * By default, setup GCR3 table to support MAX PASIDs 2198 * supported by the device/IOMMU. 2199 */ 2200 ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, 2201 max_pasids > 0 ? max_pasids : 1); 2202 if (ret) 2203 return ret; 2204 2205 /* Setup GCR3[0] only if domain is setup with v2 page table mode */ 2206 if (!pdom_is_v2_pgtbl_mode(pdom)) 2207 return ret; 2208 2209 pt_iommu_x86_64_hw_info(&pdom->amdv2, &pt_info); 2210 ret = update_gcr3(dev_data, 0, __sme_set(pt_info.gcr3_pt), true); 2211 if (ret) 2212 free_gcr3_table(&dev_data->gcr3_info); 2213 2214 return ret; 2215} 2216 2217static void destroy_gcr3_table(struct iommu_dev_data *dev_data, 2218 struct protection_domain *pdom) 2219{ 2220 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; 2221 2222 if (pdom_is_v2_pgtbl_mode(pdom)) 2223 update_gcr3(dev_data, 0, 0, false); 2224 2225 if (gcr3_info->gcr3_tbl == NULL) 2226 return; 2227 2228 free_gcr3_table(gcr3_info); 2229} 2230 2231static int pdom_attach_iommu(struct amd_iommu *iommu, 2232 struct protection_domain *pdom) 2233{ 2234 struct pdom_iommu_info *pdom_iommu_info, *curr; 2235 unsigned long flags; 2236 int ret = 0; 2237 2238 spin_lock_irqsave(&pdom->lock, flags); 2239 2240 pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index); 2241 if (pdom_iommu_info) { 2242 pdom_iommu_info->refcnt++; 2243 goto out_unlock; 2244 } 2245 2246 pdom_iommu_info = kzalloc(sizeof(*pdom_iommu_info), GFP_ATOMIC); 2247 if (!pdom_iommu_info) { 2248 ret = -ENOMEM; 2249 goto out_unlock; 2250 } 2251 2252 pdom_iommu_info->iommu = iommu; 2253 pdom_iommu_info->refcnt = 1; 2254 2255 curr = xa_cmpxchg(&pdom->iommu_array, iommu->index, 2256 NULL, pdom_iommu_info, GFP_ATOMIC); 2257 if (curr) { 2258 kfree(pdom_iommu_info); 2259 ret = -ENOSPC; 2260 goto out_unlock; 2261 } 2262 2263out_unlock: 2264 spin_unlock_irqrestore(&pdom->lock, flags); 2265 return ret; 2266} 2267 2268static void pdom_detach_iommu(struct amd_iommu *iommu, 2269 struct protection_domain *pdom) 2270{ 2271 struct pdom_iommu_info *pdom_iommu_info; 2272 unsigned long flags; 2273 2274 spin_lock_irqsave(&pdom->lock, flags); 2275 2276 pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index); 2277 if (!pdom_iommu_info) { 2278 spin_unlock_irqrestore(&pdom->lock, flags); 2279 return; 2280 } 2281 2282 pdom_iommu_info->refcnt--; 2283 if (pdom_iommu_info->refcnt == 0) { 2284 xa_erase(&pdom->iommu_array, iommu->index); 2285 kfree(pdom_iommu_info); 2286 } 2287 2288 spin_unlock_irqrestore(&pdom->lock, flags); 2289} 2290 2291/* 2292 * If a device is not yet associated with a domain, this function makes the 2293 * device visible in the domain 2294 */ 2295static int attach_device(struct device *dev, 2296 struct protection_domain *domain) 2297{ 2298 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2299 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2300 struct pci_dev *pdev; 2301 unsigned long flags; 2302 int ret = 0; 2303 2304 mutex_lock(&dev_data->mutex); 2305 2306 if (dev_data->domain != NULL) { 2307 ret = -EBUSY; 2308 goto out; 2309 } 2310 2311 /* Do reference counting */ 2312 ret = pdom_attach_iommu(iommu, domain); 2313 if (ret) 2314 goto out; 2315 2316 /* Setup GCR3 table */ 2317 if (pdom_is_sva_capable(domain)) { 2318 ret = init_gcr3_table(dev_data, domain); 2319 if (ret) { 2320 pdom_detach_iommu(iommu, domain); 2321 goto out; 2322 } 2323 } 2324 2325 pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL; 2326 if (pdev && pdom_is_sva_capable(domain)) { 2327 pdev_enable_caps(pdev); 2328 2329 /* 2330 * Device can continue to function even if IOPF 2331 * enablement failed. Hence in error path just 2332 * disable device PRI support. 2333 */ 2334 if (amd_iommu_iopf_add_device(iommu, dev_data)) 2335 pdev_disable_cap_pri(pdev); 2336 } else if (pdev) { 2337 pdev_enable_cap_ats(pdev); 2338 } 2339 2340 /* Update data structures */ 2341 dev_data->domain = domain; 2342 spin_lock_irqsave(&domain->lock, flags); 2343 list_add(&dev_data->list, &domain->dev_list); 2344 spin_unlock_irqrestore(&domain->lock, flags); 2345 2346 /* Update device table */ 2347 dev_update_dte(dev_data, true); 2348 2349out: 2350 mutex_unlock(&dev_data->mutex); 2351 2352 return ret; 2353} 2354 2355/* 2356 * Removes a device from a protection domain (with devtable_lock held) 2357 */ 2358static void detach_device(struct device *dev) 2359{ 2360 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2361 struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data); 2362 struct protection_domain *domain = dev_data->domain; 2363 unsigned long flags; 2364 2365 mutex_lock(&dev_data->mutex); 2366 2367 /* 2368 * First check if the device is still attached. It might already 2369 * be detached from its domain because the generic 2370 * iommu_detach_group code detached it and we try again here in 2371 * our alias handling. 2372 */ 2373 if (WARN_ON(!dev_data->domain)) 2374 goto out; 2375 2376 /* Remove IOPF handler */ 2377 if (dev_data->ppr) { 2378 iopf_queue_flush_dev(dev); 2379 amd_iommu_iopf_remove_device(iommu, dev_data); 2380 } 2381 2382 if (dev_is_pci(dev)) 2383 pdev_disable_caps(to_pci_dev(dev)); 2384 2385 /* Clear DTE and flush the entry */ 2386 dev_update_dte(dev_data, false); 2387 2388 /* Flush IOTLB and wait for the flushes to finish */ 2389 spin_lock_irqsave(&domain->lock, flags); 2390 amd_iommu_domain_flush_all(domain); 2391 list_del(&dev_data->list); 2392 spin_unlock_irqrestore(&domain->lock, flags); 2393 2394 /* Clear GCR3 table */ 2395 if (pdom_is_sva_capable(domain)) 2396 destroy_gcr3_table(dev_data, domain); 2397 2398 /* Update data structures */ 2399 dev_data->domain = NULL; 2400 2401 /* decrease reference counters - needs to happen after the flushes */ 2402 pdom_detach_iommu(iommu, domain); 2403 2404out: 2405 mutex_unlock(&dev_data->mutex); 2406} 2407 2408static struct iommu_device *amd_iommu_probe_device(struct device *dev) 2409{ 2410 struct iommu_device *iommu_dev; 2411 struct amd_iommu *iommu; 2412 struct iommu_dev_data *dev_data; 2413 int ret; 2414 2415 if (!check_device(dev)) 2416 return ERR_PTR(-ENODEV); 2417 2418 iommu = rlookup_amd_iommu(dev); 2419 if (!iommu) 2420 return ERR_PTR(-ENODEV); 2421 2422 /* Not registered yet? */ 2423 if (!iommu->iommu.ops) 2424 return ERR_PTR(-ENODEV); 2425 2426 if (dev_iommu_priv_get(dev)) 2427 return &iommu->iommu; 2428 2429 ret = iommu_init_device(iommu, dev); 2430 if (ret) { 2431 dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); 2432 iommu_dev = ERR_PTR(ret); 2433 iommu_ignore_device(iommu, dev); 2434 goto out_err; 2435 } 2436 2437 amd_iommu_set_pci_msi_domain(dev, iommu); 2438 iommu_dev = &iommu->iommu; 2439 2440 /* 2441 * If IOMMU and device supports PASID then it will contain max 2442 * supported PASIDs, else it will be zero. 2443 */ 2444 dev_data = dev_iommu_priv_get(dev); 2445 if (amd_iommu_pasid_supported() && dev_is_pci(dev) && 2446 pdev_pasid_supported(dev_data)) { 2447 dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, 2448 pci_max_pasids(to_pci_dev(dev))); 2449 } 2450 2451 if (amd_iommu_pgtable == PD_MODE_NONE) { 2452 pr_warn_once("%s: DMA translation not supported by iommu.\n", 2453 __func__); 2454 iommu_dev = ERR_PTR(-ENODEV); 2455 goto out_err; 2456 } 2457 2458 iommu_completion_wait(iommu); 2459 2460 if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) 2461 dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K; 2462 else 2463 dev_data->max_irqs = MAX_IRQS_PER_TABLE_512; 2464 2465 if (dev_is_pci(dev)) 2466 pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT); 2467 2468out_err: 2469 return iommu_dev; 2470} 2471 2472static void amd_iommu_release_device(struct device *dev) 2473{ 2474 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2475 2476 WARN_ON(dev_data->domain); 2477 2478 /* 2479 * We keep dev_data around for unplugged devices and reuse it when the 2480 * device is re-plugged - not doing so would introduce a ton of races. 2481 */ 2482} 2483 2484static struct iommu_group *amd_iommu_device_group(struct device *dev) 2485{ 2486 if (dev_is_pci(dev)) 2487 return pci_device_group(dev); 2488 2489 return acpihid_device_group(dev); 2490} 2491 2492/***************************************************************************** 2493 * 2494 * The following functions belong to the exported interface of AMD IOMMU 2495 * 2496 * This interface allows access to lower level functions of the IOMMU 2497 * like protection domain handling and assignement of devices to domains 2498 * which is not possible with the dma_ops interface. 2499 * 2500 *****************************************************************************/ 2501 2502static void protection_domain_init(struct protection_domain *domain) 2503{ 2504 spin_lock_init(&domain->lock); 2505 INIT_LIST_HEAD(&domain->dev_list); 2506 INIT_LIST_HEAD(&domain->dev_data_list); 2507 xa_init(&domain->iommu_array); 2508} 2509 2510struct protection_domain *protection_domain_alloc(void) 2511{ 2512 struct protection_domain *domain; 2513 int domid; 2514 2515 domain = kzalloc(sizeof(*domain), GFP_KERNEL); 2516 if (!domain) 2517 return NULL; 2518 2519 domid = amd_iommu_pdom_id_alloc(); 2520 if (domid <= 0) { 2521 kfree(domain); 2522 return NULL; 2523 } 2524 domain->id = domid; 2525 2526 protection_domain_init(domain); 2527 2528 return domain; 2529} 2530 2531static bool amd_iommu_hd_support(struct amd_iommu *iommu) 2532{ 2533 if (amd_iommu_hatdis) 2534 return false; 2535 2536 return iommu && (iommu->features & FEATURE_HDSUP); 2537} 2538 2539static spinlock_t *amd_iommu_get_top_lock(struct pt_iommu *iommupt) 2540{ 2541 struct protection_domain *pdom = 2542 container_of(iommupt, struct protection_domain, iommu); 2543 2544 return &pdom->lock; 2545} 2546 2547/* 2548 * Update all HW references to the domain with a new pgtable configuration. 2549 */ 2550static void amd_iommu_change_top(struct pt_iommu *iommu_table, 2551 phys_addr_t top_paddr, unsigned int top_level) 2552{ 2553 struct protection_domain *pdom = 2554 container_of(iommu_table, struct protection_domain, iommu); 2555 struct iommu_dev_data *dev_data; 2556 2557 lockdep_assert_held(&pdom->lock); 2558 2559 /* Update the DTE for all devices attached to this domain */ 2560 list_for_each_entry(dev_data, &pdom->dev_list, list) { 2561 struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); 2562 2563 /* Update the HW references with the new level and top ptr */ 2564 set_dte_entry(iommu, dev_data, top_paddr, top_level); 2565 clone_aliases(iommu, dev_data->dev); 2566 } 2567 2568 list_for_each_entry(dev_data, &pdom->dev_list, list) 2569 device_flush_dte(dev_data); 2570 2571 domain_flush_complete(pdom); 2572} 2573 2574/* 2575 * amd_iommu_iotlb_sync_map() is used to generate flushes for non-present to 2576 * present (ie mapping) operations. It is a NOP if the IOMMU doesn't have non 2577 * present caching (like hypervisor shadowing). 2578 */ 2579static int amd_iommu_iotlb_sync_map(struct iommu_domain *dom, 2580 unsigned long iova, size_t size) 2581{ 2582 struct protection_domain *domain = to_pdomain(dom); 2583 unsigned long flags; 2584 2585 if (likely(!amd_iommu_np_cache)) 2586 return 0; 2587 2588 spin_lock_irqsave(&domain->lock, flags); 2589 amd_iommu_domain_flush_pages(domain, iova, size); 2590 spin_unlock_irqrestore(&domain->lock, flags); 2591 return 0; 2592} 2593 2594static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain) 2595{ 2596 struct protection_domain *dom = to_pdomain(domain); 2597 unsigned long flags; 2598 2599 spin_lock_irqsave(&dom->lock, flags); 2600 amd_iommu_domain_flush_all(dom); 2601 spin_unlock_irqrestore(&dom->lock, flags); 2602} 2603 2604static void amd_iommu_iotlb_sync(struct iommu_domain *domain, 2605 struct iommu_iotlb_gather *gather) 2606{ 2607 struct protection_domain *dom = to_pdomain(domain); 2608 unsigned long flags; 2609 2610 spin_lock_irqsave(&dom->lock, flags); 2611 amd_iommu_domain_flush_pages(dom, gather->start, 2612 gather->end - gather->start + 1); 2613 spin_unlock_irqrestore(&dom->lock, flags); 2614 iommu_put_pages_list(&gather->freelist); 2615} 2616 2617static const struct pt_iommu_driver_ops amd_hw_driver_ops_v1 = { 2618 .get_top_lock = amd_iommu_get_top_lock, 2619 .change_top = amd_iommu_change_top, 2620}; 2621 2622static const struct iommu_domain_ops amdv1_ops = { 2623 IOMMU_PT_DOMAIN_OPS(amdv1), 2624 .iotlb_sync_map = amd_iommu_iotlb_sync_map, 2625 .flush_iotlb_all = amd_iommu_flush_iotlb_all, 2626 .iotlb_sync = amd_iommu_iotlb_sync, 2627 .attach_dev = amd_iommu_attach_device, 2628 .free = amd_iommu_domain_free, 2629 .enforce_cache_coherency = amd_iommu_enforce_cache_coherency, 2630}; 2631 2632static const struct iommu_dirty_ops amdv1_dirty_ops = { 2633 IOMMU_PT_DIRTY_OPS(amdv1), 2634 .set_dirty_tracking = amd_iommu_set_dirty_tracking, 2635}; 2636 2637static struct iommu_domain *amd_iommu_domain_alloc_paging_v1(struct device *dev, 2638 u32 flags) 2639{ 2640 struct pt_iommu_amdv1_cfg cfg = {}; 2641 struct protection_domain *domain; 2642 int ret; 2643 2644 if (amd_iommu_hatdis) 2645 return ERR_PTR(-EOPNOTSUPP); 2646 2647 domain = protection_domain_alloc(); 2648 if (!domain) 2649 return ERR_PTR(-ENOMEM); 2650 2651 domain->pd_mode = PD_MODE_V1; 2652 domain->iommu.driver_ops = &amd_hw_driver_ops_v1; 2653 domain->iommu.nid = dev_to_node(dev); 2654 if (flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING) 2655 domain->domain.dirty_ops = &amdv1_dirty_ops; 2656 2657 /* 2658 * Someday FORCE_COHERENCE should be set by 2659 * amd_iommu_enforce_cache_coherency() like VT-d does. 2660 */ 2661 cfg.common.features = BIT(PT_FEAT_DYNAMIC_TOP) | 2662 BIT(PT_FEAT_AMDV1_ENCRYPT_TABLES) | 2663 BIT(PT_FEAT_AMDV1_FORCE_COHERENCE); 2664 2665 /* 2666 * AMD's IOMMU can flush as many pages as necessary in a single flush. 2667 * Unless we run in a virtual machine, which can be inferred according 2668 * to whether "non-present cache" is on, it is probably best to prefer 2669 * (potentially) too extensive TLB flushing (i.e., more misses) over 2670 * multiple TLB flushes (i.e., more flushes). For virtual machines the 2671 * hypervisor needs to synchronize the host IOMMU PTEs with those of 2672 * the guest, and the trade-off is different: unnecessary TLB flushes 2673 * should be avoided. 2674 */ 2675 if (amd_iommu_np_cache) 2676 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE_NO_GAPS); 2677 else 2678 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE); 2679 2680 cfg.common.hw_max_vasz_lg2 = 2681 min(64, (amd_iommu_hpt_level - 1) * 9 + 21); 2682 cfg.common.hw_max_oasz_lg2 = 52; 2683 cfg.starting_level = 2; 2684 domain->domain.ops = &amdv1_ops; 2685 2686 ret = pt_iommu_amdv1_init(&domain->amdv1, &cfg, GFP_KERNEL); 2687 if (ret) { 2688 amd_iommu_domain_free(&domain->domain); 2689 return ERR_PTR(ret); 2690 } 2691 2692 /* 2693 * Narrow the supported page sizes to those selected by the kernel 2694 * command line. 2695 */ 2696 domain->domain.pgsize_bitmap &= amd_iommu_pgsize_bitmap; 2697 return &domain->domain; 2698} 2699 2700static const struct iommu_domain_ops amdv2_ops = { 2701 IOMMU_PT_DOMAIN_OPS(x86_64), 2702 .iotlb_sync_map = amd_iommu_iotlb_sync_map, 2703 .flush_iotlb_all = amd_iommu_flush_iotlb_all, 2704 .iotlb_sync = amd_iommu_iotlb_sync, 2705 .attach_dev = amd_iommu_attach_device, 2706 .free = amd_iommu_domain_free, 2707 /* 2708 * Note the AMDv2 page table format does not support a Force Coherency 2709 * bit, so enforce_cache_coherency should not be set. However VFIO is 2710 * not prepared to handle a case where some domains will support 2711 * enforcement and others do not. VFIO and iommufd will have to be fixed 2712 * before it can fully use the V2 page table. See the comment in 2713 * iommufd_hwpt_paging_alloc(). For now leave things as they have 2714 * historically been and lie about enforce_cache_coherencey. 2715 */ 2716 .enforce_cache_coherency = amd_iommu_enforce_cache_coherency, 2717}; 2718 2719static struct iommu_domain *amd_iommu_domain_alloc_paging_v2(struct device *dev, 2720 u32 flags) 2721{ 2722 struct pt_iommu_x86_64_cfg cfg = {}; 2723 struct protection_domain *domain; 2724 int ret; 2725 2726 if (!amd_iommu_v2_pgtbl_supported()) 2727 return ERR_PTR(-EOPNOTSUPP); 2728 2729 domain = protection_domain_alloc(); 2730 if (!domain) 2731 return ERR_PTR(-ENOMEM); 2732 2733 domain->pd_mode = PD_MODE_V2; 2734 domain->iommu.nid = dev_to_node(dev); 2735 2736 cfg.common.features = BIT(PT_FEAT_X86_64_AMD_ENCRYPT_TABLES); 2737 if (amd_iommu_np_cache) 2738 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE_NO_GAPS); 2739 else 2740 cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE); 2741 2742 /* 2743 * The v2 table behaves differently if it is attached to PASID 0 vs a 2744 * non-zero PASID. On PASID 0 it has no sign extension and the full 2745 * 57/48 bits decode the lower addresses. Otherwise it behaves like a 2746 * normal sign extended x86 page table. Since we want the domain to work 2747 * in both modes the top bit is removed and PT_FEAT_SIGN_EXTEND is not 2748 * set which creates a table that is compatible in both modes. 2749 */ 2750 if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) { 2751 cfg.common.hw_max_vasz_lg2 = 56; 2752 cfg.top_level = 4; 2753 } else { 2754 cfg.common.hw_max_vasz_lg2 = 47; 2755 cfg.top_level = 3; 2756 } 2757 cfg.common.hw_max_oasz_lg2 = 52; 2758 domain->domain.ops = &amdv2_ops; 2759 2760 ret = pt_iommu_x86_64_init(&domain->amdv2, &cfg, GFP_KERNEL); 2761 if (ret) { 2762 amd_iommu_domain_free(&domain->domain); 2763 return ERR_PTR(ret); 2764 } 2765 return &domain->domain; 2766} 2767 2768static struct iommu_domain * 2769amd_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags, 2770 const struct iommu_user_data *user_data) 2771 2772{ 2773 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 2774 const u32 supported_flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING | 2775 IOMMU_HWPT_ALLOC_PASID; 2776 2777 if ((flags & ~supported_flags) || user_data) 2778 return ERR_PTR(-EOPNOTSUPP); 2779 2780 switch (flags & supported_flags) { 2781 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING: 2782 /* Allocate domain with v1 page table for dirty tracking */ 2783 if (!amd_iommu_hd_support(iommu)) 2784 break; 2785 return amd_iommu_domain_alloc_paging_v1(dev, flags); 2786 case IOMMU_HWPT_ALLOC_PASID: 2787 /* Allocate domain with v2 page table if IOMMU supports PASID. */ 2788 if (!amd_iommu_pasid_supported()) 2789 break; 2790 return amd_iommu_domain_alloc_paging_v2(dev, flags); 2791 case 0: { 2792 struct iommu_domain *ret; 2793 2794 /* If nothing specific is required use the kernel commandline default */ 2795 if (amd_iommu_pgtable == PD_MODE_V1) { 2796 ret = amd_iommu_domain_alloc_paging_v1(dev, flags); 2797 if (ret != ERR_PTR(-EOPNOTSUPP)) 2798 return ret; 2799 return amd_iommu_domain_alloc_paging_v2(dev, flags); 2800 } 2801 ret = amd_iommu_domain_alloc_paging_v2(dev, flags); 2802 if (ret != ERR_PTR(-EOPNOTSUPP)) 2803 return ret; 2804 return amd_iommu_domain_alloc_paging_v1(dev, flags); 2805 } 2806 default: 2807 break; 2808 } 2809 return ERR_PTR(-EOPNOTSUPP); 2810} 2811 2812void amd_iommu_domain_free(struct iommu_domain *dom) 2813{ 2814 struct protection_domain *domain = to_pdomain(dom); 2815 2816 WARN_ON(!list_empty(&domain->dev_list)); 2817 pt_iommu_deinit(&domain->iommu); 2818 amd_iommu_pdom_id_free(domain->id); 2819 kfree(domain); 2820} 2821 2822static int blocked_domain_attach_device(struct iommu_domain *domain, 2823 struct device *dev, 2824 struct iommu_domain *old) 2825{ 2826 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2827 2828 if (dev_data->domain) 2829 detach_device(dev); 2830 2831 /* Clear DTE and flush the entry */ 2832 mutex_lock(&dev_data->mutex); 2833 dev_update_dte(dev_data, false); 2834 mutex_unlock(&dev_data->mutex); 2835 2836 return 0; 2837} 2838 2839static int blocked_domain_set_dev_pasid(struct iommu_domain *domain, 2840 struct device *dev, ioasid_t pasid, 2841 struct iommu_domain *old) 2842{ 2843 amd_iommu_remove_dev_pasid(dev, pasid, old); 2844 return 0; 2845} 2846 2847static struct iommu_domain blocked_domain = { 2848 .type = IOMMU_DOMAIN_BLOCKED, 2849 .ops = &(const struct iommu_domain_ops) { 2850 .attach_dev = blocked_domain_attach_device, 2851 .set_dev_pasid = blocked_domain_set_dev_pasid, 2852 } 2853}; 2854 2855static struct protection_domain identity_domain; 2856 2857static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev, 2858 struct iommu_domain *old) 2859{ 2860 /* 2861 * Don't allow attaching a device to the identity domain if SNP is 2862 * enabled. 2863 */ 2864 if (amd_iommu_snp_en) 2865 return -EINVAL; 2866 2867 return amd_iommu_attach_device(dom, dev, old); 2868} 2869 2870static const struct iommu_domain_ops identity_domain_ops = { 2871 .attach_dev = amd_iommu_identity_attach, 2872}; 2873 2874void amd_iommu_init_identity_domain(void) 2875{ 2876 struct iommu_domain *domain = &identity_domain.domain; 2877 2878 domain->type = IOMMU_DOMAIN_IDENTITY; 2879 domain->ops = &identity_domain_ops; 2880 domain->owner = &amd_iommu_ops; 2881 2882 identity_domain.id = amd_iommu_pdom_id_alloc(); 2883 2884 protection_domain_init(&identity_domain); 2885} 2886 2887static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev, 2888 struct iommu_domain *old) 2889{ 2890 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 2891 struct protection_domain *domain = to_pdomain(dom); 2892 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 2893 int ret; 2894 2895 /* 2896 * Skip attach device to domain if new domain is same as 2897 * devices current domain 2898 */ 2899 if (dev_data->domain == domain) 2900 return 0; 2901 2902 dev_data->defer_attach = false; 2903 2904 /* 2905 * Restrict to devices with compatible IOMMU hardware support 2906 * when enforcement of dirty tracking is enabled. 2907 */ 2908 if (dom->dirty_ops && !amd_iommu_hd_support(iommu)) 2909 return -EINVAL; 2910 2911 if (dev_data->domain) 2912 detach_device(dev); 2913 2914 ret = attach_device(dev, domain); 2915 2916#ifdef CONFIG_IRQ_REMAP 2917 if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) { 2918 if (dom->type == IOMMU_DOMAIN_UNMANAGED) 2919 dev_data->use_vapic = 1; 2920 else 2921 dev_data->use_vapic = 0; 2922 } 2923#endif 2924 2925 return ret; 2926} 2927 2928static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) 2929{ 2930 switch (cap) { 2931 case IOMMU_CAP_CACHE_COHERENCY: 2932 return true; 2933 case IOMMU_CAP_NOEXEC: 2934 return false; 2935 case IOMMU_CAP_PRE_BOOT_PROTECTION: 2936 return amdr_ivrs_remap_support; 2937 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: 2938 return true; 2939 case IOMMU_CAP_DEFERRED_FLUSH: 2940 return true; 2941 case IOMMU_CAP_DIRTY_TRACKING: { 2942 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); 2943 2944 return amd_iommu_hd_support(iommu); 2945 } 2946 default: 2947 break; 2948 } 2949 2950 return false; 2951} 2952 2953static int amd_iommu_set_dirty_tracking(struct iommu_domain *domain, 2954 bool enable) 2955{ 2956 struct protection_domain *pdomain = to_pdomain(domain); 2957 struct dev_table_entry *dte; 2958 struct iommu_dev_data *dev_data; 2959 bool domain_flush = false; 2960 struct amd_iommu *iommu; 2961 unsigned long flags; 2962 u64 new; 2963 2964 spin_lock_irqsave(&pdomain->lock, flags); 2965 if (!(pdomain->dirty_tracking ^ enable)) { 2966 spin_unlock_irqrestore(&pdomain->lock, flags); 2967 return 0; 2968 } 2969 2970 list_for_each_entry(dev_data, &pdomain->dev_list, list) { 2971 spin_lock(&dev_data->dte_lock); 2972 iommu = get_amd_iommu_from_dev_data(dev_data); 2973 dte = &get_dev_table(iommu)[dev_data->devid]; 2974 new = dte->data[0]; 2975 new = (enable ? new | DTE_FLAG_HAD : new & ~DTE_FLAG_HAD); 2976 dte->data[0] = new; 2977 spin_unlock(&dev_data->dte_lock); 2978 2979 /* Flush device DTE */ 2980 device_flush_dte(dev_data); 2981 domain_flush = true; 2982 } 2983 2984 /* Flush IOTLB to mark IOPTE dirty on the next translation(s) */ 2985 if (domain_flush) 2986 amd_iommu_domain_flush_all(pdomain); 2987 2988 pdomain->dirty_tracking = enable; 2989 spin_unlock_irqrestore(&pdomain->lock, flags); 2990 2991 return 0; 2992} 2993 2994static void amd_iommu_get_resv_regions(struct device *dev, 2995 struct list_head *head) 2996{ 2997 struct iommu_resv_region *region; 2998 struct unity_map_entry *entry; 2999 struct amd_iommu *iommu; 3000 struct amd_iommu_pci_seg *pci_seg; 3001 int devid, sbdf; 3002 3003 sbdf = get_device_sbdf_id(dev); 3004 if (sbdf < 0) 3005 return; 3006 3007 devid = PCI_SBDF_TO_DEVID(sbdf); 3008 iommu = get_amd_iommu_from_dev(dev); 3009 pci_seg = iommu->pci_seg; 3010 3011 list_for_each_entry(entry, &pci_seg->unity_map, list) { 3012 int type, prot = 0; 3013 size_t length; 3014 3015 if (devid < entry->devid_start || devid > entry->devid_end) 3016 continue; 3017 3018 type = IOMMU_RESV_DIRECT; 3019 length = entry->address_end - entry->address_start; 3020 if (entry->prot & IOMMU_PROT_IR) 3021 prot |= IOMMU_READ; 3022 if (entry->prot & IOMMU_PROT_IW) 3023 prot |= IOMMU_WRITE; 3024 if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE) 3025 /* Exclusion range */ 3026 type = IOMMU_RESV_RESERVED; 3027 3028 region = iommu_alloc_resv_region(entry->address_start, 3029 length, prot, type, 3030 GFP_KERNEL); 3031 if (!region) { 3032 dev_err(dev, "Out of memory allocating dm-regions\n"); 3033 return; 3034 } 3035 list_add_tail(®ion->list, head); 3036 } 3037 3038 region = iommu_alloc_resv_region(MSI_RANGE_START, 3039 MSI_RANGE_END - MSI_RANGE_START + 1, 3040 0, IOMMU_RESV_MSI, GFP_KERNEL); 3041 if (!region) 3042 return; 3043 list_add_tail(®ion->list, head); 3044 3045 if (amd_iommu_ht_range_ignore()) 3046 return; 3047 3048 region = iommu_alloc_resv_region(HT_RANGE_START, 3049 HT_RANGE_END - HT_RANGE_START + 1, 3050 0, IOMMU_RESV_RESERVED, GFP_KERNEL); 3051 if (!region) 3052 return; 3053 list_add_tail(®ion->list, head); 3054} 3055 3056static bool amd_iommu_is_attach_deferred(struct device *dev) 3057{ 3058 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 3059 3060 return dev_data->defer_attach; 3061} 3062 3063static int amd_iommu_def_domain_type(struct device *dev) 3064{ 3065 struct iommu_dev_data *dev_data; 3066 3067 dev_data = dev_iommu_priv_get(dev); 3068 if (!dev_data) 3069 return 0; 3070 3071 /* Always use DMA domain for untrusted device */ 3072 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) 3073 return IOMMU_DOMAIN_DMA; 3074 3075 /* 3076 * Do not identity map IOMMUv2 capable devices when: 3077 * - memory encryption is active, because some of those devices 3078 * (AMD GPUs) don't have the encryption bit in their DMA-mask 3079 * and require remapping. 3080 * - SNP is enabled, because it prohibits DTE[Mode]=0. 3081 */ 3082 if (pdev_pasid_supported(dev_data) && 3083 !cc_platform_has(CC_ATTR_MEM_ENCRYPT) && 3084 !amd_iommu_snp_en) { 3085 return IOMMU_DOMAIN_IDENTITY; 3086 } 3087 3088 return 0; 3089} 3090 3091static bool amd_iommu_enforce_cache_coherency(struct iommu_domain *domain) 3092{ 3093 /* IOMMU_PTE_FC is always set */ 3094 return true; 3095} 3096 3097const struct iommu_ops amd_iommu_ops = { 3098 .capable = amd_iommu_capable, 3099 .blocked_domain = &blocked_domain, 3100 .release_domain = &blocked_domain, 3101 .identity_domain = &identity_domain.domain, 3102 .domain_alloc_paging_flags = amd_iommu_domain_alloc_paging_flags, 3103 .domain_alloc_sva = amd_iommu_domain_alloc_sva, 3104 .probe_device = amd_iommu_probe_device, 3105 .release_device = amd_iommu_release_device, 3106 .device_group = amd_iommu_device_group, 3107 .get_resv_regions = amd_iommu_get_resv_regions, 3108 .is_attach_deferred = amd_iommu_is_attach_deferred, 3109 .def_domain_type = amd_iommu_def_domain_type, 3110 .page_response = amd_iommu_page_response, 3111}; 3112 3113#ifdef CONFIG_IRQ_REMAP 3114 3115/***************************************************************************** 3116 * 3117 * Interrupt Remapping Implementation 3118 * 3119 *****************************************************************************/ 3120 3121static struct irq_chip amd_ir_chip; 3122static DEFINE_SPINLOCK(iommu_table_lock); 3123 3124static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) 3125{ 3126 int ret; 3127 u64 data; 3128 unsigned long flags; 3129 struct iommu_cmd cmd, cmd2; 3130 3131 if (iommu->irtcachedis_enabled) 3132 return; 3133 3134 build_inv_irt(&cmd, devid); 3135 3136 raw_spin_lock_irqsave(&iommu->lock, flags); 3137 data = get_cmdsem_val(iommu); 3138 build_completion_wait(&cmd2, iommu, data); 3139 3140 ret = __iommu_queue_command_sync(iommu, &cmd, true); 3141 if (ret) 3142 goto out_err; 3143 ret = __iommu_queue_command_sync(iommu, &cmd2, false); 3144 if (ret) 3145 goto out_err; 3146 raw_spin_unlock_irqrestore(&iommu->lock, flags); 3147 3148 wait_on_sem(iommu, data); 3149 return; 3150 3151out_err: 3152 raw_spin_unlock_irqrestore(&iommu->lock, flags); 3153} 3154 3155static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data) 3156{ 3157 if (dev_data && dev_data->max_irqs == MAX_IRQS_PER_TABLE_2K) 3158 return DTE_INTTABLEN_2K; 3159 return DTE_INTTABLEN_512; 3160} 3161 3162static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid, 3163 struct irq_remap_table *table) 3164{ 3165 u64 new; 3166 struct dev_table_entry *dte = &get_dev_table(iommu)[devid]; 3167 struct iommu_dev_data *dev_data = search_dev_data(iommu, devid); 3168 3169 if (dev_data) 3170 spin_lock(&dev_data->dte_lock); 3171 3172 new = READ_ONCE(dte->data[2]); 3173 new &= ~DTE_IRQ_PHYS_ADDR_MASK; 3174 new |= iommu_virt_to_phys(table->table); 3175 new |= DTE_IRQ_REMAP_INTCTL; 3176 new |= iommu_get_int_tablen(dev_data); 3177 new |= DTE_IRQ_REMAP_ENABLE; 3178 WRITE_ONCE(dte->data[2], new); 3179 3180 if (dev_data) 3181 spin_unlock(&dev_data->dte_lock); 3182} 3183 3184static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid) 3185{ 3186 struct irq_remap_table *table; 3187 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 3188 3189 if (WARN_ONCE(!pci_seg->rlookup_table[devid], 3190 "%s: no iommu for devid %x:%x\n", 3191 __func__, pci_seg->id, devid)) 3192 return NULL; 3193 3194 table = pci_seg->irq_lookup_table[devid]; 3195 if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n", 3196 __func__, pci_seg->id, devid)) 3197 return NULL; 3198 3199 return table; 3200} 3201 3202static struct irq_remap_table *__alloc_irq_table(int nid, size_t size) 3203{ 3204 struct irq_remap_table *table; 3205 3206 table = kzalloc(sizeof(*table), GFP_KERNEL); 3207 if (!table) 3208 return NULL; 3209 3210 table->table = iommu_alloc_pages_node_sz( 3211 nid, GFP_KERNEL, max(DTE_INTTAB_ALIGNMENT, size)); 3212 if (!table->table) { 3213 kfree(table); 3214 return NULL; 3215 } 3216 raw_spin_lock_init(&table->lock); 3217 3218 return table; 3219} 3220 3221static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, 3222 struct irq_remap_table *table) 3223{ 3224 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; 3225 3226 pci_seg->irq_lookup_table[devid] = table; 3227 set_dte_irq_entry(iommu, devid, table); 3228 iommu_flush_dte(iommu, devid); 3229} 3230 3231static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias, 3232 void *data) 3233{ 3234 struct irq_remap_table *table = data; 3235 struct amd_iommu_pci_seg *pci_seg; 3236 struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); 3237 3238 if (!iommu) 3239 return -EINVAL; 3240 3241 pci_seg = iommu->pci_seg; 3242 pci_seg->irq_lookup_table[alias] = table; 3243 set_dte_irq_entry(iommu, alias, table); 3244 iommu_flush_dte(pci_seg->rlookup_table[alias], alias); 3245 3246 return 0; 3247} 3248 3249static inline size_t get_irq_table_size(unsigned int max_irqs) 3250{ 3251 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) 3252 return max_irqs * sizeof(u32); 3253 3254 return max_irqs * (sizeof(u64) * 2); 3255} 3256 3257static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu, 3258 u16 devid, struct pci_dev *pdev, 3259 unsigned int max_irqs) 3260{ 3261 struct irq_remap_table *table = NULL; 3262 struct irq_remap_table *new_table = NULL; 3263 struct amd_iommu_pci_seg *pci_seg; 3264 unsigned long flags; 3265 int nid = iommu && iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; 3266 u16 alias; 3267 3268 spin_lock_irqsave(&iommu_table_lock, flags); 3269 3270 pci_seg = iommu->pci_seg; 3271 table = pci_seg->irq_lookup_table[devid]; 3272 if (table) 3273 goto out_unlock; 3274 3275 alias = pci_seg->alias_table[devid]; 3276 table = pci_seg->irq_lookup_table[alias]; 3277 if (table) { 3278 set_remap_table_entry(iommu, devid, table); 3279 goto out_wait; 3280 } 3281 spin_unlock_irqrestore(&iommu_table_lock, flags); 3282 3283 /* Nothing there yet, allocate new irq remapping table */ 3284 new_table = __alloc_irq_table(nid, get_irq_table_size(max_irqs)); 3285 if (!new_table) 3286 return NULL; 3287 3288 spin_lock_irqsave(&iommu_table_lock, flags); 3289 3290 table = pci_seg->irq_lookup_table[devid]; 3291 if (table) 3292 goto out_unlock; 3293 3294 table = pci_seg->irq_lookup_table[alias]; 3295 if (table) { 3296 set_remap_table_entry(iommu, devid, table); 3297 goto out_wait; 3298 } 3299 3300 table = new_table; 3301 new_table = NULL; 3302 3303 if (pdev) 3304 pci_for_each_dma_alias(pdev, set_remap_table_entry_alias, 3305 table); 3306 else 3307 set_remap_table_entry(iommu, devid, table); 3308 3309 if (devid != alias) 3310 set_remap_table_entry(iommu, alias, table); 3311 3312out_wait: 3313 iommu_completion_wait(iommu); 3314 3315out_unlock: 3316 spin_unlock_irqrestore(&iommu_table_lock, flags); 3317 3318 if (new_table) { 3319 iommu_free_pages(new_table->table); 3320 kfree(new_table); 3321 } 3322 return table; 3323} 3324 3325static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count, 3326 bool align, struct pci_dev *pdev, 3327 unsigned long max_irqs) 3328{ 3329 struct irq_remap_table *table; 3330 int index, c, alignment = 1; 3331 unsigned long flags; 3332 3333 table = alloc_irq_table(iommu, devid, pdev, max_irqs); 3334 if (!table) 3335 return -ENODEV; 3336 3337 if (align) 3338 alignment = roundup_pow_of_two(count); 3339 3340 raw_spin_lock_irqsave(&table->lock, flags); 3341 3342 /* Scan table for free entries */ 3343 for (index = ALIGN(table->min_index, alignment), c = 0; 3344 index < max_irqs;) { 3345 if (!iommu->irte_ops->is_allocated(table, index)) { 3346 c += 1; 3347 } else { 3348 c = 0; 3349 index = ALIGN(index + 1, alignment); 3350 continue; 3351 } 3352 3353 if (c == count) { 3354 for (; c != 0; --c) 3355 iommu->irte_ops->set_allocated(table, index - c + 1); 3356 3357 index -= count - 1; 3358 goto out; 3359 } 3360 3361 index++; 3362 } 3363 3364 index = -ENOSPC; 3365 3366out: 3367 raw_spin_unlock_irqrestore(&table->lock, flags); 3368 3369 return index; 3370} 3371 3372static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, 3373 struct irte_ga *irte) 3374{ 3375 struct irq_remap_table *table; 3376 struct irte_ga *entry; 3377 unsigned long flags; 3378 u128 old; 3379 3380 table = get_irq_table(iommu, devid); 3381 if (!table) 3382 return -ENOMEM; 3383 3384 raw_spin_lock_irqsave(&table->lock, flags); 3385 3386 entry = (struct irte_ga *)table->table; 3387 entry = &entry[index]; 3388 3389 /* 3390 * We use cmpxchg16 to atomically update the 128-bit IRTE, 3391 * and it cannot be updated by the hardware or other processors 3392 * behind us, so the return value of cmpxchg16 should be the 3393 * same as the old value. 3394 */ 3395 old = entry->irte; 3396 WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte)); 3397 3398 raw_spin_unlock_irqrestore(&table->lock, flags); 3399 3400 return 0; 3401} 3402 3403static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, 3404 struct irte_ga *irte) 3405{ 3406 int ret; 3407 3408 ret = __modify_irte_ga(iommu, devid, index, irte); 3409 if (ret) 3410 return ret; 3411 3412 iommu_flush_irt_and_complete(iommu, devid); 3413 3414 return 0; 3415} 3416 3417static int modify_irte(struct amd_iommu *iommu, 3418 u16 devid, int index, union irte *irte) 3419{ 3420 struct irq_remap_table *table; 3421 unsigned long flags; 3422 3423 table = get_irq_table(iommu, devid); 3424 if (!table) 3425 return -ENOMEM; 3426 3427 raw_spin_lock_irqsave(&table->lock, flags); 3428 table->table[index] = irte->val; 3429 raw_spin_unlock_irqrestore(&table->lock, flags); 3430 3431 iommu_flush_irt_and_complete(iommu, devid); 3432 3433 return 0; 3434} 3435 3436static void free_irte(struct amd_iommu *iommu, u16 devid, int index) 3437{ 3438 struct irq_remap_table *table; 3439 unsigned long flags; 3440 3441 table = get_irq_table(iommu, devid); 3442 if (!table) 3443 return; 3444 3445 raw_spin_lock_irqsave(&table->lock, flags); 3446 iommu->irte_ops->clear_allocated(table, index); 3447 raw_spin_unlock_irqrestore(&table->lock, flags); 3448 3449 iommu_flush_irt_and_complete(iommu, devid); 3450} 3451 3452static void irte_prepare(void *entry, 3453 u32 delivery_mode, bool dest_mode, 3454 u8 vector, u32 dest_apicid, int devid) 3455{ 3456 union irte *irte = (union irte *) entry; 3457 3458 irte->val = 0; 3459 irte->fields.vector = vector; 3460 irte->fields.int_type = delivery_mode; 3461 irte->fields.destination = dest_apicid; 3462 irte->fields.dm = dest_mode; 3463 irte->fields.valid = 1; 3464} 3465 3466static void irte_ga_prepare(void *entry, 3467 u32 delivery_mode, bool dest_mode, 3468 u8 vector, u32 dest_apicid, int devid) 3469{ 3470 struct irte_ga *irte = (struct irte_ga *) entry; 3471 3472 irte->lo.val = 0; 3473 irte->hi.val = 0; 3474 irte->lo.fields_remap.int_type = delivery_mode; 3475 irte->lo.fields_remap.dm = dest_mode; 3476 irte->hi.fields.vector = vector; 3477 irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid); 3478 irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid); 3479 irte->lo.fields_remap.valid = 1; 3480} 3481 3482static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3483{ 3484 union irte *irte = (union irte *) entry; 3485 3486 irte->fields.valid = 1; 3487 modify_irte(iommu, devid, index, irte); 3488} 3489 3490static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3491{ 3492 struct irte_ga *irte = (struct irte_ga *) entry; 3493 3494 irte->lo.fields_remap.valid = 1; 3495 modify_irte_ga(iommu, devid, index, irte); 3496} 3497 3498static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3499{ 3500 union irte *irte = (union irte *) entry; 3501 3502 irte->fields.valid = 0; 3503 modify_irte(iommu, devid, index, irte); 3504} 3505 3506static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) 3507{ 3508 struct irte_ga *irte = (struct irte_ga *) entry; 3509 3510 irte->lo.fields_remap.valid = 0; 3511 modify_irte_ga(iommu, devid, index, irte); 3512} 3513 3514static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, 3515 u8 vector, u32 dest_apicid) 3516{ 3517 union irte *irte = (union irte *) entry; 3518 3519 irte->fields.vector = vector; 3520 irte->fields.destination = dest_apicid; 3521 modify_irte(iommu, devid, index, irte); 3522} 3523 3524static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, 3525 u8 vector, u32 dest_apicid) 3526{ 3527 struct irte_ga *irte = (struct irte_ga *) entry; 3528 3529 if (!irte->lo.fields_remap.guest_mode) { 3530 irte->hi.fields.vector = vector; 3531 irte->lo.fields_remap.destination = 3532 APICID_TO_IRTE_DEST_LO(dest_apicid); 3533 irte->hi.fields.destination = 3534 APICID_TO_IRTE_DEST_HI(dest_apicid); 3535 modify_irte_ga(iommu, devid, index, irte); 3536 } 3537} 3538 3539#define IRTE_ALLOCATED (~1U) 3540static void irte_set_allocated(struct irq_remap_table *table, int index) 3541{ 3542 table->table[index] = IRTE_ALLOCATED; 3543} 3544 3545static void irte_ga_set_allocated(struct irq_remap_table *table, int index) 3546{ 3547 struct irte_ga *ptr = (struct irte_ga *)table->table; 3548 struct irte_ga *irte = &ptr[index]; 3549 3550 memset(&irte->lo.val, 0, sizeof(u64)); 3551 memset(&irte->hi.val, 0, sizeof(u64)); 3552 irte->hi.fields.vector = 0xff; 3553} 3554 3555static bool irte_is_allocated(struct irq_remap_table *table, int index) 3556{ 3557 union irte *ptr = (union irte *)table->table; 3558 union irte *irte = &ptr[index]; 3559 3560 return irte->val != 0; 3561} 3562 3563static bool irte_ga_is_allocated(struct irq_remap_table *table, int index) 3564{ 3565 struct irte_ga *ptr = (struct irte_ga *)table->table; 3566 struct irte_ga *irte = &ptr[index]; 3567 3568 return irte->hi.fields.vector != 0; 3569} 3570 3571static void irte_clear_allocated(struct irq_remap_table *table, int index) 3572{ 3573 table->table[index] = 0; 3574} 3575 3576static void irte_ga_clear_allocated(struct irq_remap_table *table, int index) 3577{ 3578 struct irte_ga *ptr = (struct irte_ga *)table->table; 3579 struct irte_ga *irte = &ptr[index]; 3580 3581 memset(&irte->lo.val, 0, sizeof(u64)); 3582 memset(&irte->hi.val, 0, sizeof(u64)); 3583} 3584 3585static int get_devid(struct irq_alloc_info *info) 3586{ 3587 switch (info->type) { 3588 case X86_IRQ_ALLOC_TYPE_IOAPIC: 3589 return get_ioapic_devid(info->devid); 3590 case X86_IRQ_ALLOC_TYPE_HPET: 3591 return get_hpet_devid(info->devid); 3592 case X86_IRQ_ALLOC_TYPE_PCI_MSI: 3593 case X86_IRQ_ALLOC_TYPE_PCI_MSIX: 3594 return get_device_sbdf_id(msi_desc_to_dev(info->desc)); 3595 default: 3596 WARN_ON_ONCE(1); 3597 return -1; 3598 } 3599} 3600 3601struct irq_remap_ops amd_iommu_irq_ops = { 3602 .prepare = amd_iommu_prepare, 3603 .enable = amd_iommu_enable, 3604 .disable = amd_iommu_disable, 3605 .reenable = amd_iommu_reenable, 3606 .enable_faulting = amd_iommu_enable_faulting, 3607}; 3608 3609static void fill_msi_msg(struct msi_msg *msg, u32 index) 3610{ 3611 msg->data = index; 3612 msg->address_lo = 0; 3613 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; 3614 /* 3615 * The struct msi_msg.dest_mode_logical is used to set the DM bit 3616 * in MSI Message Address Register. For device w/ 2K int-remap support, 3617 * this is bit must be set to 1 regardless of the actual destination 3618 * mode, which is signified by the IRTE[DM]. 3619 */ 3620 if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2)) 3621 msg->arch_addr_lo.dest_mode_logical = true; 3622 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; 3623} 3624 3625static void irq_remapping_prepare_irte(struct amd_ir_data *data, 3626 struct irq_cfg *irq_cfg, 3627 struct irq_alloc_info *info, 3628 int devid, int index, int sub_handle) 3629{ 3630 struct irq_2_irte *irte_info = &data->irq_2_irte; 3631 struct amd_iommu *iommu = data->iommu; 3632 3633 if (!iommu) 3634 return; 3635 3636 data->irq_2_irte.devid = devid; 3637 data->irq_2_irte.index = index + sub_handle; 3638 iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED, 3639 apic->dest_mode_logical, irq_cfg->vector, 3640 irq_cfg->dest_apicid, devid); 3641 3642 switch (info->type) { 3643 case X86_IRQ_ALLOC_TYPE_IOAPIC: 3644 case X86_IRQ_ALLOC_TYPE_HPET: 3645 case X86_IRQ_ALLOC_TYPE_PCI_MSI: 3646 case X86_IRQ_ALLOC_TYPE_PCI_MSIX: 3647 fill_msi_msg(&data->msi_entry, irte_info->index); 3648 break; 3649 3650 default: 3651 BUG_ON(1); 3652 break; 3653 } 3654} 3655 3656struct amd_irte_ops irte_32_ops = { 3657 .prepare = irte_prepare, 3658 .activate = irte_activate, 3659 .deactivate = irte_deactivate, 3660 .set_affinity = irte_set_affinity, 3661 .set_allocated = irte_set_allocated, 3662 .is_allocated = irte_is_allocated, 3663 .clear_allocated = irte_clear_allocated, 3664}; 3665 3666struct amd_irte_ops irte_128_ops = { 3667 .prepare = irte_ga_prepare, 3668 .activate = irte_ga_activate, 3669 .deactivate = irte_ga_deactivate, 3670 .set_affinity = irte_ga_set_affinity, 3671 .set_allocated = irte_ga_set_allocated, 3672 .is_allocated = irte_ga_is_allocated, 3673 .clear_allocated = irte_ga_clear_allocated, 3674}; 3675 3676static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, 3677 unsigned int nr_irqs, void *arg) 3678{ 3679 struct irq_alloc_info *info = arg; 3680 struct irq_data *irq_data; 3681 struct amd_ir_data *data = NULL; 3682 struct amd_iommu *iommu; 3683 struct irq_cfg *cfg; 3684 struct iommu_dev_data *dev_data; 3685 unsigned long max_irqs; 3686 int i, ret, devid, seg, sbdf; 3687 int index; 3688 3689 if (!info) 3690 return -EINVAL; 3691 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI) 3692 return -EINVAL; 3693 3694 sbdf = get_devid(info); 3695 if (sbdf < 0) 3696 return -EINVAL; 3697 3698 seg = PCI_SBDF_TO_SEGID(sbdf); 3699 devid = PCI_SBDF_TO_DEVID(sbdf); 3700 iommu = __rlookup_amd_iommu(seg, devid); 3701 if (!iommu) 3702 return -EINVAL; 3703 3704 dev_data = search_dev_data(iommu, devid); 3705 max_irqs = dev_data ? dev_data->max_irqs : MAX_IRQS_PER_TABLE_512; 3706 3707 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); 3708 if (ret < 0) 3709 return ret; 3710 3711 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { 3712 struct irq_remap_table *table; 3713 3714 table = alloc_irq_table(iommu, devid, NULL, max_irqs); 3715 if (table) { 3716 if (!table->min_index) { 3717 /* 3718 * Keep the first 32 indexes free for IOAPIC 3719 * interrupts. 3720 */ 3721 table->min_index = 32; 3722 for (i = 0; i < 32; ++i) 3723 iommu->irte_ops->set_allocated(table, i); 3724 } 3725 WARN_ON(table->min_index != 32); 3726 index = info->ioapic.pin; 3727 } else { 3728 index = -ENOMEM; 3729 } 3730 } else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI || 3731 info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) { 3732 bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI); 3733 3734 index = alloc_irq_index(iommu, devid, nr_irqs, align, 3735 msi_desc_to_pci_dev(info->desc), 3736 max_irqs); 3737 } else { 3738 index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL, 3739 max_irqs); 3740 } 3741 3742 if (index < 0) { 3743 pr_warn("Failed to allocate IRTE\n"); 3744 ret = index; 3745 goto out_free_parent; 3746 } 3747 3748 for (i = 0; i < nr_irqs; i++) { 3749 irq_data = irq_domain_get_irq_data(domain, virq + i); 3750 cfg = irq_data ? irqd_cfg(irq_data) : NULL; 3751 if (!cfg) { 3752 ret = -EINVAL; 3753 goto out_free_data; 3754 } 3755 3756 ret = -ENOMEM; 3757 data = kzalloc(sizeof(*data), GFP_KERNEL); 3758 if (!data) 3759 goto out_free_data; 3760 3761 if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) 3762 data->entry = kzalloc(sizeof(union irte), GFP_KERNEL); 3763 else 3764 data->entry = kzalloc(sizeof(struct irte_ga), 3765 GFP_KERNEL); 3766 if (!data->entry) { 3767 kfree(data); 3768 goto out_free_data; 3769 } 3770 3771 data->iommu = iommu; 3772 irq_data->hwirq = (devid << 16) + i; 3773 irq_data->chip_data = data; 3774 irq_data->chip = &amd_ir_chip; 3775 irq_remapping_prepare_irte(data, cfg, info, devid, index, i); 3776 } 3777 3778 return 0; 3779 3780out_free_data: 3781 for (i--; i >= 0; i--) { 3782 irq_data = irq_domain_get_irq_data(domain, virq + i); 3783 if (irq_data) 3784 kfree(irq_data->chip_data); 3785 } 3786 for (i = 0; i < nr_irqs; i++) 3787 free_irte(iommu, devid, index + i); 3788out_free_parent: 3789 irq_domain_free_irqs_common(domain, virq, nr_irqs); 3790 return ret; 3791} 3792 3793static void irq_remapping_free(struct irq_domain *domain, unsigned int virq, 3794 unsigned int nr_irqs) 3795{ 3796 struct irq_2_irte *irte_info; 3797 struct irq_data *irq_data; 3798 struct amd_ir_data *data; 3799 int i; 3800 3801 for (i = 0; i < nr_irqs; i++) { 3802 irq_data = irq_domain_get_irq_data(domain, virq + i); 3803 if (irq_data && irq_data->chip_data) { 3804 data = irq_data->chip_data; 3805 irte_info = &data->irq_2_irte; 3806 free_irte(data->iommu, irte_info->devid, irte_info->index); 3807 kfree(data->entry); 3808 kfree(data); 3809 } 3810 } 3811 irq_domain_free_irqs_common(domain, virq, nr_irqs); 3812} 3813 3814static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu, 3815 struct amd_ir_data *ir_data, 3816 struct irq_2_irte *irte_info, 3817 struct irq_cfg *cfg); 3818 3819static int irq_remapping_activate(struct irq_domain *domain, 3820 struct irq_data *irq_data, bool reserve) 3821{ 3822 struct amd_ir_data *data = irq_data->chip_data; 3823 struct irq_2_irte *irte_info = &data->irq_2_irte; 3824 struct amd_iommu *iommu = data->iommu; 3825 struct irq_cfg *cfg = irqd_cfg(irq_data); 3826 3827 if (!iommu) 3828 return 0; 3829 3830 iommu->irte_ops->activate(iommu, data->entry, irte_info->devid, 3831 irte_info->index); 3832 amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg); 3833 return 0; 3834} 3835 3836static void irq_remapping_deactivate(struct irq_domain *domain, 3837 struct irq_data *irq_data) 3838{ 3839 struct amd_ir_data *data = irq_data->chip_data; 3840 struct irq_2_irte *irte_info = &data->irq_2_irte; 3841 struct amd_iommu *iommu = data->iommu; 3842 3843 if (iommu) 3844 iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid, 3845 irte_info->index); 3846} 3847 3848static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec, 3849 enum irq_domain_bus_token bus_token) 3850{ 3851 struct amd_iommu *iommu; 3852 int devid = -1; 3853 3854 if (!amd_iommu_irq_remap) 3855 return 0; 3856 3857 if (x86_fwspec_is_ioapic(fwspec)) 3858 devid = get_ioapic_devid(fwspec->param[0]); 3859 else if (x86_fwspec_is_hpet(fwspec)) 3860 devid = get_hpet_devid(fwspec->param[0]); 3861 3862 if (devid < 0) 3863 return 0; 3864 iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff)); 3865 3866 return iommu && iommu->ir_domain == d; 3867} 3868 3869static const struct irq_domain_ops amd_ir_domain_ops = { 3870 .select = irq_remapping_select, 3871 .alloc = irq_remapping_alloc, 3872 .free = irq_remapping_free, 3873 .activate = irq_remapping_activate, 3874 .deactivate = irq_remapping_deactivate, 3875}; 3876 3877static void __amd_iommu_update_ga(struct irte_ga *entry, int cpu, 3878 bool ga_log_intr) 3879{ 3880 if (cpu >= 0) { 3881 entry->lo.fields_vapic.destination = 3882 APICID_TO_IRTE_DEST_LO(cpu); 3883 entry->hi.fields.destination = 3884 APICID_TO_IRTE_DEST_HI(cpu); 3885 entry->lo.fields_vapic.is_run = true; 3886 entry->lo.fields_vapic.ga_log_intr = false; 3887 } else { 3888 entry->lo.fields_vapic.is_run = false; 3889 entry->lo.fields_vapic.ga_log_intr = ga_log_intr; 3890 } 3891} 3892 3893/* 3894 * Update the pCPU information for an IRTE that is configured to post IRQs to 3895 * a vCPU, without issuing an IOMMU invalidation for the IRTE. 3896 * 3897 * If the vCPU is associated with a pCPU (@cpu >= 0), configure the Destination 3898 * with the pCPU's APIC ID, set IsRun, and clear GALogIntr. If the vCPU isn't 3899 * associated with a pCPU (@cpu < 0), clear IsRun and set/clear GALogIntr based 3900 * on input from the caller (e.g. KVM only requests GALogIntr when the vCPU is 3901 * blocking and requires a notification wake event). I.e. treat vCPUs that are 3902 * associated with a pCPU as running. This API is intended to be used when a 3903 * vCPU is scheduled in/out (or stops running for any reason), to do a fast 3904 * update of IsRun, GALogIntr, and (conditionally) Destination. 3905 * 3906 * Per the IOMMU spec, the Destination, IsRun, and GATag fields are not cached 3907 * and thus don't require an invalidation to ensure the IOMMU consumes fresh 3908 * information. 3909 */ 3910int amd_iommu_update_ga(void *data, int cpu, bool ga_log_intr) 3911{ 3912 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3913 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3914 3915 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 3916 return -EINVAL; 3917 3918 if (!entry || !entry->lo.fields_vapic.guest_mode) 3919 return 0; 3920 3921 if (!ir_data->iommu) 3922 return -ENODEV; 3923 3924 __amd_iommu_update_ga(entry, cpu, ga_log_intr); 3925 3926 return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 3927 ir_data->irq_2_irte.index, entry); 3928} 3929EXPORT_SYMBOL(amd_iommu_update_ga); 3930 3931int amd_iommu_activate_guest_mode(void *data, int cpu, bool ga_log_intr) 3932{ 3933 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3934 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3935 u64 valid; 3936 3937 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 3938 return -EINVAL; 3939 3940 if (!entry) 3941 return 0; 3942 3943 valid = entry->lo.fields_vapic.valid; 3944 3945 entry->lo.val = 0; 3946 entry->hi.val = 0; 3947 3948 entry->lo.fields_vapic.valid = valid; 3949 entry->lo.fields_vapic.guest_mode = 1; 3950 entry->hi.fields.ga_root_ptr = ir_data->ga_root_ptr; 3951 entry->hi.fields.vector = ir_data->ga_vector; 3952 entry->lo.fields_vapic.ga_tag = ir_data->ga_tag; 3953 3954 __amd_iommu_update_ga(entry, cpu, ga_log_intr); 3955 3956 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 3957 ir_data->irq_2_irte.index, entry); 3958} 3959EXPORT_SYMBOL(amd_iommu_activate_guest_mode); 3960 3961int amd_iommu_deactivate_guest_mode(void *data) 3962{ 3963 struct amd_ir_data *ir_data = (struct amd_ir_data *)data; 3964 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; 3965 struct irq_cfg *cfg = ir_data->cfg; 3966 u64 valid; 3967 3968 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 3969 return -EINVAL; 3970 3971 if (!entry || !entry->lo.fields_vapic.guest_mode) 3972 return 0; 3973 3974 valid = entry->lo.fields_remap.valid; 3975 3976 entry->lo.val = 0; 3977 entry->hi.val = 0; 3978 3979 entry->lo.fields_remap.valid = valid; 3980 entry->lo.fields_remap.dm = apic->dest_mode_logical; 3981 entry->lo.fields_remap.int_type = APIC_DELIVERY_MODE_FIXED; 3982 entry->hi.fields.vector = cfg->vector; 3983 entry->lo.fields_remap.destination = 3984 APICID_TO_IRTE_DEST_LO(cfg->dest_apicid); 3985 entry->hi.fields.destination = 3986 APICID_TO_IRTE_DEST_HI(cfg->dest_apicid); 3987 3988 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, 3989 ir_data->irq_2_irte.index, entry); 3990} 3991EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode); 3992 3993static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *info) 3994{ 3995 int ret; 3996 struct amd_iommu_pi_data *pi_data = info; 3997 struct amd_ir_data *ir_data = data->chip_data; 3998 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; 3999 struct iommu_dev_data *dev_data; 4000 4001 if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))) 4002 return -EINVAL; 4003 4004 if (ir_data->iommu == NULL) 4005 return -EINVAL; 4006 4007 dev_data = search_dev_data(ir_data->iommu, irte_info->devid); 4008 4009 /* Note: 4010 * This device has never been set up for guest mode. 4011 * we should not modify the IRTE 4012 */ 4013 if (!dev_data || !dev_data->use_vapic) 4014 return -EINVAL; 4015 4016 ir_data->cfg = irqd_cfg(data); 4017 4018 if (pi_data) { 4019 pi_data->ir_data = ir_data; 4020 4021 ir_data->ga_root_ptr = (pi_data->vapic_addr >> 12); 4022 ir_data->ga_vector = pi_data->vector; 4023 ir_data->ga_tag = pi_data->ga_tag; 4024 if (pi_data->is_guest_mode) 4025 ret = amd_iommu_activate_guest_mode(ir_data, pi_data->cpu, 4026 pi_data->ga_log_intr); 4027 else 4028 ret = amd_iommu_deactivate_guest_mode(ir_data); 4029 } else { 4030 ret = amd_iommu_deactivate_guest_mode(ir_data); 4031 } 4032 4033 return ret; 4034} 4035 4036 4037static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu, 4038 struct amd_ir_data *ir_data, 4039 struct irq_2_irte *irte_info, 4040 struct irq_cfg *cfg) 4041{ 4042 4043 /* 4044 * Atomically updates the IRTE with the new destination, vector 4045 * and flushes the interrupt entry cache. 4046 */ 4047 iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid, 4048 irte_info->index, cfg->vector, 4049 cfg->dest_apicid); 4050} 4051 4052static int amd_ir_set_affinity(struct irq_data *data, 4053 const struct cpumask *mask, bool force) 4054{ 4055 struct amd_ir_data *ir_data = data->chip_data; 4056 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; 4057 struct irq_cfg *cfg = irqd_cfg(data); 4058 struct irq_data *parent = data->parent_data; 4059 struct amd_iommu *iommu = ir_data->iommu; 4060 int ret; 4061 4062 if (!iommu) 4063 return -ENODEV; 4064 4065 ret = parent->chip->irq_set_affinity(parent, mask, force); 4066 if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) 4067 return ret; 4068 4069 amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg); 4070 /* 4071 * After this point, all the interrupts will start arriving 4072 * at the new destination. So, time to cleanup the previous 4073 * vector allocation. 4074 */ 4075 vector_schedule_cleanup(cfg); 4076 4077 return IRQ_SET_MASK_OK_DONE; 4078} 4079 4080static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg) 4081{ 4082 struct amd_ir_data *ir_data = irq_data->chip_data; 4083 4084 *msg = ir_data->msi_entry; 4085} 4086 4087static struct irq_chip amd_ir_chip = { 4088 .name = "AMD-IR", 4089 .irq_ack = apic_ack_irq, 4090 .irq_set_affinity = amd_ir_set_affinity, 4091 .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity, 4092 .irq_compose_msi_msg = ir_compose_msi_msg, 4093}; 4094 4095static const struct msi_parent_ops amdvi_msi_parent_ops = { 4096 .supported_flags = X86_VECTOR_MSI_FLAGS_SUPPORTED | MSI_FLAG_MULTI_PCI_MSI, 4097 .bus_select_token = DOMAIN_BUS_AMDVI, 4098 .bus_select_mask = MATCH_PCI_MSI, 4099 .prefix = "IR-", 4100 .init_dev_msi_info = msi_parent_init_dev_msi_info, 4101}; 4102 4103int amd_iommu_create_irq_domain(struct amd_iommu *iommu) 4104{ 4105 struct irq_domain_info info = { 4106 .fwnode = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index), 4107 .ops = &amd_ir_domain_ops, 4108 .domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI, 4109 .host_data = iommu, 4110 .parent = arch_get_ir_parent_domain(), 4111 }; 4112 4113 if (!info.fwnode) 4114 return -ENOMEM; 4115 4116 iommu->ir_domain = msi_create_parent_irq_domain(&info, &amdvi_msi_parent_ops); 4117 if (!iommu->ir_domain) { 4118 irq_domain_free_fwnode(info.fwnode); 4119 return -ENOMEM; 4120 } 4121 return 0; 4122} 4123#endif 4124 4125MODULE_IMPORT_NS("GENERIC_PT_IOMMU");
~/
Read-only file preview.