FILE
pci.c
File preview with syntax highlighting.
linux-6.19.14/drivers/pci/pci.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * PCI Bus Services, see include/linux/pci.h for further explanation. 4 * 5 * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter, 6 * David Mosberger-Tang 7 * 8 * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz> 9 */ 10 11#include <linux/acpi.h> 12#include <linux/kernel.h> 13#include <linux/delay.h> 14#include <linux/dmi.h> 15#include <linux/init.h> 16#include <linux/msi.h> 17#include <linux/of.h> 18#include <linux/pci.h> 19#include <linux/pm.h> 20#include <linux/slab.h> 21#include <linux/module.h> 22#include <linux/spinlock.h> 23#include <linux/string.h> 24#include <linux/log2.h> 25#include <linux/logic_pio.h> 26#include <linux/device.h> 27#include <linux/pm_runtime.h> 28#include <linux/pci_hotplug.h> 29#include <linux/vmalloc.h> 30#include <asm/dma.h> 31#include <linux/aer.h> 32#include <linux/bitfield.h> 33#include "pci.h" 34 35DEFINE_MUTEX(pci_slot_mutex); 36 37const char *pci_power_names[] = { 38 "error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown", 39}; 40EXPORT_SYMBOL_GPL(pci_power_names); 41 42#ifdef CONFIG_X86_32 43int isa_dma_bridge_buggy; 44EXPORT_SYMBOL(isa_dma_bridge_buggy); 45#endif 46 47int pci_pci_problems; 48EXPORT_SYMBOL(pci_pci_problems); 49 50unsigned int pci_pm_d3hot_delay; 51 52static void pci_pme_list_scan(struct work_struct *work); 53 54static LIST_HEAD(pci_pme_list); 55static DEFINE_MUTEX(pci_pme_list_mutex); 56static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan); 57 58struct pci_pme_device { 59 struct list_head list; 60 struct pci_dev *dev; 61}; 62 63#define PME_TIMEOUT 1000 /* How long between PME checks */ 64 65/* 66 * Following exit from Conventional Reset, devices must be ready within 1 sec 67 * (PCIe r6.0 sec 6.6.1). A D3cold to D0 transition implies a Conventional 68 * Reset (PCIe r6.0 sec 5.8). 69 */ 70#define PCI_RESET_WAIT 1000 /* msec */ 71 72/* 73 * Devices may extend the 1 sec period through Request Retry Status 74 * completions (PCIe r6.0 sec 2.3.1). The spec does not provide an upper 75 * limit, but 60 sec ought to be enough for any device to become 76 * responsive. 77 */ 78#define PCIE_RESET_READY_POLL_MS 60000 /* msec */ 79 80static void pci_dev_d3_sleep(struct pci_dev *dev) 81{ 82 unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay); 83 unsigned int upper; 84 85 if (delay_ms) { 86 /* Use a 20% upper bound, 1ms minimum */ 87 upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U); 88 usleep_range(delay_ms * USEC_PER_MSEC, 89 (delay_ms + upper) * USEC_PER_MSEC); 90 } 91} 92 93bool pci_reset_supported(struct pci_dev *dev) 94{ 95 return dev->reset_methods[0] != 0; 96} 97 98#ifdef CONFIG_PCI_DOMAINS 99int pci_domains_supported = 1; 100#endif 101 102#define DEFAULT_CARDBUS_IO_SIZE (256) 103#define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024) 104/* pci=cbmemsize=nnM,cbiosize=nn can override this */ 105unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE; 106unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE; 107 108#define DEFAULT_HOTPLUG_IO_SIZE (256) 109#define DEFAULT_HOTPLUG_MMIO_SIZE (2*1024*1024) 110#define DEFAULT_HOTPLUG_MMIO_PREF_SIZE (2*1024*1024) 111/* hpiosize=nn can override this */ 112unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE; 113/* 114 * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size, 115 * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size; 116 * pci=hpmemsize=nnM overrides both 117 */ 118unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE; 119unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE; 120 121#define DEFAULT_HOTPLUG_BUS_SIZE 1 122unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 123 124 125/* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */ 126#ifdef CONFIG_PCIE_BUS_TUNE_OFF 127enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF; 128#elif defined CONFIG_PCIE_BUS_SAFE 129enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE; 130#elif defined CONFIG_PCIE_BUS_PERFORMANCE 131enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE; 132#elif defined CONFIG_PCIE_BUS_PEER2PEER 133enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER; 134#else 135enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT; 136#endif 137 138/* 139 * The default CLS is used if arch didn't set CLS explicitly and not 140 * all pci devices agree on the same value. Arch can override either 141 * the dfl or actual value as it sees fit. Don't forget this is 142 * measured in 32-bit words, not bytes. 143 */ 144u8 pci_dfl_cache_line_size __ro_after_init = L1_CACHE_BYTES >> 2; 145u8 pci_cache_line_size __ro_after_init ; 146 147/* 148 * If we set up a device for bus mastering, we need to check the latency 149 * timer as certain BIOSes forget to set it properly. 150 */ 151unsigned int pcibios_max_latency = 255; 152 153/* If set, the PCIe ARI capability will not be used. */ 154static bool pcie_ari_disabled; 155 156/* If set, the PCIe ATS capability will not be used. */ 157static bool pcie_ats_disabled; 158 159/* If set, the PCI config space of each device is printed during boot. */ 160bool pci_early_dump; 161 162bool pci_ats_disabled(void) 163{ 164 return pcie_ats_disabled; 165} 166EXPORT_SYMBOL_GPL(pci_ats_disabled); 167 168/* Disable bridge_d3 for all PCIe ports */ 169static bool pci_bridge_d3_disable; 170/* Force bridge_d3 for all PCIe ports */ 171static bool pci_bridge_d3_force; 172 173static int __init pcie_port_pm_setup(char *str) 174{ 175 if (!strcmp(str, "off")) 176 pci_bridge_d3_disable = true; 177 else if (!strcmp(str, "force")) 178 pci_bridge_d3_force = true; 179 return 1; 180} 181__setup("pcie_port_pm=", pcie_port_pm_setup); 182 183/** 184 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children 185 * @bus: pointer to PCI bus structure to search 186 * 187 * Given a PCI bus, returns the highest PCI bus number present in the set 188 * including the given PCI bus and its list of child PCI buses. 189 */ 190unsigned char pci_bus_max_busnr(struct pci_bus *bus) 191{ 192 struct pci_bus *tmp; 193 unsigned char max, n; 194 195 max = bus->busn_res.end; 196 list_for_each_entry(tmp, &bus->children, node) { 197 n = pci_bus_max_busnr(tmp); 198 if (n > max) 199 max = n; 200 } 201 return max; 202} 203EXPORT_SYMBOL_GPL(pci_bus_max_busnr); 204 205/** 206 * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS 207 * @pdev: the PCI device 208 * 209 * Returns error bits set in PCI_STATUS and clears them. 210 */ 211int pci_status_get_and_clear_errors(struct pci_dev *pdev) 212{ 213 u16 status; 214 int ret; 215 216 ret = pci_read_config_word(pdev, PCI_STATUS, &status); 217 if (ret != PCIBIOS_SUCCESSFUL) 218 return -EIO; 219 220 status &= PCI_STATUS_ERROR_BITS; 221 if (status) 222 pci_write_config_word(pdev, PCI_STATUS, status); 223 224 return status; 225} 226EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors); 227 228#ifdef CONFIG_HAS_IOMEM 229static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar, 230 bool write_combine) 231{ 232 struct resource *res = &pdev->resource[bar]; 233 resource_size_t start = res->start; 234 resource_size_t size = resource_size(res); 235 236 /* 237 * Make sure the BAR is actually a memory resource, not an IO resource 238 */ 239 if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) { 240 pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res); 241 return NULL; 242 } 243 244 if (write_combine) 245 return ioremap_wc(start, size); 246 247 return ioremap(start, size); 248} 249 250void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) 251{ 252 return __pci_ioremap_resource(pdev, bar, false); 253} 254EXPORT_SYMBOL_GPL(pci_ioremap_bar); 255 256void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar) 257{ 258 return __pci_ioremap_resource(pdev, bar, true); 259} 260EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar); 261#endif 262 263/** 264 * pci_dev_str_match_path - test if a path string matches a device 265 * @dev: the PCI device to test 266 * @path: string to match the device against 267 * @endptr: pointer to the string after the match 268 * 269 * Test if a string (typically from a kernel parameter) formatted as a 270 * path of device/function addresses matches a PCI device. The string must 271 * be of the form: 272 * 273 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 274 * 275 * A path for a device can be obtained using 'lspci -t'. Using a path 276 * is more robust against bus renumbering than using only a single bus, 277 * device and function address. 278 * 279 * Returns 1 if the string matches the device, 0 if it does not and 280 * a negative error code if it fails to parse the string. 281 */ 282static int pci_dev_str_match_path(struct pci_dev *dev, const char *path, 283 const char **endptr) 284{ 285 int ret; 286 unsigned int seg, bus, slot, func; 287 char *wpath, *p; 288 char end; 289 290 *endptr = strchrnul(path, ';'); 291 292 wpath = kmemdup_nul(path, *endptr - path, GFP_ATOMIC); 293 if (!wpath) 294 return -ENOMEM; 295 296 while (1) { 297 p = strrchr(wpath, '/'); 298 if (!p) 299 break; 300 ret = sscanf(p, "/%x.%x%c", &slot, &func, &end); 301 if (ret != 2) { 302 ret = -EINVAL; 303 goto free_and_exit; 304 } 305 306 if (dev->devfn != PCI_DEVFN(slot, func)) { 307 ret = 0; 308 goto free_and_exit; 309 } 310 311 /* 312 * Note: we don't need to get a reference to the upstream 313 * bridge because we hold a reference to the top level 314 * device which should hold a reference to the bridge, 315 * and so on. 316 */ 317 dev = pci_upstream_bridge(dev); 318 if (!dev) { 319 ret = 0; 320 goto free_and_exit; 321 } 322 323 *p = 0; 324 } 325 326 ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot, 327 &func, &end); 328 if (ret != 4) { 329 seg = 0; 330 ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end); 331 if (ret != 3) { 332 ret = -EINVAL; 333 goto free_and_exit; 334 } 335 } 336 337 ret = (seg == pci_domain_nr(dev->bus) && 338 bus == dev->bus->number && 339 dev->devfn == PCI_DEVFN(slot, func)); 340 341free_and_exit: 342 kfree(wpath); 343 return ret; 344} 345 346/** 347 * pci_dev_str_match - test if a string matches a device 348 * @dev: the PCI device to test 349 * @p: string to match the device against 350 * @endptr: pointer to the string after the match 351 * 352 * Test if a string (typically from a kernel parameter) matches a specified 353 * PCI device. The string may be of one of the following formats: 354 * 355 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 356 * pci:<vendor>:<device>[:<subvendor>:<subdevice>] 357 * 358 * The first format specifies a PCI bus/device/function address which 359 * may change if new hardware is inserted, if motherboard firmware changes, 360 * or due to changes caused in kernel parameters. If the domain is 361 * left unspecified, it is taken to be 0. In order to be robust against 362 * bus renumbering issues, a path of PCI device/function numbers may be used 363 * to address the specific device. The path for a device can be determined 364 * through the use of 'lspci -t'. 365 * 366 * The second format matches devices using IDs in the configuration 367 * space which may match multiple devices in the system. A value of 0 368 * for any field will match all devices. (Note: this differs from 369 * in-kernel code that uses PCI_ANY_ID which is ~0; this is for 370 * legacy reasons and convenience so users don't have to specify 371 * FFFFFFFFs on the command line.) 372 * 373 * Returns 1 if the string matches the device, 0 if it does not and 374 * a negative error code if the string cannot be parsed. 375 */ 376static int pci_dev_str_match(struct pci_dev *dev, const char *p, 377 const char **endptr) 378{ 379 int ret; 380 int count; 381 unsigned short vendor, device, subsystem_vendor, subsystem_device; 382 383 if (strncmp(p, "pci:", 4) == 0) { 384 /* PCI vendor/device (subvendor/subdevice) IDs are specified */ 385 p += 4; 386 ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device, 387 &subsystem_vendor, &subsystem_device, &count); 388 if (ret != 4) { 389 ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count); 390 if (ret != 2) 391 return -EINVAL; 392 393 subsystem_vendor = 0; 394 subsystem_device = 0; 395 } 396 397 p += count; 398 399 if ((!vendor || vendor == dev->vendor) && 400 (!device || device == dev->device) && 401 (!subsystem_vendor || 402 subsystem_vendor == dev->subsystem_vendor) && 403 (!subsystem_device || 404 subsystem_device == dev->subsystem_device)) 405 goto found; 406 } else { 407 /* 408 * PCI Bus, Device, Function IDs are specified 409 * (optionally, may include a path of devfns following it) 410 */ 411 ret = pci_dev_str_match_path(dev, p, &p); 412 if (ret < 0) 413 return ret; 414 else if (ret) 415 goto found; 416 } 417 418 *endptr = p; 419 return 0; 420 421found: 422 *endptr = p; 423 return 1; 424} 425 426static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, 427 u8 pos, int cap) 428{ 429 return PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, cap, NULL, bus, devfn); 430} 431 432u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) 433{ 434 return __pci_find_next_cap(dev->bus, dev->devfn, 435 pos + PCI_CAP_LIST_NEXT, cap); 436} 437EXPORT_SYMBOL_GPL(pci_find_next_capability); 438 439static u8 __pci_bus_find_cap_start(struct pci_bus *bus, 440 unsigned int devfn, u8 hdr_type) 441{ 442 u16 status; 443 444 pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status); 445 if (!(status & PCI_STATUS_CAP_LIST)) 446 return 0; 447 448 switch (hdr_type) { 449 case PCI_HEADER_TYPE_NORMAL: 450 case PCI_HEADER_TYPE_BRIDGE: 451 return PCI_CAPABILITY_LIST; 452 case PCI_HEADER_TYPE_CARDBUS: 453 return PCI_CB_CAPABILITY_LIST; 454 } 455 456 return 0; 457} 458 459/** 460 * pci_find_capability - query for devices' capabilities 461 * @dev: PCI device to query 462 * @cap: capability code 463 * 464 * Tell if a device supports a given PCI capability. 465 * Returns the address of the requested capability structure within the 466 * device's PCI configuration space or 0 in case the device does not 467 * support it. Possible values for @cap include: 468 * 469 * %PCI_CAP_ID_PM Power Management 470 * %PCI_CAP_ID_AGP Accelerated Graphics Port 471 * %PCI_CAP_ID_VPD Vital Product Data 472 * %PCI_CAP_ID_SLOTID Slot Identification 473 * %PCI_CAP_ID_MSI Message Signalled Interrupts 474 * %PCI_CAP_ID_CHSWP CompactPCI HotSwap 475 * %PCI_CAP_ID_PCIX PCI-X 476 * %PCI_CAP_ID_EXP PCI Express 477 */ 478u8 pci_find_capability(struct pci_dev *dev, int cap) 479{ 480 u8 pos; 481 482 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 483 if (pos) 484 pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap); 485 486 return pos; 487} 488EXPORT_SYMBOL(pci_find_capability); 489 490/** 491 * pci_bus_find_capability - query for devices' capabilities 492 * @bus: the PCI bus to query 493 * @devfn: PCI device to query 494 * @cap: capability code 495 * 496 * Like pci_find_capability() but works for PCI devices that do not have a 497 * pci_dev structure set up yet. 498 * 499 * Returns the address of the requested capability structure within the 500 * device's PCI configuration space or 0 in case the device does not 501 * support it. 502 */ 503u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap) 504{ 505 u8 hdr_type, pos; 506 507 pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type); 508 509 pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & PCI_HEADER_TYPE_MASK); 510 if (pos) 511 pos = __pci_find_next_cap(bus, devfn, pos, cap); 512 513 return pos; 514} 515EXPORT_SYMBOL(pci_bus_find_capability); 516 517/** 518 * pci_find_next_ext_capability - Find an extended capability 519 * @dev: PCI device to query 520 * @start: address at which to start looking (0 to start at beginning of list) 521 * @cap: capability code 522 * 523 * Returns the address of the next matching extended capability structure 524 * within the device's PCI configuration space or 0 if the device does 525 * not support it. Some capabilities can occur several times, e.g., the 526 * vendor-specific capability, and this provides a way to find them all. 527 */ 528u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) 529{ 530 if (dev->cfg_size <= PCI_CFG_SPACE_SIZE) 531 return 0; 532 533 return PCI_FIND_NEXT_EXT_CAP(pci_bus_read_config, start, cap, 534 NULL, dev->bus, dev->devfn); 535} 536EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); 537 538/** 539 * pci_find_ext_capability - Find an extended capability 540 * @dev: PCI device to query 541 * @cap: capability code 542 * 543 * Returns the address of the requested extended capability structure 544 * within the device's PCI configuration space or 0 if the device does 545 * not support it. Possible values for @cap include: 546 * 547 * %PCI_EXT_CAP_ID_ERR Advanced Error Reporting 548 * %PCI_EXT_CAP_ID_VC Virtual Channel 549 * %PCI_EXT_CAP_ID_DSN Device Serial Number 550 * %PCI_EXT_CAP_ID_PWR Power Budgeting 551 */ 552u16 pci_find_ext_capability(struct pci_dev *dev, int cap) 553{ 554 return pci_find_next_ext_capability(dev, 0, cap); 555} 556EXPORT_SYMBOL_GPL(pci_find_ext_capability); 557 558/** 559 * pci_get_dsn - Read and return the 8-byte Device Serial Number 560 * @dev: PCI device to query 561 * 562 * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial 563 * Number. 564 * 565 * Returns the DSN, or zero if the capability does not exist. 566 */ 567u64 pci_get_dsn(struct pci_dev *dev) 568{ 569 u32 dword; 570 u64 dsn; 571 int pos; 572 573 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN); 574 if (!pos) 575 return 0; 576 577 /* 578 * The Device Serial Number is two dwords offset 4 bytes from the 579 * capability position. The specification says that the first dword is 580 * the lower half, and the second dword is the upper half. 581 */ 582 pos += 4; 583 pci_read_config_dword(dev, pos, &dword); 584 dsn = (u64)dword; 585 pci_read_config_dword(dev, pos + 4, &dword); 586 dsn |= ((u64)dword) << 32; 587 588 return dsn; 589} 590EXPORT_SYMBOL_GPL(pci_get_dsn); 591 592static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) 593{ 594 int rc; 595 u8 cap, mask; 596 597 if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST) 598 mask = HT_3BIT_CAP_MASK; 599 else 600 mask = HT_5BIT_CAP_MASK; 601 602 pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, pos, 603 PCI_CAP_ID_HT, NULL, dev->bus, dev->devfn); 604 while (pos) { 605 rc = pci_read_config_byte(dev, pos + 3, &cap); 606 if (rc != PCIBIOS_SUCCESSFUL) 607 return 0; 608 609 if ((cap & mask) == ht_cap) 610 return pos; 611 612 pos = PCI_FIND_NEXT_CAP(pci_bus_read_config, 613 pos + PCI_CAP_LIST_NEXT, 614 PCI_CAP_ID_HT, NULL, dev->bus, 615 dev->devfn); 616 } 617 618 return 0; 619} 620 621/** 622 * pci_find_next_ht_capability - query a device's HyperTransport capabilities 623 * @dev: PCI device to query 624 * @pos: Position from which to continue searching 625 * @ht_cap: HyperTransport capability code 626 * 627 * To be used in conjunction with pci_find_ht_capability() to search for 628 * all capabilities matching @ht_cap. @pos should always be a value returned 629 * from pci_find_ht_capability(). 630 * 631 * NB. To be 100% safe against broken PCI devices, the caller should take 632 * steps to avoid an infinite loop. 633 */ 634u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap) 635{ 636 return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap); 637} 638EXPORT_SYMBOL_GPL(pci_find_next_ht_capability); 639 640/** 641 * pci_find_ht_capability - query a device's HyperTransport capabilities 642 * @dev: PCI device to query 643 * @ht_cap: HyperTransport capability code 644 * 645 * Tell if a device supports a given HyperTransport capability. 646 * Returns an address within the device's PCI configuration space 647 * or 0 in case the device does not support the request capability. 648 * The address points to the PCI capability, of type PCI_CAP_ID_HT, 649 * which has a HyperTransport capability matching @ht_cap. 650 */ 651u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap) 652{ 653 u8 pos; 654 655 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 656 if (pos) 657 pos = __pci_find_next_ht_cap(dev, pos, ht_cap); 658 659 return pos; 660} 661EXPORT_SYMBOL_GPL(pci_find_ht_capability); 662 663/** 664 * pci_find_vsec_capability - Find a vendor-specific extended capability 665 * @dev: PCI device to query 666 * @vendor: Vendor ID for which capability is defined 667 * @cap: Vendor-specific capability ID 668 * 669 * If @dev has Vendor ID @vendor, search for a VSEC capability with 670 * VSEC ID @cap. If found, return the capability offset in 671 * config space; otherwise return 0. 672 */ 673u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap) 674{ 675 u16 vsec = 0; 676 u32 header; 677 int ret; 678 679 if (vendor != dev->vendor) 680 return 0; 681 682 while ((vsec = pci_find_next_ext_capability(dev, vsec, 683 PCI_EXT_CAP_ID_VNDR))) { 684 ret = pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER, &header); 685 if (ret != PCIBIOS_SUCCESSFUL) 686 continue; 687 688 if (PCI_VNDR_HEADER_ID(header) == cap) 689 return vsec; 690 } 691 692 return 0; 693} 694EXPORT_SYMBOL_GPL(pci_find_vsec_capability); 695 696/** 697 * pci_find_dvsec_capability - Find DVSEC for vendor 698 * @dev: PCI device to query 699 * @vendor: Vendor ID to match for the DVSEC 700 * @dvsec: Designated Vendor-specific capability ID 701 * 702 * If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability 703 * offset in config space; otherwise return 0. 704 */ 705u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec) 706{ 707 int pos; 708 709 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC); 710 if (!pos) 711 return 0; 712 713 while (pos) { 714 u16 v, id; 715 716 pci_read_config_word(dev, pos + PCI_DVSEC_HEADER1, &v); 717 pci_read_config_word(dev, pos + PCI_DVSEC_HEADER2, &id); 718 if (vendor == v && dvsec == id) 719 return pos; 720 721 pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC); 722 } 723 724 return 0; 725} 726EXPORT_SYMBOL_GPL(pci_find_dvsec_capability); 727 728/** 729 * pci_find_parent_resource - return resource region of parent bus of given 730 * region 731 * @dev: PCI device structure contains resources to be searched 732 * @res: child resource record for which parent is sought 733 * 734 * For given resource region of given device, return the resource region of 735 * parent bus the given region is contained in. 736 */ 737struct resource *pci_find_parent_resource(const struct pci_dev *dev, 738 struct resource *res) 739{ 740 const struct pci_bus *bus = dev->bus; 741 struct resource *r; 742 743 pci_bus_for_each_resource(bus, r) { 744 if (!r) 745 continue; 746 if (resource_contains(r, res)) { 747 748 /* 749 * If the window is prefetchable but the BAR is 750 * not, the allocator made a mistake. 751 */ 752 if (r->flags & IORESOURCE_PREFETCH && 753 !(res->flags & IORESOURCE_PREFETCH)) 754 return NULL; 755 756 /* 757 * If we're below a transparent bridge, there may 758 * be both a positively-decoded aperture and a 759 * subtractively-decoded region that contain the BAR. 760 * We want the positively-decoded one, so this depends 761 * on pci_bus_for_each_resource() giving us those 762 * first. 763 */ 764 return r; 765 } 766 } 767 return NULL; 768} 769EXPORT_SYMBOL(pci_find_parent_resource); 770 771/** 772 * pci_find_resource - Return matching PCI device resource 773 * @dev: PCI device to query 774 * @res: Resource to look for 775 * 776 * Goes over standard PCI resources (BARs) and checks if the given resource 777 * is partially or fully contained in any of them. In that case the 778 * matching resource is returned, %NULL otherwise. 779 */ 780struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res) 781{ 782 int i; 783 784 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 785 struct resource *r = &dev->resource[i]; 786 787 if (r->start && resource_contains(r, res)) 788 return r; 789 } 790 791 return NULL; 792} 793EXPORT_SYMBOL(pci_find_resource); 794 795/** 796 * pci_resource_name - Return the name of the PCI resource 797 * @dev: PCI device to query 798 * @i: index of the resource 799 * 800 * Return the standard PCI resource (BAR) name according to their index. 801 */ 802const char *pci_resource_name(struct pci_dev *dev, unsigned int i) 803{ 804 static const char * const bar_name[] = { 805 "BAR 0", 806 "BAR 1", 807 "BAR 2", 808 "BAR 3", 809 "BAR 4", 810 "BAR 5", 811 "ROM", 812#ifdef CONFIG_PCI_IOV 813 "VF BAR 0", 814 "VF BAR 1", 815 "VF BAR 2", 816 "VF BAR 3", 817 "VF BAR 4", 818 "VF BAR 5", 819#endif 820 "bridge window", /* "io" included in %pR */ 821 "bridge window", /* "mem" included in %pR */ 822 "bridge window", /* "mem pref" included in %pR */ 823 }; 824 static const char * const cardbus_name[] = { 825 "BAR 1", 826 "unknown", 827 "unknown", 828 "unknown", 829 "unknown", 830 "unknown", 831#ifdef CONFIG_PCI_IOV 832 "unknown", 833 "unknown", 834 "unknown", 835 "unknown", 836 "unknown", 837 "unknown", 838#endif 839 "CardBus bridge window 0", /* I/O */ 840 "CardBus bridge window 1", /* I/O */ 841 "CardBus bridge window 0", /* mem */ 842 "CardBus bridge window 1", /* mem */ 843 }; 844 845 if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS && 846 i < ARRAY_SIZE(cardbus_name)) 847 return cardbus_name[i]; 848 849 if (i < ARRAY_SIZE(bar_name)) 850 return bar_name[i]; 851 852 return "unknown"; 853} 854 855/** 856 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos 857 * @dev: the PCI device to operate on 858 * @pos: config space offset of status word 859 * @mask: mask of bit(s) to care about in status word 860 * 861 * Return 1 when mask bit(s) in status word clear, 0 otherwise. 862 */ 863int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask) 864{ 865 int i; 866 867 /* Wait for Transaction Pending bit clean */ 868 for (i = 0; i < 4; i++) { 869 u16 status; 870 if (i) 871 msleep((1 << (i - 1)) * 100); 872 873 pci_read_config_word(dev, pos, &status); 874 if (!(status & mask)) 875 return 1; 876 } 877 878 return 0; 879} 880 881static int pci_acs_enable; 882 883/** 884 * pci_request_acs - ask for ACS to be enabled if supported 885 */ 886void pci_request_acs(void) 887{ 888 pci_acs_enable = 1; 889} 890 891static const char *disable_acs_redir_param; 892static const char *config_acs_param; 893 894struct pci_acs { 895 u16 cap; 896 u16 ctrl; 897 u16 fw_ctrl; 898}; 899 900static void __pci_config_acs(struct pci_dev *dev, struct pci_acs *caps, 901 const char *p, const u16 acs_mask, const u16 acs_flags) 902{ 903 u16 flags = acs_flags; 904 u16 mask = acs_mask; 905 char *delimit; 906 int ret = 0; 907 908 if (!p) 909 return; 910 911 while (*p) { 912 if (!acs_mask) { 913 /* Check for ACS flags */ 914 delimit = strstr(p, "@"); 915 if (delimit) { 916 int end; 917 u32 shift = 0; 918 919 end = delimit - p - 1; 920 mask = 0; 921 flags = 0; 922 923 while (end > -1) { 924 if (*(p + end) == '0') { 925 mask |= 1 << shift; 926 shift++; 927 end--; 928 } else if (*(p + end) == '1') { 929 mask |= 1 << shift; 930 flags |= 1 << shift; 931 shift++; 932 end--; 933 } else if ((*(p + end) == 'x') || (*(p + end) == 'X')) { 934 shift++; 935 end--; 936 } else { 937 pci_err(dev, "Invalid ACS flags... Ignoring\n"); 938 return; 939 } 940 } 941 p = delimit + 1; 942 } else { 943 pci_err(dev, "ACS Flags missing\n"); 944 return; 945 } 946 } 947 948 if (mask & ~(PCI_ACS_SV | PCI_ACS_TB | PCI_ACS_RR | PCI_ACS_CR | 949 PCI_ACS_UF | PCI_ACS_EC | PCI_ACS_DT)) { 950 pci_err(dev, "Invalid ACS flags specified\n"); 951 return; 952 } 953 954 ret = pci_dev_str_match(dev, p, &p); 955 if (ret < 0) { 956 pr_info_once("PCI: Can't parse ACS command line parameter\n"); 957 break; 958 } else if (ret == 1) { 959 /* Found a match */ 960 break; 961 } 962 963 if (*p != ';' && *p != ',') { 964 /* End of param or invalid format */ 965 break; 966 } 967 p++; 968 } 969 970 if (ret != 1) 971 return; 972 973 if (!pci_dev_specific_disable_acs_redir(dev)) 974 return; 975 976 pci_dbg(dev, "ACS mask = %#06x\n", mask); 977 pci_dbg(dev, "ACS flags = %#06x\n", flags); 978 pci_dbg(dev, "ACS control = %#06x\n", caps->ctrl); 979 pci_dbg(dev, "ACS fw_ctrl = %#06x\n", caps->fw_ctrl); 980 981 /* 982 * For mask bits that are 0, copy them from the firmware setting 983 * and apply flags for all the mask bits that are 1. 984 */ 985 caps->ctrl = (caps->fw_ctrl & ~mask) | (flags & mask); 986 987 pci_info(dev, "Configured ACS to %#06x\n", caps->ctrl); 988} 989 990/** 991 * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities 992 * @dev: the PCI device 993 * @caps: default ACS controls 994 */ 995static void pci_std_enable_acs(struct pci_dev *dev, struct pci_acs *caps) 996{ 997 /* Source Validation */ 998 caps->ctrl |= (caps->cap & PCI_ACS_SV); 999 1000 /* P2P Request Redirect */ 1001 caps->ctrl |= (caps->cap & PCI_ACS_RR); 1002 1003 /* P2P Completion Redirect */ 1004 caps->ctrl |= (caps->cap & PCI_ACS_CR); 1005 1006 /* Upstream Forwarding */ 1007 caps->ctrl |= (caps->cap & PCI_ACS_UF); 1008 1009 /* Enable Translation Blocking for external devices and noats */ 1010 if (pci_ats_disabled() || dev->external_facing || dev->untrusted) 1011 caps->ctrl |= (caps->cap & PCI_ACS_TB); 1012} 1013 1014/** 1015 * pci_enable_acs - enable ACS if hardware support it 1016 * @dev: the PCI device 1017 */ 1018void pci_enable_acs(struct pci_dev *dev) 1019{ 1020 struct pci_acs caps; 1021 bool enable_acs = false; 1022 int pos; 1023 1024 /* If an iommu is present we start with kernel default caps */ 1025 if (pci_acs_enable) { 1026 if (pci_dev_specific_enable_acs(dev)) 1027 enable_acs = true; 1028 } 1029 1030 pos = dev->acs_cap; 1031 if (!pos) 1032 return; 1033 1034 pci_read_config_word(dev, pos + PCI_ACS_CAP, &caps.cap); 1035 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &caps.ctrl); 1036 caps.fw_ctrl = caps.ctrl; 1037 1038 if (enable_acs) 1039 pci_std_enable_acs(dev, &caps); 1040 1041 /* 1042 * Always apply caps from the command line, even if there is no iommu. 1043 * Trust that the admin has a reason to change the ACS settings. 1044 */ 1045 __pci_config_acs(dev, &caps, disable_acs_redir_param, 1046 PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC, 1047 ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC)); 1048 __pci_config_acs(dev, &caps, config_acs_param, 0, 0); 1049 1050 pci_write_config_word(dev, pos + PCI_ACS_CTRL, caps.ctrl); 1051} 1052 1053/** 1054 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up) 1055 * @dev: PCI device to have its BARs restored 1056 * 1057 * Restore the BAR values for a given device, so as to make it 1058 * accessible by its driver. 1059 */ 1060static void pci_restore_bars(struct pci_dev *dev) 1061{ 1062 int i; 1063 1064 for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) 1065 pci_update_resource(dev, i); 1066} 1067 1068static inline bool platform_pci_power_manageable(struct pci_dev *dev) 1069{ 1070 if (pci_use_mid_pm()) 1071 return true; 1072 1073 return acpi_pci_power_manageable(dev); 1074} 1075 1076static inline int platform_pci_set_power_state(struct pci_dev *dev, 1077 pci_power_t t) 1078{ 1079 if (pci_use_mid_pm()) 1080 return mid_pci_set_power_state(dev, t); 1081 1082 return acpi_pci_set_power_state(dev, t); 1083} 1084 1085static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev) 1086{ 1087 if (pci_use_mid_pm()) 1088 return mid_pci_get_power_state(dev); 1089 1090 return acpi_pci_get_power_state(dev); 1091} 1092 1093static inline void platform_pci_refresh_power_state(struct pci_dev *dev) 1094{ 1095 if (!pci_use_mid_pm()) 1096 acpi_pci_refresh_power_state(dev); 1097} 1098 1099static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev) 1100{ 1101 if (pci_use_mid_pm()) 1102 return PCI_POWER_ERROR; 1103 1104 return acpi_pci_choose_state(dev); 1105} 1106 1107static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable) 1108{ 1109 if (pci_use_mid_pm()) 1110 return PCI_POWER_ERROR; 1111 1112 return acpi_pci_wakeup(dev, enable); 1113} 1114 1115static inline bool platform_pci_need_resume(struct pci_dev *dev) 1116{ 1117 if (pci_use_mid_pm()) 1118 return false; 1119 1120 return acpi_pci_need_resume(dev); 1121} 1122 1123static inline bool platform_pci_bridge_d3(struct pci_dev *dev) 1124{ 1125 if (pci_use_mid_pm()) 1126 return false; 1127 1128 return acpi_pci_bridge_d3(dev); 1129} 1130 1131/** 1132 * pci_update_current_state - Read power state of given device and cache it 1133 * @dev: PCI device to handle. 1134 * @state: State to cache in case the device doesn't have the PM capability 1135 * 1136 * The power state is read from the PMCSR register, which however is 1137 * inaccessible in D3cold. The platform firmware is therefore queried first 1138 * to detect accessibility of the register. In case the platform firmware 1139 * reports an incorrect state or the device isn't power manageable by the 1140 * platform at all, we try to detect D3cold by testing accessibility of the 1141 * vendor ID in config space. 1142 */ 1143void pci_update_current_state(struct pci_dev *dev, pci_power_t state) 1144{ 1145 if (platform_pci_get_power_state(dev) == PCI_D3cold) { 1146 dev->current_state = PCI_D3cold; 1147 } else if (dev->pm_cap) { 1148 u16 pmcsr; 1149 1150 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1151 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1152 dev->current_state = PCI_D3cold; 1153 return; 1154 } 1155 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1156 } else { 1157 dev->current_state = state; 1158 } 1159} 1160 1161/** 1162 * pci_refresh_power_state - Refresh the given device's power state data 1163 * @dev: Target PCI device. 1164 * 1165 * Ask the platform to refresh the devices power state information and invoke 1166 * pci_update_current_state() to update its current PCI power state. 1167 */ 1168void pci_refresh_power_state(struct pci_dev *dev) 1169{ 1170 platform_pci_refresh_power_state(dev); 1171 pci_update_current_state(dev, dev->current_state); 1172} 1173 1174/** 1175 * pci_platform_power_transition - Use platform to change device power state 1176 * @dev: PCI device to handle. 1177 * @state: State to put the device into. 1178 */ 1179int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state) 1180{ 1181 int error; 1182 1183 error = platform_pci_set_power_state(dev, state); 1184 if (!error) 1185 pci_update_current_state(dev, state); 1186 else if (!dev->pm_cap) /* Fall back to PCI_D0 */ 1187 dev->current_state = PCI_D0; 1188 1189 return error; 1190} 1191EXPORT_SYMBOL_GPL(pci_platform_power_transition); 1192 1193static int pci_resume_one(struct pci_dev *pci_dev, void *ign) 1194{ 1195 pm_request_resume(&pci_dev->dev); 1196 return 0; 1197} 1198 1199/** 1200 * pci_resume_bus - Walk given bus and runtime resume devices on it 1201 * @bus: Top bus of the subtree to walk. 1202 */ 1203void pci_resume_bus(struct pci_bus *bus) 1204{ 1205 if (bus) 1206 pci_walk_bus(bus, pci_resume_one, NULL); 1207} 1208 1209static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) 1210{ 1211 int delay = 1; 1212 bool retrain = false; 1213 struct pci_dev *root, *bridge; 1214 1215 root = pcie_find_root_port(dev); 1216 1217 if (pci_is_pcie(dev)) { 1218 bridge = pci_upstream_bridge(dev); 1219 if (bridge) 1220 retrain = true; 1221 } 1222 1223 /* 1224 * The caller has already waited long enough after a reset that the 1225 * device should respond to config requests, but it may respond 1226 * with Request Retry Status (RRS) if it needs more time to 1227 * initialize. 1228 * 1229 * If the device is below a Root Port with Configuration RRS 1230 * Software Visibility enabled, reading the Vendor ID returns a 1231 * special data value if the device responded with RRS. Read the 1232 * Vendor ID until we get non-RRS status. 1233 * 1234 * If there's no Root Port or Configuration RRS Software Visibility 1235 * is not enabled, the device may still respond with RRS, but 1236 * hardware may retry the config request. If no retries receive 1237 * Successful Completion, hardware generally synthesizes ~0 1238 * (PCI_ERROR_RESPONSE) data to complete the read. Reading Vendor 1239 * ID for VFs and non-existent devices also returns ~0, so read the 1240 * Command register until it returns something other than ~0. 1241 */ 1242 for (;;) { 1243 u32 id; 1244 1245 if (pci_dev_is_disconnected(dev)) { 1246 pci_dbg(dev, "disconnected; not waiting\n"); 1247 return -ENOTTY; 1248 } 1249 1250 if (root && root->config_rrs_sv) { 1251 pci_read_config_dword(dev, PCI_VENDOR_ID, &id); 1252 if (!pci_bus_rrs_vendor_id(id)) 1253 break; 1254 } else { 1255 pci_read_config_dword(dev, PCI_COMMAND, &id); 1256 if (!PCI_POSSIBLE_ERROR(id)) 1257 break; 1258 } 1259 1260 if (delay > timeout) { 1261 pci_warn(dev, "not ready %dms after %s; giving up\n", 1262 delay - 1, reset_type); 1263 return -ENOTTY; 1264 } 1265 1266 if (delay > PCI_RESET_WAIT) { 1267 if (retrain) { 1268 retrain = false; 1269 if (pcie_failed_link_retrain(bridge) == 0) { 1270 delay = 1; 1271 continue; 1272 } 1273 } 1274 pci_info(dev, "not ready %dms after %s; waiting\n", 1275 delay - 1, reset_type); 1276 } 1277 1278 msleep(delay); 1279 delay *= 2; 1280 } 1281 1282 if (delay > PCI_RESET_WAIT) 1283 pci_info(dev, "ready %dms after %s\n", delay - 1, 1284 reset_type); 1285 else 1286 pci_dbg(dev, "ready %dms after %s\n", delay - 1, 1287 reset_type); 1288 1289 return 0; 1290} 1291 1292/** 1293 * pci_power_up - Put the given device into D0 1294 * @dev: PCI device to power up 1295 * 1296 * On success, return 0 or 1, depending on whether or not it is necessary to 1297 * restore the device's BARs subsequently (1 is returned in that case). 1298 * 1299 * On failure, return a negative error code. Always return failure if @dev 1300 * lacks a Power Management Capability, even if the platform was able to 1301 * put the device in D0 via non-PCI means. 1302 */ 1303int pci_power_up(struct pci_dev *dev) 1304{ 1305 bool need_restore; 1306 pci_power_t state; 1307 u16 pmcsr; 1308 1309 platform_pci_set_power_state(dev, PCI_D0); 1310 1311 if (!dev->pm_cap) { 1312 state = platform_pci_get_power_state(dev); 1313 if (state == PCI_UNKNOWN) 1314 dev->current_state = PCI_D0; 1315 else 1316 dev->current_state = state; 1317 1318 return -EIO; 1319 } 1320 1321 if (pci_dev_is_disconnected(dev)) { 1322 dev->current_state = PCI_D3cold; 1323 return -EIO; 1324 } 1325 1326 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1327 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1328 pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n", 1329 pci_power_name(dev->current_state)); 1330 dev->current_state = PCI_D3cold; 1331 return -EIO; 1332 } 1333 1334 state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1335 1336 need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) && 1337 !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 1338 1339 if (state == PCI_D0) 1340 goto end; 1341 1342 /* 1343 * Force the entire word to 0. This doesn't affect PME_Status, disables 1344 * PME_En, and sets PowerState to 0. 1345 */ 1346 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0); 1347 1348 /* Mandatory transition delays; see PCI PM 1.2. */ 1349 if (state == PCI_D3hot) 1350 pci_dev_d3_sleep(dev); 1351 else if (state == PCI_D2) 1352 udelay(PCI_PM_D2_DELAY); 1353 1354end: 1355 dev->current_state = PCI_D0; 1356 if (need_restore) 1357 return 1; 1358 1359 return 0; 1360} 1361 1362/** 1363 * pci_set_full_power_state - Put a PCI device into D0 and update its state 1364 * @dev: PCI device to power up 1365 * @locked: whether pci_bus_sem is held 1366 * 1367 * Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register 1368 * to confirm the state change, restore its BARs if they might be lost and 1369 * reconfigure ASPM in accordance with the new power state. 1370 * 1371 * If pci_restore_state() is going to be called right after a power state change 1372 * to D0, it is more efficient to use pci_power_up() directly instead of this 1373 * function. 1374 */ 1375static int pci_set_full_power_state(struct pci_dev *dev, bool locked) 1376{ 1377 u16 pmcsr; 1378 int ret; 1379 1380 ret = pci_power_up(dev); 1381 if (ret < 0) { 1382 if (dev->current_state == PCI_D0) 1383 return 0; 1384 1385 return ret; 1386 } 1387 1388 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1389 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1390 if (dev->current_state != PCI_D0) { 1391 pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n", 1392 pci_power_name(dev->current_state)); 1393 } else if (ret > 0) { 1394 /* 1395 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT 1396 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning 1397 * from D3hot to D0 _may_ perform an internal reset, thereby 1398 * going to "D0 Uninitialized" rather than "D0 Initialized". 1399 * For example, at least some versions of the 3c905B and the 1400 * 3c556B exhibit this behaviour. 1401 * 1402 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave 1403 * devices in a D3hot state at boot. Consequently, we need to 1404 * restore at least the BARs so that the device will be 1405 * accessible to its driver. 1406 */ 1407 pci_restore_bars(dev); 1408 } 1409 1410 if (dev->bus->self) 1411 pcie_aspm_pm_state_change(dev->bus->self, locked); 1412 1413 return 0; 1414} 1415 1416/** 1417 * __pci_dev_set_current_state - Set current state of a PCI device 1418 * @dev: Device to handle 1419 * @data: pointer to state to be set 1420 */ 1421static int __pci_dev_set_current_state(struct pci_dev *dev, void *data) 1422{ 1423 pci_power_t state = *(pci_power_t *)data; 1424 1425 dev->current_state = state; 1426 return 0; 1427} 1428 1429/** 1430 * pci_bus_set_current_state - Walk given bus and set current state of devices 1431 * @bus: Top bus of the subtree to walk. 1432 * @state: state to be set 1433 */ 1434void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state) 1435{ 1436 if (bus) 1437 pci_walk_bus(bus, __pci_dev_set_current_state, &state); 1438} 1439 1440static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state, bool locked) 1441{ 1442 if (!bus) 1443 return; 1444 1445 if (locked) 1446 pci_walk_bus_locked(bus, __pci_dev_set_current_state, &state); 1447 else 1448 pci_walk_bus(bus, __pci_dev_set_current_state, &state); 1449} 1450 1451/** 1452 * pci_set_low_power_state - Put a PCI device into a low-power state. 1453 * @dev: PCI device to handle. 1454 * @state: PCI power state (D1, D2, D3hot) to put the device into. 1455 * @locked: whether pci_bus_sem is held 1456 * 1457 * Use the device's PCI_PM_CTRL register to put it into a low-power state. 1458 * 1459 * RETURN VALUE: 1460 * -EINVAL if the requested state is invalid. 1461 * -EIO if device does not support PCI PM or its PM capabilities register has a 1462 * wrong version, or device doesn't support the requested state. 1463 * 0 if device already is in the requested state. 1464 * 0 if device's power state has been successfully changed. 1465 */ 1466static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool locked) 1467{ 1468 u16 pmcsr; 1469 1470 if (!dev->pm_cap) 1471 return -EIO; 1472 1473 /* 1474 * Validate transition: We can enter D0 from any state, but if 1475 * we're already in a low-power state, we can only go deeper. E.g., 1476 * we can go from D1 to D3, but we can't go directly from D3 to D1; 1477 * we'd have to go from D3 to D0, then to D1. 1478 */ 1479 if (dev->current_state <= PCI_D3cold && dev->current_state > state) { 1480 pci_dbg(dev, "Invalid power transition (from %s to %s)\n", 1481 pci_power_name(dev->current_state), 1482 pci_power_name(state)); 1483 return -EINVAL; 1484 } 1485 1486 /* Check if this device supports the desired state */ 1487 if ((state == PCI_D1 && !dev->d1_support) 1488 || (state == PCI_D2 && !dev->d2_support)) 1489 return -EIO; 1490 1491 if (dev->current_state == state) 1492 return 0; 1493 1494 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1495 if (PCI_POSSIBLE_ERROR(pmcsr)) { 1496 pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n", 1497 pci_power_name(dev->current_state), 1498 pci_power_name(state)); 1499 dev->current_state = PCI_D3cold; 1500 return -EIO; 1501 } 1502 1503 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 1504 pmcsr |= state; 1505 1506 /* Enter specified state */ 1507 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 1508 1509 /* Mandatory power management transition delays; see PCI PM 1.2. */ 1510 if (state == PCI_D3hot) 1511 pci_dev_d3_sleep(dev); 1512 else if (state == PCI_D2) 1513 udelay(PCI_PM_D2_DELAY); 1514 1515 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1516 dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK; 1517 if (dev->current_state != state) 1518 pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n", 1519 pci_power_name(dev->current_state), 1520 pci_power_name(state)); 1521 1522 if (dev->bus->self) 1523 pcie_aspm_pm_state_change(dev->bus->self, locked); 1524 1525 return 0; 1526} 1527 1528static int __pci_set_power_state(struct pci_dev *dev, pci_power_t state, bool locked) 1529{ 1530 int error; 1531 1532 /* Bound the state we're entering */ 1533 if (state > PCI_D3cold) 1534 state = PCI_D3cold; 1535 else if (state < PCI_D0) 1536 state = PCI_D0; 1537 else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev)) 1538 1539 /* 1540 * If the device or the parent bridge do not support PCI 1541 * PM, ignore the request if we're doing anything other 1542 * than putting it into D0 (which would only happen on 1543 * boot). 1544 */ 1545 return 0; 1546 1547 /* Check if we're already there */ 1548 if (dev->current_state == state) 1549 return 0; 1550 1551 if (state == PCI_D0) 1552 return pci_set_full_power_state(dev, locked); 1553 1554 /* 1555 * This device is quirked not to be put into D3, so don't put it in 1556 * D3 1557 */ 1558 if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3)) 1559 return 0; 1560 1561 if (state == PCI_D3cold) { 1562 /* 1563 * To put the device in D3cold, put it into D3hot in the native 1564 * way, then put it into D3cold using platform ops. 1565 */ 1566 error = pci_set_low_power_state(dev, PCI_D3hot, locked); 1567 1568 if (pci_platform_power_transition(dev, PCI_D3cold)) 1569 return error; 1570 1571 /* Powering off a bridge may power off the whole hierarchy */ 1572 if (dev->current_state == PCI_D3cold) 1573 __pci_bus_set_current_state(dev->subordinate, PCI_D3cold, locked); 1574 } else { 1575 error = pci_set_low_power_state(dev, state, locked); 1576 1577 if (pci_platform_power_transition(dev, state)) 1578 return error; 1579 } 1580 1581 return 0; 1582} 1583 1584/** 1585 * pci_set_power_state - Set the power state of a PCI device 1586 * @dev: PCI device to handle. 1587 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. 1588 * 1589 * Transition a device to a new power state, using the platform firmware and/or 1590 * the device's PCI PM registers. 1591 * 1592 * RETURN VALUE: 1593 * -EINVAL if the requested state is invalid. 1594 * -EIO if device does not support PCI PM or its PM capabilities register has a 1595 * wrong version, or device doesn't support the requested state. 1596 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported. 1597 * 0 if device already is in the requested state. 1598 * 0 if the transition is to D3 but D3 is not supported. 1599 * 0 if device's power state has been successfully changed. 1600 */ 1601int pci_set_power_state(struct pci_dev *dev, pci_power_t state) 1602{ 1603 return __pci_set_power_state(dev, state, false); 1604} 1605EXPORT_SYMBOL(pci_set_power_state); 1606 1607int pci_set_power_state_locked(struct pci_dev *dev, pci_power_t state) 1608{ 1609 lockdep_assert_held(&pci_bus_sem); 1610 1611 return __pci_set_power_state(dev, state, true); 1612} 1613EXPORT_SYMBOL(pci_set_power_state_locked); 1614 1615#define PCI_EXP_SAVE_REGS 7 1616 1617static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev, 1618 u16 cap, bool extended) 1619{ 1620 struct pci_cap_saved_state *tmp; 1621 1622 hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) { 1623 if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap) 1624 return tmp; 1625 } 1626 return NULL; 1627} 1628 1629struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap) 1630{ 1631 return _pci_find_saved_cap(dev, cap, false); 1632} 1633 1634struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap) 1635{ 1636 return _pci_find_saved_cap(dev, cap, true); 1637} 1638 1639static int pci_save_pcie_state(struct pci_dev *dev) 1640{ 1641 int i = 0; 1642 struct pci_cap_saved_state *save_state; 1643 u16 *cap; 1644 1645 if (!pci_is_pcie(dev)) 1646 return 0; 1647 1648 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1649 if (!save_state) { 1650 pci_err(dev, "buffer not found in %s\n", __func__); 1651 return -ENOMEM; 1652 } 1653 1654 cap = (u16 *)&save_state->cap.data[0]; 1655 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]); 1656 pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]); 1657 pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]); 1658 pcie_capability_read_word(dev, PCI_EXP_RTCTL, &cap[i++]); 1659 pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]); 1660 pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]); 1661 pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]); 1662 1663 pci_save_aspm_l1ss_state(dev); 1664 pci_save_ltr_state(dev); 1665 1666 return 0; 1667} 1668 1669static void pci_restore_pcie_state(struct pci_dev *dev) 1670{ 1671 int i = 0; 1672 struct pci_cap_saved_state *save_state; 1673 u16 *cap; 1674 1675 /* 1676 * Restore max latencies (in the LTR capability) before enabling 1677 * LTR itself in PCI_EXP_DEVCTL2. 1678 */ 1679 pci_restore_ltr_state(dev); 1680 pci_restore_aspm_l1ss_state(dev); 1681 1682 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1683 if (!save_state) 1684 return; 1685 1686 /* 1687 * Downstream ports reset the LTR enable bit when link goes down. 1688 * Check and re-configure the bit here before restoring device. 1689 * PCIe r5.0, sec 7.5.3.16. 1690 */ 1691 pci_bridge_reconfigure_ltr(dev); 1692 1693 cap = (u16 *)&save_state->cap.data[0]; 1694 pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]); 1695 pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]); 1696 pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]); 1697 pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]); 1698 pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]); 1699 pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]); 1700 pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]); 1701} 1702 1703static int pci_save_pcix_state(struct pci_dev *dev) 1704{ 1705 int pos; 1706 struct pci_cap_saved_state *save_state; 1707 1708 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1709 if (!pos) 1710 return 0; 1711 1712 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1713 if (!save_state) { 1714 pci_err(dev, "buffer not found in %s\n", __func__); 1715 return -ENOMEM; 1716 } 1717 1718 pci_read_config_word(dev, pos + PCI_X_CMD, 1719 (u16 *)save_state->cap.data); 1720 1721 return 0; 1722} 1723 1724static void pci_restore_pcix_state(struct pci_dev *dev) 1725{ 1726 int i = 0, pos; 1727 struct pci_cap_saved_state *save_state; 1728 u16 *cap; 1729 1730 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1731 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1732 if (!save_state || !pos) 1733 return; 1734 cap = (u16 *)&save_state->cap.data[0]; 1735 1736 pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]); 1737} 1738 1739/** 1740 * pci_save_state - save the PCI configuration space of a device before 1741 * suspending 1742 * @dev: PCI device that we're dealing with 1743 */ 1744int pci_save_state(struct pci_dev *dev) 1745{ 1746 int i; 1747 /* XXX: 100% dword access ok here? */ 1748 for (i = 0; i < 16; i++) { 1749 pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]); 1750 pci_dbg(dev, "save config %#04x: %#010x\n", 1751 i * 4, dev->saved_config_space[i]); 1752 } 1753 dev->state_saved = true; 1754 1755 i = pci_save_pcie_state(dev); 1756 if (i != 0) 1757 return i; 1758 1759 i = pci_save_pcix_state(dev); 1760 if (i != 0) 1761 return i; 1762 1763 pci_save_dpc_state(dev); 1764 pci_save_aer_state(dev); 1765 pci_save_ptm_state(dev); 1766 pci_save_tph_state(dev); 1767 return pci_save_vc_state(dev); 1768} 1769EXPORT_SYMBOL(pci_save_state); 1770 1771static void pci_restore_config_dword(struct pci_dev *pdev, int offset, 1772 u32 saved_val, int retry, bool force) 1773{ 1774 u32 val; 1775 1776 pci_read_config_dword(pdev, offset, &val); 1777 if (!force && val == saved_val) 1778 return; 1779 1780 for (;;) { 1781 pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n", 1782 offset, val, saved_val); 1783 pci_write_config_dword(pdev, offset, saved_val); 1784 if (retry-- <= 0) 1785 return; 1786 1787 pci_read_config_dword(pdev, offset, &val); 1788 if (val == saved_val) 1789 return; 1790 1791 mdelay(1); 1792 } 1793} 1794 1795static void pci_restore_config_space_range(struct pci_dev *pdev, 1796 int start, int end, int retry, 1797 bool force) 1798{ 1799 int index; 1800 1801 for (index = end; index >= start; index--) 1802 pci_restore_config_dword(pdev, 4 * index, 1803 pdev->saved_config_space[index], 1804 retry, force); 1805} 1806 1807static void pci_restore_config_space(struct pci_dev *pdev) 1808{ 1809 if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) { 1810 pci_restore_config_space_range(pdev, 10, 15, 0, false); 1811 /* Restore BARs before the command register. */ 1812 pci_restore_config_space_range(pdev, 4, 9, 10, false); 1813 pci_restore_config_space_range(pdev, 0, 3, 0, false); 1814 } else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 1815 pci_restore_config_space_range(pdev, 12, 15, 0, false); 1816 1817 /* 1818 * Force rewriting of prefetch registers to avoid S3 resume 1819 * issues on Intel PCI bridges that occur when these 1820 * registers are not explicitly written. 1821 */ 1822 pci_restore_config_space_range(pdev, 9, 11, 0, true); 1823 pci_restore_config_space_range(pdev, 0, 8, 0, false); 1824 } else { 1825 pci_restore_config_space_range(pdev, 0, 15, 0, false); 1826 } 1827} 1828 1829/** 1830 * pci_restore_state - Restore the saved state of a PCI device 1831 * @dev: PCI device that we're dealing with 1832 */ 1833void pci_restore_state(struct pci_dev *dev) 1834{ 1835 pci_restore_pcie_state(dev); 1836 pci_restore_pasid_state(dev); 1837 pci_restore_pri_state(dev); 1838 pci_restore_ats_state(dev); 1839 pci_restore_vc_state(dev); 1840 pci_restore_rebar_state(dev); 1841 pci_restore_dpc_state(dev); 1842 pci_restore_ptm_state(dev); 1843 pci_restore_tph_state(dev); 1844 1845 pci_aer_clear_status(dev); 1846 pci_restore_aer_state(dev); 1847 1848 pci_restore_config_space(dev); 1849 1850 pci_restore_pcix_state(dev); 1851 pci_restore_msi_state(dev); 1852 1853 /* Restore ACS and IOV configuration state */ 1854 pci_enable_acs(dev); 1855 pci_restore_iov_state(dev); 1856 1857 dev->state_saved = false; 1858} 1859EXPORT_SYMBOL(pci_restore_state); 1860 1861struct pci_saved_state { 1862 u32 config_space[16]; 1863 struct pci_cap_saved_data cap[]; 1864}; 1865 1866/** 1867 * pci_store_saved_state - Allocate and return an opaque struct containing 1868 * the device saved state. 1869 * @dev: PCI device that we're dealing with 1870 * 1871 * Return NULL if no state or error. 1872 */ 1873struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev) 1874{ 1875 struct pci_saved_state *state; 1876 struct pci_cap_saved_state *tmp; 1877 struct pci_cap_saved_data *cap; 1878 size_t size; 1879 1880 if (!dev->state_saved) 1881 return NULL; 1882 1883 size = sizeof(*state) + sizeof(struct pci_cap_saved_data); 1884 1885 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) 1886 size += sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1887 1888 state = kzalloc(size, GFP_KERNEL); 1889 if (!state) 1890 return NULL; 1891 1892 memcpy(state->config_space, dev->saved_config_space, 1893 sizeof(state->config_space)); 1894 1895 cap = state->cap; 1896 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) { 1897 size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1898 memcpy(cap, &tmp->cap, len); 1899 cap = (struct pci_cap_saved_data *)((u8 *)cap + len); 1900 } 1901 /* Empty cap_save terminates list */ 1902 1903 return state; 1904} 1905EXPORT_SYMBOL_GPL(pci_store_saved_state); 1906 1907/** 1908 * pci_load_saved_state - Reload the provided save state into struct pci_dev. 1909 * @dev: PCI device that we're dealing with 1910 * @state: Saved state returned from pci_store_saved_state() 1911 */ 1912int pci_load_saved_state(struct pci_dev *dev, 1913 struct pci_saved_state *state) 1914{ 1915 struct pci_cap_saved_data *cap; 1916 1917 dev->state_saved = false; 1918 1919 if (!state) 1920 return 0; 1921 1922 memcpy(dev->saved_config_space, state->config_space, 1923 sizeof(state->config_space)); 1924 1925 cap = state->cap; 1926 while (cap->size) { 1927 struct pci_cap_saved_state *tmp; 1928 1929 tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended); 1930 if (!tmp || tmp->cap.size != cap->size) 1931 return -EINVAL; 1932 1933 memcpy(tmp->cap.data, cap->data, tmp->cap.size); 1934 cap = (struct pci_cap_saved_data *)((u8 *)cap + 1935 sizeof(struct pci_cap_saved_data) + cap->size); 1936 } 1937 1938 dev->state_saved = true; 1939 return 0; 1940} 1941EXPORT_SYMBOL_GPL(pci_load_saved_state); 1942 1943/** 1944 * pci_load_and_free_saved_state - Reload the save state pointed to by state, 1945 * and free the memory allocated for it. 1946 * @dev: PCI device that we're dealing with 1947 * @state: Pointer to saved state returned from pci_store_saved_state() 1948 */ 1949int pci_load_and_free_saved_state(struct pci_dev *dev, 1950 struct pci_saved_state **state) 1951{ 1952 int ret = pci_load_saved_state(dev, *state); 1953 kfree(*state); 1954 *state = NULL; 1955 return ret; 1956} 1957EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state); 1958 1959int __weak pcibios_enable_device(struct pci_dev *dev, int bars) 1960{ 1961 return pci_enable_resources(dev, bars); 1962} 1963 1964static int pci_host_bridge_enable_device(struct pci_dev *dev) 1965{ 1966 struct pci_host_bridge *host_bridge = pci_find_host_bridge(dev->bus); 1967 int err; 1968 1969 if (host_bridge && host_bridge->enable_device) { 1970 err = host_bridge->enable_device(host_bridge, dev); 1971 if (err) 1972 return err; 1973 } 1974 1975 return 0; 1976} 1977 1978static void pci_host_bridge_disable_device(struct pci_dev *dev) 1979{ 1980 struct pci_host_bridge *host_bridge = pci_find_host_bridge(dev->bus); 1981 1982 if (host_bridge && host_bridge->disable_device) 1983 host_bridge->disable_device(host_bridge, dev); 1984} 1985 1986static int do_pci_enable_device(struct pci_dev *dev, int bars) 1987{ 1988 int err; 1989 struct pci_dev *bridge; 1990 u16 cmd; 1991 u8 pin; 1992 1993 err = pci_set_power_state(dev, PCI_D0); 1994 if (err < 0 && err != -EIO) 1995 return err; 1996 1997 bridge = pci_upstream_bridge(dev); 1998 if (bridge) 1999 pcie_aspm_powersave_config_link(bridge); 2000 2001 err = pci_host_bridge_enable_device(dev); 2002 if (err) 2003 return err; 2004 2005 err = pcibios_enable_device(dev, bars); 2006 if (err < 0) 2007 goto err_enable; 2008 pci_fixup_device(pci_fixup_enable, dev); 2009 2010 if (dev->msi_enabled || dev->msix_enabled) 2011 return 0; 2012 2013 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin); 2014 if (pin) { 2015 pci_read_config_word(dev, PCI_COMMAND, &cmd); 2016 if (cmd & PCI_COMMAND_INTX_DISABLE) 2017 pci_write_config_word(dev, PCI_COMMAND, 2018 cmd & ~PCI_COMMAND_INTX_DISABLE); 2019 } 2020 2021 return 0; 2022 2023err_enable: 2024 pci_host_bridge_disable_device(dev); 2025 2026 return err; 2027 2028} 2029 2030/** 2031 * pci_reenable_device - Resume abandoned device 2032 * @dev: PCI device to be resumed 2033 * 2034 * NOTE: This function is a backend of pci_default_resume() and is not supposed 2035 * to be called by normal code, write proper resume handler and use it instead. 2036 */ 2037int pci_reenable_device(struct pci_dev *dev) 2038{ 2039 if (pci_is_enabled(dev)) 2040 return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1); 2041 return 0; 2042} 2043EXPORT_SYMBOL(pci_reenable_device); 2044 2045static void pci_enable_bridge(struct pci_dev *dev) 2046{ 2047 struct pci_dev *bridge; 2048 int retval; 2049 2050 bridge = pci_upstream_bridge(dev); 2051 if (bridge) 2052 pci_enable_bridge(bridge); 2053 2054 if (pci_is_enabled(dev)) { 2055 if (!dev->is_busmaster) 2056 pci_set_master(dev); 2057 return; 2058 } 2059 2060 retval = pci_enable_device(dev); 2061 if (retval) 2062 pci_err(dev, "Error enabling bridge (%d), continuing\n", 2063 retval); 2064 pci_set_master(dev); 2065} 2066 2067static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) 2068{ 2069 struct pci_dev *bridge; 2070 int err; 2071 int i, bars = 0; 2072 2073 /* 2074 * Power state could be unknown at this point, either due to a fresh 2075 * boot or a device removal call. So get the current power state 2076 * so that things like MSI message writing will behave as expected 2077 * (e.g. if the device really is in D0 at enable time). 2078 */ 2079 pci_update_current_state(dev, dev->current_state); 2080 2081 if (atomic_inc_return(&dev->enable_cnt) > 1) 2082 return 0; /* already enabled */ 2083 2084 bridge = pci_upstream_bridge(dev); 2085 if (bridge) 2086 pci_enable_bridge(bridge); 2087 2088 /* only skip sriov related */ 2089 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 2090 if (dev->resource[i].flags & flags) 2091 bars |= (1 << i); 2092 for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++) 2093 if (dev->resource[i].flags & flags) 2094 bars |= (1 << i); 2095 2096 err = do_pci_enable_device(dev, bars); 2097 if (err < 0) 2098 atomic_dec(&dev->enable_cnt); 2099 return err; 2100} 2101 2102/** 2103 * pci_enable_device_mem - Initialize a device for use with Memory space 2104 * @dev: PCI device to be initialized 2105 * 2106 * Initialize device before it's used by a driver. Ask low-level code 2107 * to enable Memory resources. Wake up the device if it was suspended. 2108 * Beware, this function can fail. 2109 */ 2110int pci_enable_device_mem(struct pci_dev *dev) 2111{ 2112 return pci_enable_device_flags(dev, IORESOURCE_MEM); 2113} 2114EXPORT_SYMBOL(pci_enable_device_mem); 2115 2116/** 2117 * pci_enable_device - Initialize device before it's used by a driver. 2118 * @dev: PCI device to be initialized 2119 * 2120 * Initialize device before it's used by a driver. Ask low-level code 2121 * to enable I/O and memory. Wake up the device if it was suspended. 2122 * Beware, this function can fail. 2123 * 2124 * Note we don't actually enable the device many times if we call 2125 * this function repeatedly (we just increment the count). 2126 */ 2127int pci_enable_device(struct pci_dev *dev) 2128{ 2129 return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO); 2130} 2131EXPORT_SYMBOL(pci_enable_device); 2132 2133/* 2134 * pcibios_device_add - provide arch specific hooks when adding device dev 2135 * @dev: the PCI device being added 2136 * 2137 * Permits the platform to provide architecture specific functionality when 2138 * devices are added. This is the default implementation. Architecture 2139 * implementations can override this. 2140 */ 2141int __weak pcibios_device_add(struct pci_dev *dev) 2142{ 2143 return 0; 2144} 2145 2146/** 2147 * pcibios_release_device - provide arch specific hooks when releasing 2148 * device dev 2149 * @dev: the PCI device being released 2150 * 2151 * Permits the platform to provide architecture specific functionality when 2152 * devices are released. This is the default implementation. Architecture 2153 * implementations can override this. 2154 */ 2155void __weak pcibios_release_device(struct pci_dev *dev) {} 2156 2157/** 2158 * pcibios_disable_device - disable arch specific PCI resources for device dev 2159 * @dev: the PCI device to disable 2160 * 2161 * Disables architecture specific PCI resources for the device. This 2162 * is the default implementation. Architecture implementations can 2163 * override this. 2164 */ 2165void __weak pcibios_disable_device(struct pci_dev *dev) {} 2166 2167static void do_pci_disable_device(struct pci_dev *dev) 2168{ 2169 u16 pci_command; 2170 2171 pci_read_config_word(dev, PCI_COMMAND, &pci_command); 2172 if (pci_command & PCI_COMMAND_MASTER) { 2173 pci_command &= ~PCI_COMMAND_MASTER; 2174 pci_write_config_word(dev, PCI_COMMAND, pci_command); 2175 } 2176 2177 pcibios_disable_device(dev); 2178} 2179 2180/** 2181 * pci_disable_enabled_device - Disable device without updating enable_cnt 2182 * @dev: PCI device to disable 2183 * 2184 * NOTE: This function is a backend of PCI power management routines and is 2185 * not supposed to be called drivers. 2186 */ 2187void pci_disable_enabled_device(struct pci_dev *dev) 2188{ 2189 if (pci_is_enabled(dev)) 2190 do_pci_disable_device(dev); 2191} 2192 2193/** 2194 * pci_disable_device - Disable PCI device after use 2195 * @dev: PCI device to be disabled 2196 * 2197 * Signal to the system that the PCI device is not in use by the system 2198 * anymore. This only involves disabling PCI bus-mastering, if active. 2199 * 2200 * Note we don't actually disable the device until all callers of 2201 * pci_enable_device() have called pci_disable_device(). 2202 */ 2203void pci_disable_device(struct pci_dev *dev) 2204{ 2205 dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0, 2206 "disabling already-disabled device"); 2207 2208 if (atomic_dec_return(&dev->enable_cnt) != 0) 2209 return; 2210 2211 pci_host_bridge_disable_device(dev); 2212 2213 do_pci_disable_device(dev); 2214 2215 dev->is_busmaster = 0; 2216} 2217EXPORT_SYMBOL(pci_disable_device); 2218 2219/** 2220 * pcibios_set_pcie_reset_state - set reset state for device dev 2221 * @dev: the PCIe device reset 2222 * @state: Reset state to enter into 2223 * 2224 * Set the PCIe reset state for the device. This is the default 2225 * implementation. Architecture implementations can override this. 2226 */ 2227int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev, 2228 enum pcie_reset_state state) 2229{ 2230 return -EINVAL; 2231} 2232 2233/** 2234 * pci_set_pcie_reset_state - set reset state for device dev 2235 * @dev: the PCIe device reset 2236 * @state: Reset state to enter into 2237 * 2238 * Sets the PCI reset state for the device. 2239 */ 2240int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 2241{ 2242 return pcibios_set_pcie_reset_state(dev, state); 2243} 2244EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state); 2245 2246#ifdef CONFIG_PCIEAER 2247void pcie_clear_device_status(struct pci_dev *dev) 2248{ 2249 u16 sta; 2250 2251 pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta); 2252 pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta); 2253} 2254#endif 2255 2256/** 2257 * pcie_clear_root_pme_status - Clear root port PME interrupt status. 2258 * @dev: PCIe root port or event collector. 2259 */ 2260void pcie_clear_root_pme_status(struct pci_dev *dev) 2261{ 2262 pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME); 2263} 2264 2265/** 2266 * pci_check_pme_status - Check if given device has generated PME. 2267 * @dev: Device to check. 2268 * 2269 * Check the PME status of the device and if set, clear it and clear PME enable 2270 * (if set). Return 'true' if PME status and PME enable were both set or 2271 * 'false' otherwise. 2272 */ 2273bool pci_check_pme_status(struct pci_dev *dev) 2274{ 2275 int pmcsr_pos; 2276 u16 pmcsr; 2277 bool ret = false; 2278 2279 if (!dev->pm_cap) 2280 return false; 2281 2282 pmcsr_pos = dev->pm_cap + PCI_PM_CTRL; 2283 pci_read_config_word(dev, pmcsr_pos, &pmcsr); 2284 if (!(pmcsr & PCI_PM_CTRL_PME_STATUS)) 2285 return false; 2286 2287 /* Clear PME status. */ 2288 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2289 if (pmcsr & PCI_PM_CTRL_PME_ENABLE) { 2290 /* Disable PME to avoid interrupt flood. */ 2291 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2292 ret = true; 2293 } 2294 2295 pci_write_config_word(dev, pmcsr_pos, pmcsr); 2296 2297 return ret; 2298} 2299 2300/** 2301 * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set. 2302 * @dev: Device to handle. 2303 * @pme_poll_reset: Whether or not to reset the device's pme_poll flag. 2304 * 2305 * Check if @dev has generated PME and queue a resume request for it in that 2306 * case. 2307 */ 2308static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset) 2309{ 2310 if (pme_poll_reset && dev->pme_poll) 2311 dev->pme_poll = false; 2312 2313 if (pci_check_pme_status(dev)) { 2314 pci_wakeup_event(dev); 2315 pm_request_resume(&dev->dev); 2316 } 2317 return 0; 2318} 2319 2320/** 2321 * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary. 2322 * @bus: Top bus of the subtree to walk. 2323 */ 2324void pci_pme_wakeup_bus(struct pci_bus *bus) 2325{ 2326 if (bus) 2327 pci_walk_bus(bus, pci_pme_wakeup, (void *)true); 2328} 2329 2330 2331/** 2332 * pci_pme_capable - check the capability of PCI device to generate PME# 2333 * @dev: PCI device to handle. 2334 * @state: PCI state from which device will issue PME#. 2335 */ 2336bool pci_pme_capable(struct pci_dev *dev, pci_power_t state) 2337{ 2338 if (!dev->pm_cap) 2339 return false; 2340 2341 return !!(dev->pme_support & (1 << state)); 2342} 2343EXPORT_SYMBOL(pci_pme_capable); 2344 2345static void pci_pme_list_scan(struct work_struct *work) 2346{ 2347 struct pci_pme_device *pme_dev, *n; 2348 2349 mutex_lock(&pci_pme_list_mutex); 2350 list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 2351 struct pci_dev *pdev = pme_dev->dev; 2352 2353 if (pdev->pme_poll) { 2354 struct pci_dev *bridge = pdev->bus->self; 2355 struct device *dev = &pdev->dev; 2356 struct device *bdev = bridge ? &bridge->dev : NULL; 2357 int bref = 0; 2358 2359 /* 2360 * If we have a bridge, it should be in an active/D0 2361 * state or the configuration space of subordinate 2362 * devices may not be accessible or stable over the 2363 * course of the call. 2364 */ 2365 if (bdev) { 2366 bref = pm_runtime_get_if_active(bdev); 2367 if (!bref) 2368 continue; 2369 2370 if (bridge->current_state != PCI_D0) 2371 goto put_bridge; 2372 } 2373 2374 /* 2375 * The device itself should be suspended but config 2376 * space must be accessible, therefore it cannot be in 2377 * D3cold. 2378 */ 2379 if (pm_runtime_suspended(dev) && 2380 pdev->current_state != PCI_D3cold) 2381 pci_pme_wakeup(pdev, NULL); 2382 2383put_bridge: 2384 if (bref > 0) 2385 pm_runtime_put(bdev); 2386 } else { 2387 list_del(&pme_dev->list); 2388 kfree(pme_dev); 2389 } 2390 } 2391 if (!list_empty(&pci_pme_list)) 2392 queue_delayed_work(system_freezable_wq, &pci_pme_work, 2393 msecs_to_jiffies(PME_TIMEOUT)); 2394 mutex_unlock(&pci_pme_list_mutex); 2395} 2396 2397static void __pci_pme_active(struct pci_dev *dev, bool enable) 2398{ 2399 u16 pmcsr; 2400 2401 if (!dev->pme_support) 2402 return; 2403 2404 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2405 /* Clear PME_Status by writing 1 to it and enable PME# */ 2406 pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE; 2407 if (!enable) 2408 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2409 2410 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2411} 2412 2413/** 2414 * pci_pme_restore - Restore PME configuration after config space restore. 2415 * @dev: PCI device to update. 2416 */ 2417void pci_pme_restore(struct pci_dev *dev) 2418{ 2419 u16 pmcsr; 2420 2421 if (!dev->pme_support) 2422 return; 2423 2424 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2425 if (dev->wakeup_prepared) { 2426 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 2427 pmcsr &= ~PCI_PM_CTRL_PME_STATUS; 2428 } else { 2429 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2430 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2431 } 2432 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2433} 2434 2435/** 2436 * pci_pme_active - enable or disable PCI device's PME# function 2437 * @dev: PCI device to handle. 2438 * @enable: 'true' to enable PME# generation; 'false' to disable it. 2439 * 2440 * The caller must verify that the device is capable of generating PME# before 2441 * calling this function with @enable equal to 'true'. 2442 */ 2443void pci_pme_active(struct pci_dev *dev, bool enable) 2444{ 2445 __pci_pme_active(dev, enable); 2446 2447 /* 2448 * PCI (as opposed to PCIe) PME requires that the device have 2449 * its PME# line hooked up correctly. Not all hardware vendors 2450 * do this, so the PME never gets delivered and the device 2451 * remains asleep. The easiest way around this is to 2452 * periodically walk the list of suspended devices and check 2453 * whether any have their PME flag set. The assumption is that 2454 * we'll wake up often enough anyway that this won't be a huge 2455 * hit, and the power savings from the devices will still be a 2456 * win. 2457 * 2458 * Although PCIe uses in-band PME message instead of PME# line 2459 * to report PME, PME does not work for some PCIe devices in 2460 * reality. For example, there are devices that set their PME 2461 * status bits, but don't really bother to send a PME message; 2462 * there are PCI Express Root Ports that don't bother to 2463 * trigger interrupts when they receive PME messages from the 2464 * devices below. So PME poll is used for PCIe devices too. 2465 */ 2466 2467 if (dev->pme_poll) { 2468 struct pci_pme_device *pme_dev; 2469 if (enable) { 2470 pme_dev = kmalloc(sizeof(struct pci_pme_device), 2471 GFP_KERNEL); 2472 if (!pme_dev) { 2473 pci_warn(dev, "can't enable PME#\n"); 2474 return; 2475 } 2476 pme_dev->dev = dev; 2477 mutex_lock(&pci_pme_list_mutex); 2478 list_add(&pme_dev->list, &pci_pme_list); 2479 if (list_is_singular(&pci_pme_list)) 2480 queue_delayed_work(system_freezable_wq, 2481 &pci_pme_work, 2482 msecs_to_jiffies(PME_TIMEOUT)); 2483 mutex_unlock(&pci_pme_list_mutex); 2484 } else { 2485 mutex_lock(&pci_pme_list_mutex); 2486 list_for_each_entry(pme_dev, &pci_pme_list, list) { 2487 if (pme_dev->dev == dev) { 2488 list_del(&pme_dev->list); 2489 kfree(pme_dev); 2490 break; 2491 } 2492 } 2493 mutex_unlock(&pci_pme_list_mutex); 2494 } 2495 } 2496 2497 pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled"); 2498} 2499EXPORT_SYMBOL(pci_pme_active); 2500 2501/** 2502 * __pci_enable_wake - enable PCI device as wakeup event source 2503 * @dev: PCI device affected 2504 * @state: PCI state from which device will issue wakeup events 2505 * @enable: True to enable event generation; false to disable 2506 * 2507 * This enables the device as a wakeup event source, or disables it. 2508 * When such events involves platform-specific hooks, those hooks are 2509 * called automatically by this routine. 2510 * 2511 * Devices with legacy power management (no standard PCI PM capabilities) 2512 * always require such platform hooks. 2513 * 2514 * RETURN VALUE: 2515 * 0 is returned on success 2516 * -EINVAL is returned if device is not supposed to wake up the system 2517 * Error code depending on the platform is returned if both the platform and 2518 * the native mechanism fail to enable the generation of wake-up events 2519 */ 2520static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable) 2521{ 2522 int ret = 0; 2523 2524 /* 2525 * Bridges that are not power-manageable directly only signal 2526 * wakeup on behalf of subordinate devices which is set up 2527 * elsewhere, so skip them. However, bridges that are 2528 * power-manageable may signal wakeup for themselves (for example, 2529 * on a hotplug event) and they need to be covered here. 2530 */ 2531 if (!pci_power_manageable(dev)) 2532 return 0; 2533 2534 /* Don't do the same thing twice in a row for one device. */ 2535 if (!!enable == !!dev->wakeup_prepared) 2536 return 0; 2537 2538 /* 2539 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don 2540 * Anderson we should be doing PME# wake enable followed by ACPI wake 2541 * enable. To disable wake-up we call the platform first, for symmetry. 2542 */ 2543 2544 if (enable) { 2545 int error; 2546 2547 /* 2548 * Enable PME signaling if the device can signal PME from 2549 * D3cold regardless of whether or not it can signal PME from 2550 * the current target state, because that will allow it to 2551 * signal PME when the hierarchy above it goes into D3cold and 2552 * the device itself ends up in D3cold as a result of that. 2553 */ 2554 if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold)) 2555 pci_pme_active(dev, true); 2556 else 2557 ret = 1; 2558 error = platform_pci_set_wakeup(dev, true); 2559 if (ret) 2560 ret = error; 2561 if (!ret) 2562 dev->wakeup_prepared = true; 2563 } else { 2564 platform_pci_set_wakeup(dev, false); 2565 pci_pme_active(dev, false); 2566 dev->wakeup_prepared = false; 2567 } 2568 2569 return ret; 2570} 2571 2572/** 2573 * pci_enable_wake - change wakeup settings for a PCI device 2574 * @pci_dev: Target device 2575 * @state: PCI state from which device will issue wakeup events 2576 * @enable: Whether or not to enable event generation 2577 * 2578 * If @enable is set, check device_may_wakeup() for the device before calling 2579 * __pci_enable_wake() for it. 2580 */ 2581int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable) 2582{ 2583 if (enable && !device_may_wakeup(&pci_dev->dev)) 2584 return -EINVAL; 2585 2586 return __pci_enable_wake(pci_dev, state, enable); 2587} 2588EXPORT_SYMBOL(pci_enable_wake); 2589 2590/** 2591 * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold 2592 * @dev: PCI device to prepare 2593 * @enable: True to enable wake-up event generation; false to disable 2594 * 2595 * Many drivers want the device to wake up the system from D3_hot or D3_cold 2596 * and this function allows them to set that up cleanly - pci_enable_wake() 2597 * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI 2598 * ordering constraints. 2599 * 2600 * This function only returns error code if the device is not allowed to wake 2601 * up the system from sleep or it is not capable of generating PME# from both 2602 * D3_hot and D3_cold and the platform is unable to enable wake-up power for it. 2603 */ 2604int pci_wake_from_d3(struct pci_dev *dev, bool enable) 2605{ 2606 return pci_pme_capable(dev, PCI_D3cold) ? 2607 pci_enable_wake(dev, PCI_D3cold, enable) : 2608 pci_enable_wake(dev, PCI_D3hot, enable); 2609} 2610EXPORT_SYMBOL(pci_wake_from_d3); 2611 2612/** 2613 * pci_target_state - find an appropriate low power state for a given PCI dev 2614 * @dev: PCI device 2615 * @wakeup: Whether or not wakeup functionality will be enabled for the device. 2616 * 2617 * Use underlying platform code to find a supported low power state for @dev. 2618 * If the platform can't manage @dev, return the deepest state from which it 2619 * can generate wake events, based on any available PME info. 2620 */ 2621static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup) 2622{ 2623 if (platform_pci_power_manageable(dev)) { 2624 /* 2625 * Call the platform to find the target state for the device. 2626 */ 2627 pci_power_t state = platform_pci_choose_state(dev); 2628 2629 switch (state) { 2630 case PCI_POWER_ERROR: 2631 case PCI_UNKNOWN: 2632 return PCI_D3hot; 2633 2634 case PCI_D1: 2635 case PCI_D2: 2636 if (pci_no_d1d2(dev)) 2637 return PCI_D3hot; 2638 } 2639 2640 return state; 2641 } 2642 2643 /* 2644 * If the device is in D3cold even though it's not power-manageable by 2645 * the platform, it may have been powered down by non-standard means. 2646 * Best to let it slumber. 2647 */ 2648 if (dev->current_state == PCI_D3cold) 2649 return PCI_D3cold; 2650 else if (!dev->pm_cap) 2651 return PCI_D0; 2652 2653 if (wakeup && dev->pme_support) { 2654 pci_power_t state = PCI_D3hot; 2655 2656 /* 2657 * Find the deepest state from which the device can generate 2658 * PME#. 2659 */ 2660 while (state && !(dev->pme_support & (1 << state))) 2661 state--; 2662 2663 if (state) 2664 return state; 2665 else if (dev->pme_support & 1) 2666 return PCI_D0; 2667 } 2668 2669 return PCI_D3hot; 2670} 2671 2672/** 2673 * pci_prepare_to_sleep - prepare PCI device for system-wide transition 2674 * into a sleep state 2675 * @dev: Device to handle. 2676 * 2677 * Choose the power state appropriate for the device depending on whether 2678 * it can wake up the system and/or is power manageable by the platform 2679 * (PCI_D3hot is the default) and put the device into that state. 2680 */ 2681int pci_prepare_to_sleep(struct pci_dev *dev) 2682{ 2683 bool wakeup = device_may_wakeup(&dev->dev); 2684 pci_power_t target_state = pci_target_state(dev, wakeup); 2685 int error; 2686 2687 if (target_state == PCI_POWER_ERROR) 2688 return -EIO; 2689 2690 pci_enable_wake(dev, target_state, wakeup); 2691 2692 error = pci_set_power_state(dev, target_state); 2693 2694 if (error) 2695 pci_enable_wake(dev, target_state, false); 2696 2697 return error; 2698} 2699EXPORT_SYMBOL(pci_prepare_to_sleep); 2700 2701/** 2702 * pci_back_from_sleep - turn PCI device on during system-wide transition 2703 * into working state 2704 * @dev: Device to handle. 2705 * 2706 * Disable device's system wake-up capability and put it into D0. 2707 */ 2708int pci_back_from_sleep(struct pci_dev *dev) 2709{ 2710 int ret = pci_set_power_state(dev, PCI_D0); 2711 2712 if (ret) 2713 return ret; 2714 2715 pci_enable_wake(dev, PCI_D0, false); 2716 return 0; 2717} 2718EXPORT_SYMBOL(pci_back_from_sleep); 2719 2720/** 2721 * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend. 2722 * @dev: PCI device being suspended. 2723 * 2724 * Prepare @dev to generate wake-up events at run time and put it into a low 2725 * power state. 2726 */ 2727int pci_finish_runtime_suspend(struct pci_dev *dev) 2728{ 2729 pci_power_t target_state; 2730 int error; 2731 2732 target_state = pci_target_state(dev, device_can_wakeup(&dev->dev)); 2733 if (target_state == PCI_POWER_ERROR) 2734 return -EIO; 2735 2736 __pci_enable_wake(dev, target_state, pci_dev_run_wake(dev)); 2737 2738 error = pci_set_power_state(dev, target_state); 2739 2740 if (error) 2741 pci_enable_wake(dev, target_state, false); 2742 2743 return error; 2744} 2745 2746/** 2747 * pci_dev_run_wake - Check if device can generate run-time wake-up events. 2748 * @dev: Device to check. 2749 * 2750 * Return true if the device itself is capable of generating wake-up events 2751 * (through the platform or using the native PCIe PME) or if the device supports 2752 * PME and one of its upstream bridges can generate wake-up events. 2753 */ 2754bool pci_dev_run_wake(struct pci_dev *dev) 2755{ 2756 struct pci_bus *bus = dev->bus; 2757 2758 if (!dev->pme_support) 2759 return false; 2760 2761 /* PME-capable in principle, but not from the target power state */ 2762 if (!pci_pme_capable(dev, pci_target_state(dev, true))) 2763 return false; 2764 2765 if (device_can_wakeup(&dev->dev)) 2766 return true; 2767 2768 while (bus->parent) { 2769 struct pci_dev *bridge = bus->self; 2770 2771 if (device_can_wakeup(&bridge->dev)) 2772 return true; 2773 2774 bus = bus->parent; 2775 } 2776 2777 /* We have reached the root bus. */ 2778 if (bus->bridge) 2779 return device_can_wakeup(bus->bridge); 2780 2781 return false; 2782} 2783EXPORT_SYMBOL_GPL(pci_dev_run_wake); 2784 2785/** 2786 * pci_dev_need_resume - Check if it is necessary to resume the device. 2787 * @pci_dev: Device to check. 2788 * 2789 * Return 'true' if the device is not runtime-suspended or it has to be 2790 * reconfigured due to wakeup settings difference between system and runtime 2791 * suspend, or the current power state of it is not suitable for the upcoming 2792 * (system-wide) transition. 2793 */ 2794bool pci_dev_need_resume(struct pci_dev *pci_dev) 2795{ 2796 struct device *dev = &pci_dev->dev; 2797 pci_power_t target_state; 2798 2799 if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev)) 2800 return true; 2801 2802 target_state = pci_target_state(pci_dev, device_may_wakeup(dev)); 2803 2804 /* 2805 * If the earlier platform check has not triggered, D3cold is just power 2806 * removal on top of D3hot, so no need to resume the device in that 2807 * case. 2808 */ 2809 return target_state != pci_dev->current_state && 2810 target_state != PCI_D3cold && 2811 pci_dev->current_state != PCI_D3hot; 2812} 2813 2814/** 2815 * pci_dev_adjust_pme - Adjust PME setting for a suspended device. 2816 * @pci_dev: Device to check. 2817 * 2818 * If the device is suspended and it is not configured for system wakeup, 2819 * disable PME for it to prevent it from waking up the system unnecessarily. 2820 * 2821 * Note that if the device's power state is D3cold and the platform check in 2822 * pci_dev_need_resume() has not triggered, the device's configuration need not 2823 * be changed. 2824 */ 2825void pci_dev_adjust_pme(struct pci_dev *pci_dev) 2826{ 2827 struct device *dev = &pci_dev->dev; 2828 2829 spin_lock_irq(&dev->power.lock); 2830 2831 if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) && 2832 pci_dev->current_state < PCI_D3cold) 2833 __pci_pme_active(pci_dev, false); 2834 2835 spin_unlock_irq(&dev->power.lock); 2836} 2837 2838/** 2839 * pci_dev_complete_resume - Finalize resume from system sleep for a device. 2840 * @pci_dev: Device to handle. 2841 * 2842 * If the device is runtime suspended and wakeup-capable, enable PME for it as 2843 * it might have been disabled during the prepare phase of system suspend if 2844 * the device was not configured for system wakeup. 2845 */ 2846void pci_dev_complete_resume(struct pci_dev *pci_dev) 2847{ 2848 struct device *dev = &pci_dev->dev; 2849 2850 if (!pci_dev_run_wake(pci_dev)) 2851 return; 2852 2853 spin_lock_irq(&dev->power.lock); 2854 2855 if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold) 2856 __pci_pme_active(pci_dev, true); 2857 2858 spin_unlock_irq(&dev->power.lock); 2859} 2860 2861/** 2862 * pci_choose_state - Choose the power state of a PCI device. 2863 * @dev: Target PCI device. 2864 * @state: Target state for the whole system. 2865 * 2866 * Returns PCI power state suitable for @dev and @state. 2867 */ 2868pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) 2869{ 2870 if (state.event == PM_EVENT_ON) 2871 return PCI_D0; 2872 2873 return pci_target_state(dev, false); 2874} 2875EXPORT_SYMBOL(pci_choose_state); 2876 2877void pci_config_pm_runtime_get(struct pci_dev *pdev) 2878{ 2879 struct device *dev = &pdev->dev; 2880 struct device *parent = dev->parent; 2881 2882 if (parent) 2883 pm_runtime_get_sync(parent); 2884 pm_runtime_get_noresume(dev); 2885 /* 2886 * pdev->current_state is set to PCI_D3cold during suspending, 2887 * so wait until suspending completes 2888 */ 2889 pm_runtime_barrier(dev); 2890 /* 2891 * Only need to resume devices in D3cold, because config 2892 * registers are still accessible for devices suspended but 2893 * not in D3cold. 2894 */ 2895 if (pdev->current_state == PCI_D3cold) 2896 pm_runtime_resume(dev); 2897} 2898 2899void pci_config_pm_runtime_put(struct pci_dev *pdev) 2900{ 2901 struct device *dev = &pdev->dev; 2902 struct device *parent = dev->parent; 2903 2904 pm_runtime_put(dev); 2905 if (parent) 2906 pm_runtime_put_sync(parent); 2907} 2908 2909static const struct dmi_system_id bridge_d3_blacklist[] = { 2910#ifdef CONFIG_X86 2911 { 2912 /* 2913 * Gigabyte X299 root port is not marked as hotplug capable 2914 * which allows Linux to power manage it. However, this 2915 * confuses the BIOS SMI handler so don't power manage root 2916 * ports on that system. 2917 */ 2918 .ident = "X299 DESIGNARE EX-CF", 2919 .matches = { 2920 DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."), 2921 DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"), 2922 }, 2923 }, 2924 { 2925 /* 2926 * Downstream device is not accessible after putting a root port 2927 * into D3cold and back into D0 on Elo Continental Z2 board 2928 */ 2929 .ident = "Elo Continental Z2", 2930 .matches = { 2931 DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"), 2932 DMI_MATCH(DMI_BOARD_NAME, "Geminilake"), 2933 DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"), 2934 }, 2935 }, 2936 { 2937 /* 2938 * Changing power state of root port dGPU is connected fails 2939 * https://gitlab.freedesktop.org/drm/amd/-/issues/3229 2940 */ 2941 .ident = "Hewlett-Packard HP Pavilion 17 Notebook PC/1972", 2942 .matches = { 2943 DMI_MATCH(DMI_BOARD_VENDOR, "Hewlett-Packard"), 2944 DMI_MATCH(DMI_BOARD_NAME, "1972"), 2945 DMI_MATCH(DMI_BOARD_VERSION, "95.33"), 2946 }, 2947 }, 2948#endif 2949 { } 2950}; 2951 2952/** 2953 * pci_bridge_d3_possible - Is it possible to put the bridge into D3 2954 * @bridge: Bridge to check 2955 * 2956 * Currently we only allow D3 for some PCIe ports and for Thunderbolt. 2957 * 2958 * Return: Whether it is possible to move the bridge to D3. 2959 * 2960 * The return value is guaranteed to be constant across the entire lifetime 2961 * of the bridge, including its hot-removal. 2962 */ 2963bool pci_bridge_d3_possible(struct pci_dev *bridge) 2964{ 2965 if (!pci_is_pcie(bridge)) 2966 return false; 2967 2968 switch (pci_pcie_type(bridge)) { 2969 case PCI_EXP_TYPE_ROOT_PORT: 2970 case PCI_EXP_TYPE_UPSTREAM: 2971 case PCI_EXP_TYPE_DOWNSTREAM: 2972 if (pci_bridge_d3_disable) 2973 return false; 2974 2975 /* 2976 * Hotplug ports handled by platform firmware may not be put 2977 * into D3 by the OS, e.g. ACPI slots ... 2978 */ 2979 if (bridge->is_hotplug_bridge && !bridge->is_pciehp) 2980 return false; 2981 2982 /* ... or PCIe hotplug ports not handled natively by the OS. */ 2983 if (bridge->is_pciehp && !pciehp_is_native(bridge)) 2984 return false; 2985 2986 if (pci_bridge_d3_force) 2987 return true; 2988 2989 /* Even the oldest 2010 Thunderbolt controller supports D3. */ 2990 if (bridge->is_thunderbolt) 2991 return true; 2992 2993 /* Platform might know better if the bridge supports D3 */ 2994 if (platform_pci_bridge_d3(bridge)) 2995 return true; 2996 2997 /* 2998 * Hotplug ports handled natively by the OS were not validated 2999 * by vendors for runtime D3 at least until 2018 because there 3000 * was no OS support. 3001 */ 3002 if (bridge->is_pciehp) 3003 return false; 3004 3005 if (dmi_check_system(bridge_d3_blacklist)) 3006 return false; 3007 3008 /* 3009 * Out of caution, we only allow PCIe ports from 2015 or newer 3010 * into D3 on x86. 3011 */ 3012 if (!IS_ENABLED(CONFIG_X86) || dmi_get_bios_year() >= 2015) 3013 return true; 3014 break; 3015 } 3016 3017 return false; 3018} 3019 3020static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) 3021{ 3022 bool *d3cold_ok = data; 3023 3024 if (/* The device needs to be allowed to go D3cold ... */ 3025 dev->no_d3cold || !dev->d3cold_allowed || 3026 3027 /* ... and if it is wakeup capable to do so from D3cold. */ 3028 (device_may_wakeup(&dev->dev) && 3029 !pci_pme_capable(dev, PCI_D3cold)) || 3030 3031 /* If it is a bridge it must be allowed to go to D3. */ 3032 !pci_power_manageable(dev)) 3033 3034 *d3cold_ok = false; 3035 3036 return !*d3cold_ok; 3037} 3038 3039/* 3040 * pci_bridge_d3_update - Update bridge D3 capabilities 3041 * @dev: PCI device which is changed 3042 * 3043 * Update upstream bridge PM capabilities accordingly depending on if the 3044 * device PM configuration was changed or the device is being removed. The 3045 * change is also propagated upstream. 3046 */ 3047void pci_bridge_d3_update(struct pci_dev *dev) 3048{ 3049 bool remove = !device_is_registered(&dev->dev); 3050 struct pci_dev *bridge; 3051 bool d3cold_ok = true; 3052 3053 bridge = pci_upstream_bridge(dev); 3054 if (!bridge || !pci_bridge_d3_possible(bridge)) 3055 return; 3056 3057 /* 3058 * If D3 is currently allowed for the bridge, removing one of its 3059 * children won't change that. 3060 */ 3061 if (remove && bridge->bridge_d3) 3062 return; 3063 3064 /* 3065 * If D3 is currently allowed for the bridge and a child is added or 3066 * changed, disallowance of D3 can only be caused by that child, so 3067 * we only need to check that single device, not any of its siblings. 3068 * 3069 * If D3 is currently not allowed for the bridge, checking the device 3070 * first may allow us to skip checking its siblings. 3071 */ 3072 if (!remove) 3073 pci_dev_check_d3cold(dev, &d3cold_ok); 3074 3075 /* 3076 * If D3 is currently not allowed for the bridge, this may be caused 3077 * either by the device being changed/removed or any of its siblings, 3078 * so we need to go through all children to find out if one of them 3079 * continues to block D3. 3080 */ 3081 if (d3cold_ok && !bridge->bridge_d3) 3082 pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold, 3083 &d3cold_ok); 3084 3085 if (bridge->bridge_d3 != d3cold_ok) { 3086 bridge->bridge_d3 = d3cold_ok; 3087 /* Propagate change to upstream bridges */ 3088 pci_bridge_d3_update(bridge); 3089 } 3090} 3091 3092/** 3093 * pci_d3cold_enable - Enable D3cold for device 3094 * @dev: PCI device to handle 3095 * 3096 * This function can be used in drivers to enable D3cold from the device 3097 * they handle. It also updates upstream PCI bridge PM capabilities 3098 * accordingly. 3099 */ 3100void pci_d3cold_enable(struct pci_dev *dev) 3101{ 3102 if (dev->no_d3cold) { 3103 dev->no_d3cold = false; 3104 pci_bridge_d3_update(dev); 3105 } 3106} 3107EXPORT_SYMBOL_GPL(pci_d3cold_enable); 3108 3109/** 3110 * pci_d3cold_disable - Disable D3cold for device 3111 * @dev: PCI device to handle 3112 * 3113 * This function can be used in drivers to disable D3cold from the device 3114 * they handle. It also updates upstream PCI bridge PM capabilities 3115 * accordingly. 3116 */ 3117void pci_d3cold_disable(struct pci_dev *dev) 3118{ 3119 if (!dev->no_d3cold) { 3120 dev->no_d3cold = true; 3121 pci_bridge_d3_update(dev); 3122 } 3123} 3124EXPORT_SYMBOL_GPL(pci_d3cold_disable); 3125 3126void pci_pm_power_up_and_verify_state(struct pci_dev *pci_dev) 3127{ 3128 pci_power_up(pci_dev); 3129 pci_update_current_state(pci_dev, PCI_D0); 3130} 3131 3132/** 3133 * pci_pm_init - Initialize PM functions of given PCI device 3134 * @dev: PCI device to handle. 3135 */ 3136void pci_pm_init(struct pci_dev *dev) 3137{ 3138 int pm; 3139 u16 pmc; 3140 3141 device_enable_async_suspend(&dev->dev); 3142 dev->wakeup_prepared = false; 3143 3144 dev->pm_cap = 0; 3145 dev->pme_support = 0; 3146 3147 /* find PCI PM capability in list */ 3148 pm = pci_find_capability(dev, PCI_CAP_ID_PM); 3149 if (!pm) 3150 goto poweron; 3151 /* Check device's ability to generate PME# */ 3152 pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc); 3153 3154 if ((pmc & PCI_PM_CAP_VER_MASK) > 3) { 3155 pci_err(dev, "unsupported PM cap regs version (%u)\n", 3156 pmc & PCI_PM_CAP_VER_MASK); 3157 goto poweron; 3158 } 3159 3160 dev->pm_cap = pm; 3161 dev->d3hot_delay = PCI_PM_D3HOT_WAIT; 3162 dev->d3cold_delay = PCI_PM_D3COLD_WAIT; 3163 dev->bridge_d3 = pci_bridge_d3_possible(dev); 3164 dev->d3cold_allowed = true; 3165 3166 dev->d1_support = false; 3167 dev->d2_support = false; 3168 if (!pci_no_d1d2(dev)) { 3169 if (pmc & PCI_PM_CAP_D1) 3170 dev->d1_support = true; 3171 if (pmc & PCI_PM_CAP_D2) 3172 dev->d2_support = true; 3173 3174 if (dev->d1_support || dev->d2_support) 3175 pci_info(dev, "supports%s%s\n", 3176 dev->d1_support ? " D1" : "", 3177 dev->d2_support ? " D2" : ""); 3178 } 3179 3180 pmc &= PCI_PM_CAP_PME_MASK; 3181 if (pmc) { 3182 pci_info(dev, "PME# supported from%s%s%s%s%s\n", 3183 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "", 3184 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "", 3185 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "", 3186 (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "", 3187 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : ""); 3188 dev->pme_support = FIELD_GET(PCI_PM_CAP_PME_MASK, pmc); 3189 dev->pme_poll = true; 3190 /* 3191 * Make device's PM flags reflect the wake-up capability, but 3192 * let the user space enable it to wake up the system as needed. 3193 */ 3194 device_set_wakeup_capable(&dev->dev, true); 3195 /* Disable the PME# generation functionality */ 3196 pci_pme_active(dev, false); 3197 } 3198 3199poweron: 3200 pci_pm_power_up_and_verify_state(dev); 3201 pm_runtime_forbid(&dev->dev); 3202 3203 /* 3204 * Runtime PM will be enabled for the device when it has been fully 3205 * configured, but since its parent and suppliers may suspend in 3206 * the meantime, prevent them from doing so by changing the 3207 * device's runtime PM status to "active". 3208 */ 3209 pm_runtime_set_active(&dev->dev); 3210} 3211 3212static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) 3213{ 3214 unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI; 3215 3216 switch (prop) { 3217 case PCI_EA_P_MEM: 3218 case PCI_EA_P_VF_MEM: 3219 flags |= IORESOURCE_MEM; 3220 break; 3221 case PCI_EA_P_MEM_PREFETCH: 3222 case PCI_EA_P_VF_MEM_PREFETCH: 3223 flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH; 3224 break; 3225 case PCI_EA_P_IO: 3226 flags |= IORESOURCE_IO; 3227 break; 3228 default: 3229 return 0; 3230 } 3231 3232 return flags; 3233} 3234 3235static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei, 3236 u8 prop) 3237{ 3238 if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO) 3239 return &dev->resource[bei]; 3240#ifdef CONFIG_PCI_IOV 3241 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 && 3242 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH)) 3243 return &dev->resource[PCI_IOV_RESOURCES + 3244 bei - PCI_EA_BEI_VF_BAR0]; 3245#endif 3246 else if (bei == PCI_EA_BEI_ROM) 3247 return &dev->resource[PCI_ROM_RESOURCE]; 3248 else 3249 return NULL; 3250} 3251 3252/* Read an Enhanced Allocation (EA) entry */ 3253static int pci_ea_read(struct pci_dev *dev, int offset) 3254{ 3255 struct resource *res; 3256 const char *res_name; 3257 int ent_size, ent_offset = offset; 3258 resource_size_t start, end; 3259 unsigned long flags; 3260 u32 dw0, bei, base, max_offset; 3261 u8 prop; 3262 bool support_64 = (sizeof(resource_size_t) >= 8); 3263 3264 pci_read_config_dword(dev, ent_offset, &dw0); 3265 ent_offset += 4; 3266 3267 /* Entry size field indicates DWORDs after 1st */ 3268 ent_size = (FIELD_GET(PCI_EA_ES, dw0) + 1) << 2; 3269 3270 if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */ 3271 goto out; 3272 3273 bei = FIELD_GET(PCI_EA_BEI, dw0); 3274 prop = FIELD_GET(PCI_EA_PP, dw0); 3275 3276 /* 3277 * If the Property is in the reserved range, try the Secondary 3278 * Property instead. 3279 */ 3280 if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED) 3281 prop = FIELD_GET(PCI_EA_SP, dw0); 3282 if (prop > PCI_EA_P_BRIDGE_IO) 3283 goto out; 3284 3285 res = pci_ea_get_resource(dev, bei, prop); 3286 res_name = pci_resource_name(dev, bei); 3287 if (!res) { 3288 pci_err(dev, "Unsupported EA entry BEI: %u\n", bei); 3289 goto out; 3290 } 3291 3292 flags = pci_ea_flags(dev, prop); 3293 if (!flags) { 3294 pci_err(dev, "Unsupported EA properties: %#x\n", prop); 3295 goto out; 3296 } 3297 3298 /* Read Base */ 3299 pci_read_config_dword(dev, ent_offset, &base); 3300 start = (base & PCI_EA_FIELD_MASK); 3301 ent_offset += 4; 3302 3303 /* Read MaxOffset */ 3304 pci_read_config_dword(dev, ent_offset, &max_offset); 3305 ent_offset += 4; 3306 3307 /* Read Base MSBs (if 64-bit entry) */ 3308 if (base & PCI_EA_IS_64) { 3309 u32 base_upper; 3310 3311 pci_read_config_dword(dev, ent_offset, &base_upper); 3312 ent_offset += 4; 3313 3314 flags |= IORESOURCE_MEM_64; 3315 3316 /* entry starts above 32-bit boundary, can't use */ 3317 if (!support_64 && base_upper) 3318 goto out; 3319 3320 if (support_64) 3321 start |= ((u64)base_upper << 32); 3322 } 3323 3324 end = start + (max_offset | 0x03); 3325 3326 /* Read MaxOffset MSBs (if 64-bit entry) */ 3327 if (max_offset & PCI_EA_IS_64) { 3328 u32 max_offset_upper; 3329 3330 pci_read_config_dword(dev, ent_offset, &max_offset_upper); 3331 ent_offset += 4; 3332 3333 flags |= IORESOURCE_MEM_64; 3334 3335 /* entry too big, can't use */ 3336 if (!support_64 && max_offset_upper) 3337 goto out; 3338 3339 if (support_64) 3340 end += ((u64)max_offset_upper << 32); 3341 } 3342 3343 if (end < start) { 3344 pci_err(dev, "EA Entry crosses address boundary\n"); 3345 goto out; 3346 } 3347 3348 if (ent_size != ent_offset - offset) { 3349 pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n", 3350 ent_size, ent_offset - offset); 3351 goto out; 3352 } 3353 3354 res->name = pci_name(dev); 3355 res->start = start; 3356 res->end = end; 3357 res->flags = flags; 3358 3359 if (bei <= PCI_EA_BEI_BAR5) 3360 pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n", 3361 res_name, res, prop); 3362 else if (bei == PCI_EA_BEI_ROM) 3363 pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n", 3364 res_name, res, prop); 3365 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5) 3366 pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n", 3367 res_name, res, prop); 3368 else 3369 pci_info(dev, "BEI %d %pR: from Enhanced Allocation, properties %#02x\n", 3370 bei, res, prop); 3371 3372out: 3373 return offset + ent_size; 3374} 3375 3376/* Enhanced Allocation Initialization */ 3377void pci_ea_init(struct pci_dev *dev) 3378{ 3379 int ea; 3380 u8 num_ent; 3381 int offset; 3382 int i; 3383 3384 /* find PCI EA capability in list */ 3385 ea = pci_find_capability(dev, PCI_CAP_ID_EA); 3386 if (!ea) 3387 return; 3388 3389 /* determine the number of entries */ 3390 pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT, 3391 &num_ent); 3392 num_ent &= PCI_EA_NUM_ENT_MASK; 3393 3394 offset = ea + PCI_EA_FIRST_ENT; 3395 3396 /* Skip DWORD 2 for type 1 functions */ 3397 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 3398 offset += 4; 3399 3400 /* parse each EA entry */ 3401 for (i = 0; i < num_ent; ++i) 3402 offset = pci_ea_read(dev, offset); 3403} 3404 3405static void pci_add_saved_cap(struct pci_dev *pci_dev, 3406 struct pci_cap_saved_state *new_cap) 3407{ 3408 hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space); 3409} 3410 3411/** 3412 * _pci_add_cap_save_buffer - allocate buffer for saving given 3413 * capability registers 3414 * @dev: the PCI device 3415 * @cap: the capability to allocate the buffer for 3416 * @extended: Standard or Extended capability ID 3417 * @size: requested size of the buffer 3418 */ 3419static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap, 3420 bool extended, unsigned int size) 3421{ 3422 int pos; 3423 struct pci_cap_saved_state *save_state; 3424 3425 if (extended) 3426 pos = pci_find_ext_capability(dev, cap); 3427 else 3428 pos = pci_find_capability(dev, cap); 3429 3430 if (!pos) 3431 return 0; 3432 3433 save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL); 3434 if (!save_state) 3435 return -ENOMEM; 3436 3437 save_state->cap.cap_nr = cap; 3438 save_state->cap.cap_extended = extended; 3439 save_state->cap.size = size; 3440 pci_add_saved_cap(dev, save_state); 3441 3442 return 0; 3443} 3444 3445int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size) 3446{ 3447 return _pci_add_cap_save_buffer(dev, cap, false, size); 3448} 3449 3450int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size) 3451{ 3452 return _pci_add_cap_save_buffer(dev, cap, true, size); 3453} 3454 3455/** 3456 * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities 3457 * @dev: the PCI device 3458 */ 3459void pci_allocate_cap_save_buffers(struct pci_dev *dev) 3460{ 3461 int error; 3462 3463 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP, 3464 PCI_EXP_SAVE_REGS * sizeof(u16)); 3465 if (error) 3466 pci_err(dev, "unable to preallocate PCI Express save buffer\n"); 3467 3468 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16)); 3469 if (error) 3470 pci_err(dev, "unable to preallocate PCI-X save buffer\n"); 3471 3472 error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR, 3473 2 * sizeof(u16)); 3474 if (error) 3475 pci_err(dev, "unable to allocate suspend buffer for LTR\n"); 3476 3477 pci_allocate_vc_save_buffers(dev); 3478} 3479 3480void pci_free_cap_save_buffers(struct pci_dev *dev) 3481{ 3482 struct pci_cap_saved_state *tmp; 3483 struct hlist_node *n; 3484 3485 hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next) 3486 kfree(tmp); 3487} 3488 3489/** 3490 * pci_configure_ari - enable or disable ARI forwarding 3491 * @dev: the PCI device 3492 * 3493 * If @dev and its upstream bridge both support ARI, enable ARI in the 3494 * bridge. Otherwise, disable ARI in the bridge. 3495 */ 3496void pci_configure_ari(struct pci_dev *dev) 3497{ 3498 u32 cap; 3499 struct pci_dev *bridge; 3500 3501 if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn) 3502 return; 3503 3504 bridge = dev->bus->self; 3505 if (!bridge) 3506 return; 3507 3508 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3509 if (!(cap & PCI_EXP_DEVCAP2_ARI)) 3510 return; 3511 3512 if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) { 3513 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, 3514 PCI_EXP_DEVCTL2_ARI); 3515 bridge->ari_enabled = 1; 3516 } else { 3517 pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2, 3518 PCI_EXP_DEVCTL2_ARI); 3519 bridge->ari_enabled = 0; 3520 } 3521} 3522 3523static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags) 3524{ 3525 int pos; 3526 u16 cap, ctrl; 3527 3528 pos = pdev->acs_cap; 3529 if (!pos) 3530 return false; 3531 3532 /* 3533 * Except for egress control, capabilities are either required 3534 * or only required if controllable. Features missing from the 3535 * capability field can therefore be assumed as hard-wired enabled. 3536 */ 3537 pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap); 3538 acs_flags &= (cap | PCI_ACS_EC); 3539 3540 pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl); 3541 return (ctrl & acs_flags) == acs_flags; 3542} 3543 3544/** 3545 * pci_acs_enabled - test ACS against required flags for a given device 3546 * @pdev: device to test 3547 * @acs_flags: required PCI ACS flags 3548 * 3549 * Return true if the device supports the provided flags. Automatically 3550 * filters out flags that are not implemented on multifunction devices. 3551 * 3552 * Note that this interface checks the effective ACS capabilities of the 3553 * device rather than the actual capabilities. For instance, most single 3554 * function endpoints are not required to support ACS because they have no 3555 * opportunity for peer-to-peer access. We therefore return 'true' 3556 * regardless of whether the device exposes an ACS capability. This makes 3557 * it much easier for callers of this function to ignore the actual type 3558 * or topology of the device when testing ACS support. 3559 */ 3560bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags) 3561{ 3562 int ret; 3563 3564 ret = pci_dev_specific_acs_enabled(pdev, acs_flags); 3565 if (ret >= 0) 3566 return ret > 0; 3567 3568 /* 3569 * Conventional PCI and PCI-X devices never support ACS, either 3570 * effectively or actually. The shared bus topology implies that 3571 * any device on the bus can receive or snoop DMA. 3572 */ 3573 if (!pci_is_pcie(pdev)) 3574 return false; 3575 3576 switch (pci_pcie_type(pdev)) { 3577 /* 3578 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec, 3579 * but since their primary interface is PCI/X, we conservatively 3580 * handle them as we would a non-PCIe device. 3581 */ 3582 case PCI_EXP_TYPE_PCIE_BRIDGE: 3583 /* 3584 * PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never 3585 * applicable... must never implement an ACS Extended Capability...". 3586 * This seems arbitrary, but we take a conservative interpretation 3587 * of this statement. 3588 */ 3589 case PCI_EXP_TYPE_PCI_BRIDGE: 3590 case PCI_EXP_TYPE_RC_EC: 3591 return false; 3592 /* 3593 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should 3594 * implement ACS in order to indicate their peer-to-peer capabilities, 3595 * regardless of whether they are single- or multi-function devices. 3596 */ 3597 case PCI_EXP_TYPE_DOWNSTREAM: 3598 case PCI_EXP_TYPE_ROOT_PORT: 3599 return pci_acs_flags_enabled(pdev, acs_flags); 3600 /* 3601 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be 3602 * implemented by the remaining PCIe types to indicate peer-to-peer 3603 * capabilities, but only when they are part of a multifunction 3604 * device. The footnote for section 6.12 indicates the specific 3605 * PCIe types included here. 3606 */ 3607 case PCI_EXP_TYPE_ENDPOINT: 3608 case PCI_EXP_TYPE_UPSTREAM: 3609 case PCI_EXP_TYPE_LEG_END: 3610 case PCI_EXP_TYPE_RC_END: 3611 if (!pdev->multifunction) 3612 break; 3613 3614 return pci_acs_flags_enabled(pdev, acs_flags); 3615 } 3616 3617 /* 3618 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable 3619 * to single function devices with the exception of downstream ports. 3620 */ 3621 return true; 3622} 3623 3624/** 3625 * pci_acs_path_enabled - test ACS flags from start to end in a hierarchy 3626 * @start: starting downstream device 3627 * @end: ending upstream device or NULL to search to the root bus 3628 * @acs_flags: required flags 3629 * 3630 * Walk up a device tree from start to end testing PCI ACS support. If 3631 * any step along the way does not support the required flags, return false. 3632 */ 3633bool pci_acs_path_enabled(struct pci_dev *start, 3634 struct pci_dev *end, u16 acs_flags) 3635{ 3636 struct pci_dev *pdev, *parent = start; 3637 3638 do { 3639 pdev = parent; 3640 3641 if (!pci_acs_enabled(pdev, acs_flags)) 3642 return false; 3643 3644 if (pci_is_root_bus(pdev->bus)) 3645 return (end == NULL); 3646 3647 parent = pdev->bus->self; 3648 } while (pdev != end); 3649 3650 return true; 3651} 3652 3653/** 3654 * pci_acs_init - Initialize ACS if hardware supports it 3655 * @dev: the PCI device 3656 */ 3657void pci_acs_init(struct pci_dev *dev) 3658{ 3659 dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); 3660} 3661 3662/** 3663 * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port 3664 * @dev: the PCI device 3665 * @cap_mask: mask of desired AtomicOp sizes, including one or more of: 3666 * PCI_EXP_DEVCAP2_ATOMIC_COMP32 3667 * PCI_EXP_DEVCAP2_ATOMIC_COMP64 3668 * PCI_EXP_DEVCAP2_ATOMIC_COMP128 3669 * 3670 * Return 0 if all upstream bridges support AtomicOp routing, egress 3671 * blocking is disabled on all upstream ports, and the root port supports 3672 * the requested completion capabilities (32-bit, 64-bit and/or 128-bit 3673 * AtomicOp completion), or negative otherwise. 3674 */ 3675int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) 3676{ 3677 struct pci_bus *bus = dev->bus; 3678 struct pci_dev *bridge; 3679 u32 cap, ctl2; 3680 3681 /* 3682 * Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit 3683 * in Device Control 2 is reserved in VFs and the PF value applies 3684 * to all associated VFs. 3685 */ 3686 if (dev->is_virtfn) 3687 return -EINVAL; 3688 3689 if (!pci_is_pcie(dev)) 3690 return -EINVAL; 3691 3692 /* 3693 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be 3694 * AtomicOp requesters. For now, we only support endpoints as 3695 * requesters and root ports as completers. No endpoints as 3696 * completers, and no peer-to-peer. 3697 */ 3698 3699 switch (pci_pcie_type(dev)) { 3700 case PCI_EXP_TYPE_ENDPOINT: 3701 case PCI_EXP_TYPE_LEG_END: 3702 case PCI_EXP_TYPE_RC_END: 3703 break; 3704 default: 3705 return -EINVAL; 3706 } 3707 3708 while (bus->parent) { 3709 bridge = bus->self; 3710 3711 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3712 3713 switch (pci_pcie_type(bridge)) { 3714 /* Ensure switch ports support AtomicOp routing */ 3715 case PCI_EXP_TYPE_UPSTREAM: 3716 case PCI_EXP_TYPE_DOWNSTREAM: 3717 if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) 3718 return -EINVAL; 3719 break; 3720 3721 /* Ensure root port supports all the sizes we care about */ 3722 case PCI_EXP_TYPE_ROOT_PORT: 3723 if ((cap & cap_mask) != cap_mask) 3724 return -EINVAL; 3725 break; 3726 } 3727 3728 /* Ensure upstream ports don't block AtomicOps on egress */ 3729 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) { 3730 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, 3731 &ctl2); 3732 if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK) 3733 return -EINVAL; 3734 } 3735 3736 bus = bus->parent; 3737 } 3738 3739 pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, 3740 PCI_EXP_DEVCTL2_ATOMIC_REQ); 3741 return 0; 3742} 3743EXPORT_SYMBOL(pci_enable_atomic_ops_to_root); 3744 3745/** 3746 * pci_release_region - Release a PCI bar 3747 * @pdev: PCI device whose resources were previously reserved by 3748 * pci_request_region() 3749 * @bar: BAR to release 3750 * 3751 * Releases the PCI I/O and memory resources previously reserved by a 3752 * successful call to pci_request_region(). Call this function only 3753 * after all use of the PCI regions has ceased. 3754 */ 3755void pci_release_region(struct pci_dev *pdev, int bar) 3756{ 3757 if (!pci_bar_index_is_valid(bar)) 3758 return; 3759 3760 if (pci_resource_len(pdev, bar) == 0) 3761 return; 3762 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) 3763 release_region(pci_resource_start(pdev, bar), 3764 pci_resource_len(pdev, bar)); 3765 else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) 3766 release_mem_region(pci_resource_start(pdev, bar), 3767 pci_resource_len(pdev, bar)); 3768} 3769EXPORT_SYMBOL(pci_release_region); 3770 3771/** 3772 * __pci_request_region - Reserved PCI I/O and memory resource 3773 * @pdev: PCI device whose resources are to be reserved 3774 * @bar: BAR to be reserved 3775 * @name: name of the driver requesting the resource 3776 * @exclusive: whether the region access is exclusive or not 3777 * 3778 * Returns: 0 on success, negative error code on failure. 3779 * 3780 * Mark the PCI region associated with PCI device @pdev BAR @bar as being 3781 * reserved by owner @name. Do not access any address inside the PCI regions 3782 * unless this call returns successfully. 3783 * 3784 * If @exclusive is set, then the region is marked so that userspace 3785 * is explicitly not allowed to map the resource via /dev/mem or 3786 * sysfs MMIO access. 3787 * 3788 * Returns 0 on success, or %EBUSY on error. A warning 3789 * message is also printed on failure. 3790 */ 3791static int __pci_request_region(struct pci_dev *pdev, int bar, 3792 const char *name, int exclusive) 3793{ 3794 if (!pci_bar_index_is_valid(bar)) 3795 return -EINVAL; 3796 3797 if (pci_resource_len(pdev, bar) == 0) 3798 return 0; 3799 3800 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) { 3801 if (!request_region(pci_resource_start(pdev, bar), 3802 pci_resource_len(pdev, bar), name)) 3803 goto err_out; 3804 } else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) { 3805 if (!__request_mem_region(pci_resource_start(pdev, bar), 3806 pci_resource_len(pdev, bar), name, 3807 exclusive)) 3808 goto err_out; 3809 } 3810 3811 return 0; 3812 3813err_out: 3814 pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar, 3815 &pdev->resource[bar]); 3816 return -EBUSY; 3817} 3818 3819/** 3820 * pci_request_region - Reserve PCI I/O and memory resource 3821 * @pdev: PCI device whose resources are to be reserved 3822 * @bar: BAR to be reserved 3823 * @name: name of the driver requesting the resource 3824 * 3825 * Returns: 0 on success, negative error code on failure. 3826 * 3827 * Mark the PCI region associated with PCI device @pdev BAR @bar as being 3828 * reserved by owner @name. Do not access any address inside the PCI regions 3829 * unless this call returns successfully. 3830 * 3831 * Returns 0 on success, or %EBUSY on error. A warning 3832 * message is also printed on failure. 3833 */ 3834int pci_request_region(struct pci_dev *pdev, int bar, const char *name) 3835{ 3836 return __pci_request_region(pdev, bar, name, 0); 3837} 3838EXPORT_SYMBOL(pci_request_region); 3839 3840/** 3841 * pci_release_selected_regions - Release selected PCI I/O and memory resources 3842 * @pdev: PCI device whose resources were previously reserved 3843 * @bars: Bitmask of BARs to be released 3844 * 3845 * Release selected PCI I/O and memory resources previously reserved. 3846 * Call this function only after all use of the PCI regions has ceased. 3847 */ 3848void pci_release_selected_regions(struct pci_dev *pdev, int bars) 3849{ 3850 int i; 3851 3852 for (i = 0; i < PCI_STD_NUM_BARS; i++) 3853 if (bars & (1 << i)) 3854 pci_release_region(pdev, i); 3855} 3856EXPORT_SYMBOL(pci_release_selected_regions); 3857 3858static int __pci_request_selected_regions(struct pci_dev *pdev, int bars, 3859 const char *name, int excl) 3860{ 3861 int i; 3862 3863 for (i = 0; i < PCI_STD_NUM_BARS; i++) 3864 if (bars & (1 << i)) 3865 if (__pci_request_region(pdev, i, name, excl)) 3866 goto err_out; 3867 return 0; 3868 3869err_out: 3870 while (--i >= 0) 3871 if (bars & (1 << i)) 3872 pci_release_region(pdev, i); 3873 3874 return -EBUSY; 3875} 3876 3877 3878/** 3879 * pci_request_selected_regions - Reserve selected PCI I/O and memory resources 3880 * @pdev: PCI device whose resources are to be reserved 3881 * @bars: Bitmask of BARs to be requested 3882 * @name: Name of the driver requesting the resources 3883 * 3884 * Returns: 0 on success, negative error code on failure. 3885 */ 3886int pci_request_selected_regions(struct pci_dev *pdev, int bars, 3887 const char *name) 3888{ 3889 return __pci_request_selected_regions(pdev, bars, name, 0); 3890} 3891EXPORT_SYMBOL(pci_request_selected_regions); 3892 3893/** 3894 * pci_request_selected_regions_exclusive - Request regions exclusively 3895 * @pdev: PCI device to request regions from 3896 * @bars: bit mask of BARs to request 3897 * @name: name of the driver requesting the resources 3898 * 3899 * Returns: 0 on success, negative error code on failure. 3900 */ 3901int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars, 3902 const char *name) 3903{ 3904 return __pci_request_selected_regions(pdev, bars, name, 3905 IORESOURCE_EXCLUSIVE); 3906} 3907EXPORT_SYMBOL(pci_request_selected_regions_exclusive); 3908 3909/** 3910 * pci_release_regions - Release reserved PCI I/O and memory resources 3911 * @pdev: PCI device whose resources were previously reserved by 3912 * pci_request_regions() 3913 * 3914 * Releases all PCI I/O and memory resources previously reserved by a 3915 * successful call to pci_request_regions(). Call this function only 3916 * after all use of the PCI regions has ceased. 3917 */ 3918void pci_release_regions(struct pci_dev *pdev) 3919{ 3920 pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1); 3921} 3922EXPORT_SYMBOL(pci_release_regions); 3923 3924/** 3925 * pci_request_regions - Reserve PCI I/O and memory resources 3926 * @pdev: PCI device whose resources are to be reserved 3927 * @name: name of the driver requesting the resources 3928 * 3929 * Mark all PCI regions associated with PCI device @pdev as being reserved by 3930 * owner @name. Do not access any address inside the PCI regions unless this 3931 * call returns successfully. 3932 * 3933 * Returns 0 on success, or %EBUSY on error. A warning 3934 * message is also printed on failure. 3935 */ 3936int pci_request_regions(struct pci_dev *pdev, const char *name) 3937{ 3938 return pci_request_selected_regions(pdev, 3939 ((1 << PCI_STD_NUM_BARS) - 1), name); 3940} 3941EXPORT_SYMBOL(pci_request_regions); 3942 3943/** 3944 * pci_request_regions_exclusive - Reserve PCI I/O and memory resources 3945 * @pdev: PCI device whose resources are to be reserved 3946 * @name: name of the driver requesting the resources 3947 * 3948 * Returns: 0 on success, negative error code on failure. 3949 * 3950 * Mark all PCI regions associated with PCI device @pdev as being reserved 3951 * by owner @name. Do not access any address inside the PCI regions 3952 * unless this call returns successfully. 3953 * 3954 * pci_request_regions_exclusive() will mark the region so that /dev/mem 3955 * and the sysfs MMIO access will not be allowed. 3956 * 3957 * Returns 0 on success, or %EBUSY on error. A warning message is also 3958 * printed on failure. 3959 */ 3960int pci_request_regions_exclusive(struct pci_dev *pdev, const char *name) 3961{ 3962 return pci_request_selected_regions_exclusive(pdev, 3963 ((1 << PCI_STD_NUM_BARS) - 1), name); 3964} 3965EXPORT_SYMBOL(pci_request_regions_exclusive); 3966 3967/* 3968 * Record the PCI IO range (expressed as CPU physical address + size). 3969 * Return a negative value if an error has occurred, zero otherwise 3970 */ 3971int pci_register_io_range(const struct fwnode_handle *fwnode, phys_addr_t addr, 3972 resource_size_t size) 3973{ 3974 int ret = 0; 3975#ifdef PCI_IOBASE 3976 struct logic_pio_hwaddr *range; 3977 3978 if (!size || addr + size < addr) 3979 return -EINVAL; 3980 3981 range = kzalloc(sizeof(*range), GFP_ATOMIC); 3982 if (!range) 3983 return -ENOMEM; 3984 3985 range->fwnode = fwnode; 3986 range->size = size; 3987 range->hw_start = addr; 3988 range->flags = LOGIC_PIO_CPU_MMIO; 3989 3990 ret = logic_pio_register_range(range); 3991 if (ret) 3992 kfree(range); 3993 3994 /* Ignore duplicates due to deferred probing */ 3995 if (ret == -EEXIST) 3996 ret = 0; 3997#endif 3998 3999 return ret; 4000} 4001 4002phys_addr_t pci_pio_to_address(unsigned long pio) 4003{ 4004#ifdef PCI_IOBASE 4005 if (pio < MMIO_UPPER_LIMIT) 4006 return logic_pio_to_hwaddr(pio); 4007#endif 4008 4009 return (phys_addr_t) OF_BAD_ADDR; 4010} 4011EXPORT_SYMBOL_GPL(pci_pio_to_address); 4012 4013unsigned long __weak pci_address_to_pio(phys_addr_t address) 4014{ 4015#ifdef PCI_IOBASE 4016 return logic_pio_trans_cpuaddr(address); 4017#else 4018 if (address > IO_SPACE_LIMIT) 4019 return (unsigned long)-1; 4020 4021 return (unsigned long) address; 4022#endif 4023} 4024 4025/** 4026 * pci_remap_iospace - Remap the memory mapped I/O space 4027 * @res: Resource describing the I/O space 4028 * @phys_addr: physical address of range to be mapped 4029 * 4030 * Remap the memory mapped I/O space described by the @res and the CPU 4031 * physical address @phys_addr into virtual address space. Only 4032 * architectures that have memory mapped IO functions defined (and the 4033 * PCI_IOBASE value defined) should call this function. 4034 */ 4035#ifndef pci_remap_iospace 4036int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) 4037{ 4038#if defined(PCI_IOBASE) 4039 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4040 4041 if (!(res->flags & IORESOURCE_IO)) 4042 return -EINVAL; 4043 4044 if (res->end > IO_SPACE_LIMIT) 4045 return -EINVAL; 4046 4047 return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr, 4048 pgprot_device(PAGE_KERNEL)); 4049#else 4050 /* 4051 * This architecture does not have memory mapped I/O space, 4052 * so this function should never be called 4053 */ 4054 WARN_ONCE(1, "This architecture does not support memory mapped I/O\n"); 4055 return -ENODEV; 4056#endif 4057} 4058EXPORT_SYMBOL(pci_remap_iospace); 4059#endif 4060 4061/** 4062 * pci_unmap_iospace - Unmap the memory mapped I/O space 4063 * @res: resource to be unmapped 4064 * 4065 * Unmap the CPU virtual address @res from virtual address space. Only 4066 * architectures that have memory mapped IO functions defined (and the 4067 * PCI_IOBASE value defined) should call this function. 4068 */ 4069void pci_unmap_iospace(struct resource *res) 4070{ 4071#if defined(PCI_IOBASE) 4072 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4073 4074 vunmap_range(vaddr, vaddr + resource_size(res)); 4075#endif 4076} 4077EXPORT_SYMBOL(pci_unmap_iospace); 4078 4079static void __pci_set_master(struct pci_dev *dev, bool enable) 4080{ 4081 u16 old_cmd, cmd; 4082 4083 pci_read_config_word(dev, PCI_COMMAND, &old_cmd); 4084 if (enable) 4085 cmd = old_cmd | PCI_COMMAND_MASTER; 4086 else 4087 cmd = old_cmd & ~PCI_COMMAND_MASTER; 4088 if (cmd != old_cmd) { 4089 pci_dbg(dev, "%s bus mastering\n", 4090 enable ? "enabling" : "disabling"); 4091 pci_write_config_word(dev, PCI_COMMAND, cmd); 4092 } 4093 dev->is_busmaster = enable; 4094} 4095 4096/** 4097 * pcibios_setup - process "pci=" kernel boot arguments 4098 * @str: string used to pass in "pci=" kernel boot arguments 4099 * 4100 * Process kernel boot arguments. This is the default implementation. 4101 * Architecture specific implementations can override this as necessary. 4102 */ 4103char * __weak __init pcibios_setup(char *str) 4104{ 4105 return str; 4106} 4107 4108/** 4109 * pcibios_set_master - enable PCI bus-mastering for device dev 4110 * @dev: the PCI device to enable 4111 * 4112 * Enables PCI bus-mastering for the device. This is the default 4113 * implementation. Architecture specific implementations can override 4114 * this if necessary. 4115 */ 4116void __weak pcibios_set_master(struct pci_dev *dev) 4117{ 4118 u8 lat; 4119 4120 /* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */ 4121 if (pci_is_pcie(dev)) 4122 return; 4123 4124 pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat); 4125 if (lat < 16) 4126 lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency; 4127 else if (lat > pcibios_max_latency) 4128 lat = pcibios_max_latency; 4129 else 4130 return; 4131 4132 pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat); 4133} 4134 4135/** 4136 * pci_set_master - enables bus-mastering for device dev 4137 * @dev: the PCI device to enable 4138 * 4139 * Enables bus-mastering on the device and calls pcibios_set_master() 4140 * to do the needed arch specific settings. 4141 */ 4142void pci_set_master(struct pci_dev *dev) 4143{ 4144 __pci_set_master(dev, true); 4145 pcibios_set_master(dev); 4146} 4147EXPORT_SYMBOL(pci_set_master); 4148 4149/** 4150 * pci_clear_master - disables bus-mastering for device dev 4151 * @dev: the PCI device to disable 4152 */ 4153void pci_clear_master(struct pci_dev *dev) 4154{ 4155 __pci_set_master(dev, false); 4156} 4157EXPORT_SYMBOL(pci_clear_master); 4158 4159/** 4160 * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed 4161 * @dev: the PCI device for which MWI is to be enabled 4162 * 4163 * Helper function for pci_set_mwi. 4164 * Originally copied from drivers/net/acenic.c. 4165 * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>. 4166 * 4167 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4168 */ 4169int pci_set_cacheline_size(struct pci_dev *dev) 4170{ 4171 u8 cacheline_size; 4172 4173 if (!pci_cache_line_size) 4174 return -EINVAL; 4175 4176 /* Validate current setting: the PCI_CACHE_LINE_SIZE must be 4177 equal to or multiple of the right value. */ 4178 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4179 if (cacheline_size >= pci_cache_line_size && 4180 (cacheline_size % pci_cache_line_size) == 0) 4181 return 0; 4182 4183 /* Write the correct value. */ 4184 pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size); 4185 /* Read it back. */ 4186 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4187 if (cacheline_size == pci_cache_line_size) 4188 return 0; 4189 4190 pci_dbg(dev, "cache line size of %d is not supported\n", 4191 pci_cache_line_size << 2); 4192 4193 return -EINVAL; 4194} 4195EXPORT_SYMBOL_GPL(pci_set_cacheline_size); 4196 4197/** 4198 * pci_set_mwi - enables memory-write-invalidate PCI transaction 4199 * @dev: the PCI device for which MWI is enabled 4200 * 4201 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4202 * 4203 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4204 */ 4205int pci_set_mwi(struct pci_dev *dev) 4206{ 4207#ifdef PCI_DISABLE_MWI 4208 return 0; 4209#else 4210 int rc; 4211 u16 cmd; 4212 4213 rc = pci_set_cacheline_size(dev); 4214 if (rc) 4215 return rc; 4216 4217 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4218 if (!(cmd & PCI_COMMAND_INVALIDATE)) { 4219 pci_dbg(dev, "enabling Mem-Wr-Inval\n"); 4220 cmd |= PCI_COMMAND_INVALIDATE; 4221 pci_write_config_word(dev, PCI_COMMAND, cmd); 4222 } 4223 return 0; 4224#endif 4225} 4226EXPORT_SYMBOL(pci_set_mwi); 4227 4228/** 4229 * pci_try_set_mwi - enables memory-write-invalidate PCI transaction 4230 * @dev: the PCI device for which MWI is enabled 4231 * 4232 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4233 * Callers are not required to check the return value. 4234 * 4235 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4236 */ 4237int pci_try_set_mwi(struct pci_dev *dev) 4238{ 4239#ifdef PCI_DISABLE_MWI 4240 return 0; 4241#else 4242 return pci_set_mwi(dev); 4243#endif 4244} 4245EXPORT_SYMBOL(pci_try_set_mwi); 4246 4247/** 4248 * pci_clear_mwi - disables Memory-Write-Invalidate for device dev 4249 * @dev: the PCI device to disable 4250 * 4251 * Disables PCI Memory-Write-Invalidate transaction on the device 4252 */ 4253void pci_clear_mwi(struct pci_dev *dev) 4254{ 4255#ifndef PCI_DISABLE_MWI 4256 u16 cmd; 4257 4258 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4259 if (cmd & PCI_COMMAND_INVALIDATE) { 4260 cmd &= ~PCI_COMMAND_INVALIDATE; 4261 pci_write_config_word(dev, PCI_COMMAND, cmd); 4262 } 4263#endif 4264} 4265EXPORT_SYMBOL(pci_clear_mwi); 4266 4267/** 4268 * pci_disable_parity - disable parity checking for device 4269 * @dev: the PCI device to operate on 4270 * 4271 * Disable parity checking for device @dev 4272 */ 4273void pci_disable_parity(struct pci_dev *dev) 4274{ 4275 u16 cmd; 4276 4277 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4278 if (cmd & PCI_COMMAND_PARITY) { 4279 cmd &= ~PCI_COMMAND_PARITY; 4280 pci_write_config_word(dev, PCI_COMMAND, cmd); 4281 } 4282} 4283 4284/** 4285 * pci_intx - enables/disables PCI INTx for device dev 4286 * @pdev: the PCI device to operate on 4287 * @enable: boolean: whether to enable or disable PCI INTx 4288 * 4289 * Enables/disables PCI INTx for device @pdev 4290 */ 4291void pci_intx(struct pci_dev *pdev, int enable) 4292{ 4293 u16 pci_command, new; 4294 4295 pci_read_config_word(pdev, PCI_COMMAND, &pci_command); 4296 4297 if (enable) 4298 new = pci_command & ~PCI_COMMAND_INTX_DISABLE; 4299 else 4300 new = pci_command | PCI_COMMAND_INTX_DISABLE; 4301 4302 if (new == pci_command) 4303 return; 4304 4305 pci_write_config_word(pdev, PCI_COMMAND, new); 4306} 4307EXPORT_SYMBOL_GPL(pci_intx); 4308 4309/** 4310 * pci_wait_for_pending_transaction - wait for pending transaction 4311 * @dev: the PCI device to operate on 4312 * 4313 * Return 0 if transaction is pending 1 otherwise. 4314 */ 4315int pci_wait_for_pending_transaction(struct pci_dev *dev) 4316{ 4317 if (!pci_is_pcie(dev)) 4318 return 1; 4319 4320 return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA, 4321 PCI_EXP_DEVSTA_TRPND); 4322} 4323EXPORT_SYMBOL(pci_wait_for_pending_transaction); 4324 4325/** 4326 * pcie_flr - initiate a PCIe function level reset 4327 * @dev: device to reset 4328 * 4329 * Initiate a function level reset unconditionally on @dev without 4330 * checking any flags and DEVCAP 4331 */ 4332int pcie_flr(struct pci_dev *dev) 4333{ 4334 if (!pci_wait_for_pending_transaction(dev)) 4335 pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n"); 4336 4337 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); 4338 4339 if (dev->imm_ready) 4340 return 0; 4341 4342 /* 4343 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within 4344 * 100ms, but may silently discard requests while the FLR is in 4345 * progress. Wait 100ms before trying to access the device. 4346 */ 4347 msleep(100); 4348 4349 return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS); 4350} 4351EXPORT_SYMBOL_GPL(pcie_flr); 4352 4353/** 4354 * pcie_reset_flr - initiate a PCIe function level reset 4355 * @dev: device to reset 4356 * @probe: if true, return 0 if device can be reset this way 4357 * 4358 * Initiate a function level reset on @dev. 4359 */ 4360int pcie_reset_flr(struct pci_dev *dev, bool probe) 4361{ 4362 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4363 return -ENOTTY; 4364 4365 if (!(dev->devcap & PCI_EXP_DEVCAP_FLR)) 4366 return -ENOTTY; 4367 4368 if (probe) 4369 return 0; 4370 4371 return pcie_flr(dev); 4372} 4373EXPORT_SYMBOL_GPL(pcie_reset_flr); 4374 4375static int pci_af_flr(struct pci_dev *dev, bool probe) 4376{ 4377 int pos; 4378 u8 cap; 4379 4380 pos = pci_find_capability(dev, PCI_CAP_ID_AF); 4381 if (!pos) 4382 return -ENOTTY; 4383 4384 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4385 return -ENOTTY; 4386 4387 pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); 4388 if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR)) 4389 return -ENOTTY; 4390 4391 if (probe) 4392 return 0; 4393 4394 /* 4395 * Wait for Transaction Pending bit to clear. A word-aligned test 4396 * is used, so we use the control offset rather than status and shift 4397 * the test bit to match. 4398 */ 4399 if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL, 4400 PCI_AF_STATUS_TP << 8)) 4401 pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n"); 4402 4403 pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR); 4404 4405 if (dev->imm_ready) 4406 return 0; 4407 4408 /* 4409 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006, 4410 * updated 27 July 2006; a device must complete an FLR within 4411 * 100ms, but may silently discard requests while the FLR is in 4412 * progress. Wait 100ms before trying to access the device. 4413 */ 4414 msleep(100); 4415 4416 return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS); 4417} 4418 4419/** 4420 * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0. 4421 * @dev: Device to reset. 4422 * @probe: if true, return 0 if the device can be reset this way. 4423 * 4424 * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is 4425 * unset, it will be reinitialized internally when going from PCI_D3hot to 4426 * PCI_D0. If that's the case and the device is not in a low-power state 4427 * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset. 4428 * 4429 * NOTE: This causes the caller to sleep for twice the device power transition 4430 * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms 4431 * by default (i.e. unless the @dev's d3hot_delay field has a different value). 4432 * Moreover, only devices in D0 can be reset by this function. 4433 */ 4434static int pci_pm_reset(struct pci_dev *dev, bool probe) 4435{ 4436 u16 csr; 4437 4438 if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET) 4439 return -ENOTTY; 4440 4441 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr); 4442 if (csr & PCI_PM_CTRL_NO_SOFT_RESET) 4443 return -ENOTTY; 4444 4445 if (probe) 4446 return 0; 4447 4448 if (dev->current_state != PCI_D0) 4449 return -EINVAL; 4450 4451 csr &= ~PCI_PM_CTRL_STATE_MASK; 4452 csr |= PCI_D3hot; 4453 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4454 pci_dev_d3_sleep(dev); 4455 4456 csr &= ~PCI_PM_CTRL_STATE_MASK; 4457 csr |= PCI_D0; 4458 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4459 pci_dev_d3_sleep(dev); 4460 4461 return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS); 4462} 4463 4464/** 4465 * pcie_wait_for_link_status - Wait for link status change 4466 * @pdev: Device whose link to wait for. 4467 * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE. 4468 * @active: Waiting for active or inactive? 4469 * 4470 * Return 0 if successful, or -ETIMEDOUT if status has not changed within 4471 * PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds. 4472 */ 4473static int pcie_wait_for_link_status(struct pci_dev *pdev, 4474 bool use_lt, bool active) 4475{ 4476 u16 lnksta_mask, lnksta_match; 4477 unsigned long end_jiffies; 4478 u16 lnksta; 4479 4480 lnksta_mask = use_lt ? PCI_EXP_LNKSTA_LT : PCI_EXP_LNKSTA_DLLLA; 4481 lnksta_match = active ? lnksta_mask : 0; 4482 4483 end_jiffies = jiffies + msecs_to_jiffies(PCIE_LINK_RETRAIN_TIMEOUT_MS); 4484 do { 4485 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); 4486 if ((lnksta & lnksta_mask) == lnksta_match) 4487 return 0; 4488 msleep(1); 4489 } while (time_before(jiffies, end_jiffies)); 4490 4491 return -ETIMEDOUT; 4492} 4493 4494/** 4495 * pcie_retrain_link - Request a link retrain and wait for it to complete 4496 * @pdev: Device whose link to retrain. 4497 * @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE, for status. 4498 * 4499 * Trigger retraining of the PCIe Link and wait for the completion of the 4500 * retraining. As link retraining is known to asserts LBMS and may change 4501 * the Link Speed, LBMS is cleared after the retraining and the Link Speed 4502 * of the subordinate bus is updated. 4503 * 4504 * Retrain completion status is retrieved from the Link Status Register 4505 * according to @use_lt. It is not verified whether the use of the DLLLA 4506 * bit is valid. 4507 * 4508 * Return 0 if successful, or -ETIMEDOUT if training has not completed 4509 * within PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds. 4510 */ 4511int pcie_retrain_link(struct pci_dev *pdev, bool use_lt) 4512{ 4513 int rc; 4514 4515 /* 4516 * Ensure the updated LNKCTL parameters are used during link 4517 * training by checking that there is no ongoing link training that 4518 * may have started before link parameters were changed, so as to 4519 * avoid LTSSM race as recommended in Implementation Note at the end 4520 * of PCIe r6.1 sec 7.5.3.7. 4521 */ 4522 rc = pcie_wait_for_link_status(pdev, true, false); 4523 if (rc) 4524 return rc; 4525 4526 pcie_capability_set_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL); 4527 if (pdev->clear_retrain_link) { 4528 /* 4529 * Due to an erratum in some devices the Retrain Link bit 4530 * needs to be cleared again manually to allow the link 4531 * training to succeed. 4532 */ 4533 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL); 4534 } 4535 4536 rc = pcie_wait_for_link_status(pdev, use_lt, !use_lt); 4537 4538 /* 4539 * Clear LBMS after a manual retrain so that the bit can be used 4540 * to track link speed or width changes made by hardware itself 4541 * in attempt to correct unreliable link operation. 4542 */ 4543 pcie_reset_lbms(pdev); 4544 4545 /* 4546 * Ensure the Link Speed updates after retraining in case the Link 4547 * Speed was changed because of the retraining. While the bwctrl's 4548 * IRQ handler normally picks up the new Link Speed, clearing LBMS 4549 * races with the IRQ handler reading the Link Status register and 4550 * can result in the handler returning early without updating the 4551 * Link Speed. 4552 */ 4553 if (pdev->subordinate) 4554 pcie_update_link_speed(pdev->subordinate); 4555 4556 return rc; 4557} 4558 4559/** 4560 * pcie_wait_for_link_delay - Wait until link is active or inactive 4561 * @pdev: Bridge device 4562 * @active: waiting for active or inactive? 4563 * @delay: Delay to wait after link has become active (in ms) 4564 * 4565 * Use this to wait till link becomes active or inactive. 4566 */ 4567static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active, 4568 int delay) 4569{ 4570 int rc; 4571 4572 /* 4573 * Some controllers might not implement link active reporting. In this 4574 * case, we wait for 1000 ms + any delay requested by the caller. 4575 */ 4576 if (!pdev->link_active_reporting) { 4577 msleep(PCIE_LINK_RETRAIN_TIMEOUT_MS + delay); 4578 return true; 4579 } 4580 4581 /* 4582 * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms, 4583 * after which we should expect the link to be active if the reset was 4584 * successful. If so, software must wait a minimum 100ms before sending 4585 * configuration requests to devices downstream this port. 4586 * 4587 * If the link fails to activate, either the device was physically 4588 * removed or the link is permanently failed. 4589 */ 4590 if (active) 4591 msleep(20); 4592 rc = pcie_wait_for_link_status(pdev, false, active); 4593 if (active) { 4594 if (rc) 4595 rc = pcie_failed_link_retrain(pdev); 4596 if (rc) 4597 return false; 4598 4599 msleep(delay); 4600 return true; 4601 } 4602 4603 if (rc) 4604 return false; 4605 4606 return true; 4607} 4608 4609/** 4610 * pcie_wait_for_link - Wait until link is active or inactive 4611 * @pdev: Bridge device 4612 * @active: waiting for active or inactive? 4613 * 4614 * Use this to wait till link becomes active or inactive. 4615 */ 4616bool pcie_wait_for_link(struct pci_dev *pdev, bool active) 4617{ 4618 return pcie_wait_for_link_delay(pdev, active, 100); 4619} 4620 4621/* 4622 * Find maximum D3cold delay required by all the devices on the bus. The 4623 * spec says 100 ms, but firmware can lower it and we allow drivers to 4624 * increase it as well. 4625 * 4626 * Called with @pci_bus_sem locked for reading. 4627 */ 4628static int pci_bus_max_d3cold_delay(const struct pci_bus *bus) 4629{ 4630 const struct pci_dev *pdev; 4631 int min_delay = 100; 4632 int max_delay = 0; 4633 4634 list_for_each_entry(pdev, &bus->devices, bus_list) { 4635 if (pdev->d3cold_delay < min_delay) 4636 min_delay = pdev->d3cold_delay; 4637 if (pdev->d3cold_delay > max_delay) 4638 max_delay = pdev->d3cold_delay; 4639 } 4640 4641 return max(min_delay, max_delay); 4642} 4643 4644/** 4645 * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible 4646 * @dev: PCI bridge 4647 * @reset_type: reset type in human-readable form 4648 * 4649 * Handle necessary delays before access to the devices on the secondary 4650 * side of the bridge are permitted after D3cold to D0 transition 4651 * or Conventional Reset. 4652 * 4653 * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For 4654 * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section 4655 * 4.3.2. 4656 * 4657 * Return 0 on success or -ENOTTY if the first device on the secondary bus 4658 * failed to become accessible. 4659 */ 4660int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type) 4661{ 4662 struct pci_dev *child __free(pci_dev_put) = NULL; 4663 int delay; 4664 4665 if (pci_dev_is_disconnected(dev)) 4666 return 0; 4667 4668 if (!pci_is_bridge(dev)) 4669 return 0; 4670 4671 down_read(&pci_bus_sem); 4672 4673 /* 4674 * We only deal with devices that are present currently on the bus. 4675 * For any hot-added devices the access delay is handled in pciehp 4676 * board_added(). In case of ACPI hotplug the firmware is expected 4677 * to configure the devices before OS is notified. 4678 */ 4679 if (!dev->subordinate || list_empty(&dev->subordinate->devices)) { 4680 up_read(&pci_bus_sem); 4681 return 0; 4682 } 4683 4684 /* Take d3cold_delay requirements into account */ 4685 delay = pci_bus_max_d3cold_delay(dev->subordinate); 4686 if (!delay) { 4687 up_read(&pci_bus_sem); 4688 return 0; 4689 } 4690 4691 child = pci_dev_get(list_first_entry(&dev->subordinate->devices, 4692 struct pci_dev, bus_list)); 4693 up_read(&pci_bus_sem); 4694 4695 /* 4696 * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before 4697 * accessing the device after reset (that is 1000 ms + 100 ms). 4698 */ 4699 if (!pci_is_pcie(dev)) { 4700 pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay); 4701 msleep(1000 + delay); 4702 return 0; 4703 } 4704 4705 /* 4706 * For PCIe downstream and root ports that do not support speeds 4707 * greater than 5 GT/s need to wait minimum 100 ms. For higher 4708 * speeds (gen3) we need to wait first for the data link layer to 4709 * become active. 4710 * 4711 * However, 100 ms is the minimum and the PCIe spec says the 4712 * software must allow at least 1s before it can determine that the 4713 * device that did not respond is a broken device. Also device can 4714 * take longer than that to respond if it indicates so through Request 4715 * Retry Status completions. 4716 * 4717 * Therefore we wait for 100 ms and check for the device presence 4718 * until the timeout expires. 4719 */ 4720 if (!pcie_downstream_port(dev)) 4721 return 0; 4722 4723 if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) { 4724 u16 status; 4725 4726 pci_dbg(dev, "waiting %d ms for downstream link\n", delay); 4727 msleep(delay); 4728 4729 if (!pci_dev_wait(child, reset_type, PCI_RESET_WAIT - delay)) 4730 return 0; 4731 4732 /* 4733 * If the port supports active link reporting we now check 4734 * whether the link is active and if not bail out early with 4735 * the assumption that the device is not present anymore. 4736 */ 4737 if (!dev->link_active_reporting) 4738 return -ENOTTY; 4739 4740 pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &status); 4741 if (!(status & PCI_EXP_LNKSTA_DLLLA)) 4742 return -ENOTTY; 4743 4744 return pci_dev_wait(child, reset_type, 4745 PCIE_RESET_READY_POLL_MS - PCI_RESET_WAIT); 4746 } 4747 4748 pci_dbg(dev, "waiting %d ms for downstream link, after activation\n", 4749 delay); 4750 if (!pcie_wait_for_link_delay(dev, true, delay)) { 4751 /* Did not train, no need to wait any further */ 4752 pci_info(dev, "Data Link Layer Link Active not set in %d msec\n", delay); 4753 return -ENOTTY; 4754 } 4755 4756 return pci_dev_wait(child, reset_type, 4757 PCIE_RESET_READY_POLL_MS - delay); 4758} 4759 4760void pci_reset_secondary_bus(struct pci_dev *dev) 4761{ 4762 u16 ctrl; 4763 4764 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl); 4765 ctrl |= PCI_BRIDGE_CTL_BUS_RESET; 4766 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 4767 4768 /* 4769 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double 4770 * this to 2ms to ensure that we meet the minimum requirement. 4771 */ 4772 msleep(2); 4773 4774 ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; 4775 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 4776} 4777 4778void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) 4779{ 4780 pci_reset_secondary_bus(dev); 4781} 4782 4783/** 4784 * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge. 4785 * @dev: Bridge device 4786 * 4787 * Use the bridge control register to assert reset on the secondary bus. 4788 * Devices on the secondary bus are left in power-on state. 4789 */ 4790int pci_bridge_secondary_bus_reset(struct pci_dev *dev) 4791{ 4792 if (!dev->block_cfg_access) 4793 pci_warn_once(dev, "unlocked secondary bus reset via: %pS\n", 4794 __builtin_return_address(0)); 4795 pcibios_reset_secondary_bus(dev); 4796 4797 return pci_bridge_wait_for_secondary_bus(dev, "bus reset"); 4798} 4799EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset); 4800 4801static int pci_parent_bus_reset(struct pci_dev *dev, bool probe) 4802{ 4803 struct pci_dev *pdev; 4804 4805 if (pci_is_root_bus(dev->bus) || dev->subordinate || 4806 !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 4807 return -ENOTTY; 4808 4809 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 4810 if (pdev != dev) 4811 return -ENOTTY; 4812 4813 if (probe) 4814 return 0; 4815 4816 return pci_bridge_secondary_bus_reset(dev->bus->self); 4817} 4818 4819static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe) 4820{ 4821 int rc = -ENOTTY; 4822 4823 if (!hotplug || !try_module_get(hotplug->owner)) 4824 return rc; 4825 4826 if (hotplug->ops->reset_slot) 4827 rc = hotplug->ops->reset_slot(hotplug, probe); 4828 4829 module_put(hotplug->owner); 4830 4831 return rc; 4832} 4833 4834static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe) 4835{ 4836 if (dev->multifunction || dev->subordinate || !dev->slot || 4837 dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 4838 return -ENOTTY; 4839 4840 return pci_reset_hotplug_slot(dev->slot->hotplug, probe); 4841} 4842 4843static u16 cxl_port_dvsec(struct pci_dev *dev) 4844{ 4845 return pci_find_dvsec_capability(dev, PCI_VENDOR_ID_CXL, 4846 PCI_DVSEC_CXL_PORT); 4847} 4848 4849static bool cxl_sbr_masked(struct pci_dev *dev) 4850{ 4851 u16 dvsec, reg; 4852 int rc; 4853 4854 dvsec = cxl_port_dvsec(dev); 4855 if (!dvsec) 4856 return false; 4857 4858 rc = pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_PORT_CTL, ®); 4859 if (rc || PCI_POSSIBLE_ERROR(reg)) 4860 return false; 4861 4862 /* 4863 * Per CXL spec r3.1, sec 8.1.5.2, when "Unmask SBR" is 0, the SBR 4864 * bit in Bridge Control has no effect. When 1, the Port generates 4865 * hot reset when the SBR bit is set to 1. 4866 */ 4867 if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) 4868 return false; 4869 4870 return true; 4871} 4872 4873static int pci_reset_bus_function(struct pci_dev *dev, bool probe) 4874{ 4875 struct pci_dev *bridge = pci_upstream_bridge(dev); 4876 int rc; 4877 4878 /* 4879 * If "dev" is below a CXL port that has SBR control masked, SBR 4880 * won't do anything, so return error. 4881 */ 4882 if (bridge && cxl_sbr_masked(bridge)) { 4883 if (probe) 4884 return 0; 4885 4886 return -ENOTTY; 4887 } 4888 4889 rc = pci_dev_reset_slot_function(dev, probe); 4890 if (rc != -ENOTTY) 4891 return rc; 4892 return pci_parent_bus_reset(dev, probe); 4893} 4894 4895static int cxl_reset_bus_function(struct pci_dev *dev, bool probe) 4896{ 4897 struct pci_dev *bridge; 4898 u16 dvsec, reg, val; 4899 int rc; 4900 4901 bridge = pci_upstream_bridge(dev); 4902 if (!bridge) 4903 return -ENOTTY; 4904 4905 dvsec = cxl_port_dvsec(bridge); 4906 if (!dvsec) 4907 return -ENOTTY; 4908 4909 if (probe) 4910 return 0; 4911 4912 rc = pci_read_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, ®); 4913 if (rc) 4914 return -ENOTTY; 4915 4916 if (reg & PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR) { 4917 val = reg; 4918 } else { 4919 val = reg | PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR; 4920 pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, 4921 val); 4922 } 4923 4924 rc = pci_reset_bus_function(dev, probe); 4925 4926 if (reg != val) 4927 pci_write_config_word(bridge, dvsec + PCI_DVSEC_CXL_PORT_CTL, 4928 reg); 4929 4930 return rc; 4931} 4932 4933void pci_dev_lock(struct pci_dev *dev) 4934{ 4935 /* block PM suspend, driver probe, etc. */ 4936 device_lock(&dev->dev); 4937 pci_cfg_access_lock(dev); 4938} 4939EXPORT_SYMBOL_GPL(pci_dev_lock); 4940 4941/* Return 1 on successful lock, 0 on contention */ 4942int pci_dev_trylock(struct pci_dev *dev) 4943{ 4944 if (device_trylock(&dev->dev)) { 4945 if (pci_cfg_access_trylock(dev)) 4946 return 1; 4947 device_unlock(&dev->dev); 4948 } 4949 4950 return 0; 4951} 4952EXPORT_SYMBOL_GPL(pci_dev_trylock); 4953 4954void pci_dev_unlock(struct pci_dev *dev) 4955{ 4956 pci_cfg_access_unlock(dev); 4957 device_unlock(&dev->dev); 4958} 4959EXPORT_SYMBOL_GPL(pci_dev_unlock); 4960 4961static void pci_dev_save_and_disable(struct pci_dev *dev) 4962{ 4963 const struct pci_error_handlers *err_handler = 4964 dev->driver ? dev->driver->err_handler : NULL; 4965 4966 /* 4967 * dev->driver->err_handler->reset_prepare() is protected against 4968 * races with ->remove() by the device lock, which must be held by 4969 * the caller. 4970 */ 4971 if (err_handler && err_handler->reset_prepare) 4972 err_handler->reset_prepare(dev); 4973 else if (dev->driver) 4974 pci_warn(dev, "resetting"); 4975 4976 /* 4977 * Wake-up device prior to save. PM registers default to D0 after 4978 * reset and a simple register restore doesn't reliably return 4979 * to a non-D0 state anyway. 4980 */ 4981 pci_set_power_state(dev, PCI_D0); 4982 4983 pci_save_state(dev); 4984 /* 4985 * Disable the device by clearing the Command register, except for 4986 * INTx-disable which is set. This not only disables MMIO and I/O port 4987 * BARs, but also prevents the device from being Bus Master, preventing 4988 * DMA from the device including MSI/MSI-X interrupts. For PCI 2.3 4989 * compliant devices, INTx-disable prevents legacy interrupts. 4990 */ 4991 pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 4992} 4993 4994static void pci_dev_restore(struct pci_dev *dev) 4995{ 4996 const struct pci_error_handlers *err_handler = 4997 dev->driver ? dev->driver->err_handler : NULL; 4998 4999 pci_restore_state(dev); 5000 5001 /* 5002 * dev->driver->err_handler->reset_done() is protected against 5003 * races with ->remove() by the device lock, which must be held by 5004 * the caller. 5005 */ 5006 if (err_handler && err_handler->reset_done) 5007 err_handler->reset_done(dev); 5008 else if (dev->driver) 5009 pci_warn(dev, "reset done"); 5010} 5011 5012/* dev->reset_methods[] is a 0-terminated list of indices into this array */ 5013const struct pci_reset_fn_method pci_reset_fn_methods[] = { 5014 { }, 5015 { pci_dev_specific_reset, .name = "device_specific" }, 5016 { pci_dev_acpi_reset, .name = "acpi" }, 5017 { pcie_reset_flr, .name = "flr" }, 5018 { pci_af_flr, .name = "af_flr" }, 5019 { pci_pm_reset, .name = "pm" }, 5020 { pci_reset_bus_function, .name = "bus" }, 5021 { cxl_reset_bus_function, .name = "cxl_bus" }, 5022}; 5023 5024/** 5025 * __pci_reset_function_locked - reset a PCI device function while holding 5026 * the @dev mutex lock. 5027 * @dev: PCI device to reset 5028 * 5029 * Some devices allow an individual function to be reset without affecting 5030 * other functions in the same device. The PCI device must be responsive 5031 * to PCI config space in order to use this function. 5032 * 5033 * The device function is presumed to be unused and the caller is holding 5034 * the device mutex lock when this function is called. 5035 * 5036 * Resetting the device will make the contents of PCI configuration space 5037 * random, so any caller of this must be prepared to reinitialise the 5038 * device including MSI, bus mastering, BARs, decoding IO and memory spaces, 5039 * etc. 5040 * 5041 * Returns 0 if the device function was successfully reset or negative if the 5042 * device doesn't support resetting a single function. 5043 */ 5044int __pci_reset_function_locked(struct pci_dev *dev) 5045{ 5046 int i, m, rc; 5047 const struct pci_reset_fn_method *method; 5048 5049 might_sleep(); 5050 5051 /* 5052 * A reset method returns -ENOTTY if it doesn't support this device and 5053 * we should try the next method. 5054 * 5055 * If it returns 0 (success), we're finished. If it returns any other 5056 * error, we're also finished: this indicates that further reset 5057 * mechanisms might be broken on the device. 5058 */ 5059 for (i = 0; i < PCI_NUM_RESET_METHODS; i++) { 5060 m = dev->reset_methods[i]; 5061 if (!m) 5062 return -ENOTTY; 5063 5064 method = &pci_reset_fn_methods[m]; 5065 pci_dbg(dev, "reset via %s\n", method->name); 5066 rc = method->reset_fn(dev, PCI_RESET_DO_RESET); 5067 if (!rc) 5068 return 0; 5069 5070 pci_dbg(dev, "%s failed with %d\n", method->name, rc); 5071 if (rc != -ENOTTY) 5072 return rc; 5073 } 5074 5075 return -ENOTTY; 5076} 5077EXPORT_SYMBOL_GPL(__pci_reset_function_locked); 5078 5079/** 5080 * pci_init_reset_methods - check whether device can be safely reset 5081 * and store supported reset mechanisms. 5082 * @dev: PCI device to check for reset mechanisms 5083 * 5084 * Some devices allow an individual function to be reset without affecting 5085 * other functions in the same device. The PCI device must be in D0-D3hot 5086 * state. 5087 * 5088 * Stores reset mechanisms supported by device in reset_methods byte array 5089 * which is a member of struct pci_dev. 5090 */ 5091void pci_init_reset_methods(struct pci_dev *dev) 5092{ 5093 int m, i, rc; 5094 5095 BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS); 5096 5097 might_sleep(); 5098 5099 i = 0; 5100 for (m = 1; m < PCI_NUM_RESET_METHODS; m++) { 5101 rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE); 5102 if (!rc) 5103 dev->reset_methods[i++] = m; 5104 else if (rc != -ENOTTY) 5105 break; 5106 } 5107 5108 dev->reset_methods[i] = 0; 5109} 5110 5111/** 5112 * pci_reset_function - quiesce and reset a PCI device function 5113 * @dev: PCI device to reset 5114 * 5115 * Some devices allow an individual function to be reset without affecting 5116 * other functions in the same device. The PCI device must be responsive 5117 * to PCI config space in order to use this function. 5118 * 5119 * This function does not just reset the PCI portion of a device, but 5120 * clears all the state associated with the device. This function differs 5121 * from __pci_reset_function_locked() in that it saves and restores device state 5122 * over the reset and takes the PCI device lock. 5123 * 5124 * Returns 0 if the device function was successfully reset or negative if the 5125 * device doesn't support resetting a single function. 5126 */ 5127int pci_reset_function(struct pci_dev *dev) 5128{ 5129 struct pci_dev *bridge; 5130 int rc; 5131 5132 if (!pci_reset_supported(dev)) 5133 return -ENOTTY; 5134 5135 /* 5136 * If there's no upstream bridge, no locking is needed since there is 5137 * no upstream bridge configuration to hold consistent. 5138 */ 5139 bridge = pci_upstream_bridge(dev); 5140 if (bridge) 5141 pci_dev_lock(bridge); 5142 5143 pci_dev_lock(dev); 5144 pci_dev_save_and_disable(dev); 5145 5146 rc = __pci_reset_function_locked(dev); 5147 5148 pci_dev_restore(dev); 5149 pci_dev_unlock(dev); 5150 5151 if (bridge) 5152 pci_dev_unlock(bridge); 5153 5154 return rc; 5155} 5156EXPORT_SYMBOL_GPL(pci_reset_function); 5157 5158/** 5159 * pci_reset_function_locked - quiesce and reset a PCI device function 5160 * @dev: PCI device to reset 5161 * 5162 * Some devices allow an individual function to be reset without affecting 5163 * other functions in the same device. The PCI device must be responsive 5164 * to PCI config space in order to use this function. 5165 * 5166 * This function does not just reset the PCI portion of a device, but 5167 * clears all the state associated with the device. This function differs 5168 * from __pci_reset_function_locked() in that it saves and restores device state 5169 * over the reset. It also differs from pci_reset_function() in that it 5170 * requires the PCI device lock to be held. 5171 * 5172 * Returns 0 if the device function was successfully reset or negative if the 5173 * device doesn't support resetting a single function. 5174 */ 5175int pci_reset_function_locked(struct pci_dev *dev) 5176{ 5177 int rc; 5178 5179 if (!pci_reset_supported(dev)) 5180 return -ENOTTY; 5181 5182 pci_dev_save_and_disable(dev); 5183 5184 rc = __pci_reset_function_locked(dev); 5185 5186 pci_dev_restore(dev); 5187 5188 return rc; 5189} 5190EXPORT_SYMBOL_GPL(pci_reset_function_locked); 5191 5192/** 5193 * pci_try_reset_function - quiesce and reset a PCI device function 5194 * @dev: PCI device to reset 5195 * 5196 * Same as above, except return -EAGAIN if unable to lock device. 5197 */ 5198int pci_try_reset_function(struct pci_dev *dev) 5199{ 5200 int rc; 5201 5202 if (!pci_reset_supported(dev)) 5203 return -ENOTTY; 5204 5205 if (!pci_dev_trylock(dev)) 5206 return -EAGAIN; 5207 5208 pci_dev_save_and_disable(dev); 5209 rc = __pci_reset_function_locked(dev); 5210 pci_dev_restore(dev); 5211 pci_dev_unlock(dev); 5212 5213 return rc; 5214} 5215EXPORT_SYMBOL_GPL(pci_try_reset_function); 5216 5217/* Do any devices on or below this bus prevent a bus reset? */ 5218static bool pci_bus_resettable(struct pci_bus *bus) 5219{ 5220 struct pci_dev *dev; 5221 5222 5223 if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5224 return false; 5225 5226 list_for_each_entry(dev, &bus->devices, bus_list) { 5227 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5228 (dev->subordinate && !pci_bus_resettable(dev->subordinate))) 5229 return false; 5230 } 5231 5232 return true; 5233} 5234 5235/* Lock devices from the top of the tree down */ 5236static void pci_bus_lock(struct pci_bus *bus) 5237{ 5238 struct pci_dev *dev; 5239 5240 pci_dev_lock(bus->self); 5241 list_for_each_entry(dev, &bus->devices, bus_list) { 5242 if (dev->subordinate) 5243 pci_bus_lock(dev->subordinate); 5244 else 5245 pci_dev_lock(dev); 5246 } 5247} 5248 5249/* Unlock devices from the bottom of the tree up */ 5250static void pci_bus_unlock(struct pci_bus *bus) 5251{ 5252 struct pci_dev *dev; 5253 5254 list_for_each_entry(dev, &bus->devices, bus_list) { 5255 if (dev->subordinate) 5256 pci_bus_unlock(dev->subordinate); 5257 else 5258 pci_dev_unlock(dev); 5259 } 5260 pci_dev_unlock(bus->self); 5261} 5262 5263/* Return 1 on successful lock, 0 on contention */ 5264static int pci_bus_trylock(struct pci_bus *bus) 5265{ 5266 struct pci_dev *dev; 5267 5268 if (!pci_dev_trylock(bus->self)) 5269 return 0; 5270 5271 list_for_each_entry(dev, &bus->devices, bus_list) { 5272 if (dev->subordinate) { 5273 if (!pci_bus_trylock(dev->subordinate)) 5274 goto unlock; 5275 } else if (!pci_dev_trylock(dev)) 5276 goto unlock; 5277 } 5278 return 1; 5279 5280unlock: 5281 list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) { 5282 if (dev->subordinate) 5283 pci_bus_unlock(dev->subordinate); 5284 else 5285 pci_dev_unlock(dev); 5286 } 5287 pci_dev_unlock(bus->self); 5288 return 0; 5289} 5290 5291/* Do any devices on or below this slot prevent a bus reset? */ 5292static bool pci_slot_resettable(struct pci_slot *slot) 5293{ 5294 struct pci_dev *dev, *bridge = slot->bus->self; 5295 5296 if (bridge && (bridge->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5297 return false; 5298 5299 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5300 if (!dev->slot || dev->slot != slot) 5301 continue; 5302 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5303 (dev->subordinate && !pci_bus_resettable(dev->subordinate))) 5304 return false; 5305 } 5306 5307 return true; 5308} 5309 5310/* Lock devices from the top of the tree down */ 5311static void pci_slot_lock(struct pci_slot *slot) 5312{ 5313 struct pci_dev *dev, *bridge = slot->bus->self; 5314 5315 if (bridge) 5316 pci_dev_lock(bridge); 5317 5318 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5319 if (!dev->slot || dev->slot != slot) 5320 continue; 5321 if (dev->subordinate) 5322 pci_bus_lock(dev->subordinate); 5323 else 5324 pci_dev_lock(dev); 5325 } 5326} 5327 5328/* Unlock devices from the bottom of the tree up */ 5329static void pci_slot_unlock(struct pci_slot *slot) 5330{ 5331 struct pci_dev *dev, *bridge = slot->bus->self; 5332 5333 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5334 if (!dev->slot || dev->slot != slot) 5335 continue; 5336 if (dev->subordinate) 5337 pci_bus_unlock(dev->subordinate); 5338 else 5339 pci_dev_unlock(dev); 5340 } 5341 5342 if (bridge) 5343 pci_dev_unlock(bridge); 5344} 5345 5346/* Return 1 on successful lock, 0 on contention */ 5347static int pci_slot_trylock(struct pci_slot *slot) 5348{ 5349 struct pci_dev *dev, *bridge = slot->bus->self; 5350 5351 if (bridge && !pci_dev_trylock(bridge)) 5352 return 0; 5353 5354 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5355 if (!dev->slot || dev->slot != slot) 5356 continue; 5357 if (dev->subordinate) { 5358 if (!pci_bus_trylock(dev->subordinate)) 5359 goto unlock; 5360 } else if (!pci_dev_trylock(dev)) 5361 goto unlock; 5362 } 5363 return 1; 5364 5365unlock: 5366 list_for_each_entry_continue_reverse(dev, 5367 &slot->bus->devices, bus_list) { 5368 if (!dev->slot || dev->slot != slot) 5369 continue; 5370 if (dev->subordinate) 5371 pci_bus_unlock(dev->subordinate); 5372 else 5373 pci_dev_unlock(dev); 5374 } 5375 5376 if (bridge) 5377 pci_dev_unlock(bridge); 5378 return 0; 5379} 5380 5381/* 5382 * Save and disable devices from the top of the tree down while holding 5383 * the @dev mutex lock for the entire tree. 5384 */ 5385static void pci_bus_save_and_disable_locked(struct pci_bus *bus) 5386{ 5387 struct pci_dev *dev; 5388 5389 list_for_each_entry(dev, &bus->devices, bus_list) { 5390 pci_dev_save_and_disable(dev); 5391 if (dev->subordinate) 5392 pci_bus_save_and_disable_locked(dev->subordinate); 5393 } 5394} 5395 5396/* 5397 * Restore devices from top of the tree down while holding @dev mutex lock 5398 * for the entire tree. Parent bridges need to be restored before we can 5399 * get to subordinate devices. 5400 */ 5401static void pci_bus_restore_locked(struct pci_bus *bus) 5402{ 5403 struct pci_dev *dev; 5404 5405 list_for_each_entry(dev, &bus->devices, bus_list) { 5406 pci_dev_restore(dev); 5407 if (dev->subordinate) { 5408 pci_bridge_wait_for_secondary_bus(dev, "bus reset"); 5409 pci_bus_restore_locked(dev->subordinate); 5410 } 5411 } 5412} 5413 5414/* 5415 * Save and disable devices from the top of the tree down while holding 5416 * the @dev mutex lock for the entire tree. 5417 */ 5418static void pci_slot_save_and_disable_locked(struct pci_slot *slot) 5419{ 5420 struct pci_dev *dev; 5421 5422 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5423 if (!dev->slot || dev->slot != slot) 5424 continue; 5425 pci_dev_save_and_disable(dev); 5426 if (dev->subordinate) 5427 pci_bus_save_and_disable_locked(dev->subordinate); 5428 } 5429} 5430 5431/* 5432 * Restore devices from top of the tree down while holding @dev mutex lock 5433 * for the entire tree. Parent bridges need to be restored before we can 5434 * get to subordinate devices. 5435 */ 5436static void pci_slot_restore_locked(struct pci_slot *slot) 5437{ 5438 struct pci_dev *dev; 5439 5440 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5441 if (!dev->slot || dev->slot != slot) 5442 continue; 5443 pci_dev_restore(dev); 5444 if (dev->subordinate) { 5445 pci_bridge_wait_for_secondary_bus(dev, "slot reset"); 5446 pci_bus_restore_locked(dev->subordinate); 5447 } 5448 } 5449} 5450 5451static int pci_slot_reset(struct pci_slot *slot, bool probe) 5452{ 5453 int rc; 5454 5455 if (!slot || !pci_slot_resettable(slot)) 5456 return -ENOTTY; 5457 5458 if (!probe) 5459 pci_slot_lock(slot); 5460 5461 might_sleep(); 5462 5463 rc = pci_reset_hotplug_slot(slot->hotplug, probe); 5464 5465 if (!probe) 5466 pci_slot_unlock(slot); 5467 5468 return rc; 5469} 5470 5471/** 5472 * pci_probe_reset_slot - probe whether a PCI slot can be reset 5473 * @slot: PCI slot to probe 5474 * 5475 * Return 0 if slot can be reset, negative if a slot reset is not supported. 5476 */ 5477int pci_probe_reset_slot(struct pci_slot *slot) 5478{ 5479 return pci_slot_reset(slot, PCI_RESET_PROBE); 5480} 5481EXPORT_SYMBOL_GPL(pci_probe_reset_slot); 5482 5483/** 5484 * __pci_reset_slot - Try to reset a PCI slot 5485 * @slot: PCI slot to reset 5486 * 5487 * A PCI bus may host multiple slots, each slot may support a reset mechanism 5488 * independent of other slots. For instance, some slots may support slot power 5489 * control. In the case of a 1:1 bus to slot architecture, this function may 5490 * wrap the bus reset to avoid spurious slot related events such as hotplug. 5491 * Generally a slot reset should be attempted before a bus reset. All of the 5492 * function of the slot and any subordinate buses behind the slot are reset 5493 * through this function. PCI config space of all devices in the slot and 5494 * behind the slot is saved before and restored after reset. 5495 * 5496 * Same as above except return -EAGAIN if the slot cannot be locked 5497 */ 5498static int __pci_reset_slot(struct pci_slot *slot) 5499{ 5500 int rc; 5501 5502 rc = pci_slot_reset(slot, PCI_RESET_PROBE); 5503 if (rc) 5504 return rc; 5505 5506 if (pci_slot_trylock(slot)) { 5507 pci_slot_save_and_disable_locked(slot); 5508 might_sleep(); 5509 rc = pci_reset_hotplug_slot(slot->hotplug, PCI_RESET_DO_RESET); 5510 pci_slot_restore_locked(slot); 5511 pci_slot_unlock(slot); 5512 } else 5513 rc = -EAGAIN; 5514 5515 return rc; 5516} 5517 5518static int pci_bus_reset(struct pci_bus *bus, bool probe) 5519{ 5520 int ret; 5521 5522 if (!bus->self || !pci_bus_resettable(bus)) 5523 return -ENOTTY; 5524 5525 if (probe) 5526 return 0; 5527 5528 pci_bus_lock(bus); 5529 5530 might_sleep(); 5531 5532 ret = pci_bridge_secondary_bus_reset(bus->self); 5533 5534 pci_bus_unlock(bus); 5535 5536 return ret; 5537} 5538 5539/** 5540 * pci_bus_error_reset - reset the bridge's subordinate bus 5541 * @bridge: The parent device that connects to the bus to reset 5542 * 5543 * This function will first try to reset the slots on this bus if the method is 5544 * available. If slot reset fails or is not available, this will fall back to a 5545 * secondary bus reset. 5546 */ 5547int pci_bus_error_reset(struct pci_dev *bridge) 5548{ 5549 struct pci_bus *bus = bridge->subordinate; 5550 struct pci_slot *slot; 5551 5552 if (!bus) 5553 return -ENOTTY; 5554 5555 mutex_lock(&pci_slot_mutex); 5556 if (list_empty(&bus->slots)) 5557 goto bus_reset; 5558 5559 list_for_each_entry(slot, &bus->slots, list) 5560 if (pci_probe_reset_slot(slot)) 5561 goto bus_reset; 5562 5563 list_for_each_entry(slot, &bus->slots, list) 5564 if (pci_slot_reset(slot, PCI_RESET_DO_RESET)) 5565 goto bus_reset; 5566 5567 mutex_unlock(&pci_slot_mutex); 5568 return 0; 5569bus_reset: 5570 mutex_unlock(&pci_slot_mutex); 5571 return pci_bus_reset(bridge->subordinate, PCI_RESET_DO_RESET); 5572} 5573 5574/** 5575 * pci_probe_reset_bus - probe whether a PCI bus can be reset 5576 * @bus: PCI bus to probe 5577 * 5578 * Return 0 if bus can be reset, negative if a bus reset is not supported. 5579 */ 5580int pci_probe_reset_bus(struct pci_bus *bus) 5581{ 5582 return pci_bus_reset(bus, PCI_RESET_PROBE); 5583} 5584EXPORT_SYMBOL_GPL(pci_probe_reset_bus); 5585 5586/** 5587 * __pci_reset_bus - Try to reset a PCI bus 5588 * @bus: top level PCI bus to reset 5589 * 5590 * Same as above except return -EAGAIN if the bus cannot be locked 5591 */ 5592int __pci_reset_bus(struct pci_bus *bus) 5593{ 5594 int rc; 5595 5596 rc = pci_bus_reset(bus, PCI_RESET_PROBE); 5597 if (rc) 5598 return rc; 5599 5600 if (pci_bus_trylock(bus)) { 5601 pci_bus_save_and_disable_locked(bus); 5602 might_sleep(); 5603 rc = pci_bridge_secondary_bus_reset(bus->self); 5604 pci_bus_restore_locked(bus); 5605 pci_bus_unlock(bus); 5606 } else 5607 rc = -EAGAIN; 5608 5609 return rc; 5610} 5611 5612/** 5613 * pci_reset_bus - Try to reset a PCI bus 5614 * @pdev: top level PCI device to reset via slot/bus 5615 * 5616 * Same as above except return -EAGAIN if the bus cannot be locked 5617 */ 5618int pci_reset_bus(struct pci_dev *pdev) 5619{ 5620 return (!pci_probe_reset_slot(pdev->slot)) ? 5621 __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus); 5622} 5623EXPORT_SYMBOL_GPL(pci_reset_bus); 5624 5625/** 5626 * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count 5627 * @dev: PCI device to query 5628 * 5629 * Returns mmrbc: maximum designed memory read count in bytes or 5630 * appropriate error value. 5631 */ 5632int pcix_get_max_mmrbc(struct pci_dev *dev) 5633{ 5634 int cap; 5635 u32 stat; 5636 5637 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5638 if (!cap) 5639 return -EINVAL; 5640 5641 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 5642 return -EINVAL; 5643 5644 return 512 << FIELD_GET(PCI_X_STATUS_MAX_READ, stat); 5645} 5646EXPORT_SYMBOL(pcix_get_max_mmrbc); 5647 5648/** 5649 * pcix_get_mmrbc - get PCI-X maximum memory read byte count 5650 * @dev: PCI device to query 5651 * 5652 * Returns mmrbc: maximum memory read count in bytes or appropriate error 5653 * value. 5654 */ 5655int pcix_get_mmrbc(struct pci_dev *dev) 5656{ 5657 int cap; 5658 u16 cmd; 5659 5660 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5661 if (!cap) 5662 return -EINVAL; 5663 5664 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 5665 return -EINVAL; 5666 5667 return 512 << FIELD_GET(PCI_X_CMD_MAX_READ, cmd); 5668} 5669EXPORT_SYMBOL(pcix_get_mmrbc); 5670 5671/** 5672 * pcix_set_mmrbc - set PCI-X maximum memory read byte count 5673 * @dev: PCI device to query 5674 * @mmrbc: maximum memory read count in bytes 5675 * valid values are 512, 1024, 2048, 4096 5676 * 5677 * If possible sets maximum memory read byte count, some bridges have errata 5678 * that prevent this. 5679 */ 5680int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc) 5681{ 5682 int cap; 5683 u32 stat, v, o; 5684 u16 cmd; 5685 5686 if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc)) 5687 return -EINVAL; 5688 5689 v = ffs(mmrbc) - 10; 5690 5691 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5692 if (!cap) 5693 return -EINVAL; 5694 5695 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 5696 return -EINVAL; 5697 5698 if (v > FIELD_GET(PCI_X_STATUS_MAX_READ, stat)) 5699 return -E2BIG; 5700 5701 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 5702 return -EINVAL; 5703 5704 o = FIELD_GET(PCI_X_CMD_MAX_READ, cmd); 5705 if (o != v) { 5706 if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC)) 5707 return -EIO; 5708 5709 cmd &= ~PCI_X_CMD_MAX_READ; 5710 cmd |= FIELD_PREP(PCI_X_CMD_MAX_READ, v); 5711 if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd)) 5712 return -EIO; 5713 } 5714 return 0; 5715} 5716EXPORT_SYMBOL(pcix_set_mmrbc); 5717 5718/** 5719 * pcie_get_readrq - get PCI Express read request size 5720 * @dev: PCI device to query 5721 * 5722 * Returns maximum memory read request in bytes or appropriate error value. 5723 */ 5724int pcie_get_readrq(struct pci_dev *dev) 5725{ 5726 u16 ctl; 5727 5728 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 5729 5730 return 128 << FIELD_GET(PCI_EXP_DEVCTL_READRQ, ctl); 5731} 5732EXPORT_SYMBOL(pcie_get_readrq); 5733 5734/** 5735 * pcie_set_readrq - set PCI Express maximum memory read request 5736 * @dev: PCI device to query 5737 * @rq: maximum memory read count in bytes 5738 * valid values are 128, 256, 512, 1024, 2048, 4096 5739 * 5740 * If possible sets maximum memory read request in bytes 5741 */ 5742int pcie_set_readrq(struct pci_dev *dev, int rq) 5743{ 5744 u16 v; 5745 int ret; 5746 unsigned int firstbit; 5747 struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus); 5748 5749 if (rq < 128 || rq > 4096 || !is_power_of_2(rq)) 5750 return -EINVAL; 5751 5752 /* 5753 * If using the "performance" PCIe config, we clamp the read rq 5754 * size to the max packet size to keep the host bridge from 5755 * generating requests larger than we can cope with. 5756 */ 5757 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) { 5758 int mps = pcie_get_mps(dev); 5759 5760 if (mps < rq) 5761 rq = mps; 5762 } 5763 5764 firstbit = ffs(rq); 5765 if (firstbit < 8) 5766 return -EINVAL; 5767 v = FIELD_PREP(PCI_EXP_DEVCTL_READRQ, firstbit - 8); 5768 5769 if (bridge->no_inc_mrrs) { 5770 int max_mrrs = pcie_get_readrq(dev); 5771 5772 if (rq > max_mrrs) { 5773 pci_info(dev, "can't set Max_Read_Request_Size to %d; max is %d\n", rq, max_mrrs); 5774 return -EINVAL; 5775 } 5776 } 5777 5778 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 5779 PCI_EXP_DEVCTL_READRQ, v); 5780 5781 return pcibios_err_to_errno(ret); 5782} 5783EXPORT_SYMBOL(pcie_set_readrq); 5784 5785/** 5786 * pcie_get_mps - get PCI Express maximum payload size 5787 * @dev: PCI device to query 5788 * 5789 * Returns maximum payload size in bytes 5790 */ 5791int pcie_get_mps(struct pci_dev *dev) 5792{ 5793 u16 ctl; 5794 5795 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 5796 5797 return 128 << FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, ctl); 5798} 5799EXPORT_SYMBOL(pcie_get_mps); 5800 5801/** 5802 * pcie_set_mps - set PCI Express maximum payload size 5803 * @dev: PCI device to query 5804 * @mps: maximum payload size in bytes 5805 * valid values are 128, 256, 512, 1024, 2048, 4096 5806 * 5807 * If possible sets maximum payload size 5808 */ 5809int pcie_set_mps(struct pci_dev *dev, int mps) 5810{ 5811 u16 v; 5812 int ret; 5813 5814 if (mps < 128 || mps > 4096 || !is_power_of_2(mps)) 5815 return -EINVAL; 5816 5817 v = ffs(mps) - 8; 5818 if (v > dev->pcie_mpss) 5819 return -EINVAL; 5820 v = FIELD_PREP(PCI_EXP_DEVCTL_PAYLOAD, v); 5821 5822 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 5823 PCI_EXP_DEVCTL_PAYLOAD, v); 5824 5825 return pcibios_err_to_errno(ret); 5826} 5827EXPORT_SYMBOL(pcie_set_mps); 5828 5829static enum pci_bus_speed to_pcie_link_speed(u16 lnksta) 5830{ 5831 return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)]; 5832} 5833 5834int pcie_link_speed_mbps(struct pci_dev *pdev) 5835{ 5836 u16 lnksta; 5837 int err; 5838 5839 err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); 5840 if (err) 5841 return err; 5842 5843 return pcie_dev_speed_mbps(to_pcie_link_speed(lnksta)); 5844} 5845EXPORT_SYMBOL(pcie_link_speed_mbps); 5846 5847/** 5848 * pcie_bandwidth_available - determine minimum link settings of a PCIe 5849 * device and its bandwidth limitation 5850 * @dev: PCI device to query 5851 * @limiting_dev: storage for device causing the bandwidth limitation 5852 * @speed: storage for speed of limiting device 5853 * @width: storage for width of limiting device 5854 * 5855 * Walk up the PCI device chain and find the point where the minimum 5856 * bandwidth is available. Return the bandwidth available there and (if 5857 * limiting_dev, speed, and width pointers are supplied) information about 5858 * that point. The bandwidth returned is in Mb/s, i.e., megabits/second of 5859 * raw bandwidth. 5860 */ 5861u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, 5862 enum pci_bus_speed *speed, 5863 enum pcie_link_width *width) 5864{ 5865 u16 lnksta; 5866 enum pci_bus_speed next_speed; 5867 enum pcie_link_width next_width; 5868 u32 bw, next_bw; 5869 5870 if (speed) 5871 *speed = PCI_SPEED_UNKNOWN; 5872 if (width) 5873 *width = PCIE_LNK_WIDTH_UNKNOWN; 5874 5875 bw = 0; 5876 5877 while (dev) { 5878 pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); 5879 5880 next_speed = to_pcie_link_speed(lnksta); 5881 next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); 5882 5883 next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed); 5884 5885 /* Check if current device limits the total bandwidth */ 5886 if (!bw || next_bw <= bw) { 5887 bw = next_bw; 5888 5889 if (limiting_dev) 5890 *limiting_dev = dev; 5891 if (speed) 5892 *speed = next_speed; 5893 if (width) 5894 *width = next_width; 5895 } 5896 5897 dev = pci_upstream_bridge(dev); 5898 } 5899 5900 return bw; 5901} 5902EXPORT_SYMBOL(pcie_bandwidth_available); 5903 5904/** 5905 * pcie_get_supported_speeds - query Supported Link Speed Vector 5906 * @dev: PCI device to query 5907 * 5908 * Query @dev supported link speeds. 5909 * 5910 * Implementation Note in PCIe r6.0 sec 7.5.3.18 recommends determining 5911 * supported link speeds using the Supported Link Speeds Vector in the Link 5912 * Capabilities 2 Register (when available). 5913 * 5914 * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. 5915 * 5916 * Without Link Capabilities 2, i.e., prior to PCIe r3.0, Supported Link 5917 * Speeds field in Link Capabilities is used and only 2.5 GT/s and 5.0 GT/s 5918 * speeds were defined. 5919 * 5920 * For @dev without Supported Link Speed Vector, the field is synthesized 5921 * from the Max Link Speed field in the Link Capabilities Register. 5922 * 5923 * Return: Supported Link Speeds Vector (+ reserved 0 at LSB). 5924 */ 5925u8 pcie_get_supported_speeds(struct pci_dev *dev) 5926{ 5927 u32 lnkcap2, lnkcap; 5928 u8 speeds; 5929 5930 /* 5931 * Speeds retain the reserved 0 at LSB before PCIe Supported Link 5932 * Speeds Vector to allow using SLS Vector bit defines directly. 5933 */ 5934 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2); 5935 speeds = lnkcap2 & PCI_EXP_LNKCAP2_SLS; 5936 5937 /* Ignore speeds higher than Max Link Speed */ 5938 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 5939 speeds &= GENMASK(lnkcap & PCI_EXP_LNKCAP_SLS, 0); 5940 5941 /* PCIe r3.0-compliant */ 5942 if (speeds) 5943 return speeds; 5944 5945 /* Synthesize from the Max Link Speed field */ 5946 if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB) 5947 speeds = PCI_EXP_LNKCAP2_SLS_5_0GB | PCI_EXP_LNKCAP2_SLS_2_5GB; 5948 else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB) 5949 speeds = PCI_EXP_LNKCAP2_SLS_2_5GB; 5950 5951 return speeds; 5952} 5953 5954/** 5955 * pcie_get_speed_cap - query for the PCI device's link speed capability 5956 * @dev: PCI device to query 5957 * 5958 * Query the PCI device speed capability. 5959 * 5960 * Return: the maximum link speed supported by the device. 5961 */ 5962enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev) 5963{ 5964 return PCIE_LNKCAP2_SLS2SPEED(dev->supported_speeds); 5965} 5966EXPORT_SYMBOL(pcie_get_speed_cap); 5967 5968/** 5969 * pcie_get_width_cap - query for the PCI device's link width capability 5970 * @dev: PCI device to query 5971 * 5972 * Query the PCI device width capability. Return the maximum link width 5973 * supported by the device. 5974 */ 5975enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev) 5976{ 5977 u32 lnkcap; 5978 5979 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 5980 if (lnkcap) 5981 return FIELD_GET(PCI_EXP_LNKCAP_MLW, lnkcap); 5982 5983 return PCIE_LNK_WIDTH_UNKNOWN; 5984} 5985EXPORT_SYMBOL(pcie_get_width_cap); 5986 5987/** 5988 * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability 5989 * @dev: PCI device 5990 * @speed: storage for link speed 5991 * @width: storage for link width 5992 * 5993 * Calculate a PCI device's link bandwidth by querying for its link speed 5994 * and width, multiplying them, and applying encoding overhead. The result 5995 * is in Mb/s, i.e., megabits/second of raw bandwidth. 5996 */ 5997static u32 pcie_bandwidth_capable(struct pci_dev *dev, 5998 enum pci_bus_speed *speed, 5999 enum pcie_link_width *width) 6000{ 6001 *speed = pcie_get_speed_cap(dev); 6002 *width = pcie_get_width_cap(dev); 6003 6004 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 6005 return 0; 6006 6007 return *width * PCIE_SPEED2MBS_ENC(*speed); 6008} 6009 6010/** 6011 * __pcie_print_link_status - Report the PCI device's link speed and width 6012 * @dev: PCI device to query 6013 * @verbose: Print info even when enough bandwidth is available 6014 * 6015 * If the available bandwidth at the device is less than the device is 6016 * capable of, report the device's maximum possible bandwidth and the 6017 * upstream link that limits its performance. If @verbose, always print 6018 * the available bandwidth, even if the device isn't constrained. 6019 */ 6020void __pcie_print_link_status(struct pci_dev *dev, bool verbose) 6021{ 6022 enum pcie_link_width width, width_cap; 6023 enum pci_bus_speed speed, speed_cap; 6024 struct pci_dev *limiting_dev = NULL; 6025 u32 bw_avail, bw_cap; 6026 char *flit_mode = ""; 6027 6028 bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap); 6029 bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width); 6030 6031 if (dev->bus && dev->bus->flit_mode) 6032 flit_mode = ", in Flit mode"; 6033 6034 if (bw_avail >= bw_cap && verbose) 6035 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)%s\n", 6036 bw_cap / 1000, bw_cap % 1000, 6037 pci_speed_string(speed_cap), width_cap, flit_mode); 6038 else if (bw_avail < bw_cap) 6039 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)%s\n", 6040 bw_avail / 1000, bw_avail % 1000, 6041 pci_speed_string(speed), width, 6042 limiting_dev ? pci_name(limiting_dev) : "<unknown>", 6043 bw_cap / 1000, bw_cap % 1000, 6044 pci_speed_string(speed_cap), width_cap, flit_mode); 6045} 6046 6047/** 6048 * pcie_print_link_status - Report the PCI device's link speed and width 6049 * @dev: PCI device to query 6050 * 6051 * Report the available bandwidth at the device. 6052 */ 6053void pcie_print_link_status(struct pci_dev *dev) 6054{ 6055 __pcie_print_link_status(dev, true); 6056} 6057EXPORT_SYMBOL(pcie_print_link_status); 6058 6059/** 6060 * pci_select_bars - Make BAR mask from the type of resource 6061 * @dev: the PCI device for which BAR mask is made 6062 * @flags: resource type mask to be selected 6063 * 6064 * This helper routine makes bar mask from the type of resource. 6065 */ 6066int pci_select_bars(struct pci_dev *dev, unsigned long flags) 6067{ 6068 int i, bars = 0; 6069 for (i = 0; i < PCI_NUM_RESOURCES; i++) 6070 if (pci_resource_flags(dev, i) & flags) 6071 bars |= (1 << i); 6072 return bars; 6073} 6074EXPORT_SYMBOL(pci_select_bars); 6075 6076/* Some architectures require additional programming to enable VGA */ 6077static arch_set_vga_state_t arch_set_vga_state; 6078 6079void __init pci_register_set_vga_state(arch_set_vga_state_t func) 6080{ 6081 arch_set_vga_state = func; /* NULL disables */ 6082} 6083 6084static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode, 6085 unsigned int command_bits, u32 flags) 6086{ 6087 if (arch_set_vga_state) 6088 return arch_set_vga_state(dev, decode, command_bits, 6089 flags); 6090 return 0; 6091} 6092 6093/** 6094 * pci_set_vga_state - set VGA decode state on device and parents if requested 6095 * @dev: the PCI device 6096 * @decode: true = enable decoding, false = disable decoding 6097 * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY 6098 * @flags: traverse ancestors and change bridges 6099 * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE 6100 */ 6101int pci_set_vga_state(struct pci_dev *dev, bool decode, 6102 unsigned int command_bits, u32 flags) 6103{ 6104 struct pci_bus *bus; 6105 struct pci_dev *bridge; 6106 u16 cmd; 6107 int rc; 6108 6109 WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY))); 6110 6111 /* ARCH specific VGA enables */ 6112 rc = pci_set_vga_state_arch(dev, decode, command_bits, flags); 6113 if (rc) 6114 return rc; 6115 6116 if (flags & PCI_VGA_STATE_CHANGE_DECODES) { 6117 pci_read_config_word(dev, PCI_COMMAND, &cmd); 6118 if (decode) 6119 cmd |= command_bits; 6120 else 6121 cmd &= ~command_bits; 6122 pci_write_config_word(dev, PCI_COMMAND, cmd); 6123 } 6124 6125 if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE)) 6126 return 0; 6127 6128 bus = dev->bus; 6129 while (bus) { 6130 bridge = bus->self; 6131 if (bridge) { 6132 pci_read_config_word(bridge, PCI_BRIDGE_CONTROL, 6133 &cmd); 6134 if (decode) 6135 cmd |= PCI_BRIDGE_CTL_VGA; 6136 else 6137 cmd &= ~PCI_BRIDGE_CTL_VGA; 6138 pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, 6139 cmd); 6140 } 6141 bus = bus->parent; 6142 } 6143 return 0; 6144} 6145 6146#ifdef CONFIG_ACPI 6147bool pci_pr3_present(struct pci_dev *pdev) 6148{ 6149 struct acpi_device *adev; 6150 6151 if (acpi_disabled) 6152 return false; 6153 6154 adev = ACPI_COMPANION(&pdev->dev); 6155 if (!adev) 6156 return false; 6157 6158 return adev->power.flags.power_resources && 6159 acpi_has_method(adev->handle, "_PR3"); 6160} 6161EXPORT_SYMBOL_GPL(pci_pr3_present); 6162#endif 6163 6164/** 6165 * pci_add_dma_alias - Add a DMA devfn alias for a device 6166 * @dev: the PCI device for which alias is added 6167 * @devfn_from: alias slot and function 6168 * @nr_devfns: number of subsequent devfns to alias 6169 * 6170 * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask 6171 * which is used to program permissible bus-devfn source addresses for DMA 6172 * requests in an IOMMU. These aliases factor into IOMMU group creation 6173 * and are useful for devices generating DMA requests beyond or different 6174 * from their logical bus-devfn. Examples include device quirks where the 6175 * device simply uses the wrong devfn, as well as non-transparent bridges 6176 * where the alias may be a proxy for devices in another domain. 6177 * 6178 * IOMMU group creation is performed during device discovery or addition, 6179 * prior to any potential DMA mapping and therefore prior to driver probing 6180 * (especially for userspace assigned devices where IOMMU group definition 6181 * cannot be left as a userspace activity). DMA aliases should therefore 6182 * be configured via quirks, such as the PCI fixup header quirk. 6183 */ 6184void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, 6185 unsigned int nr_devfns) 6186{ 6187 int devfn_to; 6188 6189 nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from); 6190 devfn_to = devfn_from + nr_devfns - 1; 6191 6192 if (!dev->dma_alias_mask) 6193 dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL); 6194 if (!dev->dma_alias_mask) { 6195 pci_warn(dev, "Unable to allocate DMA alias mask\n"); 6196 return; 6197 } 6198 6199 bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns); 6200 6201 if (nr_devfns == 1) 6202 pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n", 6203 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from)); 6204 else if (nr_devfns > 1) 6205 pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n", 6206 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from), 6207 PCI_SLOT(devfn_to), PCI_FUNC(devfn_to)); 6208} 6209 6210bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) 6211{ 6212 return (dev1->dma_alias_mask && 6213 test_bit(dev2->devfn, dev1->dma_alias_mask)) || 6214 (dev2->dma_alias_mask && 6215 test_bit(dev1->devfn, dev2->dma_alias_mask)) || 6216 pci_real_dma_dev(dev1) == dev2 || 6217 pci_real_dma_dev(dev2) == dev1; 6218} 6219 6220bool pci_device_is_present(struct pci_dev *pdev) 6221{ 6222 u32 v; 6223 6224 /* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */ 6225 pdev = pci_physfn(pdev); 6226 if (pci_dev_is_disconnected(pdev)) 6227 return false; 6228 return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0); 6229} 6230EXPORT_SYMBOL_GPL(pci_device_is_present); 6231 6232void pci_ignore_hotplug(struct pci_dev *dev) 6233{ 6234 struct pci_dev *bridge = dev->bus->self; 6235 6236 dev->ignore_hotplug = 1; 6237 /* Propagate the "ignore hotplug" setting to the parent bridge. */ 6238 if (bridge) 6239 bridge->ignore_hotplug = 1; 6240} 6241EXPORT_SYMBOL_GPL(pci_ignore_hotplug); 6242 6243/** 6244 * pci_real_dma_dev - Get PCI DMA device for PCI device 6245 * @dev: the PCI device that may have a PCI DMA alias 6246 * 6247 * Permits the platform to provide architecture-specific functionality to 6248 * devices needing to alias DMA to another PCI device on another PCI bus. If 6249 * the PCI device is on the same bus, it is recommended to use 6250 * pci_add_dma_alias(). This is the default implementation. Architecture 6251 * implementations can override this. 6252 */ 6253struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev) 6254{ 6255 return dev; 6256} 6257 6258resource_size_t __weak pcibios_default_alignment(void) 6259{ 6260 return 0; 6261} 6262 6263/* 6264 * Arches that don't want to expose struct resource to userland as-is in 6265 * sysfs and /proc can implement their own pci_resource_to_user(). 6266 */ 6267void __weak pci_resource_to_user(const struct pci_dev *dev, int bar, 6268 const struct resource *rsrc, 6269 resource_size_t *start, resource_size_t *end) 6270{ 6271 *start = rsrc->start; 6272 *end = rsrc->end; 6273} 6274 6275static char *resource_alignment_param; 6276static DEFINE_SPINLOCK(resource_alignment_lock); 6277 6278/** 6279 * pci_specified_resource_alignment - get resource alignment specified by user. 6280 * @dev: the PCI device to get 6281 * @resize: whether or not to change resources' size when reassigning alignment 6282 * 6283 * RETURNS: Resource alignment if it is specified. 6284 * Zero if it is not specified. 6285 */ 6286static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev, 6287 bool *resize) 6288{ 6289 int align_order, count; 6290 resource_size_t align = pcibios_default_alignment(); 6291 const char *p; 6292 int ret; 6293 6294 spin_lock(&resource_alignment_lock); 6295 p = resource_alignment_param; 6296 if (!p || !*p) 6297 goto out; 6298 if (pci_has_flag(PCI_PROBE_ONLY)) { 6299 align = 0; 6300 pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n"); 6301 goto out; 6302 } 6303 6304 while (*p) { 6305 count = 0; 6306 if (sscanf(p, "%d%n", &align_order, &count) == 1 && 6307 p[count] == '@') { 6308 p += count + 1; 6309 if (align_order > 63) { 6310 pr_err("PCI: Invalid requested alignment (order %d)\n", 6311 align_order); 6312 align_order = PAGE_SHIFT; 6313 } 6314 } else { 6315 align_order = PAGE_SHIFT; 6316 } 6317 6318 ret = pci_dev_str_match(dev, p, &p); 6319 if (ret == 1) { 6320 *resize = true; 6321 align = 1ULL << align_order; 6322 break; 6323 } else if (ret < 0) { 6324 pr_err("PCI: Can't parse resource_alignment parameter: %s\n", 6325 p); 6326 break; 6327 } 6328 6329 if (*p != ';' && *p != ',') { 6330 /* End of param or invalid format */ 6331 break; 6332 } 6333 p++; 6334 } 6335out: 6336 spin_unlock(&resource_alignment_lock); 6337 return align; 6338} 6339 6340static void pci_request_resource_alignment(struct pci_dev *dev, int bar, 6341 resource_size_t align, bool resize) 6342{ 6343 struct resource *r = &dev->resource[bar]; 6344 const char *r_name = pci_resource_name(dev, bar); 6345 resource_size_t size; 6346 6347 if (!(r->flags & IORESOURCE_MEM)) 6348 return; 6349 6350 if (r->flags & IORESOURCE_PCI_FIXED) { 6351 pci_info(dev, "%s %pR: ignoring requested alignment %#llx\n", 6352 r_name, r, (unsigned long long)align); 6353 return; 6354 } 6355 6356 size = resource_size(r); 6357 if (size >= align) 6358 return; 6359 6360 /* 6361 * Increase the alignment of the resource. There are two ways we 6362 * can do this: 6363 * 6364 * 1) Increase the size of the resource. BARs are aligned on their 6365 * size, so when we reallocate space for this resource, we'll 6366 * allocate it with the larger alignment. This also prevents 6367 * assignment of any other BARs inside the alignment region, so 6368 * if we're requesting page alignment, this means no other BARs 6369 * will share the page. 6370 * 6371 * The disadvantage is that this makes the resource larger than 6372 * the hardware BAR, which may break drivers that compute things 6373 * based on the resource size, e.g., to find registers at a 6374 * fixed offset before the end of the BAR. 6375 * 6376 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and 6377 * set r->start to the desired alignment. By itself this 6378 * doesn't prevent other BARs being put inside the alignment 6379 * region, but if we realign *every* resource of every device in 6380 * the system, none of them will share an alignment region. 6381 * 6382 * When the user has requested alignment for only some devices via 6383 * the "pci=resource_alignment" argument, "resize" is true and we 6384 * use the first method. Otherwise we assume we're aligning all 6385 * devices and we use the second. 6386 */ 6387 6388 pci_info(dev, "%s %pR: requesting alignment to %#llx\n", 6389 r_name, r, (unsigned long long)align); 6390 6391 if (resize) { 6392 r->start = 0; 6393 r->end = align - 1; 6394 } else { 6395 r->flags &= ~IORESOURCE_SIZEALIGN; 6396 r->flags |= IORESOURCE_STARTALIGN; 6397 resource_set_range(r, align, size); 6398 } 6399 r->flags |= IORESOURCE_UNSET; 6400} 6401 6402/* 6403 * This function disables memory decoding and releases memory resources 6404 * of the device specified by kernel's boot parameter 'pci=resource_alignment='. 6405 * It also rounds up size to specified alignment. 6406 * Later on, the kernel will assign page-aligned memory resource back 6407 * to the device. 6408 */ 6409void pci_reassigndev_resource_alignment(struct pci_dev *dev) 6410{ 6411 int i; 6412 struct resource *r; 6413 resource_size_t align; 6414 u16 command; 6415 bool resize = false; 6416 6417 /* 6418 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec 6419 * 3.4.1.11. Their resources are allocated from the space 6420 * described by the VF BARx register in the PF's SR-IOV capability. 6421 * We can't influence their alignment here. 6422 */ 6423 if (dev->is_virtfn) 6424 return; 6425 6426 /* check if specified PCI is target device to reassign */ 6427 align = pci_specified_resource_alignment(dev, &resize); 6428 if (!align) 6429 return; 6430 6431 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL && 6432 (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) { 6433 pci_warn(dev, "Can't reassign resources to host bridge\n"); 6434 return; 6435 } 6436 6437 pci_read_config_word(dev, PCI_COMMAND, &command); 6438 command &= ~PCI_COMMAND_MEMORY; 6439 pci_write_config_word(dev, PCI_COMMAND, command); 6440 6441 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 6442 pci_request_resource_alignment(dev, i, align, resize); 6443 6444 /* 6445 * Need to disable bridge's resource window, 6446 * to enable the kernel to reassign new resource 6447 * window later on. 6448 */ 6449 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 6450 for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) { 6451 r = &dev->resource[i]; 6452 if (!(r->flags & IORESOURCE_MEM)) 6453 continue; 6454 r->flags |= IORESOURCE_UNSET; 6455 r->end = resource_size(r) - 1; 6456 r->start = 0; 6457 } 6458 pci_disable_bridge_window(dev); 6459 } 6460} 6461 6462static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf) 6463{ 6464 size_t count = 0; 6465 6466 spin_lock(&resource_alignment_lock); 6467 if (resource_alignment_param) 6468 count = sysfs_emit(buf, "%s\n", resource_alignment_param); 6469 spin_unlock(&resource_alignment_lock); 6470 6471 return count; 6472} 6473 6474static ssize_t resource_alignment_store(const struct bus_type *bus, 6475 const char *buf, size_t count) 6476{ 6477 char *param, *old, *end; 6478 6479 if (count >= (PAGE_SIZE - 1)) 6480 return -EINVAL; 6481 6482 param = kstrndup(buf, count, GFP_KERNEL); 6483 if (!param) 6484 return -ENOMEM; 6485 6486 end = strchr(param, '\n'); 6487 if (end) 6488 *end = '\0'; 6489 6490 spin_lock(&resource_alignment_lock); 6491 old = resource_alignment_param; 6492 if (strlen(param)) { 6493 resource_alignment_param = param; 6494 } else { 6495 kfree(param); 6496 resource_alignment_param = NULL; 6497 } 6498 spin_unlock(&resource_alignment_lock); 6499 6500 kfree(old); 6501 6502 return count; 6503} 6504 6505static BUS_ATTR_RW(resource_alignment); 6506 6507static int __init pci_resource_alignment_sysfs_init(void) 6508{ 6509 return bus_create_file(&pci_bus_type, 6510 &bus_attr_resource_alignment); 6511} 6512late_initcall(pci_resource_alignment_sysfs_init); 6513 6514static void pci_no_domains(void) 6515{ 6516#ifdef CONFIG_PCI_DOMAINS 6517 pci_domains_supported = 0; 6518#endif 6519} 6520 6521#ifdef CONFIG_PCI_DOMAINS 6522static DEFINE_IDA(pci_domain_nr_dynamic_ida); 6523 6524/** 6525 * pci_bus_find_emul_domain_nr() - allocate a PCI domain number per constraints 6526 * @hint: desired domain, 0 if any ID in the range of @min to @max is acceptable 6527 * @min: minimum allowable domain 6528 * @max: maximum allowable domain, no IDs higher than INT_MAX will be returned 6529 */ 6530int pci_bus_find_emul_domain_nr(u32 hint, u32 min, u32 max) 6531{ 6532 return ida_alloc_range(&pci_domain_nr_dynamic_ida, max(hint, min), max, 6533 GFP_KERNEL); 6534} 6535EXPORT_SYMBOL_GPL(pci_bus_find_emul_domain_nr); 6536 6537void pci_bus_release_emul_domain_nr(int domain_nr) 6538{ 6539 ida_free(&pci_domain_nr_dynamic_ida, domain_nr); 6540} 6541EXPORT_SYMBOL_GPL(pci_bus_release_emul_domain_nr); 6542#endif 6543 6544#ifdef CONFIG_PCI_DOMAINS_GENERIC 6545static DEFINE_IDA(pci_domain_nr_static_ida); 6546 6547static void of_pci_reserve_static_domain_nr(void) 6548{ 6549 struct device_node *np; 6550 int domain_nr; 6551 6552 for_each_node_by_type(np, "pci") { 6553 domain_nr = of_get_pci_domain_nr(np); 6554 if (domain_nr < 0) 6555 continue; 6556 /* 6557 * Permanently allocate domain_nr in dynamic_ida 6558 * to prevent it from dynamic allocation. 6559 */ 6560 ida_alloc_range(&pci_domain_nr_dynamic_ida, 6561 domain_nr, domain_nr, GFP_KERNEL); 6562 } 6563} 6564 6565static int of_pci_bus_find_domain_nr(struct device *parent) 6566{ 6567 static bool static_domains_reserved = false; 6568 int domain_nr; 6569 6570 /* On the first call scan device tree for static allocations. */ 6571 if (!static_domains_reserved) { 6572 of_pci_reserve_static_domain_nr(); 6573 static_domains_reserved = true; 6574 } 6575 6576 if (parent) { 6577 /* 6578 * If domain is in DT, allocate it in static IDA. This 6579 * prevents duplicate static allocations in case of errors 6580 * in DT. 6581 */ 6582 domain_nr = of_get_pci_domain_nr(parent->of_node); 6583 if (domain_nr >= 0) 6584 return ida_alloc_range(&pci_domain_nr_static_ida, 6585 domain_nr, domain_nr, 6586 GFP_KERNEL); 6587 } 6588 6589 /* 6590 * If domain was not specified in DT, choose a free ID from dynamic 6591 * allocations. All domain numbers from DT are permanently in 6592 * dynamic allocations to prevent assigning them to other DT nodes 6593 * without static domain. 6594 */ 6595 return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL); 6596} 6597 6598static void of_pci_bus_release_domain_nr(struct device *parent, int domain_nr) 6599{ 6600 if (domain_nr < 0) 6601 return; 6602 6603 /* Release domain from IDA where it was allocated. */ 6604 if (parent && of_get_pci_domain_nr(parent->of_node) == domain_nr) 6605 ida_free(&pci_domain_nr_static_ida, domain_nr); 6606 else 6607 ida_free(&pci_domain_nr_dynamic_ida, domain_nr); 6608} 6609 6610int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent) 6611{ 6612 return acpi_disabled ? of_pci_bus_find_domain_nr(parent) : 6613 acpi_pci_bus_find_domain_nr(bus); 6614} 6615 6616void pci_bus_release_domain_nr(struct device *parent, int domain_nr) 6617{ 6618 if (!acpi_disabled) 6619 return; 6620 of_pci_bus_release_domain_nr(parent, domain_nr); 6621} 6622#endif 6623 6624/** 6625 * pci_ext_cfg_avail - can we access extended PCI config space? 6626 * 6627 * Returns 1 if we can access PCI extended config space (offsets 6628 * greater than 0xff). This is the default implementation. Architecture 6629 * implementations can override this. 6630 */ 6631int __weak pci_ext_cfg_avail(void) 6632{ 6633 return 1; 6634} 6635 6636static int __init pci_setup(char *str) 6637{ 6638 while (str) { 6639 char *k = strchr(str, ','); 6640 if (k) 6641 *k++ = 0; 6642 if (*str && (str = pcibios_setup(str)) && *str) { 6643 if (!strcmp(str, "nomsi")) { 6644 pci_no_msi(); 6645 } else if (!strncmp(str, "noats", 5)) { 6646 pr_info("PCIe: ATS is disabled\n"); 6647 pcie_ats_disabled = true; 6648 } else if (!strcmp(str, "noaer")) { 6649 pci_no_aer(); 6650 } else if (!strcmp(str, "earlydump")) { 6651 pci_early_dump = true; 6652 } else if (!strncmp(str, "realloc=", 8)) { 6653 pci_realloc_get_opt(str + 8); 6654 } else if (!strncmp(str, "realloc", 7)) { 6655 pci_realloc_get_opt("on"); 6656 } else if (!strcmp(str, "nodomains")) { 6657 pci_no_domains(); 6658 } else if (!strncmp(str, "noari", 5)) { 6659 pcie_ari_disabled = true; 6660 } else if (!strncmp(str, "notph", 5)) { 6661 pci_no_tph(); 6662 } else if (!strncmp(str, "cbiosize=", 9)) { 6663 pci_cardbus_io_size = memparse(str + 9, &str); 6664 } else if (!strncmp(str, "cbmemsize=", 10)) { 6665 pci_cardbus_mem_size = memparse(str + 10, &str); 6666 } else if (!strncmp(str, "resource_alignment=", 19)) { 6667 resource_alignment_param = str + 19; 6668 } else if (!strncmp(str, "ecrc=", 5)) { 6669 pcie_ecrc_get_policy(str + 5); 6670 } else if (!strncmp(str, "hpiosize=", 9)) { 6671 pci_hotplug_io_size = memparse(str + 9, &str); 6672 } else if (!strncmp(str, "hpmmiosize=", 11)) { 6673 pci_hotplug_mmio_size = memparse(str + 11, &str); 6674 } else if (!strncmp(str, "hpmmioprefsize=", 15)) { 6675 pci_hotplug_mmio_pref_size = memparse(str + 15, &str); 6676 } else if (!strncmp(str, "hpmemsize=", 10)) { 6677 pci_hotplug_mmio_size = memparse(str + 10, &str); 6678 pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size; 6679 } else if (!strncmp(str, "hpbussize=", 10)) { 6680 pci_hotplug_bus_size = 6681 simple_strtoul(str + 10, &str, 0); 6682 if (pci_hotplug_bus_size > 0xff) 6683 pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 6684 } else if (!strncmp(str, "pcie_bus_tune_off", 17)) { 6685 pcie_bus_config = PCIE_BUS_TUNE_OFF; 6686 } else if (!strncmp(str, "pcie_bus_safe", 13)) { 6687 pcie_bus_config = PCIE_BUS_SAFE; 6688 } else if (!strncmp(str, "pcie_bus_perf", 13)) { 6689 pcie_bus_config = PCIE_BUS_PERFORMANCE; 6690 } else if (!strncmp(str, "pcie_bus_peer2peer", 18)) { 6691 pcie_bus_config = PCIE_BUS_PEER2PEER; 6692 } else if (!strncmp(str, "pcie_scan_all", 13)) { 6693 pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS); 6694 } else if (!strncmp(str, "disable_acs_redir=", 18)) { 6695 disable_acs_redir_param = str + 18; 6696 } else if (!strncmp(str, "config_acs=", 11)) { 6697 config_acs_param = str + 11; 6698 } else { 6699 pr_err("PCI: Unknown option `%s'\n", str); 6700 } 6701 } 6702 str = k; 6703 } 6704 return 0; 6705} 6706early_param("pci", pci_setup); 6707 6708/* 6709 * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized 6710 * in pci_setup(), above, to point to data in the __initdata section which 6711 * will be freed after the init sequence is complete. We can't allocate memory 6712 * in pci_setup() because some architectures do not have any memory allocation 6713 * service available during an early_param() call. So we allocate memory and 6714 * copy the variable here before the init section is freed. 6715 * 6716 */ 6717static int __init pci_realloc_setup_params(void) 6718{ 6719 resource_alignment_param = kstrdup(resource_alignment_param, 6720 GFP_KERNEL); 6721 disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL); 6722 config_acs_param = kstrdup(config_acs_param, GFP_KERNEL); 6723 6724 return 0; 6725} 6726pure_initcall(pci_realloc_setup_params);
~/
Read-only file preview.