portdrv.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Purpose:	PCI Express Port Bus Driver
  4 *
  5 * Copyright (C) 2004 Intel
  6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
  7 */
  8
  9#include <linux/bitfield.h>
 10#include <linux/dmi.h>
 11#include <linux/init.h>
 12#include <linux/module.h>
 13#include <linux/pci.h>
 14#include <linux/kernel.h>
 15#include <linux/errno.h>
 16#include <linux/pm.h>
 17#include <linux/pm_runtime.h>
 18#include <linux/string.h>
 19#include <linux/slab.h>
 20#include <linux/aer.h>
 21
 22#include "../pci.h"
 23#include "portdrv.h"
 24
 25/*
 26 * The PCIe Capability Interrupt Message Number (PCIe r3.1, sec 7.8.2) must
 27 * be one of the first 32 MSI-X entries.  Per PCI r3.0, sec 6.8.3.1, MSI
 28 * supports a maximum of 32 vectors per function.
 29 */
 30#define PCIE_PORT_MAX_MSI_ENTRIES	32
 31
 32#define get_descriptor_id(type, service) (((type - 4) << 8) | service)
 33
 34struct portdrv_service_data {
 35	struct pcie_port_service_driver *drv;
 36	struct device *dev;
 37	u32 service;
 38};
 39
 40/**
 41 * release_pcie_device - free PCI Express port service device structure
 42 * @dev: Port service device to release
 43 *
 44 * Invoked automatically when device is being removed in response to
 45 * device_unregister(dev).  Release all resources being claimed.
 46 */
 47static void release_pcie_device(struct device *dev)
 48{
 49	kfree(to_pcie_device(dev));
 50}
 51
 52/*
 53 * Fill in *pme, *aer, *dpc with the relevant Interrupt Message Numbers if
 54 * services are enabled in "mask".  Return the number of MSI/MSI-X vectors
 55 * required to accommodate the largest Message Number.
 56 */
 57static int pcie_message_numbers(struct pci_dev *dev, int mask,
 58				u32 *pme, u32 *aer, u32 *dpc)
 59{
 60	u32 nvec = 0, pos;
 61	u16 reg16;
 62
 63	/*
 64	 * The Interrupt Message Number indicates which vector is used, i.e.,
 65	 * the MSI-X table entry or the MSI offset between the base Message
 66	 * Data and the generated interrupt message.  See PCIe r3.1, sec
 67	 * 7.8.2, 7.10.10, 7.31.2.
 68	 */
 69
 70	if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
 71		    PCIE_PORT_SERVICE_BWCTRL)) {
 72		pcie_capability_read_word(dev, PCI_EXP_FLAGS, &reg16);
 73		*pme = FIELD_GET(PCI_EXP_FLAGS_IRQ, reg16);
 74		nvec = *pme + 1;
 75	}
 76
 77#ifdef CONFIG_PCIEAER
 78	if (mask & PCIE_PORT_SERVICE_AER) {
 79		u32 reg32;
 80
 81		pos = dev->aer_cap;
 82		if (pos) {
 83			pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS,
 84					      &reg32);
 85			*aer = FIELD_GET(PCI_ERR_ROOT_AER_IRQ, reg32);
 86			nvec = max(nvec, *aer + 1);
 87		}
 88	}
 89#endif
 90
 91	if (mask & PCIE_PORT_SERVICE_DPC) {
 92		pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC);
 93		if (pos) {
 94			pci_read_config_word(dev, pos + PCI_EXP_DPC_CAP,
 95					     &reg16);
 96			*dpc = FIELD_GET(PCI_EXP_DPC_IRQ, reg16);
 97			nvec = max(nvec, *dpc + 1);
 98		}
 99	}
100
101	return nvec;
102}
103
104/**
105 * pcie_port_enable_irq_vec - try to set up MSI-X or MSI as interrupt mode
106 * for given port
107 * @dev: PCI Express port to handle
108 * @irqs: Array of interrupt vectors to populate
109 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
110 *
111 * Return value: 0 on success, error code on failure
112 */
113static int pcie_port_enable_irq_vec(struct pci_dev *dev, int *irqs, int mask)
114{
115	int nr_entries, nvec, pcie_irq;
116	u32 pme = 0, aer = 0, dpc = 0;
117
118	/* Allocate the maximum possible number of MSI/MSI-X vectors */
119	nr_entries = pci_alloc_irq_vectors(dev, 1, PCIE_PORT_MAX_MSI_ENTRIES,
120			PCI_IRQ_MSIX | PCI_IRQ_MSI);
121	if (nr_entries < 0)
122		return nr_entries;
123
124	/* See how many and which Interrupt Message Numbers we actually use */
125	nvec = pcie_message_numbers(dev, mask, &pme, &aer, &dpc);
126	if (nvec > nr_entries) {
127		pci_free_irq_vectors(dev);
128		return -EIO;
129	}
130
131	/*
132	 * If we allocated more than we need, free them and reallocate fewer.
133	 *
134	 * Reallocating may change the specific vectors we get, so
135	 * pci_irq_vector() must be done *after* the reallocation.
136	 *
137	 * If we're using MSI, hardware is *allowed* to change the Interrupt
138	 * Message Numbers when we free and reallocate the vectors, but we
139	 * assume it won't because we allocate enough vectors for the
140	 * biggest Message Number we found.
141	 */
142	if (nvec != nr_entries) {
143		pci_free_irq_vectors(dev);
144
145		nr_entries = pci_alloc_irq_vectors(dev, nvec, nvec,
146				PCI_IRQ_MSIX | PCI_IRQ_MSI);
147		if (nr_entries < 0)
148			return nr_entries;
149	}
150
151	/* PME, hotplug and bandwidth notification share an MSI/MSI-X vector */
152	if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
153		    PCIE_PORT_SERVICE_BWCTRL)) {
154		pcie_irq = pci_irq_vector(dev, pme);
155		irqs[PCIE_PORT_SERVICE_PME_SHIFT] = pcie_irq;
156		irqs[PCIE_PORT_SERVICE_HP_SHIFT] = pcie_irq;
157		irqs[PCIE_PORT_SERVICE_BWCTRL_SHIFT] = pcie_irq;
158	}
159
160	if (mask & PCIE_PORT_SERVICE_AER)
161		irqs[PCIE_PORT_SERVICE_AER_SHIFT] = pci_irq_vector(dev, aer);
162
163	if (mask & PCIE_PORT_SERVICE_DPC)
164		irqs[PCIE_PORT_SERVICE_DPC_SHIFT] = pci_irq_vector(dev, dpc);
165
166	return 0;
167}
168
169/**
170 * pcie_init_service_irqs - initialize irqs for PCI Express port services
171 * @dev: PCI Express port to handle
172 * @irqs: Array of irqs to populate
173 * @mask: Bitmask of port capabilities returned by get_port_device_capability()
174 *
175 * Return value: Interrupt mode associated with the port
176 */
177static int pcie_init_service_irqs(struct pci_dev *dev, int *irqs, int mask)
178{
179	int ret, i;
180
181	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
182		irqs[i] = -1;
183
184	/*
185	 * If we support PME but can't use MSI/MSI-X for it, we have to
186	 * fall back to INTx or other interrupts, e.g., a system shared
187	 * interrupt.
188	 */
189	if ((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi())
190		goto intx_irq;
191
192	/* Try to use MSI-X or MSI if supported */
193	if (pcie_port_enable_irq_vec(dev, irqs, mask) == 0)
194		return 0;
195
196intx_irq:
197	/* fall back to INTX IRQ */
198	ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_INTX);
199	if (ret < 0)
200		return -ENODEV;
201
202	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
203		irqs[i] = pci_irq_vector(dev, 0);
204
205	return 0;
206}
207
208/**
209 * get_port_device_capability - discover capabilities of a PCI Express port
210 * @dev: PCI Express port to examine
211 *
212 * The capabilities are read from the port's PCI Express configuration registers
213 * as described in PCI Express Base Specification 1.0a sections 7.8.2, 7.8.9 and
214 * 7.9 - 7.11.
215 *
216 * Return value: Bitmask of discovered port capabilities
217 */
218static int get_port_device_capability(struct pci_dev *dev)
219{
220	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
221	int services = 0;
222
223	if (dev->is_pciehp &&
224	    (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
225	     pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) &&
226	    (pcie_ports_native || host->native_pcie_hotplug)) {
227		services |= PCIE_PORT_SERVICE_HP;
228
229		/*
230		 * Disable hot-plug interrupts in case they have been enabled
231		 * by the BIOS and the hot-plug service driver won't be loaded
232		 * to handle them.
233		 */
234		if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE))
235			pcie_capability_clear_word(dev, PCI_EXP_SLTCTL,
236				PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE);
237	}
238
239#ifdef CONFIG_PCIEAER
240	if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
241             pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC) &&
242	    dev->aer_cap && pci_aer_available() &&
243	    (pcie_ports_native || host->native_aer))
244		services |= PCIE_PORT_SERVICE_AER;
245#endif
246
247	/* Root Ports and Root Complex Event Collectors may generate PMEs */
248	if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
249	     pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC) &&
250	    (pcie_ports_native || host->native_pme)) {
251		services |= PCIE_PORT_SERVICE_PME;
252
253		/*
254		 * Disable PME interrupt on this port in case it's been enabled
255		 * by the BIOS (the PME service driver will enable it when
256		 * necessary).
257		 */
258		pcie_pme_interrupt_enable(dev, false);
259	}
260
261	/*
262	 * With dpc-native, allow Linux to use DPC even if it doesn't have
263	 * permission to use AER.
264	 */
265	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) &&
266	    pci_aer_available() &&
267	    (pcie_ports_dpc_native || (services & PCIE_PORT_SERVICE_AER)))
268		services |= PCIE_PORT_SERVICE_DPC;
269
270	/* Enable bandwidth control if more than one speed is supported. */
271	if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM ||
272	    pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
273		u32 linkcap;
274
275		pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &linkcap);
276		if (linkcap & PCI_EXP_LNKCAP_LBNC &&
277		    hweight8(dev->supported_speeds) > 1)
278			services |= PCIE_PORT_SERVICE_BWCTRL;
279	}
280
281	return services;
282}
283
284/**
285 * pcie_device_init - allocate and initialize PCI Express port service device
286 * @pdev: PCI Express port to associate the service device with
287 * @service: Type of service to associate with the service device
288 * @irq: Interrupt vector to associate with the service device
289 */
290static int pcie_device_init(struct pci_dev *pdev, int service, int irq)
291{
292	int retval;
293	struct pcie_device *pcie;
294	struct device *device;
295
296	pcie = kzalloc(sizeof(*pcie), GFP_KERNEL);
297	if (!pcie)
298		return -ENOMEM;
299	pcie->port = pdev;
300	pcie->irq = irq;
301	pcie->service = service;
302
303	/* Initialize generic device interface */
304	device = &pcie->device;
305	device->bus = &pcie_port_bus_type;
306	device->release = release_pcie_device;	/* callback to free pcie dev */
307	dev_set_name(device, "%s:pcie%03x",
308		     pci_name(pdev),
309		     get_descriptor_id(pci_pcie_type(pdev), service));
310	device->parent = &pdev->dev;
311	device_enable_async_suspend(device);
312
313	retval = device_register(device);
314	if (retval) {
315		put_device(device);
316		return retval;
317	}
318
319	pm_runtime_no_callbacks(device);
320
321	return 0;
322}
323
324/**
325 * pcie_port_device_register - register PCI Express port
326 * @dev: PCI Express port to register
327 *
328 * Allocate the port extension structure and register services associated with
329 * the port.
330 */
331static int pcie_port_device_register(struct pci_dev *dev)
332{
333	int status, capabilities, i, nr_service;
334	int irqs[PCIE_PORT_DEVICE_MAXSERVICES];
335
336	/* Enable PCI Express port device */
337	status = pci_enable_device(dev);
338	if (status)
339		return status;
340
341	/* Get and check PCI Express port services */
342	capabilities = get_port_device_capability(dev);
343	if (!capabilities)
344		return 0;
345
346	pci_set_master(dev);
347	/*
348	 * Initialize service irqs. Don't use service devices that
349	 * require interrupts if there is no way to generate them.
350	 * However, some drivers may have a polling mode (e.g. pciehp_poll_mode)
351	 * that can be used in the absence of irqs.  Allow them to determine
352	 * if that is to be used.
353	 */
354	status = pcie_init_service_irqs(dev, irqs, capabilities);
355	if (status) {
356		capabilities &= PCIE_PORT_SERVICE_HP;
357		if (!capabilities)
358			goto error_disable;
359	}
360
361	/* Allocate child services if any */
362	status = -ENODEV;
363	nr_service = 0;
364	for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) {
365		int service = 1 << i;
366		if (!(capabilities & service))
367			continue;
368		if (!pcie_device_init(dev, service, irqs[i]))
369			nr_service++;
370	}
371	if (!nr_service)
372		goto error_cleanup_irqs;
373
374	return 0;
375
376error_cleanup_irqs:
377	pci_free_irq_vectors(dev);
378error_disable:
379	pci_disable_device(dev);
380	return status;
381}
382
383typedef int (*pcie_callback_t)(struct pcie_device *);
384
385static int pcie_port_device_iter(struct device *dev, void *data)
386{
387	struct pcie_port_service_driver *service_driver;
388	size_t offset = *(size_t *)data;
389	pcie_callback_t cb;
390
391	if ((dev->bus == &pcie_port_bus_type) && dev->driver) {
392		service_driver = to_service_driver(dev->driver);
393		cb = *(pcie_callback_t *)((void *)service_driver + offset);
394		if (cb)
395			return cb(to_pcie_device(dev));
396	}
397	return 0;
398}
399
400#ifdef CONFIG_PM
401/**
402 * pcie_port_device_suspend - suspend port services associated with a PCIe port
403 * @dev: PCI Express port to handle
404 */
405static int pcie_port_device_suspend(struct device *dev)
406{
407	size_t off = offsetof(struct pcie_port_service_driver, suspend);
408	return device_for_each_child(dev, &off, pcie_port_device_iter);
409}
410
411static int pcie_port_device_resume_noirq(struct device *dev)
412{
413	size_t off = offsetof(struct pcie_port_service_driver, resume_noirq);
414	return device_for_each_child(dev, &off, pcie_port_device_iter);
415}
416
417/**
418 * pcie_port_device_resume - resume port services associated with a PCIe port
419 * @dev: PCI Express port to handle
420 */
421static int pcie_port_device_resume(struct device *dev)
422{
423	size_t off = offsetof(struct pcie_port_service_driver, resume);
424	return device_for_each_child(dev, &off, pcie_port_device_iter);
425}
426
427/**
428 * pcie_port_device_runtime_suspend - runtime suspend port services
429 * @dev: PCI Express port to handle
430 */
431static int pcie_port_device_runtime_suspend(struct device *dev)
432{
433	size_t off = offsetof(struct pcie_port_service_driver, runtime_suspend);
434	return device_for_each_child(dev, &off, pcie_port_device_iter);
435}
436
437/**
438 * pcie_port_device_runtime_resume - runtime resume port services
439 * @dev: PCI Express port to handle
440 */
441static int pcie_port_device_runtime_resume(struct device *dev)
442{
443	size_t off = offsetof(struct pcie_port_service_driver, runtime_resume);
444	return device_for_each_child(dev, &off, pcie_port_device_iter);
445}
446#endif /* PM */
447
448static int remove_iter(struct device *dev, void *data)
449{
450	if (dev->bus == &pcie_port_bus_type)
451		device_unregister(dev);
452	return 0;
453}
454
455static int find_service_iter(struct device *device, void *data)
456{
457	struct pcie_port_service_driver *service_driver;
458	struct portdrv_service_data *pdrvs;
459	u32 service;
460
461	pdrvs = (struct portdrv_service_data *) data;
462	service = pdrvs->service;
463
464	if (device->bus == &pcie_port_bus_type && device->driver) {
465		service_driver = to_service_driver(device->driver);
466		if (service_driver->service == service) {
467			pdrvs->drv = service_driver;
468			pdrvs->dev = device;
469			return 1;
470		}
471	}
472
473	return 0;
474}
475
476/**
477 * pcie_port_find_device - find the struct device
478 * @dev: PCI Express port the service is associated with
479 * @service: For the service to find
480 *
481 * Find the struct device associated with given service on a pci_dev
482 */
483struct device *pcie_port_find_device(struct pci_dev *dev,
484				      u32 service)
485{
486	struct device *device;
487	struct portdrv_service_data pdrvs;
488
489	pdrvs.dev = NULL;
490	pdrvs.service = service;
491	device_for_each_child(&dev->dev, &pdrvs, find_service_iter);
492
493	device = pdrvs.dev;
494	return device;
495}
496EXPORT_SYMBOL_GPL(pcie_port_find_device);
497
498/**
499 * pcie_port_device_remove - unregister PCI Express port service devices
500 * @dev: PCI Express port the service devices to unregister are associated with
501 *
502 * Remove PCI Express port service devices associated with given port and
503 * disable MSI-X or MSI for the port.
504 */
505static void pcie_port_device_remove(struct pci_dev *dev)
506{
507	device_for_each_child(&dev->dev, NULL, remove_iter);
508	pci_free_irq_vectors(dev);
509}
510
511/**
512 * pcie_port_probe_service - probe driver for given PCI Express port service
513 * @dev: PCI Express port service device to probe against
514 *
515 * If PCI Express port service driver is registered with
516 * pcie_port_service_register(), this function will be called by the driver core
517 * whenever match is found between the driver and a port service device.
518 */
519static int pcie_port_probe_service(struct device *dev)
520{
521	struct pcie_device *pciedev;
522	struct pcie_port_service_driver *driver;
523	int status;
524
525	if (!dev || !dev->driver)
526		return -ENODEV;
527
528	driver = to_service_driver(dev->driver);
529	if (!driver || !driver->probe)
530		return -ENODEV;
531
532	pciedev = to_pcie_device(dev);
533	status = driver->probe(pciedev);
534	if (status)
535		return status;
536
537	get_device(dev);
538	return 0;
539}
540
541/**
542 * pcie_port_remove_service - detach driver from given PCI Express port service
543 * @dev: PCI Express port service device to handle
544 *
545 * If PCI Express port service driver is registered with
546 * pcie_port_service_register(), this function will be called by the driver core
547 * when device_unregister() is called for the port service device associated
548 * with the driver.
549 */
550static int pcie_port_remove_service(struct device *dev)
551{
552	struct pcie_device *pciedev;
553	struct pcie_port_service_driver *driver;
554
555	if (!dev || !dev->driver)
556		return 0;
557
558	pciedev = to_pcie_device(dev);
559	driver = to_service_driver(dev->driver);
560	if (driver && driver->remove)
561		driver->remove(pciedev);
562
563	put_device(dev);
564	return 0;
565}
566
567/**
568 * pcie_port_shutdown_service - shut down given PCI Express port service
569 * @dev: PCI Express port service device to handle
570 *
571 * If PCI Express port service driver is registered with
572 * pcie_port_service_register(), this function will be called by the driver core
573 * when device_shutdown() is called for the port service device associated
574 * with the driver.
575 */
576static void pcie_port_shutdown_service(struct device *dev) {}
577
578/**
579 * pcie_port_service_register - register PCI Express port service driver
580 * @new: PCI Express port service driver to register
581 */
582int pcie_port_service_register(struct pcie_port_service_driver *new)
583{
584	if (pcie_ports_disabled)
585		return -ENODEV;
586
587	new->driver.name = new->name;
588	new->driver.bus = &pcie_port_bus_type;
589	new->driver.probe = pcie_port_probe_service;
590	new->driver.remove = pcie_port_remove_service;
591	new->driver.shutdown = pcie_port_shutdown_service;
592
593	return driver_register(&new->driver);
594}
595
596/**
597 * pcie_port_service_unregister - unregister PCI Express port service driver
598 * @drv: PCI Express port service driver to unregister
599 */
600void pcie_port_service_unregister(struct pcie_port_service_driver *drv)
601{
602	driver_unregister(&drv->driver);
603}
604
605/* If this switch is set, PCIe port native services should not be enabled. */
606bool pcie_ports_disabled;
607
608/*
609 * If the user specified "pcie_ports=native", use the PCIe services regardless
610 * of whether the platform has given us permission.  On ACPI systems, this
611 * means we ignore _OSC.
612 */
613bool pcie_ports_native;
614
615/*
616 * If the user specified "pcie_ports=dpc-native", use the Linux DPC PCIe
617 * service even if the platform hasn't given us permission.
618 */
619bool pcie_ports_dpc_native;
620
621static int __init pcie_port_setup(char *str)
622{
623	if (!strncmp(str, "compat", 6))
624		pcie_ports_disabled = true;
625	else if (!strncmp(str, "native", 6))
626		pcie_ports_native = true;
627	else if (!strncmp(str, "dpc-native", 10))
628		pcie_ports_dpc_native = true;
629
630	return 1;
631}
632__setup("pcie_ports=", pcie_port_setup);
633
634/* global data */
635
636#ifdef CONFIG_PM
637static int pcie_port_runtime_suspend(struct device *dev)
638{
639	if (!to_pci_dev(dev)->bridge_d3)
640		return -EBUSY;
641
642	return pcie_port_device_runtime_suspend(dev);
643}
644
645static int pcie_port_runtime_idle(struct device *dev)
646{
647	/*
648	 * Assume the PCI core has set bridge_d3 whenever it thinks the port
649	 * should be good to go to D3.  Everything else, including moving
650	 * the port to D3, is handled by the PCI core.
651	 */
652	return to_pci_dev(dev)->bridge_d3 ? 0 : -EBUSY;
653}
654
655static const struct dev_pm_ops pcie_portdrv_pm_ops = {
656	.suspend	= pcie_port_device_suspend,
657	.resume_noirq	= pcie_port_device_resume_noirq,
658	.resume		= pcie_port_device_resume,
659	.freeze		= pcie_port_device_suspend,
660	.thaw		= pcie_port_device_resume,
661	.poweroff	= pcie_port_device_suspend,
662	.restore_noirq	= pcie_port_device_resume_noirq,
663	.restore	= pcie_port_device_resume,
664	.runtime_suspend = pcie_port_runtime_suspend,
665	.runtime_resume	= pcie_port_device_runtime_resume,
666	.runtime_idle	= pcie_port_runtime_idle,
667};
668
669#define PCIE_PORTDRV_PM_OPS	(&pcie_portdrv_pm_ops)
670
671#else /* !PM */
672
673#define PCIE_PORTDRV_PM_OPS	NULL
674#endif /* !PM */
675
676/*
677 * pcie_portdrv_probe - Probe PCI-Express port devices
678 * @dev: PCI-Express port device being probed
679 *
680 * If detected invokes the pcie_port_device_register() method for
681 * this port device.
682 *
683 */
684static int pcie_portdrv_probe(struct pci_dev *dev,
685					const struct pci_device_id *id)
686{
687	int type = pci_pcie_type(dev);
688	int status;
689
690	if (!pci_is_pcie(dev) ||
691	    ((type != PCI_EXP_TYPE_ROOT_PORT) &&
692	     (type != PCI_EXP_TYPE_UPSTREAM) &&
693	     (type != PCI_EXP_TYPE_DOWNSTREAM) &&
694	     (type != PCI_EXP_TYPE_RC_EC)))
695		return -ENODEV;
696
697	if (type == PCI_EXP_TYPE_RC_EC)
698		pcie_link_rcec(dev);
699
700	status = pcie_port_device_register(dev);
701	if (status)
702		return status;
703
704	pci_save_state(dev);
705
706	dev_pm_set_driver_flags(&dev->dev, DPM_FLAG_NO_DIRECT_COMPLETE |
707					   DPM_FLAG_SMART_SUSPEND);
708
709	if (pci_bridge_d3_possible(dev)) {
710		/*
711		 * Keep the port resumed 100ms to make sure things like
712		 * config space accesses from userspace (lspci) will not
713		 * cause the port to repeatedly suspend and resume.
714		 */
715		pm_runtime_set_autosuspend_delay(&dev->dev, 100);
716		pm_runtime_use_autosuspend(&dev->dev);
717		pm_runtime_mark_last_busy(&dev->dev);
718		pm_runtime_put_autosuspend(&dev->dev);
719		pm_runtime_allow(&dev->dev);
720	}
721
722	return 0;
723}
724
725static void pcie_portdrv_remove(struct pci_dev *dev)
726{
727	if (pci_bridge_d3_possible(dev)) {
728		pm_runtime_forbid(&dev->dev);
729		pm_runtime_get_noresume(&dev->dev);
730		pm_runtime_dont_use_autosuspend(&dev->dev);
731	}
732
733	pcie_port_device_remove(dev);
734
735	pci_disable_device(dev);
736}
737
738static void pcie_portdrv_shutdown(struct pci_dev *dev)
739{
740	if (pci_bridge_d3_possible(dev)) {
741		pm_runtime_forbid(&dev->dev);
742		pm_runtime_get_noresume(&dev->dev);
743		pm_runtime_dont_use_autosuspend(&dev->dev);
744	}
745
746	pcie_port_device_remove(dev);
747}
748
749static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev,
750					pci_channel_state_t error)
751{
752	if (error == pci_channel_io_frozen)
753		return PCI_ERS_RESULT_NEED_RESET;
754	return PCI_ERS_RESULT_CAN_RECOVER;
755}
756
757static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev)
758{
759	size_t off = offsetof(struct pcie_port_service_driver, slot_reset);
760	device_for_each_child(&dev->dev, &off, pcie_port_device_iter);
761
762	pci_restore_state(dev);
763	return PCI_ERS_RESULT_RECOVERED;
764}
765
766static pci_ers_result_t pcie_portdrv_mmio_enabled(struct pci_dev *dev)
767{
768	return PCI_ERS_RESULT_RECOVERED;
769}
770
771/*
772 * LINUX Device Driver Model
773 */
774static const struct pci_device_id port_pci_ids[] = {
775	/* handle any PCI-Express port */
776	{ PCI_DEVICE_CLASS(PCI_CLASS_BRIDGE_PCI_NORMAL, ~0) },
777	/* subtractive decode PCI-to-PCI bridge, class type is 060401h */
778	{ PCI_DEVICE_CLASS(PCI_CLASS_BRIDGE_PCI_SUBTRACTIVE, ~0) },
779	/* handle any Root Complex Event Collector */
780	{ PCI_DEVICE_CLASS(((PCI_CLASS_SYSTEM_RCEC << 8) | 0x00), ~0) },
781	{ },
782};
783
784static const struct pci_error_handlers pcie_portdrv_err_handler = {
785	.error_detected = pcie_portdrv_error_detected,
786	.slot_reset = pcie_portdrv_slot_reset,
787	.mmio_enabled = pcie_portdrv_mmio_enabled,
788};
789
790static struct pci_driver pcie_portdriver = {
791	.name		= "pcieport",
792	.id_table	= port_pci_ids,
793
794	.probe		= pcie_portdrv_probe,
795	.remove		= pcie_portdrv_remove,
796	.shutdown	= pcie_portdrv_shutdown,
797
798	.err_handler	= &pcie_portdrv_err_handler,
799
800	.driver_managed_dma = true,
801
802	.driver.pm	= PCIE_PORTDRV_PM_OPS,
803};
804
805static int __init dmi_pcie_pme_disable_msi(const struct dmi_system_id *d)
806{
807	pr_notice("%s detected: will not use MSI for PCIe PME signaling\n",
808		  d->ident);
809	pcie_pme_disable_msi();
810	return 0;
811}
812
813static const struct dmi_system_id pcie_portdrv_dmi_table[] __initconst = {
814	/*
815	 * Boxes that should not use MSI for PCIe PME signaling.
816	 */
817	{
818	 .callback = dmi_pcie_pme_disable_msi,
819	 .ident = "MSI Wind U-100",
820	 .matches = {
821		     DMI_MATCH(DMI_SYS_VENDOR,
822				"MICRO-STAR INTERNATIONAL CO., LTD"),
823		     DMI_MATCH(DMI_PRODUCT_NAME, "U-100"),
824		     },
825	 },
826	 {}
827};
828
829static void __init pcie_init_services(void)
830{
831	pcie_aer_init();
832	pcie_pme_init();
833	pcie_dpc_init();
834	pcie_bwctrl_init();
835	pcie_hp_init();
836}
837
838static int __init pcie_portdrv_init(void)
839{
840	if (pcie_ports_disabled)
841		return -EACCES;
842
843	pcie_init_services();
844	dmi_check_system(pcie_portdrv_dmi_table);
845
846	return pci_register_driver(&pcie_portdriver);
847}
848device_initcall(pcie_portdrv_init);