dpc.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * PCI Express Downstream Port Containment services driver
  4 * Author: Keith Busch <keith.busch@intel.com>
  5 *
  6 * Copyright (C) 2016 Intel Corp.
  7 */
  8
  9#define dev_fmt(fmt) "DPC: " fmt
 10
 11#include <linux/aer.h>
 12#include <linux/bitfield.h>
 13#include <linux/delay.h>
 14#include <linux/interrupt.h>
 15#include <linux/init.h>
 16#include <linux/pci.h>
 17
 18#include "portdrv.h"
 19#include "../pci.h"
 20
 21#define PCI_EXP_DPC_CTL_EN_MASK	(PCI_EXP_DPC_CTL_EN_FATAL | \
 22				 PCI_EXP_DPC_CTL_EN_NONFATAL)
 23
 24static const char * const rp_pio_error_string[] = {
 25	"Configuration Request received UR Completion",	 /* Bit Position 0  */
 26	"Configuration Request received CA Completion",	 /* Bit Position 1  */
 27	"Configuration Request Completion Timeout",	 /* Bit Position 2  */
 28	NULL,
 29	NULL,
 30	NULL,
 31	NULL,
 32	NULL,
 33	"I/O Request received UR Completion",		 /* Bit Position 8  */
 34	"I/O Request received CA Completion",		 /* Bit Position 9  */
 35	"I/O Request Completion Timeout",		 /* Bit Position 10 */
 36	NULL,
 37	NULL,
 38	NULL,
 39	NULL,
 40	NULL,
 41	"Memory Request received UR Completion",	 /* Bit Position 16 */
 42	"Memory Request received CA Completion",	 /* Bit Position 17 */
 43	"Memory Request Completion Timeout",		 /* Bit Position 18 */
 44};
 45
 46void pci_save_dpc_state(struct pci_dev *dev)
 47{
 48	struct pci_cap_saved_state *save_state;
 49	u16 *cap;
 50
 51	if (!pci_is_pcie(dev))
 52		return;
 53
 54	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_DPC);
 55	if (!save_state)
 56		return;
 57
 58	cap = (u16 *)&save_state->cap.data[0];
 59	pci_read_config_word(dev, dev->dpc_cap + PCI_EXP_DPC_CTL, cap);
 60}
 61
 62void pci_restore_dpc_state(struct pci_dev *dev)
 63{
 64	struct pci_cap_saved_state *save_state;
 65	u16 *cap;
 66
 67	if (!pci_is_pcie(dev))
 68		return;
 69
 70	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_DPC);
 71	if (!save_state)
 72		return;
 73
 74	cap = (u16 *)&save_state->cap.data[0];
 75	pci_write_config_word(dev, dev->dpc_cap + PCI_EXP_DPC_CTL, *cap);
 76}
 77
 78static DECLARE_WAIT_QUEUE_HEAD(dpc_completed_waitqueue);
 79
 80#ifdef CONFIG_HOTPLUG_PCI_PCIE
 81static bool dpc_completed(struct pci_dev *pdev)
 82{
 83	u16 status;
 84
 85	pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_STATUS, &status);
 86	if ((!PCI_POSSIBLE_ERROR(status)) && (status & PCI_EXP_DPC_STATUS_TRIGGER))
 87		return false;
 88
 89	if (test_bit(PCI_DPC_RECOVERING, &pdev->priv_flags))
 90		return false;
 91
 92	return true;
 93}
 94
 95/**
 96 * pci_dpc_recovered - whether DPC triggered and has recovered successfully
 97 * @pdev: PCI device
 98 *
 99 * Return true if DPC was triggered for @pdev and has recovered successfully.
100 * Wait for recovery if it hasn't completed yet.  Called from the PCIe hotplug
101 * driver to recognize and ignore Link Down/Up events caused by DPC.
102 */
103bool pci_dpc_recovered(struct pci_dev *pdev)
104{
105	struct pci_host_bridge *host;
106
107	if (!pdev->dpc_cap)
108		return false;
109
110	/*
111	 * Synchronization between hotplug and DPC is not supported
112	 * if DPC is owned by firmware and EDR is not enabled.
113	 */
114	host = pci_find_host_bridge(pdev->bus);
115	if (!host->native_dpc && !IS_ENABLED(CONFIG_PCIE_EDR))
116		return false;
117
118	/*
119	 * Need a timeout in case DPC never completes due to failure of
120	 * dpc_wait_rp_inactive().  The spec doesn't mandate a time limit,
121	 * but reports indicate that DPC completes within 4 seconds.
122	 */
123	wait_event_timeout(dpc_completed_waitqueue, dpc_completed(pdev),
124			   msecs_to_jiffies(4000));
125
126	return test_and_clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
127}
128#endif /* CONFIG_HOTPLUG_PCI_PCIE */
129
130static int dpc_wait_rp_inactive(struct pci_dev *pdev)
131{
132	unsigned long timeout = jiffies + HZ;
133	u16 cap = pdev->dpc_cap, status;
134
135	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
136	while (status & PCI_EXP_DPC_RP_BUSY &&
137					!time_after(jiffies, timeout)) {
138		msleep(10);
139		pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
140	}
141	if (status & PCI_EXP_DPC_RP_BUSY) {
142		pci_warn(pdev, "root port still busy\n");
143		return -EBUSY;
144	}
145	return 0;
146}
147
148pci_ers_result_t dpc_reset_link(struct pci_dev *pdev)
149{
150	pci_ers_result_t ret;
151	u16 cap;
152
153	set_bit(PCI_DPC_RECOVERING, &pdev->priv_flags);
154
155	/*
156	 * DPC disables the Link automatically in hardware, so it has
157	 * already been reset by the time we get here.
158	 */
159	cap = pdev->dpc_cap;
160
161	/*
162	 * Wait until the Link is inactive, then clear DPC Trigger Status
163	 * to allow the Port to leave DPC.
164	 */
165	if (!pcie_wait_for_link(pdev, false))
166		pci_info(pdev, "Data Link Layer Link Active not cleared in 1000 msec\n");
167
168	if (pdev->dpc_rp_extensions && dpc_wait_rp_inactive(pdev)) {
169		clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
170		ret = PCI_ERS_RESULT_DISCONNECT;
171		goto out;
172	}
173
174	pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS,
175			      PCI_EXP_DPC_STATUS_TRIGGER);
176
177	if (pci_bridge_wait_for_secondary_bus(pdev, "DPC")) {
178		clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
179		ret = PCI_ERS_RESULT_DISCONNECT;
180	} else {
181		set_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
182		ret = PCI_ERS_RESULT_RECOVERED;
183	}
184out:
185	clear_bit(PCI_DPC_RECOVERING, &pdev->priv_flags);
186	wake_up_all(&dpc_completed_waitqueue);
187	return ret;
188}
189
190static void dpc_process_rp_pio_error(struct pci_dev *pdev)
191{
192	u16 cap = pdev->dpc_cap, dpc_status, first_error;
193	u32 status, mask, sev, syserr, exc, log;
194	struct pcie_tlp_log tlp_log;
195	int i;
196
197	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_STATUS, &status);
198	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_MASK, &mask);
199	pci_err(pdev, "rp_pio_status: %#010x, rp_pio_mask: %#010x\n",
200		status, mask);
201
202	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_SEVERITY, &sev);
203	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_SYSERROR, &syserr);
204	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_EXCEPTION, &exc);
205	pci_err(pdev, "RP PIO severity=%#010x, syserror=%#010x, exception=%#010x\n",
206		sev, syserr, exc);
207
208	/* Get First Error Pointer */
209	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &dpc_status);
210	first_error = FIELD_GET(PCI_EXP_DPC_RP_PIO_FEP, dpc_status);
211
212	for (i = 0; i < ARRAY_SIZE(rp_pio_error_string); i++) {
213		if ((status & ~mask) & (1 << i))
214			pci_err(pdev, "[%2d] %s%s\n", i, rp_pio_error_string[i],
215				first_error == i ? " (First)" : "");
216	}
217
218	if (pdev->dpc_rp_log_size < PCIE_STD_NUM_TLP_HEADERLOG)
219		goto clear_status;
220	pcie_read_tlp_log(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG,
221			  cap + PCI_EXP_DPC_RP_PIO_TLPPREFIX_LOG,
222			  dpc_tlp_log_len(pdev),
223			  pdev->subordinate->flit_mode,
224			  &tlp_log);
225	pcie_print_tlp_log(pdev, &tlp_log, KERN_ERR, dev_fmt(""));
226
227	if (pdev->dpc_rp_log_size < PCIE_STD_NUM_TLP_HEADERLOG + 1)
228		goto clear_status;
229	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_IMPSPEC_LOG, &log);
230	pci_err(pdev, "RP PIO ImpSpec Log %#010x\n", log);
231
232 clear_status:
233	pci_write_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_STATUS, status);
234}
235
236static int dpc_get_aer_uncorrect_severity(struct pci_dev *dev,
237					  struct aer_err_info *info)
238{
239	int pos = dev->aer_cap;
240	u32 status, mask, sev;
241
242	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
243	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, &mask);
244	status &= ~mask;
245	if (!status)
246		return 0;
247
248	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &sev);
249	status &= sev;
250	if (status)
251		info->severity = AER_FATAL;
252	else
253		info->severity = AER_NONFATAL;
254
255	info->level = KERN_ERR;
256
257	info->dev[0] = dev;
258	info->error_dev_num = 1;
259
260	return 1;
261}
262
263void dpc_process_error(struct pci_dev *pdev)
264{
265	u16 cap = pdev->dpc_cap, status, source, reason, ext_reason;
266	struct aer_err_info info = {};
267
268	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
269
270	reason = status & PCI_EXP_DPC_STATUS_TRIGGER_RSN;
271
272	switch (reason) {
273	case PCI_EXP_DPC_STATUS_TRIGGER_RSN_UNCOR:
274		pci_warn(pdev, "containment event, status:%#06x: unmasked uncorrectable error detected\n",
275			 status);
276		if (dpc_get_aer_uncorrect_severity(pdev, &info) &&
277		    aer_get_device_error_info(&info, 0)) {
278			aer_print_error(&info, 0);
279			pci_aer_clear_nonfatal_status(pdev);
280			pci_aer_clear_fatal_status(pdev);
281		}
282		break;
283	case PCI_EXP_DPC_STATUS_TRIGGER_RSN_NFE:
284	case PCI_EXP_DPC_STATUS_TRIGGER_RSN_FE:
285		pci_read_config_word(pdev, cap + PCI_EXP_DPC_SOURCE_ID,
286				     &source);
287		pci_warn(pdev, "containment event, status:%#06x, %s received from %04x:%02x:%02x.%d\n",
288			 status,
289			 (reason == PCI_EXP_DPC_STATUS_TRIGGER_RSN_FE) ?
290				"ERR_FATAL" : "ERR_NONFATAL",
291			 pci_domain_nr(pdev->bus), PCI_BUS_NUM(source),
292			 PCI_SLOT(source), PCI_FUNC(source));
293		break;
294	case PCI_EXP_DPC_STATUS_TRIGGER_RSN_IN_EXT:
295		ext_reason = status & PCI_EXP_DPC_STATUS_TRIGGER_RSN_EXT;
296		pci_warn(pdev, "containment event, status:%#06x: %s detected\n",
297			 status,
298			 (ext_reason == PCI_EXP_DPC_STATUS_TRIGGER_RSN_RP_PIO) ?
299			 "RP PIO error" :
300			 (ext_reason == PCI_EXP_DPC_STATUS_TRIGGER_RSN_SW_TRIGGER) ?
301			 "software trigger" :
302			 "reserved error");
303		/* show RP PIO error detail information */
304		if (ext_reason == PCI_EXP_DPC_STATUS_TRIGGER_RSN_RP_PIO &&
305		    pdev->dpc_rp_extensions)
306			dpc_process_rp_pio_error(pdev);
307		break;
308	}
309}
310
311static void pci_clear_surpdn_errors(struct pci_dev *pdev)
312{
313	if (pdev->dpc_rp_extensions)
314		pci_write_config_dword(pdev, pdev->dpc_cap +
315				       PCI_EXP_DPC_RP_PIO_STATUS, ~0);
316
317	/*
318	 * In practice, Surprise Down errors have been observed to also set
319	 * error bits in the Status Register as well as the Fatal Error
320	 * Detected bit in the Device Status Register.
321	 */
322	pci_write_config_word(pdev, PCI_STATUS, 0xffff);
323
324	pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, PCI_EXP_DEVSTA_FED);
325}
326
327static void dpc_handle_surprise_removal(struct pci_dev *pdev)
328{
329	if (!pcie_wait_for_link(pdev, false)) {
330		pci_info(pdev, "Data Link Layer Link Active not cleared in 1000 msec\n");
331		goto out;
332	}
333
334	if (pdev->dpc_rp_extensions && dpc_wait_rp_inactive(pdev))
335		goto out;
336
337	pci_aer_raw_clear_status(pdev);
338	pci_clear_surpdn_errors(pdev);
339
340	pci_write_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_STATUS,
341			      PCI_EXP_DPC_STATUS_TRIGGER);
342
343out:
344	clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
345	wake_up_all(&dpc_completed_waitqueue);
346}
347
348static bool dpc_is_surprise_removal(struct pci_dev *pdev)
349{
350	u16 status;
351
352	if (!pdev->is_hotplug_bridge)
353		return false;
354
355	if (pci_read_config_word(pdev, pdev->aer_cap + PCI_ERR_UNCOR_STATUS,
356				 &status))
357		return false;
358
359	return status & PCI_ERR_UNC_SURPDN;
360}
361
362static irqreturn_t dpc_handler(int irq, void *context)
363{
364	struct pci_dev *pdev = context;
365
366	/*
367	 * According to PCIe r6.0 sec 6.7.6, errors are an expected side effect
368	 * of async removal and should be ignored by software.
369	 */
370	if (dpc_is_surprise_removal(pdev)) {
371		dpc_handle_surprise_removal(pdev);
372		return IRQ_HANDLED;
373	}
374
375	dpc_process_error(pdev);
376
377	/* We configure DPC so it only triggers on ERR_FATAL */
378	pcie_do_recovery(pdev, pci_channel_io_frozen, dpc_reset_link);
379
380	return IRQ_HANDLED;
381}
382
383static irqreturn_t dpc_irq(int irq, void *context)
384{
385	struct pci_dev *pdev = context;
386	u16 cap = pdev->dpc_cap, status;
387
388	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
389
390	if (!(status & PCI_EXP_DPC_STATUS_INTERRUPT) || PCI_POSSIBLE_ERROR(status))
391		return IRQ_NONE;
392
393	pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS,
394			      PCI_EXP_DPC_STATUS_INTERRUPT);
395	if (status & PCI_EXP_DPC_STATUS_TRIGGER)
396		return IRQ_WAKE_THREAD;
397	return IRQ_HANDLED;
398}
399
400void pci_dpc_init(struct pci_dev *pdev)
401{
402	u16 cap;
403
404	pdev->dpc_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DPC);
405	if (!pdev->dpc_cap)
406		return;
407
408	pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CAP, &cap);
409	if (!(cap & PCI_EXP_DPC_CAP_RP_EXT))
410		return;
411
412	pdev->dpc_rp_extensions = true;
413
414	/* Quirks may set dpc_rp_log_size if device or firmware is buggy */
415	if (!pdev->dpc_rp_log_size) {
416		u16 flags;
417		int ret;
418
419		ret = pcie_capability_read_word(pdev, PCI_EXP_FLAGS, &flags);
420		if (ret)
421			return;
422
423		pdev->dpc_rp_log_size =
424				FIELD_GET(PCI_EXP_DPC_RP_PIO_LOG_SIZE, cap);
425		if (FIELD_GET(PCI_EXP_FLAGS_FLIT, flags))
426			pdev->dpc_rp_log_size += FIELD_GET(PCI_EXP_DPC_RP_PIO_LOG_SIZE4,
427							   cap) << 4;
428
429		if (pdev->dpc_rp_log_size < PCIE_STD_NUM_TLP_HEADERLOG ||
430		    pdev->dpc_rp_log_size > PCIE_STD_MAX_TLP_HEADERLOG + 1) {
431			pci_err(pdev, "RP PIO log size %u is invalid\n",
432				pdev->dpc_rp_log_size);
433			pdev->dpc_rp_log_size = 0;
434		}
435	}
436}
437
438static void dpc_enable(struct pcie_device *dev)
439{
440	struct pci_dev *pdev = dev->port;
441	int dpc = pdev->dpc_cap;
442	u16 ctl;
443
444	/*
445	 * Clear DPC Interrupt Status so we don't get an interrupt for an
446	 * old event when setting DPC Interrupt Enable.
447	 */
448	pci_write_config_word(pdev, dpc + PCI_EXP_DPC_STATUS,
449			      PCI_EXP_DPC_STATUS_INTERRUPT);
450
451	pci_read_config_word(pdev, dpc + PCI_EXP_DPC_CTL, &ctl);
452	ctl &= ~PCI_EXP_DPC_CTL_EN_MASK;
453	ctl |= PCI_EXP_DPC_CTL_EN_FATAL | PCI_EXP_DPC_CTL_INT_EN;
454	pci_write_config_word(pdev, dpc + PCI_EXP_DPC_CTL, ctl);
455}
456
457static void dpc_disable(struct pcie_device *dev)
458{
459	struct pci_dev *pdev = dev->port;
460	int dpc = pdev->dpc_cap;
461	u16 ctl;
462
463	/* Disable DPC triggering and DPC interrupts */
464	pci_read_config_word(pdev, dpc + PCI_EXP_DPC_CTL, &ctl);
465	ctl &= ~(PCI_EXP_DPC_CTL_EN_FATAL | PCI_EXP_DPC_CTL_INT_EN);
466	pci_write_config_word(pdev, dpc + PCI_EXP_DPC_CTL, ctl);
467}
468
469#define FLAG(x, y) (((x) & (y)) ? '+' : '-')
470static int dpc_probe(struct pcie_device *dev)
471{
472	struct pci_dev *pdev = dev->port;
473	struct device *device = &dev->device;
474	int status;
475	u16 cap;
476
477	if (!pcie_aer_is_native(pdev) && !pcie_ports_dpc_native)
478		return -ENOTSUPP;
479
480	status = devm_request_threaded_irq(device, dev->irq, dpc_irq,
481					   dpc_handler, IRQF_SHARED,
482					   "pcie-dpc", pdev);
483	if (status) {
484		pci_warn(pdev, "request IRQ%d failed: %d\n", dev->irq,
485			 status);
486		return status;
487	}
488
489	pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CAP, &cap);
490	dpc_enable(dev);
491
492	pci_info(pdev, "enabled with IRQ %d\n", dev->irq);
493	pci_info(pdev, "error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
494		 cap & PCI_EXP_DPC_IRQ, FLAG(cap, PCI_EXP_DPC_CAP_RP_EXT),
495		 FLAG(cap, PCI_EXP_DPC_CAP_POISONED_TLP),
496		 FLAG(cap, PCI_EXP_DPC_CAP_SW_TRIGGER), pdev->dpc_rp_log_size,
497		 FLAG(cap, PCI_EXP_DPC_CAP_DL_ACTIVE));
498
499	pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_DPC, sizeof(u16));
500	return status;
501}
502
503static int dpc_suspend(struct pcie_device *dev)
504{
505	dpc_disable(dev);
506	return 0;
507}
508
509static int dpc_resume(struct pcie_device *dev)
510{
511	dpc_enable(dev);
512	return 0;
513}
514
515static void dpc_remove(struct pcie_device *dev)
516{
517	dpc_disable(dev);
518}
519
520static struct pcie_port_service_driver dpcdriver = {
521	.name		= "dpc",
522	.port_type	= PCIE_ANY_PORT,
523	.service	= PCIE_PORT_SERVICE_DPC,
524	.probe		= dpc_probe,
525	.suspend	= dpc_suspend,
526	.resume		= dpc_resume,
527	.remove		= dpc_remove,
528};
529
530int __init pcie_dpc_init(void)
531{
532	return pcie_port_service_register(&dpcdriver);
533}