doe.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Data Object Exchange
  4 *	PCIe r6.0, sec 6.30 DOE
  5 *
  6 * Copyright (C) 2021 Huawei
  7 *	Jonathan Cameron <Jonathan.Cameron@huawei.com>
  8 *
  9 * Copyright (C) 2022 Intel Corporation
 10 *	Ira Weiny <ira.weiny@intel.com>
 11 */
 12
 13#define dev_fmt(fmt) "DOE: " fmt
 14
 15#include <linux/bitfield.h>
 16#include <linux/delay.h>
 17#include <linux/device.h>
 18#include <linux/jiffies.h>
 19#include <linux/mutex.h>
 20#include <linux/pci.h>
 21#include <linux/pci-doe.h>
 22#include <linux/sysfs.h>
 23#include <linux/workqueue.h>
 24
 25#include "pci.h"
 26
 27/* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
 28#define PCI_DOE_TIMEOUT HZ
 29#define PCI_DOE_POLL_INTERVAL	(PCI_DOE_TIMEOUT / 128)
 30
 31#define PCI_DOE_FLAG_CANCEL	0
 32#define PCI_DOE_FLAG_DEAD	1
 33
 34/* Max data object length is 2^18 dwords */
 35#define PCI_DOE_MAX_LENGTH	(1 << 18)
 36
 37/**
 38 * struct pci_doe_mb - State for a single DOE mailbox
 39 *
 40 * This state is used to manage a single DOE mailbox capability.  All fields
 41 * should be considered opaque to the consumers and the structure passed into
 42 * the helpers below after being created by pci_doe_create_mb().
 43 *
 44 * @pdev: PCI device this mailbox belongs to
 45 * @cap_offset: Capability offset
 46 * @feats: Array of features supported (encoded as long values)
 47 * @wq: Wait queue for work item
 48 * @work_queue: Queue of pci_doe_work items
 49 * @flags: Bit array of PCI_DOE_FLAG_* flags
 50 * @sysfs_attrs: Array of sysfs device attributes
 51 */
 52struct pci_doe_mb {
 53	struct pci_dev *pdev;
 54	u16 cap_offset;
 55	struct xarray feats;
 56
 57	wait_queue_head_t wq;
 58	struct workqueue_struct *work_queue;
 59	unsigned long flags;
 60
 61#ifdef CONFIG_SYSFS
 62	struct device_attribute *sysfs_attrs;
 63#endif
 64};
 65
 66struct pci_doe_feature {
 67	u16 vid;
 68	u8 type;
 69};
 70
 71/**
 72 * struct pci_doe_task - represents a single query/response
 73 *
 74 * @feat: DOE Feature
 75 * @request_pl: The request payload
 76 * @request_pl_sz: Size of the request payload (bytes)
 77 * @response_pl: The response payload
 78 * @response_pl_sz: Size of the response payload (bytes)
 79 * @rv: Return value.  Length of received response or error (bytes)
 80 * @complete: Called when task is complete
 81 * @private: Private data for the consumer
 82 * @work: Used internally by the mailbox
 83 * @doe_mb: Used internally by the mailbox
 84 */
 85struct pci_doe_task {
 86	struct pci_doe_feature feat;
 87	const __le32 *request_pl;
 88	size_t request_pl_sz;
 89	__le32 *response_pl;
 90	size_t response_pl_sz;
 91	int rv;
 92	void (*complete)(struct pci_doe_task *task);
 93	void *private;
 94
 95	/* initialized by pci_doe_submit_task() */
 96	struct work_struct work;
 97	struct pci_doe_mb *doe_mb;
 98};
 99
100#ifdef CONFIG_SYSFS
101static ssize_t doe_discovery_show(struct device *dev,
102				  struct device_attribute *attr,
103				  char *buf)
104{
105	return sysfs_emit(buf, "0001:00\n");
106}
107static DEVICE_ATTR_RO(doe_discovery);
108
109static struct attribute *pci_doe_sysfs_feature_attrs[] = {
110	&dev_attr_doe_discovery.attr,
111	NULL
112};
113
114static bool pci_doe_features_sysfs_group_visible(struct kobject *kobj)
115{
116	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
117
118	return !xa_empty(&pdev->doe_mbs);
119}
120DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(pci_doe_features_sysfs)
121
122const struct attribute_group pci_doe_sysfs_group = {
123	.name	    = "doe_features",
124	.attrs	    = pci_doe_sysfs_feature_attrs,
125	.is_visible = SYSFS_GROUP_VISIBLE(pci_doe_features_sysfs),
126};
127
128static ssize_t pci_doe_sysfs_feature_show(struct device *dev,
129					  struct device_attribute *attr,
130					  char *buf)
131{
132	return sysfs_emit(buf, "%s\n", attr->attr.name);
133}
134
135static void pci_doe_sysfs_feature_remove(struct pci_dev *pdev,
136					 struct pci_doe_mb *doe_mb)
137{
138	struct device_attribute *attrs = doe_mb->sysfs_attrs;
139	struct device *dev = &pdev->dev;
140	unsigned long i;
141	void *entry;
142
143	if (!attrs)
144		return;
145
146	doe_mb->sysfs_attrs = NULL;
147	xa_for_each(&doe_mb->feats, i, entry) {
148		if (attrs[i].show)
149			sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr,
150						     pci_doe_sysfs_group.name);
151		kfree(attrs[i].attr.name);
152	}
153	kfree(attrs);
154}
155
156static int pci_doe_sysfs_feature_populate(struct pci_dev *pdev,
157					  struct pci_doe_mb *doe_mb)
158{
159	struct device *dev = &pdev->dev;
160	struct device_attribute *attrs;
161	unsigned long num_features = 0;
162	unsigned long vid, type;
163	unsigned long i;
164	void *entry;
165	int ret;
166
167	xa_for_each(&doe_mb->feats, i, entry)
168		num_features++;
169
170	attrs = kcalloc(num_features, sizeof(*attrs), GFP_KERNEL);
171	if (!attrs) {
172		pci_warn(pdev, "Failed allocating the device_attribute array\n");
173		return -ENOMEM;
174	}
175
176	doe_mb->sysfs_attrs = attrs;
177	xa_for_each(&doe_mb->feats, i, entry) {
178		sysfs_attr_init(&attrs[i].attr);
179		vid = xa_to_value(entry) >> 8;
180		type = xa_to_value(entry) & 0xFF;
181
182		if (vid == PCI_VENDOR_ID_PCI_SIG &&
183		    type == PCI_DOE_FEATURE_DISCOVERY) {
184
185			/*
186			 * DOE Discovery, manually displayed by
187			 * `dev_attr_doe_discovery`
188			 */
189			continue;
190		}
191
192		attrs[i].attr.name = kasprintf(GFP_KERNEL,
193					       "%04lx:%02lx", vid, type);
194		if (!attrs[i].attr.name) {
195			ret = -ENOMEM;
196			pci_warn(pdev, "Failed allocating the attribute name\n");
197			goto fail;
198		}
199
200		attrs[i].attr.mode = 0444;
201		attrs[i].show = pci_doe_sysfs_feature_show;
202
203		ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr,
204					      pci_doe_sysfs_group.name);
205		if (ret) {
206			attrs[i].show = NULL;
207			if (ret != -EEXIST) {
208				pci_warn(pdev, "Failed adding %s to sysfs group\n",
209					 attrs[i].attr.name);
210				goto fail;
211			} else
212				kfree(attrs[i].attr.name);
213		}
214	}
215
216	return 0;
217
218fail:
219	pci_doe_sysfs_feature_remove(pdev, doe_mb);
220	return ret;
221}
222
223void pci_doe_sysfs_teardown(struct pci_dev *pdev)
224{
225	struct pci_doe_mb *doe_mb;
226	unsigned long index;
227
228	xa_for_each(&pdev->doe_mbs, index, doe_mb)
229		pci_doe_sysfs_feature_remove(pdev, doe_mb);
230}
231
232void pci_doe_sysfs_init(struct pci_dev *pdev)
233{
234	struct pci_doe_mb *doe_mb;
235	unsigned long index;
236	int ret;
237
238	xa_for_each(&pdev->doe_mbs, index, doe_mb) {
239		ret = pci_doe_sysfs_feature_populate(pdev, doe_mb);
240		if (ret)
241			return;
242	}
243}
244#endif
245
246static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
247{
248	if (wait_event_timeout(doe_mb->wq,
249			       test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
250			       timeout))
251		return -EIO;
252	return 0;
253}
254
255static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
256{
257	struct pci_dev *pdev = doe_mb->pdev;
258	int offset = doe_mb->cap_offset;
259
260	pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
261}
262
263static int pci_doe_abort(struct pci_doe_mb *doe_mb)
264{
265	struct pci_dev *pdev = doe_mb->pdev;
266	int offset = doe_mb->cap_offset;
267	unsigned long timeout_jiffies;
268
269	pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
270
271	timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
272	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
273
274	do {
275		int rc;
276		u32 val;
277
278		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
279		if (rc)
280			return rc;
281		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
282
283		/* Abort success! */
284		if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
285		    !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
286			return 0;
287
288	} while (!time_after(jiffies, timeout_jiffies));
289
290	/* Abort has timed out and the MB is dead */
291	pci_err(pdev, "[%x] ABORT timed out\n", offset);
292	return -EIO;
293}
294
295static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
296			    struct pci_doe_task *task)
297{
298	struct pci_dev *pdev = doe_mb->pdev;
299	int offset = doe_mb->cap_offset;
300	unsigned long timeout_jiffies;
301	size_t length, remainder;
302	u32 val;
303	int i;
304
305	/*
306	 * Check the DOE busy bit is not set. If it is set, this could indicate
307	 * someone other than Linux (e.g. firmware) is using the mailbox. Note
308	 * it is expected that firmware and OS will negotiate access rights via
309	 * an, as yet to be defined, method.
310	 *
311	 * Wait up to one PCI_DOE_TIMEOUT period to allow the prior command to
312	 * finish. Otherwise, simply error out as unable to field the request.
313	 *
314	 * PCIe r6.2 sec 6.30.3 states no interrupt is raised when the DOE Busy
315	 * bit is cleared, so polling here is our best option for the moment.
316	 */
317	timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
318	do {
319		pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
320	} while (FIELD_GET(PCI_DOE_STATUS_BUSY, val) &&
321		 !time_after(jiffies, timeout_jiffies));
322
323	if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
324		return -EBUSY;
325
326	if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
327		return -EIO;
328
329	/* Length is 2 DW of header + length of payload in DW */
330	length = 2 + DIV_ROUND_UP(task->request_pl_sz, sizeof(__le32));
331	if (length > PCI_DOE_MAX_LENGTH)
332		return -EIO;
333	if (length == PCI_DOE_MAX_LENGTH)
334		length = 0;
335
336	/* Write DOE Header */
337	val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->feat.vid) |
338		FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->feat.type);
339	pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
340	pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
341			       FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
342					  length));
343
344	/* Write payload */
345	for (i = 0; i < task->request_pl_sz / sizeof(__le32); i++)
346		pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
347				       le32_to_cpu(task->request_pl[i]));
348
349	/* Write last payload dword */
350	remainder = task->request_pl_sz % sizeof(__le32);
351	if (remainder) {
352		val = 0;
353		memcpy(&val, &task->request_pl[i], remainder);
354		le32_to_cpus(&val);
355		pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
356	}
357
358	pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
359
360	return 0;
361}
362
363static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
364{
365	struct pci_dev *pdev = doe_mb->pdev;
366	int offset = doe_mb->cap_offset;
367	u32 val;
368
369	pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
370	if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
371		return true;
372	return false;
373}
374
375static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
376{
377	size_t length, payload_length, remainder, received;
378	struct pci_dev *pdev = doe_mb->pdev;
379	int offset = doe_mb->cap_offset;
380	int i = 0;
381	u32 val;
382
383	/* Read the first dword to get the feature */
384	pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
385	if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->feat.vid) ||
386	    (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->feat.type)) {
387		dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Feature] = [%04x, %02x], got [%04x, %02x]\n",
388				    doe_mb->cap_offset, task->feat.vid, task->feat.type,
389				    FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
390				    FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
391		return -EIO;
392	}
393
394	pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
395	/* Read the second dword to get the length */
396	pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
397	pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
398
399	length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
400	/* A value of 0x0 indicates max data object length */
401	if (!length)
402		length = PCI_DOE_MAX_LENGTH;
403	if (length < 2)
404		return -EIO;
405
406	/* First 2 dwords have already been read */
407	length -= 2;
408	received = task->response_pl_sz;
409	payload_length = DIV_ROUND_UP(task->response_pl_sz, sizeof(__le32));
410	remainder = task->response_pl_sz % sizeof(__le32);
411
412	/* remainder signifies number of data bytes in last payload dword */
413	if (!remainder)
414		remainder = sizeof(__le32);
415
416	if (length < payload_length) {
417		received = length * sizeof(__le32);
418		payload_length = length;
419		remainder = sizeof(__le32);
420	}
421
422	if (payload_length) {
423		/* Read all payload dwords except the last */
424		for (; i < payload_length - 1; i++) {
425			pci_read_config_dword(pdev, offset + PCI_DOE_READ,
426					      &val);
427			task->response_pl[i] = cpu_to_le32(val);
428			pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
429		}
430
431		/* Read last payload dword */
432		pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
433		cpu_to_le32s(&val);
434		memcpy(&task->response_pl[i], &val, remainder);
435		/* Prior to the last ack, ensure Data Object Ready */
436		if (!pci_doe_data_obj_ready(doe_mb))
437			return -EIO;
438		pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
439		i++;
440	}
441
442	/* Flush excess length */
443	for (; i < length; i++) {
444		pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
445		pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
446	}
447
448	/* Final error check to pick up on any since Data Object Ready */
449	pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
450	if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
451		return -EIO;
452
453	return received;
454}
455
456static void signal_task_complete(struct pci_doe_task *task, int rv)
457{
458	task->rv = rv;
459	destroy_work_on_stack(&task->work);
460	task->complete(task);
461}
462
463static void signal_task_abort(struct pci_doe_task *task, int rv)
464{
465	struct pci_doe_mb *doe_mb = task->doe_mb;
466	struct pci_dev *pdev = doe_mb->pdev;
467
468	if (pci_doe_abort(doe_mb)) {
469		/*
470		 * If the device can't process an abort; set the mailbox dead
471		 *	- no more submissions
472		 */
473		pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
474			doe_mb->cap_offset);
475		set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
476	}
477	signal_task_complete(task, rv);
478}
479
480static void doe_statemachine_work(struct work_struct *work)
481{
482	struct pci_doe_task *task = container_of(work, struct pci_doe_task,
483						 work);
484	struct pci_doe_mb *doe_mb = task->doe_mb;
485	struct pci_dev *pdev = doe_mb->pdev;
486	int offset = doe_mb->cap_offset;
487	unsigned long timeout_jiffies;
488	u32 val;
489	int rc;
490
491	if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
492		signal_task_complete(task, -EIO);
493		return;
494	}
495
496	/* Send request */
497	rc = pci_doe_send_req(doe_mb, task);
498	if (rc) {
499		/*
500		 * The specification does not provide any guidance on how to
501		 * resolve conflicting requests from other entities.
502		 * Furthermore, it is likely that busy will not be detected
503		 * most of the time.  Flag any detection of status busy with an
504		 * error.
505		 */
506		if (rc == -EBUSY)
507			dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
508					    offset);
509		signal_task_abort(task, rc);
510		return;
511	}
512
513	timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
514	/* Poll for response */
515retry_resp:
516	pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
517	if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
518		signal_task_abort(task, -EIO);
519		return;
520	}
521
522	if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
523		if (time_after(jiffies, timeout_jiffies)) {
524			signal_task_abort(task, -EIO);
525			return;
526		}
527		rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
528		if (rc) {
529			signal_task_abort(task, rc);
530			return;
531		}
532		goto retry_resp;
533	}
534
535	rc  = pci_doe_recv_resp(doe_mb, task);
536	if (rc < 0) {
537		signal_task_abort(task, rc);
538		return;
539	}
540
541	signal_task_complete(task, rc);
542}
543
544static void pci_doe_task_complete(struct pci_doe_task *task)
545{
546	complete(task->private);
547}
548
549static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 capver, u8 *index, u16 *vid,
550			     u8 *feature)
551{
552	u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
553				    *index) |
554			 FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_VER,
555				    (capver >= 2) ? 2 : 0);
556	__le32 request_pl_le = cpu_to_le32(request_pl);
557	__le32 response_pl_le;
558	u32 response_pl;
559	int rc;
560
561	rc = pci_doe(doe_mb, PCI_VENDOR_ID_PCI_SIG, PCI_DOE_FEATURE_DISCOVERY,
562		     &request_pl_le, sizeof(request_pl_le),
563		     &response_pl_le, sizeof(response_pl_le));
564	if (rc < 0)
565		return rc;
566
567	if (rc != sizeof(response_pl_le))
568		return -EIO;
569
570	response_pl = le32_to_cpu(response_pl_le);
571	*vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
572	*feature = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_TYPE,
573			      response_pl);
574	*index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
575			   response_pl);
576
577	return 0;
578}
579
580static void *pci_doe_xa_feat_entry(u16 vid, u8 type)
581{
582	return xa_mk_value((vid << 8) | type);
583}
584
585static int pci_doe_cache_features(struct pci_doe_mb *doe_mb)
586{
587	u8 index = 0;
588	u8 xa_idx = 0;
589	u32 hdr = 0;
590
591	pci_read_config_dword(doe_mb->pdev, doe_mb->cap_offset, &hdr);
592
593	do {
594		int rc;
595		u16 vid;
596		u8 type;
597
598		rc = pci_doe_discovery(doe_mb, PCI_EXT_CAP_VER(hdr), &index,
599				       &vid, &type);
600		if (rc)
601			return rc;
602
603		pci_dbg(doe_mb->pdev,
604			"[%x] Found feature %d vid: %x type: %x\n",
605			doe_mb->cap_offset, xa_idx, vid, type);
606
607		rc = xa_insert(&doe_mb->feats, xa_idx++,
608			       pci_doe_xa_feat_entry(vid, type), GFP_KERNEL);
609		if (rc)
610			return rc;
611	} while (index);
612
613	return 0;
614}
615
616static void pci_doe_cancel_tasks(struct pci_doe_mb *doe_mb)
617{
618	/* Stop all pending work items from starting */
619	set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
620
621	/* Cancel an in progress work item, if necessary */
622	set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
623	wake_up(&doe_mb->wq);
624}
625
626/**
627 * pci_doe_create_mb() - Create a DOE mailbox object
628 *
629 * @pdev: PCI device to create the DOE mailbox for
630 * @cap_offset: Offset of the DOE mailbox
631 *
632 * Create a single mailbox object to manage the mailbox feature at the
633 * cap_offset specified.
634 *
635 * RETURNS: created mailbox object on success
636 *	    ERR_PTR(-errno) on failure
637 */
638static struct pci_doe_mb *pci_doe_create_mb(struct pci_dev *pdev,
639					    u16 cap_offset)
640{
641	struct pci_doe_mb *doe_mb;
642	int rc;
643
644	doe_mb = kzalloc(sizeof(*doe_mb), GFP_KERNEL);
645	if (!doe_mb)
646		return ERR_PTR(-ENOMEM);
647
648	doe_mb->pdev = pdev;
649	doe_mb->cap_offset = cap_offset;
650	init_waitqueue_head(&doe_mb->wq);
651	xa_init(&doe_mb->feats);
652
653	doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
654						dev_bus_name(&pdev->dev),
655						pci_name(pdev),
656						doe_mb->cap_offset);
657	if (!doe_mb->work_queue) {
658		pci_err(pdev, "[%x] failed to allocate work queue\n",
659			doe_mb->cap_offset);
660		rc = -ENOMEM;
661		goto err_free;
662	}
663
664	/* Reset the mailbox by issuing an abort */
665	rc = pci_doe_abort(doe_mb);
666	if (rc) {
667		pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
668			doe_mb->cap_offset, rc);
669		goto err_destroy_wq;
670	}
671
672	/*
673	 * The state machine and the mailbox should be in sync now;
674	 * Use the mailbox to query features.
675	 */
676	rc = pci_doe_cache_features(doe_mb);
677	if (rc) {
678		pci_err(pdev, "[%x] failed to cache features : %d\n",
679			doe_mb->cap_offset, rc);
680		goto err_cancel;
681	}
682
683	return doe_mb;
684
685err_cancel:
686	pci_doe_cancel_tasks(doe_mb);
687	xa_destroy(&doe_mb->feats);
688err_destroy_wq:
689	destroy_workqueue(doe_mb->work_queue);
690err_free:
691	kfree(doe_mb);
692	return ERR_PTR(rc);
693}
694
695/**
696 * pci_doe_destroy_mb() - Destroy a DOE mailbox object
697 *
698 * @doe_mb: DOE mailbox
699 *
700 * Destroy all internal data structures created for the DOE mailbox.
701 */
702static void pci_doe_destroy_mb(struct pci_doe_mb *doe_mb)
703{
704	pci_doe_cancel_tasks(doe_mb);
705	xa_destroy(&doe_mb->feats);
706	destroy_workqueue(doe_mb->work_queue);
707	kfree(doe_mb);
708}
709
710/**
711 * pci_doe_supports_feat() - Return if the DOE instance supports the given
712 *			     feature
713 * @doe_mb: DOE mailbox capability to query
714 * @vid: Feature Vendor ID
715 * @type: Feature type
716 *
717 * RETURNS: True if the DOE mailbox supports the feature specified
718 */
719static bool pci_doe_supports_feat(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
720{
721	unsigned long index;
722	void *entry;
723
724	/* The discovery feature must always be supported */
725	if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_FEATURE_DISCOVERY)
726		return true;
727
728	xa_for_each(&doe_mb->feats, index, entry)
729		if (entry == pci_doe_xa_feat_entry(vid, type))
730			return true;
731
732	return false;
733}
734
735/**
736 * pci_doe_submit_task() - Submit a task to be processed by the state machine
737 *
738 * @doe_mb: DOE mailbox capability to submit to
739 * @task: task to be queued
740 *
741 * Submit a DOE task (request/response) to the DOE mailbox to be processed.
742 * Returns upon queueing the task object.  If the queue is full this function
743 * will sleep until there is room in the queue.
744 *
745 * task->complete will be called when the state machine is done processing this
746 * task.
747 *
748 * @task must be allocated on the stack.
749 *
750 * Excess data will be discarded.
751 *
752 * RETURNS: 0 when task has been successfully queued, -ERRNO on error
753 */
754static int pci_doe_submit_task(struct pci_doe_mb *doe_mb,
755			       struct pci_doe_task *task)
756{
757	if (!pci_doe_supports_feat(doe_mb, task->feat.vid, task->feat.type))
758		return -EINVAL;
759
760	if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
761		return -EIO;
762
763	task->doe_mb = doe_mb;
764	INIT_WORK_ONSTACK(&task->work, doe_statemachine_work);
765	queue_work(doe_mb->work_queue, &task->work);
766	return 0;
767}
768
769/**
770 * pci_doe() - Perform Data Object Exchange
771 *
772 * @doe_mb: DOE Mailbox
773 * @vendor: Vendor ID
774 * @type: Data Object Type
775 * @request: Request payload
776 * @request_sz: Size of request payload (bytes)
777 * @response: Response payload
778 * @response_sz: Size of response payload (bytes)
779 *
780 * Submit @request to @doe_mb and store the @response.
781 * The DOE exchange is performed synchronously and may therefore sleep.
782 *
783 * Payloads are treated as opaque byte streams which are transmitted verbatim,
784 * without byte-swapping.  If payloads contain little-endian register values,
785 * the caller is responsible for conversion with cpu_to_le32() / le32_to_cpu().
786 *
787 * For convenience, arbitrary payload sizes are allowed even though PCIe r6.0
788 * sec 6.30.1 specifies the Data Object Header 2 "Length" in dwords.  The last
789 * (partial) dword is copied with byte granularity and padded with zeroes if
790 * necessary.  Callers are thus relieved of using dword-sized bounce buffers.
791 *
792 * RETURNS: Length of received response or negative errno.
793 * Received data in excess of @response_sz is discarded.
794 * The length may be smaller than @response_sz and the caller
795 * is responsible for checking that.
796 */
797int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type,
798	    const void *request, size_t request_sz,
799	    void *response, size_t response_sz)
800{
801	DECLARE_COMPLETION_ONSTACK(c);
802	struct pci_doe_task task = {
803		.feat.vid = vendor,
804		.feat.type = type,
805		.request_pl = request,
806		.request_pl_sz = request_sz,
807		.response_pl = response,
808		.response_pl_sz = response_sz,
809		.complete = pci_doe_task_complete,
810		.private = &c,
811	};
812	int rc;
813
814	rc = pci_doe_submit_task(doe_mb, &task);
815	if (rc)
816		return rc;
817
818	wait_for_completion(&c);
819
820	return task.rv;
821}
822EXPORT_SYMBOL_GPL(pci_doe);
823
824/**
825 * pci_find_doe_mailbox() - Find Data Object Exchange mailbox
826 *
827 * @pdev: PCI device
828 * @vendor: Vendor ID
829 * @type: Data Object Type
830 *
831 * Find first DOE mailbox of a PCI device which supports the given feature.
832 *
833 * RETURNS: Pointer to the DOE mailbox or NULL if none was found.
834 */
835struct pci_doe_mb *pci_find_doe_mailbox(struct pci_dev *pdev, u16 vendor,
836					u8 type)
837{
838	struct pci_doe_mb *doe_mb;
839	unsigned long index;
840
841	xa_for_each(&pdev->doe_mbs, index, doe_mb)
842		if (pci_doe_supports_feat(doe_mb, vendor, type))
843			return doe_mb;
844
845	return NULL;
846}
847EXPORT_SYMBOL_GPL(pci_find_doe_mailbox);
848
849void pci_doe_init(struct pci_dev *pdev)
850{
851	struct pci_doe_mb *doe_mb;
852	u16 offset = 0;
853	int rc;
854
855	xa_init(&pdev->doe_mbs);
856
857	while ((offset = pci_find_next_ext_capability(pdev, offset,
858						      PCI_EXT_CAP_ID_DOE))) {
859		doe_mb = pci_doe_create_mb(pdev, offset);
860		if (IS_ERR(doe_mb)) {
861			pci_err(pdev, "[%x] failed to create mailbox: %ld\n",
862				offset, PTR_ERR(doe_mb));
863			continue;
864		}
865
866		rc = xa_insert(&pdev->doe_mbs, offset, doe_mb, GFP_KERNEL);
867		if (rc) {
868			pci_err(pdev, "[%x] failed to insert mailbox: %d\n",
869				offset, rc);
870			pci_doe_destroy_mb(doe_mb);
871		}
872	}
873}
874
875void pci_doe_destroy(struct pci_dev *pdev)
876{
877	struct pci_doe_mb *doe_mb;
878	unsigned long index;
879
880	xa_for_each(&pdev->doe_mbs, index, doe_mb)
881		pci_doe_destroy_mb(doe_mb);
882
883	xa_destroy(&pdev->doe_mbs);
884}
885
886void pci_doe_disconnected(struct pci_dev *pdev)
887{
888	struct pci_doe_mb *doe_mb;
889	unsigned long index;
890
891	xa_for_each(&pdev->doe_mbs, index, doe_mb)
892		pci_doe_cancel_tasks(doe_mb);
893}