efa_main.c

  1// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
  2/*
  3 * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
  4 */
  5
  6#include <linux/module.h>
  7#include <linux/pci.h>
  8#include <linux/utsname.h>
  9#include <linux/version.h>
 10
 11#include <rdma/ib_user_verbs.h>
 12#include <rdma/uverbs_ioctl.h>
 13
 14#include "efa.h"
 15
 16#define PCI_DEV_ID_EFA0_VF 0xefa0
 17#define PCI_DEV_ID_EFA1_VF 0xefa1
 18#define PCI_DEV_ID_EFA2_VF 0xefa2
 19#define PCI_DEV_ID_EFA3_VF 0xefa3
 20
 21static const struct pci_device_id efa_pci_tbl[] = {
 22	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) },
 23	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) },
 24	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA2_VF) },
 25	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA3_VF) },
 26	{ }
 27};
 28
 29MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
 30MODULE_LICENSE("Dual BSD/GPL");
 31MODULE_DESCRIPTION(DEVICE_NAME);
 32MODULE_DEVICE_TABLE(pci, efa_pci_tbl);
 33
 34#define EFA_REG_BAR 0
 35#define EFA_MEM_BAR 2
 36#define EFA_BASE_BAR_MASK (BIT(EFA_REG_BAR) | BIT(EFA_MEM_BAR))
 37
 38#define EFA_AENQ_ENABLED_GROUPS \
 39	(BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \
 40	 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))
 41
 42extern const struct uapi_definition efa_uapi_defs[];
 43
 44/* This handler will called for unknown event group or unimplemented handlers */
 45static void unimplemented_aenq_handler(void *data,
 46				       struct efa_admin_aenq_entry *aenq_e)
 47{
 48	struct efa_dev *dev = (struct efa_dev *)data;
 49
 50	ibdev_err(&dev->ibdev,
 51		  "Unknown event was received or event with unimplemented handler\n");
 52}
 53
 54static void efa_keep_alive(void *data, struct efa_admin_aenq_entry *aenq_e)
 55{
 56	struct efa_dev *dev = (struct efa_dev *)data;
 57
 58	atomic64_inc(&dev->stats.keep_alive_rcvd);
 59}
 60
 61static struct efa_aenq_handlers aenq_handlers = {
 62	.handlers = {
 63		[EFA_ADMIN_KEEP_ALIVE] = efa_keep_alive,
 64	},
 65	.unimplemented_handler = unimplemented_aenq_handler
 66};
 67
 68static void efa_release_bars(struct efa_dev *dev, int bars_mask)
 69{
 70	struct pci_dev *pdev = dev->pdev;
 71	int release_bars;
 72
 73	release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & bars_mask;
 74	pci_release_selected_regions(pdev, release_bars);
 75}
 76
 77static void efa_process_comp_eqe(struct efa_dev *dev, struct efa_admin_eqe *eqe)
 78{
 79	u16 cqn = eqe->u.comp_event.cqn;
 80	struct efa_cq *cq;
 81
 82	/* Safe to load as we're in irq and removal calls synchronize_irq() */
 83	cq = xa_load(&dev->cqs_xa, cqn);
 84	if (unlikely(!cq)) {
 85		ibdev_err_ratelimited(&dev->ibdev,
 86				      "Completion event on non-existent CQ[%u]",
 87				      cqn);
 88		return;
 89	}
 90
 91	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 92}
 93
 94static void efa_process_eqe(struct efa_com_eq *eeq, struct efa_admin_eqe *eqe)
 95{
 96	struct efa_dev *dev = container_of(eeq->edev, struct efa_dev, edev);
 97
 98	if (likely(EFA_GET(&eqe->common, EFA_ADMIN_EQE_EVENT_TYPE) ==
 99			   EFA_ADMIN_EQE_EVENT_TYPE_COMPLETION))
100		efa_process_comp_eqe(dev, eqe);
101	else
102		ibdev_err_ratelimited(&dev->ibdev,
103				      "Unknown event type received %lu",
104				      EFA_GET(&eqe->common,
105					      EFA_ADMIN_EQE_EVENT_TYPE));
106}
107
108static irqreturn_t efa_intr_msix_comp(int irq, void *data)
109{
110	struct efa_eq *eq = data;
111	struct efa_com_dev *edev = eq->eeq.edev;
112
113	efa_com_eq_comp_intr_handler(edev, &eq->eeq);
114
115	return IRQ_HANDLED;
116}
117
118static irqreturn_t efa_intr_msix_mgmnt(int irq, void *data)
119{
120	struct efa_dev *dev = data;
121
122	efa_com_admin_q_comp_intr_handler(&dev->edev);
123	efa_com_aenq_intr_handler(&dev->edev, data);
124
125	return IRQ_HANDLED;
126}
127
128static int efa_request_irq(struct efa_dev *dev, struct efa_irq *irq)
129{
130	int err;
131
132	err = request_irq(irq->irqn, irq->handler, 0, irq->name, irq->data);
133	if (err) {
134		dev_err(&dev->pdev->dev, "Failed to request irq %s (%d)\n",
135			irq->name, err);
136		return err;
137	}
138
139	irq_set_affinity_hint(irq->irqn, &irq->affinity_hint_mask);
140
141	return 0;
142}
143
144static void efa_setup_comp_irq(struct efa_dev *dev, struct efa_eq *eq, u32 vector)
145{
146	u32 cpu;
147
148	cpu = vector - EFA_COMP_EQS_VEC_BASE;
149	snprintf(eq->irq.name, EFA_IRQNAME_SIZE, "efa-comp%d@pci:%s", cpu,
150		 pci_name(dev->pdev));
151	eq->irq.handler = efa_intr_msix_comp;
152	eq->irq.data = eq;
153	eq->irq.vector = vector;
154	eq->irq.irqn = pci_irq_vector(dev->pdev, vector);
155	cpumask_set_cpu(cpu, &eq->irq.affinity_hint_mask);
156}
157
158static void efa_free_irq(struct efa_dev *dev, struct efa_irq *irq)
159{
160	irq_set_affinity_hint(irq->irqn, NULL);
161	free_irq(irq->irqn, irq->data);
162}
163
164static void efa_setup_mgmnt_irq(struct efa_dev *dev)
165{
166	u32 cpu;
167
168	snprintf(dev->admin_irq.name, EFA_IRQNAME_SIZE,
169		 "efa-mgmnt@pci:%s", pci_name(dev->pdev));
170	dev->admin_irq.handler = efa_intr_msix_mgmnt;
171	dev->admin_irq.data = dev;
172	dev->admin_irq.vector = dev->admin_msix_vector_idx;
173	dev->admin_irq.irqn = pci_irq_vector(dev->pdev,
174					     dev->admin_msix_vector_idx);
175	cpu = cpumask_first(cpu_online_mask);
176	cpumask_set_cpu(cpu,
177			&dev->admin_irq.affinity_hint_mask);
178	dev_info(&dev->pdev->dev, "Setup irq:%d name:%s\n",
179		 dev->admin_irq.irqn,
180		 dev->admin_irq.name);
181}
182
183static int efa_set_mgmnt_irq(struct efa_dev *dev)
184{
185	efa_setup_mgmnt_irq(dev);
186
187	return efa_request_irq(dev, &dev->admin_irq);
188}
189
190static int efa_request_doorbell_bar(struct efa_dev *dev)
191{
192	u8 db_bar_idx = dev->dev_attr.db_bar;
193	struct pci_dev *pdev = dev->pdev;
194	int pci_mem_bars;
195	int db_bar;
196	int err;
197
198	db_bar = BIT(db_bar_idx);
199	if (!(db_bar & EFA_BASE_BAR_MASK)) {
200		pci_mem_bars = pci_select_bars(pdev, IORESOURCE_MEM);
201		if (db_bar & ~pci_mem_bars) {
202			dev_err(&pdev->dev,
203				"Doorbells BAR unavailable. Requested %#x, available %#x\n",
204				db_bar, pci_mem_bars);
205			return -ENODEV;
206		}
207
208		err = pci_request_selected_regions(pdev, db_bar, DRV_MODULE_NAME);
209		if (err) {
210			dev_err(&pdev->dev,
211				"pci_request_selected_regions for bar %d failed %d\n",
212				db_bar_idx, err);
213			return err;
214		}
215	}
216
217	dev->db_bar_addr = pci_resource_start(dev->pdev, db_bar_idx);
218	dev->db_bar_len = pci_resource_len(dev->pdev, db_bar_idx);
219
220	return 0;
221}
222
223static void efa_release_doorbell_bar(struct efa_dev *dev)
224{
225	if (!(BIT(dev->dev_attr.db_bar) & EFA_BASE_BAR_MASK))
226		efa_release_bars(dev, BIT(dev->dev_attr.db_bar));
227}
228
229static void efa_update_hw_hints(struct efa_dev *dev,
230				struct efa_com_get_hw_hints_result *hw_hints)
231{
232	struct efa_com_dev *edev = &dev->edev;
233
234	if (hw_hints->mmio_read_timeout)
235		edev->mmio_read.mmio_read_timeout =
236			hw_hints->mmio_read_timeout * 1000;
237
238	if (hw_hints->poll_interval)
239		edev->aq.poll_interval = hw_hints->poll_interval;
240
241	if (hw_hints->admin_completion_timeout)
242		edev->aq.completion_timeout =
243			hw_hints->admin_completion_timeout;
244}
245
246static void efa_stats_init(struct efa_dev *dev)
247{
248	atomic64_t *s = (atomic64_t *)&dev->stats;
249	int i;
250
251	for (i = 0; i < sizeof(dev->stats) / sizeof(*s); i++, s++)
252		atomic64_set(s, 0);
253}
254
255static void efa_set_host_info(struct efa_dev *dev)
256{
257	struct efa_admin_set_feature_resp resp = {};
258	struct efa_admin_set_feature_cmd cmd = {};
259	struct efa_admin_host_info *hinf;
260	u32 bufsz = sizeof(*hinf);
261	dma_addr_t hinf_dma;
262
263	if (!efa_com_check_supported_feature_id(&dev->edev,
264						EFA_ADMIN_HOST_INFO))
265		return;
266
267	/* Failures in host info set shall not disturb probe */
268	hinf = dma_alloc_coherent(&dev->pdev->dev, bufsz, &hinf_dma,
269				  GFP_KERNEL);
270	if (!hinf)
271		return;
272
273	strscpy(hinf->os_dist_str, utsname()->release,
274		sizeof(hinf->os_dist_str));
275	hinf->os_type = EFA_ADMIN_OS_LINUX;
276	strscpy(hinf->kernel_ver_str, utsname()->version,
277		sizeof(hinf->kernel_ver_str));
278	hinf->kernel_ver = LINUX_VERSION_CODE;
279	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MAJOR, 0);
280	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MINOR, 0);
281	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_SUB_MINOR, 0);
282	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE, 0);
283	EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_BUS, dev->pdev->bus->number);
284	EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_DEVICE,
285		PCI_SLOT(dev->pdev->devfn));
286	EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_FUNCTION,
287		PCI_FUNC(dev->pdev->devfn));
288	EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MAJOR,
289		EFA_COMMON_SPEC_VERSION_MAJOR);
290	EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MINOR,
291		EFA_COMMON_SPEC_VERSION_MINOR);
292	EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_INTREE, 1);
293	EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_GDR, 0);
294
295	efa_com_set_feature_ex(&dev->edev, &resp, &cmd, EFA_ADMIN_HOST_INFO,
296			       hinf_dma, bufsz);
297
298	dma_free_coherent(&dev->pdev->dev, bufsz, hinf, hinf_dma);
299}
300
301static void efa_destroy_eq(struct efa_dev *dev, struct efa_eq *eq)
302{
303	efa_com_eq_destroy(&dev->edev, &eq->eeq);
304	efa_free_irq(dev, &eq->irq);
305}
306
307static int efa_create_eq(struct efa_dev *dev, struct efa_eq *eq, u32 msix_vec)
308{
309	int err;
310
311	efa_setup_comp_irq(dev, eq, msix_vec);
312	err = efa_request_irq(dev, &eq->irq);
313	if (err)
314		return err;
315
316	err = efa_com_eq_init(&dev->edev, &eq->eeq, efa_process_eqe,
317			      dev->dev_attr.max_eq_depth, msix_vec);
318	if (err)
319		goto err_free_comp_irq;
320
321	return 0;
322
323err_free_comp_irq:
324	efa_free_irq(dev, &eq->irq);
325	return err;
326}
327
328static int efa_create_eqs(struct efa_dev *dev)
329{
330	u32 neqs = dev->dev_attr.max_eq;
331	int err, i;
332
333	neqs = min_t(u32, neqs, dev->num_irq_vectors - EFA_COMP_EQS_VEC_BASE);
334	dev->neqs = neqs;
335	dev->eqs = kcalloc(neqs, sizeof(*dev->eqs), GFP_KERNEL);
336	if (!dev->eqs)
337		return -ENOMEM;
338
339	for (i = 0; i < neqs; i++) {
340		err = efa_create_eq(dev, &dev->eqs[i], i + EFA_COMP_EQS_VEC_BASE);
341		if (err)
342			goto err_destroy_eqs;
343	}
344
345	return 0;
346
347err_destroy_eqs:
348	for (i--; i >= 0; i--)
349		efa_destroy_eq(dev, &dev->eqs[i]);
350	kfree(dev->eqs);
351
352	return err;
353}
354
355static void efa_destroy_eqs(struct efa_dev *dev)
356{
357	int i;
358
359	for (i = 0; i < dev->neqs; i++)
360		efa_destroy_eq(dev, &dev->eqs[i]);
361
362	kfree(dev->eqs);
363}
364
365static const struct ib_device_ops efa_dev_ops = {
366	.owner = THIS_MODULE,
367	.driver_id = RDMA_DRIVER_EFA,
368	.uverbs_abi_ver = EFA_UVERBS_ABI_VERSION,
369
370	.alloc_hw_port_stats = efa_alloc_hw_port_stats,
371	.alloc_hw_device_stats = efa_alloc_hw_device_stats,
372	.alloc_pd = efa_alloc_pd,
373	.alloc_ucontext = efa_alloc_ucontext,
374	.create_cq = efa_create_cq,
375	.create_cq_umem = efa_create_cq_umem,
376	.create_qp = efa_create_qp,
377	.create_user_ah = efa_create_ah,
378	.dealloc_pd = efa_dealloc_pd,
379	.dealloc_ucontext = efa_dealloc_ucontext,
380	.dereg_mr = efa_dereg_mr,
381	.destroy_ah = efa_destroy_ah,
382	.destroy_cq = efa_destroy_cq,
383	.destroy_qp = efa_destroy_qp,
384	.get_hw_stats = efa_get_hw_stats,
385	.get_link_layer = efa_port_link_layer,
386	.get_port_immutable = efa_get_port_immutable,
387	.mmap = efa_mmap,
388	.mmap_free = efa_mmap_free,
389	.modify_qp = efa_modify_qp,
390	.query_device = efa_query_device,
391	.query_gid = efa_query_gid,
392	.query_pkey = efa_query_pkey,
393	.query_port = efa_query_port,
394	.query_qp = efa_query_qp,
395	.reg_user_mr = efa_reg_mr,
396	.reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf,
397
398	INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),
399	INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),
400	INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),
401	INIT_RDMA_OBJ_SIZE(ib_qp, efa_qp, ibqp),
402	INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext),
403};
404
405static int efa_ib_device_add(struct efa_dev *dev)
406{
407	struct efa_com_get_hw_hints_result hw_hints;
408	struct pci_dev *pdev = dev->pdev;
409	int err;
410
411	efa_stats_init(dev);
412
413	err = efa_com_get_device_attr(&dev->edev, &dev->dev_attr);
414	if (err)
415		return err;
416
417	dev_dbg(&dev->pdev->dev, "Doorbells bar (%d)\n", dev->dev_attr.db_bar);
418	err = efa_request_doorbell_bar(dev);
419	if (err)
420		return err;
421
422	err = efa_com_get_hw_hints(&dev->edev, &hw_hints);
423	if (err)
424		goto err_release_doorbell_bar;
425
426	efa_update_hw_hints(dev, &hw_hints);
427
428	/* Try to enable all the available aenq groups */
429	err = efa_com_set_aenq_config(&dev->edev, EFA_AENQ_ENABLED_GROUPS);
430	if (err)
431		goto err_release_doorbell_bar;
432
433	err = efa_create_eqs(dev);
434	if (err)
435		goto err_release_doorbell_bar;
436
437	efa_set_host_info(dev);
438
439	dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED;
440	dev->ibdev.node_guid = dev->dev_attr.guid;
441	dev->ibdev.phys_port_cnt = 1;
442	dev->ibdev.num_comp_vectors = dev->neqs ?: 1;
443	dev->ibdev.dev.parent = &pdev->dev;
444
445	ib_set_device_ops(&dev->ibdev, &efa_dev_ops);
446
447	dev->ibdev.driver_def = efa_uapi_defs;
448
449	err = ib_register_device(&dev->ibdev, "efa_%d", &pdev->dev);
450	if (err)
451		goto err_destroy_eqs;
452
453	ibdev_info(&dev->ibdev, "IB device registered\n");
454
455	return 0;
456
457err_destroy_eqs:
458	efa_destroy_eqs(dev);
459err_release_doorbell_bar:
460	efa_release_doorbell_bar(dev);
461	return err;
462}
463
464static void efa_ib_device_remove(struct efa_dev *dev)
465{
466	ibdev_info(&dev->ibdev, "Unregister ib device\n");
467	ib_unregister_device(&dev->ibdev);
468	efa_destroy_eqs(dev);
469	efa_release_doorbell_bar(dev);
470}
471
472static void efa_disable_msix(struct efa_dev *dev)
473{
474	pci_free_irq_vectors(dev->pdev);
475}
476
477static int efa_enable_msix(struct efa_dev *dev)
478{
479	int max_vecs, num_vecs;
480
481	/*
482	 * Reserve the max msix vectors we might need, one vector is reserved
483	 * for admin.
484	 */
485	max_vecs = min_t(int, pci_msix_vec_count(dev->pdev),
486			 num_online_cpus() + 1);
487	dev_dbg(&dev->pdev->dev, "Trying to enable MSI-X, vectors %d\n",
488		max_vecs);
489
490	dev->admin_msix_vector_idx = EFA_MGMNT_MSIX_VEC_IDX;
491	num_vecs = pci_alloc_irq_vectors(dev->pdev, 1,
492					 max_vecs, PCI_IRQ_MSIX);
493
494	if (num_vecs < 0) {
495		dev_err(&dev->pdev->dev, "Failed to enable MSI-X. error %d\n",
496			num_vecs);
497		return -ENOSPC;
498	}
499
500	dev_dbg(&dev->pdev->dev, "Allocated %d MSI-X vectors\n", num_vecs);
501
502	dev->num_irq_vectors = num_vecs;
503
504	return 0;
505}
506
507static int efa_device_init(struct efa_com_dev *edev, struct pci_dev *pdev)
508{
509	int dma_width;
510	int err;
511
512	err = efa_com_dev_reset(edev, EFA_REGS_RESET_NORMAL);
513	if (err)
514		return err;
515
516	err = efa_com_validate_version(edev);
517	if (err)
518		return err;
519
520	dma_width = efa_com_get_dma_width(edev);
521	if (dma_width < 0) {
522		err = dma_width;
523		return err;
524	}
525
526	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_width));
527	if (err) {
528		dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", err);
529		return err;
530	}
531
532	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
533	return 0;
534}
535
536static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
537{
538	struct efa_com_dev *edev;
539	struct efa_dev *dev;
540	int pci_mem_bars;
541	int err;
542
543	err = pci_enable_device_mem(pdev);
544	if (err) {
545		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
546		return ERR_PTR(err);
547	}
548
549	pci_set_master(pdev);
550
551	dev = ib_alloc_device(efa_dev, ibdev);
552	if (!dev) {
553		dev_err(&pdev->dev, "Device alloc failed\n");
554		err = -ENOMEM;
555		goto err_disable_device;
556	}
557
558	pci_set_drvdata(pdev, dev);
559	edev = &dev->edev;
560	edev->efa_dev = dev;
561	edev->dmadev = &pdev->dev;
562	dev->pdev = pdev;
563	xa_init(&dev->cqs_xa);
564
565	pci_mem_bars = pci_select_bars(pdev, IORESOURCE_MEM);
566	if (EFA_BASE_BAR_MASK & ~pci_mem_bars) {
567		dev_err(&pdev->dev, "BARs unavailable. Requested %#x, available %#x\n",
568			(int)EFA_BASE_BAR_MASK, pci_mem_bars);
569		err = -ENODEV;
570		goto err_ibdev_destroy;
571	}
572	err = pci_request_selected_regions(pdev, EFA_BASE_BAR_MASK, DRV_MODULE_NAME);
573	if (err) {
574		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
575			err);
576		goto err_ibdev_destroy;
577	}
578
579	dev->reg_bar_addr = pci_resource_start(pdev, EFA_REG_BAR);
580	dev->reg_bar_len = pci_resource_len(pdev, EFA_REG_BAR);
581	dev->mem_bar_addr = pci_resource_start(pdev, EFA_MEM_BAR);
582	dev->mem_bar_len = pci_resource_len(pdev, EFA_MEM_BAR);
583
584	edev->reg_bar = devm_ioremap(&pdev->dev,
585				     dev->reg_bar_addr,
586				     dev->reg_bar_len);
587	if (!edev->reg_bar) {
588		dev_err(&pdev->dev, "Failed to remap register bar\n");
589		err = -EFAULT;
590		goto err_release_bars;
591	}
592
593	err = efa_com_mmio_reg_read_init(edev);
594	if (err) {
595		dev_err(&pdev->dev, "Failed to init readless MMIO\n");
596		goto err_iounmap;
597	}
598
599	err = efa_device_init(edev, pdev);
600	if (err) {
601		dev_err(&pdev->dev, "EFA device init failed\n");
602		if (err == -ETIME)
603			err = -EPROBE_DEFER;
604		goto err_reg_read_destroy;
605	}
606
607	err = efa_enable_msix(dev);
608	if (err)
609		goto err_reg_read_destroy;
610
611	edev->aq.msix_vector_idx = dev->admin_msix_vector_idx;
612	edev->aenq.msix_vector_idx = dev->admin_msix_vector_idx;
613
614	err = efa_set_mgmnt_irq(dev);
615	if (err)
616		goto err_disable_msix;
617
618	err = efa_com_admin_init(edev, &aenq_handlers);
619	if (err)
620		goto err_free_mgmnt_irq;
621
622	return dev;
623
624err_free_mgmnt_irq:
625	efa_free_irq(dev, &dev->admin_irq);
626err_disable_msix:
627	efa_disable_msix(dev);
628err_reg_read_destroy:
629	efa_com_mmio_reg_read_destroy(edev);
630err_iounmap:
631	devm_iounmap(&pdev->dev, edev->reg_bar);
632err_release_bars:
633	efa_release_bars(dev, EFA_BASE_BAR_MASK);
634err_ibdev_destroy:
635	ib_dealloc_device(&dev->ibdev);
636err_disable_device:
637	pci_disable_device(pdev);
638	return ERR_PTR(err);
639}
640
641static void efa_remove_device(struct pci_dev *pdev,
642			      enum efa_regs_reset_reason_types reset_reason)
643{
644	struct efa_dev *dev = pci_get_drvdata(pdev);
645	struct efa_com_dev *edev;
646
647	edev = &dev->edev;
648	efa_com_dev_reset(edev, reset_reason);
649	efa_com_admin_destroy(edev);
650	efa_free_irq(dev, &dev->admin_irq);
651	efa_disable_msix(dev);
652	efa_com_mmio_reg_read_destroy(edev);
653	devm_iounmap(&pdev->dev, edev->reg_bar);
654	efa_release_bars(dev, EFA_BASE_BAR_MASK);
655	xa_destroy(&dev->cqs_xa);
656	ib_dealloc_device(&dev->ibdev);
657	pci_disable_device(pdev);
658}
659
660static int efa_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
661{
662	struct efa_dev *dev;
663	int err;
664
665	dev = efa_probe_device(pdev);
666	if (IS_ERR(dev))
667		return PTR_ERR(dev);
668
669	err = efa_ib_device_add(dev);
670	if (err)
671		goto err_remove_device;
672
673	return 0;
674
675err_remove_device:
676	efa_remove_device(pdev, EFA_REGS_RESET_INIT_ERR);
677	return err;
678}
679
680static void efa_remove(struct pci_dev *pdev)
681{
682	struct efa_dev *dev = pci_get_drvdata(pdev);
683
684	efa_ib_device_remove(dev);
685	efa_remove_device(pdev, EFA_REGS_RESET_NORMAL);
686}
687
688static void efa_shutdown(struct pci_dev *pdev)
689{
690	struct efa_dev *dev = pci_get_drvdata(pdev);
691
692	efa_destroy_eqs(dev);
693	efa_com_dev_reset(&dev->edev, EFA_REGS_RESET_SHUTDOWN);
694	efa_free_irq(dev, &dev->admin_irq);
695	efa_disable_msix(dev);
696}
697
698static struct pci_driver efa_pci_driver = {
699	.name           = DRV_MODULE_NAME,
700	.id_table       = efa_pci_tbl,
701	.probe          = efa_probe,
702	.remove         = efa_remove,
703	.shutdown       = efa_shutdown,
704};
705
706module_pci_driver(efa_pci_driver);