erdma_main.c

  1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2
  3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
  4/*          Kai Shen <kaishen@linux.alibaba.com> */
  5/* Copyright (c) 2020-2022, Alibaba Group. */
  6
  7#include <linux/module.h>
  8#include <net/addrconf.h>
  9#include <rdma/erdma-abi.h>
 10
 11#include "erdma.h"
 12#include "erdma_cm.h"
 13#include "erdma_verbs.h"
 14
 15MODULE_AUTHOR("Cheng Xu <chengyou@linux.alibaba.com>");
 16MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver");
 17MODULE_LICENSE("Dual BSD/GPL");
 18
 19static int erdma_netdev_event(struct notifier_block *nb, unsigned long event,
 20			      void *arg)
 21{
 22	struct net_device *netdev = netdev_notifier_info_to_dev(arg);
 23	struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb);
 24
 25	if (dev->netdev == NULL || dev->netdev != netdev)
 26		goto done;
 27
 28	switch (event) {
 29	case NETDEV_CHANGEMTU:
 30		if (dev->mtu != netdev->mtu) {
 31			erdma_set_mtu(dev, netdev->mtu);
 32			dev->mtu = netdev->mtu;
 33		}
 34		break;
 35	case NETDEV_REGISTER:
 36	case NETDEV_UNREGISTER:
 37	case NETDEV_CHANGEADDR:
 38	case NETDEV_GOING_DOWN:
 39	case NETDEV_CHANGE:
 40	default:
 41		break;
 42	}
 43
 44done:
 45	return NOTIFY_OK;
 46}
 47
 48static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
 49{
 50	struct net_device *netdev;
 51	int ret = -EPROBE_DEFER;
 52
 53	/* Already binded to a net_device, so we skip. */
 54	if (dev->netdev)
 55		return 0;
 56
 57	rtnl_lock();
 58	for_each_netdev(&init_net, netdev) {
 59		/*
 60		 * In erdma, the paired netdev and ibdev should have the same
 61		 * MAC address. erdma can get the value from its PCIe bar
 62		 * registers. Since erdma can not get the paired netdev
 63		 * reference directly, we do a traverse here to get the paired
 64		 * netdev.
 65		 */
 66		if (ether_addr_equal_unaligned(netdev->perm_addr,
 67					       dev->attrs.peer_addr)) {
 68			ret = ib_device_set_netdev(&dev->ibdev, netdev, 1);
 69			if (ret) {
 70				rtnl_unlock();
 71				ibdev_warn(&dev->ibdev,
 72					   "failed (%d) to link netdev", ret);
 73				return ret;
 74			}
 75
 76			dev->netdev = netdev;
 77			break;
 78		}
 79	}
 80
 81	rtnl_unlock();
 82
 83	return ret;
 84}
 85
 86static int erdma_device_register(struct erdma_dev *dev)
 87{
 88	struct ib_device *ibdev = &dev->ibdev;
 89	int ret;
 90
 91	ret = erdma_enum_and_get_netdev(dev);
 92	if (ret)
 93		return ret;
 94
 95	dev->mtu = dev->netdev->mtu;
 96	addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr);
 97
 98	ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev);
 99	if (ret) {
100		dev_err(&dev->pdev->dev,
101			"ib_register_device failed: ret = %d\n", ret);
102		return ret;
103	}
104
105	dev->netdev_nb.notifier_call = erdma_netdev_event;
106	ret = register_netdevice_notifier(&dev->netdev_nb);
107	if (ret) {
108		ibdev_err(&dev->ibdev, "failed to register notifier.\n");
109		ib_unregister_device(ibdev);
110	}
111
112	return ret;
113}
114
115static irqreturn_t erdma_comm_irq_handler(int irq, void *data)
116{
117	struct erdma_dev *dev = data;
118
119	erdma_cmdq_completion_handler(&dev->cmdq);
120	erdma_aeq_event_handler(dev);
121
122	return IRQ_HANDLED;
123}
124
125static int erdma_request_vectors(struct erdma_dev *dev)
126{
127	int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC);
128	int ret;
129
130	ret = pci_alloc_irq_vectors(dev->pdev, 1, expect_irq_num, PCI_IRQ_MSIX);
131	if (ret < 0) {
132		dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n",
133			ret);
134		return ret;
135	}
136	dev->attrs.irq_num = ret;
137
138	return 0;
139}
140
141static int erdma_comm_irq_init(struct erdma_dev *dev)
142{
143	snprintf(dev->comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s",
144		 pci_name(dev->pdev));
145	dev->comm_irq.msix_vector =
146		pci_irq_vector(dev->pdev, ERDMA_MSIX_VECTOR_CMDQ);
147
148	cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev->pdev->bus)),
149			&dev->comm_irq.affinity_hint_mask);
150	irq_set_affinity_hint(dev->comm_irq.msix_vector,
151			      &dev->comm_irq.affinity_hint_mask);
152
153	return request_irq(dev->comm_irq.msix_vector, erdma_comm_irq_handler, 0,
154			   dev->comm_irq.name, dev);
155}
156
157static void erdma_comm_irq_uninit(struct erdma_dev *dev)
158{
159	irq_set_affinity_hint(dev->comm_irq.msix_vector, NULL);
160	free_irq(dev->comm_irq.msix_vector, dev);
161}
162
163static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
164{
165	int ret;
166
167	dev->proto = erdma_reg_read32(dev, ERDMA_REGS_DEV_PROTO_REG);
168
169	dev->resp_pool = dma_pool_create("erdma_resp_pool", &pdev->dev,
170					 ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE,
171					 0);
172	if (!dev->resp_pool)
173		return -ENOMEM;
174
175	dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev,
176				       ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0);
177	if (!dev->db_pool) {
178		ret = -ENOMEM;
179		goto destroy_resp_pool;
180	}
181
182	ret = dma_set_mask_and_coherent(&pdev->dev,
183					DMA_BIT_MASK(ERDMA_PCI_WIDTH));
184	if (ret)
185		goto destroy_db_pool;
186
187	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
188
189	return 0;
190
191destroy_db_pool:
192	dma_pool_destroy(dev->db_pool);
193
194destroy_resp_pool:
195	dma_pool_destroy(dev->resp_pool);
196
197	return ret;
198}
199
200static void erdma_device_uninit(struct erdma_dev *dev)
201{
202	dma_pool_destroy(dev->db_pool);
203	dma_pool_destroy(dev->resp_pool);
204}
205
206static void erdma_hw_reset(struct erdma_dev *dev)
207{
208	u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);
209
210	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
211}
212
213static int erdma_wait_hw_init_done(struct erdma_dev *dev)
214{
215	int i;
216
217	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,
218			  FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));
219
220	for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
221		if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
222					   ERDMA_REG_DEV_ST_INIT_DONE_MASK))
223			break;
224
225		msleep(ERDMA_REG_ACCESS_WAIT_MS);
226	}
227
228	if (i == ERDMA_WAIT_DEV_DONE_CNT) {
229		dev_err(&dev->pdev->dev, "wait init done failed.\n");
230		return -ETIMEDOUT;
231	}
232
233	return 0;
234}
235
236static const struct pci_device_id erdma_pci_tbl[] = {
237	{ PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },
238	{}
239};
240
241static int erdma_probe_dev(struct pci_dev *pdev)
242{
243	struct erdma_dev *dev;
244	int bars, err;
245	u32 version;
246
247	err = pci_enable_device(pdev);
248	if (err) {
249		dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err);
250		return err;
251	}
252
253	pci_set_master(pdev);
254
255	dev = ib_alloc_device(erdma_dev, ibdev);
256	if (!dev) {
257		dev_err(&pdev->dev, "ib_alloc_device failed\n");
258		err = -ENOMEM;
259		goto err_disable_device;
260	}
261
262	pci_set_drvdata(pdev, dev);
263	dev->pdev = pdev;
264	dev->attrs.numa_node = dev_to_node(&pdev->dev);
265
266	bars = pci_select_bars(pdev, IORESOURCE_MEM);
267	err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
268	if (bars != ERDMA_BAR_MASK || err) {
269		err = err ? err : -EINVAL;
270		goto err_ib_device_release;
271	}
272
273	dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR);
274	dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR);
275
276	dev->func_bar =
277		devm_ioremap(&pdev->dev, dev->func_bar_addr, dev->func_bar_len);
278	if (!dev->func_bar) {
279		dev_err(&pdev->dev, "devm_ioremap failed.\n");
280		err = -EFAULT;
281		goto err_release_bars;
282	}
283
284	version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG);
285	if (version == 0) {
286		/* we knows that it is a non-functional function. */
287		err = -ENODEV;
288		goto err_iounmap_func_bar;
289	}
290
291	err = erdma_device_init(dev, pdev);
292	if (err)
293		goto err_iounmap_func_bar;
294
295	err = erdma_request_vectors(dev);
296	if (err)
297		goto err_uninit_device;
298
299	err = erdma_comm_irq_init(dev);
300	if (err)
301		goto err_free_vectors;
302
303	err = erdma_aeq_init(dev);
304	if (err)
305		goto err_uninit_comm_irq;
306
307	err = erdma_cmdq_init(dev);
308	if (err)
309		goto err_uninit_aeq;
310
311	err = erdma_wait_hw_init_done(dev);
312	if (err)
313		goto err_uninit_cmdq;
314
315	err = erdma_ceqs_init(dev);
316	if (err)
317		goto err_reset_hw;
318
319	erdma_finish_cmdq_init(dev);
320
321	return 0;
322
323err_reset_hw:
324	erdma_hw_reset(dev);
325
326err_uninit_cmdq:
327	erdma_cmdq_destroy(dev);
328
329err_uninit_aeq:
330	erdma_eq_destroy(dev, &dev->aeq);
331
332err_uninit_comm_irq:
333	erdma_comm_irq_uninit(dev);
334
335err_free_vectors:
336	pci_free_irq_vectors(dev->pdev);
337
338err_uninit_device:
339	erdma_device_uninit(dev);
340
341err_iounmap_func_bar:
342	devm_iounmap(&pdev->dev, dev->func_bar);
343
344err_release_bars:
345	pci_release_selected_regions(pdev, bars);
346
347err_ib_device_release:
348	ib_dealloc_device(&dev->ibdev);
349
350err_disable_device:
351	pci_disable_device(pdev);
352
353	return err;
354}
355
356static void erdma_remove_dev(struct pci_dev *pdev)
357{
358	struct erdma_dev *dev = pci_get_drvdata(pdev);
359
360	erdma_ceqs_uninit(dev);
361	erdma_hw_reset(dev);
362	erdma_cmdq_destroy(dev);
363	erdma_eq_destroy(dev, &dev->aeq);
364	erdma_comm_irq_uninit(dev);
365	pci_free_irq_vectors(dev->pdev);
366	erdma_device_uninit(dev);
367
368	devm_iounmap(&pdev->dev, dev->func_bar);
369	pci_release_selected_regions(pdev, ERDMA_BAR_MASK);
370
371	ib_dealloc_device(&dev->ibdev);
372
373	pci_disable_device(pdev);
374}
375
376#define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap)
377
378static int erdma_dev_attrs_init(struct erdma_dev *dev)
379{
380	int err;
381	u64 req_hdr, cap0, cap1;
382
383	erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_RDMA,
384				CMDQ_OPCODE_QUERY_DEVICE);
385
386	err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,
387				  &cap1, true);
388	if (err)
389		return err;
390
391	dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0);
392	dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0);
393	dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1);
394	dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0);
395	dev->attrs.max_gid = 1 << ERDMA_GET_CAP(MAX_GID, cap0);
396	dev->attrs.max_ah = 1 << ERDMA_GET_CAP(MAX_AH, cap0);
397	dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1);
398	dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1);
399	dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);
400	dev->attrs.max_mr = dev->attrs.max_qp << 1;
401	dev->attrs.max_cq = dev->attrs.max_qp << 1;
402	dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);
403
404	dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;
405	dev->attrs.max_ord = ERDMA_MAX_ORD;
406	dev->attrs.max_ird = ERDMA_MAX_IRD;
407	dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE;
408	dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE;
409	dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD;
410	dev->attrs.max_pd = ERDMA_MAX_PD;
411
412	dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD;
413	dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr;
414	dev->res_cb[ERDMA_RES_TYPE_AH].max_cap = dev->attrs.max_ah;
415
416	erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_COMMON,
417				CMDQ_OPCODE_QUERY_FW_INFO);
418
419	err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,
420				  &cap1, true);
421	if (!err)
422		dev->attrs.fw_version =
423			FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0);
424
425	return err;
426}
427
428static int erdma_device_config(struct erdma_dev *dev)
429{
430	struct erdma_cmdq_config_device_req req = {};
431
432	if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB))
433		return 0;
434
435	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
436				CMDQ_OPCODE_CONF_DEVICE);
437
438	req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) |
439		  FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1);
440
441	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
442				   true);
443}
444
445static int erdma_res_cb_init(struct erdma_dev *dev)
446{
447	int i, j;
448
449	for (i = 0; i < ERDMA_RES_CNT; i++) {
450		dev->res_cb[i].next_alloc_idx = 1;
451		spin_lock_init(&dev->res_cb[i].lock);
452		dev->res_cb[i].bitmap =
453			bitmap_zalloc(dev->res_cb[i].max_cap, GFP_KERNEL);
454		if (!dev->res_cb[i].bitmap)
455			goto err;
456	}
457
458	return 0;
459
460err:
461	for (j = 0; j < i; j++)
462		bitmap_free(dev->res_cb[j].bitmap);
463
464	return -ENOMEM;
465}
466
467static void erdma_res_cb_free(struct erdma_dev *dev)
468{
469	int i;
470
471	for (i = 0; i < ERDMA_RES_CNT; i++)
472		bitmap_free(dev->res_cb[i].bitmap);
473}
474
475static const struct ib_device_ops erdma_device_ops_rocev2 = {
476	.get_link_layer = erdma_get_link_layer,
477	.add_gid = erdma_add_gid,
478	.del_gid = erdma_del_gid,
479	.query_pkey = erdma_query_pkey,
480	.create_ah = erdma_create_ah,
481	.destroy_ah = erdma_destroy_ah,
482	.query_ah = erdma_query_ah,
483
484	INIT_RDMA_OBJ_SIZE(ib_ah, erdma_ah, ibah),
485};
486
487static const struct ib_device_ops erdma_device_ops_iwarp = {
488	.iw_accept = erdma_accept,
489	.iw_add_ref = erdma_qp_get_ref,
490	.iw_connect = erdma_connect,
491	.iw_create_listen = erdma_create_listen,
492	.iw_destroy_listen = erdma_destroy_listen,
493	.iw_get_qp = erdma_get_ibqp,
494	.iw_reject = erdma_reject,
495	.iw_rem_ref = erdma_qp_put_ref,
496};
497
498static const struct ib_device_ops erdma_device_ops = {
499	.owner = THIS_MODULE,
500	.driver_id = RDMA_DRIVER_ERDMA,
501	.uverbs_abi_ver = ERDMA_ABI_VERSION,
502
503	.alloc_hw_port_stats = erdma_alloc_hw_port_stats,
504	.alloc_mr = erdma_ib_alloc_mr,
505	.alloc_pd = erdma_alloc_pd,
506	.alloc_ucontext = erdma_alloc_ucontext,
507	.create_cq = erdma_create_cq,
508	.create_qp = erdma_create_qp,
509	.dealloc_pd = erdma_dealloc_pd,
510	.dealloc_ucontext = erdma_dealloc_ucontext,
511	.dereg_mr = erdma_dereg_mr,
512	.destroy_cq = erdma_destroy_cq,
513	.destroy_qp = erdma_destroy_qp,
514	.disassociate_ucontext = erdma_disassociate_ucontext,
515	.get_dma_mr = erdma_get_dma_mr,
516	.get_hw_stats = erdma_get_hw_stats,
517	.get_port_immutable = erdma_get_port_immutable,
518	.map_mr_sg = erdma_map_mr_sg,
519	.mmap = erdma_mmap,
520	.mmap_free = erdma_mmap_free,
521	.post_recv = erdma_post_recv,
522	.post_send = erdma_post_send,
523	.poll_cq = erdma_poll_cq,
524	.query_device = erdma_query_device,
525	.query_gid = erdma_query_gid,
526	.query_port = erdma_query_port,
527	.query_qp = erdma_query_qp,
528	.req_notify_cq = erdma_req_notify_cq,
529	.reg_user_mr = erdma_reg_user_mr,
530	.modify_qp = erdma_modify_qp,
531
532	INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),
533	INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd),
534	INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext),
535	INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp),
536};
537
538static int erdma_ib_device_add(struct pci_dev *pdev)
539{
540	struct erdma_dev *dev = pci_get_drvdata(pdev);
541	struct ib_device *ibdev = &dev->ibdev;
542	u64 mac;
543	int ret;
544
545	ret = erdma_dev_attrs_init(dev);
546	if (ret)
547		return ret;
548
549	ret = erdma_device_config(dev);
550	if (ret)
551		return ret;
552
553	if (erdma_device_iwarp(dev)) {
554		ibdev->node_type = RDMA_NODE_RNIC;
555		ib_set_device_ops(ibdev, &erdma_device_ops_iwarp);
556	} else {
557		ibdev->node_type = RDMA_NODE_IB_CA;
558		ib_set_device_ops(ibdev, &erdma_device_ops_rocev2);
559	}
560
561	memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC));
562
563	/*
564	 * Current model (one-to-one device association):
565	 * One ERDMA device per net_device or, equivalently,
566	 * per physical port.
567	 */
568	ibdev->phys_port_cnt = 1;
569	ibdev->num_comp_vectors = dev->attrs.irq_num - 1;
570
571	ib_set_device_ops(ibdev, &erdma_device_ops);
572
573	INIT_LIST_HEAD(&dev->cep_list);
574
575	spin_lock_init(&dev->lock);
576	xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1);
577	xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1);
578	dev->next_alloc_cqn = 1;
579	dev->next_alloc_qpn = 1;
580
581	ret = erdma_res_cb_init(dev);
582	if (ret)
583		return ret;
584
585	atomic_set(&dev->num_ctx, 0);
586
587	mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG);
588	mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32;
589
590	u64_to_ether_addr(mac, dev->attrs.peer_addr);
591
592	dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,
593					  WQ_UNBOUND_MAX_ACTIVE);
594	if (!dev->reflush_wq) {
595		ret = -ENOMEM;
596		goto err_alloc_workqueue;
597	}
598
599	ret = erdma_device_register(dev);
600	if (ret)
601		goto err_register;
602
603	return 0;
604
605err_register:
606	destroy_workqueue(dev->reflush_wq);
607err_alloc_workqueue:
608	xa_destroy(&dev->qp_xa);
609	xa_destroy(&dev->cq_xa);
610
611	erdma_res_cb_free(dev);
612
613	return ret;
614}
615
616static void erdma_ib_device_remove(struct pci_dev *pdev)
617{
618	struct erdma_dev *dev = pci_get_drvdata(pdev);
619
620	unregister_netdevice_notifier(&dev->netdev_nb);
621	ib_unregister_device(&dev->ibdev);
622
623	destroy_workqueue(dev->reflush_wq);
624	erdma_res_cb_free(dev);
625	xa_destroy(&dev->qp_xa);
626	xa_destroy(&dev->cq_xa);
627}
628
629static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
630{
631	int ret;
632
633	ret = erdma_probe_dev(pdev);
634	if (ret)
635		return ret;
636
637	ret = erdma_ib_device_add(pdev);
638	if (ret) {
639		erdma_remove_dev(pdev);
640		return ret;
641	}
642
643	return 0;
644}
645
646static void erdma_remove(struct pci_dev *pdev)
647{
648	erdma_ib_device_remove(pdev);
649	erdma_remove_dev(pdev);
650}
651
652static struct pci_driver erdma_pci_driver = {
653	.name = DRV_MODULE_NAME,
654	.id_table = erdma_pci_tbl,
655	.probe = erdma_probe,
656	.remove = erdma_remove
657};
658
659MODULE_DEVICE_TABLE(pci, erdma_pci_tbl);
660
661static __init int erdma_init_module(void)
662{
663	int ret;
664
665	ret = erdma_cm_init();
666	if (ret)
667		return ret;
668
669	ret = pci_register_driver(&erdma_pci_driver);
670	if (ret)
671		erdma_cm_exit();
672
673	return ret;
674}
675
676static void __exit erdma_exit_module(void)
677{
678	pci_unregister_driver(&erdma_pci_driver);
679
680	erdma_cm_exit();
681}
682
683module_init(erdma_init_module);
684module_exit(erdma_exit_module);