device.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
  4 */
  5
  6#include "mana_ib.h"
  7#include <net/mana/mana_auxiliary.h>
  8#include <net/addrconf.h>
  9
 10MODULE_DESCRIPTION("Microsoft Azure Network Adapter IB driver");
 11MODULE_LICENSE("GPL");
 12MODULE_IMPORT_NS("NET_MANA");
 13
 14static const struct ib_device_ops mana_ib_dev_ops = {
 15	.owner = THIS_MODULE,
 16	.driver_id = RDMA_DRIVER_MANA,
 17	.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
 18
 19	.add_gid = mana_ib_gd_add_gid,
 20	.alloc_pd = mana_ib_alloc_pd,
 21	.alloc_ucontext = mana_ib_alloc_ucontext,
 22	.create_ah = mana_ib_create_ah,
 23	.create_cq = mana_ib_create_cq,
 24	.create_qp = mana_ib_create_qp,
 25	.create_rwq_ind_table = mana_ib_create_rwq_ind_table,
 26	.create_wq = mana_ib_create_wq,
 27	.dealloc_pd = mana_ib_dealloc_pd,
 28	.dealloc_ucontext = mana_ib_dealloc_ucontext,
 29	.del_gid = mana_ib_gd_del_gid,
 30	.dereg_mr = mana_ib_dereg_mr,
 31	.destroy_ah = mana_ib_destroy_ah,
 32	.destroy_cq = mana_ib_destroy_cq,
 33	.destroy_qp = mana_ib_destroy_qp,
 34	.destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,
 35	.destroy_wq = mana_ib_destroy_wq,
 36	.disassociate_ucontext = mana_ib_disassociate_ucontext,
 37	.get_dma_mr = mana_ib_get_dma_mr,
 38	.get_link_layer = mana_ib_get_link_layer,
 39	.get_port_immutable = mana_ib_get_port_immutable,
 40	.mmap = mana_ib_mmap,
 41	.modify_qp = mana_ib_modify_qp,
 42	.modify_wq = mana_ib_modify_wq,
 43	.poll_cq = mana_ib_poll_cq,
 44	.post_recv = mana_ib_post_recv,
 45	.post_send = mana_ib_post_send,
 46	.query_device = mana_ib_query_device,
 47	.query_gid = mana_ib_query_gid,
 48	.query_pkey = mana_ib_query_pkey,
 49	.query_port = mana_ib_query_port,
 50	.reg_user_mr = mana_ib_reg_user_mr,
 51	.reg_user_mr_dmabuf = mana_ib_reg_user_mr_dmabuf,
 52	.req_notify_cq = mana_ib_arm_cq,
 53
 54	INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah),
 55	INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
 56	INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
 57	INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
 58	INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
 59	INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mana_ib_rwq_ind_table,
 60			   ib_ind_table),
 61};
 62
 63static const struct ib_device_ops mana_ib_stats_ops = {
 64	.alloc_hw_port_stats = mana_ib_alloc_hw_port_stats,
 65	.get_hw_stats = mana_ib_get_hw_stats,
 66};
 67
 68static const struct ib_device_ops mana_ib_device_stats_ops = {
 69	.alloc_hw_device_stats = mana_ib_alloc_hw_device_stats,
 70};
 71
 72static int mana_ib_netdev_event(struct notifier_block *this,
 73				unsigned long event, void *ptr)
 74{
 75	struct mana_ib_dev *dev = container_of(this, struct mana_ib_dev, nb);
 76	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
 77	struct gdma_context *gc = dev->gdma_dev->gdma_context;
 78	struct mana_context *mc = gc->mana.driver_data;
 79	struct net_device *ndev;
 80	int i;
 81
 82	/* Only process events from our parent device */
 83	for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
 84		if (event_dev == mc->ports[i]) {
 85			switch (event) {
 86			case NETDEV_CHANGEUPPER:
 87				ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
 88				/*
 89				 * RDMA core will setup GID based on updated netdev.
 90				 * It's not possible to race with the core as rtnl lock is being
 91				 * held.
 92				 */
 93				ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
 94
 95				/* mana_get_primary_netdev() returns ndev with refcount held */
 96				if (ndev)
 97					netdev_put(ndev, &dev->dev_tracker);
 98
 99				return NOTIFY_OK;
100			default:
101				return NOTIFY_DONE;
102			}
103		}
104	return NOTIFY_DONE;
105}
106
107static int mana_ib_probe(struct auxiliary_device *adev,
108			 const struct auxiliary_device_id *id)
109{
110	struct mana_adev *madev = container_of(adev, struct mana_adev, adev);
111	struct gdma_context *gc = madev->mdev->gdma_context;
112	struct mana_context *mc = gc->mana.driver_data;
113	struct gdma_dev *mdev = madev->mdev;
114	struct net_device *ndev;
115	struct mana_ib_dev *dev;
116	u8 mac_addr[ETH_ALEN];
117	int ret, i;
118
119	dev = ib_alloc_device(mana_ib_dev, ib_dev);
120	if (!dev)
121		return -ENOMEM;
122
123	ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops);
124	dev->ib_dev.node_type = RDMA_NODE_IB_CA;
125	dev->ib_dev.num_comp_vectors = gc->max_num_queues;
126	dev->ib_dev.dev.parent = gc->dev;
127	dev->gdma_dev = mdev;
128	xa_init_flags(&dev->qp_table_wq, XA_FLAGS_LOCK_IRQ);
129
130	if (mana_ib_is_rnic(dev)) {
131		dev->ib_dev.phys_port_cnt = 1;
132		addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
133		ret = mana_ib_gd_query_adapter_caps(dev);
134		if (ret) {
135			ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
136			goto free_ib_device;
137		}
138
139		ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
140		if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT)
141			ib_set_device_ops(&dev->ib_dev, &mana_ib_device_stats_ops);
142
143		ret = mana_ib_create_eqs(dev);
144		if (ret) {
145			ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
146			goto free_ib_device;
147		}
148
149		ret = mana_ib_gd_create_rnic_adapter(dev);
150		if (ret)
151			goto destroy_eqs;
152
153		if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
154			dev->ib_dev.phys_port_cnt = mc->num_ports;
155
156		for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
157			ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
158			if (!ndev) {
159				ret = -ENODEV;
160				ibdev_err(&dev->ib_dev,
161					  "Failed to get netdev for IB port %d", i + 1);
162				goto destroy_rnic;
163			}
164			ether_addr_copy(mac_addr, ndev->dev_addr);
165			ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
166			/* mana_get_primary_netdev() returns ndev with refcount held */
167			netdev_put(ndev, &dev->dev_tracker);
168			if (ret) {
169				ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
170				goto destroy_rnic;
171			}
172			ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
173			if (ret) {
174				ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
175				goto destroy_rnic;
176			}
177		}
178		dev->nb.notifier_call = mana_ib_netdev_event;
179		ret = register_netdevice_notifier(&dev->nb);
180		if (ret) {
181			ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
182			goto destroy_rnic;
183		}
184	} else {
185		dev->ib_dev.phys_port_cnt = mc->num_ports;
186		ret = mana_eth_query_adapter_caps(dev);
187		if (ret) {
188			ibdev_err(&dev->ib_dev, "Failed to query ETH device caps, ret %d", ret);
189			goto free_ib_device;
190		}
191	}
192
193	dev->av_pool = dma_pool_create("mana_ib_av", gc->dev, MANA_AV_BUFFER_SIZE,
194				       MANA_AV_BUFFER_SIZE, 0);
195	if (!dev->av_pool) {
196		ret = -ENOMEM;
197		goto deregister_net_notifier;
198	}
199
200	ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
201		  mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt);
202
203	ret = ib_register_device(&dev->ib_dev, mana_ib_is_rnic(dev) ? "mana_%d" : "manae_%d",
204				 gc->dev);
205	if (ret)
206		goto deallocate_pool;
207
208	dev_set_drvdata(&adev->dev, dev);
209
210	return 0;
211
212deallocate_pool:
213	dma_pool_destroy(dev->av_pool);
214deregister_net_notifier:
215	if (mana_ib_is_rnic(dev))
216		unregister_netdevice_notifier(&dev->nb);
217destroy_rnic:
218	if (mana_ib_is_rnic(dev))
219		mana_ib_gd_destroy_rnic_adapter(dev);
220destroy_eqs:
221	if (mana_ib_is_rnic(dev))
222		mana_ib_destroy_eqs(dev);
223free_ib_device:
224	xa_destroy(&dev->qp_table_wq);
225	ib_dealloc_device(&dev->ib_dev);
226	return ret;
227}
228
229static void mana_ib_remove(struct auxiliary_device *adev)
230{
231	struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
232
233	if (mana_ib_is_rnic(dev))
234		mana_drain_gsi_sqs(dev);
235
236	ib_unregister_device(&dev->ib_dev);
237	dma_pool_destroy(dev->av_pool);
238	if (mana_ib_is_rnic(dev)) {
239		unregister_netdevice_notifier(&dev->nb);
240		mana_ib_gd_destroy_rnic_adapter(dev);
241		mana_ib_destroy_eqs(dev);
242	}
243	xa_destroy(&dev->qp_table_wq);
244	ib_dealloc_device(&dev->ib_dev);
245}
246
247static const struct auxiliary_device_id mana_id_table[] = {
248	{ .name = "mana.rdma", },
249	{ .name = "mana.eth", },
250	{},
251};
252
253MODULE_DEVICE_TABLE(auxiliary, mana_id_table);
254
255static struct auxiliary_driver mana_driver = {
256	.probe = mana_ib_probe,
257	.remove = mana_ib_remove,
258	.id_table = mana_id_table,
259};
260
261module_auxiliary_driver(mana_driver);