bnxt_ulp.c

  1/* Broadcom NetXtreme-C/E network driver.
  2 *
  3 * Copyright (c) 2016-2018 Broadcom Limited
  4 *
  5 * This program is free software; you can redistribute it and/or modify
  6 * it under the terms of the GNU General Public License as published by
  7 * the Free Software Foundation.
  8 */
  9
 10#include <linux/module.h>
 11
 12#include <linux/kernel.h>
 13#include <linux/errno.h>
 14#include <linux/interrupt.h>
 15#include <linux/pci.h>
 16#include <linux/netdevice.h>
 17#include <linux/rtnetlink.h>
 18#include <linux/bitops.h>
 19#include <linux/irq.h>
 20#include <asm/byteorder.h>
 21#include <linux/bitmap.h>
 22#include <linux/auxiliary_bus.h>
 23#include <net/netdev_lock.h>
 24#include <linux/bnxt/hsi.h>
 25
 26#include "bnxt.h"
 27#include "bnxt_hwrm.h"
 28#include "bnxt_ulp.h"
 29
 30static DEFINE_IDA(bnxt_aux_dev_ids);
 31
 32static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent)
 33{
 34	struct bnxt_en_dev *edev = bp->edev;
 35	int num_msix, i;
 36
 37	if (!edev->ulp_tbl->msix_requested) {
 38		netdev_warn(bp->dev, "Requested MSI-X vectors insufficient\n");
 39		return;
 40	}
 41	num_msix = edev->ulp_tbl->msix_requested;
 42	for (i = 0; i < num_msix; i++) {
 43		ent[i].vector = bp->irq_tbl[i].vector;
 44		ent[i].ring_idx = i;
 45		if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
 46			ent[i].db_offset = bp->db_offset;
 47		else
 48			ent[i].db_offset = i * 0x80;
 49	}
 50}
 51
 52int bnxt_get_ulp_msix_num(struct bnxt *bp)
 53{
 54	if (bp->edev)
 55		return bp->edev->ulp_num_msix_vec;
 56	return 0;
 57}
 58
 59void bnxt_set_ulp_msix_num(struct bnxt *bp, int num)
 60{
 61	if (bp->edev)
 62		bp->edev->ulp_num_msix_vec = num;
 63}
 64
 65int bnxt_get_ulp_msix_num_in_use(struct bnxt *bp)
 66{
 67	if (bnxt_ulp_registered(bp->edev))
 68		return bp->edev->ulp_num_msix_vec;
 69	return 0;
 70}
 71
 72int bnxt_get_ulp_stat_ctxs(struct bnxt *bp)
 73{
 74	if (bp->edev)
 75		return bp->edev->ulp_num_ctxs;
 76	return 0;
 77}
 78
 79void bnxt_set_ulp_stat_ctxs(struct bnxt *bp, int num_ulp_ctx)
 80{
 81	if (bp->edev)
 82		bp->edev->ulp_num_ctxs = num_ulp_ctx;
 83}
 84
 85int bnxt_get_ulp_stat_ctxs_in_use(struct bnxt *bp)
 86{
 87	if (bnxt_ulp_registered(bp->edev))
 88		return bp->edev->ulp_num_ctxs;
 89	return 0;
 90}
 91
 92void bnxt_set_dflt_ulp_stat_ctxs(struct bnxt *bp)
 93{
 94	if (bp->edev) {
 95		bp->edev->ulp_num_ctxs = BNXT_MIN_ROCE_STAT_CTXS;
 96		/* Reserve one additional stat_ctx for PF0 (except
 97		 * on 1-port NICs) as it also creates one stat_ctx
 98		 * for PF1 in case of RoCE bonding.
 99		 */
100		if (BNXT_PF(bp) && !bp->pf.port_id &&
101		    bp->port_count > 1)
102			bp->edev->ulp_num_ctxs++;
103
104		/* Reserve one additional stat_ctx when the device is capable
105		 * of supporting port mirroring on RDMA device.
106		 */
107		if (BNXT_MIRROR_ON_ROCE_CAP(bp))
108			bp->edev->ulp_num_ctxs++;
109	}
110}
111
112int bnxt_register_dev(struct bnxt_en_dev *edev,
113		      struct bnxt_ulp_ops *ulp_ops,
114		      void *handle)
115{
116	struct net_device *dev = edev->net;
117	struct bnxt *bp = netdev_priv(dev);
118	unsigned int max_stat_ctxs;
119	struct bnxt_ulp *ulp;
120	int rc = 0;
121
122	netdev_lock(dev);
123	mutex_lock(&edev->en_dev_lock);
124	if (!bp->irq_tbl) {
125		rc = -ENODEV;
126		goto exit;
127	}
128	max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp);
129	if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS ||
130	    bp->cp_nr_rings == max_stat_ctxs) {
131		rc = -ENOMEM;
132		goto exit;
133	}
134
135	ulp = edev->ulp_tbl;
136	ulp->handle = handle;
137	rcu_assign_pointer(ulp->ulp_ops, ulp_ops);
138
139	if (test_bit(BNXT_STATE_OPEN, &bp->state))
140		bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info[BNXT_VNIC_DEFAULT]);
141
142	edev->ulp_tbl->msix_requested = bnxt_get_ulp_msix_num(bp);
143
144	bnxt_fill_msix_vecs(bp, bp->edev->msix_entries);
145exit:
146	mutex_unlock(&edev->en_dev_lock);
147	netdev_unlock(dev);
148	return rc;
149}
150EXPORT_SYMBOL(bnxt_register_dev);
151
152void bnxt_unregister_dev(struct bnxt_en_dev *edev)
153{
154	struct net_device *dev = edev->net;
155	struct bnxt *bp = netdev_priv(dev);
156	struct bnxt_ulp *ulp;
157
158	ulp = edev->ulp_tbl;
159	netdev_lock(dev);
160	mutex_lock(&edev->en_dev_lock);
161	edev->ulp_tbl->msix_requested = 0;
162
163	if (ulp->max_async_event_id)
164		bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, true);
165
166	RCU_INIT_POINTER(ulp->ulp_ops, NULL);
167	synchronize_rcu();
168	ulp->max_async_event_id = 0;
169	ulp->async_events_bmap = NULL;
170	mutex_unlock(&edev->en_dev_lock);
171	netdev_unlock(dev);
172	return;
173}
174EXPORT_SYMBOL(bnxt_unregister_dev);
175
176static int bnxt_set_dflt_ulp_msix(struct bnxt *bp)
177{
178	int roce_msix = BNXT_MAX_ROCE_MSIX;
179
180	if (BNXT_VF(bp))
181		roce_msix = BNXT_MAX_ROCE_MSIX_VF;
182	else if (bp->port_partition_type)
183		roce_msix = BNXT_MAX_ROCE_MSIX_NPAR_PF;
184
185	/* NQ MSIX vectors should match the number of CPUs plus 1 more for
186	 * the CREQ MSIX, up to the default.
187	 */
188	return min_t(int, roce_msix, num_online_cpus() + 1);
189}
190
191int bnxt_send_msg(struct bnxt_en_dev *edev,
192			 struct bnxt_fw_msg *fw_msg)
193{
194	struct net_device *dev = edev->net;
195	struct bnxt *bp = netdev_priv(dev);
196	struct output *resp;
197	struct input *req;
198	u32 resp_len;
199	int rc;
200
201	if (bp->fw_reset_state)
202		return -EBUSY;
203
204	rc = hwrm_req_init(bp, req, 0 /* don't care */);
205	if (rc)
206		return rc;
207
208	rc = hwrm_req_replace(bp, req, fw_msg->msg, fw_msg->msg_len);
209	if (rc)
210		goto drop_req;
211
212	hwrm_req_timeout(bp, req, fw_msg->timeout);
213	resp = hwrm_req_hold(bp, req);
214	rc = hwrm_req_send(bp, req);
215	resp_len = le16_to_cpu(resp->resp_len);
216	if (resp_len) {
217		if (fw_msg->resp_max_len < resp_len)
218			resp_len = fw_msg->resp_max_len;
219
220		memcpy(fw_msg->resp, resp, resp_len);
221	}
222drop_req:
223	hwrm_req_drop(bp, req);
224	return rc;
225}
226EXPORT_SYMBOL(bnxt_send_msg);
227
228void bnxt_ulp_stop(struct bnxt *bp)
229{
230	struct bnxt_aux_priv *aux_priv = bp->aux_priv;
231	struct bnxt_en_dev *edev = bp->edev;
232
233	if (!edev)
234		return;
235
236	mutex_lock(&edev->en_dev_lock);
237	if (!bnxt_ulp_registered(edev) ||
238	    (edev->flags & BNXT_EN_FLAG_ULP_STOPPED))
239		goto ulp_stop_exit;
240
241	edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;
242	if (aux_priv) {
243		struct auxiliary_device *adev;
244
245		adev = &aux_priv->aux_dev;
246		if (adev->dev.driver) {
247			const struct auxiliary_driver *adrv;
248			pm_message_t pm = {};
249
250			adrv = to_auxiliary_drv(adev->dev.driver);
251			edev->en_state = bp->state;
252			adrv->suspend(adev, pm);
253		}
254	}
255ulp_stop_exit:
256	mutex_unlock(&edev->en_dev_lock);
257}
258
259void bnxt_ulp_start(struct bnxt *bp, int err)
260{
261	struct bnxt_aux_priv *aux_priv = bp->aux_priv;
262	struct bnxt_en_dev *edev = bp->edev;
263
264	if (!edev || err)
265		return;
266
267	mutex_lock(&edev->en_dev_lock);
268	if (!bnxt_ulp_registered(edev) ||
269	    !(edev->flags & BNXT_EN_FLAG_ULP_STOPPED))
270		goto ulp_start_exit;
271
272	if (edev->ulp_tbl->msix_requested)
273		bnxt_fill_msix_vecs(bp, edev->msix_entries);
274
275	if (aux_priv) {
276		struct auxiliary_device *adev;
277
278		adev = &aux_priv->aux_dev;
279		if (adev->dev.driver) {
280			const struct auxiliary_driver *adrv;
281
282			adrv = to_auxiliary_drv(adev->dev.driver);
283			edev->en_state = bp->state;
284			adrv->resume(adev);
285		}
286	}
287ulp_start_exit:
288	edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED;
289	mutex_unlock(&edev->en_dev_lock);
290}
291
292void bnxt_ulp_irq_stop(struct bnxt *bp)
293{
294	struct bnxt_en_dev *edev = bp->edev;
295	struct bnxt_ulp_ops *ops;
296	bool reset = false;
297
298	if (!edev)
299		return;
300
301	if (bnxt_ulp_registered(bp->edev)) {
302		struct bnxt_ulp *ulp = edev->ulp_tbl;
303
304		if (!ulp->msix_requested)
305			return;
306
307		ops = netdev_lock_dereference(ulp->ulp_ops, bp->dev);
308		if (!ops || !ops->ulp_irq_stop)
309			return;
310		if (test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
311			reset = true;
312		ops->ulp_irq_stop(ulp->handle, reset);
313	}
314}
315
316void bnxt_ulp_irq_restart(struct bnxt *bp, int err)
317{
318	struct bnxt_en_dev *edev = bp->edev;
319	struct bnxt_ulp_ops *ops;
320
321	if (!edev)
322		return;
323
324	if (bnxt_ulp_registered(bp->edev)) {
325		struct bnxt_ulp *ulp = edev->ulp_tbl;
326		struct bnxt_msix_entry *ent = NULL;
327
328		if (!ulp->msix_requested)
329			return;
330
331		ops = netdev_lock_dereference(ulp->ulp_ops, bp->dev);
332		if (!ops || !ops->ulp_irq_restart)
333			return;
334
335		if (!err) {
336			ent = kcalloc(ulp->msix_requested, sizeof(*ent),
337				      GFP_KERNEL);
338			if (!ent)
339				return;
340			bnxt_fill_msix_vecs(bp, ent);
341		}
342		ops->ulp_irq_restart(ulp->handle, ent);
343		kfree(ent);
344	}
345}
346
347void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl)
348{
349	u16 event_id = le16_to_cpu(cmpl->event_id);
350	struct bnxt_en_dev *edev = bp->edev;
351	struct bnxt_ulp_ops *ops;
352	struct bnxt_ulp *ulp;
353
354	if (!bnxt_ulp_registered(edev))
355		return;
356	ulp = edev->ulp_tbl;
357
358	rcu_read_lock();
359
360	ops = rcu_dereference(ulp->ulp_ops);
361	if (!ops || !ops->ulp_async_notifier)
362		goto exit_unlock_rcu;
363	if (!ulp->async_events_bmap || event_id > ulp->max_async_event_id)
364		goto exit_unlock_rcu;
365
366	/* Read max_async_event_id first before testing the bitmap. */
367	smp_rmb();
368
369	if (test_bit(event_id, ulp->async_events_bmap))
370		ops->ulp_async_notifier(ulp->handle, cmpl);
371exit_unlock_rcu:
372	rcu_read_unlock();
373}
374
375void bnxt_register_async_events(struct bnxt_en_dev *edev,
376				unsigned long *events_bmap, u16 max_id)
377{
378	struct net_device *dev = edev->net;
379	struct bnxt *bp = netdev_priv(dev);
380	struct bnxt_ulp *ulp;
381
382	ulp = edev->ulp_tbl;
383	ulp->async_events_bmap = events_bmap;
384	/* Make sure bnxt_ulp_async_events() sees this order */
385	smp_wmb();
386	ulp->max_async_event_id = max_id;
387	bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true);
388}
389EXPORT_SYMBOL(bnxt_register_async_events);
390
391void bnxt_rdma_aux_device_uninit(struct bnxt *bp)
392{
393	struct bnxt_aux_priv *aux_priv;
394	struct auxiliary_device *adev;
395
396	/* Skip if no auxiliary device init was done. */
397	if (!bp->aux_priv)
398		return;
399
400	aux_priv = bp->aux_priv;
401	adev = &aux_priv->aux_dev;
402	auxiliary_device_uninit(adev);
403}
404
405static void bnxt_aux_dev_release(struct device *dev)
406{
407	struct bnxt_aux_priv *aux_priv =
408		container_of(dev, struct bnxt_aux_priv, aux_dev.dev);
409	struct bnxt *bp = netdev_priv(aux_priv->edev->net);
410
411	ida_free(&bnxt_aux_dev_ids, aux_priv->id);
412	kfree(aux_priv->edev->ulp_tbl);
413	bp->edev = NULL;
414	kfree(aux_priv->edev);
415	kfree(aux_priv);
416	bp->aux_priv = NULL;
417}
418
419void bnxt_rdma_aux_device_del(struct bnxt *bp)
420{
421	if (!bp->edev)
422		return;
423
424	auxiliary_device_delete(&bp->aux_priv->aux_dev);
425}
426
427static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp)
428{
429	edev->net = bp->dev;
430	edev->pdev = bp->pdev;
431	edev->l2_db_size = bp->db_size;
432	edev->l2_db_size_nc = bp->db_size;
433	edev->l2_db_offset = bp->db_offset;
434	mutex_init(&edev->en_dev_lock);
435
436	if (bp->flags & BNXT_FLAG_ROCEV1_CAP)
437		edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP;
438	if (bp->flags & BNXT_FLAG_ROCEV2_CAP)
439		edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;
440	if (bp->flags & BNXT_FLAG_VF)
441		edev->flags |= BNXT_EN_FLAG_VF;
442	if (BNXT_ROCE_VF_RESC_CAP(bp))
443		edev->flags |= BNXT_EN_FLAG_ROCE_VF_RES_MGMT;
444	if (BNXT_SW_RES_LMT(bp))
445		edev->flags |= BNXT_EN_FLAG_SW_RES_LMT;
446
447	edev->chip_num = bp->chip_num;
448	edev->hw_ring_stats_size = bp->hw_ring_stats_size;
449	edev->pf_port_id = bp->pf.port_id;
450	edev->en_state = bp->state;
451	edev->bar0 = bp->bar0;
452}
453
454void bnxt_rdma_aux_device_add(struct bnxt *bp)
455{
456	struct auxiliary_device *aux_dev;
457	int rc;
458
459	if (!bp->edev)
460		return;
461
462	aux_dev = &bp->aux_priv->aux_dev;
463	rc = auxiliary_device_add(aux_dev);
464	if (rc) {
465		netdev_warn(bp->dev, "Failed to add auxiliary device for ROCE\n");
466		auxiliary_device_uninit(aux_dev);
467		bp->flags &= ~BNXT_FLAG_ROCE_CAP;
468	}
469}
470
471void bnxt_rdma_aux_device_init(struct bnxt *bp)
472{
473	struct auxiliary_device *aux_dev;
474	struct bnxt_aux_priv *aux_priv;
475	struct bnxt_en_dev *edev;
476	struct bnxt_ulp *ulp;
477	int rc;
478
479	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
480		return;
481
482	aux_priv = kzalloc(sizeof(*bp->aux_priv), GFP_KERNEL);
483	if (!aux_priv)
484		goto exit;
485
486	aux_priv->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
487	if (aux_priv->id < 0) {
488		netdev_warn(bp->dev,
489			    "ida alloc failed for ROCE auxiliary device\n");
490		kfree(aux_priv);
491		goto exit;
492	}
493
494	aux_dev = &aux_priv->aux_dev;
495	aux_dev->id = aux_priv->id;
496	aux_dev->name = "rdma";
497	aux_dev->dev.parent = &bp->pdev->dev;
498	aux_dev->dev.release = bnxt_aux_dev_release;
499
500	rc = auxiliary_device_init(aux_dev);
501	if (rc) {
502		ida_free(&bnxt_aux_dev_ids, aux_priv->id);
503		kfree(aux_priv);
504		goto exit;
505	}
506	bp->aux_priv = aux_priv;
507
508	/* From this point, all cleanup will happen via the .release callback &
509	 * any error unwinding will need to include a call to
510	 * auxiliary_device_uninit.
511	 */
512	edev = kzalloc(sizeof(*edev), GFP_KERNEL);
513	if (!edev)
514		goto aux_dev_uninit;
515
516	aux_priv->edev = edev;
517
518	ulp = kzalloc(sizeof(*ulp), GFP_KERNEL);
519	if (!ulp)
520		goto aux_dev_uninit;
521
522	edev->ulp_tbl = ulp;
523	bp->edev = edev;
524	bnxt_set_edev_info(edev, bp);
525	bp->ulp_num_msix_want = bnxt_set_dflt_ulp_msix(bp);
526
527	return;
528
529aux_dev_uninit:
530	auxiliary_device_uninit(aux_dev);
531exit:
532	bp->flags &= ~BNXT_FLAG_ROCE_CAP;
533}