cn10k.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Marvell RVU Ethernet driver
  3 *
  4 * Copyright (C) 2021 Marvell.
  5 *
  6 */
  7
  8#include "cn10k.h"
  9#include "otx2_reg.h"
 10#include "otx2_struct.h"
 11
 12static struct dev_hw_ops	otx2_hw_ops = {
 13	.sq_aq_init = otx2_sq_aq_init,
 14	.sqe_flush = otx2_sqe_flush,
 15	.aura_freeptr = otx2_aura_freeptr,
 16	.refill_pool_ptrs = otx2_refill_pool_ptrs,
 17	.pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler,
 18	.aura_aq_init = otx2_aura_aq_init,
 19	.pool_aq_init = otx2_pool_aq_init,
 20};
 21
 22static struct dev_hw_ops cn10k_hw_ops = {
 23	.sq_aq_init = cn10k_sq_aq_init,
 24	.sqe_flush = cn10k_sqe_flush,
 25	.aura_freeptr = cn10k_aura_freeptr,
 26	.refill_pool_ptrs = cn10k_refill_pool_ptrs,
 27	.pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler,
 28	.aura_aq_init = otx2_aura_aq_init,
 29	.pool_aq_init = otx2_pool_aq_init,
 30};
 31
 32void otx2_init_hw_ops(struct otx2_nic *pfvf)
 33{
 34	if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
 35		pfvf->hw_ops = &otx2_hw_ops;
 36		return;
 37	}
 38
 39	pfvf->hw_ops = &cn10k_hw_ops;
 40}
 41EXPORT_SYMBOL(otx2_init_hw_ops);
 42
 43int cn10k_lmtst_init(struct otx2_nic *pfvf)
 44{
 45
 46	struct lmtst_tbl_setup_req *req;
 47	struct otx2_lmt_info *lmt_info;
 48	int err, cpu;
 49
 50	if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag))
 51		return 0;
 52
 53	/* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/
 54	pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE);
 55	pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info);
 56
 57	mutex_lock(&pfvf->mbox.lock);
 58	req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox);
 59	if (!req) {
 60		mutex_unlock(&pfvf->mbox.lock);
 61		return -ENOMEM;
 62	}
 63
 64	req->use_local_lmt_region = true;
 65
 66	err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines,
 67			 LMT_LINE_SIZE);
 68	if (err) {
 69		mutex_unlock(&pfvf->mbox.lock);
 70		return err;
 71	}
 72	pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base;
 73	req->lmt_iova = (u64)pfvf->dync_lmt->iova;
 74
 75	err = otx2_sync_mbox_msg(&pfvf->mbox);
 76	mutex_unlock(&pfvf->mbox.lock);
 77
 78	for_each_possible_cpu(cpu) {
 79		lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu);
 80		lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base +
 81				      (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE));
 82		lmt_info->lmt_id = cpu * LMT_BURST_SIZE;
 83	}
 84
 85	return 0;
 86}
 87EXPORT_SYMBOL(cn10k_lmtst_init);
 88
 89int cn10k_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura)
 90{
 91	struct nix_cn10k_aq_enq_req *aq;
 92	struct otx2_nic *pfvf = dev;
 93
 94	/* Get memory to put this msg */
 95	aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
 96	if (!aq)
 97		return -ENOMEM;
 98
 99	aq->sq.cq = pfvf->hw.rx_queues + qidx;
100	aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */
101	aq->sq.cq_ena = 1;
102	aq->sq.ena = 1;
103	aq->sq.smq = otx2_get_smq_idx(pfvf, qidx);
104	aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen);
105	aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset;
106	aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */
107	aq->sq.sqb_aura = sqb_aura;
108	aq->sq.sq_int_ena = NIX_SQINT_BITS;
109	aq->sq.qint_idx = 0;
110	/* Due pipelining impact minimum 2000 unused SQ CQE's
111	 * need to maintain to avoid CQ overflow.
112	 */
113	aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt));
114
115	/* Fill AQ info */
116	aq->qidx = qidx;
117	aq->ctype = NIX_AQ_CTYPE_SQ;
118	aq->op = NIX_AQ_INSTOP_INIT;
119
120	return otx2_sync_mbox_msg(&pfvf->mbox);
121}
122
123#define NPA_MAX_BURST 16
124int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
125{
126	struct otx2_nic *pfvf = dev;
127	int cnt = cq->pool_ptrs;
128	u64 ptrs[NPA_MAX_BURST];
129	struct otx2_pool *pool;
130	dma_addr_t bufptr;
131	int num_ptrs = 1;
132
133	pool = &pfvf->qset.pool[cq->cq_idx];
134
135	/* Refill pool with new buffers */
136	while (cq->pool_ptrs) {
137		if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
138			if (num_ptrs--)
139				__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
140						     num_ptrs);
141			break;
142		}
143		cq->pool_ptrs--;
144		ptrs[num_ptrs] = pool->xsk_pool ?
145				 (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM;
146
147		num_ptrs++;
148		if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
149			__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
150					     num_ptrs);
151			num_ptrs = 1;
152		}
153	}
154	return cnt - cq->pool_ptrs;
155}
156
157void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx)
158{
159	struct otx2_lmt_info *lmt_info;
160	struct otx2_nic *pfvf = dev;
161	u64 val = 0, tar_addr = 0;
162
163	lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id());
164	/* FIXME: val[0:10] LMT_ID.
165	 * [12:15] no of LMTST - 1 in the burst.
166	 * [19:63] data size of each LMTST in the burst except first.
167	 */
168	val = (lmt_info->lmt_id & 0x7FF);
169	/* Target address for LMTST flush tells HW how many 128bit
170	 * words are present.
171	 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
172	 */
173	tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4;
174	dma_wmb();
175	memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size);
176	cn10k_lmt_flush(val, tar_addr);
177
178	sq->head++;
179	sq->head &= (sq->sqe_cnt - 1);
180}
181
182int cn10k_free_all_ipolicers(struct otx2_nic *pfvf)
183{
184	struct nix_bandprof_free_req *req;
185	int rc;
186
187	if (is_dev_otx2(pfvf->pdev))
188		return 0;
189
190	mutex_lock(&pfvf->mbox.lock);
191
192	req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
193	if (!req) {
194		rc =  -ENOMEM;
195		goto out;
196	}
197
198	/* Free all bandwidth profiles allocated */
199	req->free_all = true;
200
201	rc = otx2_sync_mbox_msg(&pfvf->mbox);
202out:
203	mutex_unlock(&pfvf->mbox.lock);
204	return rc;
205}
206
207int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf)
208{
209	struct nix_bandprof_alloc_req *req;
210	struct nix_bandprof_alloc_rsp *rsp;
211	int rc;
212
213	req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox);
214	if (!req)
215		return  -ENOMEM;
216
217	req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
218
219	rc = otx2_sync_mbox_msg(&pfvf->mbox);
220	if (rc)
221		goto out;
222
223	rsp = (struct  nix_bandprof_alloc_rsp *)
224	       otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr);
225	if (IS_ERR(rsp)) {
226		rc = PTR_ERR(rsp);
227		goto out;
228	}
229
230	if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) {
231		rc = -EIO;
232		goto out;
233	}
234
235	*leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0];
236out:
237	if (rc) {
238		dev_warn(pfvf->dev,
239			 "Failed to allocate ingress bandwidth policer\n");
240	}
241
242	return rc;
243}
244
245int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf)
246{
247	struct otx2_hw *hw = &pfvf->hw;
248	int ret;
249
250	mutex_lock(&pfvf->mbox.lock);
251
252	ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer);
253
254	mutex_unlock(&pfvf->mbox.lock);
255
256	return ret;
257}
258
259#define POLICER_TIMESTAMP	  1  /* 1 second */
260#define MAX_RATE_EXP		  22 /* Valid rate exponent range: 0 - 22 */
261
262static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp,
263					u32 *burst_mantissa)
264{
265	int tmp;
266
267	/* Burst is calculated as
268	 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT]
269	 * This is the upper limit on number tokens (bytes) that
270	 * can be accumulated in the bucket.
271	 */
272	*burst_exp = ilog2(burst);
273	if (burst < 256) {
274		/* No float: can't express mantissa in this case */
275		*burst_mantissa = 0;
276		return;
277	}
278
279	if (*burst_exp > MAX_RATE_EXP)
280		*burst_exp = MAX_RATE_EXP;
281
282	/* Calculate mantissa
283	 * Find remaining bytes 'burst - 2^burst_exp'
284	 * mantissa = (remaining bytes) / 2^ (burst_exp - 8)
285	 */
286	tmp = burst - rounddown_pow_of_two(burst);
287	*burst_mantissa = tmp / (1UL << (*burst_exp - 8));
288}
289
290static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp,
291				       u32 *rate_mantissa, u32 *rdiv)
292{
293	u32 div = 0;
294	u32 exp = 0;
295	u64 tmp;
296
297	/* Figure out mantissa, exponent and divider from given max pkt rate
298	 *
299	 * To achieve desired rate HW adds
300	 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every
301	 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket.
302	 * Here policer timeunit is 2 usecs and rate is in bits per sec.
303	 * Since floating point cannot be used below algorithm uses 1000000
304	 * scale factor to support rates upto 100Gbps.
305	 */
306	tmp = rate * 32 * 2;
307	if (tmp < 256000000) {
308		while (tmp < 256000000) {
309			tmp = tmp * 2;
310			div++;
311		}
312	} else {
313		for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++)
314			tmp = tmp / 2;
315
316		if (exp > MAX_RATE_EXP)
317			exp = MAX_RATE_EXP;
318	}
319
320	*rate_mantissa = (tmp - 256000000) / 1000000;
321	*rate_exp = exp;
322	*rdiv = div;
323}
324
325int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx,
326			       u16 policer, bool map)
327{
328	struct nix_cn10k_aq_enq_req *aq;
329
330	aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
331	if (!aq)
332		return -ENOMEM;
333
334	/* Enable policing and set the bandwidth profile (policer) index */
335	if (map)
336		aq->rq.policer_ena = 1;
337	else
338		aq->rq.policer_ena = 0;
339	aq->rq_mask.policer_ena = 1;
340
341	aq->rq.band_prof_id = policer;
342	aq->rq_mask.band_prof_id = GENMASK(9, 0);
343
344	/* If policer id is greater than 1023 then it implies hardware supports
345	 * more leaf profiles. In that case use band_prof_id_h for 4 MSBs.
346	 */
347	aq->rq.band_prof_id_h = policer >> 10;
348	aq->rq_mask.band_prof_id_h = GENMASK(3, 0);
349
350	/* Fill AQ info */
351	aq->qidx = rq_idx;
352	aq->ctype = NIX_AQ_CTYPE_RQ;
353	aq->op = NIX_AQ_INSTOP_WRITE;
354
355	return otx2_sync_mbox_msg(&pfvf->mbox);
356}
357
358int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf)
359{
360	struct nix_bandprof_free_req *req;
361
362	req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
363	if (!req)
364		return -ENOMEM;
365
366	req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
367	req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf;
368
369	return otx2_sync_mbox_msg(&pfvf->mbox);
370}
371
372int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf)
373{
374	struct otx2_hw *hw = &pfvf->hw;
375	int qidx, rc;
376
377	mutex_lock(&pfvf->mbox.lock);
378
379	/* Remove RQ's policer mapping */
380	for (qidx = 0; qidx < hw->rx_queues; qidx++) {
381		rc = cn10k_map_unmap_rq_policer(pfvf, qidx, hw->matchall_ipolicer, false);
382		if (rc)
383			dev_warn(pfvf->dev, "Failed to unmap RQ %d's policer (error %d).",
384				 qidx, rc);
385	}
386
387	rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer);
388
389	mutex_unlock(&pfvf->mbox.lock);
390	return rc;
391}
392
393int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile,
394			    u32 burst, u64 rate, bool pps)
395{
396	struct nix_cn10k_aq_enq_req *aq;
397	u32 burst_exp, burst_mantissa;
398	u32 rate_exp, rate_mantissa;
399	u32 rdiv;
400
401	/* Get exponent and mantissa values for the desired rate */
402	cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa);
403	cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv);
404
405	/* Init bandwidth profile */
406	aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
407	if (!aq)
408		return -ENOMEM;
409
410	/* Set initial color mode to blind */
411	aq->prof.icolor = 0x03;
412	aq->prof_mask.icolor = 0x03;
413
414	/* Set rate and burst values */
415	aq->prof.cir_exponent = rate_exp;
416	aq->prof_mask.cir_exponent = 0x1F;
417
418	aq->prof.cir_mantissa = rate_mantissa;
419	aq->prof_mask.cir_mantissa = 0xFF;
420
421	aq->prof.cbs_exponent = burst_exp;
422	aq->prof_mask.cbs_exponent = 0x1F;
423
424	aq->prof.cbs_mantissa = burst_mantissa;
425	aq->prof_mask.cbs_mantissa = 0xFF;
426
427	aq->prof.rdiv = rdiv;
428	aq->prof_mask.rdiv = 0xF;
429
430	if (pps) {
431		/* The amount of decremented tokens is calculated according to
432		 * the following equation:
433		 * max([ LMODE ? 0 : (packet_length - LXPTR)] +
434		 *	     ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT],
435		 *	1/256)
436		 * if LMODE is 1 then rate limiting will be based on
437		 * PPS otherwise bps.
438		 * The aim of the ADJUST value is to specify a token cost per
439		 * packet in contrary to the packet length that specifies a
440		 * cost per byte. To rate limit based on PPS adjust mantissa
441		 * is set as 384 and exponent as 1 so that number of tokens
442		 * decremented becomes 1 i.e, 1 token per packeet.
443		 */
444		aq->prof.adjust_exponent = 1;
445		aq->prof_mask.adjust_exponent = 0x1F;
446
447		aq->prof.adjust_mantissa = 384;
448		aq->prof_mask.adjust_mantissa = 0x1FF;
449
450		aq->prof.lmode = 0x1;
451		aq->prof_mask.lmode = 0x1;
452	}
453
454	/* Two rate three color marker
455	 * With PEIR/EIR set to zero, color will be either green or red
456	 */
457	aq->prof.meter_algo = 2;
458	aq->prof_mask.meter_algo = 0x3;
459
460	aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP;
461	aq->prof_mask.rc_action = 0x3;
462
463	aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
464	aq->prof_mask.yc_action = 0x3;
465
466	aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
467	aq->prof_mask.gc_action = 0x3;
468
469	/* Setting exponent value as 24 and mantissa as 0 configures
470	 * the bucket with zero values making bucket unused. Peak
471	 * information rate and Excess information rate buckets are
472	 * unused here.
473	 */
474	aq->prof.peir_exponent = 24;
475	aq->prof_mask.peir_exponent = 0x1F;
476
477	aq->prof.peir_mantissa = 0;
478	aq->prof_mask.peir_mantissa = 0xFF;
479
480	aq->prof.pebs_exponent = 24;
481	aq->prof_mask.pebs_exponent = 0x1F;
482
483	aq->prof.pebs_mantissa = 0;
484	aq->prof_mask.pebs_mantissa = 0xFF;
485
486	aq->prof.hl_en = 0;
487	aq->prof_mask.hl_en = 1;
488
489	/* Fill AQ info */
490	aq->qidx = profile;
491	aq->ctype = NIX_AQ_CTYPE_BANDPROF;
492	aq->op = NIX_AQ_INSTOP_WRITE;
493
494	return otx2_sync_mbox_msg(&pfvf->mbox);
495}
496
497int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf,
498				     u32 burst, u64 rate)
499{
500	struct otx2_hw *hw = &pfvf->hw;
501	int qidx, rc;
502
503	mutex_lock(&pfvf->mbox.lock);
504
505	rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst,
506				     rate, false);
507	if (rc)
508		goto out;
509
510	for (qidx = 0; qidx < hw->rx_queues; qidx++) {
511		rc = cn10k_map_unmap_rq_policer(pfvf, qidx,
512						hw->matchall_ipolicer, true);
513		if (rc)
514			break;
515	}
516
517out:
518	mutex_unlock(&pfvf->mbox.lock);
519	return rc;
520}