octep_rx.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Marvell Octeon EP (EndPoint) Ethernet Driver
  3 *
  4 * Copyright (C) 2020 Marvell.
  5 *
  6 */
  7
  8#include <linux/pci.h>
  9#include <linux/etherdevice.h>
 10#include <linux/vmalloc.h>
 11
 12#include "octep_config.h"
 13#include "octep_main.h"
 14
 15static void octep_oq_free_ring_buffers(struct octep_oq *oq);
 16
 17static void octep_oq_reset_indices(struct octep_oq *oq)
 18{
 19	oq->host_read_idx = 0;
 20	oq->host_refill_idx = 0;
 21	oq->refill_count = 0;
 22	oq->last_pkt_count = 0;
 23	oq->pkts_pending = 0;
 24}
 25
 26/**
 27 * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
 28 *
 29 * @oq: Octeon Rx queue data structure.
 30 *
 31 * Return: 0, if successfully filled receive buffers for all descriptors.
 32 *         -1, if failed to allocate a buffer or failed to map for DMA.
 33 */
 34static int octep_oq_fill_ring_buffers(struct octep_oq *oq)
 35{
 36	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
 37	struct page *page;
 38	u32 i;
 39
 40	for (i = 0; i < oq->max_count; i++) {
 41		page = dev_alloc_page();
 42		if (unlikely(!page)) {
 43			dev_err(oq->dev, "Rx buffer alloc failed\n");
 44			goto rx_buf_alloc_err;
 45		}
 46		desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
 47						       PAGE_SIZE,
 48						       DMA_FROM_DEVICE);
 49		if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
 50			dev_err(oq->dev,
 51				"OQ-%d buffer alloc: DMA mapping error!\n",
 52				oq->q_no);
 53			put_page(page);
 54			goto dma_map_err;
 55		}
 56		oq->buff_info[i].page = page;
 57	}
 58
 59	return 0;
 60
 61dma_map_err:
 62rx_buf_alloc_err:
 63	while (i) {
 64		i--;
 65		dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
 66		put_page(oq->buff_info[i].page);
 67		oq->buff_info[i].page = NULL;
 68	}
 69
 70	return -1;
 71}
 72
 73/**
 74 * octep_oq_refill() - refill buffers for used Rx ring descriptors.
 75 *
 76 * @oct: Octeon device private data structure.
 77 * @oq: Octeon Rx queue data structure.
 78 *
 79 * Return: number of descriptors successfully refilled with receive buffers.
 80 */
 81static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)
 82{
 83	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
 84	struct page *page;
 85	u32 refill_idx, i;
 86
 87	refill_idx = oq->host_refill_idx;
 88	for (i = 0; i < oq->refill_count; i++) {
 89		page = dev_alloc_page();
 90		if (unlikely(!page)) {
 91			dev_err(oq->dev, "refill: rx buffer alloc failed\n");
 92			oq->stats->alloc_failures++;
 93			break;
 94		}
 95
 96		desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
 97								PAGE_SIZE, DMA_FROM_DEVICE);
 98		if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
 99			dev_err(oq->dev,
100				"OQ-%d buffer refill: DMA mapping error!\n",
101				oq->q_no);
102			put_page(page);
103			oq->stats->alloc_failures++;
104			break;
105		}
106		oq->buff_info[refill_idx].page = page;
107		refill_idx++;
108		if (refill_idx == oq->max_count)
109			refill_idx = 0;
110	}
111	oq->host_refill_idx = refill_idx;
112	oq->refill_count -= i;
113
114	return i;
115}
116
117/**
118 * octep_setup_oq() - Setup a Rx queue.
119 *
120 * @oct: Octeon device private data structure.
121 * @q_no: Rx queue number to be setup.
122 *
123 * Allocate resources for a Rx queue.
124 */
125static int octep_setup_oq(struct octep_device *oct, int q_no)
126{
127	struct octep_oq *oq;
128	u32 desc_ring_size;
129
130	oq = vzalloc(sizeof(*oq));
131	if (!oq)
132		goto create_oq_fail;
133	oct->oq[q_no] = oq;
134
135	oq->octep_dev = oct;
136	oq->netdev = oct->netdev;
137	oq->dev = &oct->pdev->dev;
138	oq->q_no = q_no;
139	oq->stats = &oct->stats_oq[q_no];
140	oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
141	oq->ring_size_mask = oq->max_count - 1;
142	oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
143	oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;
144
145	/* When the hardware/firmware supports additional capabilities,
146	 * additional header is filled-in by Octeon after length field in
147	 * Rx packets. this header contains additional packet information.
148	 */
149	if (oct->conf->fw_info.rx_ol_flags)
150		oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;
151
152	oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
153
154	desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;
155	oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
156					   &oq->desc_ring_dma, GFP_KERNEL);
157
158	if (unlikely(!oq->desc_ring)) {
159		dev_err(oq->dev,
160			"Failed to allocate DMA memory for OQ-%d !!\n", q_no);
161		goto desc_dma_alloc_err;
162	}
163
164	oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE);
165	if (unlikely(!oq->buff_info)) {
166		dev_err(&oct->pdev->dev,
167			"Failed to allocate buffer info for OQ-%d\n", q_no);
168		goto buf_list_err;
169	}
170
171	if (octep_oq_fill_ring_buffers(oq))
172		goto oq_fill_buff_err;
173
174	octep_oq_reset_indices(oq);
175	if (oct->hw_ops.setup_oq_regs(oct, q_no))
176		goto oq_setup_err;
177
178	oct->num_oqs++;
179
180	return 0;
181
182oq_setup_err:
183	octep_oq_free_ring_buffers(oq);
184oq_fill_buff_err:
185	vfree(oq->buff_info);
186	oq->buff_info = NULL;
187buf_list_err:
188	dma_free_coherent(oq->dev, desc_ring_size,
189			  oq->desc_ring, oq->desc_ring_dma);
190	oq->desc_ring = NULL;
191desc_dma_alloc_err:
192	vfree(oq);
193	oct->oq[q_no] = NULL;
194create_oq_fail:
195	return -1;
196}
197
198/**
199 * octep_oq_free_ring_buffers() - Free ring buffers.
200 *
201 * @oq: Octeon Rx queue data structure.
202 *
203 * Free receive buffers in unused Rx queue descriptors.
204 */
205static void octep_oq_free_ring_buffers(struct octep_oq *oq)
206{
207	struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
208	int  i;
209
210	if (!oq->desc_ring || !oq->buff_info)
211		return;
212
213	for (i = 0; i < oq->max_count; i++)  {
214		if (oq->buff_info[i].page) {
215			dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
216				       PAGE_SIZE, DMA_FROM_DEVICE);
217			put_page(oq->buff_info[i].page);
218			oq->buff_info[i].page = NULL;
219			desc_ring[i].buffer_ptr = 0;
220		}
221	}
222	octep_oq_reset_indices(oq);
223}
224
225/**
226 * octep_free_oq() - Free Rx queue resources.
227 *
228 * @oq: Octeon Rx queue data structure.
229 *
230 * Free all resources of a Rx queue.
231 */
232static int octep_free_oq(struct octep_oq *oq)
233{
234	struct octep_device *oct = oq->octep_dev;
235	int q_no = oq->q_no;
236
237	octep_oq_free_ring_buffers(oq);
238
239	vfree(oq->buff_info);
240
241	if (oq->desc_ring)
242		dma_free_coherent(oq->dev,
243				  oq->max_count * OCTEP_OQ_DESC_SIZE,
244				  oq->desc_ring, oq->desc_ring_dma);
245
246	vfree(oq);
247	oct->oq[q_no] = NULL;
248	oct->num_oqs--;
249	return 0;
250}
251
252/**
253 * octep_setup_oqs() - setup resources for all Rx queues.
254 *
255 * @oct: Octeon device private data structure.
256 */
257int octep_setup_oqs(struct octep_device *oct)
258{
259	int i, retval = 0;
260
261	oct->num_oqs = 0;
262	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
263		retval = octep_setup_oq(oct, i);
264		if (retval) {
265			dev_err(&oct->pdev->dev,
266				"Failed to setup OQ(RxQ)-%d.\n", i);
267			goto oq_setup_err;
268		}
269		dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
270	}
271
272	return 0;
273
274oq_setup_err:
275	while (i) {
276		i--;
277		octep_free_oq(oct->oq[i]);
278	}
279	return -1;
280}
281
282/**
283 * octep_oq_dbell_init() - Initialize Rx queue doorbell.
284 *
285 * @oct: Octeon device private data structure.
286 *
287 * Write number of descriptors to Rx queue doorbell register.
288 */
289void octep_oq_dbell_init(struct octep_device *oct)
290{
291	int i;
292
293	for (i = 0; i < oct->num_oqs; i++)
294		writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
295}
296
297/**
298 * octep_free_oqs() - Free resources of all Rx queues.
299 *
300 * @oct: Octeon device private data structure.
301 */
302void octep_free_oqs(struct octep_device *oct)
303{
304	int i;
305
306	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
307		if (!oct->oq[i])
308			continue;
309		octep_free_oq(oct->oq[i]);
310		dev_dbg(&oct->pdev->dev,
311			"Successfully freed OQ(RxQ)-%d.\n", i);
312	}
313}
314
315/**
316 * octep_oq_check_hw_for_pkts() - Check for new Rx packets.
317 *
318 * @oct: Octeon device private data structure.
319 * @oq: Octeon Rx queue data structure.
320 *
321 * Return: packets received after previous check.
322 */
323static int octep_oq_check_hw_for_pkts(struct octep_device *oct,
324				      struct octep_oq *oq)
325{
326	u32 pkt_count, new_pkts;
327	u32 last_pkt_count, pkts_pending;
328
329	pkt_count = readl(oq->pkts_sent_reg);
330	last_pkt_count = READ_ONCE(oq->last_pkt_count);
331	new_pkts = pkt_count - last_pkt_count;
332
333	if (pkt_count < last_pkt_count) {
334		dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n",
335			oq->q_no, pkt_count, last_pkt_count);
336	}
337	/* Clear the hardware packets counter register if the rx queue is
338	 * being processed continuously with-in a single interrupt and
339	 * reached half its max value.
340	 * this counter is not cleared every time read, to save write cycles.
341	 */
342	if (unlikely(pkt_count > 0xF0000000U)) {
343		writel(pkt_count, oq->pkts_sent_reg);
344		pkt_count = readl(oq->pkts_sent_reg);
345		new_pkts += pkt_count;
346	}
347	WRITE_ONCE(oq->last_pkt_count, pkt_count);
348	pkts_pending = READ_ONCE(oq->pkts_pending);
349	WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts));
350	return new_pkts;
351}
352
353/**
354 * octep_oq_next_pkt() - Move to the next packet in Rx queue.
355 *
356 * @oq: Octeon Rx queue data structure.
357 * @buff_info: Current packet buffer info.
358 * @read_idx: Current packet index in the ring.
359 * @desc_used: Current packet descriptor number.
360 *
361 * Free the resources associated with a packet.
362 * Increment packet index in the ring and packet descriptor number.
363 */
364static void octep_oq_next_pkt(struct octep_oq *oq,
365			      struct octep_rx_buffer *buff_info,
366			      u32 *read_idx, u32 *desc_used)
367{
368	dma_unmap_page(oq->dev, oq->desc_ring[*read_idx].buffer_ptr,
369		       PAGE_SIZE, DMA_FROM_DEVICE);
370	buff_info->page = NULL;
371	(*read_idx)++;
372	(*desc_used)++;
373	if (*read_idx == oq->max_count)
374		*read_idx = 0;
375}
376
377/**
378 * octep_oq_drop_rx() - Free the resources associated with a packet.
379 *
380 * @oq: Octeon Rx queue data structure.
381 * @buff_info: Current packet buffer info.
382 * @read_idx: Current packet index in the ring.
383 * @desc_used: Current packet descriptor number.
384 *
385 */
386static void octep_oq_drop_rx(struct octep_oq *oq,
387			     struct octep_rx_buffer *buff_info,
388			     u32 *read_idx, u32 *desc_used)
389{
390	int data_len = buff_info->len - oq->max_single_buffer_size;
391
392	while (data_len > 0) {
393		octep_oq_next_pkt(oq, buff_info, read_idx, desc_used);
394		data_len -= oq->buffer_size;
395	};
396}
397
398/**
399 * __octep_oq_process_rx() - Process hardware Rx queue and push to stack.
400 *
401 * @oct: Octeon device private data structure.
402 * @oq: Octeon Rx queue data structure.
403 * @pkts_to_process: number of packets to be processed.
404 *
405 * Process the new packets in Rx queue.
406 * Packets larger than single Rx buffer arrive in consecutive descriptors.
407 * But, count returned by the API only accounts full packets, not fragments.
408 *
409 * Return: number of packets processed and pushed to stack.
410 */
411static int __octep_oq_process_rx(struct octep_device *oct,
412				 struct octep_oq *oq, u16 pkts_to_process)
413{
414	struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;
415	netdev_features_t feat = oq->netdev->features;
416	struct octep_rx_buffer *buff_info;
417	struct octep_oq_resp_hw *resp_hw;
418	u32 pkt, rx_bytes, desc_used;
419	struct sk_buff *skb;
420	u16 data_offset;
421	u16 rx_ol_flags;
422	u32 read_idx;
423
424	read_idx = READ_ONCE(oq->host_read_idx);
425	rx_bytes = 0;
426	desc_used = 0;
427	for (pkt = 0; pkt < pkts_to_process; pkt++) {
428		buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];
429		resp_hw = page_address(buff_info->page);
430
431		/* Swap the length field that is in Big-Endian to CPU */
432		buff_info->len = be64_to_cpu(resp_hw->length);
433		if (oct->conf->fw_info.rx_ol_flags) {
434			/* Extended response header is immediately after
435			 * response header (resp_hw)
436			 */
437			resp_hw_ext = (struct octep_oq_resp_hw_ext *)
438				      (resp_hw + 1);
439			buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;
440			/* Packet Data is immediately after
441			 * extended response header.
442			 */
443			data_offset = OCTEP_OQ_RESP_HW_SIZE +
444				      OCTEP_OQ_RESP_HW_EXT_SIZE;
445			rx_ol_flags = resp_hw_ext->rx_ol_flags;
446		} else {
447			/* Data is immediately after
448			 * Hardware Rx response header.
449			 */
450			data_offset = OCTEP_OQ_RESP_HW_SIZE;
451			rx_ol_flags = 0;
452		}
453
454		octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
455
456		skb = build_skb((void *)resp_hw, PAGE_SIZE);
457		if (!skb) {
458			octep_oq_drop_rx(oq, buff_info,
459					 &read_idx, &desc_used);
460			oq->stats->alloc_failures++;
461			continue;
462		}
463		skb_reserve(skb, data_offset);
464
465		rx_bytes += buff_info->len;
466
467		if (buff_info->len <= oq->max_single_buffer_size) {
468			skb_put(skb, buff_info->len);
469		} else {
470			struct skb_shared_info *shinfo;
471			u16 data_len;
472
473			/* Head fragment includes response header(s);
474			 * subsequent fragments contains only data.
475			 */
476			skb_put(skb, oq->max_single_buffer_size);
477			shinfo = skb_shinfo(skb);
478			data_len = buff_info->len - oq->max_single_buffer_size;
479			while (data_len) {
480				buff_info = (struct octep_rx_buffer *)
481					    &oq->buff_info[read_idx];
482				if (data_len < oq->buffer_size) {
483					buff_info->len = data_len;
484					data_len = 0;
485				} else {
486					buff_info->len = oq->buffer_size;
487					data_len -= oq->buffer_size;
488				}
489
490				skb_add_rx_frag(skb, shinfo->nr_frags,
491						buff_info->page, 0,
492						buff_info->len,
493						buff_info->len);
494
495				octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
496			}
497		}
498
499		skb->dev = oq->netdev;
500		skb->protocol =  eth_type_trans(skb, skb->dev);
501		if (feat & NETIF_F_RXCSUM &&
502		    OCTEP_RX_CSUM_VERIFIED(rx_ol_flags))
503			skb->ip_summed = CHECKSUM_UNNECESSARY;
504		else
505			skb->ip_summed = CHECKSUM_NONE;
506		napi_gro_receive(oq->napi, skb);
507	}
508
509	WRITE_ONCE(oq->host_read_idx, read_idx);
510	oq->refill_count += desc_used;
511	oq->stats->packets += pkt;
512	oq->stats->bytes += rx_bytes;
513
514	return pkt;
515}
516
517/**
518 * octep_oq_process_rx() - Process Rx queue.
519 *
520 * @oq: Octeon Rx queue data structure.
521 * @budget: max number of packets can be processed in one invocation.
522 *
523 * Check for newly received packets and process them.
524 * Keeps checking for new packets until budget is used or no new packets seen.
525 *
526 * Return: number of packets processed.
527 */
528int octep_oq_process_rx(struct octep_oq *oq, int budget)
529{
530	u32 pkts_available, pkts_processed, total_pkts_processed;
531	struct octep_device *oct = oq->octep_dev;
532	u32 pkts_pending;
533
534	pkts_available = 0;
535	pkts_processed = 0;
536	total_pkts_processed = 0;
537	while (total_pkts_processed < budget) {
538		 /* update pending count only when current one exhausted */
539		pkts_pending = READ_ONCE(oq->pkts_pending);
540		if (pkts_pending == 0)
541			octep_oq_check_hw_for_pkts(oct, oq);
542		pkts_pending = READ_ONCE(oq->pkts_pending);
543		pkts_available = min(budget - total_pkts_processed,
544				     pkts_pending);
545		if (!pkts_available)
546			break;
547
548		pkts_processed = __octep_oq_process_rx(oct, oq,
549						       pkts_available);
550		pkts_pending = READ_ONCE(oq->pkts_pending);
551		WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed));
552		total_pkts_processed += pkts_processed;
553	}
554
555	if (oq->refill_count >= oq->refill_threshold) {
556		u32 desc_refilled = octep_oq_refill(oct, oq);
557
558		/* flush pending writes before updating credits */
559		wmb();
560		writel(desc_refilled, oq->pkts_credit_reg);
561	}
562
563	return total_pkts_processed;
564}