octep_vf_rx.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Marvell Octeon EP (EndPoint) VF Ethernet Driver
  3 *
  4 * Copyright (C) 2020 Marvell.
  5 *
  6 */
  7
  8#include <linux/pci.h>
  9#include <linux/etherdevice.h>
 10#include <linux/vmalloc.h>
 11
 12#include "octep_vf_config.h"
 13#include "octep_vf_main.h"
 14
 15static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq);
 16
 17static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq)
 18{
 19	oq->host_read_idx = 0;
 20	oq->host_refill_idx = 0;
 21	oq->refill_count = 0;
 22	oq->last_pkt_count = 0;
 23	oq->pkts_pending = 0;
 24}
 25
 26/**
 27 * octep_vf_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
 28 *
 29 * @oq: Octeon Rx queue data structure.
 30 *
 31 * Return: 0, if successfully filled receive buffers for all descriptors.
 32 *         -ENOMEM, if failed to allocate a buffer or failed to map for DMA.
 33 */
 34static int octep_vf_oq_fill_ring_buffers(struct octep_vf_oq *oq)
 35{
 36	struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;
 37	struct page *page;
 38	u32 i;
 39
 40	for (i = 0; i < oq->max_count; i++) {
 41		page = dev_alloc_page();
 42		if (unlikely(!page)) {
 43			dev_err(oq->dev, "Rx buffer alloc failed\n");
 44			goto rx_buf_alloc_err;
 45		}
 46		desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
 47						       PAGE_SIZE,
 48						       DMA_FROM_DEVICE);
 49		if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
 50			dev_err(oq->dev,
 51				"OQ-%d buffer alloc: DMA mapping error!\n",
 52				oq->q_no);
 53			goto dma_map_err;
 54		}
 55		oq->buff_info[i].page = page;
 56	}
 57
 58	return 0;
 59
 60dma_map_err:
 61	put_page(page);
 62rx_buf_alloc_err:
 63	while (i) {
 64		i--;
 65		dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
 66		put_page(oq->buff_info[i].page);
 67		oq->buff_info[i].page = NULL;
 68	}
 69
 70	return -ENOMEM;
 71}
 72
 73/**
 74 * octep_vf_oq_refill() - refill buffers for used Rx ring descriptors.
 75 *
 76 * @oct: Octeon device private data structure.
 77 * @oq: Octeon Rx queue data structure.
 78 *
 79 * Return: number of descriptors successfully refilled with receive buffers.
 80 */
 81static int octep_vf_oq_refill(struct octep_vf_device *oct, struct octep_vf_oq *oq)
 82{
 83	struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;
 84	struct page *page;
 85	u32 refill_idx, i;
 86
 87	refill_idx = oq->host_refill_idx;
 88	for (i = 0; i < oq->refill_count; i++) {
 89		page = dev_alloc_page();
 90		if (unlikely(!page)) {
 91			dev_err(oq->dev, "refill: rx buffer alloc failed\n");
 92			oq->stats->alloc_failures++;
 93			break;
 94		}
 95
 96		desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
 97								PAGE_SIZE, DMA_FROM_DEVICE);
 98		if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
 99			dev_err(oq->dev,
100				"OQ-%d buffer refill: DMA mapping error!\n",
101				oq->q_no);
102			put_page(page);
103			oq->stats->alloc_failures++;
104			break;
105		}
106		oq->buff_info[refill_idx].page = page;
107		refill_idx++;
108		if (refill_idx == oq->max_count)
109			refill_idx = 0;
110	}
111	oq->host_refill_idx = refill_idx;
112	oq->refill_count -= i;
113
114	return i;
115}
116
117/**
118 * octep_vf_setup_oq() - Setup a Rx queue.
119 *
120 * @oct: Octeon device private data structure.
121 * @q_no: Rx queue number to be setup.
122 *
123 * Allocate resources for a Rx queue.
124 */
125static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no)
126{
127	struct octep_vf_oq *oq;
128	u32 desc_ring_size;
129
130	oq = vzalloc(sizeof(*oq));
131	if (!oq)
132		goto create_oq_fail;
133	oct->oq[q_no] = oq;
134
135	oq->octep_vf_dev = oct;
136	oq->netdev = oct->netdev;
137	oq->dev = &oct->pdev->dev;
138	oq->q_no = q_no;
139	oq->stats = &oct->stats_oq[q_no];
140	oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
141	oq->ring_size_mask = oq->max_count - 1;
142	oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
143	oq->max_single_buffer_size = oq->buffer_size - OCTEP_VF_OQ_RESP_HW_SIZE;
144
145	/* When the hardware/firmware supports additional capabilities,
146	 * additional header is filled-in by Octeon after length field in
147	 * Rx packets. this header contains additional packet information.
148	 */
149	if (oct->fw_info.rx_ol_flags)
150		oq->max_single_buffer_size -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE;
151
152	oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
153
154	desc_ring_size = oq->max_count * OCTEP_VF_OQ_DESC_SIZE;
155	oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
156					   &oq->desc_ring_dma, GFP_KERNEL);
157
158	if (unlikely(!oq->desc_ring)) {
159		dev_err(oq->dev,
160			"Failed to allocate DMA memory for OQ-%d !!\n", q_no);
161		goto desc_dma_alloc_err;
162	}
163
164	oq->buff_info = vzalloc(oq->max_count * OCTEP_VF_OQ_RECVBUF_SIZE);
165
166	if (unlikely(!oq->buff_info)) {
167		dev_err(&oct->pdev->dev,
168			"Failed to allocate buffer info for OQ-%d\n", q_no);
169		goto buf_list_err;
170	}
171
172	if (octep_vf_oq_fill_ring_buffers(oq))
173		goto oq_fill_buff_err;
174
175	octep_vf_oq_reset_indices(oq);
176	if (oct->hw_ops.setup_oq_regs(oct, q_no))
177		goto oq_setup_err;
178
179	oct->num_oqs++;
180
181	return 0;
182
183oq_setup_err:
184	octep_vf_oq_free_ring_buffers(oq);
185oq_fill_buff_err:
186	vfree(oq->buff_info);
187	oq->buff_info = NULL;
188buf_list_err:
189	dma_free_coherent(oq->dev, desc_ring_size,
190			  oq->desc_ring, oq->desc_ring_dma);
191	oq->desc_ring = NULL;
192desc_dma_alloc_err:
193	vfree(oq);
194	oct->oq[q_no] = NULL;
195create_oq_fail:
196	return -ENOMEM;
197}
198
199/**
200 * octep_vf_oq_free_ring_buffers() - Free ring buffers.
201 *
202 * @oq: Octeon Rx queue data structure.
203 *
204 * Free receive buffers in unused Rx queue descriptors.
205 */
206static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq)
207{
208	struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;
209	int  i;
210
211	if (!oq->desc_ring || !oq->buff_info)
212		return;
213
214	for (i = 0; i < oq->max_count; i++)  {
215		if (oq->buff_info[i].page) {
216			dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
217				       PAGE_SIZE, DMA_FROM_DEVICE);
218			put_page(oq->buff_info[i].page);
219			oq->buff_info[i].page = NULL;
220			desc_ring[i].buffer_ptr = 0;
221		}
222	}
223	octep_vf_oq_reset_indices(oq);
224}
225
226/**
227 * octep_vf_free_oq() - Free Rx queue resources.
228 *
229 * @oq: Octeon Rx queue data structure.
230 *
231 * Free all resources of a Rx queue.
232 */
233static int octep_vf_free_oq(struct octep_vf_oq *oq)
234{
235	struct octep_vf_device *oct = oq->octep_vf_dev;
236	int q_no = oq->q_no;
237
238	octep_vf_oq_free_ring_buffers(oq);
239
240	vfree(oq->buff_info);
241
242	if (oq->desc_ring)
243		dma_free_coherent(oq->dev,
244				  oq->max_count * OCTEP_VF_OQ_DESC_SIZE,
245				  oq->desc_ring, oq->desc_ring_dma);
246
247	vfree(oq);
248	oct->oq[q_no] = NULL;
249	oct->num_oqs--;
250	return 0;
251}
252
253/**
254 * octep_vf_setup_oqs() - setup resources for all Rx queues.
255 *
256 * @oct: Octeon device private data structure.
257 */
258int octep_vf_setup_oqs(struct octep_vf_device *oct)
259{
260	int i, retval = 0;
261
262	oct->num_oqs = 0;
263	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
264		retval = octep_vf_setup_oq(oct, i);
265		if (retval) {
266			dev_err(&oct->pdev->dev,
267				"Failed to setup OQ(RxQ)-%d.\n", i);
268			goto oq_setup_err;
269		}
270		dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
271	}
272
273	return 0;
274
275oq_setup_err:
276	while (i) {
277		i--;
278		octep_vf_free_oq(oct->oq[i]);
279	}
280	return retval;
281}
282
283/**
284 * octep_vf_oq_dbell_init() - Initialize Rx queue doorbell.
285 *
286 * @oct: Octeon device private data structure.
287 *
288 * Write number of descriptors to Rx queue doorbell register.
289 */
290void octep_vf_oq_dbell_init(struct octep_vf_device *oct)
291{
292	int i;
293
294	for (i = 0; i < oct->num_oqs; i++)
295		writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
296}
297
298/**
299 * octep_vf_free_oqs() - Free resources of all Rx queues.
300 *
301 * @oct: Octeon device private data structure.
302 */
303void octep_vf_free_oqs(struct octep_vf_device *oct)
304{
305	int i;
306
307	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
308		if (!oct->oq[i])
309			continue;
310		octep_vf_free_oq(oct->oq[i]);
311		dev_dbg(&oct->pdev->dev,
312			"Successfully freed OQ(RxQ)-%d.\n", i);
313	}
314}
315
316/**
317 * octep_vf_oq_check_hw_for_pkts() - Check for new Rx packets.
318 *
319 * @oct: Octeon device private data structure.
320 * @oq: Octeon Rx queue data structure.
321 *
322 * Return: packets received after previous check.
323 */
324static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct,
325					 struct octep_vf_oq *oq)
326{
327	u32 pkt_count, new_pkts;
328	u32 last_pkt_count, pkts_pending;
329
330	pkt_count = readl(oq->pkts_sent_reg);
331	last_pkt_count = READ_ONCE(oq->last_pkt_count);
332	new_pkts = pkt_count - last_pkt_count;
333
334	if (pkt_count < last_pkt_count) {
335		dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n",
336			oq->q_no, pkt_count, last_pkt_count);
337	}
338
339	/* Clear the hardware packets counter register if the rx queue is
340	 * being processed continuously with-in a single interrupt and
341	 * reached half its max value.
342	 * this counter is not cleared every time read, to save write cycles.
343	 */
344	if (unlikely(pkt_count > 0xF0000000U)) {
345		writel(pkt_count, oq->pkts_sent_reg);
346		pkt_count = readl(oq->pkts_sent_reg);
347		new_pkts += pkt_count;
348	}
349	WRITE_ONCE(oq->last_pkt_count, pkt_count);
350	pkts_pending = READ_ONCE(oq->pkts_pending);
351	WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts));
352	return new_pkts;
353}
354
355/**
356 * __octep_vf_oq_process_rx() - Process hardware Rx queue and push to stack.
357 *
358 * @oct: Octeon device private data structure.
359 * @oq: Octeon Rx queue data structure.
360 * @pkts_to_process: number of packets to be processed.
361 *
362 * Process the new packets in Rx queue.
363 * Packets larger than single Rx buffer arrive in consecutive descriptors.
364 * But, count returned by the API only accounts full packets, not fragments.
365 *
366 * Return: number of packets processed and pushed to stack.
367 */
368static int __octep_vf_oq_process_rx(struct octep_vf_device *oct,
369				    struct octep_vf_oq *oq, u16 pkts_to_process)
370{
371	struct octep_vf_oq_resp_hw_ext *resp_hw_ext = NULL;
372	netdev_features_t feat = oq->netdev->features;
373	struct octep_vf_rx_buffer *buff_info;
374	struct octep_vf_oq_resp_hw *resp_hw;
375	u32 pkt, rx_bytes, desc_used;
376	u16 data_offset, rx_ol_flags;
377	struct sk_buff *skb;
378	u32 read_idx;
379
380	read_idx = READ_ONCE(oq->host_read_idx);
381	rx_bytes = 0;
382	desc_used = 0;
383	for (pkt = 0; pkt < pkts_to_process; pkt++) {
384		buff_info = (struct octep_vf_rx_buffer *)&oq->buff_info[read_idx];
385		dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
386			       PAGE_SIZE, DMA_FROM_DEVICE);
387		resp_hw = page_address(buff_info->page);
388		buff_info->page = NULL;
389
390		/* Swap the length field that is in Big-Endian to CPU */
391		buff_info->len = be64_to_cpu(resp_hw->length);
392		if (oct->fw_info.rx_ol_flags) {
393			/* Extended response header is immediately after
394			 * response header (resp_hw)
395			 */
396			resp_hw_ext = (struct octep_vf_oq_resp_hw_ext *)
397				      (resp_hw + 1);
398			buff_info->len -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE;
399			/* Packet Data is immediately after
400			 * extended response header.
401			 */
402			data_offset = OCTEP_VF_OQ_RESP_HW_SIZE +
403				      OCTEP_VF_OQ_RESP_HW_EXT_SIZE;
404			rx_ol_flags = resp_hw_ext->rx_ol_flags;
405		} else {
406			/* Data is immediately after
407			 * Hardware Rx response header.
408			 */
409			data_offset = OCTEP_VF_OQ_RESP_HW_SIZE;
410			rx_ol_flags = 0;
411		}
412		rx_bytes += buff_info->len;
413
414		if (buff_info->len <= oq->max_single_buffer_size) {
415			skb = napi_build_skb((void *)resp_hw, PAGE_SIZE);
416			skb_reserve(skb, data_offset);
417			skb_put(skb, buff_info->len);
418			read_idx++;
419			desc_used++;
420			if (read_idx == oq->max_count)
421				read_idx = 0;
422		} else {
423			struct skb_shared_info *shinfo;
424			u16 data_len;
425
426			skb = napi_build_skb((void *)resp_hw, PAGE_SIZE);
427			skb_reserve(skb, data_offset);
428			/* Head fragment includes response header(s);
429			 * subsequent fragments contains only data.
430			 */
431			skb_put(skb, oq->max_single_buffer_size);
432			read_idx++;
433			desc_used++;
434			if (read_idx == oq->max_count)
435				read_idx = 0;
436
437			shinfo = skb_shinfo(skb);
438			data_len = buff_info->len - oq->max_single_buffer_size;
439			while (data_len) {
440				dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
441					       PAGE_SIZE, DMA_FROM_DEVICE);
442				buff_info = (struct octep_vf_rx_buffer *)
443					    &oq->buff_info[read_idx];
444				if (data_len < oq->buffer_size) {
445					buff_info->len = data_len;
446					data_len = 0;
447				} else {
448					buff_info->len = oq->buffer_size;
449					data_len -= oq->buffer_size;
450				}
451
452				skb_add_rx_frag(skb, shinfo->nr_frags,
453						buff_info->page, 0,
454						buff_info->len,
455						buff_info->len);
456				buff_info->page = NULL;
457				read_idx++;
458				desc_used++;
459				if (read_idx == oq->max_count)
460					read_idx = 0;
461			}
462		}
463
464		skb->dev = oq->netdev;
465		skb->protocol = eth_type_trans(skb, skb->dev);
466		if (feat & NETIF_F_RXCSUM &&
467		    OCTEP_VF_RX_CSUM_VERIFIED(rx_ol_flags))
468			skb->ip_summed = CHECKSUM_UNNECESSARY;
469		else
470			skb->ip_summed = CHECKSUM_NONE;
471		napi_gro_receive(oq->napi, skb);
472	}
473
474	WRITE_ONCE(oq->host_read_idx, read_idx);
475	oq->refill_count += desc_used;
476	oq->stats->packets += pkt;
477	oq->stats->bytes += rx_bytes;
478
479	return pkt;
480}
481
482/**
483 * octep_vf_oq_process_rx() - Process Rx queue.
484 *
485 * @oq: Octeon Rx queue data structure.
486 * @budget: max number of packets can be processed in one invocation.
487 *
488 * Check for newly received packets and process them.
489 * Keeps checking for new packets until budget is used or no new packets seen.
490 *
491 * Return: number of packets processed.
492 */
493int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget)
494{
495	u32 pkts_available, pkts_processed, total_pkts_processed;
496	struct octep_vf_device *oct = oq->octep_vf_dev;
497	u32 pkts_pending;
498
499	pkts_available = 0;
500	pkts_processed = 0;
501	total_pkts_processed = 0;
502	while (total_pkts_processed < budget) {
503		 /* update pending count only when current one exhausted */
504		pkts_pending = READ_ONCE(oq->pkts_pending);
505		if (pkts_pending == 0)
506			octep_vf_oq_check_hw_for_pkts(oct, oq);
507		pkts_pending = READ_ONCE(oq->pkts_pending);
508		pkts_available = min(budget - total_pkts_processed,
509				     pkts_pending);
510		if (!pkts_available)
511			break;
512
513		pkts_processed = __octep_vf_oq_process_rx(oct, oq,
514							  pkts_available);
515		pkts_pending = READ_ONCE(oq->pkts_pending);
516		WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed));
517		total_pkts_processed += pkts_processed;
518	}
519
520	if (oq->refill_count >= oq->refill_threshold) {
521		u32 desc_refilled = octep_vf_oq_refill(oct, oq);
522
523		/* flush pending writes before updating credits */
524		smp_wmb();
525		writel(desc_refilled, oq->pkts_credit_reg);
526	}
527
528	return total_pkts_processed;
529}