mlxbf_gige_rx.c

  1// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
  2
  3/* Packet receive logic for Mellanox Gigabit Ethernet driver
  4 *
  5 * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
  6 */
  7
  8#include <linux/etherdevice.h>
  9#include <linux/skbuff.h>
 10
 11#include "mlxbf_gige.h"
 12#include "mlxbf_gige_regs.h"
 13
 14void mlxbf_gige_enable_multicast_rx(struct mlxbf_gige *priv)
 15{
 16	void __iomem *base = priv->base;
 17	u64 data;
 18
 19	data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
 20	data |= MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST;
 21	writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
 22}
 23
 24void mlxbf_gige_disable_multicast_rx(struct mlxbf_gige *priv)
 25{
 26	void __iomem *base = priv->base;
 27	u64 data;
 28
 29	data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
 30	data &= ~MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST;
 31	writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
 32}
 33
 34void mlxbf_gige_enable_mac_rx_filter(struct mlxbf_gige *priv,
 35				     unsigned int index)
 36{
 37	void __iomem *base = priv->base;
 38	u64 control;
 39
 40	/* Enable MAC receive filter mask for specified index */
 41	control = readq(base + MLXBF_GIGE_CONTROL);
 42	control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
 43	writeq(control, base + MLXBF_GIGE_CONTROL);
 44}
 45
 46void mlxbf_gige_disable_mac_rx_filter(struct mlxbf_gige *priv,
 47				      unsigned int index)
 48{
 49	void __iomem *base = priv->base;
 50	u64 control;
 51
 52	/* Disable MAC receive filter mask for specified index */
 53	control = readq(base + MLXBF_GIGE_CONTROL);
 54	control &= ~(MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
 55	writeq(control, base + MLXBF_GIGE_CONTROL);
 56}
 57
 58void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
 59				  unsigned int index, u64 dmac)
 60{
 61	void __iomem *base = priv->base;
 62
 63	/* Write destination MAC to specified MAC RX filter */
 64	writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +
 65	       (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
 66}
 67
 68void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
 69				  unsigned int index, u64 *dmac)
 70{
 71	void __iomem *base = priv->base;
 72
 73	/* Read destination MAC from specified MAC RX filter */
 74	*dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +
 75		      (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
 76}
 77
 78void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)
 79{
 80	void __iomem *base = priv->base;
 81	u64 control;
 82	u64 end_mac;
 83
 84	/* Enable MAC_ID_RANGE match functionality */
 85	control = readq(base + MLXBF_GIGE_CONTROL);
 86	control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
 87	writeq(control, base + MLXBF_GIGE_CONTROL);
 88
 89	/* Set start of destination MAC range check to 0 */
 90	writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);
 91
 92	/* Set end of destination MAC range check to all FFs */
 93	end_mac = BCAST_MAC_ADDR;
 94	writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);
 95}
 96
 97void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)
 98{
 99	void __iomem *base = priv->base;
100	u64 control;
101
102	/* Disable MAC_ID_RANGE match functionality */
103	control = readq(base + MLXBF_GIGE_CONTROL);
104	control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
105	writeq(control, base + MLXBF_GIGE_CONTROL);
106
107	/* NOTE: no need to change DMAC_RANGE_START or END;
108	 * those values are ignored since MAC_ID_RANGE_EN=0
109	 */
110}
111
112/* Receive Initialization
113 * 1) Configures RX MAC filters via MMIO registers
114 * 2) Allocates RX WQE array using coherent DMA mapping
115 * 3) Initializes each element of RX WQE array with a receive
116 *    buffer pointer (also using coherent DMA mapping)
117 * 4) Allocates RX CQE array using coherent DMA mapping
118 * 5) Completes other misc receive initialization
119 */
120int mlxbf_gige_rx_init(struct mlxbf_gige *priv)
121{
122	size_t wq_size, cq_size;
123	dma_addr_t *rx_wqe_ptr;
124	dma_addr_t rx_buf_dma;
125	u64 data;
126	int i, j;
127
128	/* Configure MAC RX filter #0 to allow RX of broadcast pkts */
129	mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,
130				     BCAST_MAC_ADDR);
131
132	wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
133	priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,
134					       &priv->rx_wqe_base_dma,
135					       GFP_KERNEL);
136	if (!priv->rx_wqe_base)
137		return -ENOMEM;
138
139	/* Initialize 'rx_wqe_ptr' to point to first RX WQE in array
140	 * Each RX WQE is simply a receive buffer pointer, so walk
141	 * the entire array, allocating a 2KB buffer for each element
142	 */
143	rx_wqe_ptr = priv->rx_wqe_base;
144
145	for (i = 0; i < priv->rx_q_entries; i++) {
146		priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
147						       &rx_buf_dma, DMA_FROM_DEVICE);
148		if (!priv->rx_skb[i])
149			goto free_wqe_and_skb;
150		*rx_wqe_ptr++ = rx_buf_dma;
151	}
152
153	/* Write RX WQE base address into MMIO reg */
154	writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);
155
156	cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
157	priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,
158					       &priv->rx_cqe_base_dma,
159					       GFP_KERNEL);
160	if (!priv->rx_cqe_base)
161		goto free_wqe_and_skb;
162
163	for (i = 0; i < priv->rx_q_entries; i++)
164		priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK;
165
166	/* Write RX CQE base address into MMIO reg */
167	writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);
168
169	/* Write RX_WQE_PI with current number of replenished buffers */
170	writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);
171
172	/* Enable removal of CRC during RX */
173	data = readq(priv->base + MLXBF_GIGE_RX);
174	data |= MLXBF_GIGE_RX_STRIP_CRC_EN;
175	writeq(data, priv->base + MLXBF_GIGE_RX);
176
177	/* Enable RX MAC filter pass and discard counters */
178	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,
179	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);
180	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,
181	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);
182
183	writeq(ilog2(priv->rx_q_entries),
184	       priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);
185
186	/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
187	 * indicate readiness to receive interrupts
188	 */
189	data = readq(priv->base + MLXBF_GIGE_INT_MASK);
190	data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
191	writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
192
193	/* Enable RX DMA to write new packets to memory */
194	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
195	data |= MLXBF_GIGE_RX_DMA_EN;
196	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
197
198	return 0;
199
200free_wqe_and_skb:
201	rx_wqe_ptr = priv->rx_wqe_base;
202	for (j = 0; j < i; j++) {
203		dma_unmap_single(priv->dev, *rx_wqe_ptr,
204				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
205		dev_kfree_skb(priv->rx_skb[j]);
206		rx_wqe_ptr++;
207	}
208	dma_free_coherent(priv->dev, wq_size,
209			  priv->rx_wqe_base, priv->rx_wqe_base_dma);
210	return -ENOMEM;
211}
212
213/* Receive Deinitialization
214 * This routine will free allocations done by mlxbf_gige_rx_init(),
215 * namely the RX WQE and RX CQE arrays, as well as all RX buffers
216 */
217void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)
218{
219	dma_addr_t *rx_wqe_ptr;
220	size_t size;
221	u64 data;
222	int i;
223
224	/* Disable RX DMA to prevent packet transfers to memory */
225	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
226	data &= ~MLXBF_GIGE_RX_DMA_EN;
227	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
228
229	rx_wqe_ptr = priv->rx_wqe_base;
230
231	for (i = 0; i < priv->rx_q_entries; i++) {
232		dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,
233				 DMA_FROM_DEVICE);
234		dev_kfree_skb(priv->rx_skb[i]);
235		rx_wqe_ptr++;
236	}
237
238	size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
239	dma_free_coherent(priv->dev, size,
240			  priv->rx_wqe_base, priv->rx_wqe_base_dma);
241
242	size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
243	dma_free_coherent(priv->dev, size,
244			  priv->rx_cqe_base, priv->rx_cqe_base_dma);
245
246	priv->rx_wqe_base = NULL;
247	priv->rx_wqe_base_dma = 0;
248	priv->rx_cqe_base = NULL;
249	priv->rx_cqe_base_dma = 0;
250	writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);
251	writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);
252}
253
254static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts)
255{
256	struct net_device *netdev = priv->netdev;
257	struct sk_buff *skb = NULL, *rx_skb;
258	u16 rx_pi_rem, rx_ci_rem;
259	dma_addr_t *rx_wqe_addr;
260	dma_addr_t rx_buf_dma;
261	u64 *rx_cqe_addr;
262	u64 datalen;
263	u64 rx_cqe;
264	u16 rx_ci;
265	u16 rx_pi;
266
267	/* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */
268	rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);
269	rx_pi_rem = rx_pi % priv->rx_q_entries;
270
271	rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;
272	rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;
273	rx_cqe = *rx_cqe_addr;
274
275	if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)
276		return false;
277
278	if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {
279		/* Packet is OK, increment stats */
280		datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;
281		netdev->stats.rx_packets++;
282		netdev->stats.rx_bytes += datalen;
283
284		skb = priv->rx_skb[rx_pi_rem];
285
286		/* Alloc another RX SKB for this same index */
287		rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
288					      &rx_buf_dma, DMA_FROM_DEVICE);
289		if (!rx_skb)
290			return false;
291		priv->rx_skb[rx_pi_rem] = rx_skb;
292		dma_unmap_single(priv->dev, *rx_wqe_addr,
293				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
294
295		skb_put(skb, datalen);
296
297		skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */
298
299		skb->protocol = eth_type_trans(skb, netdev);
300
301		*rx_wqe_addr = rx_buf_dma;
302	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {
303		priv->stats.rx_mac_errors++;
304	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {
305		priv->stats.rx_truncate_errors++;
306	}
307
308	/* Read receive consumer index before replenish so that this routine
309	 * returns accurate return value even if packet is received into
310	 * just-replenished buffer prior to exiting this routine.
311	 */
312	rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);
313	rx_ci_rem = rx_ci % priv->rx_q_entries;
314
315	/* Let hardware know we've replenished one buffer */
316	rx_pi++;
317
318	/* Ensure completion of all writes before notifying HW of replenish */
319	wmb();
320	writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);
321
322	(*rx_pkts)++;
323
324	rx_pi_rem = rx_pi % priv->rx_q_entries;
325	if (rx_pi_rem == 0)
326		priv->valid_polarity ^= 1;
327
328	if (skb)
329		netif_receive_skb(skb);
330
331	return rx_pi_rem != rx_ci_rem;
332}
333
334/* Driver poll() function called by NAPI infrastructure */
335int mlxbf_gige_poll(struct napi_struct *napi, int budget)
336{
337	struct mlxbf_gige *priv;
338	bool remaining_pkts;
339	int work_done = 0;
340	u64 data;
341
342	priv = container_of(napi, struct mlxbf_gige, napi);
343
344	mlxbf_gige_handle_tx_complete(priv);
345
346	do {
347		remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);
348	} while (remaining_pkts && work_done < budget);
349
350	/* If amount of work done < budget, turn off NAPI polling
351	 * via napi_complete_done(napi, work_done) and then
352	 * re-enable interrupts.
353	 */
354	if (work_done < budget && napi_complete_done(napi, work_done)) {
355		/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
356		 * indicate receive readiness
357		 */
358		data = readq(priv->base + MLXBF_GIGE_INT_MASK);
359		data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
360		writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
361	}
362
363	return work_done;
364}