npu.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2025 AIROHA Inc
  4 * Author: Lorenzo Bianconi <lorenzo@kernel.org>
  5 */
  6#include <linux/kernel.h>
  7#include <net/flow_offload.h>
  8#include <net/pkt_cls.h>
  9
 10#include "mt76.h"
 11#include "dma.h"
 12#include "mt76_connac.h"
 13
 14#define MT76_NPU_RX_BUF_SIZE	(1800 + \
 15				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 16
 17int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q)
 18{
 19	int nframes = 0;
 20
 21	while (q->queued < q->ndesc - 1) {
 22		struct airoha_npu_rx_dma_desc *desc = (void *)q->desc;
 23		struct mt76_queue_entry *e = &q->entry[q->head];
 24		struct page *page;
 25		int offset;
 26
 27		e->buf = mt76_get_page_pool_buf(q, &offset, q->buf_size);
 28		if (!e->buf)
 29			break;
 30
 31		e->dma_len[0] = SKB_WITH_OVERHEAD(q->buf_size);
 32		page = virt_to_head_page(e->buf);
 33		e->dma_addr[0] = page_pool_get_dma_addr(page) + offset;
 34
 35		memset(&desc[q->head], 0, sizeof(*desc));
 36		desc[q->head].addr = e->dma_addr[0];
 37
 38		q->head = (q->head + 1) % q->ndesc;
 39		q->queued++;
 40		nframes++;
 41	}
 42
 43	return nframes;
 44}
 45
 46void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
 47{
 48	spin_lock_bh(&q->lock);
 49	while (q->queued > 0) {
 50		struct mt76_queue_entry *e = &q->entry[q->tail];
 51
 52		dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0],
 53					e->dma_len[0],
 54					page_pool_get_dma_dir(q->page_pool));
 55		mt76_put_page_pool_buf(e->buf, false);
 56		q->tail = (q->tail + 1) % q->ndesc;
 57		q->queued--;
 58	}
 59	spin_unlock_bh(&q->lock);
 60}
 61
 62static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev,
 63					struct mt76_queue *q,
 64					u32 *info)
 65{
 66	struct airoha_npu_rx_dma_desc *desc = (void *)q->desc;
 67	int i, nframes, index = q->tail;
 68	struct sk_buff *skb = NULL;
 69
 70	nframes = FIELD_GET(NPU_RX_DMA_PKT_COUNT_MASK, desc[index].info);
 71	nframes = max_t(int, nframes, 1);
 72
 73	for (i = 0; i < nframes; i++) {
 74		struct mt76_queue_entry *e = &q->entry[index];
 75		int len = FIELD_GET(NPU_RX_DMA_DESC_CUR_LEN_MASK,
 76				    desc[index].ctrl);
 77
 78		if (!FIELD_GET(NPU_RX_DMA_DESC_DONE_MASK, desc[index].ctrl)) {
 79			dev_kfree_skb(skb);
 80			return NULL;
 81		}
 82
 83		dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0],
 84					e->dma_len[0],
 85					page_pool_get_dma_dir(q->page_pool));
 86
 87		if (!skb) {
 88			skb = napi_build_skb(e->buf, q->buf_size);
 89			if (!skb)
 90				return NULL;
 91
 92			__skb_put(skb, len);
 93			skb_reset_mac_header(skb);
 94			skb_mark_for_recycle(skb);
 95		} else {
 96			struct skb_shared_info *shinfo = skb_shinfo(skb);
 97			struct page *page = virt_to_head_page(e->buf);
 98			int nr_frags = shinfo->nr_frags;
 99
100			if (nr_frags < ARRAY_SIZE(shinfo->frags))
101				skb_add_rx_frag(skb, nr_frags, page,
102						e->buf - page_address(page),
103						len, q->buf_size);
104		}
105
106		*info = desc[index].info;
107		index = (index + 1) % q->ndesc;
108	}
109	q->tail = index;
110	q->queued -= i;
111	Q_WRITE(q, dma_idx, q->tail);
112
113	return skb;
114}
115
116void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb,
117			u32 info)
118{
119	struct airoha_ppe_dev *ppe_dev;
120	u16 reason, hash;
121
122	if (!mt76_npu_device_active(dev))
123		return;
124
125	rcu_read_lock();
126
127	ppe_dev = rcu_dereference(dev->mmio.ppe_dev);
128	if (!ppe_dev)
129		goto out;
130
131	hash = FIELD_GET(NPU_RX_DMA_FOE_ID_MASK, info);
132	skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
133
134	reason = FIELD_GET(NPU_RX_DMA_CRSN_MASK, info);
135	if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) {
136		skb_set_mac_header(skb, 0);
137		airoha_ppe_dev_check_skb(ppe_dev, skb, hash, true);
138	}
139out:
140	rcu_read_unlock();
141}
142EXPORT_SYMBOL_GPL(mt76_npu_check_ppe);
143
144static int mt76_npu_rx_poll(struct napi_struct *napi, int budget)
145{
146	struct mt76_dev *dev = mt76_priv(napi->dev);
147	enum mt76_rxq_id qid = napi - dev->napi;
148	struct airoha_npu *npu;
149	int done = 0;
150
151	rcu_read_lock();
152
153	npu = rcu_dereference(dev->mmio.npu);
154	if (!npu)
155		goto out;
156
157	while (done < budget) {
158		struct sk_buff *skb;
159		u32 info = 0;
160
161		skb = mt76_npu_dequeue(dev, &dev->q_rx[qid], &info);
162		if (!skb)
163			break;
164
165		dev->drv->rx_skb(dev, qid, skb, &info);
166		mt76_rx_poll_complete(dev, qid, napi);
167		done++;
168	}
169
170	mt76_npu_fill_rx_queue(dev, &dev->q_rx[qid]);
171out:
172	if (done < budget && napi_complete(napi))
173		dev->drv->rx_poll_complete(dev, qid);
174
175	rcu_read_unlock();
176
177	return done;
178}
179
180static irqreturn_t mt76_npu_irq_handler(int irq, void *q_instance)
181{
182	struct mt76_queue *q = q_instance;
183	struct mt76_dev *dev = q->dev;
184	int qid = q - &dev->q_rx[0];
185	int index = qid - MT_RXQ_NPU0;
186	struct airoha_npu *npu;
187	u32 status;
188
189	rcu_read_lock();
190
191	npu = rcu_dereference(dev->mmio.npu);
192	if (!npu)
193		goto out;
194
195	status = airoha_npu_wlan_get_irq_status(npu, index);
196	airoha_npu_wlan_set_irq_status(npu, status);
197
198	airoha_npu_wlan_disable_irq(npu, index);
199	napi_schedule(&dev->napi[qid]);
200out:
201	rcu_read_unlock();
202
203	return IRQ_HANDLED;
204}
205
206int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q,
207			 struct sk_buff *skb, struct mt76_queue_buf *buf,
208			 void *txwi_ptr)
209{
210	u16 txwi_len = min_t(u16, phy->dev->drv->txwi_size, NPU_TXWI_LEN);
211	struct airoha_npu_tx_dma_desc *desc = (void *)q->desc;
212	int ret;
213
214	/* TODO: Take into account unlinear skbs */
215	memcpy(desc[q->head].txwi, txwi_ptr, txwi_len);
216	desc[q->head].addr = buf->addr;
217	desc[q->head].ctrl = FIELD_PREP(NPU_TX_DMA_DESC_VEND_LEN_MASK, txwi_len) |
218			     FIELD_PREP(NPU_TX_DMA_DESC_LEN_MASK, skb->len) |
219			     NPU_TX_DMA_DESC_DONE_MASK;
220
221	ret = q->head;
222	q->entry[q->head].skip_buf0 = true;
223	q->entry[q->head].skip_buf1 = true;
224	q->entry[q->head].txwi = NULL;
225	q->entry[q->head].skb = NULL;
226	q->entry[q->head].wcid = 0xffff;
227
228	q->head = (q->head + 1) % q->ndesc;
229	q->queued++;
230
231	return ret;
232}
233
234void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index)
235{
236	struct airoha_npu_tx_dma_desc *desc = (void *)q->desc;
237
238	if (!mt76_queue_is_npu_tx(q))
239		return;
240
241	desc[index].ctrl &= ~NPU_TX_DMA_DESC_DONE_MASK;
242}
243
244void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q)
245{
246	int qid = FIELD_GET(MT_QFLAG_WED_RING, q->flags);
247	bool xmit = mt76_queue_is_npu_tx(q);
248	struct airoha_npu *npu;
249
250	if (!mt76_queue_is_npu(q))
251		return;
252
253	npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex);
254	if (npu)
255		q->wed_regs = airoha_npu_wlan_get_queue_addr(npu, qid, xmit);
256}
257
258int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q)
259{
260	int err, irq, qid = q - &dev->q_rx[0];
261	int size, index = qid - MT_RXQ_NPU0;
262	struct airoha_npu *npu;
263	const char *name;
264
265	mutex_lock(&dev->mutex);
266
267	npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex);
268	irq = npu && index < ARRAY_SIZE(npu->irqs) ? npu->irqs[index]
269						   : -EINVAL;
270	if (irq < 0) {
271		err = irq;
272		goto out;
273	}
274
275	q->flags = MT_NPU_Q_RX(index);
276	size = qid == MT_RXQ_NPU1 ? NPU_RX1_DESC_NUM : NPU_RX0_DESC_NUM;
277	err = dev->queue_ops->alloc(dev, q, 0, size,
278				    MT76_NPU_RX_BUF_SIZE, 0);
279	if (err)
280		goto out;
281
282	name = devm_kasprintf(dev->dev, GFP_KERNEL, "mt76-npu.%d", index);
283	if (!name) {
284		err = -ENOMEM;
285		goto out;
286	}
287
288	err = devm_request_irq(dev->dev, irq, mt76_npu_irq_handler,
289			       IRQF_SHARED, name, q);
290	if (err)
291		goto out;
292
293	netif_napi_add(dev->napi_dev, &dev->napi[qid], mt76_npu_rx_poll);
294	mt76_npu_fill_rx_queue(dev, q);
295	napi_enable(&dev->napi[qid]);
296out:
297	mutex_unlock(&dev->mutex);
298
299	return err;
300}
301EXPORT_SYMBOL_GPL(mt76_npu_rx_queue_init);
302
303static int mt76_npu_setup_tc_block_cb(enum tc_setup_type type,
304				      void *type_data, void *cb_priv)
305{
306	struct mt76_phy *phy = cb_priv;
307	struct mt76_dev *dev = phy->dev;
308	struct airoha_ppe_dev *ppe_dev;
309	int err = -EOPNOTSUPP;
310
311	if (type != TC_SETUP_CLSFLOWER)
312		return -EOPNOTSUPP;
313
314	mutex_lock(&dev->mutex);
315
316	ppe_dev = rcu_dereference_protected(dev->mmio.ppe_dev, &dev->mutex);
317	if (ppe_dev)
318		err = airoha_ppe_dev_setup_tc_block_cb(ppe_dev, type_data);
319
320	mutex_unlock(&dev->mutex);
321
322	return err;
323}
324
325static int mt76_npu_setup_tc_block(struct mt76_phy *phy,
326				   struct net_device *dev,
327				   struct flow_block_offload *f)
328{
329	flow_setup_cb_t *cb = mt76_npu_setup_tc_block_cb;
330	static LIST_HEAD(block_cb_list);
331	struct flow_block_cb *block_cb;
332
333	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
334		return -EOPNOTSUPP;
335
336	if (!tc_can_offload(dev))
337		return -EOPNOTSUPP;
338
339	f->driver_block_list = &block_cb_list;
340	switch (f->command) {
341	case FLOW_BLOCK_BIND:
342		block_cb = flow_block_cb_lookup(f->block, cb, dev);
343		if (block_cb) {
344			flow_block_cb_incref(block_cb);
345			return 0;
346		}
347
348		block_cb = flow_block_cb_alloc(cb, dev, phy, NULL);
349		if (IS_ERR(block_cb))
350			return PTR_ERR(block_cb);
351
352		flow_block_cb_incref(block_cb);
353		flow_block_cb_add(block_cb, f);
354		list_add_tail(&block_cb->driver_list, &block_cb_list);
355		return 0;
356	case FLOW_BLOCK_UNBIND:
357		block_cb = flow_block_cb_lookup(f->block, cb, dev);
358		if (!block_cb)
359			return -ENOENT;
360
361		if (!flow_block_cb_decref(block_cb)) {
362			flow_block_cb_remove(block_cb, f);
363			list_del(&block_cb->driver_list);
364		}
365		return 0;
366	default:
367		return -EOPNOTSUPP;
368	}
369}
370
371int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
372			  struct net_device *dev, enum tc_setup_type type,
373			  void *type_data)
374{
375	struct mt76_phy *phy = hw->priv;
376
377	if (!tc_can_offload(dev))
378		return -EOPNOTSUPP;
379
380	if (!mt76_npu_device_active(phy->dev))
381		return -EOPNOTSUPP;
382
383	switch (type) {
384	case TC_SETUP_BLOCK:
385	case TC_SETUP_FT:
386		return mt76_npu_setup_tc_block(phy, dev, type_data);
387	default:
388		return -EOPNOTSUPP;
389	}
390}
391EXPORT_SYMBOL_GPL(mt76_npu_net_setup_tc);
392
393void mt76_npu_disable_irqs(struct mt76_dev *dev)
394{
395	struct airoha_npu *npu;
396	int i;
397
398	rcu_read_lock();
399
400	npu = rcu_dereference(dev->mmio.npu);
401	if (!npu)
402		goto unlock;
403
404	for (i = MT_RXQ_NPU0; i <= MT_RXQ_NPU1; i++) {
405		int qid = i - MT_RXQ_NPU0;
406		u32 status;
407
408		status = airoha_npu_wlan_get_irq_status(npu, qid);
409		airoha_npu_wlan_set_irq_status(npu, status);
410		airoha_npu_wlan_disable_irq(npu, qid);
411	}
412unlock:
413	rcu_read_unlock();
414}
415EXPORT_SYMBOL_GPL(mt76_npu_disable_irqs);
416
417int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type)
418{
419	struct airoha_ppe_dev *ppe_dev;
420	struct airoha_npu *npu;
421	int err = 0;
422
423	/* NPU offloading is only supported by MT7992 */
424	if (!is_mt7992(dev))
425		return 0;
426
427	mutex_lock(&dev->mutex);
428
429	npu = airoha_npu_get(dev->dev);
430	if (IS_ERR(npu)) {
431		request_module("airoha-npu");
432		npu = airoha_npu_get(dev->dev);
433	}
434
435	if (IS_ERR(npu)) {
436		err = PTR_ERR(npu);
437		goto error_unlock;
438	}
439
440	ppe_dev = airoha_ppe_get_dev(dev->dev);
441	if (IS_ERR(ppe_dev)) {
442		request_module("airoha-eth");
443		ppe_dev = airoha_ppe_get_dev(dev->dev);
444	}
445
446	if (IS_ERR(ppe_dev)) {
447		err = PTR_ERR(ppe_dev);
448		goto error_npu_put;
449	}
450
451	err = airoha_npu_wlan_init_reserved_memory(npu);
452	if (err)
453		goto error_ppe_put;
454
455	dev->dma_dev = npu->dev;
456	dev->mmio.phy_addr = phy_addr;
457	dev->mmio.npu_type = type;
458	/* NPU offloading requires HW-RRO for RX packet reordering. */
459	dev->hwrro_mode = MT76_HWRRO_V3_1;
460
461	rcu_assign_pointer(dev->mmio.npu, npu);
462	rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev);
463	synchronize_rcu();
464
465	mutex_unlock(&dev->mutex);
466
467	return 0;
468
469error_ppe_put:
470	airoha_ppe_put_dev(ppe_dev);
471error_npu_put:
472	airoha_npu_put(npu);
473error_unlock:
474	mutex_unlock(&dev->mutex);
475
476	return err;
477}
478EXPORT_SYMBOL_GPL(mt76_npu_init);
479
480void mt76_npu_deinit(struct mt76_dev *dev)
481{
482	struct airoha_ppe_dev *ppe_dev;
483	struct airoha_npu *npu;
484
485	mutex_lock(&dev->mutex);
486
487	npu = rcu_replace_pointer(dev->mmio.npu, NULL,
488				  lockdep_is_held(&dev->mutex));
489	if (npu)
490		airoha_npu_put(npu);
491
492	ppe_dev = rcu_replace_pointer(dev->mmio.ppe_dev, NULL,
493				      lockdep_is_held(&dev->mutex));
494	if (ppe_dev)
495		airoha_ppe_put_dev(ppe_dev);
496
497	mutex_unlock(&dev->mutex);
498
499	mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU0]);
500	mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU1]);
501}