t7xx_hif_dpmaif_tx.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2021, MediaTek Inc.
  4 * Copyright (c) 2021-2022, Intel Corporation.
  5 *
  6 * Authors:
  7 *  Amir Hanania <amir.hanania@intel.com>
  8 *  Haijun Liu <haijun.liu@mediatek.com>
  9 *  Eliot Lee <eliot.lee@intel.com>
 10 *  Moises Veleta <moises.veleta@intel.com>
 11 *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
 12 *
 13 * Contributors:
 14 *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
 15 *  Sreehari Kancharla <sreehari.kancharla@intel.com>
 16 */
 17
 18#include <linux/atomic.h>
 19#include <linux/bitfield.h>
 20#include <linux/delay.h>
 21#include <linux/device.h>
 22#include <linux/dma-direction.h>
 23#include <linux/dma-mapping.h>
 24#include <linux/err.h>
 25#include <linux/gfp.h>
 26#include <linux/kernel.h>
 27#include <linux/kthread.h>
 28#include <linux/list.h>
 29#include <linux/minmax.h>
 30#include <linux/netdevice.h>
 31#include <linux/pm_runtime.h>
 32#include <linux/sched.h>
 33#include <linux/spinlock.h>
 34#include <linux/skbuff.h>
 35#include <linux/types.h>
 36#include <linux/wait.h>
 37#include <linux/workqueue.h>
 38
 39#include "t7xx_dpmaif.h"
 40#include "t7xx_hif_dpmaif.h"
 41#include "t7xx_hif_dpmaif_tx.h"
 42#include "t7xx_pci.h"
 43
 44#define DPMAIF_SKB_TX_BURST_CNT	5
 45#define DPMAIF_DRB_LIST_LEN	6144
 46
 47/* DRB dtype */
 48#define DES_DTYP_PD		0
 49#define DES_DTYP_MSG		1
 50
 51static unsigned int t7xx_dpmaif_update_drb_rd_idx(struct dpmaif_ctrl *dpmaif_ctrl,
 52						  unsigned int q_num)
 53{
 54	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num];
 55	unsigned int old_sw_rd_idx, new_hw_rd_idx, drb_cnt;
 56	unsigned long flags;
 57
 58	if (!txq->que_started)
 59		return 0;
 60
 61	old_sw_rd_idx = txq->drb_rd_idx;
 62	new_hw_rd_idx = t7xx_dpmaif_ul_get_rd_idx(&dpmaif_ctrl->hw_info, q_num);
 63	if (new_hw_rd_idx >= DPMAIF_DRB_LIST_LEN) {
 64		dev_err(dpmaif_ctrl->dev, "Out of range read index: %u\n", new_hw_rd_idx);
 65		return 0;
 66	}
 67
 68	if (old_sw_rd_idx <= new_hw_rd_idx)
 69		drb_cnt = new_hw_rd_idx - old_sw_rd_idx;
 70	else
 71		drb_cnt = txq->drb_size_cnt - old_sw_rd_idx + new_hw_rd_idx;
 72
 73	spin_lock_irqsave(&txq->tx_lock, flags);
 74	txq->drb_rd_idx = new_hw_rd_idx;
 75	spin_unlock_irqrestore(&txq->tx_lock, flags);
 76
 77	return drb_cnt;
 78}
 79
 80static unsigned int t7xx_dpmaif_release_tx_buffer(struct dpmaif_ctrl *dpmaif_ctrl,
 81						  unsigned int q_num, unsigned int release_cnt)
 82{
 83	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num];
 84	struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks;
 85	struct dpmaif_drb_skb *cur_drb_skb, *drb_skb_base;
 86	struct dpmaif_drb *cur_drb, *drb_base;
 87	unsigned int drb_cnt, i, cur_idx;
 88	unsigned long flags;
 89
 90	drb_skb_base = txq->drb_skb_base;
 91	drb_base = txq->drb_base;
 92
 93	spin_lock_irqsave(&txq->tx_lock, flags);
 94	drb_cnt = txq->drb_size_cnt;
 95	cur_idx = txq->drb_release_rd_idx;
 96	spin_unlock_irqrestore(&txq->tx_lock, flags);
 97
 98	for (i = 0; i < release_cnt; i++) {
 99		cur_drb = drb_base + cur_idx;
100		if (FIELD_GET(DRB_HDR_DTYP, le32_to_cpu(cur_drb->header)) == DES_DTYP_PD) {
101			cur_drb_skb = drb_skb_base + cur_idx;
102			if (!cur_drb_skb->is_msg)
103				dma_unmap_single(dpmaif_ctrl->dev, cur_drb_skb->bus_addr,
104						 cur_drb_skb->data_len, DMA_TO_DEVICE);
105
106			if (!FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) {
107				if (!cur_drb_skb->skb) {
108					dev_err(dpmaif_ctrl->dev,
109						"txq%u: DRB check fail, invalid skb\n", q_num);
110					continue;
111				}
112
113				dev_kfree_skb_any(cur_drb_skb->skb);
114			}
115
116			cur_drb_skb->skb = NULL;
117		}
118
119		spin_lock_irqsave(&txq->tx_lock, flags);
120		cur_idx = t7xx_ring_buf_get_next_wr_idx(drb_cnt, cur_idx);
121		txq->drb_release_rd_idx = cur_idx;
122		spin_unlock_irqrestore(&txq->tx_lock, flags);
123
124		if (atomic_inc_return(&txq->tx_budget) > txq->drb_size_cnt / 8)
125			cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_IRQ, txq->index);
126	}
127
128	if (FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header)))
129		dev_err(dpmaif_ctrl->dev, "txq%u: DRB not marked as the last one\n", q_num);
130
131	return i;
132}
133
134static int t7xx_dpmaif_tx_release(struct dpmaif_ctrl *dpmaif_ctrl,
135				  unsigned int q_num, unsigned int budget)
136{
137	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num];
138	unsigned int rel_cnt, real_rel_cnt;
139
140	/* Update read index from HW */
141	t7xx_dpmaif_update_drb_rd_idx(dpmaif_ctrl, q_num);
142
143	rel_cnt = t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx,
144					    txq->drb_rd_idx, DPMAIF_READ);
145
146	real_rel_cnt = min_not_zero(budget, rel_cnt);
147	if (real_rel_cnt)
148		real_rel_cnt = t7xx_dpmaif_release_tx_buffer(dpmaif_ctrl, q_num, real_rel_cnt);
149
150	return real_rel_cnt < rel_cnt ? -EAGAIN : 0;
151}
152
153static bool t7xx_dpmaif_drb_ring_not_empty(struct dpmaif_tx_queue *txq)
154{
155	return !!t7xx_dpmaif_update_drb_rd_idx(txq->dpmaif_ctrl, txq->index);
156}
157
158static void t7xx_dpmaif_tx_done(struct work_struct *work)
159{
160	struct dpmaif_tx_queue *txq = container_of(work, struct dpmaif_tx_queue, dpmaif_tx_work);
161	struct dpmaif_ctrl *dpmaif_ctrl = txq->dpmaif_ctrl;
162	struct dpmaif_hw_info *hw_info;
163	int ret;
164
165	ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev);
166	if (ret < 0 && ret != -EACCES)
167		return;
168
169	/* The device may be in low power state. Disable sleep if needed */
170	t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev);
171	if (t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) {
172		hw_info = &dpmaif_ctrl->hw_info;
173		ret = t7xx_dpmaif_tx_release(dpmaif_ctrl, txq->index, txq->drb_size_cnt);
174		if (ret == -EAGAIN ||
175		    (t7xx_dpmaif_ul_clr_done(hw_info, txq->index) &&
176		     t7xx_dpmaif_drb_ring_not_empty(txq))) {
177			queue_work(dpmaif_ctrl->txq[txq->index].worker,
178				   &dpmaif_ctrl->txq[txq->index].dpmaif_tx_work);
179			/* Give the device time to enter the low power state */
180			t7xx_dpmaif_clr_ip_busy_sts(hw_info);
181		} else {
182			t7xx_dpmaif_clr_ip_busy_sts(hw_info);
183			t7xx_dpmaif_unmask_ulq_intr(hw_info, txq->index);
184		}
185	}
186
187	t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev);
188	pm_runtime_put_autosuspend(dpmaif_ctrl->dev);
189}
190
191static void t7xx_setup_msg_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num,
192			       unsigned int cur_idx, unsigned int pkt_len, unsigned int count_l,
193			       unsigned int channel_id)
194{
195	struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base;
196	struct dpmaif_drb *drb = drb_base + cur_idx;
197
198	drb->header = cpu_to_le32(FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_MSG) |
199				  FIELD_PREP(DRB_HDR_CONT, 1) |
200				  FIELD_PREP(DRB_HDR_DATA_LEN, pkt_len));
201
202	drb->msg.msg_hdr = cpu_to_le32(FIELD_PREP(DRB_MSG_COUNT_L, count_l) |
203				       FIELD_PREP(DRB_MSG_CHANNEL_ID, channel_id) |
204				       FIELD_PREP(DRB_MSG_L4_CHK, 1));
205}
206
207static void t7xx_setup_payload_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num,
208				   unsigned int cur_idx, dma_addr_t data_addr,
209				   unsigned int pkt_size, bool last_one)
210{
211	struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base;
212	struct dpmaif_drb *drb = drb_base + cur_idx;
213	u32 header;
214
215	header = FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_PD) | FIELD_PREP(DRB_HDR_DATA_LEN, pkt_size);
216	if (!last_one)
217		header |= FIELD_PREP(DRB_HDR_CONT, 1);
218
219	drb->header = cpu_to_le32(header);
220	drb->pd.data_addr_l = cpu_to_le32(lower_32_bits(data_addr));
221	drb->pd.data_addr_h = cpu_to_le32(upper_32_bits(data_addr));
222}
223
224static void t7xx_record_drb_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num,
225				unsigned int cur_idx, struct sk_buff *skb, bool is_msg,
226				bool is_frag, bool is_last_one, dma_addr_t bus_addr,
227				unsigned int data_len)
228{
229	struct dpmaif_drb_skb *drb_skb_base = dpmaif_ctrl->txq[q_num].drb_skb_base;
230	struct dpmaif_drb_skb *drb_skb = drb_skb_base + cur_idx;
231
232	drb_skb->skb = skb;
233	drb_skb->bus_addr = bus_addr;
234	drb_skb->data_len = data_len;
235	drb_skb->index = cur_idx;
236	drb_skb->is_msg = is_msg;
237	drb_skb->is_frag = is_frag;
238	drb_skb->is_last = is_last_one;
239}
240
241static int t7xx_dpmaif_add_skb_to_ring(struct dpmaif_ctrl *dpmaif_ctrl, struct sk_buff *skb)
242{
243	struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks;
244	unsigned int wr_cnt, send_cnt, payload_cnt;
245	unsigned int cur_idx, drb_wr_idx_backup;
246	struct skb_shared_info *shinfo;
247	struct dpmaif_tx_queue *txq;
248	struct t7xx_skb_cb *skb_cb;
249	unsigned long flags;
250
251	skb_cb = T7XX_SKB_CB(skb);
252	txq = &dpmaif_ctrl->txq[skb_cb->txq_number];
253	if (!txq->que_started || dpmaif_ctrl->state != DPMAIF_STATE_PWRON)
254		return -ENODEV;
255
256	atomic_set(&txq->tx_processing, 1);
257	 /* Ensure tx_processing is changed to 1 before actually begin TX flow */
258	smp_mb();
259
260	shinfo = skb_shinfo(skb);
261	if (shinfo->frag_list)
262		dev_warn_ratelimited(dpmaif_ctrl->dev, "frag_list not supported\n");
263
264	payload_cnt = shinfo->nr_frags + 1;
265	/* nr_frags: frag cnt, 1: skb->data, 1: msg DRB */
266	send_cnt = payload_cnt + 1;
267
268	spin_lock_irqsave(&txq->tx_lock, flags);
269	cur_idx = txq->drb_wr_idx;
270	drb_wr_idx_backup = cur_idx;
271	txq->drb_wr_idx += send_cnt;
272	if (txq->drb_wr_idx >= txq->drb_size_cnt)
273		txq->drb_wr_idx -= txq->drb_size_cnt;
274	t7xx_setup_msg_drb(dpmaif_ctrl, txq->index, cur_idx, skb->len, 0, skb_cb->netif_idx);
275	t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, true, 0, 0, 0, 0);
276	spin_unlock_irqrestore(&txq->tx_lock, flags);
277
278	for (wr_cnt = 0; wr_cnt < payload_cnt; wr_cnt++) {
279		bool is_frag, is_last_one = wr_cnt == payload_cnt - 1;
280		unsigned int data_len;
281		dma_addr_t bus_addr;
282		void *data_addr;
283
284		if (!wr_cnt) {
285			data_len = skb_headlen(skb);
286			data_addr = skb->data;
287			is_frag = false;
288		} else {
289			skb_frag_t *frag = shinfo->frags + wr_cnt - 1;
290
291			data_len = skb_frag_size(frag);
292			data_addr = skb_frag_address(frag);
293			is_frag = true;
294		}
295
296		bus_addr = dma_map_single(dpmaif_ctrl->dev, data_addr, data_len, DMA_TO_DEVICE);
297		if (dma_mapping_error(dpmaif_ctrl->dev, bus_addr))
298			goto unmap_buffers;
299
300		cur_idx = t7xx_ring_buf_get_next_wr_idx(txq->drb_size_cnt, cur_idx);
301
302		spin_lock_irqsave(&txq->tx_lock, flags);
303		t7xx_setup_payload_drb(dpmaif_ctrl, txq->index, cur_idx, bus_addr, data_len,
304				       is_last_one);
305		t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, false, is_frag,
306				    is_last_one, bus_addr, data_len);
307		spin_unlock_irqrestore(&txq->tx_lock, flags);
308	}
309
310	if (atomic_sub_return(send_cnt, &txq->tx_budget) <= (MAX_SKB_FRAGS + 2))
311		cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq->index);
312
313	atomic_set(&txq->tx_processing, 0);
314
315	return 0;
316
317unmap_buffers:
318	while (wr_cnt--) {
319		struct dpmaif_drb_skb *drb_skb = txq->drb_skb_base;
320
321		cur_idx = cur_idx ? cur_idx - 1 : txq->drb_size_cnt - 1;
322		drb_skb += cur_idx;
323		dma_unmap_single(dpmaif_ctrl->dev, drb_skb->bus_addr,
324				 drb_skb->data_len, DMA_TO_DEVICE);
325	}
326
327	txq->drb_wr_idx = drb_wr_idx_backup;
328	atomic_set(&txq->tx_processing, 0);
329
330	return -ENOMEM;
331}
332
333static bool t7xx_tx_lists_are_all_empty(const struct dpmaif_ctrl *dpmaif_ctrl)
334{
335	int i;
336
337	for (i = 0; i < DPMAIF_TXQ_NUM; i++) {
338		if (!skb_queue_empty(&dpmaif_ctrl->txq[i].tx_skb_head))
339			return false;
340	}
341
342	return true;
343}
344
345/* Currently, only the default TX queue is used */
346static struct dpmaif_tx_queue *t7xx_select_tx_queue(struct dpmaif_ctrl *dpmaif_ctrl)
347{
348	struct dpmaif_tx_queue *txq;
349
350	txq = &dpmaif_ctrl->txq[DPMAIF_TX_DEFAULT_QUEUE];
351	if (!txq->que_started)
352		return NULL;
353
354	return txq;
355}
356
357static unsigned int t7xx_txq_drb_wr_available(struct dpmaif_tx_queue *txq)
358{
359	return t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx,
360					 txq->drb_wr_idx, DPMAIF_WRITE);
361}
362
363static unsigned int t7xx_skb_drb_cnt(struct sk_buff *skb)
364{
365	/* Normal DRB (frags data + skb linear data) + msg DRB */
366	return skb_shinfo(skb)->nr_frags + 2;
367}
368
369static int t7xx_txq_burst_send_skb(struct dpmaif_tx_queue *txq)
370{
371	unsigned int drb_remain_cnt, i;
372	unsigned int send_drb_cnt;
373	int drb_cnt = 0;
374	int ret = 0;
375
376	drb_remain_cnt = t7xx_txq_drb_wr_available(txq);
377
378	for (i = 0; i < DPMAIF_SKB_TX_BURST_CNT; i++) {
379		struct sk_buff *skb;
380
381		skb = skb_peek(&txq->tx_skb_head);
382		if (!skb)
383			break;
384
385		send_drb_cnt = t7xx_skb_drb_cnt(skb);
386		if (drb_remain_cnt < send_drb_cnt) {
387			drb_remain_cnt = t7xx_txq_drb_wr_available(txq);
388			continue;
389		}
390
391		drb_remain_cnt -= send_drb_cnt;
392
393		ret = t7xx_dpmaif_add_skb_to_ring(txq->dpmaif_ctrl, skb);
394		if (ret < 0) {
395			dev_err(txq->dpmaif_ctrl->dev,
396				"Failed to add skb to device's ring: %d\n", ret);
397			break;
398		}
399
400		drb_cnt += send_drb_cnt;
401		skb_unlink(skb, &txq->tx_skb_head);
402	}
403
404	if (drb_cnt > 0)
405		return drb_cnt;
406
407	return ret;
408}
409
410static void t7xx_do_tx_hw_push(struct dpmaif_ctrl *dpmaif_ctrl)
411{
412	bool wait_disable_sleep = true;
413
414	do {
415		struct dpmaif_tx_queue *txq;
416		int drb_send_cnt;
417
418		txq = t7xx_select_tx_queue(dpmaif_ctrl);
419		if (!txq)
420			return;
421
422		drb_send_cnt = t7xx_txq_burst_send_skb(txq);
423		if (drb_send_cnt <= 0) {
424			usleep_range(10, 20);
425			cond_resched();
426			continue;
427		}
428
429		/* Wait for the PCIe resource to unlock */
430		if (wait_disable_sleep) {
431			if (!t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev))
432				return;
433
434			wait_disable_sleep = false;
435		}
436
437		t7xx_dpmaif_ul_update_hw_drb_cnt(&dpmaif_ctrl->hw_info, txq->index,
438						 drb_send_cnt * DPMAIF_UL_DRB_SIZE_WORD);
439
440		cond_resched();
441	} while (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && !kthread_should_stop() &&
442		 (dpmaif_ctrl->state == DPMAIF_STATE_PWRON));
443}
444
445static int t7xx_dpmaif_tx_hw_push_thread(void *arg)
446{
447	struct dpmaif_ctrl *dpmaif_ctrl = arg;
448	int ret;
449
450	while (!kthread_should_stop()) {
451		if (t7xx_tx_lists_are_all_empty(dpmaif_ctrl) ||
452		    dpmaif_ctrl->state != DPMAIF_STATE_PWRON) {
453			if (wait_event_interruptible(dpmaif_ctrl->tx_wq,
454						     (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) &&
455						     dpmaif_ctrl->state == DPMAIF_STATE_PWRON) ||
456						     kthread_should_stop()))
457				continue;
458
459			if (kthread_should_stop())
460				break;
461		}
462
463		ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev);
464		if (ret < 0 && ret != -EACCES)
465			return ret;
466
467		t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev);
468		t7xx_do_tx_hw_push(dpmaif_ctrl);
469		t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev);
470		pm_runtime_put_autosuspend(dpmaif_ctrl->dev);
471	}
472
473	return 0;
474}
475
476int t7xx_dpmaif_tx_thread_init(struct dpmaif_ctrl *dpmaif_ctrl)
477{
478	init_waitqueue_head(&dpmaif_ctrl->tx_wq);
479	dpmaif_ctrl->tx_thread = kthread_run(t7xx_dpmaif_tx_hw_push_thread,
480					     dpmaif_ctrl, "dpmaif_tx_hw_push");
481	return PTR_ERR_OR_ZERO(dpmaif_ctrl->tx_thread);
482}
483
484void t7xx_dpmaif_tx_thread_rel(struct dpmaif_ctrl *dpmaif_ctrl)
485{
486	if (dpmaif_ctrl->tx_thread)
487		kthread_stop(dpmaif_ctrl->tx_thread);
488}
489
490/**
491 * t7xx_dpmaif_tx_send_skb() - Add skb to the transmit queue.
492 * @dpmaif_ctrl: Pointer to struct dpmaif_ctrl.
493 * @txq_number: Queue number to xmit on.
494 * @skb: Pointer to the skb to transmit.
495 *
496 * Add the skb to the queue of the skbs to be transmit.
497 * Wake up the thread that push the skbs from the queue to the HW.
498 *
499 * Return:
500 * * 0		- Success.
501 * * -EBUSY	- Tx budget exhausted.
502 *		  In normal circumstances t7xx_dpmaif_add_skb_to_ring() must report the txq full
503 *		  state to prevent this error condition.
504 */
505int t7xx_dpmaif_tx_send_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int txq_number,
506			    struct sk_buff *skb)
507{
508	struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[txq_number];
509	struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks;
510	struct t7xx_skb_cb *skb_cb;
511
512	if (atomic_read(&txq->tx_budget) <= t7xx_skb_drb_cnt(skb)) {
513		cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq_number);
514		return -EBUSY;
515	}
516
517	skb_cb = T7XX_SKB_CB(skb);
518	skb_cb->txq_number = txq_number;
519	skb_queue_tail(&txq->tx_skb_head, skb);
520	wake_up(&dpmaif_ctrl->tx_wq);
521
522	return 0;
523}
524
525void t7xx_dpmaif_irq_tx_done(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int que_mask)
526{
527	int i;
528
529	for (i = 0; i < DPMAIF_TXQ_NUM; i++) {
530		if (que_mask & BIT(i))
531			queue_work(dpmaif_ctrl->txq[i].worker, &dpmaif_ctrl->txq[i].dpmaif_tx_work);
532	}
533}
534
535static int t7xx_dpmaif_tx_drb_buf_init(struct dpmaif_tx_queue *txq)
536{
537	size_t brb_skb_size, brb_pd_size;
538
539	brb_pd_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb);
540	brb_skb_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb_skb);
541
542	txq->drb_size_cnt = DPMAIF_DRB_LIST_LEN;
543
544	/* For HW && AP SW */
545	txq->drb_base = dma_alloc_coherent(txq->dpmaif_ctrl->dev, brb_pd_size,
546					   &txq->drb_bus_addr, GFP_KERNEL | __GFP_ZERO);
547	if (!txq->drb_base)
548		return -ENOMEM;
549
550	/* For AP SW to record the skb information */
551	txq->drb_skb_base = devm_kzalloc(txq->dpmaif_ctrl->dev, brb_skb_size, GFP_KERNEL);
552	if (!txq->drb_skb_base) {
553		dma_free_coherent(txq->dpmaif_ctrl->dev, brb_pd_size,
554				  txq->drb_base, txq->drb_bus_addr);
555		return -ENOMEM;
556	}
557
558	return 0;
559}
560
561static void t7xx_dpmaif_tx_free_drb_skb(struct dpmaif_tx_queue *txq)
562{
563	struct dpmaif_drb_skb *drb_skb, *drb_skb_base = txq->drb_skb_base;
564	unsigned int i;
565
566	if (!drb_skb_base)
567		return;
568
569	for (i = 0; i < txq->drb_size_cnt; i++) {
570		drb_skb = drb_skb_base + i;
571		if (!drb_skb->skb)
572			continue;
573
574		if (!drb_skb->is_msg)
575			dma_unmap_single(txq->dpmaif_ctrl->dev, drb_skb->bus_addr,
576					 drb_skb->data_len, DMA_TO_DEVICE);
577
578		if (drb_skb->is_last) {
579			dev_kfree_skb(drb_skb->skb);
580			drb_skb->skb = NULL;
581		}
582	}
583}
584
585static void t7xx_dpmaif_tx_drb_buf_rel(struct dpmaif_tx_queue *txq)
586{
587	if (txq->drb_base)
588		dma_free_coherent(txq->dpmaif_ctrl->dev,
589				  txq->drb_size_cnt * sizeof(struct dpmaif_drb),
590				  txq->drb_base, txq->drb_bus_addr);
591
592	t7xx_dpmaif_tx_free_drb_skb(txq);
593}
594
595/**
596 * t7xx_dpmaif_txq_init() - Initialize TX queue.
597 * @txq: Pointer to struct dpmaif_tx_queue.
598 *
599 * Initialize the TX queue data structure and allocate memory for it to use.
600 *
601 * Return:
602 * * 0		- Success.
603 * * -ERROR	- Error code from failure sub-initializations.
604 */
605int t7xx_dpmaif_txq_init(struct dpmaif_tx_queue *txq)
606{
607	int ret;
608
609	skb_queue_head_init(&txq->tx_skb_head);
610	init_waitqueue_head(&txq->req_wq);
611	atomic_set(&txq->tx_budget, DPMAIF_DRB_LIST_LEN);
612
613	ret = t7xx_dpmaif_tx_drb_buf_init(txq);
614	if (ret) {
615		dev_err(txq->dpmaif_ctrl->dev, "Failed to initialize DRB buffers: %d\n", ret);
616		return ret;
617	}
618
619	txq->worker = alloc_ordered_workqueue("md_dpmaif_tx%d_worker",
620				WQ_MEM_RECLAIM | (txq->index ? 0 : WQ_HIGHPRI),
621				txq->index);
622	if (!txq->worker)
623		return -ENOMEM;
624
625	INIT_WORK(&txq->dpmaif_tx_work, t7xx_dpmaif_tx_done);
626	spin_lock_init(&txq->tx_lock);
627
628	return 0;
629}
630
631void t7xx_dpmaif_txq_free(struct dpmaif_tx_queue *txq)
632{
633	if (txq->worker)
634		destroy_workqueue(txq->worker);
635
636	skb_queue_purge(&txq->tx_skb_head);
637	t7xx_dpmaif_tx_drb_buf_rel(txq);
638}
639
640void t7xx_dpmaif_tx_stop(struct dpmaif_ctrl *dpmaif_ctrl)
641{
642	int i;
643
644	for (i = 0; i < DPMAIF_TXQ_NUM; i++) {
645		struct dpmaif_tx_queue *txq;
646		int count = 0;
647
648		txq = &dpmaif_ctrl->txq[i];
649		txq->que_started = false;
650		/* Make sure TXQ is disabled */
651		smp_mb();
652
653		/* Wait for active Tx to be done */
654		while (atomic_read(&txq->tx_processing)) {
655			if (++count >= DPMAIF_MAX_CHECK_COUNT) {
656				dev_err(dpmaif_ctrl->dev, "TX queue stop failed\n");
657				break;
658			}
659		}
660	}
661}
662
663static void t7xx_dpmaif_txq_flush_rel(struct dpmaif_tx_queue *txq)
664{
665	txq->que_started = false;
666
667	cancel_work_sync(&txq->dpmaif_tx_work);
668	flush_work(&txq->dpmaif_tx_work);
669	t7xx_dpmaif_tx_free_drb_skb(txq);
670
671	txq->drb_rd_idx = 0;
672	txq->drb_wr_idx = 0;
673	txq->drb_release_rd_idx = 0;
674}
675
676void t7xx_dpmaif_tx_clear(struct dpmaif_ctrl *dpmaif_ctrl)
677{
678	int i;
679
680	for (i = 0; i < DPMAIF_TXQ_NUM; i++)
681		t7xx_dpmaif_txq_flush_rel(&dpmaif_ctrl->txq[i]);
682}