t7xx_hif_cldma.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2021, MediaTek Inc.
   4 * Copyright (c) 2021-2022, Intel Corporation.
   5 *
   6 * Authors:
   7 *  Amir Hanania <amir.hanania@intel.com>
   8 *  Haijun Liu <haijun.liu@mediatek.com>
   9 *  Moises Veleta <moises.veleta@intel.com>
  10 *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
  11 *  Sreehari Kancharla <sreehari.kancharla@intel.com>
  12 *
  13 * Contributors:
  14 *  Andy Shevchenko <andriy.shevchenko@linux.intel.com>
  15 *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
  16 *  Eliot Lee <eliot.lee@intel.com>
  17 */
  18
  19#include <linux/bits.h>
  20#include <linux/bitops.h>
  21#include <linux/delay.h>
  22#include <linux/device.h>
  23#include <linux/dmapool.h>
  24#include <linux/dma-mapping.h>
  25#include <linux/dma-direction.h>
  26#include <linux/gfp.h>
  27#include <linux/io.h>
  28#include <linux/io-64-nonatomic-lo-hi.h>
  29#include <linux/iopoll.h>
  30#include <linux/irqreturn.h>
  31#include <linux/kernel.h>
  32#include <linux/kthread.h>
  33#include <linux/list.h>
  34#include <linux/netdevice.h>
  35#include <linux/pci.h>
  36#include <linux/pm_runtime.h>
  37#include <linux/sched.h>
  38#include <linux/skbuff.h>
  39#include <linux/slab.h>
  40#include <linux/spinlock.h>
  41#include <linux/types.h>
  42#include <linux/wait.h>
  43#include <linux/workqueue.h>
  44
  45#include "t7xx_cldma.h"
  46#include "t7xx_hif_cldma.h"
  47#include "t7xx_mhccif.h"
  48#include "t7xx_pci.h"
  49#include "t7xx_pcie_mac.h"
  50#include "t7xx_port_proxy.h"
  51#include "t7xx_reg.h"
  52#include "t7xx_state_monitor.h"
  53
  54#define MAX_TX_BUDGET			16
  55#define MAX_RX_BUDGET			16
  56
  57#define CHECK_Q_STOP_TIMEOUT_US		1000000
  58#define CHECK_Q_STOP_STEP_US		10000
  59
  60static void md_cd_queue_struct_reset(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
  61				     enum mtk_txrx tx_rx, unsigned int index)
  62{
  63	queue->dir = tx_rx;
  64	queue->index = index;
  65	queue->md_ctrl = md_ctrl;
  66	queue->tr_ring = NULL;
  67	queue->tr_done = NULL;
  68	queue->tx_next = NULL;
  69}
  70
  71static void md_cd_queue_struct_init(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl,
  72				    enum mtk_txrx tx_rx, unsigned int index)
  73{
  74	md_cd_queue_struct_reset(queue, md_ctrl, tx_rx, index);
  75	init_waitqueue_head(&queue->req_wq);
  76	spin_lock_init(&queue->ring_lock);
  77}
  78
  79static void t7xx_cldma_gpd_set_data_ptr(struct cldma_gpd *gpd, dma_addr_t data_ptr)
  80{
  81	gpd->data_buff_bd_ptr_h = cpu_to_le32(upper_32_bits(data_ptr));
  82	gpd->data_buff_bd_ptr_l = cpu_to_le32(lower_32_bits(data_ptr));
  83}
  84
  85static void t7xx_cldma_gpd_set_next_ptr(struct cldma_gpd *gpd, dma_addr_t next_ptr)
  86{
  87	gpd->next_gpd_ptr_h = cpu_to_le32(upper_32_bits(next_ptr));
  88	gpd->next_gpd_ptr_l = cpu_to_le32(lower_32_bits(next_ptr));
  89}
  90
  91static int t7xx_cldma_alloc_and_map_skb(struct cldma_ctrl *md_ctrl, struct cldma_request *req,
  92					size_t size, gfp_t gfp_mask)
  93{
  94	req->skb = __dev_alloc_skb(size, gfp_mask);
  95	if (!req->skb)
  96		return -ENOMEM;
  97
  98	req->mapped_buff = dma_map_single(md_ctrl->dev, req->skb->data, size, DMA_FROM_DEVICE);
  99	if (dma_mapping_error(md_ctrl->dev, req->mapped_buff)) {
 100		dev_kfree_skb_any(req->skb);
 101		req->skb = NULL;
 102		req->mapped_buff = 0;
 103		dev_err(md_ctrl->dev, "DMA mapping failed\n");
 104		return -ENOMEM;
 105	}
 106
 107	return 0;
 108}
 109
 110static int t7xx_cldma_gpd_rx_from_q(struct cldma_queue *queue, int budget, bool *over_budget)
 111{
 112	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 113	unsigned int hwo_polling_count = 0;
 114	struct t7xx_cldma_hw *hw_info;
 115	bool rx_not_done = true;
 116	unsigned long flags;
 117	int count = 0;
 118
 119	hw_info = &md_ctrl->hw_info;
 120
 121	do {
 122		struct cldma_request *req;
 123		struct cldma_gpd *gpd;
 124		struct sk_buff *skb;
 125		int ret;
 126
 127		req = queue->tr_done;
 128		if (!req)
 129			return -ENODATA;
 130
 131		gpd = req->gpd;
 132		if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
 133			dma_addr_t gpd_addr;
 134
 135			if (!pci_device_is_present(to_pci_dev(md_ctrl->dev))) {
 136				dev_err(md_ctrl->dev, "PCIe Link disconnected\n");
 137				return -ENODEV;
 138			}
 139
 140			gpd_addr = ioread64_lo_hi(hw_info->ap_pdn_base +
 141						  REG_CLDMA_DL_CURRENT_ADDRL_0 +
 142						  queue->index * sizeof(u64));
 143			if (req->gpd_addr == gpd_addr || hwo_polling_count++ >= 100)
 144				return 0;
 145
 146			udelay(1);
 147			continue;
 148		}
 149
 150		hwo_polling_count = 0;
 151		skb = req->skb;
 152
 153		if (req->mapped_buff) {
 154			dma_unmap_single(md_ctrl->dev, req->mapped_buff,
 155					 queue->tr_ring->pkt_size, DMA_FROM_DEVICE);
 156			req->mapped_buff = 0;
 157		}
 158
 159		skb->len = 0;
 160		skb_reset_tail_pointer(skb);
 161		skb_put(skb, le16_to_cpu(gpd->data_buff_len));
 162
 163		ret = queue->recv_skb(queue, skb);
 164		/* Break processing, will try again later */
 165		if (ret < 0)
 166			return ret;
 167
 168		req->skb = NULL;
 169		t7xx_cldma_gpd_set_data_ptr(gpd, 0);
 170
 171		spin_lock_irqsave(&queue->ring_lock, flags);
 172		queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 173		spin_unlock_irqrestore(&queue->ring_lock, flags);
 174		req = queue->rx_refill;
 175
 176		ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, queue->tr_ring->pkt_size, GFP_KERNEL);
 177		if (ret)
 178			return ret;
 179
 180		gpd = req->gpd;
 181		t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
 182		gpd->data_buff_len = 0;
 183		gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 184
 185		spin_lock_irqsave(&queue->ring_lock, flags);
 186		queue->rx_refill = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 187		spin_unlock_irqrestore(&queue->ring_lock, flags);
 188
 189		rx_not_done = ++count < budget || !need_resched();
 190	} while (rx_not_done);
 191
 192	*over_budget = true;
 193	return 0;
 194}
 195
 196static int t7xx_cldma_gpd_rx_collect(struct cldma_queue *queue, int budget)
 197{
 198	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 199	struct t7xx_cldma_hw *hw_info;
 200	unsigned int pending_rx_int;
 201	bool over_budget = false;
 202	unsigned long flags;
 203	int ret;
 204
 205	hw_info = &md_ctrl->hw_info;
 206
 207	do {
 208		ret = t7xx_cldma_gpd_rx_from_q(queue, budget, &over_budget);
 209		if (ret == -ENODATA)
 210			return 0;
 211		else if (ret)
 212			return ret;
 213
 214		pending_rx_int = 0;
 215
 216		spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 217		if (md_ctrl->rxq_active & BIT(queue->index)) {
 218			if (!t7xx_cldma_hw_queue_status(hw_info, queue->index, MTK_RX))
 219				t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_RX);
 220
 221			pending_rx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index),
 222								  MTK_RX);
 223			if (pending_rx_int) {
 224				t7xx_cldma_hw_rx_done(hw_info, pending_rx_int);
 225
 226				if (over_budget) {
 227					spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 228					return -EAGAIN;
 229				}
 230			}
 231		}
 232		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 233	} while (pending_rx_int);
 234
 235	return 0;
 236}
 237
 238static void t7xx_cldma_rx_done(struct work_struct *work)
 239{
 240	struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
 241	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 242	int value;
 243
 244	value = t7xx_cldma_gpd_rx_collect(queue, queue->budget);
 245	if (value && md_ctrl->rxq_active & BIT(queue->index)) {
 246		queue_work(queue->worker, &queue->cldma_work);
 247		return;
 248	}
 249
 250	t7xx_cldma_clear_ip_busy(&md_ctrl->hw_info);
 251	t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, queue->index, MTK_RX);
 252	t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, queue->index, MTK_RX);
 253	pm_runtime_put_autosuspend(md_ctrl->dev);
 254}
 255
 256static int t7xx_cldma_gpd_tx_collect(struct cldma_queue *queue)
 257{
 258	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 259	unsigned int dma_len, count = 0;
 260	struct cldma_request *req;
 261	struct cldma_gpd *gpd;
 262	unsigned long flags;
 263	dma_addr_t dma_free;
 264	struct sk_buff *skb;
 265
 266	while (!kthread_should_stop()) {
 267		spin_lock_irqsave(&queue->ring_lock, flags);
 268		req = queue->tr_done;
 269		if (!req) {
 270			spin_unlock_irqrestore(&queue->ring_lock, flags);
 271			break;
 272		}
 273		gpd = req->gpd;
 274		if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) {
 275			spin_unlock_irqrestore(&queue->ring_lock, flags);
 276			break;
 277		}
 278		queue->budget++;
 279		dma_free = req->mapped_buff;
 280		dma_len = le16_to_cpu(gpd->data_buff_len);
 281		skb = req->skb;
 282		req->skb = NULL;
 283		queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry);
 284		spin_unlock_irqrestore(&queue->ring_lock, flags);
 285
 286		count++;
 287		dma_unmap_single(md_ctrl->dev, dma_free, dma_len, DMA_TO_DEVICE);
 288		dev_kfree_skb_any(skb);
 289	}
 290
 291	if (count)
 292		wake_up_nr(&queue->req_wq, count);
 293
 294	return count;
 295}
 296
 297static void t7xx_cldma_txq_empty_hndl(struct cldma_queue *queue)
 298{
 299	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 300	struct cldma_request *req;
 301	dma_addr_t ul_curr_addr;
 302	unsigned long flags;
 303	bool pending_gpd;
 304
 305	if (!(md_ctrl->txq_active & BIT(queue->index)))
 306		return;
 307
 308	spin_lock_irqsave(&queue->ring_lock, flags);
 309	req = list_prev_entry_circular(queue->tx_next, &queue->tr_ring->gpd_ring, entry);
 310	spin_unlock_irqrestore(&queue->ring_lock, flags);
 311
 312	pending_gpd = (req->gpd->flags & GPD_FLAGS_HWO) && req->skb;
 313
 314	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 315	if (pending_gpd) {
 316		struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 317
 318		/* Check current processing TGPD, 64-bit address is in a table by Q index */
 319		ul_curr_addr = ioread64_lo_hi(hw_info->ap_pdn_base + REG_CLDMA_UL_CURRENT_ADDRL_0 +
 320					      queue->index * sizeof(u64));
 321		if (req->gpd_addr != ul_curr_addr) {
 322			spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 323			dev_err(md_ctrl->dev, "CLDMA%d queue %d is not empty\n",
 324				md_ctrl->hif_id, queue->index);
 325			return;
 326		}
 327
 328		t7xx_cldma_hw_resume_queue(hw_info, queue->index, MTK_TX);
 329	}
 330	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 331}
 332
 333static void t7xx_cldma_tx_done(struct work_struct *work)
 334{
 335	struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work);
 336	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 337	struct t7xx_cldma_hw *hw_info;
 338	unsigned int l2_tx_int;
 339	unsigned long flags;
 340
 341	hw_info = &md_ctrl->hw_info;
 342	t7xx_cldma_gpd_tx_collect(queue);
 343	l2_tx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index) | EQ_STA_BIT(queue->index),
 344					     MTK_TX);
 345	if (l2_tx_int & EQ_STA_BIT(queue->index)) {
 346		t7xx_cldma_hw_tx_done(hw_info, EQ_STA_BIT(queue->index));
 347		t7xx_cldma_txq_empty_hndl(queue);
 348	}
 349
 350	if (l2_tx_int & BIT(queue->index)) {
 351		t7xx_cldma_hw_tx_done(hw_info, BIT(queue->index));
 352		queue_work(queue->worker, &queue->cldma_work);
 353		return;
 354	}
 355
 356	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 357	if (md_ctrl->txq_active & BIT(queue->index)) {
 358		t7xx_cldma_clear_ip_busy(hw_info);
 359		t7xx_cldma_hw_irq_en_eq(hw_info, queue->index, MTK_TX);
 360		t7xx_cldma_hw_irq_en_txrx(hw_info, queue->index, MTK_TX);
 361	}
 362	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 363
 364	pm_runtime_put_autosuspend(md_ctrl->dev);
 365}
 366
 367static void t7xx_cldma_ring_free(struct cldma_ctrl *md_ctrl,
 368				 struct cldma_ring *ring, enum dma_data_direction tx_rx)
 369{
 370	struct cldma_request *req_cur, *req_next;
 371
 372	list_for_each_entry_safe(req_cur, req_next, &ring->gpd_ring, entry) {
 373		if (req_cur->mapped_buff && req_cur->skb) {
 374			dma_unmap_single(md_ctrl->dev, req_cur->mapped_buff,
 375					 ring->pkt_size, tx_rx);
 376			req_cur->mapped_buff = 0;
 377		}
 378
 379		dev_kfree_skb_any(req_cur->skb);
 380
 381		if (req_cur->gpd)
 382			dma_pool_free(md_ctrl->gpd_dmapool, req_cur->gpd, req_cur->gpd_addr);
 383
 384		list_del(&req_cur->entry);
 385		kfree(req_cur);
 386	}
 387}
 388
 389static struct cldma_request *t7xx_alloc_rx_request(struct cldma_ctrl *md_ctrl, size_t pkt_size)
 390{
 391	struct cldma_request *req;
 392	int val;
 393
 394	req = kzalloc(sizeof(*req), GFP_KERNEL);
 395	if (!req)
 396		return NULL;
 397
 398	req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
 399	if (!req->gpd)
 400		goto err_free_req;
 401
 402	val = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, pkt_size, GFP_KERNEL);
 403	if (val)
 404		goto err_free_pool;
 405
 406	return req;
 407
 408err_free_pool:
 409	dma_pool_free(md_ctrl->gpd_dmapool, req->gpd, req->gpd_addr);
 410
 411err_free_req:
 412	kfree(req);
 413
 414	return NULL;
 415}
 416
 417static int t7xx_cldma_rx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
 418{
 419	struct cldma_request *req;
 420	struct cldma_gpd *gpd;
 421	int i;
 422
 423	INIT_LIST_HEAD(&ring->gpd_ring);
 424	ring->length = MAX_RX_BUDGET;
 425
 426	for (i = 0; i < ring->length; i++) {
 427		req = t7xx_alloc_rx_request(md_ctrl, ring->pkt_size);
 428		if (!req) {
 429			t7xx_cldma_ring_free(md_ctrl, ring, DMA_FROM_DEVICE);
 430			return -ENOMEM;
 431		}
 432
 433		gpd = req->gpd;
 434		t7xx_cldma_gpd_set_data_ptr(gpd, req->mapped_buff);
 435		gpd->rx_data_allow_len = cpu_to_le16(ring->pkt_size);
 436		gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 437		INIT_LIST_HEAD(&req->entry);
 438		list_add_tail(&req->entry, &ring->gpd_ring);
 439	}
 440
 441	/* Link previous GPD to next GPD, circular */
 442	list_for_each_entry(req, &ring->gpd_ring, entry) {
 443		t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
 444		gpd = req->gpd;
 445	}
 446
 447	return 0;
 448}
 449
 450static struct cldma_request *t7xx_alloc_tx_request(struct cldma_ctrl *md_ctrl)
 451{
 452	struct cldma_request *req;
 453
 454	req = kzalloc(sizeof(*req), GFP_KERNEL);
 455	if (!req)
 456		return NULL;
 457
 458	req->gpd = dma_pool_zalloc(md_ctrl->gpd_dmapool, GFP_KERNEL, &req->gpd_addr);
 459	if (!req->gpd) {
 460		kfree(req);
 461		return NULL;
 462	}
 463
 464	return req;
 465}
 466
 467static int t7xx_cldma_tx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring)
 468{
 469	struct cldma_request *req;
 470	struct cldma_gpd *gpd;
 471	int i;
 472
 473	INIT_LIST_HEAD(&ring->gpd_ring);
 474	ring->length = MAX_TX_BUDGET;
 475
 476	for (i = 0; i < ring->length; i++) {
 477		req = t7xx_alloc_tx_request(md_ctrl);
 478		if (!req) {
 479			t7xx_cldma_ring_free(md_ctrl, ring, DMA_TO_DEVICE);
 480			return -ENOMEM;
 481		}
 482
 483		gpd = req->gpd;
 484		gpd->flags = GPD_FLAGS_IOC;
 485		INIT_LIST_HEAD(&req->entry);
 486		list_add_tail(&req->entry, &ring->gpd_ring);
 487	}
 488
 489	/* Link previous GPD to next GPD, circular */
 490	list_for_each_entry(req, &ring->gpd_ring, entry) {
 491		t7xx_cldma_gpd_set_next_ptr(gpd, req->gpd_addr);
 492		gpd = req->gpd;
 493	}
 494
 495	return 0;
 496}
 497
 498/**
 499 * t7xx_cldma_q_reset() - Reset CLDMA request pointers to their initial values.
 500 * @queue: Pointer to the queue structure.
 501 *
 502 * Called with ring_lock (unless called during initialization phase)
 503 */
 504static void t7xx_cldma_q_reset(struct cldma_queue *queue)
 505{
 506	struct cldma_request *req;
 507
 508	req = list_first_entry(&queue->tr_ring->gpd_ring, struct cldma_request, entry);
 509	queue->tr_done = req;
 510	queue->budget = queue->tr_ring->length;
 511
 512	if (queue->dir == MTK_TX)
 513		queue->tx_next = req;
 514	else
 515		queue->rx_refill = req;
 516}
 517
 518static void t7xx_cldma_rxq_init(struct cldma_queue *queue)
 519{
 520	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 521
 522	queue->dir = MTK_RX;
 523	queue->tr_ring = &md_ctrl->rx_ring[queue->index];
 524	t7xx_cldma_q_reset(queue);
 525}
 526
 527static void t7xx_cldma_txq_init(struct cldma_queue *queue)
 528{
 529	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 530
 531	queue->dir = MTK_TX;
 532	queue->tr_ring = &md_ctrl->tx_ring[queue->index];
 533	t7xx_cldma_q_reset(queue);
 534}
 535
 536static void t7xx_cldma_enable_irq(struct cldma_ctrl *md_ctrl)
 537{
 538	t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
 539}
 540
 541static void t7xx_cldma_disable_irq(struct cldma_ctrl *md_ctrl)
 542{
 543	t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, md_ctrl->hw_info.phy_interrupt_id);
 544}
 545
 546static void t7xx_cldma_irq_work_cb(struct cldma_ctrl *md_ctrl)
 547{
 548	unsigned long l2_tx_int_msk, l2_rx_int_msk, l2_tx_int, l2_rx_int, val;
 549	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 550	int i;
 551
 552	/* L2 raw interrupt status */
 553	l2_tx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TISAR0);
 554	l2_rx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2RISAR0);
 555	l2_tx_int_msk = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TIMR0);
 556	l2_rx_int_msk = ioread32(hw_info->ap_ao_base + REG_CLDMA_L2RIMR0);
 557	l2_tx_int &= ~l2_tx_int_msk;
 558	l2_rx_int &= ~l2_rx_int_msk;
 559
 560	if (l2_tx_int) {
 561		if (l2_tx_int & (TQ_ERR_INT_BITMASK | TQ_ACTIVE_START_ERR_INT_BITMASK)) {
 562			/* Read and clear L3 TX interrupt status */
 563			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
 564			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0);
 565			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
 566			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1);
 567		}
 568
 569		t7xx_cldma_hw_tx_done(hw_info, l2_tx_int);
 570		if (l2_tx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
 571			for_each_set_bit(i, &l2_tx_int, L2_INT_BIT_COUNT) {
 572				if (i < CLDMA_TXQ_NUM) {
 573					pm_runtime_get(md_ctrl->dev);
 574					t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_TX);
 575					t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_TX);
 576					queue_work(md_ctrl->txq[i].worker,
 577						   &md_ctrl->txq[i].cldma_work);
 578				} else {
 579					t7xx_cldma_txq_empty_hndl(&md_ctrl->txq[i - CLDMA_TXQ_NUM]);
 580				}
 581			}
 582		}
 583	}
 584
 585	if (l2_rx_int) {
 586		if (l2_rx_int & (RQ_ERR_INT_BITMASK | RQ_ACTIVE_START_ERR_INT_BITMASK)) {
 587			/* Read and clear L3 RX interrupt status */
 588			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
 589			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0);
 590			val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
 591			iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1);
 592		}
 593
 594		t7xx_cldma_hw_rx_done(hw_info, l2_rx_int);
 595		if (l2_rx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) {
 596			l2_rx_int |= l2_rx_int >> CLDMA_RXQ_NUM;
 597			for_each_set_bit(i, &l2_rx_int, CLDMA_RXQ_NUM) {
 598				pm_runtime_get(md_ctrl->dev);
 599				t7xx_cldma_hw_irq_dis_eq(hw_info, i, MTK_RX);
 600				t7xx_cldma_hw_irq_dis_txrx(hw_info, i, MTK_RX);
 601				queue_work(md_ctrl->rxq[i].worker, &md_ctrl->rxq[i].cldma_work);
 602			}
 603		}
 604	}
 605}
 606
 607static bool t7xx_cldma_qs_are_active(struct cldma_ctrl *md_ctrl)
 608{
 609	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 610	unsigned int tx_active;
 611	unsigned int rx_active;
 612
 613	if (!pci_device_is_present(to_pci_dev(md_ctrl->dev)))
 614		return false;
 615
 616	tx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_TX);
 617	rx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, MTK_RX);
 618
 619	return tx_active || rx_active;
 620}
 621
 622/**
 623 * t7xx_cldma_stop() - Stop CLDMA.
 624 * @md_ctrl: CLDMA context structure.
 625 *
 626 * Stop TX and RX queues. Disable L1 and L2 interrupts.
 627 * Clear status registers.
 628 *
 629 * Return:
 630 * * 0		- Success.
 631 * * -ERROR	- Error code from polling cldma_queues_active.
 632 */
 633int t7xx_cldma_stop(struct cldma_ctrl *md_ctrl)
 634{
 635	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 636	bool active;
 637	int i, ret;
 638
 639	md_ctrl->rxq_active = 0;
 640	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
 641	md_ctrl->txq_active = 0;
 642	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
 643	md_ctrl->txq_started = 0;
 644	t7xx_cldma_disable_irq(md_ctrl);
 645	t7xx_cldma_hw_stop(hw_info, MTK_RX);
 646	t7xx_cldma_hw_stop(hw_info, MTK_TX);
 647	t7xx_cldma_hw_tx_done(hw_info, CLDMA_L2TISAR0_ALL_INT_MASK);
 648	t7xx_cldma_hw_rx_done(hw_info, CLDMA_L2RISAR0_ALL_INT_MASK);
 649
 650	if (md_ctrl->is_late_init) {
 651		for (i = 0; i < CLDMA_TXQ_NUM; i++)
 652			flush_work(&md_ctrl->txq[i].cldma_work);
 653
 654		for (i = 0; i < CLDMA_RXQ_NUM; i++)
 655			flush_work(&md_ctrl->rxq[i].cldma_work);
 656	}
 657
 658	ret = read_poll_timeout(t7xx_cldma_qs_are_active, active, !active, CHECK_Q_STOP_STEP_US,
 659				CHECK_Q_STOP_TIMEOUT_US, true, md_ctrl);
 660	if (ret)
 661		dev_err(md_ctrl->dev, "Could not stop CLDMA%d queues", md_ctrl->hif_id);
 662
 663	return ret;
 664}
 665
 666static void t7xx_cldma_late_release(struct cldma_ctrl *md_ctrl)
 667{
 668	int i;
 669
 670	if (!md_ctrl->is_late_init)
 671		return;
 672
 673	for (i = 0; i < CLDMA_TXQ_NUM; i++)
 674		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
 675
 676	for (i = 0; i < CLDMA_RXQ_NUM; i++)
 677		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[i], DMA_FROM_DEVICE);
 678
 679	dma_pool_destroy(md_ctrl->gpd_dmapool);
 680	md_ctrl->gpd_dmapool = NULL;
 681	md_ctrl->is_late_init = false;
 682}
 683
 684void t7xx_cldma_reset(struct cldma_ctrl *md_ctrl)
 685{
 686	unsigned long flags;
 687	int i;
 688
 689	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 690	md_ctrl->txq_active = 0;
 691	md_ctrl->rxq_active = 0;
 692	t7xx_cldma_disable_irq(md_ctrl);
 693	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 694
 695	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
 696		cancel_work_sync(&md_ctrl->txq[i].cldma_work);
 697
 698		spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 699		md_cd_queue_struct_reset(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
 700		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 701	}
 702
 703	for (i = 0; i < CLDMA_RXQ_NUM; i++) {
 704		cancel_work_sync(&md_ctrl->rxq[i].cldma_work);
 705
 706		spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 707		md_cd_queue_struct_reset(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
 708		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 709	}
 710
 711	t7xx_cldma_late_release(md_ctrl);
 712}
 713
 714/**
 715 * t7xx_cldma_start() - Start CLDMA.
 716 * @md_ctrl: CLDMA context structure.
 717 *
 718 * Set TX/RX start address.
 719 * Start all RX queues and enable L2 interrupt.
 720 */
 721void t7xx_cldma_start(struct cldma_ctrl *md_ctrl)
 722{
 723	unsigned long flags;
 724
 725	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 726	if (md_ctrl->is_late_init) {
 727		struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 728		int i;
 729
 730		t7xx_cldma_enable_irq(md_ctrl);
 731
 732		for (i = 0; i < CLDMA_TXQ_NUM; i++) {
 733			if (md_ctrl->txq[i].tr_done)
 734				t7xx_cldma_hw_set_start_addr(hw_info, i,
 735							     md_ctrl->txq[i].tr_done->gpd_addr,
 736							     MTK_TX);
 737		}
 738
 739		for (i = 0; i < CLDMA_RXQ_NUM; i++) {
 740			if (md_ctrl->rxq[i].tr_done)
 741				t7xx_cldma_hw_set_start_addr(hw_info, i,
 742							     md_ctrl->rxq[i].tr_done->gpd_addr,
 743							     MTK_RX);
 744		}
 745
 746		/* Enable L2 interrupt */
 747		t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
 748		t7xx_cldma_hw_start(hw_info);
 749		md_ctrl->txq_started = 0;
 750		md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
 751		md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
 752	}
 753	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 754}
 755
 756static void t7xx_cldma_clear_txq(struct cldma_ctrl *md_ctrl, int qnum)
 757{
 758	struct cldma_queue *txq = &md_ctrl->txq[qnum];
 759	struct cldma_request *req;
 760	struct cldma_gpd *gpd;
 761	unsigned long flags;
 762
 763	spin_lock_irqsave(&txq->ring_lock, flags);
 764	t7xx_cldma_q_reset(txq);
 765	list_for_each_entry(req, &txq->tr_ring->gpd_ring, entry) {
 766		gpd = req->gpd;
 767		gpd->flags &= ~GPD_FLAGS_HWO;
 768		t7xx_cldma_gpd_set_data_ptr(gpd, 0);
 769		gpd->data_buff_len = 0;
 770		dev_kfree_skb_any(req->skb);
 771		req->skb = NULL;
 772	}
 773	spin_unlock_irqrestore(&txq->ring_lock, flags);
 774}
 775
 776static int t7xx_cldma_clear_rxq(struct cldma_ctrl *md_ctrl, int qnum)
 777{
 778	struct cldma_queue *rxq = &md_ctrl->rxq[qnum];
 779	struct cldma_request *req;
 780	struct cldma_gpd *gpd;
 781	unsigned long flags;
 782	int ret = 0;
 783
 784	spin_lock_irqsave(&rxq->ring_lock, flags);
 785	t7xx_cldma_q_reset(rxq);
 786	list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
 787		gpd = req->gpd;
 788		gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO;
 789		gpd->data_buff_len = 0;
 790
 791		if (req->skb) {
 792			req->skb->len = 0;
 793			skb_reset_tail_pointer(req->skb);
 794		}
 795	}
 796
 797	list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) {
 798		if (req->skb)
 799			continue;
 800
 801		ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, rxq->tr_ring->pkt_size, GFP_ATOMIC);
 802		if (ret)
 803			break;
 804
 805		t7xx_cldma_gpd_set_data_ptr(req->gpd, req->mapped_buff);
 806	}
 807	spin_unlock_irqrestore(&rxq->ring_lock, flags);
 808
 809	return ret;
 810}
 811
 812void t7xx_cldma_clear_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
 813{
 814	int i;
 815
 816	if (tx_rx == MTK_TX) {
 817		for (i = 0; i < CLDMA_TXQ_NUM; i++)
 818			t7xx_cldma_clear_txq(md_ctrl, i);
 819	} else {
 820		for (i = 0; i < CLDMA_RXQ_NUM; i++)
 821			t7xx_cldma_clear_rxq(md_ctrl, i);
 822	}
 823}
 824
 825void t7xx_cldma_stop_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx)
 826{
 827	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 828	unsigned long flags;
 829
 830	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 831	t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, tx_rx);
 832	t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, tx_rx);
 833	if (tx_rx == MTK_RX)
 834		md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
 835	else
 836		md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
 837	t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx);
 838	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 839}
 840
 841static int t7xx_cldma_gpd_handle_tx_request(struct cldma_queue *queue, struct cldma_request *tx_req,
 842					    struct sk_buff *skb)
 843{
 844	struct cldma_ctrl *md_ctrl = queue->md_ctrl;
 845	struct cldma_gpd *gpd = tx_req->gpd;
 846	unsigned long flags;
 847
 848	/* Update GPD */
 849	tx_req->mapped_buff = dma_map_single(md_ctrl->dev, skb->data, skb->len, DMA_TO_DEVICE);
 850
 851	if (dma_mapping_error(md_ctrl->dev, tx_req->mapped_buff)) {
 852		dev_err(md_ctrl->dev, "DMA mapping failed\n");
 853		return -ENOMEM;
 854	}
 855
 856	t7xx_cldma_gpd_set_data_ptr(gpd, tx_req->mapped_buff);
 857	gpd->data_buff_len = cpu_to_le16(skb->len);
 858
 859	/* This lock must cover TGPD setting, as even without a resume operation,
 860	 * CLDMA can send next HWO=1 if last TGPD just finished.
 861	 */
 862	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 863	if (md_ctrl->txq_active & BIT(queue->index))
 864		gpd->flags |= GPD_FLAGS_HWO;
 865
 866	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 867
 868	tx_req->skb = skb;
 869	return 0;
 870}
 871
 872/* Called with cldma_lock */
 873static void t7xx_cldma_hw_start_send(struct cldma_ctrl *md_ctrl, int qno,
 874				     struct cldma_request *prev_req)
 875{
 876	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
 877
 878	/* Check whether the device was powered off (CLDMA start address is not set) */
 879	if (!t7xx_cldma_tx_addr_is_set(hw_info, qno)) {
 880		t7xx_cldma_hw_init(hw_info);
 881		t7xx_cldma_hw_set_start_addr(hw_info, qno, prev_req->gpd_addr, MTK_TX);
 882		md_ctrl->txq_started &= ~BIT(qno);
 883	}
 884
 885	if (!t7xx_cldma_hw_queue_status(hw_info, qno, MTK_TX)) {
 886		if (md_ctrl->txq_started & BIT(qno))
 887			t7xx_cldma_hw_resume_queue(hw_info, qno, MTK_TX);
 888		else
 889			t7xx_cldma_hw_start_queue(hw_info, qno, MTK_TX);
 890
 891		md_ctrl->txq_started |= BIT(qno);
 892	}
 893}
 894
 895/**
 896 * t7xx_cldma_set_recv_skb() - Set the callback to handle RX packets.
 897 * @queue: CLDMA queue.
 898 * @recv_skb: Receiving skb callback.
 899 */
 900static void t7xx_cldma_set_recv_skb(struct cldma_queue *queue,
 901			     int (*recv_skb)(struct cldma_queue *queue, struct sk_buff *skb))
 902{
 903	queue->recv_skb = recv_skb;
 904}
 905
 906/**
 907 * t7xx_cldma_send_skb() - Send control data to modem.
 908 * @md_ctrl: CLDMA context structure.
 909 * @qno: Queue number.
 910 * @skb: Socket buffer.
 911 *
 912 * Return:
 913 * * 0		- Success.
 914 * * -ENOMEM	- Allocation failure.
 915 * * -EINVAL	- Invalid queue request.
 916 * * -EIO	- Queue is not active.
 917 * * -ETIMEDOUT	- Timeout waiting for the device to wake up.
 918 */
 919int t7xx_cldma_send_skb(struct cldma_ctrl *md_ctrl, int qno, struct sk_buff *skb)
 920{
 921	struct cldma_request *tx_req;
 922	struct cldma_queue *queue;
 923	unsigned long flags;
 924	int ret;
 925
 926	if (qno >= CLDMA_TXQ_NUM)
 927		return -EINVAL;
 928
 929	ret = pm_runtime_resume_and_get(md_ctrl->dev);
 930	if (ret < 0 && ret != -EACCES)
 931		return ret;
 932
 933	t7xx_pci_disable_sleep(md_ctrl->t7xx_dev);
 934	queue = &md_ctrl->txq[qno];
 935
 936	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 937	if (!(md_ctrl->txq_active & BIT(qno))) {
 938		ret = -EIO;
 939		spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 940		goto allow_sleep;
 941	}
 942	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 943
 944	do {
 945		spin_lock_irqsave(&queue->ring_lock, flags);
 946		tx_req = queue->tx_next;
 947		if (queue->budget > 0 && !tx_req->skb) {
 948			struct list_head *gpd_ring = &queue->tr_ring->gpd_ring;
 949
 950			queue->budget--;
 951			t7xx_cldma_gpd_handle_tx_request(queue, tx_req, skb);
 952			queue->tx_next = list_next_entry_circular(tx_req, gpd_ring, entry);
 953			spin_unlock_irqrestore(&queue->ring_lock, flags);
 954
 955			if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
 956				ret = -ETIMEDOUT;
 957				break;
 958			}
 959
 960			/* Protect the access to the modem for queues operations (resume/start)
 961			 * which access shared locations by all the queues.
 962			 * cldma_lock is independent of ring_lock which is per queue.
 963			 */
 964			spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 965			t7xx_cldma_hw_start_send(md_ctrl, qno, tx_req);
 966			spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 967
 968			break;
 969		}
 970		spin_unlock_irqrestore(&queue->ring_lock, flags);
 971
 972		if (!t7xx_pci_sleep_disable_complete(md_ctrl->t7xx_dev)) {
 973			ret = -ETIMEDOUT;
 974			break;
 975		}
 976
 977		if (!t7xx_cldma_hw_queue_status(&md_ctrl->hw_info, qno, MTK_TX)) {
 978			spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
 979			t7xx_cldma_hw_resume_queue(&md_ctrl->hw_info, qno, MTK_TX);
 980			spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
 981		}
 982
 983		ret = wait_event_interruptible_exclusive(queue->req_wq, queue->budget > 0);
 984	} while (!ret);
 985
 986allow_sleep:
 987	t7xx_pci_enable_sleep(md_ctrl->t7xx_dev);
 988	pm_runtime_put_autosuspend(md_ctrl->dev);
 989	return ret;
 990}
 991
 992static void t7xx_cldma_adjust_config(struct cldma_ctrl *md_ctrl, enum cldma_cfg cfg_id)
 993{
 994	int qno;
 995
 996	for (qno = 0; qno < CLDMA_RXQ_NUM; qno++) {
 997		md_ctrl->rx_ring[qno].pkt_size = CLDMA_SHARED_Q_BUFF_SZ;
 998		t7xx_cldma_set_recv_skb(&md_ctrl->rxq[qno], t7xx_port_proxy_recv_skb);
 999	}
1000
1001	md_ctrl->rx_ring[CLDMA_RXQ_NUM - 1].pkt_size = CLDMA_JUMBO_BUFF_SZ;
1002
1003	for (qno = 0; qno < CLDMA_TXQ_NUM; qno++)
1004		md_ctrl->tx_ring[qno].pkt_size = CLDMA_SHARED_Q_BUFF_SZ;
1005
1006	if (cfg_id == CLDMA_DEDICATED_Q_CFG) {
1007		md_ctrl->tx_ring[CLDMA_Q_IDX_DUMP].pkt_size = CLDMA_DEDICATED_Q_BUFF_SZ;
1008		md_ctrl->rx_ring[CLDMA_Q_IDX_DUMP].pkt_size = CLDMA_DEDICATED_Q_BUFF_SZ;
1009		t7xx_cldma_set_recv_skb(&md_ctrl->rxq[CLDMA_Q_IDX_DUMP],
1010					t7xx_port_proxy_recv_skb_from_dedicated_queue);
1011	}
1012}
1013
1014static int t7xx_cldma_late_init(struct cldma_ctrl *md_ctrl)
1015{
1016	char dma_pool_name[32];
1017	int i, j, ret;
1018
1019	if (md_ctrl->is_late_init) {
1020		dev_err(md_ctrl->dev, "CLDMA late init was already done\n");
1021		return -EALREADY;
1022	}
1023
1024	snprintf(dma_pool_name, sizeof(dma_pool_name), "cldma_req_hif%d", md_ctrl->hif_id);
1025
1026	md_ctrl->gpd_dmapool = dma_pool_create(dma_pool_name, md_ctrl->dev,
1027					       sizeof(struct cldma_gpd), GPD_DMAPOOL_ALIGN, 0);
1028	if (!md_ctrl->gpd_dmapool) {
1029		dev_err(md_ctrl->dev, "DMA pool alloc fail\n");
1030		return -ENOMEM;
1031	}
1032
1033	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1034		ret = t7xx_cldma_tx_ring_init(md_ctrl, &md_ctrl->tx_ring[i]);
1035		if (ret) {
1036			dev_err(md_ctrl->dev, "control TX ring init fail\n");
1037			goto err_free_tx_ring;
1038		}
1039	}
1040
1041	for (j = 0; j < CLDMA_RXQ_NUM; j++) {
1042		ret = t7xx_cldma_rx_ring_init(md_ctrl, &md_ctrl->rx_ring[j]);
1043		if (ret) {
1044			dev_err(md_ctrl->dev, "Control RX ring init fail\n");
1045			goto err_free_rx_ring;
1046		}
1047	}
1048
1049	for (i = 0; i < CLDMA_TXQ_NUM; i++)
1050		t7xx_cldma_txq_init(&md_ctrl->txq[i]);
1051
1052	for (j = 0; j < CLDMA_RXQ_NUM; j++)
1053		t7xx_cldma_rxq_init(&md_ctrl->rxq[j]);
1054
1055	md_ctrl->is_late_init = true;
1056	return 0;
1057
1058err_free_rx_ring:
1059	while (j--)
1060		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->rx_ring[j], DMA_FROM_DEVICE);
1061
1062err_free_tx_ring:
1063	while (i--)
1064		t7xx_cldma_ring_free(md_ctrl, &md_ctrl->tx_ring[i], DMA_TO_DEVICE);
1065
1066	return ret;
1067}
1068
1069static void __iomem *t7xx_pcie_addr_transfer(void __iomem *addr, u32 addr_trs1, u32 phy_addr)
1070{
1071	return addr + phy_addr - addr_trs1;
1072}
1073
1074static void t7xx_hw_info_init(struct cldma_ctrl *md_ctrl)
1075{
1076	struct t7xx_addr_base *pbase = &md_ctrl->t7xx_dev->base_addr;
1077	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1078	u32 phy_ao_base, phy_pd_base;
1079
1080	hw_info->hw_mode = MODE_BIT_64;
1081
1082	if (md_ctrl->hif_id == CLDMA_ID_MD) {
1083		phy_ao_base = CLDMA1_AO_BASE;
1084		phy_pd_base = CLDMA1_PD_BASE;
1085		hw_info->phy_interrupt_id = CLDMA1_INT;
1086	} else {
1087		phy_ao_base = CLDMA0_AO_BASE;
1088		phy_pd_base = CLDMA0_PD_BASE;
1089		hw_info->phy_interrupt_id = CLDMA0_INT;
1090	}
1091
1092	hw_info->ap_ao_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1093						      pbase->pcie_dev_reg_trsl_addr, phy_ao_base);
1094	hw_info->ap_pdn_base = t7xx_pcie_addr_transfer(pbase->pcie_ext_reg_base,
1095						       pbase->pcie_dev_reg_trsl_addr, phy_pd_base);
1096}
1097
1098static int t7xx_cldma_default_recv_skb(struct cldma_queue *queue, struct sk_buff *skb)
1099{
1100	dev_kfree_skb_any(skb);
1101	return 0;
1102}
1103
1104int t7xx_cldma_alloc(enum cldma_id hif_id, struct t7xx_pci_dev *t7xx_dev)
1105{
1106	struct device *dev = &t7xx_dev->pdev->dev;
1107	struct cldma_ctrl *md_ctrl;
1108	int qno;
1109
1110	md_ctrl = devm_kzalloc(dev, sizeof(*md_ctrl), GFP_KERNEL);
1111	if (!md_ctrl)
1112		return -ENOMEM;
1113
1114	md_ctrl->t7xx_dev = t7xx_dev;
1115	md_ctrl->dev = dev;
1116	md_ctrl->hif_id = hif_id;
1117	for (qno = 0; qno < CLDMA_RXQ_NUM; qno++)
1118		md_ctrl->rxq[qno].recv_skb = t7xx_cldma_default_recv_skb;
1119
1120	t7xx_hw_info_init(md_ctrl);
1121	t7xx_dev->md->md_ctrl[hif_id] = md_ctrl;
1122	return 0;
1123}
1124
1125static void t7xx_cldma_resume_early(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1126{
1127	struct cldma_ctrl *md_ctrl = entity_param;
1128	struct t7xx_cldma_hw *hw_info;
1129	unsigned long flags;
1130	int qno_t;
1131
1132	hw_info = &md_ctrl->hw_info;
1133
1134	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1135	t7xx_cldma_hw_restore(hw_info);
1136	for (qno_t = 0; qno_t < CLDMA_TXQ_NUM; qno_t++) {
1137		t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->txq[qno_t].tx_next->gpd_addr,
1138					     MTK_TX);
1139		t7xx_cldma_hw_set_start_addr(hw_info, qno_t, md_ctrl->rxq[qno_t].tr_done->gpd_addr,
1140					     MTK_RX);
1141	}
1142	t7xx_cldma_enable_irq(md_ctrl);
1143	t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, MTK_RX);
1144	md_ctrl->rxq_active |= TXRX_STATUS_BITMASK;
1145	t7xx_cldma_hw_irq_en_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1146	t7xx_cldma_hw_irq_en_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1147	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1148}
1149
1150static int t7xx_cldma_resume(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1151{
1152	struct cldma_ctrl *md_ctrl = entity_param;
1153	unsigned long flags;
1154
1155	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1156	md_ctrl->txq_active |= TXRX_STATUS_BITMASK;
1157	t7xx_cldma_hw_irq_en_txrx(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1158	t7xx_cldma_hw_irq_en_eq(&md_ctrl->hw_info, CLDMA_ALL_Q, MTK_TX);
1159	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1160
1161	if (md_ctrl->hif_id == CLDMA_ID_MD)
1162		t7xx_mhccif_mask_clr(t7xx_dev, D2H_SW_INT_MASK);
1163
1164	return 0;
1165}
1166
1167static void t7xx_cldma_suspend_late(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1168{
1169	struct cldma_ctrl *md_ctrl = entity_param;
1170	struct t7xx_cldma_hw *hw_info;
1171	unsigned long flags;
1172
1173	hw_info = &md_ctrl->hw_info;
1174
1175	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1176	t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_RX);
1177	t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_RX);
1178	md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK;
1179	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_RX);
1180	t7xx_cldma_clear_ip_busy(hw_info);
1181	t7xx_cldma_disable_irq(md_ctrl);
1182	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1183}
1184
1185static int t7xx_cldma_suspend(struct t7xx_pci_dev *t7xx_dev, void *entity_param)
1186{
1187	struct cldma_ctrl *md_ctrl = entity_param;
1188	struct t7xx_cldma_hw *hw_info;
1189	unsigned long flags;
1190
1191	if (md_ctrl->hif_id == CLDMA_ID_MD)
1192		t7xx_mhccif_mask_set(t7xx_dev, D2H_SW_INT_MASK);
1193
1194	hw_info = &md_ctrl->hw_info;
1195
1196	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1197	t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, MTK_TX);
1198	t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, MTK_TX);
1199	md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK;
1200	t7xx_cldma_hw_stop_all_qs(hw_info, MTK_TX);
1201	md_ctrl->txq_started = 0;
1202	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1203
1204	return 0;
1205}
1206
1207static int t7xx_cldma_pm_init(struct cldma_ctrl *md_ctrl)
1208{
1209	md_ctrl->pm_entity = kzalloc(sizeof(*md_ctrl->pm_entity), GFP_KERNEL);
1210	if (!md_ctrl->pm_entity)
1211		return -ENOMEM;
1212
1213	md_ctrl->pm_entity->entity_param = md_ctrl;
1214
1215	if (md_ctrl->hif_id == CLDMA_ID_MD)
1216		md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL1;
1217	else
1218		md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL2;
1219
1220	md_ctrl->pm_entity->suspend = t7xx_cldma_suspend;
1221	md_ctrl->pm_entity->suspend_late = t7xx_cldma_suspend_late;
1222	md_ctrl->pm_entity->resume = t7xx_cldma_resume;
1223	md_ctrl->pm_entity->resume_early = t7xx_cldma_resume_early;
1224
1225	return t7xx_pci_pm_entity_register(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1226}
1227
1228static int t7xx_cldma_pm_uninit(struct cldma_ctrl *md_ctrl)
1229{
1230	if (!md_ctrl->pm_entity)
1231		return -EINVAL;
1232
1233	t7xx_pci_pm_entity_unregister(md_ctrl->t7xx_dev, md_ctrl->pm_entity);
1234	kfree(md_ctrl->pm_entity);
1235	md_ctrl->pm_entity = NULL;
1236	return 0;
1237}
1238
1239void t7xx_cldma_hif_hw_init(struct cldma_ctrl *md_ctrl)
1240{
1241	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1242	unsigned long flags;
1243
1244	spin_lock_irqsave(&md_ctrl->cldma_lock, flags);
1245	t7xx_cldma_hw_stop(hw_info, MTK_TX);
1246	t7xx_cldma_hw_stop(hw_info, MTK_RX);
1247	t7xx_cldma_hw_rx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1248	t7xx_cldma_hw_tx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK);
1249	t7xx_cldma_hw_init(hw_info);
1250	spin_unlock_irqrestore(&md_ctrl->cldma_lock, flags);
1251}
1252
1253static irqreturn_t t7xx_cldma_isr_handler(int irq, void *data)
1254{
1255	struct cldma_ctrl *md_ctrl = data;
1256	u32 interrupt;
1257
1258	interrupt = md_ctrl->hw_info.phy_interrupt_id;
1259	t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, interrupt);
1260	t7xx_cldma_irq_work_cb(md_ctrl);
1261	t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, interrupt);
1262	t7xx_pcie_mac_set_int(md_ctrl->t7xx_dev, interrupt);
1263	return IRQ_HANDLED;
1264}
1265
1266static void t7xx_cldma_destroy_wqs(struct cldma_ctrl *md_ctrl)
1267{
1268	int i;
1269
1270	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1271		if (md_ctrl->txq[i].worker) {
1272			destroy_workqueue(md_ctrl->txq[i].worker);
1273			md_ctrl->txq[i].worker = NULL;
1274		}
1275	}
1276
1277	for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1278		if (md_ctrl->rxq[i].worker) {
1279			destroy_workqueue(md_ctrl->rxq[i].worker);
1280			md_ctrl->rxq[i].worker = NULL;
1281		}
1282	}
1283}
1284
1285/**
1286 * t7xx_cldma_init() - Initialize CLDMA.
1287 * @md_ctrl: CLDMA context structure.
1288 *
1289 * Allocate and initialize device power management entity.
1290 * Initialize HIF TX/RX queue structure.
1291 * Register CLDMA callback ISR with PCIe driver.
1292 *
1293 * Return:
1294 * * 0		- Success.
1295 * * -ERROR	- Error code from failure sub-initializations.
1296 */
1297int t7xx_cldma_init(struct cldma_ctrl *md_ctrl)
1298{
1299	struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info;
1300	int ret, i;
1301
1302	md_ctrl->txq_active = 0;
1303	md_ctrl->rxq_active = 0;
1304	md_ctrl->is_late_init = false;
1305
1306	ret = t7xx_cldma_pm_init(md_ctrl);
1307	if (ret)
1308		return ret;
1309
1310	spin_lock_init(&md_ctrl->cldma_lock);
1311
1312	for (i = 0; i < CLDMA_TXQ_NUM; i++) {
1313		md_cd_queue_struct_init(&md_ctrl->txq[i], md_ctrl, MTK_TX, i);
1314		md_ctrl->txq[i].worker =
1315			alloc_ordered_workqueue("md_hif%d_tx%d_worker",
1316					WQ_MEM_RECLAIM | (i ? 0 : WQ_HIGHPRI),
1317					md_ctrl->hif_id, i);
1318		if (!md_ctrl->txq[i].worker)
1319			goto err_workqueue;
1320
1321		INIT_WORK(&md_ctrl->txq[i].cldma_work, t7xx_cldma_tx_done);
1322	}
1323
1324	for (i = 0; i < CLDMA_RXQ_NUM; i++) {
1325		md_cd_queue_struct_init(&md_ctrl->rxq[i], md_ctrl, MTK_RX, i);
1326		INIT_WORK(&md_ctrl->rxq[i].cldma_work, t7xx_cldma_rx_done);
1327
1328		md_ctrl->rxq[i].worker =
1329			alloc_ordered_workqueue("md_hif%d_rx%d_worker",
1330						WQ_MEM_RECLAIM,
1331						md_ctrl->hif_id, i);
1332		if (!md_ctrl->rxq[i].worker)
1333			goto err_workqueue;
1334	}
1335
1336	t7xx_pcie_mac_clear_int(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1337	md_ctrl->t7xx_dev->intr_handler[hw_info->phy_interrupt_id] = t7xx_cldma_isr_handler;
1338	md_ctrl->t7xx_dev->intr_thread[hw_info->phy_interrupt_id] = NULL;
1339	md_ctrl->t7xx_dev->callback_param[hw_info->phy_interrupt_id] = md_ctrl;
1340	t7xx_pcie_mac_clear_int_status(md_ctrl->t7xx_dev, hw_info->phy_interrupt_id);
1341	return 0;
1342
1343err_workqueue:
1344	t7xx_cldma_destroy_wqs(md_ctrl);
1345	t7xx_cldma_pm_uninit(md_ctrl);
1346	return -ENOMEM;
1347}
1348
1349void t7xx_cldma_switch_cfg(struct cldma_ctrl *md_ctrl, enum cldma_cfg cfg_id)
1350{
1351	t7xx_cldma_late_release(md_ctrl);
1352	t7xx_cldma_adjust_config(md_ctrl, cfg_id);
1353	t7xx_cldma_late_init(md_ctrl);
1354}
1355
1356void t7xx_cldma_exit(struct cldma_ctrl *md_ctrl)
1357{
1358	t7xx_cldma_stop(md_ctrl);
1359	t7xx_cldma_late_release(md_ctrl);
1360	t7xx_cldma_destroy_wqs(md_ctrl);
1361	t7xx_cldma_pm_uninit(md_ctrl);
1362}