ae4dma-dev.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * AMD AE4DMA driver
  4 *
  5 * Copyright (c) 2024, Advanced Micro Devices, Inc.
  6 * All Rights Reserved.
  7 *
  8 * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com>
  9 */
 10
 11#include "ae4dma.h"
 12
 13static unsigned int max_hw_q = 1;
 14module_param(max_hw_q, uint, 0444);
 15MODULE_PARM_DESC(max_hw_q, "max hw queues supported by engine (any non-zero value, default: 1)");
 16
 17static void ae4_pending_work(struct work_struct *work)
 18{
 19	struct ae4_cmd_queue *ae4cmd_q = container_of(work, struct ae4_cmd_queue, p_work.work);
 20	struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q;
 21	struct pt_cmd *cmd;
 22	u32 cridx;
 23
 24	for (;;) {
 25		wait_event_interruptible(ae4cmd_q->q_w,
 26					 ((atomic64_read(&ae4cmd_q->done_cnt)) <
 27					   atomic64_read(&ae4cmd_q->intr_cnt)));
 28
 29		atomic64_inc(&ae4cmd_q->done_cnt);
 30
 31		mutex_lock(&ae4cmd_q->cmd_lock);
 32		cridx = readl(cmd_q->reg_control + AE4_RD_IDX_OFF);
 33		while ((ae4cmd_q->dridx != cridx) && !list_empty(&ae4cmd_q->cmd)) {
 34			cmd = list_first_entry(&ae4cmd_q->cmd, struct pt_cmd, entry);
 35			list_del(&cmd->entry);
 36
 37			ae4_check_status_error(ae4cmd_q, ae4cmd_q->dridx);
 38			cmd->pt_cmd_callback(cmd->data, cmd->ret);
 39
 40			ae4cmd_q->q_cmd_count--;
 41			ae4cmd_q->dridx = (ae4cmd_q->dridx + 1) % CMD_Q_LEN;
 42
 43			complete_all(&ae4cmd_q->cmp);
 44		}
 45		mutex_unlock(&ae4cmd_q->cmd_lock);
 46	}
 47}
 48
 49static irqreturn_t ae4_core_irq_handler(int irq, void *data)
 50{
 51	struct ae4_cmd_queue *ae4cmd_q = data;
 52	struct pt_cmd_queue *cmd_q;
 53	struct pt_device *pt;
 54	u32 status;
 55
 56	cmd_q = &ae4cmd_q->cmd_q;
 57	pt = cmd_q->pt;
 58
 59	pt->total_interrupts++;
 60	atomic64_inc(&ae4cmd_q->intr_cnt);
 61
 62	status = readl(cmd_q->reg_control + AE4_INTR_STS_OFF);
 63	if (status & BIT(0)) {
 64		status &= GENMASK(31, 1);
 65		writel(status, cmd_q->reg_control + AE4_INTR_STS_OFF);
 66	}
 67
 68	wake_up(&ae4cmd_q->q_w);
 69
 70	return IRQ_HANDLED;
 71}
 72
 73void ae4_destroy_work(struct ae4_device *ae4)
 74{
 75	struct ae4_cmd_queue *ae4cmd_q;
 76	int i;
 77
 78	for (i = 0; i < ae4->cmd_q_count; i++) {
 79		ae4cmd_q = &ae4->ae4cmd_q[i];
 80
 81		if (!ae4cmd_q->pws)
 82			break;
 83
 84		cancel_delayed_work_sync(&ae4cmd_q->p_work);
 85		destroy_workqueue(ae4cmd_q->pws);
 86	}
 87}
 88
 89int ae4_core_init(struct ae4_device *ae4)
 90{
 91	struct pt_device *pt = &ae4->pt;
 92	struct ae4_cmd_queue *ae4cmd_q;
 93	struct device *dev = pt->dev;
 94	struct pt_cmd_queue *cmd_q;
 95	int i, ret = 0;
 96
 97	writel(max_hw_q, pt->io_regs);
 98
 99	for (i = 0; i < max_hw_q; i++) {
100		ae4cmd_q = &ae4->ae4cmd_q[i];
101		ae4cmd_q->id = ae4->cmd_q_count;
102		ae4->cmd_q_count++;
103
104		cmd_q = &ae4cmd_q->cmd_q;
105		cmd_q->pt = pt;
106
107		cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ);
108
109		ret = devm_request_irq(dev, ae4->ae4_irq[i], ae4_core_irq_handler, 0,
110				       dev_name(pt->dev), ae4cmd_q);
111		if (ret)
112			return ret;
113
114		cmd_q->qsize = Q_SIZE(sizeof(struct ae4dma_desc));
115
116		cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, &cmd_q->qbase_dma,
117						   GFP_KERNEL);
118		if (!cmd_q->qbase)
119			return -ENOMEM;
120	}
121
122	for (i = 0; i < ae4->cmd_q_count; i++) {
123		ae4cmd_q = &ae4->ae4cmd_q[i];
124
125		cmd_q = &ae4cmd_q->cmd_q;
126
127		cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ);
128
129		/* Update the device registers with queue information. */
130		writel(CMD_Q_LEN, cmd_q->reg_control + AE4_MAX_IDX_OFF);
131
132		cmd_q->qdma_tail = cmd_q->qbase_dma;
133		writel(lower_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_L_OFF);
134		writel(upper_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_H_OFF);
135
136		INIT_LIST_HEAD(&ae4cmd_q->cmd);
137		init_waitqueue_head(&ae4cmd_q->q_w);
138
139		ae4cmd_q->pws = alloc_ordered_workqueue("ae4dma_%d", WQ_MEM_RECLAIM, ae4cmd_q->id);
140		if (!ae4cmd_q->pws) {
141			ae4_destroy_work(ae4);
142			return -ENOMEM;
143		}
144		INIT_DELAYED_WORK(&ae4cmd_q->p_work, ae4_pending_work);
145		queue_delayed_work(ae4cmd_q->pws, &ae4cmd_q->p_work,  usecs_to_jiffies(100));
146
147		init_completion(&ae4cmd_q->cmp);
148	}
149
150	ret = pt_dmaengine_register(pt);
151	if (ret)
152		ae4_destroy_work(ae4);
153	else
154		ptdma_debugfs_setup(pt);
155
156	return ret;
157}