qdma.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * DMA driver for AMD Queue-based DMA Subsystem
   4 *
   5 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
   6 */
   7#include <linux/bitfield.h>
   8#include <linux/bitops.h>
   9#include <linux/dmaengine.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/module.h>
  12#include <linux/mod_devicetable.h>
  13#include <linux/platform_device.h>
  14#include <linux/platform_data/amd_qdma.h>
  15#include <linux/regmap.h>
  16
  17#include "qdma.h"
  18
  19#define CHAN_STR(q)		(((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H")
  20#define QDMA_REG_OFF(d, r)	((d)->roffs[r].off)
  21
  22/* MMIO regmap config for all QDMA registers */
  23static const struct regmap_config qdma_regmap_config = {
  24	.reg_bits = 32,
  25	.val_bits = 32,
  26	.reg_stride = 4,
  27};
  28
  29static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan)
  30{
  31	return container_of(chan, struct qdma_queue, vchan.chan);
  32}
  33
  34static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc)
  35{
  36	return container_of(vdesc, struct qdma_mm_vdesc, vdesc);
  37}
  38
  39static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev)
  40{
  41	u32 idx;
  42
  43	idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx;
  44	qdev->qintr_ring_idx %= qdev->qintr_ring_num;
  45
  46	return idx;
  47}
  48
  49static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data,
  50			  enum qdma_reg_fields field)
  51{
  52	const struct qdma_reg_field *f = &qdev->rfields[field];
  53	u16 low_pos, hi_pos, low_bit, hi_bit;
  54	u64 value = 0, mask;
  55
  56	low_pos = f->lsb / BITS_PER_TYPE(*data);
  57	hi_pos = f->msb / BITS_PER_TYPE(*data);
  58
  59	if (low_pos == hi_pos) {
  60		low_bit = f->lsb % BITS_PER_TYPE(*data);
  61		hi_bit = f->msb % BITS_PER_TYPE(*data);
  62		mask = GENMASK(hi_bit, low_bit);
  63		value = (data[low_pos] & mask) >> low_bit;
  64	} else if (hi_pos == low_pos + 1) {
  65		low_bit = f->lsb % BITS_PER_TYPE(*data);
  66		hi_bit = low_bit + (f->msb - f->lsb);
  67		value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) |
  68			data[low_pos];
  69		mask = GENMASK_ULL(hi_bit, low_bit);
  70		value = (value & mask) >> low_bit;
  71	} else {
  72		hi_bit = f->msb % BITS_PER_TYPE(*data);
  73		mask = GENMASK(hi_bit, 0);
  74		value = data[hi_pos] & mask;
  75		low_bit = f->msb - f->lsb - hi_bit;
  76		value <<= low_bit;
  77		low_bit -= 32;
  78		value |= (u64)data[hi_pos - 1] << low_bit;
  79		mask = GENMASK(31, 32 - low_bit);
  80		value |= (data[hi_pos - 2] & mask) >> low_bit;
  81	}
  82
  83	return value;
  84}
  85
  86static void qdma_set_field(const struct qdma_device *qdev, u32 *data,
  87			   enum qdma_reg_fields field, u64 value)
  88{
  89	const struct qdma_reg_field *f = &qdev->rfields[field];
  90	u16 low_pos, hi_pos, low_bit;
  91
  92	low_pos = f->lsb / BITS_PER_TYPE(*data);
  93	hi_pos = f->msb / BITS_PER_TYPE(*data);
  94	low_bit = f->lsb % BITS_PER_TYPE(*data);
  95
  96	data[low_pos++] |= value << low_bit;
  97	if (low_pos <= hi_pos)
  98		data[low_pos++] |= (u32)(value >> (32 - low_bit));
  99	if (low_pos <= hi_pos)
 100		data[low_pos] |= (u32)(value >> (64 - low_bit));
 101}
 102
 103static inline int qdma_reg_write(const struct qdma_device *qdev,
 104				 const u32 *data, enum qdma_regs reg)
 105{
 106	const struct qdma_reg *r = &qdev->roffs[reg];
 107	int ret;
 108
 109	if (r->count > 1)
 110		ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count);
 111	else
 112		ret = regmap_write(qdev->regmap, r->off, *data);
 113
 114	return ret;
 115}
 116
 117static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data,
 118				enum qdma_regs reg)
 119{
 120	const struct qdma_reg *r = &qdev->roffs[reg];
 121	int ret;
 122
 123	if (r->count > 1)
 124		ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count);
 125	else
 126		ret = regmap_read(qdev->regmap, r->off, data);
 127
 128	return ret;
 129}
 130
 131static int qdma_context_cmd_execute(const struct qdma_device *qdev,
 132				    enum qdma_ctxt_type type,
 133				    enum qdma_ctxt_cmd cmd, u16 index)
 134{
 135	u32 value = 0;
 136	int ret;
 137
 138	qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index);
 139	qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd);
 140	qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type);
 141
 142	ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD);
 143	if (ret)
 144		return ret;
 145
 146	ret = regmap_read_poll_timeout(qdev->regmap,
 147				       QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD),
 148				       value,
 149				       !qdma_get_field(qdev, &value,
 150						       QDMA_REGF_CMD_BUSY),
 151				       QDMA_POLL_INTRVL_US,
 152				       QDMA_POLL_TIMEOUT_US);
 153	if (ret) {
 154		qdma_err(qdev, "Context command execution timed out");
 155		return ret;
 156	}
 157
 158	return 0;
 159}
 160
 161static int qdma_context_write_data(const struct qdma_device *qdev,
 162				   const u32 *data)
 163{
 164	u32 mask[QDMA_CTXT_REGMAP_LEN];
 165	int ret;
 166
 167	memset(mask, ~0, sizeof(mask));
 168
 169	ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK);
 170	if (ret)
 171		return ret;
 172
 173	ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA);
 174	if (ret)
 175		return ret;
 176
 177	return 0;
 178}
 179
 180static void qdma_prep_sw_desc_context(const struct qdma_device *qdev,
 181				      const struct qdma_ctxt_sw_desc *ctxt,
 182				      u32 *data)
 183{
 184	memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data));
 185	qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base);
 186	qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec);
 187	qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid);
 188
 189	qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B);
 190	qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID);
 191	qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM);
 192	qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1);
 193	qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1);
 194	qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1);
 195	qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1);
 196	qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1);
 197	qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1);
 198	qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1);
 199	qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1);
 200}
 201
 202static void qdma_prep_intr_context(const struct qdma_device *qdev,
 203				   const struct qdma_ctxt_intr *ctxt,
 204				   u32 *data)
 205{
 206	memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data));
 207	qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base);
 208	qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec);
 209	qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size);
 210	qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid);
 211	qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color);
 212	qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid);
 213}
 214
 215static void qdma_prep_fmap_context(const struct qdma_device *qdev,
 216				   const struct qdma_ctxt_fmap *ctxt,
 217				   u32 *data)
 218{
 219	memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data));
 220	qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase);
 221	qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax);
 222}
 223
 224/*
 225 * Program the indirect context register space
 226 *
 227 * Once the queue is enabled, context is dynamically updated by hardware. Any
 228 * modification of the context through this API when the queue is enabled can
 229 * result in unexpected behavior. Reading the context when the queue is enabled
 230 * is not recommended as it can result in reduced performance.
 231 */
 232static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type,
 233			     enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt)
 234{
 235	int ret;
 236
 237	mutex_lock(&qdev->ctxt_lock);
 238	if (cmd == QDMA_CTXT_WRITE) {
 239		ret = qdma_context_write_data(qdev, ctxt);
 240		if (ret)
 241			goto failed;
 242	}
 243
 244	ret = qdma_context_cmd_execute(qdev, type, cmd, index);
 245	if (ret)
 246		goto failed;
 247
 248	if (cmd == QDMA_CTXT_READ) {
 249		ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA);
 250		if (ret)
 251			goto failed;
 252	}
 253
 254failed:
 255	mutex_unlock(&qdev->ctxt_lock);
 256
 257	return ret;
 258}
 259
 260static int qdma_check_queue_status(struct qdma_device *qdev,
 261				   enum dma_transfer_direction dir, u16 qid)
 262{
 263	u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0};
 264	enum qdma_ctxt_type type;
 265	int ret;
 266
 267	if (dir == DMA_MEM_TO_DEV)
 268		type = QDMA_CTXT_DESC_SW_H2C;
 269	else
 270		type = QDMA_CTXT_DESC_SW_C2H;
 271	ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data);
 272	if (ret)
 273		return ret;
 274
 275	status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE);
 276	if (status) {
 277		qdma_err(qdev, "queue %d already in use", qid);
 278		return -EBUSY;
 279	}
 280
 281	return 0;
 282}
 283
 284static int qdma_clear_queue_context(const struct qdma_queue *queue)
 285{
 286	static const enum qdma_ctxt_type h2c_types[] = {
 287		QDMA_CTXT_DESC_SW_H2C,
 288		QDMA_CTXT_DESC_HW_H2C,
 289		QDMA_CTXT_DESC_CR_H2C,
 290		QDMA_CTXT_PFTCH,
 291	};
 292	static const enum qdma_ctxt_type c2h_types[] = {
 293		QDMA_CTXT_DESC_SW_C2H,
 294		QDMA_CTXT_DESC_HW_C2H,
 295		QDMA_CTXT_DESC_CR_C2H,
 296		QDMA_CTXT_PFTCH,
 297	};
 298	struct qdma_device *qdev = queue->qdev;
 299	const enum qdma_ctxt_type *type;
 300	int ret, num, i;
 301
 302	if (queue->dir == DMA_MEM_TO_DEV) {
 303		type = h2c_types;
 304		num = ARRAY_SIZE(h2c_types);
 305	} else {
 306		type = c2h_types;
 307		num = ARRAY_SIZE(c2h_types);
 308	}
 309	for (i = 0; i < num; i++) {
 310		ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR,
 311					queue->qid, NULL);
 312		if (ret) {
 313			qdma_err(qdev, "Failed to clear ctxt %d", type[i]);
 314			return ret;
 315		}
 316	}
 317
 318	return 0;
 319}
 320
 321static int qdma_setup_fmap_context(struct qdma_device *qdev)
 322{
 323	u32 ctxt[QDMA_CTXT_REGMAP_LEN];
 324	struct qdma_ctxt_fmap fmap;
 325	int ret;
 326
 327	ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR,
 328				qdev->fid, NULL);
 329	if (ret) {
 330		qdma_err(qdev, "Failed clearing context");
 331		return ret;
 332	}
 333
 334	fmap.qbase = 0;
 335	fmap.qmax = qdev->chan_num * 2;
 336	qdma_prep_fmap_context(qdev, &fmap, ctxt);
 337	ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE,
 338				qdev->fid, ctxt);
 339	if (ret)
 340		qdma_err(qdev, "Failed setup fmap, ret %d", ret);
 341
 342	return ret;
 343}
 344
 345static int qdma_setup_queue_context(struct qdma_device *qdev,
 346				    const struct qdma_ctxt_sw_desc *sw_desc,
 347				    enum dma_transfer_direction dir, u16 qid)
 348{
 349	u32 ctxt[QDMA_CTXT_REGMAP_LEN];
 350	enum qdma_ctxt_type type;
 351	int ret;
 352
 353	if (dir == DMA_MEM_TO_DEV)
 354		type = QDMA_CTXT_DESC_SW_H2C;
 355	else
 356		type = QDMA_CTXT_DESC_SW_C2H;
 357
 358	qdma_prep_sw_desc_context(qdev, sw_desc, ctxt);
 359	/* Setup SW descriptor context */
 360	ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt);
 361	if (ret)
 362		qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid);
 363
 364	return ret;
 365}
 366
 367/*
 368 * Enable or disable memory-mapped DMA engines
 369 * 1: enable, 0: disable
 370 */
 371static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl)
 372{
 373	int ret;
 374
 375	ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL);
 376	ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL);
 377
 378	return ret;
 379}
 380
 381static int qdma_get_hw_info(struct qdma_device *qdev)
 382{
 383	struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev);
 384	u32 value = 0;
 385	int ret;
 386
 387	ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT);
 388	if (ret)
 389		return ret;
 390
 391	value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1;
 392	if (pdata->max_mm_channels * 2 > value) {
 393		qdma_err(qdev, "not enough hw queues %d", value);
 394		return -EINVAL;
 395	}
 396	qdev->chan_num = pdata->max_mm_channels;
 397
 398	ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID);
 399	if (ret)
 400		return ret;
 401
 402	qdma_info(qdev, "max channel %d, function id %d",
 403		  qdev->chan_num, qdev->fid);
 404
 405	return 0;
 406}
 407
 408static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx)
 409{
 410	struct qdma_device *qdev = queue->qdev;
 411
 412	return regmap_write(qdev->regmap, queue->pidx_reg,
 413			    pidx | QDMA_QUEUE_ARM_BIT);
 414}
 415
 416static inline int qdma_update_cidx(const struct qdma_queue *queue,
 417				   u16 ridx, u16 cidx)
 418{
 419	struct qdma_device *qdev = queue->qdev;
 420
 421	return regmap_write(qdev->regmap, queue->cidx_reg,
 422			    ((u32)ridx << 16) | cidx);
 423}
 424
 425/**
 426 * qdma_free_vdesc - Free descriptor
 427 * @vdesc: Virtual DMA descriptor
 428 */
 429static void qdma_free_vdesc(struct virt_dma_desc *vdesc)
 430{
 431	struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc);
 432
 433	kfree(vd);
 434}
 435
 436static int qdma_alloc_queues(struct qdma_device *qdev,
 437			     enum dma_transfer_direction dir)
 438{
 439	struct qdma_queue *q, **queues;
 440	u32 i, pidx_base;
 441	int ret;
 442
 443	if (dir == DMA_MEM_TO_DEV) {
 444		queues = &qdev->h2c_queues;
 445		pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX);
 446	} else {
 447		queues = &qdev->c2h_queues;
 448		pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX);
 449	}
 450
 451	*queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q),
 452			       GFP_KERNEL);
 453	if (!*queues)
 454		return -ENOMEM;
 455
 456	for (i = 0; i < qdev->chan_num; i++) {
 457		ret = qdma_check_queue_status(qdev, dir, i);
 458		if (ret)
 459			return ret;
 460
 461		q = &(*queues)[i];
 462		q->ring_size = QDMA_DEFAULT_RING_SIZE;
 463		q->idx_mask = q->ring_size - 2;
 464		q->qdev = qdev;
 465		q->dir = dir;
 466		q->qid = i;
 467		q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE;
 468		q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) +
 469				i * QDMA_DMAP_REG_STRIDE;
 470		q->vchan.desc_free = qdma_free_vdesc;
 471		vchan_init(&q->vchan, &qdev->dma_dev);
 472	}
 473
 474	return 0;
 475}
 476
 477static int qdma_device_verify(struct qdma_device *qdev)
 478{
 479	u32 value;
 480	int ret;
 481
 482	ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value);
 483	if (ret)
 484		return ret;
 485
 486	value = FIELD_GET(QDMA_IDENTIFIER_MASK, value);
 487	if (value != QDMA_IDENTIFIER) {
 488		qdma_err(qdev, "Invalid identifier");
 489		return -ENODEV;
 490	}
 491	qdev->rfields = qdma_regfs_default;
 492	qdev->roffs = qdma_regos_default;
 493
 494	return 0;
 495}
 496
 497static int qdma_device_setup(struct qdma_device *qdev)
 498{
 499	u32 ring_sz = QDMA_DEFAULT_RING_SIZE;
 500	int ret = 0;
 501
 502	ret = qdma_setup_fmap_context(qdev);
 503	if (ret) {
 504		qdma_err(qdev, "Failed setup fmap context");
 505		return ret;
 506	}
 507
 508	/* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */
 509	ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE);
 510	if (ret) {
 511		qdma_err(qdev, "Failed to setup ring %d of size %ld",
 512			 QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE);
 513		return ret;
 514	}
 515
 516	/* Enable memory-mapped DMA engine in both directions */
 517	ret = qdma_sgdma_control(qdev, 1);
 518	if (ret) {
 519		qdma_err(qdev, "Failed to SGDMA with error %d", ret);
 520		return ret;
 521	}
 522
 523	ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV);
 524	if (ret) {
 525		qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret);
 526		return ret;
 527	}
 528
 529	ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM);
 530	if (ret) {
 531		qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret);
 532		return ret;
 533	}
 534
 535	return 0;
 536}
 537
 538/**
 539 * qdma_free_queue_resources() - Free queue resources
 540 * @chan: DMA channel
 541 */
 542static void qdma_free_queue_resources(struct dma_chan *chan)
 543{
 544	struct qdma_queue *queue = to_qdma_queue(chan);
 545	struct qdma_device *qdev = queue->qdev;
 546	struct qdma_platdata *pdata;
 547
 548	qdma_clear_queue_context(queue);
 549	vchan_free_chan_resources(&queue->vchan);
 550	pdata = dev_get_platdata(&qdev->pdev->dev);
 551	dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE,
 552			  queue->desc_base, queue->dma_desc_base);
 553}
 554
 555/**
 556 * qdma_alloc_queue_resources() - Allocate queue resources
 557 * @chan: DMA channel
 558 */
 559static int qdma_alloc_queue_resources(struct dma_chan *chan)
 560{
 561	struct qdma_queue *queue = to_qdma_queue(chan);
 562	struct qdma_device *qdev = queue->qdev;
 563	struct qdma_ctxt_sw_desc desc;
 564	struct qdma_platdata *pdata;
 565	size_t size;
 566	int ret;
 567
 568	ret = qdma_clear_queue_context(queue);
 569	if (ret)
 570		return ret;
 571
 572	pdata = dev_get_platdata(&qdev->pdev->dev);
 573	size = queue->ring_size * QDMA_MM_DESC_SIZE;
 574	queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size,
 575					      &queue->dma_desc_base,
 576					      GFP_KERNEL);
 577	if (!queue->desc_base) {
 578		qdma_err(qdev, "Failed to allocate descriptor ring");
 579		return -ENOMEM;
 580	}
 581
 582	/* Setup SW descriptor queue context for DMA memory map */
 583	desc.vec = qdma_get_intr_ring_idx(qdev);
 584	desc.desc_base = queue->dma_desc_base;
 585	ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid);
 586	if (ret) {
 587		qdma_err(qdev, "Failed to setup SW desc ctxt for %s",
 588			 chan->name);
 589		dma_free_coherent(pdata->dma_dev, size, queue->desc_base,
 590				  queue->dma_desc_base);
 591		return ret;
 592	}
 593
 594	queue->pidx = 0;
 595	queue->cidx = 0;
 596
 597	return 0;
 598}
 599
 600static bool qdma_filter_fn(struct dma_chan *chan, void *param)
 601{
 602	struct qdma_queue *queue = to_qdma_queue(chan);
 603	struct qdma_queue_info *info = param;
 604
 605	return info->dir == queue->dir;
 606}
 607
 608static int qdma_xfer_start(struct qdma_queue *queue)
 609{
 610	struct qdma_device *qdev = queue->qdev;
 611	int ret;
 612
 613	if (!vchan_next_desc(&queue->vchan))
 614		return 0;
 615
 616	qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d",
 617		 queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid);
 618
 619	ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx);
 620	if (ret) {
 621		qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d",
 622			 queue->pidx, CHAN_STR(queue), queue->qid);
 623	}
 624
 625	return ret;
 626}
 627
 628static void qdma_issue_pending(struct dma_chan *chan)
 629{
 630	struct qdma_queue *queue = to_qdma_queue(chan);
 631	unsigned long flags;
 632
 633	spin_lock_irqsave(&queue->vchan.lock, flags);
 634	if (vchan_issue_pending(&queue->vchan)) {
 635		if (queue->submitted_vdesc) {
 636			queue->issued_vdesc = queue->submitted_vdesc;
 637			queue->submitted_vdesc = NULL;
 638		}
 639		qdma_xfer_start(queue);
 640	}
 641
 642	spin_unlock_irqrestore(&queue->vchan.lock, flags);
 643}
 644
 645static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q)
 646{
 647	struct qdma_mm_desc *desc;
 648
 649	if (((q->pidx + 1) & q->idx_mask) == q->cidx)
 650		return NULL;
 651
 652	desc = q->desc_base + q->pidx;
 653	q->pidx = (q->pidx + 1) & q->idx_mask;
 654
 655	return desc;
 656}
 657
 658static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc)
 659{
 660	struct qdma_mm_desc *desc;
 661	struct scatterlist *sg;
 662	u64 addr, *src, *dst;
 663	u32 rest, len;
 664	int ret = 0;
 665	u32 i;
 666
 667	if (!vdesc->sg_len)
 668		return 0;
 669
 670	if (q->dir == DMA_MEM_TO_DEV) {
 671		dst = &vdesc->dev_addr;
 672		src = &addr;
 673	} else {
 674		dst = &addr;
 675		src = &vdesc->dev_addr;
 676	}
 677
 678	for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) {
 679		addr = sg_dma_address(sg) + vdesc->sg_off;
 680		rest = sg_dma_len(sg) - vdesc->sg_off;
 681		while (rest) {
 682			len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN);
 683			desc = qdma_get_desc(q);
 684			if (!desc) {
 685				ret = -EBUSY;
 686				goto out;
 687			}
 688
 689			desc->src_addr = cpu_to_le64(*src);
 690			desc->dst_addr = cpu_to_le64(*dst);
 691			desc->len = cpu_to_le32(len);
 692
 693			vdesc->dev_addr += len;
 694			vdesc->sg_off += len;
 695			vdesc->pending_descs++;
 696			addr += len;
 697			rest -= len;
 698		}
 699		vdesc->sg_off = 0;
 700	}
 701out:
 702	vdesc->sg_len -= i;
 703	vdesc->pidx = q->pidx;
 704	return ret;
 705}
 706
 707static void qdma_fill_pending_vdesc(struct qdma_queue *q)
 708{
 709	struct virt_dma_chan *vc = &q->vchan;
 710	struct qdma_mm_vdesc *vdesc = NULL;
 711	struct virt_dma_desc *vd;
 712	int ret;
 713
 714	if (!list_empty(&vc->desc_issued)) {
 715		vd = &q->issued_vdesc->vdesc;
 716		list_for_each_entry_from(vd, &vc->desc_issued, node) {
 717			vdesc = to_qdma_vdesc(vd);
 718			ret = qdma_hw_enqueue(q, vdesc);
 719			if (ret) {
 720				q->issued_vdesc = vdesc;
 721				return;
 722			}
 723		}
 724		q->issued_vdesc = vdesc;
 725	}
 726
 727	if (list_empty(&vc->desc_submitted))
 728		return;
 729
 730	if (q->submitted_vdesc)
 731		vd = &q->submitted_vdesc->vdesc;
 732	else
 733		vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node);
 734
 735	list_for_each_entry_from(vd, &vc->desc_submitted, node) {
 736		vdesc = to_qdma_vdesc(vd);
 737		ret = qdma_hw_enqueue(q, vdesc);
 738		if (ret)
 739			break;
 740	}
 741	q->submitted_vdesc = vdesc;
 742}
 743
 744static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx)
 745{
 746	struct virt_dma_chan *vc = to_virt_chan(tx->chan);
 747	struct qdma_queue *q = to_qdma_queue(&vc->chan);
 748	struct virt_dma_desc *vd;
 749	unsigned long flags;
 750	dma_cookie_t cookie;
 751
 752	vd = container_of(tx, struct virt_dma_desc, tx);
 753	spin_lock_irqsave(&vc->lock, flags);
 754	cookie = dma_cookie_assign(tx);
 755
 756	list_move_tail(&vd->node, &vc->desc_submitted);
 757	qdma_fill_pending_vdesc(q);
 758	spin_unlock_irqrestore(&vc->lock, flags);
 759
 760	return cookie;
 761}
 762
 763static struct dma_async_tx_descriptor *
 764qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl,
 765		    unsigned int sg_len, enum dma_transfer_direction dir,
 766		    unsigned long flags, void *context)
 767{
 768	struct qdma_queue *q = to_qdma_queue(chan);
 769	struct dma_async_tx_descriptor *tx;
 770	struct qdma_mm_vdesc *vdesc;
 771
 772	vdesc = kzalloc(sizeof(*vdesc), GFP_NOWAIT);
 773	if (!vdesc)
 774		return NULL;
 775	vdesc->sgl = sgl;
 776	vdesc->sg_len = sg_len;
 777	if (dir == DMA_MEM_TO_DEV)
 778		vdesc->dev_addr = q->cfg.dst_addr;
 779	else
 780		vdesc->dev_addr = q->cfg.src_addr;
 781
 782	tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags);
 783	tx->tx_submit = qdma_tx_submit;
 784
 785	return tx;
 786}
 787
 788static int qdma_device_config(struct dma_chan *chan,
 789			      struct dma_slave_config *cfg)
 790{
 791	struct qdma_queue *q = to_qdma_queue(chan);
 792
 793	memcpy(&q->cfg, cfg, sizeof(*cfg));
 794
 795	return 0;
 796}
 797
 798static int qdma_arm_err_intr(const struct qdma_device *qdev)
 799{
 800	u32 value = 0;
 801
 802	qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid);
 803	qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx);
 804	qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1);
 805
 806	return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT);
 807}
 808
 809static irqreturn_t qdma_error_isr(int irq, void *data)
 810{
 811	struct qdma_device *qdev = data;
 812	u32 err_stat = 0;
 813	int ret;
 814
 815	ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT);
 816	if (ret) {
 817		qdma_err(qdev, "read error state failed, ret %d", ret);
 818		goto out;
 819	}
 820
 821	qdma_err(qdev, "global error %d", err_stat);
 822	ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT);
 823	if (ret)
 824		qdma_err(qdev, "clear error state failed, ret %d", ret);
 825
 826out:
 827	qdma_arm_err_intr(qdev);
 828	return IRQ_HANDLED;
 829}
 830
 831static irqreturn_t qdma_queue_isr(int irq, void *data)
 832{
 833	struct qdma_intr_ring *intr = data;
 834	struct qdma_queue *q = NULL;
 835	struct qdma_device *qdev;
 836	u32 index, comp_desc;
 837	u64 intr_ent;
 838	u8 color;
 839	int ret;
 840	u16 qid;
 841
 842	qdev = intr->qdev;
 843	index = intr->cidx;
 844	while (1) {
 845		struct virt_dma_desc *vd;
 846		struct qdma_mm_vdesc *vdesc;
 847		unsigned long flags;
 848		u32 cidx;
 849
 850		intr_ent = le64_to_cpu(intr->base[index]);
 851		color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent);
 852		if (color != intr->color)
 853			break;
 854
 855		qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent);
 856		if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent))
 857			q = qdev->c2h_queues;
 858		else
 859			q = qdev->h2c_queues;
 860		q += qid;
 861
 862		cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent);
 863
 864		spin_lock_irqsave(&q->vchan.lock, flags);
 865		comp_desc = (cidx - q->cidx) & q->idx_mask;
 866
 867		vd = vchan_next_desc(&q->vchan);
 868		if (!vd)
 869			goto skip;
 870
 871		vdesc = to_qdma_vdesc(vd);
 872		while (comp_desc > vdesc->pending_descs) {
 873			list_del(&vd->node);
 874			vchan_cookie_complete(vd);
 875			comp_desc -= vdesc->pending_descs;
 876			vd = vchan_next_desc(&q->vchan);
 877			vdesc = to_qdma_vdesc(vd);
 878		}
 879		vdesc->pending_descs -= comp_desc;
 880		if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) {
 881			list_del(&vd->node);
 882			vchan_cookie_complete(vd);
 883		}
 884		q->cidx = cidx;
 885
 886		qdma_fill_pending_vdesc(q);
 887		qdma_xfer_start(q);
 888
 889skip:
 890		spin_unlock_irqrestore(&q->vchan.lock, flags);
 891
 892		/*
 893		 * Wrap the index value and flip the expected color value if
 894		 * interrupt aggregation PIDX has wrapped around.
 895		 */
 896		index++;
 897		index &= QDMA_INTR_RING_IDX_MASK;
 898		if (!index)
 899			intr->color = !intr->color;
 900	}
 901
 902	/*
 903	 * Update the software interrupt aggregation ring CIDX if a valid entry
 904	 * was found.
 905	 */
 906	if (q) {
 907		qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index);
 908
 909		/*
 910		 * Record the last read index of status descriptor from the
 911		 * interrupt aggregation ring.
 912		 */
 913		intr->cidx = index;
 914
 915		ret = qdma_update_cidx(q, intr->ridx, index);
 916		if (ret) {
 917			qdma_err(qdev, "Failed to update IRQ CIDX");
 918			return IRQ_NONE;
 919		}
 920	}
 921
 922	return IRQ_HANDLED;
 923}
 924
 925static int qdma_init_error_irq(struct qdma_device *qdev)
 926{
 927	struct device *dev = &qdev->pdev->dev;
 928	int ret;
 929	u32 vec;
 930
 931	vec = qdev->queue_irq_start - 1;
 932
 933	ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr,
 934					IRQF_ONESHOT, "amd-qdma-error", qdev);
 935	if (ret) {
 936		qdma_err(qdev, "Failed to request error IRQ vector: %d", vec);
 937		return ret;
 938	}
 939
 940	ret = qdma_arm_err_intr(qdev);
 941	if (ret)
 942		qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret);
 943
 944	return ret;
 945}
 946
 947static int qdmam_alloc_qintr_rings(struct qdma_device *qdev)
 948{
 949	struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev);
 950	struct device *dev = &qdev->pdev->dev;
 951	u32 ctxt[QDMA_CTXT_REGMAP_LEN];
 952	struct qdma_intr_ring *ring;
 953	struct qdma_ctxt_intr intr_ctxt;
 954	u32 vector;
 955	int ret, i;
 956
 957	qdev->qintr_ring_num = qdev->queue_irq_num;
 958	qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num,
 959					 sizeof(*qdev->qintr_rings),
 960					 GFP_KERNEL);
 961	if (!qdev->qintr_rings)
 962		return -ENOMEM;
 963
 964	vector = qdev->queue_irq_start;
 965	for (i = 0; i < qdev->qintr_ring_num; i++, vector++) {
 966		ring = &qdev->qintr_rings[i];
 967		ring->qdev = qdev;
 968		ring->msix_id = qdev->err_irq_idx + i + 1;
 969		ring->ridx = i;
 970		ring->color = 1;
 971		ring->base = dmam_alloc_coherent(pdata->dma_dev,
 972						 QDMA_INTR_RING_SIZE,
 973						 &ring->dev_base, GFP_KERNEL);
 974		if (!ring->base) {
 975			qdma_err(qdev, "Failed to alloc intr ring %d", i);
 976			return -ENOMEM;
 977		}
 978		intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base);
 979		intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096;
 980		intr_ctxt.vec = ring->msix_id;
 981		intr_ctxt.valid = true;
 982		intr_ctxt.color = true;
 983		ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL,
 984					QDMA_CTXT_CLEAR, ring->ridx, NULL);
 985		if (ret) {
 986			qdma_err(qdev, "Failed clear intr ctx, ret %d", ret);
 987			return ret;
 988		}
 989
 990		qdma_prep_intr_context(qdev, &intr_ctxt, ctxt);
 991		ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL,
 992					QDMA_CTXT_WRITE, ring->ridx, ctxt);
 993		if (ret) {
 994			qdma_err(qdev, "Failed setup intr ctx, ret %d", ret);
 995			return ret;
 996		}
 997
 998		ret = devm_request_threaded_irq(dev, vector, NULL,
 999						qdma_queue_isr, IRQF_ONESHOT,
1000						"amd-qdma-queue", ring);
1001		if (ret) {
1002			qdma_err(qdev, "Failed to request irq %d", vector);
1003			return ret;
1004		}
1005	}
1006
1007	return 0;
1008}
1009
1010static int qdma_intr_init(struct qdma_device *qdev)
1011{
1012	int ret;
1013
1014	ret = qdma_init_error_irq(qdev);
1015	if (ret) {
1016		qdma_err(qdev, "Failed to init error IRQs, ret %d", ret);
1017		return ret;
1018	}
1019
1020	ret = qdmam_alloc_qintr_rings(qdev);
1021	if (ret) {
1022		qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret);
1023		return ret;
1024	}
1025
1026	return 0;
1027}
1028
1029static void amd_qdma_remove(struct platform_device *pdev)
1030{
1031	struct qdma_device *qdev = platform_get_drvdata(pdev);
1032
1033	qdma_sgdma_control(qdev, 0);
1034	dma_async_device_unregister(&qdev->dma_dev);
1035
1036	mutex_destroy(&qdev->ctxt_lock);
1037}
1038
1039static int amd_qdma_probe(struct platform_device *pdev)
1040{
1041	struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev);
1042	struct qdma_device *qdev;
1043	struct resource *res;
1044	void __iomem *regs;
1045	int ret;
1046
1047	qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL);
1048	if (!qdev)
1049		return -ENOMEM;
1050
1051	platform_set_drvdata(pdev, qdev);
1052	qdev->pdev = pdev;
1053	mutex_init(&qdev->ctxt_lock);
1054
1055	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1056	if (!res) {
1057		qdma_err(qdev, "Failed to get IRQ resource");
1058		ret = -ENODEV;
1059		goto failed;
1060	}
1061	qdev->err_irq_idx = pdata->irq_index;
1062	qdev->queue_irq_start = res->start + 1;
1063	qdev->queue_irq_num = resource_size(res) - 1;
1064
1065	regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
1066	if (IS_ERR(regs)) {
1067		ret = PTR_ERR(regs);
1068		qdma_err(qdev, "Failed to map IO resource, err %d", ret);
1069		goto failed;
1070	}
1071
1072	qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs,
1073					     &qdma_regmap_config);
1074	if (IS_ERR(qdev->regmap)) {
1075		ret = PTR_ERR(qdev->regmap);
1076		qdma_err(qdev, "Regmap init failed, err %d", ret);
1077		goto failed;
1078	}
1079
1080	ret = qdma_device_verify(qdev);
1081	if (ret)
1082		goto failed;
1083
1084	ret = qdma_get_hw_info(qdev);
1085	if (ret)
1086		goto failed;
1087
1088	INIT_LIST_HEAD(&qdev->dma_dev.channels);
1089
1090	ret = qdma_device_setup(qdev);
1091	if (ret)
1092		goto failed;
1093
1094	ret = qdma_intr_init(qdev);
1095	if (ret) {
1096		qdma_err(qdev, "Failed to initialize IRQs %d", ret);
1097		goto failed_disable_engine;
1098	}
1099
1100	dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask);
1101	dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask);
1102
1103	qdev->dma_dev.dev = &pdev->dev;
1104	qdev->dma_dev.filter.map = pdata->device_map;
1105	qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2;
1106	qdev->dma_dev.filter.fn = qdma_filter_fn;
1107	qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources;
1108	qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources;
1109	qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg;
1110	qdev->dma_dev.device_config = qdma_device_config;
1111	qdev->dma_dev.device_issue_pending = qdma_issue_pending;
1112	qdev->dma_dev.device_tx_status = dma_cookie_status;
1113	qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1114
1115	ret = dma_async_device_register(&qdev->dma_dev);
1116	if (ret) {
1117		qdma_err(qdev, "Failed to register AMD QDMA: %d", ret);
1118		goto failed_disable_engine;
1119	}
1120
1121	return 0;
1122
1123failed_disable_engine:
1124	qdma_sgdma_control(qdev, 0);
1125failed:
1126	mutex_destroy(&qdev->ctxt_lock);
1127	qdma_err(qdev, "Failed to probe AMD QDMA driver");
1128	return ret;
1129}
1130
1131static struct platform_driver amd_qdma_driver = {
1132	.driver		= {
1133		.name = "amd-qdma",
1134	},
1135	.probe		= amd_qdma_probe,
1136	.remove		= amd_qdma_remove,
1137};
1138
1139module_platform_driver(amd_qdma_driver);
1140
1141MODULE_DESCRIPTION("AMD QDMA driver");
1142MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>");
1143MODULE_LICENSE("GPL");