dma.c

  1// SPDX-License-Identifier: BSD-3-Clause
  2/*
  3 * Copyright (c) 2020, MIPI Alliance, Inc.
  4 *
  5 * Author: Nicolas Pitre <npitre@baylibre.com>
  6 *
  7 * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on
  8 * v1.x of the spec and v2.0 will likely be split out.
  9 */
 10
 11#include <linux/bitfield.h>
 12#include <linux/device.h>
 13#include <linux/dma-mapping.h>
 14#include <linux/errno.h>
 15#include <linux/i3c/master.h>
 16#include <linux/io.h>
 17#include <linux/pci.h>
 18
 19#include "hci.h"
 20#include "cmd.h"
 21#include "ibi.h"
 22
 23
 24/*
 25 * Software Parameter Values (somewhat arb itrary for now).
 26 * Some of them could be determined at run time eventually.
 27 */
 28
 29#define XFER_RINGS			1	/* max: 8 */
 30#define XFER_RING_ENTRIES		16	/* max: 255 */
 31
 32#define IBI_RINGS			1	/* max: 8 */
 33#define IBI_STATUS_RING_ENTRIES		32	/* max: 255 */
 34#define IBI_CHUNK_CACHELINES		1	/* max: 256 bytes equivalent */
 35#define IBI_CHUNK_POOL_SIZE		128	/* max: 1023 */
 36
 37/*
 38 * Ring Header Preamble
 39 */
 40
 41#define rhs_reg_read(r)		readl(hci->RHS_regs + (RHS_##r))
 42#define rhs_reg_write(r, v)	writel(v, hci->RHS_regs + (RHS_##r))
 43
 44#define RHS_CONTROL			0x00
 45#define PREAMBLE_SIZE			GENMASK(31, 24)	/* Preamble Section Size */
 46#define HEADER_SIZE			GENMASK(23, 16)	/* Ring Header Size */
 47#define MAX_HEADER_COUNT_CAP		GENMASK(7, 4) /* HC Max Header Count */
 48#define MAX_HEADER_COUNT		GENMASK(3, 0) /* Driver Max Header Count */
 49
 50#define RHS_RHn_OFFSET(n)		(0x04 + (n)*4)
 51
 52/*
 53 * Ring Header (Per-Ring Bundle)
 54 */
 55
 56#define rh_reg_read(r)		readl(rh->regs + (RH_##r))
 57#define rh_reg_write(r, v)	writel(v, rh->regs + (RH_##r))
 58
 59#define RH_CR_SETUP			0x00	/* Command/Response Ring */
 60#define CR_XFER_STRUCT_SIZE		GENMASK(31, 24)
 61#define CR_RESP_STRUCT_SIZE		GENMASK(23, 16)
 62#define CR_RING_SIZE			GENMASK(8, 0)
 63
 64#define RH_IBI_SETUP			0x04
 65#define IBI_STATUS_STRUCT_SIZE		GENMASK(31, 24)
 66#define IBI_STATUS_RING_SIZE		GENMASK(23, 16)
 67#define IBI_DATA_CHUNK_SIZE		GENMASK(12, 10)
 68#define IBI_DATA_CHUNK_COUNT		GENMASK(9, 0)
 69
 70#define RH_CHUNK_CONTROL			0x08
 71
 72#define RH_INTR_STATUS			0x10
 73#define RH_INTR_STATUS_ENABLE		0x14
 74#define RH_INTR_SIGNAL_ENABLE		0x18
 75#define RH_INTR_FORCE			0x1c
 76#define INTR_IBI_READY			BIT(12)
 77#define INTR_TRANSFER_COMPLETION	BIT(11)
 78#define INTR_RING_OP			BIT(10)
 79#define INTR_TRANSFER_ERR		BIT(9)
 80#define INTR_IBI_RING_FULL		BIT(6)
 81#define INTR_TRANSFER_ABORT		BIT(5)
 82
 83#define RH_RING_STATUS			0x20
 84#define RING_STATUS_LOCKED		BIT(3)
 85#define RING_STATUS_ABORTED		BIT(2)
 86#define RING_STATUS_RUNNING		BIT(1)
 87#define RING_STATUS_ENABLED		BIT(0)
 88
 89#define RH_RING_CONTROL			0x24
 90#define RING_CTRL_ABORT			BIT(2)
 91#define RING_CTRL_RUN_STOP		BIT(1)
 92#define RING_CTRL_ENABLE		BIT(0)
 93
 94#define RH_RING_OPERATION1		0x28
 95#define RING_OP1_IBI_DEQ_PTR		GENMASK(23, 16)
 96#define RING_OP1_CR_SW_DEQ_PTR		GENMASK(15, 8)
 97#define RING_OP1_CR_ENQ_PTR		GENMASK(7, 0)
 98
 99#define RH_RING_OPERATION2		0x2c
100#define RING_OP2_IBI_ENQ_PTR		GENMASK(23, 16)
101#define RING_OP2_CR_DEQ_PTR		GENMASK(7, 0)
102
103#define RH_CMD_RING_BASE_LO		0x30
104#define RH_CMD_RING_BASE_HI		0x34
105#define RH_RESP_RING_BASE_LO		0x38
106#define RH_RESP_RING_BASE_HI		0x3c
107#define RH_IBI_STATUS_RING_BASE_LO	0x40
108#define RH_IBI_STATUS_RING_BASE_HI	0x44
109#define RH_IBI_DATA_RING_BASE_LO	0x48
110#define RH_IBI_DATA_RING_BASE_HI	0x4c
111
112#define RH_CMD_RING_SG			0x50	/* Ring Scatter Gather Support */
113#define RH_RESP_RING_SG			0x54
114#define RH_IBI_STATUS_RING_SG		0x58
115#define RH_IBI_DATA_RING_SG		0x5c
116#define RING_SG_BLP			BIT(31)	/* Buffer Vs. List Pointer */
117#define RING_SG_LIST_SIZE		GENMASK(15, 0)
118
119/*
120 * Data Buffer Descriptor (in memory)
121 */
122
123#define DATA_BUF_BLP			BIT(31)	/* Buffer Vs. List Pointer */
124#define DATA_BUF_IOC			BIT(30)	/* Interrupt on Completion */
125#define DATA_BUF_BLOCK_SIZE		GENMASK(15, 0)
126
127
128struct hci_rh_data {
129	void __iomem *regs;
130	void *xfer, *resp, *ibi_status, *ibi_data;
131	dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma;
132	unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total;
133	unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz;
134	unsigned int done_ptr, ibi_chunk_ptr;
135	struct hci_xfer **src_xfers;
136	struct completion op_done;
137};
138
139struct hci_rings_data {
140	struct device *sysdev;
141	unsigned int total;
142	struct hci_rh_data headers[] __counted_by(total);
143};
144
145struct hci_dma_dev_ibi_data {
146	struct i3c_generic_ibi_pool *pool;
147	unsigned int max_len;
148};
149
150static void hci_dma_cleanup(struct i3c_hci *hci)
151{
152	struct hci_rings_data *rings = hci->io_data;
153	struct hci_rh_data *rh;
154	unsigned int i;
155
156	if (!rings)
157		return;
158
159	for (i = 0; i < rings->total; i++) {
160		rh = &rings->headers[i];
161
162		rh_reg_write(INTR_SIGNAL_ENABLE, 0);
163		rh_reg_write(RING_CONTROL, 0);
164		rh_reg_write(CR_SETUP, 0);
165		rh_reg_write(IBI_SETUP, 0);
166
167		if (rh->xfer)
168			dma_free_coherent(rings->sysdev,
169					  rh->xfer_struct_sz * rh->xfer_entries,
170					  rh->xfer, rh->xfer_dma);
171		if (rh->resp)
172			dma_free_coherent(rings->sysdev,
173					  rh->resp_struct_sz * rh->xfer_entries,
174					  rh->resp, rh->resp_dma);
175		kfree(rh->src_xfers);
176		if (rh->ibi_status)
177			dma_free_coherent(rings->sysdev,
178					  rh->ibi_status_sz * rh->ibi_status_entries,
179					  rh->ibi_status, rh->ibi_status_dma);
180		if (rh->ibi_data_dma)
181			dma_unmap_single(rings->sysdev, rh->ibi_data_dma,
182					 rh->ibi_chunk_sz * rh->ibi_chunks_total,
183					 DMA_FROM_DEVICE);
184		kfree(rh->ibi_data);
185	}
186
187	rhs_reg_write(CONTROL, 0);
188
189	kfree(rings);
190	hci->io_data = NULL;
191}
192
193static int hci_dma_init(struct i3c_hci *hci)
194{
195	struct hci_rings_data *rings;
196	struct hci_rh_data *rh;
197	struct device *sysdev;
198	u32 regval;
199	unsigned int i, nr_rings, xfers_sz, resps_sz;
200	unsigned int ibi_status_ring_sz, ibi_data_ring_sz;
201	int ret;
202
203	/*
204	 * Set pointer to a physical device that does DMA and has IOMMU setup
205	 * done for it in case of enabled IOMMU and use it with the DMA API.
206	 * Here such device is either
207	 * "mipi-i3c-hci" platform device (OF/ACPI enumeration) parent or
208	 * grandparent (PCI enumeration).
209	 */
210	sysdev = hci->master.dev.parent;
211	if (sysdev->parent && dev_is_pci(sysdev->parent))
212		sysdev = sysdev->parent;
213
214	regval = rhs_reg_read(CONTROL);
215	nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval);
216	dev_info(&hci->master.dev, "%d DMA rings available\n", nr_rings);
217	if (unlikely(nr_rings > 8)) {
218		dev_err(&hci->master.dev, "number of rings should be <= 8\n");
219		nr_rings = 8;
220	}
221	if (nr_rings > XFER_RINGS)
222		nr_rings = XFER_RINGS;
223	rings = kzalloc(struct_size(rings, headers, nr_rings), GFP_KERNEL);
224	if (!rings)
225		return -ENOMEM;
226	hci->io_data = rings;
227	rings->total = nr_rings;
228	rings->sysdev = sysdev;
229
230	regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total);
231	rhs_reg_write(CONTROL, regval);
232
233	for (i = 0; i < rings->total; i++) {
234		u32 offset = rhs_reg_read(RHn_OFFSET(i));
235
236		dev_info(&hci->master.dev, "Ring %d at offset %#x\n", i, offset);
237		ret = -EINVAL;
238		if (!offset)
239			goto err_out;
240		rh = &rings->headers[i];
241		rh->regs = hci->base_regs + offset;
242		init_completion(&rh->op_done);
243
244		rh->xfer_entries = XFER_RING_ENTRIES;
245
246		regval = rh_reg_read(CR_SETUP);
247		rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval);
248		rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval);
249		dev_dbg(&hci->master.dev,
250			"xfer_struct_sz = %d, resp_struct_sz = %d",
251			rh->xfer_struct_sz, rh->resp_struct_sz);
252		xfers_sz = rh->xfer_struct_sz * rh->xfer_entries;
253		resps_sz = rh->resp_struct_sz * rh->xfer_entries;
254
255		rh->xfer = dma_alloc_coherent(rings->sysdev, xfers_sz,
256					      &rh->xfer_dma, GFP_KERNEL);
257		rh->resp = dma_alloc_coherent(rings->sysdev, resps_sz,
258					      &rh->resp_dma, GFP_KERNEL);
259		rh->src_xfers =
260			kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers),
261				      GFP_KERNEL);
262		ret = -ENOMEM;
263		if (!rh->xfer || !rh->resp || !rh->src_xfers)
264			goto err_out;
265
266		rh_reg_write(CMD_RING_BASE_LO, lower_32_bits(rh->xfer_dma));
267		rh_reg_write(CMD_RING_BASE_HI, upper_32_bits(rh->xfer_dma));
268		rh_reg_write(RESP_RING_BASE_LO, lower_32_bits(rh->resp_dma));
269		rh_reg_write(RESP_RING_BASE_HI, upper_32_bits(rh->resp_dma));
270
271		regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries);
272		rh_reg_write(CR_SETUP, regval);
273
274		rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff);
275		rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY |
276						 INTR_TRANSFER_COMPLETION |
277						 INTR_RING_OP |
278						 INTR_TRANSFER_ERR |
279						 INTR_IBI_RING_FULL |
280						 INTR_TRANSFER_ABORT);
281
282		/* IBIs */
283
284		if (i >= IBI_RINGS)
285			goto ring_ready;
286
287		regval = rh_reg_read(IBI_SETUP);
288		rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval);
289		rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES;
290		rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE;
291
292		rh->ibi_chunk_sz = dma_get_cache_alignment();
293		rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES;
294		/*
295		 * Round IBI data chunk size to number of bytes supported by
296		 * the HW. Chunk size can be 2^n number of DWORDs which is the
297		 * same as 2^(n+2) bytes, where n is 0..6.
298		 */
299		rh->ibi_chunk_sz = umax(4, rh->ibi_chunk_sz);
300		rh->ibi_chunk_sz = roundup_pow_of_two(rh->ibi_chunk_sz);
301		if (rh->ibi_chunk_sz > 256) {
302			ret = -EINVAL;
303			goto err_out;
304		}
305
306		ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries;
307		ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total;
308
309		rh->ibi_status =
310			dma_alloc_coherent(rings->sysdev, ibi_status_ring_sz,
311					   &rh->ibi_status_dma, GFP_KERNEL);
312		rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL);
313		ret = -ENOMEM;
314		if (!rh->ibi_status || !rh->ibi_data)
315			goto err_out;
316		rh->ibi_data_dma =
317			dma_map_single(rings->sysdev, rh->ibi_data,
318				       ibi_data_ring_sz, DMA_FROM_DEVICE);
319		if (dma_mapping_error(rings->sysdev, rh->ibi_data_dma)) {
320			rh->ibi_data_dma = 0;
321			ret = -ENOMEM;
322			goto err_out;
323		}
324
325		rh_reg_write(IBI_STATUS_RING_BASE_LO, lower_32_bits(rh->ibi_status_dma));
326		rh_reg_write(IBI_STATUS_RING_BASE_HI, upper_32_bits(rh->ibi_status_dma));
327		rh_reg_write(IBI_DATA_RING_BASE_LO, lower_32_bits(rh->ibi_data_dma));
328		rh_reg_write(IBI_DATA_RING_BASE_HI, upper_32_bits(rh->ibi_data_dma));
329
330		regval = FIELD_PREP(IBI_STATUS_RING_SIZE,
331				    rh->ibi_status_entries) |
332			 FIELD_PREP(IBI_DATA_CHUNK_SIZE,
333				    ilog2(rh->ibi_chunk_sz) - 2) |
334			 FIELD_PREP(IBI_DATA_CHUNK_COUNT,
335				    rh->ibi_chunks_total);
336		rh_reg_write(IBI_SETUP, regval);
337
338		regval = rh_reg_read(INTR_SIGNAL_ENABLE);
339		regval |= INTR_IBI_READY;
340		rh_reg_write(INTR_SIGNAL_ENABLE, regval);
341
342ring_ready:
343		/*
344		 * The MIPI I3C HCI specification does not document reset values for
345		 * RING_OPERATION1 fields and some controllers (e.g. Intel controllers)
346		 * do not reset the values, so ensure the ring pointers are set to zero
347		 * here.
348		 */
349		rh_reg_write(RING_OPERATION1, 0);
350
351		rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE |
352					   RING_CTRL_RUN_STOP);
353	}
354
355	return 0;
356
357err_out:
358	hci_dma_cleanup(hci);
359	return ret;
360}
361
362static void hci_dma_unmap_xfer(struct i3c_hci *hci,
363			       struct hci_xfer *xfer_list, unsigned int n)
364{
365	struct hci_xfer *xfer;
366	unsigned int i;
367
368	for (i = 0; i < n; i++) {
369		xfer = xfer_list + i;
370		if (!xfer->data)
371			continue;
372		i3c_master_dma_unmap_single(xfer->dma);
373	}
374}
375
376static struct i3c_dma *hci_dma_map_xfer(struct device *dev, struct hci_xfer *xfer)
377{
378	enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
379	bool need_bounce = device_iommu_mapped(dev) && xfer->rnw && (xfer->data_len & 3);
380
381	return i3c_master_dma_map_single(dev, xfer->data, xfer->data_len, need_bounce, dir);
382}
383
384static int hci_dma_map_xfer_list(struct i3c_hci *hci, struct device *dev,
385				 struct hci_xfer *xfer_list, int n)
386{
387	for (int i = 0; i < n; i++) {
388		struct hci_xfer *xfer = xfer_list + i;
389
390		if (!xfer->data)
391			continue;
392
393		xfer->dma = hci_dma_map_xfer(dev, xfer);
394		if (!xfer->dma) {
395			hci_dma_unmap_xfer(hci, xfer_list, i);
396			return -ENOMEM;
397		}
398	}
399
400	return 0;
401}
402
403static int hci_dma_queue_xfer(struct i3c_hci *hci,
404			      struct hci_xfer *xfer_list, int n)
405{
406	struct hci_rings_data *rings = hci->io_data;
407	struct hci_rh_data *rh;
408	unsigned int i, ring, enqueue_ptr;
409	u32 op1_val, op2_val;
410	int ret;
411
412	ret = hci_dma_map_xfer_list(hci, rings->sysdev, xfer_list, n);
413	if (ret)
414		return ret;
415
416	/* For now we only use ring 0 */
417	ring = 0;
418	rh = &rings->headers[ring];
419
420	op1_val = rh_reg_read(RING_OPERATION1);
421	enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val);
422	for (i = 0; i < n; i++) {
423		struct hci_xfer *xfer = xfer_list + i;
424		u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr;
425
426		/* store cmd descriptor */
427		*ring_data++ = xfer->cmd_desc[0];
428		*ring_data++ = xfer->cmd_desc[1];
429		if (hci->cmd == &mipi_i3c_hci_cmd_v2) {
430			*ring_data++ = xfer->cmd_desc[2];
431			*ring_data++ = xfer->cmd_desc[3];
432		}
433
434		/* first word of Data Buffer Descriptor Structure */
435		if (!xfer->data)
436			xfer->data_len = 0;
437		*ring_data++ =
438			FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) |
439			((i == n - 1) ? DATA_BUF_IOC : 0);
440
441		/* 2nd and 3rd words of Data Buffer Descriptor Structure */
442		if (xfer->data) {
443			*ring_data++ = lower_32_bits(xfer->dma->addr);
444			*ring_data++ = upper_32_bits(xfer->dma->addr);
445		} else {
446			*ring_data++ = 0;
447			*ring_data++ = 0;
448		}
449
450		/* remember corresponding xfer struct */
451		rh->src_xfers[enqueue_ptr] = xfer;
452		/* remember corresponding ring/entry for this xfer structure */
453		xfer->ring_number = ring;
454		xfer->ring_entry = enqueue_ptr;
455
456		enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries;
457
458		/*
459		 * We may update the hardware view of the enqueue pointer
460		 * only if we didn't reach its dequeue pointer.
461		 */
462		op2_val = rh_reg_read(RING_OPERATION2);
463		if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) {
464			/* the ring is full */
465			hci_dma_unmap_xfer(hci, xfer_list, n);
466			return -EBUSY;
467		}
468	}
469
470	/* take care to update the hardware enqueue pointer atomically */
471	spin_lock_irq(&hci->lock);
472	op1_val = rh_reg_read(RING_OPERATION1);
473	op1_val &= ~RING_OP1_CR_ENQ_PTR;
474	op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr);
475	rh_reg_write(RING_OPERATION1, op1_val);
476	spin_unlock_irq(&hci->lock);
477
478	return 0;
479}
480
481static bool hci_dma_dequeue_xfer(struct i3c_hci *hci,
482				 struct hci_xfer *xfer_list, int n)
483{
484	struct hci_rings_data *rings = hci->io_data;
485	struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number];
486	unsigned int i;
487	bool did_unqueue = false;
488	u32 ring_status;
489
490	guard(mutex)(&hci->control_mutex);
491
492	ring_status = rh_reg_read(RING_STATUS);
493	if (ring_status & RING_STATUS_RUNNING) {
494		/* stop the ring */
495		reinit_completion(&rh->op_done);
496		rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_ABORT);
497		wait_for_completion_timeout(&rh->op_done, HZ);
498		ring_status = rh_reg_read(RING_STATUS);
499		if (ring_status & RING_STATUS_RUNNING) {
500			/*
501			 * We're deep in it if ever this condition is ever met.
502			 * Hardware might still be writing to memory, etc.
503			 */
504			dev_crit(&hci->master.dev, "unable to abort the ring\n");
505			WARN_ON(1);
506		}
507	}
508
509	for (i = 0; i < n; i++) {
510		struct hci_xfer *xfer = xfer_list + i;
511		int idx = xfer->ring_entry;
512
513		/*
514		 * At the time the abort happened, the xfer might have
515		 * completed already. If not then replace corresponding
516		 * descriptor entries with a no-op.
517		 */
518		if (idx >= 0) {
519			u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx;
520
521			/* store no-op cmd descriptor */
522			*ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7) | FIELD_PREP(CMD_0_TID, xfer->cmd_tid);
523			*ring_data++ = 0;
524			if (hci->cmd == &mipi_i3c_hci_cmd_v2) {
525				*ring_data++ = 0;
526				*ring_data++ = 0;
527			}
528
529			/* disassociate this xfer struct */
530			rh->src_xfers[idx] = NULL;
531
532			/* and unmap it */
533			hci_dma_unmap_xfer(hci, xfer, 1);
534
535			did_unqueue = true;
536		}
537	}
538
539	/* restart the ring */
540	mipi_i3c_hci_resume(hci);
541	rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE);
542	rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP);
543
544	return did_unqueue;
545}
546
547static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh)
548{
549	u32 op1_val, op2_val, resp, *ring_resp;
550	unsigned int tid, done_ptr = rh->done_ptr;
551	struct hci_xfer *xfer;
552
553	for (;;) {
554		op2_val = rh_reg_read(RING_OPERATION2);
555		if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val))
556			break;
557
558		ring_resp = rh->resp + rh->resp_struct_sz * done_ptr;
559		resp = *ring_resp;
560		tid = RESP_TID(resp);
561		dev_dbg(&hci->master.dev, "resp = 0x%08x", resp);
562
563		xfer = rh->src_xfers[done_ptr];
564		if (!xfer) {
565			dev_dbg(&hci->master.dev, "orphaned ring entry");
566		} else {
567			hci_dma_unmap_xfer(hci, xfer, 1);
568			xfer->ring_entry = -1;
569			xfer->response = resp;
570			if (tid != xfer->cmd_tid) {
571				dev_err(&hci->master.dev,
572					"response tid=%d when expecting %d\n",
573					tid, xfer->cmd_tid);
574				/* TODO: do something about it? */
575			}
576			if (xfer->completion)
577				complete(xfer->completion);
578		}
579
580		done_ptr = (done_ptr + 1) % rh->xfer_entries;
581		rh->done_ptr = done_ptr;
582	}
583
584	/* take care to update the software dequeue pointer atomically */
585	spin_lock(&hci->lock);
586	op1_val = rh_reg_read(RING_OPERATION1);
587	op1_val &= ~RING_OP1_CR_SW_DEQ_PTR;
588	op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr);
589	rh_reg_write(RING_OPERATION1, op1_val);
590	spin_unlock(&hci->lock);
591}
592
593static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev,
594			       const struct i3c_ibi_setup *req)
595{
596	struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
597	struct i3c_generic_ibi_pool *pool;
598	struct hci_dma_dev_ibi_data *dev_ibi;
599
600	dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL);
601	if (!dev_ibi)
602		return -ENOMEM;
603	pool = i3c_generic_ibi_alloc_pool(dev, req);
604	if (IS_ERR(pool)) {
605		kfree(dev_ibi);
606		return PTR_ERR(pool);
607	}
608	dev_ibi->pool = pool;
609	dev_ibi->max_len = req->max_payload_len;
610	dev_data->ibi_data = dev_ibi;
611	return 0;
612}
613
614static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev)
615{
616	struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
617	struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;
618
619	dev_data->ibi_data = NULL;
620	i3c_generic_ibi_free_pool(dev_ibi->pool);
621	kfree(dev_ibi);
622}
623
624static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci,
625				     struct i3c_dev_desc *dev,
626				     struct i3c_ibi_slot *slot)
627{
628	struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
629	struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;
630
631	i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot);
632}
633
634static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)
635{
636	struct hci_rings_data *rings = hci->io_data;
637	struct i3c_dev_desc *dev;
638	struct i3c_hci_dev_data *dev_data;
639	struct hci_dma_dev_ibi_data *dev_ibi;
640	struct i3c_ibi_slot *slot;
641	u32 op1_val, op2_val, ibi_status_error;
642	unsigned int ptr, enq_ptr, deq_ptr;
643	unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part;
644	int ibi_addr, last_ptr;
645	void *ring_ibi_data;
646	dma_addr_t ring_ibi_data_dma;
647
648	op1_val = rh_reg_read(RING_OPERATION1);
649	deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val);
650
651	op2_val = rh_reg_read(RING_OPERATION2);
652	enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val);
653
654	ibi_status_error = 0;
655	ibi_addr = -1;
656	ibi_chunks = 0;
657	ibi_size = 0;
658	last_ptr = -1;
659
660	/* let's find all we can about this IBI */
661	for (ptr = deq_ptr; ptr != enq_ptr;
662	     ptr = (ptr + 1) % rh->ibi_status_entries) {
663		u32 ibi_status, *ring_ibi_status;
664		unsigned int chunks;
665
666		ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr;
667		ibi_status = *ring_ibi_status;
668		dev_dbg(&hci->master.dev, "status = %#x", ibi_status);
669
670		if (ibi_status_error) {
671			/* we no longer care */
672		} else if (ibi_status & IBI_ERROR) {
673			ibi_status_error = ibi_status;
674		} else if (ibi_addr ==  -1) {
675			ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);
676		} else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) {
677			/* the address changed unexpectedly */
678			ibi_status_error = ibi_status;
679		}
680
681		chunks = FIELD_GET(IBI_CHUNKS, ibi_status);
682		ibi_chunks += chunks;
683		if (!(ibi_status & IBI_LAST_STATUS)) {
684			ibi_size += chunks * rh->ibi_chunk_sz;
685		} else {
686			ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status);
687			last_ptr = ptr;
688			break;
689		}
690	}
691
692	/* validate what we've got */
693
694	if (last_ptr == -1) {
695		/* this IBI sequence is not yet complete */
696		dev_dbg(&hci->master.dev,
697			"no LAST_STATUS available (e=%d d=%d)",
698			enq_ptr, deq_ptr);
699		return;
700	}
701	deq_ptr = last_ptr + 1;
702	deq_ptr %= rh->ibi_status_entries;
703
704	if (ibi_status_error) {
705		dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr);
706		goto done;
707	}
708
709	/* determine who this is for */
710	dev = i3c_hci_addr_to_dev(hci, ibi_addr);
711	if (!dev) {
712		dev_err(&hci->master.dev,
713			"IBI for unknown device %#x\n", ibi_addr);
714		goto done;
715	}
716
717	dev_data = i3c_dev_get_master_data(dev);
718	dev_ibi = dev_data->ibi_data;
719	if (ibi_size > dev_ibi->max_len) {
720		dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n",
721			ibi_size, dev_ibi->max_len);
722		goto done;
723	}
724
725	/*
726	 * This ring model is not suitable for zero-copy processing of IBIs.
727	 * We have the data chunk ring wrap-around to deal with, meaning
728	 * that the payload might span multiple chunks beginning at the
729	 * end of the ring and wrap to the start of the ring. Furthermore
730	 * there is no guarantee that those chunks will be released in order
731	 * and in a timely manner by the upper driver. So let's just copy
732	 * them to a discrete buffer. In practice they're supposed to be
733	 * small anyway.
734	 */
735	slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool);
736	if (!slot) {
737		dev_err(&hci->master.dev, "no free slot for IBI\n");
738		goto done;
739	}
740
741	/* copy first part of the payload */
742	ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr;
743	ring_ibi_data = rh->ibi_data + ibi_data_offset;
744	ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset;
745	first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr)
746			* rh->ibi_chunk_sz;
747	if (first_part > ibi_size)
748		first_part = ibi_size;
749	dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma,
750				first_part, DMA_FROM_DEVICE);
751	memcpy(slot->data, ring_ibi_data, first_part);
752
753	/* copy second part if any */
754	if (ibi_size > first_part) {
755		/* we wrap back to the start and copy remaining data */
756		ring_ibi_data = rh->ibi_data;
757		ring_ibi_data_dma = rh->ibi_data_dma;
758		dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma,
759					ibi_size - first_part, DMA_FROM_DEVICE);
760		memcpy(slot->data + first_part, ring_ibi_data,
761		       ibi_size - first_part);
762	}
763
764	/* submit it */
765	slot->dev = dev;
766	slot->len = ibi_size;
767	i3c_master_queue_ibi(dev, slot);
768
769done:
770	/* take care to update the ibi dequeue pointer atomically */
771	spin_lock(&hci->lock);
772	op1_val = rh_reg_read(RING_OPERATION1);
773	op1_val &= ~RING_OP1_IBI_DEQ_PTR;
774	op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr);
775	rh_reg_write(RING_OPERATION1, op1_val);
776	spin_unlock(&hci->lock);
777
778	/* update the chunk pointer */
779	rh->ibi_chunk_ptr += ibi_chunks;
780	rh->ibi_chunk_ptr %= rh->ibi_chunks_total;
781
782	/* and tell the hardware about freed chunks */
783	rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks);
784}
785
786static bool hci_dma_irq_handler(struct i3c_hci *hci)
787{
788	struct hci_rings_data *rings = hci->io_data;
789	unsigned int i;
790	bool handled = false;
791
792	for (i = 0; i < rings->total; i++) {
793		struct hci_rh_data *rh;
794		u32 status;
795
796		rh = &rings->headers[i];
797		status = rh_reg_read(INTR_STATUS);
798		dev_dbg(&hci->master.dev, "Ring %d: RH_INTR_STATUS %#x",
799			i, status);
800		if (!status)
801			continue;
802		rh_reg_write(INTR_STATUS, status);
803
804		if (status & INTR_IBI_READY)
805			hci_dma_process_ibi(hci, rh);
806		if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR))
807			hci_dma_xfer_done(hci, rh);
808		if (status & INTR_RING_OP)
809			complete(&rh->op_done);
810
811		if (status & INTR_TRANSFER_ABORT) {
812			u32 ring_status;
813
814			dev_notice_ratelimited(&hci->master.dev,
815				"Ring %d: Transfer Aborted\n", i);
816			mipi_i3c_hci_resume(hci);
817			ring_status = rh_reg_read(RING_STATUS);
818			if (!(ring_status & RING_STATUS_RUNNING) &&
819			    status & INTR_TRANSFER_COMPLETION &&
820			    status & INTR_TRANSFER_ERR) {
821				/*
822				 * Ring stop followed by run is an Intel
823				 * specific required quirk after resuming the
824				 * halted controller. Do it only when the ring
825				 * is not in running state after a transfer
826				 * error.
827				 */
828				rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE);
829				rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE |
830							   RING_CTRL_RUN_STOP);
831			}
832		}
833		if (status & INTR_IBI_RING_FULL)
834			dev_err_ratelimited(&hci->master.dev,
835				"Ring %d: IBI Ring Full Condition\n", i);
836
837		handled = true;
838	}
839
840	return handled;
841}
842
843const struct hci_io_ops mipi_i3c_hci_dma = {
844	.init			= hci_dma_init,
845	.cleanup		= hci_dma_cleanup,
846	.queue_xfer		= hci_dma_queue_xfer,
847	.dequeue_xfer		= hci_dma_dequeue_xfer,
848	.irq_handler		= hci_dma_irq_handler,
849	.request_ibi		= hci_dma_request_ibi,
850	.free_ibi		= hci_dma_free_ibi,
851	.recycle_ibi_slot	= hci_dma_recycle_ibi_slot,
852};