hidma.c

  1/*
  2 * Qualcomm Technologies HIDMA DMA engine interface
  3 *
  4 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
  5 *
  6 * This program is free software; you can redistribute it and/or modify
  7 * it under the terms of the GNU General Public License version 2 and
  8 * only version 2 as published by the Free Software Foundation.
  9 *
 10 * This program is distributed in the hope that it will be useful,
 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 * GNU General Public License for more details.
 14 */
 15
 16/*
 17 * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.
 18 * Copyright (C) Semihalf 2009
 19 * Copyright (C) Ilya Yanok, Emcraft Systems 2010
 20 * Copyright (C) Alexander Popov, Promcontroller 2014
 21 *
 22 * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
 23 * (defines, structures and comments) was taken from MPC5121 DMA driver
 24 * written by Hongjun Chen <hong-jun.chen@freescale.com>.
 25 *
 26 * Approved as OSADL project by a majority of OSADL members and funded
 27 * by OSADL membership fees in 2009;  for details see www.osadl.org.
 28 *
 29 * This program is free software; you can redistribute it and/or modify it
 30 * under the terms of the GNU General Public License as published by the Free
 31 * Software Foundation; either version 2 of the License, or (at your option)
 32 * any later version.
 33 *
 34 * This program is distributed in the hope that it will be useful, but WITHOUT
 35 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 36 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 37 * more details.
 38 *
 39 * The full GNU General Public License is included in this distribution in the
 40 * file called COPYING.
 41 */
 42
 43/* Linux Foundation elects GPLv2 license only. */
 44
 45#include <linux/dmaengine.h>
 46#include <linux/dma-mapping.h>
 47#include <linux/list.h>
 48#include <linux/mod_devicetable.h>
 49#include <linux/module.h>
 50#include <linux/platform_device.h>
 51#include <linux/slab.h>
 52#include <linux/spinlock.h>
 53#include <linux/property.h>
 54#include <linux/delay.h>
 55#include <linux/acpi.h>
 56#include <linux/irq.h>
 57#include <linux/atomic.h>
 58#include <linux/pm_runtime.h>
 59#include <linux/msi.h>
 60
 61#include "../dmaengine.h"
 62#include "hidma.h"
 63
 64/*
 65 * Default idle time is 2 seconds. This parameter can
 66 * be overridden by changing the following
 67 * /sys/bus/platform/devices/QCOM8061:<xy>/power/autosuspend_delay_ms
 68 * during kernel boot.
 69 */
 70#define HIDMA_AUTOSUSPEND_TIMEOUT		2000
 71#define HIDMA_ERR_INFO_SW			0xFF
 72#define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE	0x0
 73#define HIDMA_NR_DEFAULT_DESC			10
 74#define HIDMA_MSI_INTS				11
 75
 76static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev)
 77{
 78	return container_of(dmadev, struct hidma_dev, ddev);
 79}
 80
 81static inline
 82struct hidma_dev *to_hidma_dev_from_lldev(struct hidma_lldev **_lldevp)
 83{
 84	return container_of(_lldevp, struct hidma_dev, lldev);
 85}
 86
 87static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach)
 88{
 89	return container_of(dmach, struct hidma_chan, chan);
 90}
 91
 92static void hidma_free(struct hidma_dev *dmadev)
 93{
 94	INIT_LIST_HEAD(&dmadev->ddev.channels);
 95}
 96
 97static unsigned int nr_desc_prm;
 98module_param(nr_desc_prm, uint, 0644);
 99MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)");
100
101enum hidma_cap {
102	HIDMA_MSI_CAP = 1,
103	HIDMA_IDENTITY_CAP,
104};
105
106/* process completed descriptors */
107static void hidma_process_completed(struct hidma_chan *mchan)
108{
109	struct dma_device *ddev = mchan->chan.device;
110	struct hidma_dev *mdma = to_hidma_dev(ddev);
111	struct dma_async_tx_descriptor *desc;
112	dma_cookie_t last_cookie;
113	struct hidma_desc *mdesc;
114	struct hidma_desc *next;
115	unsigned long irqflags;
116	struct list_head list;
117
118	INIT_LIST_HEAD(&list);
119
120	/* Get all completed descriptors */
121	spin_lock_irqsave(&mchan->lock, irqflags);
122	list_splice_tail_init(&mchan->completed, &list);
123	spin_unlock_irqrestore(&mchan->lock, irqflags);
124
125	/* Execute callbacks and run dependencies */
126	list_for_each_entry_safe(mdesc, next, &list, node) {
127		enum dma_status llstat;
128		struct dmaengine_desc_callback cb;
129		struct dmaengine_result result;
130
131		desc = &mdesc->desc;
132		last_cookie = desc->cookie;
133
134		llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch);
135
136		spin_lock_irqsave(&mchan->lock, irqflags);
137		if (llstat == DMA_COMPLETE) {
138			mchan->last_success = last_cookie;
139			result.result = DMA_TRANS_NOERROR;
140		} else {
141			result.result = DMA_TRANS_ABORTED;
142		}
143
144		dma_cookie_complete(desc);
145		spin_unlock_irqrestore(&mchan->lock, irqflags);
146
147		dmaengine_desc_get_callback(desc, &cb);
148
149		dma_run_dependencies(desc);
150
151		spin_lock_irqsave(&mchan->lock, irqflags);
152		list_move(&mdesc->node, &mchan->free);
153		spin_unlock_irqrestore(&mchan->lock, irqflags);
154
155		dmaengine_desc_callback_invoke(&cb, &result);
156	}
157}
158
159/*
160 * Called once for each submitted descriptor.
161 * PM is locked once for each descriptor that is currently
162 * in execution.
163 */
164static void hidma_callback(void *data)
165{
166	struct hidma_desc *mdesc = data;
167	struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan);
168	struct dma_device *ddev = mchan->chan.device;
169	struct hidma_dev *dmadev = to_hidma_dev(ddev);
170	unsigned long irqflags;
171	bool queued = false;
172
173	spin_lock_irqsave(&mchan->lock, irqflags);
174	if (mdesc->node.next) {
175		/* Delete from the active list, add to completed list */
176		list_move_tail(&mdesc->node, &mchan->completed);
177		queued = true;
178
179		/* calculate the next running descriptor */
180		mchan->running = list_first_entry(&mchan->active,
181						  struct hidma_desc, node);
182	}
183	spin_unlock_irqrestore(&mchan->lock, irqflags);
184
185	hidma_process_completed(mchan);
186
187	if (queued) {
188		pm_runtime_mark_last_busy(dmadev->ddev.dev);
189		pm_runtime_put_autosuspend(dmadev->ddev.dev);
190	}
191}
192
193static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig)
194{
195	struct hidma_chan *mchan;
196	struct dma_device *ddev;
197
198	mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL);
199	if (!mchan)
200		return -ENOMEM;
201
202	ddev = &dmadev->ddev;
203	mchan->dma_sig = dma_sig;
204	mchan->dmadev = dmadev;
205	mchan->chan.device = ddev;
206	dma_cookie_init(&mchan->chan);
207
208	INIT_LIST_HEAD(&mchan->free);
209	INIT_LIST_HEAD(&mchan->prepared);
210	INIT_LIST_HEAD(&mchan->active);
211	INIT_LIST_HEAD(&mchan->completed);
212	INIT_LIST_HEAD(&mchan->queued);
213
214	spin_lock_init(&mchan->lock);
215	list_add_tail(&mchan->chan.device_node, &ddev->channels);
216	return 0;
217}
218
219static void hidma_issue_task(struct tasklet_struct *t)
220{
221	struct hidma_dev *dmadev = from_tasklet(dmadev, t, task);
222
223	pm_runtime_get_sync(dmadev->ddev.dev);
224	hidma_ll_start(dmadev->lldev);
225}
226
227static void hidma_issue_pending(struct dma_chan *dmach)
228{
229	struct hidma_chan *mchan = to_hidma_chan(dmach);
230	struct hidma_dev *dmadev = mchan->dmadev;
231	unsigned long flags;
232	struct hidma_desc *qdesc, *next;
233	int status;
234
235	spin_lock_irqsave(&mchan->lock, flags);
236	list_for_each_entry_safe(qdesc, next, &mchan->queued, node) {
237		hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch);
238		list_move_tail(&qdesc->node, &mchan->active);
239	}
240
241	if (!mchan->running) {
242		struct hidma_desc *desc = list_first_entry(&mchan->active,
243							   struct hidma_desc,
244							   node);
245		mchan->running = desc;
246	}
247	spin_unlock_irqrestore(&mchan->lock, flags);
248
249	/* PM will be released in hidma_callback function. */
250	status = pm_runtime_get(dmadev->ddev.dev);
251	if (status < 0)
252		tasklet_schedule(&dmadev->task);
253	else
254		hidma_ll_start(dmadev->lldev);
255}
256
257static inline bool hidma_txn_is_success(dma_cookie_t cookie,
258		dma_cookie_t last_success, dma_cookie_t last_used)
259{
260	if (last_success <= last_used) {
261		if ((cookie <= last_success) || (cookie > last_used))
262			return true;
263	} else {
264		if ((cookie <= last_success) && (cookie > last_used))
265			return true;
266	}
267	return false;
268}
269
270static enum dma_status hidma_tx_status(struct dma_chan *dmach,
271				       dma_cookie_t cookie,
272				       struct dma_tx_state *txstate)
273{
274	struct hidma_chan *mchan = to_hidma_chan(dmach);
275	enum dma_status ret;
276
277	ret = dma_cookie_status(dmach, cookie, txstate);
278	if (ret == DMA_COMPLETE) {
279		bool is_success;
280
281		is_success = hidma_txn_is_success(cookie, mchan->last_success,
282						  dmach->cookie);
283		return is_success ? ret : DMA_ERROR;
284	}
285
286	if (mchan->paused && (ret == DMA_IN_PROGRESS)) {
287		unsigned long flags;
288		dma_cookie_t runcookie;
289
290		spin_lock_irqsave(&mchan->lock, flags);
291		if (mchan->running)
292			runcookie = mchan->running->desc.cookie;
293		else
294			runcookie = -EINVAL;
295
296		if (runcookie == cookie)
297			ret = DMA_PAUSED;
298
299		spin_unlock_irqrestore(&mchan->lock, flags);
300	}
301
302	return ret;
303}
304
305/*
306 * Submit descriptor to hardware.
307 * Lock the PM for each descriptor we are sending.
308 */
309static dma_cookie_t hidma_tx_submit(struct dma_async_tx_descriptor *txd)
310{
311	struct hidma_chan *mchan = to_hidma_chan(txd->chan);
312	struct hidma_dev *dmadev = mchan->dmadev;
313	struct hidma_desc *mdesc;
314	unsigned long irqflags;
315	dma_cookie_t cookie;
316
317	pm_runtime_get_sync(dmadev->ddev.dev);
318	if (!hidma_ll_isenabled(dmadev->lldev)) {
319		pm_runtime_mark_last_busy(dmadev->ddev.dev);
320		pm_runtime_put_autosuspend(dmadev->ddev.dev);
321		return -ENODEV;
322	}
323	pm_runtime_mark_last_busy(dmadev->ddev.dev);
324	pm_runtime_put_autosuspend(dmadev->ddev.dev);
325
326	mdesc = container_of(txd, struct hidma_desc, desc);
327	spin_lock_irqsave(&mchan->lock, irqflags);
328
329	/* Move descriptor to queued */
330	list_move_tail(&mdesc->node, &mchan->queued);
331
332	/* Update cookie */
333	cookie = dma_cookie_assign(txd);
334
335	spin_unlock_irqrestore(&mchan->lock, irqflags);
336
337	return cookie;
338}
339
340static int hidma_alloc_chan_resources(struct dma_chan *dmach)
341{
342	struct hidma_chan *mchan = to_hidma_chan(dmach);
343	struct hidma_dev *dmadev = mchan->dmadev;
344	struct hidma_desc *mdesc, *tmp;
345	unsigned long irqflags;
346	LIST_HEAD(descs);
347	unsigned int i;
348	int rc = 0;
349
350	if (mchan->allocated)
351		return 0;
352
353	/* Alloc descriptors for this channel */
354	for (i = 0; i < dmadev->nr_descriptors; i++) {
355		mdesc = kzalloc(sizeof(struct hidma_desc), GFP_NOWAIT);
356		if (!mdesc) {
357			rc = -ENOMEM;
358			break;
359		}
360		dma_async_tx_descriptor_init(&mdesc->desc, dmach);
361		mdesc->desc.tx_submit = hidma_tx_submit;
362
363		rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig,
364				      "DMA engine", hidma_callback, mdesc,
365				      &mdesc->tre_ch);
366		if (rc) {
367			dev_err(dmach->device->dev,
368				"channel alloc failed at %u\n", i);
369			kfree(mdesc);
370			break;
371		}
372		list_add_tail(&mdesc->node, &descs);
373	}
374
375	if (rc) {
376		/* return the allocated descriptors */
377		list_for_each_entry_safe(mdesc, tmp, &descs, node) {
378			hidma_ll_free(dmadev->lldev, mdesc->tre_ch);
379			kfree(mdesc);
380		}
381		return rc;
382	}
383
384	spin_lock_irqsave(&mchan->lock, irqflags);
385	list_splice_tail_init(&descs, &mchan->free);
386	mchan->allocated = true;
387	spin_unlock_irqrestore(&mchan->lock, irqflags);
388	return 1;
389}
390
391static struct dma_async_tx_descriptor *
392hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src,
393		size_t len, unsigned long flags)
394{
395	struct hidma_chan *mchan = to_hidma_chan(dmach);
396	struct hidma_desc *mdesc = NULL;
397	struct hidma_dev *mdma = mchan->dmadev;
398	unsigned long irqflags;
399
400	/* Get free descriptor */
401	spin_lock_irqsave(&mchan->lock, irqflags);
402	if (!list_empty(&mchan->free)) {
403		mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);
404		list_del(&mdesc->node);
405	}
406	spin_unlock_irqrestore(&mchan->lock, irqflags);
407
408	if (!mdesc)
409		return NULL;
410
411	mdesc->desc.flags = flags;
412	hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
413				     src, dest, len, flags,
414				     HIDMA_TRE_MEMCPY);
415
416	/* Place descriptor in prepared list */
417	spin_lock_irqsave(&mchan->lock, irqflags);
418	list_add_tail(&mdesc->node, &mchan->prepared);
419	spin_unlock_irqrestore(&mchan->lock, irqflags);
420
421	return &mdesc->desc;
422}
423
424static struct dma_async_tx_descriptor *
425hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value,
426		size_t len, unsigned long flags)
427{
428	struct hidma_chan *mchan = to_hidma_chan(dmach);
429	struct hidma_desc *mdesc = NULL;
430	struct hidma_dev *mdma = mchan->dmadev;
431	unsigned long irqflags;
432	u64 byte_pattern, fill_pattern;
433
434	/* Get free descriptor */
435	spin_lock_irqsave(&mchan->lock, irqflags);
436	if (!list_empty(&mchan->free)) {
437		mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);
438		list_del(&mdesc->node);
439	}
440	spin_unlock_irqrestore(&mchan->lock, irqflags);
441
442	if (!mdesc)
443		return NULL;
444
445	byte_pattern = (char)value;
446	fill_pattern =	(byte_pattern << 56) |
447			(byte_pattern << 48) |
448			(byte_pattern << 40) |
449			(byte_pattern << 32) |
450			(byte_pattern << 24) |
451			(byte_pattern << 16) |
452			(byte_pattern << 8) |
453			byte_pattern;
454
455	mdesc->desc.flags = flags;
456	hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
457				     fill_pattern, dest, len, flags,
458				     HIDMA_TRE_MEMSET);
459
460	/* Place descriptor in prepared list */
461	spin_lock_irqsave(&mchan->lock, irqflags);
462	list_add_tail(&mdesc->node, &mchan->prepared);
463	spin_unlock_irqrestore(&mchan->lock, irqflags);
464
465	return &mdesc->desc;
466}
467
468static int hidma_terminate_channel(struct dma_chan *chan)
469{
470	struct hidma_chan *mchan = to_hidma_chan(chan);
471	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);
472	struct hidma_desc *tmp, *mdesc;
473	unsigned long irqflags;
474	LIST_HEAD(list);
475	int rc;
476
477	pm_runtime_get_sync(dmadev->ddev.dev);
478	/* give completed requests a chance to finish */
479	hidma_process_completed(mchan);
480
481	spin_lock_irqsave(&mchan->lock, irqflags);
482	mchan->last_success = 0;
483	list_splice_init(&mchan->active, &list);
484	list_splice_init(&mchan->prepared, &list);
485	list_splice_init(&mchan->completed, &list);
486	list_splice_init(&mchan->queued, &list);
487	spin_unlock_irqrestore(&mchan->lock, irqflags);
488
489	/* this suspends the existing transfer */
490	rc = hidma_ll_disable(dmadev->lldev);
491	if (rc) {
492		dev_err(dmadev->ddev.dev, "channel did not pause\n");
493		goto out;
494	}
495
496	/* return all user requests */
497	list_for_each_entry_safe(mdesc, tmp, &list, node) {
498		struct dma_async_tx_descriptor *txd = &mdesc->desc;
499
500		dma_descriptor_unmap(txd);
501		dmaengine_desc_get_callback_invoke(txd, NULL);
502		dma_run_dependencies(txd);
503
504		/* move myself to free_list */
505		list_move(&mdesc->node, &mchan->free);
506	}
507
508	rc = hidma_ll_enable(dmadev->lldev);
509out:
510	pm_runtime_mark_last_busy(dmadev->ddev.dev);
511	pm_runtime_put_autosuspend(dmadev->ddev.dev);
512	return rc;
513}
514
515static int hidma_terminate_all(struct dma_chan *chan)
516{
517	struct hidma_chan *mchan = to_hidma_chan(chan);
518	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);
519	int rc;
520
521	rc = hidma_terminate_channel(chan);
522	if (rc)
523		return rc;
524
525	/* reinitialize the hardware */
526	pm_runtime_get_sync(dmadev->ddev.dev);
527	rc = hidma_ll_setup(dmadev->lldev);
528	pm_runtime_mark_last_busy(dmadev->ddev.dev);
529	pm_runtime_put_autosuspend(dmadev->ddev.dev);
530	return rc;
531}
532
533static void hidma_free_chan_resources(struct dma_chan *dmach)
534{
535	struct hidma_chan *mchan = to_hidma_chan(dmach);
536	struct hidma_dev *mdma = mchan->dmadev;
537	struct hidma_desc *mdesc, *tmp;
538	unsigned long irqflags;
539	LIST_HEAD(descs);
540
541	/* terminate running transactions and free descriptors */
542	hidma_terminate_channel(dmach);
543
544	spin_lock_irqsave(&mchan->lock, irqflags);
545
546	/* Move data */
547	list_splice_tail_init(&mchan->free, &descs);
548
549	/* Free descriptors */
550	list_for_each_entry_safe(mdesc, tmp, &descs, node) {
551		hidma_ll_free(mdma->lldev, mdesc->tre_ch);
552		list_del(&mdesc->node);
553		kfree(mdesc);
554	}
555
556	mchan->allocated = false;
557	spin_unlock_irqrestore(&mchan->lock, irqflags);
558}
559
560static int hidma_pause(struct dma_chan *chan)
561{
562	struct hidma_chan *mchan;
563	struct hidma_dev *dmadev;
564
565	mchan = to_hidma_chan(chan);
566	dmadev = to_hidma_dev(mchan->chan.device);
567	if (!mchan->paused) {
568		pm_runtime_get_sync(dmadev->ddev.dev);
569		if (hidma_ll_disable(dmadev->lldev))
570			dev_warn(dmadev->ddev.dev, "channel did not stop\n");
571		mchan->paused = true;
572		pm_runtime_mark_last_busy(dmadev->ddev.dev);
573		pm_runtime_put_autosuspend(dmadev->ddev.dev);
574	}
575	return 0;
576}
577
578static int hidma_resume(struct dma_chan *chan)
579{
580	struct hidma_chan *mchan;
581	struct hidma_dev *dmadev;
582	int rc = 0;
583
584	mchan = to_hidma_chan(chan);
585	dmadev = to_hidma_dev(mchan->chan.device);
586	if (mchan->paused) {
587		pm_runtime_get_sync(dmadev->ddev.dev);
588		rc = hidma_ll_enable(dmadev->lldev);
589		if (!rc)
590			mchan->paused = false;
591		else
592			dev_err(dmadev->ddev.dev,
593				"failed to resume the channel");
594		pm_runtime_mark_last_busy(dmadev->ddev.dev);
595		pm_runtime_put_autosuspend(dmadev->ddev.dev);
596	}
597	return rc;
598}
599
600static irqreturn_t hidma_chirq_handler(int chirq, void *arg)
601{
602	struct hidma_lldev *lldev = arg;
603
604	/*
605	 * All interrupts are request driven.
606	 * HW doesn't send an interrupt by itself.
607	 */
608	return hidma_ll_inthandler(chirq, lldev);
609}
610
611#ifdef CONFIG_GENERIC_MSI_IRQ
612static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg)
613{
614	struct hidma_lldev **lldevp = arg;
615	struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp);
616
617	return hidma_ll_inthandler_msi(chirq, *lldevp,
618				       1 << (chirq - dmadev->msi_virqbase));
619}
620#endif
621
622static ssize_t hidma_show_values(struct device *dev,
623				 struct device_attribute *attr, char *buf)
624{
625	struct hidma_dev *mdev = dev_get_drvdata(dev);
626
627	buf[0] = 0;
628
629	if (strcmp(attr->attr.name, "chid") == 0)
630		sprintf(buf, "%d\n", mdev->chidx);
631
632	return strlen(buf);
633}
634
635static inline void  hidma_sysfs_uninit(struct hidma_dev *dev)
636{
637	device_remove_file(dev->ddev.dev, dev->chid_attrs);
638}
639
640static struct device_attribute*
641hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, int mode)
642{
643	struct device_attribute *attrs;
644	char *name_copy;
645
646	attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute),
647			     GFP_KERNEL);
648	if (!attrs)
649		return NULL;
650
651	name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL);
652	if (!name_copy)
653		return NULL;
654
655	attrs->attr.name = name_copy;
656	attrs->attr.mode = mode;
657	attrs->show = hidma_show_values;
658	sysfs_attr_init(&attrs->attr);
659
660	return attrs;
661}
662
663static int hidma_sysfs_init(struct hidma_dev *dev)
664{
665	dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO);
666	if (!dev->chid_attrs)
667		return -ENOMEM;
668
669	return device_create_file(dev->ddev.dev, dev->chid_attrs);
670}
671
672#ifdef CONFIG_GENERIC_MSI_IRQ
673static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
674{
675	struct device *dev = msi_desc_to_dev(desc);
676	struct hidma_dev *dmadev = dev_get_drvdata(dev);
677
678	if (!desc->msi_index) {
679		writel(msg->address_lo, dmadev->dev_evca + 0x118);
680		writel(msg->address_hi, dmadev->dev_evca + 0x11C);
681		writel(msg->data, dmadev->dev_evca + 0x120);
682	}
683}
684#endif
685
686static void hidma_free_msis(struct hidma_dev *dmadev)
687{
688#ifdef CONFIG_GENERIC_MSI_IRQ
689	struct device *dev = dmadev->ddev.dev;
690	int i, virq;
691
692	for (i = 0; i < HIDMA_MSI_INTS; i++) {
693		virq = msi_get_virq(dev, i);
694		if (virq)
695			devm_free_irq(dev, virq, &dmadev->lldev);
696	}
697
698	platform_device_msi_free_irqs_all(dev);
699#endif
700}
701
702static int hidma_request_msi(struct hidma_dev *dmadev,
703			     struct platform_device *pdev)
704{
705#ifdef CONFIG_GENERIC_MSI_IRQ
706	int rc, i, virq;
707
708	rc = platform_device_msi_init_and_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS,
709						     hidma_write_msi_msg);
710	if (rc)
711		return rc;
712
713	for (i = 0; i < HIDMA_MSI_INTS; i++) {
714		virq = msi_get_virq(&pdev->dev, i);
715		rc = devm_request_irq(&pdev->dev, virq,
716				       hidma_chirq_handler_msi,
717				       0, "qcom-hidma-msi",
718				       &dmadev->lldev);
719		if (rc)
720			break;
721		if (!i)
722			dmadev->msi_virqbase = virq;
723	}
724
725	if (rc) {
726		/* free allocated MSI interrupts above */
727		for (--i; i >= 0; i--) {
728			virq = msi_get_virq(&pdev->dev, i);
729			devm_free_irq(&pdev->dev, virq, &dmadev->lldev);
730		}
731		dev_warn(&pdev->dev,
732			 "failed to request MSI irq, falling back to wired IRQ\n");
733	} else {
734		/* Add callback to free MSIs on teardown */
735		hidma_ll_setup_irq(dmadev->lldev, true);
736	}
737	return rc;
738#else
739	return -EINVAL;
740#endif
741}
742
743static bool hidma_test_capability(struct device *dev, enum hidma_cap test_cap)
744{
745	enum hidma_cap cap;
746
747	cap = (uintptr_t) device_get_match_data(dev);
748	return cap ? ((cap & test_cap) > 0) : 0;
749}
750
751static int hidma_probe(struct platform_device *pdev)
752{
753	struct hidma_dev *dmadev;
754	struct resource *trca_resource;
755	struct resource *evca_resource;
756	int chirq;
757	void __iomem *evca;
758	void __iomem *trca;
759	int rc;
760	bool msi;
761
762	pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT);
763	pm_runtime_use_autosuspend(&pdev->dev);
764	pm_runtime_set_active(&pdev->dev);
765	pm_runtime_enable(&pdev->dev);
766
767	trca = devm_platform_get_and_ioremap_resource(pdev, 0, &trca_resource);
768	if (IS_ERR(trca)) {
769		rc = PTR_ERR(trca);
770		goto bailout;
771	}
772
773	evca = devm_platform_get_and_ioremap_resource(pdev, 1, &evca_resource);
774	if (IS_ERR(evca)) {
775		rc = PTR_ERR(evca);
776		goto bailout;
777	}
778
779	/*
780	 * This driver only handles the channel IRQs.
781	 * Common IRQ is handled by the management driver.
782	 */
783	chirq = platform_get_irq(pdev, 0);
784	if (chirq < 0) {
785		rc = chirq;
786		goto bailout;
787	}
788
789	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
790	if (!dmadev) {
791		rc = -ENOMEM;
792		goto bailout;
793	}
794
795	INIT_LIST_HEAD(&dmadev->ddev.channels);
796	spin_lock_init(&dmadev->lock);
797	dmadev->ddev.dev = &pdev->dev;
798	pm_runtime_get_sync(dmadev->ddev.dev);
799
800	dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask);
801	dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask);
802	if (WARN_ON(!pdev->dev.dma_mask)) {
803		rc = -ENXIO;
804		goto dmafree;
805	}
806
807	dmadev->dev_evca = evca;
808	dmadev->evca_resource = evca_resource;
809	dmadev->dev_trca = trca;
810	dmadev->trca_resource = trca_resource;
811	dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy;
812	dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset;
813	dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources;
814	dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources;
815	dmadev->ddev.device_tx_status = hidma_tx_status;
816	dmadev->ddev.device_issue_pending = hidma_issue_pending;
817	dmadev->ddev.device_pause = hidma_pause;
818	dmadev->ddev.device_resume = hidma_resume;
819	dmadev->ddev.device_terminate_all = hidma_terminate_all;
820	dmadev->ddev.copy_align = 8;
821
822	/*
823	 * Determine the MSI capability of the platform. Old HW doesn't
824	 * support MSI.
825	 */
826	msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP);
827	device_property_read_u32(&pdev->dev, "desc-count",
828				 &dmadev->nr_descriptors);
829
830	if (nr_desc_prm) {
831		dev_info(&pdev->dev, "overriding number of descriptors as %d\n",
832			 nr_desc_prm);
833		dmadev->nr_descriptors = nr_desc_prm;
834	}
835
836	if (!dmadev->nr_descriptors)
837		dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC;
838
839	if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP))
840		dmadev->chidx = readl(dmadev->dev_trca + 0x40);
841	else
842		dmadev->chidx = readl(dmadev->dev_trca + 0x28);
843
844	/* Set DMA mask to 64 bits. */
845	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
846	if (rc) {
847		dev_warn(&pdev->dev, "unable to set coherent mask to 64");
848		goto dmafree;
849	}
850
851	dmadev->lldev = hidma_ll_init(dmadev->ddev.dev,
852				      dmadev->nr_descriptors, dmadev->dev_trca,
853				      dmadev->dev_evca, dmadev->chidx);
854	if (!dmadev->lldev) {
855		rc = -EPROBE_DEFER;
856		goto dmafree;
857	}
858
859	platform_set_drvdata(pdev, dmadev);
860	if (msi)
861		rc = hidma_request_msi(dmadev, pdev);
862
863	if (!msi || rc) {
864		hidma_ll_setup_irq(dmadev->lldev, false);
865		rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler,
866				      0, "qcom-hidma", dmadev->lldev);
867		if (rc)
868			goto uninit;
869	}
870
871	INIT_LIST_HEAD(&dmadev->ddev.channels);
872	rc = hidma_chan_init(dmadev, 0);
873	if (rc)
874		goto uninit;
875
876	rc = dma_async_device_register(&dmadev->ddev);
877	if (rc)
878		goto uninit;
879
880	dmadev->irq = chirq;
881	tasklet_setup(&dmadev->task, hidma_issue_task);
882	hidma_debug_init(dmadev);
883	hidma_sysfs_init(dmadev);
884	dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n");
885	pm_runtime_mark_last_busy(dmadev->ddev.dev);
886	pm_runtime_put_autosuspend(dmadev->ddev.dev);
887	return 0;
888
889uninit:
890	if (msi)
891		hidma_free_msis(dmadev);
892
893	hidma_ll_uninit(dmadev->lldev);
894dmafree:
895	if (dmadev)
896		hidma_free(dmadev);
897bailout:
898	pm_runtime_put_sync(&pdev->dev);
899	pm_runtime_disable(&pdev->dev);
900	return rc;
901}
902
903static void hidma_shutdown(struct platform_device *pdev)
904{
905	struct hidma_dev *dmadev = platform_get_drvdata(pdev);
906
907	dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n");
908
909	pm_runtime_get_sync(dmadev->ddev.dev);
910	if (hidma_ll_disable(dmadev->lldev))
911		dev_warn(dmadev->ddev.dev, "channel did not stop\n");
912	pm_runtime_mark_last_busy(dmadev->ddev.dev);
913	pm_runtime_put_autosuspend(dmadev->ddev.dev);
914
915}
916
917static void hidma_remove(struct platform_device *pdev)
918{
919	struct hidma_dev *dmadev = platform_get_drvdata(pdev);
920
921	pm_runtime_get_sync(dmadev->ddev.dev);
922	dma_async_device_unregister(&dmadev->ddev);
923	if (!dmadev->lldev->msi_support)
924		devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev);
925	else
926		hidma_free_msis(dmadev);
927
928	tasklet_kill(&dmadev->task);
929	hidma_sysfs_uninit(dmadev);
930	hidma_debug_uninit(dmadev);
931	hidma_ll_uninit(dmadev->lldev);
932	hidma_free(dmadev);
933
934	dev_info(&pdev->dev, "HI-DMA engine removed\n");
935	pm_runtime_put_sync_suspend(&pdev->dev);
936	pm_runtime_disable(&pdev->dev);
937}
938
939#if IS_ENABLED(CONFIG_ACPI)
940static const struct acpi_device_id hidma_acpi_ids[] = {
941	{"QCOM8061"},
942	{"QCOM8062", HIDMA_MSI_CAP},
943	{"QCOM8063", (HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP)},
944	{},
945};
946MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids);
947#endif
948
949static struct platform_driver hidma_driver = {
950	.probe = hidma_probe,
951	.remove = hidma_remove,
952	.shutdown = hidma_shutdown,
953	.driver = {
954		   .name = "hidma",
955		   .acpi_match_table = ACPI_PTR(hidma_acpi_ids),
956	},
957};
958
959module_platform_driver(hidma_driver);
960MODULE_DESCRIPTION("Qualcomm Technologies HIDMA Channel support");
961MODULE_LICENSE("GPL v2");