qaic_drv.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2
  3/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
  4/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
  5
  6#include <linux/delay.h>
  7#include <linux/dma-mapping.h>
  8#include <linux/idr.h>
  9#include <linux/interrupt.h>
 10#include <linux/list.h>
 11#include <linux/kobject.h>
 12#include <linux/kref.h>
 13#include <linux/mhi.h>
 14#include <linux/module.h>
 15#include <linux/msi.h>
 16#include <linux/mutex.h>
 17#include <linux/pci.h>
 18#include <linux/spinlock.h>
 19#include <linux/workqueue.h>
 20#include <linux/wait.h>
 21#include <drm/drm_accel.h>
 22#include <drm/drm_drv.h>
 23#include <drm/drm_file.h>
 24#include <drm/drm_gem.h>
 25#include <drm/drm_ioctl.h>
 26#include <drm/drm_managed.h>
 27#include <uapi/drm/qaic_accel.h>
 28
 29#include "mhi_controller.h"
 30#include "qaic.h"
 31#include "qaic_debugfs.h"
 32#include "qaic_ras.h"
 33#include "qaic_ssr.h"
 34#include "qaic_timesync.h"
 35#include "sahara.h"
 36
 37MODULE_IMPORT_NS("DMA_BUF");
 38
 39#define PCI_DEVICE_ID_QCOM_AIC080	0xa080
 40#define PCI_DEVICE_ID_QCOM_AIC100	0xa100
 41#define PCI_DEVICE_ID_QCOM_AIC200	0xa110
 42#define QAIC_NAME			"qaic"
 43#define QAIC_DESC			"Qualcomm Cloud AI Accelerators"
 44#define CNTL_MAJOR			5
 45#define CNTL_MINOR			0
 46
 47struct qaic_device_config {
 48	/* Indicates the AIC family the device belongs to */
 49	int family;
 50	/* A bitmask representing the available BARs */
 51	int bar_mask;
 52	/* An index value used to identify the MHI controller BAR */
 53	unsigned int mhi_bar_idx;
 54	/* An index value used to identify the DBCs BAR */
 55	unsigned int dbc_bar_idx;
 56};
 57
 58static const struct qaic_device_config aic080_config = {
 59	.family = FAMILY_AIC100,
 60	.bar_mask = BIT(0) | BIT(2) | BIT(4),
 61	.mhi_bar_idx = 0,
 62	.dbc_bar_idx = 2,
 63};
 64
 65static const struct qaic_device_config aic100_config = {
 66	.family = FAMILY_AIC100,
 67	.bar_mask = BIT(0) | BIT(2) | BIT(4),
 68	.mhi_bar_idx = 0,
 69	.dbc_bar_idx = 2,
 70};
 71
 72static const struct qaic_device_config aic200_config = {
 73	.family = FAMILY_AIC200,
 74	.bar_mask = BIT(0) | BIT(1) | BIT(2) | BIT(4),
 75	.mhi_bar_idx = 1,
 76	.dbc_bar_idx = 2,
 77};
 78
 79bool datapath_polling;
 80module_param(datapath_polling, bool, 0400);
 81MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode");
 82static bool link_up;
 83static DEFINE_IDA(qaic_usrs);
 84
 85static void qaicm_wq_release(struct drm_device *dev, void *res)
 86{
 87	struct workqueue_struct *wq = res;
 88
 89	destroy_workqueue(wq);
 90}
 91
 92static struct workqueue_struct *qaicm_wq_init(struct drm_device *dev, const char *name)
 93{
 94	struct workqueue_struct *wq;
 95	int ret;
 96
 97	wq = alloc_workqueue("%s", WQ_UNBOUND, 0, name);
 98	if (!wq)
 99		return ERR_PTR(-ENOMEM);
100	ret = drmm_add_action_or_reset(dev, qaicm_wq_release, wq);
101	if (ret)
102		return ERR_PTR(ret);
103
104	return wq;
105}
106
107static void qaicm_srcu_release(struct drm_device *dev, void *res)
108{
109	struct srcu_struct *lock = res;
110
111	cleanup_srcu_struct(lock);
112}
113
114static int qaicm_srcu_init(struct drm_device *dev, struct srcu_struct *lock)
115{
116	int ret;
117
118	ret = init_srcu_struct(lock);
119	if (ret)
120		return ret;
121
122	return drmm_add_action_or_reset(dev, qaicm_srcu_release, lock);
123}
124
125static void qaicm_pci_release(struct drm_device *dev, void *res)
126{
127	struct qaic_device *qdev = to_qaic_device(dev);
128
129	pci_set_drvdata(qdev->pdev, NULL);
130}
131
132static void free_usr(struct kref *kref)
133{
134	struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count);
135
136	cleanup_srcu_struct(&usr->qddev_lock);
137	ida_free(&qaic_usrs, usr->handle);
138	kfree(usr);
139}
140
141static int qaic_open(struct drm_device *dev, struct drm_file *file)
142{
143	struct qaic_drm_device *qddev = to_qaic_drm_device(dev);
144	struct qaic_device *qdev = qddev->qdev;
145	struct qaic_user *usr;
146	int rcu_id;
147	int ret;
148
149	rcu_id = srcu_read_lock(&qdev->dev_lock);
150	if (qdev->dev_state != QAIC_ONLINE) {
151		ret = -ENODEV;
152		goto dev_unlock;
153	}
154
155	usr = kmalloc(sizeof(*usr), GFP_KERNEL);
156	if (!usr) {
157		ret = -ENOMEM;
158		goto dev_unlock;
159	}
160
161	usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL);
162	if (usr->handle < 0) {
163		ret = usr->handle;
164		goto free_usr;
165	}
166	usr->qddev = qddev;
167	atomic_set(&usr->chunk_id, 0);
168	init_srcu_struct(&usr->qddev_lock);
169	kref_init(&usr->ref_count);
170
171	ret = mutex_lock_interruptible(&qddev->users_mutex);
172	if (ret)
173		goto cleanup_usr;
174
175	list_add(&usr->node, &qddev->users);
176	mutex_unlock(&qddev->users_mutex);
177
178	file->driver_priv = usr;
179
180	srcu_read_unlock(&qdev->dev_lock, rcu_id);
181	return 0;
182
183cleanup_usr:
184	cleanup_srcu_struct(&usr->qddev_lock);
185	ida_free(&qaic_usrs, usr->handle);
186free_usr:
187	kfree(usr);
188dev_unlock:
189	srcu_read_unlock(&qdev->dev_lock, rcu_id);
190	return ret;
191}
192
193static void qaic_postclose(struct drm_device *dev, struct drm_file *file)
194{
195	struct qaic_user *usr = file->driver_priv;
196	struct qaic_drm_device *qddev;
197	struct qaic_device *qdev;
198	int qdev_rcu_id;
199	int usr_rcu_id;
200	int i;
201
202	qddev = usr->qddev;
203	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
204	if (qddev) {
205		qdev = qddev->qdev;
206		qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
207		if (qdev->dev_state == QAIC_ONLINE) {
208			qaic_release_usr(qdev, usr);
209			for (i = 0; i < qdev->num_dbc; ++i)
210				if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle)
211					release_dbc(qdev, i);
212		}
213		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
214
215		mutex_lock(&qddev->users_mutex);
216		if (!list_empty(&usr->node))
217			list_del_init(&usr->node);
218		mutex_unlock(&qddev->users_mutex);
219	}
220
221	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
222	kref_put(&usr->ref_count, free_usr);
223
224	file->driver_priv = NULL;
225}
226
227DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops);
228
229static const struct drm_ioctl_desc qaic_drm_ioctls[] = {
230	DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0),
231	DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0),
232	DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0),
233	DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0),
234	DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0),
235	DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0),
236	DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0),
237	DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0),
238	DRM_IOCTL_DEF_DRV(QAIC_DETACH_SLICE_BO, qaic_detach_slice_bo_ioctl, 0),
239};
240
241static const struct drm_driver qaic_accel_driver = {
242	.driver_features	= DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
243
244	.name			= QAIC_NAME,
245	.desc			= QAIC_DESC,
246
247	.fops			= &qaic_accel_fops,
248	.open			= qaic_open,
249	.postclose		= qaic_postclose,
250
251	.ioctls			= qaic_drm_ioctls,
252	.num_ioctls		= ARRAY_SIZE(qaic_drm_ioctls),
253	.gem_prime_import	= qaic_gem_prime_import,
254};
255
256static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id)
257{
258	struct qaic_drm_device *qddev = qdev->qddev;
259	struct drm_device *drm = to_drm(qddev);
260	int ret;
261
262	/* Hold off implementing partitions until the uapi is determined */
263	if (partition_id != QAIC_NO_PARTITION)
264		return -EINVAL;
265
266	qddev->partition_id = partition_id;
267
268	ret = drm_dev_register(drm, 0);
269	if (ret) {
270		pci_dbg(qdev->pdev, "drm_dev_register failed %d\n", ret);
271		return ret;
272	}
273
274	ret = qaic_sysfs_init(qddev);
275	if (ret) {
276		drm_dev_unregister(drm);
277		pci_dbg(qdev->pdev, "qaic_sysfs_init failed %d\n", ret);
278		return ret;
279	}
280
281	qaic_debugfs_init(qddev);
282
283	return ret;
284}
285
286static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id)
287{
288	struct qaic_drm_device *qddev = qdev->qddev;
289	struct drm_device *drm = to_drm(qddev);
290	struct qaic_user *usr;
291
292	qaic_sysfs_remove(qddev);
293	drm_dev_unregister(drm);
294	qddev->partition_id = 0;
295	/*
296	 * Existing users get unresolvable errors till they close FDs.
297	 * Need to sync carefully with users calling close(). The
298	 * list of users can be modified elsewhere when the lock isn't
299	 * held here, but the sync'ing the srcu with the mutex held
300	 * could deadlock. Grab the mutex so that the list will be
301	 * unmodified. The user we get will exist as long as the
302	 * lock is held. Signal that the qcdev is going away, and
303	 * grab a reference to the user so they don't go away for
304	 * synchronize_srcu(). Then release the mutex to avoid
305	 * deadlock and make sure the user has observed the signal.
306	 * With the lock released, we cannot maintain any state of the
307	 * user list.
308	 */
309	mutex_lock(&qddev->users_mutex);
310	while (!list_empty(&qddev->users)) {
311		usr = list_first_entry(&qddev->users, struct qaic_user, node);
312		list_del_init(&usr->node);
313		kref_get(&usr->ref_count);
314		usr->qddev = NULL;
315		mutex_unlock(&qddev->users_mutex);
316		synchronize_srcu(&usr->qddev_lock);
317		kref_put(&usr->ref_count, free_usr);
318		mutex_lock(&qddev->users_mutex);
319	}
320	mutex_unlock(&qddev->users_mutex);
321}
322
323static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
324{
325	u16 major = -1, minor = -1;
326	struct qaic_device *qdev;
327	int ret;
328
329	/*
330	 * Invoking this function indicates that the control channel to the
331	 * device is available. We use that as a signal to indicate that
332	 * the device side firmware has booted. The device side firmware
333	 * manages the device resources, so we need to communicate with it
334	 * via the control channel in order to utilize the device. Therefore
335	 * we wait until this signal to create the drm dev that userspace will
336	 * use to control the device, because without the device side firmware,
337	 * userspace can't do anything useful.
338	 */
339
340	qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
341
342	dev_set_drvdata(&mhi_dev->dev, qdev);
343	qdev->cntl_ch = mhi_dev;
344
345	ret = qaic_control_open(qdev);
346	if (ret) {
347		pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
348		return ret;
349	}
350
351	qdev->dev_state = QAIC_BOOT;
352	ret = get_cntl_version(qdev, NULL, &major, &minor);
353	if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) {
354		pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n",
355			__func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret);
356		ret = -EINVAL;
357		goto close_control;
358	}
359	qdev->dev_state = QAIC_ONLINE;
360	kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_ONLINE);
361
362	return ret;
363
364close_control:
365	qaic_control_close(qdev);
366	return ret;
367}
368
369static void qaic_mhi_remove(struct mhi_device *mhi_dev)
370{
371/* This is redundant since we have already observed the device crash */
372}
373
374static void qaic_notify_reset(struct qaic_device *qdev)
375{
376	int i;
377
378	kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_OFFLINE);
379	qdev->dev_state = QAIC_OFFLINE;
380	/* wake up any waiters to avoid waiting for timeouts at sync */
381	wake_all_cntl(qdev);
382	for (i = 0; i < qdev->num_dbc; ++i)
383		wakeup_dbc(qdev, i);
384	synchronize_srcu(&qdev->dev_lock);
385}
386
387void qaic_dev_reset_clean_local_state(struct qaic_device *qdev)
388{
389	int i;
390
391	qaic_notify_reset(qdev);
392
393	/* start tearing things down */
394	qaic_clean_up_ssr(qdev);
395	for (i = 0; i < qdev->num_dbc; ++i)
396		release_dbc(qdev, i);
397}
398
399static struct qaic_device *create_qdev(struct pci_dev *pdev,
400				       const struct qaic_device_config *config)
401{
402	struct device *dev = &pdev->dev;
403	struct qaic_drm_device *qddev;
404	struct qaic_device *qdev;
405	struct drm_device *drm;
406	int i, ret;
407
408	qdev = devm_kzalloc(dev, sizeof(*qdev), GFP_KERNEL);
409	if (!qdev)
410		return NULL;
411
412	qdev->dev_state = QAIC_OFFLINE;
413	qdev->num_dbc = 16;
414	qdev->dbc = devm_kcalloc(dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
415	if (!qdev->dbc)
416		return NULL;
417
418	qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm);
419	if (IS_ERR(qddev))
420		return NULL;
421
422	drm = to_drm(qddev);
423	pci_set_drvdata(pdev, qdev);
424
425	ret = drmm_mutex_init(drm, &qddev->users_mutex);
426	if (ret)
427		return NULL;
428	ret = drmm_add_action_or_reset(drm, qaicm_pci_release, NULL);
429	if (ret)
430		return NULL;
431	ret = drmm_mutex_init(drm, &qdev->cntl_mutex);
432	if (ret)
433		return NULL;
434	ret = drmm_mutex_init(drm, &qdev->bootlog_mutex);
435	if (ret)
436		return NULL;
437
438	qdev->cntl_wq = qaicm_wq_init(drm, "qaic_cntl");
439	if (IS_ERR(qdev->cntl_wq))
440		return NULL;
441	qdev->qts_wq = qaicm_wq_init(drm, "qaic_ts");
442	if (IS_ERR(qdev->qts_wq))
443		return NULL;
444	qdev->ssr_wq = qaicm_wq_init(drm, "qaic_ssr");
445	if (IS_ERR(qdev->ssr_wq))
446		return NULL;
447
448	ret = qaicm_srcu_init(drm, &qdev->dev_lock);
449	if (ret)
450		return NULL;
451
452	ret = qaic_ssr_init(qdev, drm);
453	if (ret)
454		pci_info(pdev, "QAIC SSR crashdump collection not supported.\n");
455
456	qdev->qddev = qddev;
457	qdev->pdev = pdev;
458	qddev->qdev = qdev;
459
460	INIT_LIST_HEAD(&qdev->cntl_xfer_list);
461	INIT_LIST_HEAD(&qdev->bootlog);
462	INIT_LIST_HEAD(&qddev->users);
463
464	for (i = 0; i < qdev->num_dbc; ++i) {
465		spin_lock_init(&qdev->dbc[i].xfer_lock);
466		qdev->dbc[i].qdev = qdev;
467		qdev->dbc[i].id = i;
468		INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
469		ret = qaicm_srcu_init(drm, &qdev->dbc[i].ch_lock);
470		if (ret)
471			return NULL;
472		init_waitqueue_head(&qdev->dbc[i].dbc_release);
473		INIT_LIST_HEAD(&qdev->dbc[i].bo_lists);
474		ret = drmm_mutex_init(drm, &qdev->dbc[i].req_lock);
475		if (ret)
476			return NULL;
477	}
478
479	return qdev;
480}
481
482static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev,
483		    const struct qaic_device_config *config)
484{
485	int bars;
486	int ret;
487
488	bars = pci_select_bars(pdev, IORESOURCE_MEM) & 0x3f;
489
490	/* make sure the device has the expected BARs */
491	if (bars != config->bar_mask) {
492		pci_dbg(pdev, "%s: expected BARs %#x not found in device. Found %#x\n",
493			__func__, config->bar_mask, bars);
494		return -EINVAL;
495	}
496
497	ret = pcim_enable_device(pdev);
498	if (ret)
499		return ret;
500
501	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
502	if (ret)
503		return ret;
504	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
505
506	qdev->bar_mhi = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->mhi_bar_idx]);
507	if (IS_ERR(qdev->bar_mhi))
508		return PTR_ERR(qdev->bar_mhi);
509
510	qdev->bar_dbc = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->dbc_bar_idx]);
511	if (IS_ERR(qdev->bar_dbc))
512		return PTR_ERR(qdev->bar_dbc);
513
514	/* Managed release since we use pcim_enable_device above */
515	pci_set_master(pdev);
516
517	return 0;
518}
519
520static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
521{
522	int irq_count = qdev->num_dbc + 1;
523	int mhi_irq;
524	int ret;
525	int i;
526
527	/* Managed release since we use pcim_enable_device */
528	ret = pci_alloc_irq_vectors(pdev, irq_count, irq_count, PCI_IRQ_MSI | PCI_IRQ_MSIX);
529	if (ret == -ENOSPC) {
530		ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX);
531		if (ret < 0)
532			return ret;
533
534		/*
535		 * Operate in one MSI mode. All interrupts will be directed to
536		 * MSI0; every interrupt will wake up all the interrupt handlers
537		 * (MHI and DBC[0-15]). Since the interrupt is now shared, it is
538		 * not disabled during DBC threaded handler, but only one thread
539		 * will be allowed to run per DBC, so while it can be
540		 * interrupted, it shouldn't race with itself.
541		 */
542		qdev->single_msi = true;
543		pci_info(pdev, "Allocating %d MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n",
544			 irq_count);
545	} else if (ret < 0) {
546		return ret;
547	}
548
549	mhi_irq = pci_irq_vector(pdev, 0);
550	if (mhi_irq < 0)
551		return mhi_irq;
552
553	for (i = 0; i < qdev->num_dbc; ++i) {
554		ret = devm_request_threaded_irq(&pdev->dev,
555						pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1),
556						dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
557						"qaic_dbc", &qdev->dbc[i]);
558		if (ret)
559			return ret;
560
561		if (datapath_polling) {
562			qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1);
563			if (!qdev->single_msi)
564				disable_irq_nosync(qdev->dbc[i].irq);
565			INIT_WORK(&qdev->dbc[i].poll_work, qaic_irq_polling_work);
566		}
567	}
568
569	return mhi_irq;
570}
571
572static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
573{
574	struct qaic_device_config *config = (struct qaic_device_config *)id->driver_data;
575	struct qaic_device *qdev;
576	int mhi_irq;
577	int ret;
578	int i;
579
580	qdev = create_qdev(pdev, config);
581	if (!qdev)
582		return -ENOMEM;
583
584	ret = init_pci(qdev, pdev, config);
585	if (ret)
586		return ret;
587
588	for (i = 0; i < qdev->num_dbc; ++i)
589		qdev->dbc[i].dbc_base = qdev->bar_dbc + QAIC_DBC_OFF(i);
590
591	mhi_irq = init_msi(qdev, pdev);
592	if (mhi_irq < 0)
593		return mhi_irq;
594
595	ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION);
596	if (ret)
597		return ret;
598
599	qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_mhi, mhi_irq,
600						       qdev->single_msi, config->family);
601	if (IS_ERR(qdev->mhi_cntrl)) {
602		ret = PTR_ERR(qdev->mhi_cntrl);
603		qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
604		return ret;
605	}
606
607	return 0;
608}
609
610static void qaic_pci_remove(struct pci_dev *pdev)
611{
612	struct qaic_device *qdev = pci_get_drvdata(pdev);
613
614	if (!qdev)
615		return;
616
617	qaic_dev_reset_clean_local_state(qdev);
618	qaic_mhi_free_controller(qdev->mhi_cntrl, link_up);
619	qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
620}
621
622static void qaic_pci_shutdown(struct pci_dev *pdev)
623{
624	/* see qaic_exit for what link_up is doing */
625	link_up = true;
626	qaic_pci_remove(pdev);
627}
628
629static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error)
630{
631	return PCI_ERS_RESULT_NEED_RESET;
632}
633
634static void qaic_pci_reset_prepare(struct pci_dev *pdev)
635{
636	struct qaic_device *qdev = pci_get_drvdata(pdev);
637
638	qaic_notify_reset(qdev);
639	qaic_mhi_start_reset(qdev->mhi_cntrl);
640	qaic_dev_reset_clean_local_state(qdev);
641}
642
643static void qaic_pci_reset_done(struct pci_dev *pdev)
644{
645	struct qaic_device *qdev = pci_get_drvdata(pdev);
646
647	qaic_mhi_reset_done(qdev->mhi_cntrl);
648}
649
650static const struct mhi_device_id qaic_mhi_match_table[] = {
651	{ .chan = "QAIC_CONTROL", },
652	{},
653};
654
655static struct mhi_driver qaic_mhi_driver = {
656	.id_table = qaic_mhi_match_table,
657	.remove = qaic_mhi_remove,
658	.probe = qaic_mhi_probe,
659	.ul_xfer_cb = qaic_mhi_ul_xfer_cb,
660	.dl_xfer_cb = qaic_mhi_dl_xfer_cb,
661	.driver = {
662		.name = "qaic_mhi",
663	},
664};
665
666static const struct pci_device_id qaic_ids[] = {
667	{ PCI_DEVICE_DATA(QCOM, AIC080, (kernel_ulong_t)&aic080_config), },
668	{ PCI_DEVICE_DATA(QCOM, AIC100, (kernel_ulong_t)&aic100_config), },
669	{ PCI_DEVICE_DATA(QCOM, AIC200, (kernel_ulong_t)&aic200_config), },
670	{ }
671};
672MODULE_DEVICE_TABLE(pci, qaic_ids);
673
674static const struct pci_error_handlers qaic_pci_err_handler = {
675	.error_detected = qaic_pci_error_detected,
676	.reset_prepare = qaic_pci_reset_prepare,
677	.reset_done = qaic_pci_reset_done,
678};
679
680static bool qaic_is_under_reset(struct qaic_device *qdev)
681{
682	int rcu_id;
683	bool ret;
684
685	rcu_id = srcu_read_lock(&qdev->dev_lock);
686	ret = qdev->dev_state != QAIC_ONLINE;
687	srcu_read_unlock(&qdev->dev_lock, rcu_id);
688	return ret;
689}
690
691static bool qaic_data_path_busy(struct qaic_device *qdev)
692{
693	bool ret = false;
694	int dev_rcu_id;
695	int i;
696
697	dev_rcu_id = srcu_read_lock(&qdev->dev_lock);
698	if (qdev->dev_state != QAIC_ONLINE) {
699		srcu_read_unlock(&qdev->dev_lock, dev_rcu_id);
700		return false;
701	}
702	for (i = 0; i < qdev->num_dbc; i++) {
703		struct dma_bridge_chan *dbc = &qdev->dbc[i];
704		unsigned long flags;
705		int ch_rcu_id;
706
707		ch_rcu_id = srcu_read_lock(&dbc->ch_lock);
708		if (!dbc->usr || !dbc->in_use) {
709			srcu_read_unlock(&dbc->ch_lock, ch_rcu_id);
710			continue;
711		}
712		spin_lock_irqsave(&dbc->xfer_lock, flags);
713		ret = !list_empty(&dbc->xfer_list);
714		spin_unlock_irqrestore(&dbc->xfer_lock, flags);
715		srcu_read_unlock(&dbc->ch_lock, ch_rcu_id);
716		if (ret)
717			break;
718	}
719	srcu_read_unlock(&qdev->dev_lock, dev_rcu_id);
720	return ret;
721}
722
723static int qaic_pm_suspend(struct device *dev)
724{
725	struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(dev));
726
727	dev_dbg(dev, "Suspending..\n");
728	if (qaic_data_path_busy(qdev)) {
729		dev_dbg(dev, "Device's datapath is busy. Aborting suspend..\n");
730		return -EBUSY;
731	}
732	if (qaic_is_under_reset(qdev)) {
733		dev_dbg(dev, "Device is under reset. Aborting suspend..\n");
734		return -EBUSY;
735	}
736	qaic_mqts_ch_stop_timer(qdev->mqts_ch);
737	qaic_pci_reset_prepare(qdev->pdev);
738	pci_save_state(qdev->pdev);
739	pci_disable_device(qdev->pdev);
740	pci_set_power_state(qdev->pdev, PCI_D3hot);
741	return 0;
742}
743
744static int qaic_pm_resume(struct device *dev)
745{
746	struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(dev));
747	int ret;
748
749	dev_dbg(dev, "Resuming..\n");
750	pci_set_power_state(qdev->pdev, PCI_D0);
751	pci_restore_state(qdev->pdev);
752	ret = pci_enable_device(qdev->pdev);
753	if (ret) {
754		dev_err(dev, "pci_enable_device failed on resume %d\n", ret);
755		return ret;
756	}
757	pci_set_master(qdev->pdev);
758	qaic_pci_reset_done(qdev->pdev);
759	return 0;
760}
761
762static const struct dev_pm_ops qaic_pm_ops = {
763	SYSTEM_SLEEP_PM_OPS(qaic_pm_suspend, qaic_pm_resume)
764};
765
766static struct pci_driver qaic_pci_driver = {
767	.name = QAIC_NAME,
768	.id_table = qaic_ids,
769	.probe = qaic_pci_probe,
770	.remove = qaic_pci_remove,
771	.shutdown = qaic_pci_shutdown,
772	.err_handler = &qaic_pci_err_handler,
773	.driver = {
774		.pm = pm_sleep_ptr(&qaic_pm_ops),
775	},
776};
777
778static int __init qaic_init(void)
779{
780	int ret;
781
782	ret = pci_register_driver(&qaic_pci_driver);
783	if (ret) {
784		pr_debug("qaic: pci_register_driver failed %d\n", ret);
785		return ret;
786	}
787
788	ret = mhi_driver_register(&qaic_mhi_driver);
789	if (ret) {
790		pr_debug("qaic: mhi_driver_register failed %d\n", ret);
791		goto free_pci;
792	}
793
794	ret = sahara_register();
795	if (ret) {
796		pr_debug("qaic: sahara_register failed %d\n", ret);
797		goto free_mhi;
798	}
799
800	ret = qaic_timesync_init();
801	if (ret)
802		pr_debug("qaic: qaic_timesync_init failed %d\n", ret);
803
804	ret = qaic_bootlog_register();
805	if (ret)
806		pr_debug("qaic: qaic_bootlog_register failed %d\n", ret);
807
808	ret = qaic_ras_register();
809	if (ret)
810		pr_debug("qaic: qaic_ras_register failed %d\n", ret);
811	ret = qaic_ssr_register();
812	if (ret) {
813		pr_debug("qaic: qaic_ssr_register failed %d\n", ret);
814		goto free_bootlog;
815	}
816
817	return 0;
818
819free_bootlog:
820	qaic_bootlog_unregister();
821free_mhi:
822	mhi_driver_unregister(&qaic_mhi_driver);
823free_pci:
824	pci_unregister_driver(&qaic_pci_driver);
825	return ret;
826}
827
828static void __exit qaic_exit(void)
829{
830	/*
831	 * We assume that qaic_pci_remove() is called due to a hotplug event
832	 * which would mean that the link is down, and thus
833	 * qaic_mhi_free_controller() should not try to access the device during
834	 * cleanup.
835	 * We call pci_unregister_driver() below, which also triggers
836	 * qaic_pci_remove(), but since this is module exit, we expect the link
837	 * to the device to be up, in which case qaic_mhi_free_controller()
838	 * should try to access the device during cleanup to put the device in
839	 * a sane state.
840	 * For that reason, we set link_up here to let qaic_mhi_free_controller
841	 * know the expected link state. Since the module is going to be
842	 * removed at the end of this, we don't need to worry about
843	 * reinitializing the link_up state after the cleanup is done.
844	 */
845	link_up = true;
846	qaic_ssr_unregister();
847	qaic_ras_unregister();
848	qaic_bootlog_unregister();
849	qaic_timesync_deinit();
850	sahara_unregister();
851	mhi_driver_unregister(&qaic_mhi_driver);
852	pci_unregister_driver(&qaic_pci_driver);
853}
854
855module_init(qaic_init);
856module_exit(qaic_exit);
857
858MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team");
859MODULE_DESCRIPTION(QAIC_DESC " Accel Driver");
860MODULE_LICENSE("GPL");