t7xx_pci.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (c) 2021, MediaTek Inc.
  4 * Copyright (c) 2021-2022, Intel Corporation.
  5 *
  6 * Authors:
  7 *  Haijun Liu <haijun.liu@mediatek.com>
  8 *  Ricardo Martinez <ricardo.martinez@linux.intel.com>
  9 *  Sreehari Kancharla <sreehari.kancharla@intel.com>
 10 *
 11 * Contributors:
 12 *  Amir Hanania <amir.hanania@intel.com>
 13 *  Andy Shevchenko <andriy.shevchenko@linux.intel.com>
 14 *  Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com>
 15 *  Eliot Lee <eliot.lee@intel.com>
 16 *  Moises Veleta <moises.veleta@intel.com>
 17 */
 18
 19#include <linux/atomic.h>
 20#include <linux/bits.h>
 21#include <linux/completion.h>
 22#include <linux/device.h>
 23#include <linux/dma-mapping.h>
 24#include <linux/gfp.h>
 25#include <linux/interrupt.h>
 26#include <linux/io.h>
 27#include <linux/iopoll.h>
 28#include <linux/jiffies.h>
 29#include <linux/list.h>
 30#include <linux/module.h>
 31#include <linux/mutex.h>
 32#include <linux/pci.h>
 33#include <linux/pm.h>
 34#include <linux/pm_runtime.h>
 35#include <linux/spinlock.h>
 36
 37#include "t7xx_mhccif.h"
 38#include "t7xx_modem_ops.h"
 39#include "t7xx_pci.h"
 40#include "t7xx_pcie_mac.h"
 41#include "t7xx_reg.h"
 42#include "t7xx_state_monitor.h"
 43#include "t7xx_port_proxy.h"
 44
 45#define DRIVER_NAME "mtk_t7xx"
 46
 47#define T7XX_PCI_IREG_BASE		0
 48#define T7XX_PCI_EREG_BASE		2
 49
 50#define T7XX_INIT_TIMEOUT		20
 51#define PM_SLEEP_DIS_TIMEOUT_MS		20
 52#define PM_ACK_TIMEOUT_MS		1500
 53#define PM_AUTOSUSPEND_MS		5000
 54#define PM_RESOURCE_POLL_TIMEOUT_US	10000
 55#define PM_RESOURCE_POLL_STEP_US	100
 56
 57static const char * const t7xx_mode_names[] = {
 58	[T7XX_UNKNOWN] = "unknown",
 59	[T7XX_READY] = "ready",
 60	[T7XX_RESET] = "reset",
 61	[T7XX_FASTBOOT_SWITCHING] = "fastboot_switching",
 62	[T7XX_FASTBOOT_DOWNLOAD] = "fastboot_download",
 63	[T7XX_FASTBOOT_DUMP] = "fastboot_dump",
 64};
 65
 66static_assert(ARRAY_SIZE(t7xx_mode_names) == T7XX_MODE_LAST);
 67
 68static ssize_t t7xx_mode_store(struct device *dev,
 69			       struct device_attribute *attr,
 70			       const char *buf, size_t count)
 71{
 72	struct t7xx_pci_dev *t7xx_dev;
 73	struct pci_dev *pdev;
 74	enum t7xx_mode mode;
 75	int index = 0;
 76
 77	pdev = to_pci_dev(dev);
 78	t7xx_dev = pci_get_drvdata(pdev);
 79	if (!t7xx_dev)
 80		return -ENODEV;
 81
 82	mode = READ_ONCE(t7xx_dev->mode);
 83
 84	index = sysfs_match_string(t7xx_mode_names, buf);
 85	if (index == mode)
 86		return -EBUSY;
 87
 88	if (index == T7XX_FASTBOOT_SWITCHING) {
 89		if (mode == T7XX_FASTBOOT_DOWNLOAD)
 90			return count;
 91
 92		WRITE_ONCE(t7xx_dev->mode, T7XX_FASTBOOT_SWITCHING);
 93		pm_runtime_resume(dev);
 94		t7xx_reset_device(t7xx_dev, FASTBOOT);
 95	} else if (index == T7XX_RESET) {
 96		pm_runtime_resume(dev);
 97		t7xx_reset_device(t7xx_dev, PLDR);
 98	}
 99
100	return count;
101};
102
103static ssize_t t7xx_mode_show(struct device *dev,
104			      struct device_attribute *attr,
105			      char *buf)
106{
107	enum t7xx_mode mode = T7XX_UNKNOWN;
108	struct t7xx_pci_dev *t7xx_dev;
109	struct pci_dev *pdev;
110
111	pdev = to_pci_dev(dev);
112	t7xx_dev = pci_get_drvdata(pdev);
113	if (!t7xx_dev)
114		return -ENODEV;
115
116	mode = READ_ONCE(t7xx_dev->mode);
117	if (mode < T7XX_MODE_LAST)
118		return sysfs_emit(buf, "%s\n", t7xx_mode_names[mode]);
119
120	return sysfs_emit(buf, "%s\n", t7xx_mode_names[T7XX_UNKNOWN]);
121}
122
123static DEVICE_ATTR_RW(t7xx_mode);
124
125static ssize_t t7xx_debug_ports_store(struct device *dev,
126				      struct device_attribute *attr,
127				      const char *buf, size_t count)
128{
129	struct t7xx_pci_dev *t7xx_dev;
130	struct pci_dev *pdev;
131	bool show;
132	int ret;
133
134	pdev = to_pci_dev(dev);
135	t7xx_dev = pci_get_drvdata(pdev);
136	if (!t7xx_dev)
137		return -ENODEV;
138
139	ret = kstrtobool(buf, &show);
140	if (ret < 0)
141		return ret;
142
143	t7xx_proxy_debug_ports_show(t7xx_dev, show);
144	WRITE_ONCE(t7xx_dev->debug_ports_show, show);
145
146	return count;
147};
148
149static ssize_t t7xx_debug_ports_show(struct device *dev,
150				     struct device_attribute *attr,
151				     char *buf)
152{
153	struct t7xx_pci_dev *t7xx_dev;
154	struct pci_dev *pdev;
155	bool show;
156
157	pdev = to_pci_dev(dev);
158	t7xx_dev = pci_get_drvdata(pdev);
159	if (!t7xx_dev)
160		return -ENODEV;
161
162	show = READ_ONCE(t7xx_dev->debug_ports_show);
163
164	return sysfs_emit(buf, "%d\n", show);
165}
166
167static DEVICE_ATTR_RW(t7xx_debug_ports);
168
169static struct attribute *t7xx_attr[] = {
170	&dev_attr_t7xx_mode.attr,
171	&dev_attr_t7xx_debug_ports.attr,
172	NULL
173};
174
175static const struct attribute_group t7xx_attribute_group = {
176	.attrs = t7xx_attr,
177};
178
179void t7xx_mode_update(struct t7xx_pci_dev *t7xx_dev, enum t7xx_mode mode)
180{
181	if (!t7xx_dev)
182		return;
183
184	WRITE_ONCE(t7xx_dev->mode, mode);
185	sysfs_notify(&t7xx_dev->pdev->dev.kobj, NULL, "t7xx_mode");
186}
187
188enum t7xx_pm_state {
189	MTK_PM_EXCEPTION,
190	MTK_PM_INIT,		/* Device initialized, but handshake not completed */
191	MTK_PM_SUSPENDED,
192	MTK_PM_RESUMED,
193};
194
195static void t7xx_dev_set_sleep_capability(struct t7xx_pci_dev *t7xx_dev, bool enable)
196{
197	void __iomem *ctrl_reg = IREG_BASE(t7xx_dev) + T7XX_PCIE_MISC_CTRL;
198	u32 value;
199
200	value = ioread32(ctrl_reg);
201
202	if (enable)
203		value &= ~T7XX_PCIE_MISC_MAC_SLEEP_DIS;
204	else
205		value |= T7XX_PCIE_MISC_MAC_SLEEP_DIS;
206
207	iowrite32(value, ctrl_reg);
208}
209
210static int t7xx_wait_pm_config(struct t7xx_pci_dev *t7xx_dev)
211{
212	int ret, val;
213
214	ret = read_poll_timeout(ioread32, val,
215				(val & T7XX_PCIE_RESOURCE_STS_MSK) == T7XX_PCIE_RESOURCE_STS_MSK,
216				PM_RESOURCE_POLL_STEP_US, PM_RESOURCE_POLL_TIMEOUT_US, true,
217				IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
218	if (ret == -ETIMEDOUT)
219		dev_err(&t7xx_dev->pdev->dev, "PM configuration timed out\n");
220
221	return ret;
222}
223
224static int t7xx_pci_pm_init(struct t7xx_pci_dev *t7xx_dev)
225{
226	struct pci_dev *pdev = t7xx_dev->pdev;
227
228	INIT_LIST_HEAD(&t7xx_dev->md_pm_entities);
229	mutex_init(&t7xx_dev->md_pm_entity_mtx);
230	spin_lock_init(&t7xx_dev->md_pm_lock);
231	init_completion(&t7xx_dev->sleep_lock_acquire);
232	init_completion(&t7xx_dev->pm_sr_ack);
233	init_completion(&t7xx_dev->init_done);
234	atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
235
236	device_init_wakeup(&pdev->dev, true);
237	dev_pm_set_driver_flags(&pdev->dev, pdev->dev.power.driver_flags |
238				DPM_FLAG_NO_DIRECT_COMPLETE);
239
240	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
241	pm_runtime_set_autosuspend_delay(&pdev->dev, PM_AUTOSUSPEND_MS);
242	pm_runtime_use_autosuspend(&pdev->dev);
243
244	return 0;
245}
246
247void t7xx_pci_pm_init_late(struct t7xx_pci_dev *t7xx_dev)
248{
249	/* Enable the PCIe resource lock only after MD deep sleep is done */
250	t7xx_mhccif_mask_clr(t7xx_dev,
251			     D2H_INT_DS_LOCK_ACK |
252			     D2H_INT_SUSPEND_ACK |
253			     D2H_INT_RESUME_ACK |
254			     D2H_INT_SUSPEND_ACK_AP |
255			     D2H_INT_RESUME_ACK_AP);
256	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
257	atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
258
259	pm_runtime_mark_last_busy(&t7xx_dev->pdev->dev);
260	pm_runtime_allow(&t7xx_dev->pdev->dev);
261	pm_runtime_put_noidle(&t7xx_dev->pdev->dev);
262	complete_all(&t7xx_dev->init_done);
263}
264
265static int t7xx_pci_pm_reinit(struct t7xx_pci_dev *t7xx_dev)
266{
267	/* The device is kept in FSM re-init flow
268	 * so just roll back PM setting to the init setting.
269	 */
270	atomic_set(&t7xx_dev->md_pm_state, MTK_PM_INIT);
271
272	pm_runtime_get_noresume(&t7xx_dev->pdev->dev);
273
274	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
275	return t7xx_wait_pm_config(t7xx_dev);
276}
277
278void t7xx_pci_pm_exp_detected(struct t7xx_pci_dev *t7xx_dev)
279{
280	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
281	t7xx_wait_pm_config(t7xx_dev);
282	atomic_set(&t7xx_dev->md_pm_state, MTK_PM_EXCEPTION);
283}
284
285int t7xx_pci_pm_entity_register(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
286{
287	struct md_pm_entity *entity;
288
289	mutex_lock(&t7xx_dev->md_pm_entity_mtx);
290	list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
291		if (entity->id == pm_entity->id) {
292			mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
293			return -EEXIST;
294		}
295	}
296
297	list_add_tail(&pm_entity->entity, &t7xx_dev->md_pm_entities);
298	mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
299	return 0;
300}
301
302int t7xx_pci_pm_entity_unregister(struct t7xx_pci_dev *t7xx_dev, struct md_pm_entity *pm_entity)
303{
304	struct md_pm_entity *entity, *tmp_entity;
305
306	mutex_lock(&t7xx_dev->md_pm_entity_mtx);
307	list_for_each_entry_safe(entity, tmp_entity, &t7xx_dev->md_pm_entities, entity) {
308		if (entity->id == pm_entity->id) {
309			list_del(&pm_entity->entity);
310			mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
311			return 0;
312		}
313	}
314
315	mutex_unlock(&t7xx_dev->md_pm_entity_mtx);
316
317	return -ENXIO;
318}
319
320int t7xx_pci_sleep_disable_complete(struct t7xx_pci_dev *t7xx_dev)
321{
322	struct device *dev = &t7xx_dev->pdev->dev;
323	int ret;
324
325	ret = wait_for_completion_timeout(&t7xx_dev->sleep_lock_acquire,
326					  msecs_to_jiffies(PM_SLEEP_DIS_TIMEOUT_MS));
327	if (!ret)
328		dev_err_ratelimited(dev, "Resource wait complete timed out\n");
329
330	return ret;
331}
332
333/**
334 * t7xx_pci_disable_sleep() - Disable deep sleep capability.
335 * @t7xx_dev: MTK device.
336 *
337 * Lock the deep sleep capability, note that the device can still go into deep sleep
338 * state while device is in D0 state, from the host's point-of-view.
339 *
340 * If device is in deep sleep state, wake up the device and disable deep sleep capability.
341 */
342void t7xx_pci_disable_sleep(struct t7xx_pci_dev *t7xx_dev)
343{
344	unsigned long flags;
345
346	spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
347	t7xx_dev->sleep_disable_count++;
348	if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
349		goto unlock_and_complete;
350
351	if (t7xx_dev->sleep_disable_count == 1) {
352		u32 status;
353
354		reinit_completion(&t7xx_dev->sleep_lock_acquire);
355		t7xx_dev_set_sleep_capability(t7xx_dev, false);
356
357		status = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_RESOURCE_STATUS);
358		if (status & T7XX_PCIE_RESOURCE_STS_MSK)
359			goto unlock_and_complete;
360
361		t7xx_mhccif_h2d_swint_trigger(t7xx_dev, H2D_CH_DS_LOCK);
362	}
363	spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
364	return;
365
366unlock_and_complete:
367	spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
368	complete_all(&t7xx_dev->sleep_lock_acquire);
369}
370
371/**
372 * t7xx_pci_enable_sleep() - Enable deep sleep capability.
373 * @t7xx_dev: MTK device.
374 *
375 * After enabling deep sleep, device can enter into deep sleep state.
376 */
377void t7xx_pci_enable_sleep(struct t7xx_pci_dev *t7xx_dev)
378{
379	unsigned long flags;
380
381	spin_lock_irqsave(&t7xx_dev->md_pm_lock, flags);
382	t7xx_dev->sleep_disable_count--;
383	if (atomic_read(&t7xx_dev->md_pm_state) < MTK_PM_RESUMED)
384		goto unlock;
385
386	if (t7xx_dev->sleep_disable_count == 0)
387		t7xx_dev_set_sleep_capability(t7xx_dev, true);
388
389unlock:
390	spin_unlock_irqrestore(&t7xx_dev->md_pm_lock, flags);
391}
392
393static int t7xx_send_pm_request(struct t7xx_pci_dev *t7xx_dev, u32 request)
394{
395	unsigned long wait_ret;
396
397	reinit_completion(&t7xx_dev->pm_sr_ack);
398	t7xx_mhccif_h2d_swint_trigger(t7xx_dev, request);
399	wait_ret = wait_for_completion_timeout(&t7xx_dev->pm_sr_ack,
400					       msecs_to_jiffies(PM_ACK_TIMEOUT_MS));
401	if (!wait_ret)
402		return -ETIMEDOUT;
403
404	return 0;
405}
406
407static int __t7xx_pci_pm_suspend(struct pci_dev *pdev)
408{
409	enum t7xx_pm_id entity_id = PM_ENTITY_ID_INVALID;
410	struct t7xx_pci_dev *t7xx_dev;
411	struct md_pm_entity *entity;
412	int ret;
413
414	t7xx_dev = pci_get_drvdata(pdev);
415	if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT ||
416	    READ_ONCE(t7xx_dev->mode) != T7XX_READY) {
417		dev_err(&pdev->dev, "[PM] Exiting suspend, modem in invalid state\n");
418		return -EFAULT;
419	}
420
421	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
422	ret = t7xx_wait_pm_config(t7xx_dev);
423	if (ret) {
424		iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
425		return ret;
426	}
427
428	atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
429	t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
430	t7xx_dev->rgu_pci_irq_en = false;
431
432	list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
433		if (!entity->suspend)
434			continue;
435
436		ret = entity->suspend(t7xx_dev, entity->entity_param);
437		if (ret) {
438			entity_id = entity->id;
439			dev_err(&pdev->dev, "[PM] Suspend error: %d, id: %d\n", ret, entity_id);
440			goto abort_suspend;
441		}
442	}
443
444	ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ);
445	if (ret) {
446		dev_err(&pdev->dev, "[PM] MD suspend error: %d\n", ret);
447		goto abort_suspend;
448	}
449
450	ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP);
451	if (ret) {
452		t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
453		dev_err(&pdev->dev, "[PM] SAP suspend error: %d\n", ret);
454		goto abort_suspend;
455	}
456
457	list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
458		if (entity->suspend_late)
459			entity->suspend_late(t7xx_dev, entity->entity_param);
460	}
461
462	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
463	return 0;
464
465abort_suspend:
466	list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
467		if (entity_id == entity->id)
468			break;
469
470		if (entity->resume)
471			entity->resume(t7xx_dev, entity->entity_param);
472	}
473
474	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
475	atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
476	t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
477	return ret;
478}
479
480static void t7xx_pcie_interrupt_reinit(struct t7xx_pci_dev *t7xx_dev)
481{
482	t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
483
484	/* Disable interrupt first and let the IPs enable them */
485	iowrite32(MSIX_MSK_SET_ALL, IREG_BASE(t7xx_dev) + IMASK_HOST_MSIX_CLR_GRP0_0);
486
487	/* Device disables PCIe interrupts during resume and
488	 * following function will re-enable PCIe interrupts.
489	 */
490	t7xx_pcie_mac_interrupts_en(t7xx_dev);
491	t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
492}
493
494static int t7xx_pcie_reinit(struct t7xx_pci_dev *t7xx_dev, bool is_d3)
495{
496	int ret;
497
498	ret = pcim_enable_device(t7xx_dev->pdev);
499	if (ret)
500		return ret;
501
502	t7xx_pcie_mac_atr_init(t7xx_dev);
503	t7xx_pcie_interrupt_reinit(t7xx_dev);
504
505	if (is_d3) {
506		t7xx_mhccif_init(t7xx_dev);
507		t7xx_pci_pm_reinit(t7xx_dev);
508	}
509
510	return 0;
511}
512
513static int t7xx_send_fsm_command(struct t7xx_pci_dev *t7xx_dev, u32 event)
514{
515	struct t7xx_fsm_ctl *fsm_ctl = t7xx_dev->md->fsm_ctl;
516	struct device *dev = &t7xx_dev->pdev->dev;
517	int ret = -EINVAL;
518
519	switch (event) {
520	case FSM_CMD_STOP:
521		ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_STOP, FSM_CMD_FLAG_WAIT_FOR_COMPLETION);
522		break;
523
524	case FSM_CMD_START:
525		t7xx_pcie_mac_clear_int(t7xx_dev, SAP_RGU_INT);
526		t7xx_pcie_mac_clear_int_status(t7xx_dev, SAP_RGU_INT);
527		t7xx_dev->rgu_pci_irq_en = true;
528		t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
529		ret = t7xx_fsm_append_cmd(fsm_ctl, FSM_CMD_START, 0);
530		break;
531
532	default:
533		break;
534	}
535
536	if (ret)
537		dev_err(dev, "Failure handling FSM command %u, %d\n", event, ret);
538
539	return ret;
540}
541
542int t7xx_pci_reprobe_early(struct t7xx_pci_dev *t7xx_dev)
543{
544	enum t7xx_mode mode = READ_ONCE(t7xx_dev->mode);
545	int ret;
546
547	if (mode == T7XX_FASTBOOT_DOWNLOAD)
548		pm_runtime_put_noidle(&t7xx_dev->pdev->dev);
549
550	ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
551	if (ret)
552		return ret;
553
554	return 0;
555}
556
557int t7xx_pci_reprobe(struct t7xx_pci_dev *t7xx_dev, bool boot)
558{
559	int ret;
560
561	ret = t7xx_pcie_reinit(t7xx_dev, boot);
562	if (ret)
563		return ret;
564
565	t7xx_clear_rgu_irq(t7xx_dev);
566	return t7xx_send_fsm_command(t7xx_dev, FSM_CMD_START);
567}
568
569static int __t7xx_pci_pm_resume(struct pci_dev *pdev, bool state_check)
570{
571	struct t7xx_pci_dev *t7xx_dev;
572	struct md_pm_entity *entity;
573	u32 prev_state;
574	int ret = 0;
575
576	t7xx_dev = pci_get_drvdata(pdev);
577	if (atomic_read(&t7xx_dev->md_pm_state) <= MTK_PM_INIT) {
578		iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
579		return 0;
580	}
581
582	t7xx_pcie_mac_interrupts_en(t7xx_dev);
583	prev_state = ioread32(IREG_BASE(t7xx_dev) + T7XX_PCIE_PM_RESUME_STATE);
584
585	if (state_check) {
586		/* For D3/L3 resume, the device could boot so quickly that the
587		 * initial value of the dummy register might be overwritten.
588		 * Identify new boots if the ATR source address register is not initialized.
589		 */
590		u32 atr_reg_val = ioread32(IREG_BASE(t7xx_dev) +
591					   ATR_PCIE_WIN0_T0_ATR_PARAM_SRC_ADDR);
592		if (prev_state == PM_RESUME_REG_STATE_L3 ||
593		    (prev_state == PM_RESUME_REG_STATE_INIT &&
594		     atr_reg_val == ATR_SRC_ADDR_INVALID)) {
595			ret = t7xx_pci_reprobe_early(t7xx_dev);
596			if (ret)
597				return ret;
598
599			return t7xx_pci_reprobe(t7xx_dev, true);
600		}
601
602		if (prev_state == PM_RESUME_REG_STATE_EXP ||
603		    prev_state == PM_RESUME_REG_STATE_L2_EXP) {
604			if (prev_state == PM_RESUME_REG_STATE_L2_EXP) {
605				ret = t7xx_pcie_reinit(t7xx_dev, false);
606				if (ret)
607					return ret;
608			}
609
610			atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
611			t7xx_dev->rgu_pci_irq_en = true;
612			t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
613
614			t7xx_mhccif_mask_clr(t7xx_dev,
615					     D2H_INT_EXCEPTION_INIT |
616					     D2H_INT_EXCEPTION_INIT_DONE |
617					     D2H_INT_EXCEPTION_CLEARQ_DONE |
618					     D2H_INT_EXCEPTION_ALLQ_RESET |
619					     D2H_INT_PORT_ENUM);
620
621			return ret;
622		}
623
624		if (prev_state == PM_RESUME_REG_STATE_L2) {
625			ret = t7xx_pcie_reinit(t7xx_dev, false);
626			if (ret)
627				return ret;
628
629		} else if (prev_state != PM_RESUME_REG_STATE_L1 &&
630			   prev_state != PM_RESUME_REG_STATE_INIT) {
631			ret = t7xx_send_fsm_command(t7xx_dev, FSM_CMD_STOP);
632			if (ret)
633				return ret;
634
635			t7xx_clear_rgu_irq(t7xx_dev);
636			atomic_set(&t7xx_dev->md_pm_state, MTK_PM_SUSPENDED);
637			return 0;
638		}
639	}
640
641	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + DISABLE_ASPM_LOWPWR);
642	t7xx_wait_pm_config(t7xx_dev);
643
644	list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
645		if (entity->resume_early)
646			entity->resume_early(t7xx_dev, entity->entity_param);
647	}
648
649	ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ);
650	if (ret)
651		dev_err(&pdev->dev, "[PM] MD resume error: %d\n", ret);
652
653	ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ_AP);
654	if (ret)
655		dev_err(&pdev->dev, "[PM] SAP resume error: %d\n", ret);
656
657	list_for_each_entry(entity, &t7xx_dev->md_pm_entities, entity) {
658		if (entity->resume) {
659			ret = entity->resume(t7xx_dev, entity->entity_param);
660			if (ret)
661				dev_err(&pdev->dev, "[PM] Resume entry ID: %d error: %d\n",
662					entity->id, ret);
663		}
664	}
665
666	t7xx_dev->rgu_pci_irq_en = true;
667	t7xx_pcie_mac_set_int(t7xx_dev, SAP_RGU_INT);
668	iowrite32(T7XX_L1_BIT(0), IREG_BASE(t7xx_dev) + ENABLE_ASPM_LOWPWR);
669	pm_runtime_mark_last_busy(&pdev->dev);
670	atomic_set(&t7xx_dev->md_pm_state, MTK_PM_RESUMED);
671
672	return ret;
673}
674
675static int t7xx_pci_pm_resume_noirq(struct device *dev)
676{
677	struct pci_dev *pdev = to_pci_dev(dev);
678	struct t7xx_pci_dev *t7xx_dev;
679
680	t7xx_dev = pci_get_drvdata(pdev);
681	t7xx_pcie_mac_interrupts_dis(t7xx_dev);
682
683	return 0;
684}
685
686static void t7xx_pci_shutdown(struct pci_dev *pdev)
687{
688	__t7xx_pci_pm_suspend(pdev);
689}
690
691static int t7xx_pci_pm_prepare(struct device *dev)
692{
693	struct pci_dev *pdev = to_pci_dev(dev);
694	struct t7xx_pci_dev *t7xx_dev;
695
696	t7xx_dev = pci_get_drvdata(pdev);
697	if (!wait_for_completion_timeout(&t7xx_dev->init_done, T7XX_INIT_TIMEOUT * HZ)) {
698		dev_warn(dev, "Not ready for system sleep.\n");
699		return -ETIMEDOUT;
700	}
701
702	return 0;
703}
704
705static int t7xx_pci_pm_suspend(struct device *dev)
706{
707	return __t7xx_pci_pm_suspend(to_pci_dev(dev));
708}
709
710static int t7xx_pci_pm_resume(struct device *dev)
711{
712	return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
713}
714
715static int t7xx_pci_pm_thaw(struct device *dev)
716{
717	return __t7xx_pci_pm_resume(to_pci_dev(dev), false);
718}
719
720static int t7xx_pci_pm_runtime_suspend(struct device *dev)
721{
722	return __t7xx_pci_pm_suspend(to_pci_dev(dev));
723}
724
725static int t7xx_pci_pm_runtime_resume(struct device *dev)
726{
727	return __t7xx_pci_pm_resume(to_pci_dev(dev), true);
728}
729
730static const struct dev_pm_ops t7xx_pci_pm_ops = {
731	.prepare = t7xx_pci_pm_prepare,
732	.suspend = t7xx_pci_pm_suspend,
733	.resume = t7xx_pci_pm_resume,
734	.resume_noirq = t7xx_pci_pm_resume_noirq,
735	.freeze = t7xx_pci_pm_suspend,
736	.thaw = t7xx_pci_pm_thaw,
737	.poweroff = t7xx_pci_pm_suspend,
738	.restore = t7xx_pci_pm_resume,
739	.restore_noirq = t7xx_pci_pm_resume_noirq,
740	.runtime_suspend = t7xx_pci_pm_runtime_suspend,
741	.runtime_resume = t7xx_pci_pm_runtime_resume
742};
743
744static int t7xx_request_irq(struct pci_dev *pdev)
745{
746	struct t7xx_pci_dev *t7xx_dev;
747	int ret = 0, i;
748
749	t7xx_dev = pci_get_drvdata(pdev);
750
751	for (i = 0; i < EXT_INT_NUM; i++) {
752		const char *irq_descr;
753		int irq_vec;
754
755		if (!t7xx_dev->intr_handler[i])
756			continue;
757
758		irq_descr = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d",
759					   dev_driver_string(&pdev->dev), i);
760		if (!irq_descr) {
761			ret = -ENOMEM;
762			break;
763		}
764
765		irq_vec = pci_irq_vector(pdev, i);
766		ret = request_threaded_irq(irq_vec, t7xx_dev->intr_handler[i],
767					   t7xx_dev->intr_thread[i], 0, irq_descr,
768					   t7xx_dev->callback_param[i]);
769		if (ret) {
770			dev_err(&pdev->dev, "Failed to request IRQ: %d\n", ret);
771			break;
772		}
773	}
774
775	if (ret) {
776		while (i--) {
777			if (!t7xx_dev->intr_handler[i])
778				continue;
779
780			free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
781		}
782	}
783
784	return ret;
785}
786
787static int t7xx_setup_msix(struct t7xx_pci_dev *t7xx_dev)
788{
789	struct pci_dev *pdev = t7xx_dev->pdev;
790	int ret;
791
792	/* Only using 6 interrupts, but HW-design requires power-of-2 IRQs allocation */
793	ret = pci_alloc_irq_vectors(pdev, EXT_INT_NUM, EXT_INT_NUM, PCI_IRQ_MSIX);
794	if (ret < 0) {
795		dev_err(&pdev->dev, "Failed to allocate MSI-X entry: %d\n", ret);
796		return ret;
797	}
798
799	ret = t7xx_request_irq(pdev);
800	if (ret) {
801		pci_free_irq_vectors(pdev);
802		return ret;
803	}
804
805	t7xx_pcie_set_mac_msix_cfg(t7xx_dev, EXT_INT_NUM);
806	return 0;
807}
808
809static int t7xx_interrupt_init(struct t7xx_pci_dev *t7xx_dev)
810{
811	int ret, i;
812
813	if (!t7xx_dev->pdev->msix_cap)
814		return -EINVAL;
815
816	ret = t7xx_setup_msix(t7xx_dev);
817	if (ret)
818		return ret;
819
820	/* IPs enable interrupts when ready */
821	for (i = 0; i < EXT_INT_NUM; i++)
822		t7xx_pcie_mac_set_int(t7xx_dev, i);
823
824	return 0;
825}
826
827static void t7xx_pci_infracfg_ao_calc(struct t7xx_pci_dev *t7xx_dev)
828{
829	t7xx_dev->base_addr.infracfg_ao_base = t7xx_dev->base_addr.pcie_ext_reg_base +
830					      INFRACFG_AO_DEV_CHIP -
831					      t7xx_dev->base_addr.pcie_dev_reg_trsl_addr;
832}
833
834static int t7xx_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
835{
836	struct t7xx_pci_dev *t7xx_dev;
837	void __iomem *iomem;
838	int ret;
839
840	t7xx_dev = devm_kzalloc(&pdev->dev, sizeof(*t7xx_dev), GFP_KERNEL);
841	if (!t7xx_dev)
842		return -ENOMEM;
843
844	pci_set_drvdata(pdev, t7xx_dev);
845	t7xx_dev->pdev = pdev;
846
847	ret = pcim_enable_device(pdev);
848	if (ret)
849		return ret;
850
851	pci_set_master(pdev);
852
853	iomem = pcim_iomap_region(pdev, T7XX_PCI_IREG_BASE, DRIVER_NAME);
854	ret = PTR_ERR_OR_ZERO(iomem);
855	if (ret) {
856		dev_err(&pdev->dev, "Could not request IREG BAR: %d\n", ret);
857		return -ENOMEM;
858	}
859	IREG_BASE(t7xx_dev) = iomem;
860
861	iomem = pcim_iomap_region(pdev, T7XX_PCI_EREG_BASE, DRIVER_NAME);
862	ret = PTR_ERR_OR_ZERO(iomem);
863	if (ret) {
864		dev_err(&pdev->dev, "Could not request EREG BAR: %d\n", ret);
865		return -ENOMEM;
866	}
867	t7xx_dev->base_addr.pcie_ext_reg_base = iomem;
868
869	ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
870	if (ret) {
871		dev_err(&pdev->dev, "Could not set PCI DMA mask: %d\n", ret);
872		return ret;
873	}
874
875	ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
876	if (ret) {
877		dev_err(&pdev->dev, "Could not set consistent PCI DMA mask: %d\n", ret);
878		return ret;
879	}
880
881	ret = t7xx_pci_pm_init(t7xx_dev);
882	if (ret)
883		return ret;
884
885	t7xx_pcie_mac_atr_init(t7xx_dev);
886	t7xx_pci_infracfg_ao_calc(t7xx_dev);
887	t7xx_mhccif_init(t7xx_dev);
888
889	ret = t7xx_md_init(t7xx_dev);
890	if (ret)
891		return ret;
892
893	t7xx_pcie_mac_interrupts_dis(t7xx_dev);
894
895	ret = sysfs_create_group(&t7xx_dev->pdev->dev.kobj,
896				 &t7xx_attribute_group);
897	if (ret)
898		goto err_md_exit;
899
900	ret = t7xx_interrupt_init(t7xx_dev);
901	if (ret)
902		goto err_remove_group;
903
904
905	t7xx_pcie_mac_set_int(t7xx_dev, MHCCIF_INT);
906	t7xx_pcie_mac_interrupts_en(t7xx_dev);
907
908	return 0;
909
910err_remove_group:
911	sysfs_remove_group(&t7xx_dev->pdev->dev.kobj,
912			   &t7xx_attribute_group);
913
914err_md_exit:
915	t7xx_md_exit(t7xx_dev);
916	return ret;
917}
918
919static void t7xx_pci_remove(struct pci_dev *pdev)
920{
921	struct t7xx_pci_dev *t7xx_dev;
922	int i;
923
924	t7xx_dev = pci_get_drvdata(pdev);
925
926	sysfs_remove_group(&t7xx_dev->pdev->dev.kobj,
927			   &t7xx_attribute_group);
928	t7xx_md_exit(t7xx_dev);
929
930	for (i = 0; i < EXT_INT_NUM; i++) {
931		if (!t7xx_dev->intr_handler[i])
932			continue;
933
934		free_irq(pci_irq_vector(pdev, i), t7xx_dev->callback_param[i]);
935	}
936
937	pci_free_irq_vectors(t7xx_dev->pdev);
938}
939
940static const struct pci_device_id t7xx_pci_table[] = {
941	{ PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x4d75) },
942	{ PCI_DEVICE(0x03f0, 0x09c8) }, // HP DRMR-H01
943	{ PCI_DEVICE(0x14c0, 0x4d75) }, // Dell DW5933e
944	{ }
945};
946MODULE_DEVICE_TABLE(pci, t7xx_pci_table);
947
948static struct pci_driver t7xx_pci_driver = {
949	.name = DRIVER_NAME,
950	.id_table = t7xx_pci_table,
951	.probe = t7xx_pci_probe,
952	.remove = t7xx_pci_remove,
953	.driver.pm = &t7xx_pci_pm_ops,
954	.shutdown = t7xx_pci_shutdown,
955};
956
957module_pci_driver(t7xx_pci_driver);
958
959MODULE_AUTHOR("MediaTek Inc");
960MODULE_DESCRIPTION("MediaTek PCIe 5G WWAN modem T7xx driver");
961MODULE_LICENSE("GPL");