pvr_power.c

  1// SPDX-License-Identifier: GPL-2.0-only OR MIT
  2/* Copyright (c) 2023 Imagination Technologies Ltd. */
  3
  4#include "pvr_device.h"
  5#include "pvr_fw.h"
  6#include "pvr_fw_startstop.h"
  7#include "pvr_power.h"
  8#include "pvr_queue.h"
  9#include "pvr_rogue_fwif.h"
 10
 11#include <drm/drm_drv.h>
 12#include <drm/drm_managed.h>
 13#include <drm/drm_print.h>
 14#include <linux/cleanup.h>
 15#include <linux/clk.h>
 16#include <linux/interrupt.h>
 17#include <linux/mutex.h>
 18#include <linux/of.h>
 19#include <linux/platform_device.h>
 20#include <linux/pm_domain.h>
 21#include <linux/pm_runtime.h>
 22#include <linux/pwrseq/consumer.h>
 23#include <linux/reset.h>
 24#include <linux/timer.h>
 25#include <linux/types.h>
 26#include <linux/workqueue.h>
 27
 28#define POWER_SYNC_TIMEOUT_US (1000000) /* 1s */
 29
 30#define WATCHDOG_TIME_MS (500)
 31
 32/**
 33 * pvr_device_lost() - Mark GPU device as lost
 34 * @pvr_dev: Target PowerVR device.
 35 *
 36 * This will cause the DRM device to be unplugged.
 37 */
 38void
 39pvr_device_lost(struct pvr_device *pvr_dev)
 40{
 41	if (!pvr_dev->lost) {
 42		pvr_dev->lost = true;
 43		drm_dev_unplug(from_pvr_device(pvr_dev));
 44	}
 45}
 46
 47static int
 48pvr_power_send_command(struct pvr_device *pvr_dev, struct rogue_fwif_kccb_cmd *pow_cmd)
 49{
 50	struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 51	u32 slot_nr;
 52	u32 value;
 53	int err;
 54
 55	WRITE_ONCE(*fw_dev->power_sync, 0);
 56
 57	err = pvr_kccb_send_cmd_powered(pvr_dev, pow_cmd, &slot_nr);
 58	if (err)
 59		return err;
 60
 61	/* Wait for FW to acknowledge. */
 62	return readl_poll_timeout(pvr_dev->fw_dev.power_sync, value, value != 0, 100,
 63				  POWER_SYNC_TIMEOUT_US);
 64}
 65
 66static int
 67pvr_power_request_idle(struct pvr_device *pvr_dev)
 68{
 69	struct rogue_fwif_kccb_cmd pow_cmd;
 70
 71	/* Send FORCED_IDLE request to FW. */
 72	pow_cmd.cmd_type = ROGUE_FWIF_KCCB_CMD_POW;
 73	pow_cmd.cmd_data.pow_data.pow_type = ROGUE_FWIF_POW_FORCED_IDLE_REQ;
 74	pow_cmd.cmd_data.pow_data.power_req_data.pow_request_type = ROGUE_FWIF_POWER_FORCE_IDLE;
 75
 76	return pvr_power_send_command(pvr_dev, &pow_cmd);
 77}
 78
 79static int
 80pvr_power_request_pwr_off(struct pvr_device *pvr_dev)
 81{
 82	struct rogue_fwif_kccb_cmd pow_cmd;
 83
 84	/* Send POW_OFF request to firmware. */
 85	pow_cmd.cmd_type = ROGUE_FWIF_KCCB_CMD_POW;
 86	pow_cmd.cmd_data.pow_data.pow_type = ROGUE_FWIF_POW_OFF_REQ;
 87	pow_cmd.cmd_data.pow_data.power_req_data.forced = true;
 88
 89	return pvr_power_send_command(pvr_dev, &pow_cmd);
 90}
 91
 92static int
 93pvr_power_fw_disable(struct pvr_device *pvr_dev, bool hard_reset, bool rpm_suspend)
 94{
 95	if (!hard_reset) {
 96		int err;
 97
 98		cancel_delayed_work_sync(&pvr_dev->watchdog.work);
 99
100		err = pvr_power_request_idle(pvr_dev);
101		if (err)
102			return err;
103
104		err = pvr_power_request_pwr_off(pvr_dev);
105		if (err)
106			return err;
107	}
108
109	if (rpm_suspend) {
110		/* Wait for late processing of GPU or firmware IRQs in other cores */
111		synchronize_irq(pvr_dev->irq);
112	}
113
114	return pvr_fw_stop(pvr_dev);
115}
116
117static int
118pvr_power_fw_enable(struct pvr_device *pvr_dev)
119{
120	int err;
121
122	err = pvr_fw_start(pvr_dev);
123	if (err)
124		return err;
125
126	err = pvr_wait_for_fw_boot(pvr_dev);
127	if (err) {
128		drm_err(from_pvr_device(pvr_dev), "Firmware failed to boot\n");
129		pvr_fw_stop(pvr_dev);
130		return err;
131	}
132
133	queue_delayed_work(pvr_dev->sched_wq, &pvr_dev->watchdog.work,
134			   msecs_to_jiffies(WATCHDOG_TIME_MS));
135
136	return 0;
137}
138
139bool
140pvr_power_is_idle(struct pvr_device *pvr_dev)
141{
142	/*
143	 * FW power state can be out of date if a KCCB command has been submitted but the FW hasn't
144	 * started processing it yet. So also check the KCCB status.
145	 */
146	enum rogue_fwif_pow_state pow_state = READ_ONCE(pvr_dev->fw_dev.fwif_sysdata->pow_state);
147	bool kccb_idle = pvr_kccb_is_idle(pvr_dev);
148
149	return (pow_state == ROGUE_FWIF_POW_IDLE) && kccb_idle;
150}
151
152static bool
153pvr_watchdog_kccb_stalled(struct pvr_device *pvr_dev)
154{
155	/* Check KCCB commands are progressing. */
156	u32 kccb_cmds_executed = pvr_dev->fw_dev.fwif_osdata->kccb_cmds_executed;
157	bool kccb_is_idle = pvr_kccb_is_idle(pvr_dev);
158
159	if (pvr_dev->watchdog.old_kccb_cmds_executed == kccb_cmds_executed && !kccb_is_idle) {
160		pvr_dev->watchdog.kccb_stall_count++;
161
162		/*
163		 * If we have commands pending with no progress for 2 consecutive polls then
164		 * consider KCCB command processing stalled.
165		 */
166		if (pvr_dev->watchdog.kccb_stall_count == 2) {
167			pvr_dev->watchdog.kccb_stall_count = 0;
168			return true;
169		}
170	} else if (pvr_dev->watchdog.old_kccb_cmds_executed == kccb_cmds_executed) {
171		bool has_active_contexts;
172
173		mutex_lock(&pvr_dev->queues.lock);
174		has_active_contexts = list_empty(&pvr_dev->queues.active);
175		mutex_unlock(&pvr_dev->queues.lock);
176
177		if (has_active_contexts) {
178			/* Send a HEALTH_CHECK command so we can verify FW is still alive. */
179			struct rogue_fwif_kccb_cmd health_check_cmd;
180
181			health_check_cmd.cmd_type = ROGUE_FWIF_KCCB_CMD_HEALTH_CHECK;
182
183			pvr_kccb_send_cmd_powered(pvr_dev, &health_check_cmd, NULL);
184		}
185	} else {
186		pvr_dev->watchdog.old_kccb_cmds_executed = kccb_cmds_executed;
187		pvr_dev->watchdog.kccb_stall_count = 0;
188	}
189
190	return false;
191}
192
193static void
194pvr_watchdog_worker(struct work_struct *work)
195{
196	struct pvr_device *pvr_dev = container_of(work, struct pvr_device,
197						  watchdog.work.work);
198	bool stalled;
199
200	if (pvr_dev->lost)
201		return;
202
203	if (pm_runtime_get_if_in_use(from_pvr_device(pvr_dev)->dev) <= 0)
204		goto out_requeue;
205
206	if (!pvr_dev->fw_dev.booted)
207		goto out_pm_runtime_put;
208
209	stalled = pvr_watchdog_kccb_stalled(pvr_dev);
210
211	if (stalled) {
212		drm_err(from_pvr_device(pvr_dev), "FW stalled, trying hard reset");
213
214		pvr_power_reset(pvr_dev, true);
215		/* Device may be lost at this point. */
216	}
217
218out_pm_runtime_put:
219	pm_runtime_put(from_pvr_device(pvr_dev)->dev);
220
221out_requeue:
222	if (!pvr_dev->lost) {
223		queue_delayed_work(pvr_dev->sched_wq, &pvr_dev->watchdog.work,
224				   msecs_to_jiffies(WATCHDOG_TIME_MS));
225	}
226}
227
228/**
229 * pvr_watchdog_init() - Initialise watchdog for device
230 * @pvr_dev: Target PowerVR device.
231 *
232 * Returns:
233 *  * 0 on success, or
234 *  * -%ENOMEM on out of memory.
235 */
236int
237pvr_watchdog_init(struct pvr_device *pvr_dev)
238{
239	INIT_DELAYED_WORK(&pvr_dev->watchdog.work, pvr_watchdog_worker);
240
241	return 0;
242}
243
244static int pvr_power_init_manual(struct pvr_device *pvr_dev)
245{
246	struct drm_device *drm_dev = from_pvr_device(pvr_dev);
247	struct reset_control *reset;
248
249	reset = devm_reset_control_get_optional_exclusive(drm_dev->dev, NULL);
250	if (IS_ERR(reset))
251		return dev_err_probe(drm_dev->dev, PTR_ERR(reset),
252				     "failed to get gpu reset line\n");
253
254	pvr_dev->reset = reset;
255
256	return 0;
257}
258
259static int pvr_power_on_sequence_manual(struct pvr_device *pvr_dev)
260{
261	int err;
262
263	err = clk_prepare_enable(pvr_dev->core_clk);
264	if (err)
265		return err;
266
267	err = clk_prepare_enable(pvr_dev->sys_clk);
268	if (err)
269		goto err_core_clk_disable;
270
271	err = clk_prepare_enable(pvr_dev->mem_clk);
272	if (err)
273		goto err_sys_clk_disable;
274
275	/*
276	 * According to the hardware manual, a delay of at least 32 clock
277	 * cycles is required between de-asserting the clkgen reset and
278	 * de-asserting the GPU reset. Assuming a worst-case scenario with
279	 * a very high GPU clock frequency, a delay of 1 microsecond is
280	 * sufficient to ensure this requirement is met across all
281	 * feasible GPU clock speeds.
282	 */
283	udelay(1);
284
285	err = reset_control_deassert(pvr_dev->reset);
286	if (err)
287		goto err_mem_clk_disable;
288
289	return 0;
290
291err_mem_clk_disable:
292	clk_disable_unprepare(pvr_dev->mem_clk);
293
294err_sys_clk_disable:
295	clk_disable_unprepare(pvr_dev->sys_clk);
296
297err_core_clk_disable:
298	clk_disable_unprepare(pvr_dev->core_clk);
299
300	return err;
301}
302
303static int pvr_power_off_sequence_manual(struct pvr_device *pvr_dev)
304{
305	int err;
306
307	err = reset_control_assert(pvr_dev->reset);
308
309	clk_disable_unprepare(pvr_dev->mem_clk);
310	clk_disable_unprepare(pvr_dev->sys_clk);
311	clk_disable_unprepare(pvr_dev->core_clk);
312
313	return err;
314}
315
316const struct pvr_power_sequence_ops pvr_power_sequence_ops_manual = {
317	.init = pvr_power_init_manual,
318	.power_on = pvr_power_on_sequence_manual,
319	.power_off = pvr_power_off_sequence_manual,
320};
321
322static int pvr_power_init_pwrseq(struct pvr_device *pvr_dev)
323{
324	struct device *dev = from_pvr_device(pvr_dev)->dev;
325
326	pvr_dev->pwrseq = devm_pwrseq_get(dev, "gpu-power");
327	if (IS_ERR(pvr_dev->pwrseq)) {
328		/*
329		 * This platform requires a sequencer. If we can't get it, we
330		 * must return the error (including -EPROBE_DEFER to wait for
331		 * the provider to appear)
332		 */
333		return dev_err_probe(dev, PTR_ERR(pvr_dev->pwrseq),
334				     "Failed to get required power sequencer\n");
335	}
336
337	return 0;
338}
339
340static int pvr_power_on_sequence_pwrseq(struct pvr_device *pvr_dev)
341{
342	return pwrseq_power_on(pvr_dev->pwrseq);
343}
344
345static int pvr_power_off_sequence_pwrseq(struct pvr_device *pvr_dev)
346{
347	return pwrseq_power_off(pvr_dev->pwrseq);
348}
349
350const struct pvr_power_sequence_ops pvr_power_sequence_ops_pwrseq = {
351	.init = pvr_power_init_pwrseq,
352	.power_on = pvr_power_on_sequence_pwrseq,
353	.power_off = pvr_power_off_sequence_pwrseq,
354};
355
356int
357pvr_power_device_suspend(struct device *dev)
358{
359	struct platform_device *plat_dev = to_platform_device(dev);
360	struct drm_device *drm_dev = platform_get_drvdata(plat_dev);
361	struct pvr_device *pvr_dev = to_pvr_device(drm_dev);
362	int err = 0;
363	int idx;
364
365	if (!drm_dev_enter(drm_dev, &idx))
366		return -EIO;
367
368	if (pvr_dev->fw_dev.booted) {
369		err = pvr_power_fw_disable(pvr_dev, false, true);
370		if (err)
371			goto err_drm_dev_exit;
372	}
373
374	err = pvr_dev->device_data->pwr_ops->power_off(pvr_dev);
375
376err_drm_dev_exit:
377	drm_dev_exit(idx);
378
379	return err;
380}
381
382int
383pvr_power_device_resume(struct device *dev)
384{
385	struct platform_device *plat_dev = to_platform_device(dev);
386	struct drm_device *drm_dev = platform_get_drvdata(plat_dev);
387	struct pvr_device *pvr_dev = to_pvr_device(drm_dev);
388	int idx;
389	int err;
390
391	if (!drm_dev_enter(drm_dev, &idx))
392		return -EIO;
393
394	err = pvr_dev->device_data->pwr_ops->power_on(pvr_dev);
395	if (err)
396		goto err_drm_dev_exit;
397
398	if (pvr_dev->fw_dev.booted) {
399		err = pvr_power_fw_enable(pvr_dev);
400		if (err)
401			goto err_power_off;
402	}
403
404	drm_dev_exit(idx);
405
406	return 0;
407
408err_power_off:
409	pvr_dev->device_data->pwr_ops->power_off(pvr_dev);
410
411err_drm_dev_exit:
412	drm_dev_exit(idx);
413
414	return err;
415}
416
417int
418pvr_power_device_idle(struct device *dev)
419{
420	struct platform_device *plat_dev = to_platform_device(dev);
421	struct drm_device *drm_dev = platform_get_drvdata(plat_dev);
422	struct pvr_device *pvr_dev = to_pvr_device(drm_dev);
423
424	return pvr_power_is_idle(pvr_dev) ? 0 : -EBUSY;
425}
426
427static int
428pvr_power_clear_error(struct pvr_device *pvr_dev)
429{
430	struct device *dev = from_pvr_device(pvr_dev)->dev;
431	int err;
432
433	/* Ensure the device state is known and nothing is happening past this point */
434	pm_runtime_disable(dev);
435
436	/* Attempt to clear the runtime PM error by setting the current state again */
437	if (pm_runtime_status_suspended(dev))
438		err = pm_runtime_set_suspended(dev);
439	else
440		err = pm_runtime_set_active(dev);
441
442	if (err) {
443		drm_err(from_pvr_device(pvr_dev),
444			"%s: Failed to clear runtime PM error (new error %d)\n",
445			__func__, err);
446	}
447
448	pm_runtime_enable(dev);
449
450	return err;
451}
452
453/**
454 * pvr_power_get_clear() - Acquire a power reference, correcting any errors
455 * @pvr_dev: Device pointer
456 *
457 * Attempt to acquire a power reference on the device. If the runtime PM
458 * is in error state, attempt to clear the error and retry.
459 *
460 * Returns:
461 *  * 0 on success, or
462 *  * Any error code returned by pvr_power_get() or the runtime PM API.
463 */
464static int
465pvr_power_get_clear(struct pvr_device *pvr_dev)
466{
467	int err;
468
469	err = pvr_power_get(pvr_dev);
470	if (err == 0)
471		return err;
472
473	drm_warn(from_pvr_device(pvr_dev),
474		 "%s: pvr_power_get returned error %d, attempting recovery\n",
475		 __func__, err);
476
477	err = pvr_power_clear_error(pvr_dev);
478	if (err)
479		return err;
480
481	return pvr_power_get(pvr_dev);
482}
483
484/**
485 * pvr_power_reset() - Reset the GPU
486 * @pvr_dev: Device pointer
487 * @hard_reset: %true for hard reset, %false for soft reset
488 *
489 * If @hard_reset is %false and the FW processor fails to respond during the reset process, this
490 * function will attempt a hard reset.
491 *
492 * If a hard reset fails then the GPU device is reported as lost.
493 *
494 * Returns:
495 *  * 0 on success, or
496 *  * Any error code returned by pvr_power_get, pvr_power_fw_disable or pvr_power_fw_enable().
497 */
498int
499pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset)
500{
501	bool queues_disabled = false;
502	int err;
503
504	/*
505	 * Take a power reference during the reset. This should prevent any interference with the
506	 * power state during reset.
507	 */
508	WARN_ON(pvr_power_get_clear(pvr_dev));
509
510	down_write(&pvr_dev->reset_sem);
511
512	if (pvr_dev->lost) {
513		err = -EIO;
514		goto err_up_write;
515	}
516
517	/* Disable IRQs for the duration of the reset. */
518	if (hard_reset) {
519		disable_irq(pvr_dev->irq);
520	} else {
521		/*
522		 * Soft reset is triggered as a response to a FW command to the Host and is
523		 * processed from the threaded IRQ handler. This code cannot (nor needs to)
524		 * wait for any IRQ processing to complete.
525		 */
526		disable_irq_nosync(pvr_dev->irq);
527	}
528
529	do {
530		if (hard_reset) {
531			pvr_queue_device_pre_reset(pvr_dev);
532			queues_disabled = true;
533		}
534
535		err = pvr_power_fw_disable(pvr_dev, hard_reset, false);
536		if (!err) {
537			if (hard_reset) {
538				pvr_dev->fw_dev.booted = false;
539				WARN_ON(pvr_power_device_suspend(from_pvr_device(pvr_dev)->dev));
540
541				err = pvr_fw_hard_reset(pvr_dev);
542				if (err)
543					goto err_device_lost;
544
545				err = pvr_power_device_resume(from_pvr_device(pvr_dev)->dev);
546				pvr_dev->fw_dev.booted = true;
547				if (err)
548					goto err_device_lost;
549			} else {
550				/* Clear the FW faulted flags. */
551				pvr_dev->fw_dev.fwif_sysdata->hwr_state_flags &=
552					~(ROGUE_FWIF_HWR_FW_FAULT |
553					  ROGUE_FWIF_HWR_RESTART_REQUESTED);
554			}
555
556			pvr_fw_irq_clear(pvr_dev);
557
558			err = pvr_power_fw_enable(pvr_dev);
559		}
560
561		if (err && hard_reset)
562			goto err_device_lost;
563
564		if (err && !hard_reset) {
565			drm_err(from_pvr_device(pvr_dev), "FW stalled, trying hard reset");
566			hard_reset = true;
567		}
568	} while (err);
569
570	if (queues_disabled)
571		pvr_queue_device_post_reset(pvr_dev);
572
573	enable_irq(pvr_dev->irq);
574
575	up_write(&pvr_dev->reset_sem);
576
577	pvr_power_put(pvr_dev);
578
579	return 0;
580
581err_device_lost:
582	drm_err(from_pvr_device(pvr_dev), "GPU device lost");
583	pvr_device_lost(pvr_dev);
584
585	/* Leave IRQs disabled if the device is lost. */
586
587	if (queues_disabled)
588		pvr_queue_device_post_reset(pvr_dev);
589
590err_up_write:
591	up_write(&pvr_dev->reset_sem);
592
593	pvr_power_put(pvr_dev);
594
595	return err;
596}
597
598/**
599 * pvr_watchdog_fini() - Shutdown watchdog for device
600 * @pvr_dev: Target PowerVR device.
601 */
602void
603pvr_watchdog_fini(struct pvr_device *pvr_dev)
604{
605	cancel_delayed_work_sync(&pvr_dev->watchdog.work);
606}
607
608int pvr_power_domains_init(struct pvr_device *pvr_dev)
609{
610	struct device *dev = from_pvr_device(pvr_dev)->dev;
611
612	struct device_link **domain_links __free(kfree) = NULL;
613	struct device **domain_devs __free(kfree) = NULL;
614	int domain_count;
615	int link_count;
616
617	char dev_name[2] = "a";
618	int err;
619	int i;
620
621	domain_count = of_count_phandle_with_args(dev->of_node, "power-domains",
622						  "#power-domain-cells");
623	if (domain_count < 0)
624		return domain_count;
625
626	if (domain_count <= 1)
627		return 0;
628
629	link_count = domain_count + (domain_count - 1);
630
631	domain_devs = kcalloc(domain_count, sizeof(*domain_devs), GFP_KERNEL);
632	if (!domain_devs)
633		return -ENOMEM;
634
635	domain_links = kcalloc(link_count, sizeof(*domain_links), GFP_KERNEL);
636	if (!domain_links)
637		return -ENOMEM;
638
639	for (i = 0; i < domain_count; i++) {
640		struct device *domain_dev;
641
642		dev_name[0] = 'a' + i;
643		domain_dev = dev_pm_domain_attach_by_name(dev, dev_name);
644		if (IS_ERR_OR_NULL(domain_dev)) {
645			err = domain_dev ? PTR_ERR(domain_dev) : -ENODEV;
646			goto err_detach;
647		}
648
649		domain_devs[i] = domain_dev;
650	}
651
652	for (i = 0; i < domain_count; i++) {
653		struct device_link *link;
654
655		link = device_link_add(dev, domain_devs[i], DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
656		if (!link) {
657			err = -ENODEV;
658			goto err_unlink;
659		}
660
661		domain_links[i] = link;
662	}
663
664	for (i = domain_count; i < link_count; i++) {
665		struct device_link *link;
666
667		link = device_link_add(domain_devs[i - domain_count + 1],
668				       domain_devs[i - domain_count],
669				       DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
670		if (!link) {
671			err = -ENODEV;
672			goto err_unlink;
673		}
674
675		domain_links[i] = link;
676	}
677
678	pvr_dev->power = (struct pvr_device_power){
679		.domain_devs = no_free_ptr(domain_devs),
680		.domain_links = no_free_ptr(domain_links),
681		.domain_count = domain_count,
682	};
683
684	return 0;
685
686err_unlink:
687	while (--i >= 0)
688		device_link_del(domain_links[i]);
689
690	i = domain_count;
691
692err_detach:
693	while (--i >= 0)
694		dev_pm_domain_detach(domain_devs[i], true);
695
696	return err;
697}
698
699void pvr_power_domains_fini(struct pvr_device *pvr_dev)
700{
701	const int domain_count = pvr_dev->power.domain_count;
702
703	int i = domain_count + (domain_count - 1);
704
705	while (--i >= 0)
706		device_link_del(pvr_dev->power.domain_links[i]);
707
708	i = domain_count;
709
710	while (--i >= 0)
711		dev_pm_domain_detach(pvr_dev->power.domain_devs[i], true);
712
713	kfree(pvr_dev->power.domain_links);
714	kfree(pvr_dev->power.domain_devs);
715
716	pvr_dev->power = (struct pvr_device_power){ 0 };
717}