panfrost_device.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
  3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
  4
  5#include <linux/clk.h>
  6#include <linux/reset.h>
  7#include <linux/platform_device.h>
  8#include <linux/pm_domain.h>
  9#include <linux/pm_runtime.h>
 10#include <linux/regulator/consumer.h>
 11
 12#include "panfrost_device.h"
 13#include "panfrost_devfreq.h"
 14#include "panfrost_features.h"
 15#include "panfrost_issues.h"
 16#include "panfrost_gpu.h"
 17#include "panfrost_job.h"
 18#include "panfrost_mmu.h"
 19#include "panfrost_perfcnt.h"
 20
 21static int panfrost_reset_init(struct panfrost_device *pfdev)
 22{
 23	pfdev->rstc = devm_reset_control_array_get_optional_exclusive(pfdev->base.dev);
 24	if (IS_ERR(pfdev->rstc)) {
 25		dev_err(pfdev->base.dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc));
 26		return PTR_ERR(pfdev->rstc);
 27	}
 28
 29	return reset_control_deassert(pfdev->rstc);
 30}
 31
 32static void panfrost_reset_fini(struct panfrost_device *pfdev)
 33{
 34	reset_control_assert(pfdev->rstc);
 35}
 36
 37static int panfrost_clk_init(struct panfrost_device *pfdev)
 38{
 39	int err;
 40	unsigned long rate;
 41
 42	pfdev->clock = devm_clk_get(pfdev->base.dev, NULL);
 43	if (IS_ERR(pfdev->clock)) {
 44		dev_err(pfdev->base.dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock));
 45		return PTR_ERR(pfdev->clock);
 46	}
 47
 48	rate = clk_get_rate(pfdev->clock);
 49	dev_info(pfdev->base.dev, "clock rate = %lu\n", rate);
 50
 51	err = clk_prepare_enable(pfdev->clock);
 52	if (err)
 53		return err;
 54
 55	pfdev->bus_clock = devm_clk_get_optional(pfdev->base.dev, "bus");
 56	if (IS_ERR(pfdev->bus_clock)) {
 57		dev_err(pfdev->base.dev, "get bus_clock failed %ld\n",
 58			PTR_ERR(pfdev->bus_clock));
 59		err = PTR_ERR(pfdev->bus_clock);
 60		goto disable_clock;
 61	}
 62
 63	if (pfdev->bus_clock) {
 64		rate = clk_get_rate(pfdev->bus_clock);
 65		dev_info(pfdev->base.dev, "bus_clock rate = %lu\n", rate);
 66
 67		err = clk_prepare_enable(pfdev->bus_clock);
 68		if (err)
 69			goto disable_clock;
 70	}
 71
 72	return 0;
 73
 74disable_clock:
 75	clk_disable_unprepare(pfdev->clock);
 76
 77	return err;
 78}
 79
 80static void panfrost_clk_fini(struct panfrost_device *pfdev)
 81{
 82	clk_disable_unprepare(pfdev->bus_clock);
 83	clk_disable_unprepare(pfdev->clock);
 84}
 85
 86static int panfrost_regulator_init(struct panfrost_device *pfdev)
 87{
 88	int ret, i;
 89
 90	pfdev->regulators = devm_kcalloc(pfdev->base.dev, pfdev->comp->num_supplies,
 91					 sizeof(*pfdev->regulators),
 92					 GFP_KERNEL);
 93	if (!pfdev->regulators)
 94		return -ENOMEM;
 95
 96	for (i = 0; i < pfdev->comp->num_supplies; i++)
 97		pfdev->regulators[i].supply = pfdev->comp->supply_names[i];
 98
 99	ret = devm_regulator_bulk_get(pfdev->base.dev,
100				      pfdev->comp->num_supplies,
101				      pfdev->regulators);
102	if (ret < 0) {
103		if (ret != -EPROBE_DEFER)
104			dev_err(pfdev->base.dev, "failed to get regulators: %d\n",
105				ret);
106		return ret;
107	}
108
109	ret = regulator_bulk_enable(pfdev->comp->num_supplies,
110				    pfdev->regulators);
111	if (ret < 0) {
112		dev_err(pfdev->base.dev, "failed to enable regulators: %d\n", ret);
113		return ret;
114	}
115
116	return 0;
117}
118
119static void panfrost_regulator_fini(struct panfrost_device *pfdev)
120{
121	if (!pfdev->regulators)
122		return;
123
124	regulator_bulk_disable(pfdev->comp->num_supplies, pfdev->regulators);
125}
126
127static void panfrost_pm_domain_fini(struct panfrost_device *pfdev)
128{
129	int i;
130
131	for (i = 0; i < ARRAY_SIZE(pfdev->pm_domain_devs); i++) {
132		if (!pfdev->pm_domain_devs[i])
133			break;
134
135		if (pfdev->pm_domain_links[i])
136			device_link_del(pfdev->pm_domain_links[i]);
137
138		dev_pm_domain_detach(pfdev->pm_domain_devs[i], true);
139	}
140}
141
142static int panfrost_pm_domain_init(struct panfrost_device *pfdev)
143{
144	int err;
145	int i, num_domains;
146
147	num_domains = of_count_phandle_with_args(pfdev->base.dev->of_node,
148						 "power-domains",
149						 "#power-domain-cells");
150
151	/*
152	 * Single domain is handled by the core, and, if only a single power
153	 * the power domain is requested, the property is optional.
154	 */
155	if (num_domains < 2 && pfdev->comp->num_pm_domains < 2)
156		return 0;
157
158	if (num_domains != pfdev->comp->num_pm_domains) {
159		dev_err(pfdev->base.dev,
160			"Incorrect number of power domains: %d provided, %d needed\n",
161			num_domains, pfdev->comp->num_pm_domains);
162		return -EINVAL;
163	}
164
165	if (WARN(num_domains > ARRAY_SIZE(pfdev->pm_domain_devs),
166			"Too many supplies in compatible structure.\n"))
167		return -EINVAL;
168
169	for (i = 0; i < num_domains; i++) {
170		pfdev->pm_domain_devs[i] =
171			dev_pm_domain_attach_by_name(pfdev->base.dev,
172						     pfdev->comp->pm_domain_names[i]);
173		if (IS_ERR_OR_NULL(pfdev->pm_domain_devs[i])) {
174			err = PTR_ERR(pfdev->pm_domain_devs[i]) ? : -ENODATA;
175			pfdev->pm_domain_devs[i] = NULL;
176			dev_err(pfdev->base.dev,
177				"failed to get pm-domain %s(%d): %d\n",
178				pfdev->comp->pm_domain_names[i], i, err);
179			goto err;
180		}
181
182		pfdev->pm_domain_links[i] =
183			device_link_add(pfdev->base.dev,
184					pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME |
185					DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE);
186		if (!pfdev->pm_domain_links[i]) {
187			dev_err(pfdev->pm_domain_devs[i],
188				"adding device link failed!\n");
189			err = -ENODEV;
190			goto err;
191		}
192	}
193
194	return 0;
195
196err:
197	panfrost_pm_domain_fini(pfdev);
198	return err;
199}
200
201int panfrost_device_init(struct panfrost_device *pfdev)
202{
203	int err;
204
205	mutex_init(&pfdev->sched_lock);
206	INIT_LIST_HEAD(&pfdev->scheduled_jobs);
207	INIT_LIST_HEAD(&pfdev->as_lru_list);
208
209	spin_lock_init(&pfdev->as_lock);
210
211	spin_lock_init(&pfdev->cycle_counter.lock);
212
213#ifdef CONFIG_DEBUG_FS
214	mutex_init(&pfdev->debugfs.gems_lock);
215	INIT_LIST_HEAD(&pfdev->debugfs.gems_list);
216#endif
217
218	err = panfrost_pm_domain_init(pfdev);
219	if (err)
220		return err;
221
222	err = panfrost_reset_init(pfdev);
223	if (err) {
224		dev_err(pfdev->base.dev, "reset init failed %d\n", err);
225		goto out_pm_domain;
226	}
227
228	err = panfrost_clk_init(pfdev);
229	if (err) {
230		dev_err(pfdev->base.dev, "clk init failed %d\n", err);
231		goto out_reset;
232	}
233
234	err = panfrost_devfreq_init(pfdev);
235	if (err) {
236		if (err != -EPROBE_DEFER)
237			dev_err(pfdev->base.dev, "devfreq init failed %d\n", err);
238		goto out_clk;
239	}
240
241	/* OPP will handle regulators */
242	if (!pfdev->pfdevfreq.opp_of_table_added) {
243		err = panfrost_regulator_init(pfdev);
244		if (err)
245			goto out_devfreq;
246	}
247
248	pfdev->iomem = devm_platform_ioremap_resource(to_platform_device(pfdev->base.dev), 0);
249	if (IS_ERR(pfdev->iomem)) {
250		err = PTR_ERR(pfdev->iomem);
251		goto out_regulator;
252	}
253
254	err = panfrost_gpu_init(pfdev);
255	if (err)
256		goto out_regulator;
257
258	err = panfrost_mmu_init(pfdev);
259	if (err)
260		goto out_gpu;
261
262	err = panfrost_jm_init(pfdev);
263	if (err)
264		goto out_mmu;
265
266	err = panfrost_perfcnt_init(pfdev);
267	if (err)
268		goto out_job;
269
270	return 0;
271out_job:
272	panfrost_jm_fini(pfdev);
273out_mmu:
274	panfrost_mmu_fini(pfdev);
275out_gpu:
276	panfrost_gpu_fini(pfdev);
277out_regulator:
278	panfrost_regulator_fini(pfdev);
279out_devfreq:
280	panfrost_devfreq_fini(pfdev);
281out_clk:
282	panfrost_clk_fini(pfdev);
283out_reset:
284	panfrost_reset_fini(pfdev);
285out_pm_domain:
286	panfrost_pm_domain_fini(pfdev);
287	return err;
288}
289
290void panfrost_device_fini(struct panfrost_device *pfdev)
291{
292	panfrost_perfcnt_fini(pfdev);
293	panfrost_jm_fini(pfdev);
294	panfrost_mmu_fini(pfdev);
295	panfrost_gpu_fini(pfdev);
296	panfrost_devfreq_fini(pfdev);
297	panfrost_regulator_fini(pfdev);
298	panfrost_clk_fini(pfdev);
299	panfrost_reset_fini(pfdev);
300	panfrost_pm_domain_fini(pfdev);
301}
302
303#define PANFROST_EXCEPTION(id) \
304	[DRM_PANFROST_EXCEPTION_ ## id] = { \
305		.name = #id, \
306	}
307
308struct panfrost_exception_info {
309	const char *name;
310};
311
312static const struct panfrost_exception_info panfrost_exception_infos[] = {
313	PANFROST_EXCEPTION(OK),
314	PANFROST_EXCEPTION(DONE),
315	PANFROST_EXCEPTION(INTERRUPTED),
316	PANFROST_EXCEPTION(STOPPED),
317	PANFROST_EXCEPTION(TERMINATED),
318	PANFROST_EXCEPTION(KABOOM),
319	PANFROST_EXCEPTION(EUREKA),
320	PANFROST_EXCEPTION(ACTIVE),
321	PANFROST_EXCEPTION(JOB_CONFIG_FAULT),
322	PANFROST_EXCEPTION(JOB_POWER_FAULT),
323	PANFROST_EXCEPTION(JOB_READ_FAULT),
324	PANFROST_EXCEPTION(JOB_WRITE_FAULT),
325	PANFROST_EXCEPTION(JOB_AFFINITY_FAULT),
326	PANFROST_EXCEPTION(JOB_BUS_FAULT),
327	PANFROST_EXCEPTION(INSTR_INVALID_PC),
328	PANFROST_EXCEPTION(INSTR_INVALID_ENC),
329	PANFROST_EXCEPTION(INSTR_TYPE_MISMATCH),
330	PANFROST_EXCEPTION(INSTR_OPERAND_FAULT),
331	PANFROST_EXCEPTION(INSTR_TLS_FAULT),
332	PANFROST_EXCEPTION(INSTR_BARRIER_FAULT),
333	PANFROST_EXCEPTION(INSTR_ALIGN_FAULT),
334	PANFROST_EXCEPTION(DATA_INVALID_FAULT),
335	PANFROST_EXCEPTION(TILE_RANGE_FAULT),
336	PANFROST_EXCEPTION(ADDR_RANGE_FAULT),
337	PANFROST_EXCEPTION(IMPRECISE_FAULT),
338	PANFROST_EXCEPTION(OOM),
339	PANFROST_EXCEPTION(OOM_AFBC),
340	PANFROST_EXCEPTION(UNKNOWN),
341	PANFROST_EXCEPTION(DELAYED_BUS_FAULT),
342	PANFROST_EXCEPTION(GPU_SHAREABILITY_FAULT),
343	PANFROST_EXCEPTION(SYS_SHAREABILITY_FAULT),
344	PANFROST_EXCEPTION(GPU_CACHEABILITY_FAULT),
345	PANFROST_EXCEPTION(TRANSLATION_FAULT_0),
346	PANFROST_EXCEPTION(TRANSLATION_FAULT_1),
347	PANFROST_EXCEPTION(TRANSLATION_FAULT_2),
348	PANFROST_EXCEPTION(TRANSLATION_FAULT_3),
349	PANFROST_EXCEPTION(TRANSLATION_FAULT_4),
350	PANFROST_EXCEPTION(TRANSLATION_FAULT_IDENTITY),
351	PANFROST_EXCEPTION(PERM_FAULT_0),
352	PANFROST_EXCEPTION(PERM_FAULT_1),
353	PANFROST_EXCEPTION(PERM_FAULT_2),
354	PANFROST_EXCEPTION(PERM_FAULT_3),
355	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_0),
356	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_1),
357	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_2),
358	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_3),
359	PANFROST_EXCEPTION(ACCESS_FLAG_0),
360	PANFROST_EXCEPTION(ACCESS_FLAG_1),
361	PANFROST_EXCEPTION(ACCESS_FLAG_2),
362	PANFROST_EXCEPTION(ACCESS_FLAG_3),
363	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN0),
364	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN1),
365	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN2),
366	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN3),
367	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
368	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
369	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
370	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
371	PANFROST_EXCEPTION(MEM_ATTR_FAULT_0),
372	PANFROST_EXCEPTION(MEM_ATTR_FAULT_1),
373	PANFROST_EXCEPTION(MEM_ATTR_FAULT_2),
374	PANFROST_EXCEPTION(MEM_ATTR_FAULT_3),
375	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_0),
376	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_1),
377	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_2),
378	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_3),
379};
380
381const char *panfrost_exception_name(u32 exception_code)
382{
383	if (WARN_ON(exception_code >= ARRAY_SIZE(panfrost_exception_infos) ||
384		    !panfrost_exception_infos[exception_code].name))
385		return "Unknown exception type";
386
387	return panfrost_exception_infos[exception_code].name;
388}
389
390bool panfrost_exception_needs_reset(const struct panfrost_device *pfdev,
391				    u32 exception_code)
392{
393	/* If an occlusion query write causes a bus fault on affected GPUs,
394	 * future fragment jobs may hang. Reset to workaround.
395	 */
396	if (exception_code == DRM_PANFROST_EXCEPTION_JOB_BUS_FAULT)
397		return panfrost_has_hw_issue(pfdev, HW_ISSUE_TTRX_3076);
398
399	/* No other GPUs we support need a reset */
400	return false;
401}
402
403void panfrost_device_reset(struct panfrost_device *pfdev, bool enable_job_int)
404{
405	panfrost_gpu_soft_reset(pfdev);
406
407	panfrost_gpu_power_on(pfdev);
408	panfrost_mmu_reset(pfdev);
409
410	panfrost_jm_reset_interrupts(pfdev);
411	if (enable_job_int)
412		panfrost_jm_enable_interrupts(pfdev);
413}
414
415static int panfrost_device_runtime_resume(struct device *dev)
416{
417	struct panfrost_device *pfdev = dev_get_drvdata(dev);
418	int ret;
419
420	if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) {
421		ret = reset_control_deassert(pfdev->rstc);
422		if (ret)
423			return ret;
424
425		ret = clk_enable(pfdev->clock);
426		if (ret)
427			goto err_clk;
428
429		if (pfdev->bus_clock) {
430			ret = clk_enable(pfdev->bus_clock);
431			if (ret)
432				goto err_bus_clk;
433		}
434	}
435
436	panfrost_device_reset(pfdev, true);
437	panfrost_devfreq_resume(pfdev);
438
439	return 0;
440
441err_bus_clk:
442	if (pfdev->comp->pm_features & BIT(GPU_PM_RT))
443		clk_disable(pfdev->clock);
444err_clk:
445	if (pfdev->comp->pm_features & BIT(GPU_PM_RT))
446		reset_control_assert(pfdev->rstc);
447	return ret;
448}
449
450static int panfrost_device_runtime_suspend(struct device *dev)
451{
452	struct panfrost_device *pfdev = dev_get_drvdata(dev);
453
454	if (!panfrost_jm_is_idle(pfdev))
455		return -EBUSY;
456
457	panfrost_devfreq_suspend(pfdev);
458	panfrost_jm_suspend_irq(pfdev);
459	panfrost_mmu_suspend_irq(pfdev);
460	panfrost_gpu_suspend_irq(pfdev);
461	panfrost_gpu_power_off(pfdev);
462
463	if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) {
464		if (pfdev->bus_clock)
465			clk_disable(pfdev->bus_clock);
466
467		clk_disable(pfdev->clock);
468		reset_control_assert(pfdev->rstc);
469	}
470
471	return 0;
472}
473
474static int panfrost_device_resume(struct device *dev)
475{
476	struct panfrost_device *pfdev = dev_get_drvdata(dev);
477	int ret;
478
479	if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) {
480		unsigned long freq = pfdev->pfdevfreq.fast_rate;
481		struct dev_pm_opp *opp;
482
483		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
484		if (IS_ERR(opp))
485			return PTR_ERR(opp);
486		dev_pm_opp_set_opp(dev, opp);
487		dev_pm_opp_put(opp);
488	}
489
490	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
491		ret = clk_enable(pfdev->clock);
492		if (ret)
493			goto err_clk;
494
495		if (pfdev->bus_clock) {
496			ret = clk_enable(pfdev->bus_clock);
497			if (ret)
498				goto err_bus_clk;
499		}
500	}
501
502	ret = pm_runtime_force_resume(dev);
503	if (ret)
504		goto err_resume;
505
506	return 0;
507
508err_resume:
509	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS) && pfdev->bus_clock)
510		clk_disable(pfdev->bus_clock);
511err_bus_clk:
512	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS))
513		clk_disable(pfdev->clock);
514err_clk:
515	if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
516		dev_pm_opp_set_opp(dev, NULL);
517	return ret;
518}
519
520static int panfrost_device_suspend(struct device *dev)
521{
522	struct panfrost_device *pfdev = dev_get_drvdata(dev);
523	int ret;
524
525	ret = pm_runtime_force_suspend(dev);
526	if (ret)
527		return ret;
528
529	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
530		if (pfdev->bus_clock)
531			clk_disable(pfdev->bus_clock);
532
533		clk_disable(pfdev->clock);
534	}
535
536	if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
537		dev_pm_opp_set_opp(dev, NULL);
538
539	return 0;
540}
541
542EXPORT_GPL_DEV_PM_OPS(panfrost_pm_ops) = {
543	RUNTIME_PM_OPS(panfrost_device_runtime_suspend, panfrost_device_runtime_resume, NULL)
544	SYSTEM_SLEEP_PM_OPS(panfrost_device_suspend, panfrost_device_resume)
545};