msm_gpu.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2013 Red Hat
  4 * Author: Rob Clark <robdclark@gmail.com>
  5 */
  6
  7#ifndef __MSM_GPU_H__
  8#define __MSM_GPU_H__
  9
 10#include <linux/adreno-smmu-priv.h>
 11#include <linux/clk.h>
 12#include <linux/devfreq.h>
 13#include <linux/interconnect.h>
 14#include <linux/pm_opp.h>
 15#include <linux/regulator/consumer.h>
 16
 17#include "msm_drv.h"
 18#include "msm_fence.h"
 19#include "msm_gpu_trace.h"
 20#include "msm_ringbuffer.h"
 21#include "msm_gem.h"
 22
 23struct msm_gem_submit;
 24struct msm_gem_vm_log_entry;
 25struct msm_gpu_perfcntr;
 26struct msm_gpu_state;
 27struct msm_context;
 28
 29struct msm_gpu_config {
 30	const char *ioname;
 31	unsigned int nr_rings;
 32};
 33
 34/* So far, with hardware that I've seen to date, we can have:
 35 *  + zero, one, or two z180 2d cores
 36 *  + a3xx or a2xx 3d core, which share a common CP (the firmware
 37 *    for the CP seems to implement some different PM4 packet types
 38 *    but the basics of cmdstream submission are the same)
 39 *
 40 * Which means that the eventual complete "class" hierarchy, once
 41 * support for all past and present hw is in place, becomes:
 42 *  + msm_gpu
 43 *    + adreno_gpu
 44 *      + a3xx_gpu
 45 *      + a2xx_gpu
 46 *    + z180_gpu
 47 */
 48struct msm_gpu_funcs {
 49	int (*get_param)(struct msm_gpu *gpu, struct msm_context *ctx,
 50			 uint32_t param, uint64_t *value, uint32_t *len);
 51	int (*set_param)(struct msm_gpu *gpu, struct msm_context *ctx,
 52			 uint32_t param, uint64_t value, uint32_t len);
 53	int (*hw_init)(struct msm_gpu *gpu);
 54
 55	/**
 56	 * @ucode_load: Optional hook to upload fw to GEM objs
 57	 */
 58	int (*ucode_load)(struct msm_gpu *gpu);
 59
 60	int (*pm_suspend)(struct msm_gpu *gpu);
 61	int (*pm_resume)(struct msm_gpu *gpu);
 62	void (*submit)(struct msm_gpu *gpu, struct msm_gem_submit *submit);
 63	void (*flush)(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
 64	irqreturn_t (*irq)(struct msm_gpu *irq);
 65	struct msm_ringbuffer *(*active_ring)(struct msm_gpu *gpu);
 66	void (*recover)(struct msm_gpu *gpu);
 67	void (*destroy)(struct msm_gpu *gpu);
 68#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
 69	/* show GPU status in debugfs: */
 70	void (*show)(struct msm_gpu *gpu, struct msm_gpu_state *state,
 71			struct drm_printer *p);
 72	/* for generation specific debugfs: */
 73	void (*debugfs_init)(struct msm_gpu *gpu, struct drm_minor *minor);
 74#endif
 75	/* note: gpu_busy() can assume that we have been pm_resumed */
 76	u64 (*gpu_busy)(struct msm_gpu *gpu, unsigned long *out_sample_rate);
 77	struct msm_gpu_state *(*gpu_state_get)(struct msm_gpu *gpu);
 78	int (*gpu_state_put)(struct msm_gpu_state *state);
 79	unsigned long (*gpu_get_freq)(struct msm_gpu *gpu);
 80	/* note: gpu_set_freq() can assume that we have been pm_resumed */
 81	void (*gpu_set_freq)(struct msm_gpu *gpu, struct dev_pm_opp *opp,
 82			     bool suspended);
 83	struct drm_gpuvm *(*create_vm)(struct msm_gpu *gpu, struct platform_device *pdev);
 84	struct drm_gpuvm *(*create_private_vm)(struct msm_gpu *gpu, bool kernel_managed);
 85	uint32_t (*get_rptr)(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
 86
 87	/**
 88	 * progress: Has the GPU made progress?
 89	 *
 90	 * Return true if GPU position in cmdstream has advanced (or changed)
 91	 * since the last call.  To avoid false negatives, this should account
 92	 * for cmdstream that is buffered in this FIFO upstream of the CP fw.
 93	 */
 94	bool (*progress)(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
 95	void (*sysprof_setup)(struct msm_gpu *gpu);
 96};
 97
 98/* Additional state for iommu faults: */
 99struct msm_gpu_fault_info {
100	u64 ttbr0;
101	unsigned long iova;
102	int flags;
103	const char *type;
104	const char *block;
105
106	/* Information about what we think/expect is the current SMMU state,
107	 * for example expected_ttbr0 should match smmu_info.ttbr0 which
108	 * was read back from SMMU registers.
109	 */
110	phys_addr_t pgtbl_ttbr0;
111	u64 ptes[4];
112	int asid;
113};
114
115/**
116 * struct msm_gpu_devfreq - devfreq related state
117 */
118struct msm_gpu_devfreq {
119	/** @devfreq: devfreq instance */
120	struct devfreq *devfreq;
121	/** @lock: lock for "suspended", "busy_cycles", and "time" */
122	struct mutex lock;
123	/**
124	 * @idle_freq:
125	 * Shadow frequency used while the GPU is idle.  From the PoV of
126	 * the devfreq governor, we are continuing to sample busyness and
127	 * adjust frequency while the GPU is idle, but we use this shadow
128	 * value as the GPU is actually clamped to minimum frequency while
129	 * it is inactive.
130	 */
131	unsigned long idle_freq;
132	/**
133	 * @boost_freq:
134	 * A PM QoS constraint to boost min freq for a period of time
135	 * until the boost expires.
136	 */
137	struct dev_pm_qos_request boost_freq;
138	/**
139	 * @busy_cycles: Last busy counter value, for calculating elapsed busy
140	 * cycles since last sampling period.
141	 */
142	u64 busy_cycles;
143	/** @time: Time of last sampling period.  */
144	ktime_t time;
145	/** @idle_time: Time of last transition to idle.  */
146	ktime_t idle_time;
147	/**
148	 * @idle_work:
149	 * Used to delay clamping to idle freq on active->idle transition.
150	 */
151	struct msm_hrtimer_work idle_work;
152	/**
153	 * @boost_work:
154	 * Used to reset the boost_constraint after the boost period has
155	 * elapsed
156	 */
157	struct msm_hrtimer_work boost_work;
158
159	/** @suspended: tracks if we're suspended */
160	bool suspended;
161};
162
163struct msm_gpu {
164	const char *name;
165	struct drm_device *dev;
166	struct platform_device *pdev;
167	const struct msm_gpu_funcs *funcs;
168
169	struct adreno_smmu_priv adreno_smmu;
170
171	/* performance counters (hw & sw): */
172	spinlock_t perf_lock;
173	bool perfcntr_active;
174	struct {
175		bool active;
176		ktime_t time;
177	} last_sample;
178	uint32_t totaltime, activetime;    /* sw counters */
179	uint32_t last_cntrs[5];            /* hw counters */
180	const struct msm_gpu_perfcntr *perfcntrs;
181	uint32_t num_perfcntrs;
182
183	struct msm_ringbuffer *rb[MSM_GPU_MAX_RINGS];
184	int nr_rings;
185
186	/**
187	 * sysprof_active:
188	 *
189	 * The count of contexts that have enabled system profiling.
190	 */
191	refcount_t sysprof_active;
192
193	/**
194	 * lock:
195	 *
196	 * General lock for serializing all the gpu things.
197	 *
198	 * TODO move to per-ring locking where feasible (ie. submit/retire
199	 * path, etc)
200	 */
201	struct mutex lock;
202
203	/**
204	 * active_submits:
205	 *
206	 * The number of submitted but not yet retired submits, used to
207	 * determine transitions between active and idle.
208	 *
209	 * Protected by active_lock
210	 */
211	int active_submits;
212
213	/** lock: protects active_submits and idle/active transitions */
214	struct mutex active_lock;
215
216	/* does gpu need hw_init? */
217	bool needs_hw_init;
218
219	/**
220	 * global_faults: number of GPU hangs not attributed to a particular
221	 * address space
222	 */
223	int global_faults;
224
225	void __iomem *mmio;
226	int irq;
227
228	struct drm_gpuvm *vm;
229
230	/* Power Control: */
231	struct regulator *gpu_reg, *gpu_cx;
232	struct clk_bulk_data *grp_clks;
233	int nr_clocks;
234	struct clk *ebi1_clk, *core_clk, *rbbmtimer_clk;
235	uint32_t fast_rate;
236
237	/* Hang and Inactivity Detection:
238	 */
239#define DRM_MSM_INACTIVE_PERIOD   66 /* in ms (roughly four frames) */
240
241#define DRM_MSM_HANGCHECK_DEFAULT_PERIOD 500 /* in ms */
242#define DRM_MSM_HANGCHECK_PROGRESS_RETRIES 3
243	struct timer_list hangcheck_timer;
244
245	/* work for handling GPU recovery: */
246	struct kthread_work recover_work;
247
248	/** retire_event: notified when submits are retired: */
249	wait_queue_head_t retire_event;
250
251	/* work for handling active-list retiring: */
252	struct kthread_work retire_work;
253
254	/* worker for retire/recover: */
255	struct kthread_worker *worker;
256
257	struct drm_gem_object *memptrs_bo;
258
259	struct msm_gpu_devfreq devfreq;
260
261	uint32_t suspend_count;
262
263	struct msm_gpu_state *crashstate;
264
265	/* True if the hardware supports expanded apriv (a650 and newer) */
266	bool hw_apriv;
267
268	/**
269	 * @allow_relocs: allow relocs in SUBMIT ioctl
270	 *
271	 * Mesa won't use relocs for driver version 1.4.0 and later.  This
272	 * switch-over happened early enough in mesa a6xx bringup that we
273	 * can disallow relocs for a6xx and newer.
274	 */
275	bool allow_relocs;
276
277	struct thermal_cooling_device *cooling;
278};
279
280static inline struct msm_gpu *dev_to_gpu(struct device *dev)
281{
282	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev);
283
284	if (!adreno_smmu)
285		return NULL;
286
287	return container_of(adreno_smmu, struct msm_gpu, adreno_smmu);
288}
289
290static inline bool
291adreno_smmu_has_prr(struct msm_gpu *gpu)
292{
293	struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(&gpu->pdev->dev);
294
295	if (!adreno_smmu)
296		return false;
297
298	return adreno_smmu && adreno_smmu->set_prr_addr;
299}
300
301/* It turns out that all targets use the same ringbuffer size */
302#define MSM_GPU_RINGBUFFER_SZ SZ_32K
303#define MSM_GPU_RINGBUFFER_BLKSIZE 32
304
305#define MSM_GPU_RB_CNTL_DEFAULT \
306		(AXXX_CP_RB_CNTL_BUFSZ(ilog2(MSM_GPU_RINGBUFFER_SZ / 8)) | \
307		AXXX_CP_RB_CNTL_BLKSZ(ilog2(MSM_GPU_RINGBUFFER_BLKSIZE / 8)))
308
309static inline bool msm_gpu_active(struct msm_gpu *gpu)
310{
311	int i;
312
313	for (i = 0; i < gpu->nr_rings; i++) {
314		struct msm_ringbuffer *ring = gpu->rb[i];
315
316		if (fence_after(ring->fctx->last_fence, ring->memptrs->fence))
317			return true;
318	}
319
320	return false;
321}
322
323/* Perf-Counters:
324 * The select_reg and select_val are just there for the benefit of the child
325 * class that actually enables the perf counter..  but msm_gpu base class
326 * will handle sampling/displaying the counters.
327 */
328
329struct msm_gpu_perfcntr {
330	uint32_t select_reg;
331	uint32_t sample_reg;
332	uint32_t select_val;
333	const char *name;
334};
335
336/*
337 * The number of priority levels provided by drm gpu scheduler.  The
338 * DRM_SCHED_PRIORITY_KERNEL priority level is treated specially in some
339 * cases, so we don't use it (no need for kernel generated jobs).
340 */
341#define NR_SCHED_PRIORITIES (1 + DRM_SCHED_PRIORITY_LOW - DRM_SCHED_PRIORITY_HIGH)
342
343/**
344 * struct msm_context - per-drm_file context
345 */
346struct msm_context {
347	/** @queuelock: synchronizes access to submitqueues list */
348	rwlock_t queuelock;
349	/** @submitqueues: list of &msm_gpu_submitqueue created by userspace */
350	struct list_head submitqueues;
351	/**
352	 * @queueid:
353	 * Counter incremented each time a submitqueue is created, used to
354	 * assign &msm_gpu_submitqueue.id
355	 */
356	int queueid;
357	/**
358	 * @closed: The device file associated with this context has been closed.
359	 * Once the device is closed, any submits that have not been written
360	 * to the ring buffer are no-op'd.
361	 */
362	bool closed;
363	/**
364	 * @userspace_managed_vm:
365	 * Has userspace opted-in to userspace managed VM (ie. VM_BIND) via
366	 * MSM_PARAM_EN_VM_BIND?
367	 */
368	bool userspace_managed_vm;
369	/**
370	 * @vm:
371	 * The per-process GPU address-space.  Do not access directly, use
372	 * msm_context_vm().
373	 */
374	struct drm_gpuvm *vm;
375	/** @ref: the reference count */
376	struct kref ref;
377	/**
378	 * @seqno:
379	 * A unique per-process sequence number.  Used to detect context
380	 * switches, without relying on keeping a, potentially dangling,
381	 * pointer to the previous context.
382	 */
383	int seqno;
384	/**
385	 * @sysprof:
386	 * The value of MSM_PARAM_SYSPROF set by userspace.  This is
387	 * intended to be used by system profiling tools like Mesa's
388	 * pps-producer (perfetto), and restricted to CAP_SYS_ADMIN.
389	 *
390	 * Setting a value of 1 will preserve performance counters across
391	 * context switches.  Setting a value of 2 will in addition
392	 * suppress suspend.  (Performance counters lose state across
393	 * power collapse, which is undesirable for profiling in some
394	 * cases.)
395	 *
396	 * The value automatically reverts to zero when the drm device
397	 * file is closed.
398	 */
399	int sysprof;
400	/**
401	 * @comm: Overridden task comm, see MSM_PARAM_COMM
402	 *
403	 * Accessed under msm_gpu::lock
404	 */
405	char *comm;
406	/**
407	 * @cmdline: Overridden task cmdline, see MSM_PARAM_CMDLINE
408	 *
409	 * Accessed under msm_gpu::lock
410	 */
411	char *cmdline;
412	/**
413	 * @elapsed_ns:
414	 * The total (cumulative) elapsed time GPU was busy with rendering
415	 * from this context in ns.
416	 */
417	uint64_t elapsed_ns;
418	/**
419	 * @cycles:
420	 * The total (cumulative) GPU cycles elapsed attributed to this
421	 * context.
422	 */
423	uint64_t cycles;
424	/**
425	 * @entities:
426	 * Table of per-priority-level sched entities used by submitqueues
427	 * associated with this &drm_file.  Because some userspace apps
428	 * make assumptions about rendering from multiple gl contexts
429	 * (of the same priority) within the process happening in FIFO
430	 * order without requiring any fencing beyond MakeCurrent(), we
431	 * create at most one &drm_sched_entity per-process per-priority-
432	 * level.
433	 */
434	struct drm_sched_entity *entities[NR_SCHED_PRIORITIES * MSM_GPU_MAX_RINGS];
435	/**
436	 * @ctx_mem:
437	 * Total amount of memory of GEM buffers with handles attached for
438	 * this context.
439	 */
440	atomic64_t ctx_mem;
441};
442
443struct drm_gpuvm *msm_context_vm(struct drm_device *dev, struct msm_context *ctx);
444
445/**
446 * msm_context_is_vmbind() - has userspace opted in to VM_BIND?
447 *
448 * @ctx: the drm_file context
449 *
450 * See MSM_PARAM_EN_VM_BIND.  If userspace is managing the VM, it can
451 * do sparse binding including having multiple, potentially partial,
452 * mappings in the VM.  Therefore certain legacy uabi (ie. GET_IOVA,
453 * SET_IOVA) are rejected because they don't have a sensible meaning.
454 *
455 * Returns: %true if userspace is managing the VM, %false otherwise.
456 */
457static inline bool
458msm_context_is_vmbind(struct msm_context *ctx)
459{
460	return ctx->userspace_managed_vm;
461}
462
463/**
464 * msm_gpu_convert_priority - Map userspace priority to ring # and sched priority
465 *
466 * @gpu:        the gpu instance
467 * @prio:       the userspace priority level
468 * @ring_nr:    [out] the ringbuffer the userspace priority maps to
469 * @sched_prio: [out] the gpu scheduler priority level which the userspace
470 *              priority maps to
471 *
472 * With drm/scheduler providing it's own level of prioritization, our total
473 * number of available priority levels is (nr_rings * NR_SCHED_PRIORITIES).
474 * Each ring is associated with it's own scheduler instance.  However, our
475 * UABI is that lower numerical values are higher priority.  So mapping the
476 * single userspace priority level into ring_nr and sched_prio takes some
477 * care.  The userspace provided priority (when a submitqueue is created)
478 * is mapped to ring nr and scheduler priority as such:
479 *
480 *   ring_nr    = userspace_prio / NR_SCHED_PRIORITIES
481 *   sched_prio = NR_SCHED_PRIORITIES -
482 *                (userspace_prio % NR_SCHED_PRIORITIES) - 1
483 *
484 * This allows generations without preemption (nr_rings==1) to have some
485 * amount of prioritization, and provides more priority levels for gens
486 * that do have preemption.
487 *
488 * Returns: %0 on success, %-errno on error.
489 */
490static inline int msm_gpu_convert_priority(struct msm_gpu *gpu, int prio,
491		unsigned *ring_nr, enum drm_sched_priority *sched_prio)
492{
493	unsigned rn, sp;
494
495	rn = div_u64_rem(prio, NR_SCHED_PRIORITIES, &sp);
496
497	/* invert sched priority to map to higher-numeric-is-higher-
498	 * priority convention
499	 */
500	sp = NR_SCHED_PRIORITIES - sp - 1;
501
502	if (rn >= gpu->nr_rings)
503		return -EINVAL;
504
505	*ring_nr = rn;
506	*sched_prio = sp;
507
508	return 0;
509}
510
511/**
512 * struct msm_gpu_submitqueue - Userspace created context.
513 *
514 * A submitqueue is associated with a gl context or vk queue (or equiv)
515 * in userspace.
516 *
517 * @id:        userspace id for the submitqueue, unique within the drm_file
518 * @flags:     userspace flags for the submitqueue, specified at creation
519 *             (currently unusued)
520 * @ring_nr:   the ringbuffer used by this submitqueue, which is determined
521 *             by the submitqueue's priority
522 * @faults:    the number of GPU hangs associated with this submitqueue
523 * @last_fence: the sequence number of the last allocated fence (for error
524 *             checking)
525 * @ctx:       the per-drm_file context associated with the submitqueue (ie.
526 *             which set of pgtables do submits jobs associated with the
527 *             submitqueue use)
528 * @node:      node in the context's list of submitqueues
529 * @fence_idr: maps fence-id to dma_fence for userspace visible fence
530 *             seqno, protected by submitqueue lock
531 * @idr_lock:  for serializing access to fence_idr
532 * @lock:      submitqueue lock for serializing submits on a queue
533 * @ref:       reference count
534 * @entity:    the submit job-queue
535 */
536struct msm_gpu_submitqueue {
537	int id;
538	u32 flags;
539	u32 ring_nr;
540	int faults;
541	uint32_t last_fence;
542	struct msm_context *ctx;
543	struct list_head node;
544	struct idr fence_idr;
545	struct spinlock idr_lock;
546	struct mutex lock;
547	struct kref ref;
548	struct drm_sched_entity *entity;
549
550	/** @_vm_bind_entity: used for @entity pointer for VM_BIND queues */
551	struct drm_sched_entity _vm_bind_entity[0];
552};
553
554struct msm_gpu_state_bo {
555	u64 iova;
556	size_t size;
557	u32 flags;
558	void *data;
559	bool encoded;
560	char name[32];
561};
562
563struct msm_gpu_state {
564	struct kref ref;
565	struct timespec64 time;
566
567	struct {
568		u64 iova;
569		u32 fence;
570		u32 seqno;
571		u32 rptr;
572		u32 wptr;
573		void *data;
574		int data_size;
575		bool encoded;
576	} ring[MSM_GPU_MAX_RINGS];
577
578	int nr_registers;
579	u32 *registers;
580
581	u32 rbbm_status;
582
583	char *comm;
584	char *cmd;
585
586	struct msm_gpu_fault_info fault_info;
587
588	int nr_vm_logs;
589	struct msm_gem_vm_log_entry *vm_logs;
590
591	int nr_bos;
592	struct msm_gpu_state_bo *bos;
593};
594
595static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
596{
597	trace_msm_gpu_regaccess(reg);
598	writel(data, gpu->mmio + (reg << 2));
599}
600
601static inline u32 gpu_read(struct msm_gpu *gpu, u32 reg)
602{
603	trace_msm_gpu_regaccess(reg);
604	return readl(gpu->mmio + (reg << 2));
605}
606
607static inline void gpu_rmw(struct msm_gpu *gpu, u32 reg, u32 mask, u32 or)
608{
609	trace_msm_gpu_regaccess(reg);
610	msm_rmw(gpu->mmio + (reg << 2), mask, or);
611}
612
613static inline u64 gpu_read64(struct msm_gpu *gpu, u32 reg)
614{
615	u64 val;
616
617	/*
618	 * Why not a readq here? Two reasons: 1) many of the LO registers are
619	 * not quad word aligned and 2) the GPU hardware designers have a bit
620	 * of a history of putting registers where they fit, especially in
621	 * spins. The longer a GPU family goes the higher the chance that
622	 * we'll get burned.  We could do a series of validity checks if we
623	 * wanted to, but really is a readq() that much better? Nah.
624	 */
625
626	/*
627	 * For some lo/hi registers (like perfcounters), the hi value is latched
628	 * when the lo is read, so make sure to read the lo first to trigger
629	 * that
630	 */
631	trace_msm_gpu_regaccess(reg);
632	val = (u64) readl(gpu->mmio + (reg << 2));
633	trace_msm_gpu_regaccess(reg+1);
634	val |= ((u64) readl(gpu->mmio + ((reg + 1) << 2)) << 32);
635
636	return val;
637}
638
639static inline void gpu_write64(struct msm_gpu *gpu, u32 reg, u64 val)
640{
641	trace_msm_gpu_regaccess(reg);
642	/* Why not a writeq here? Read the screed above */
643	writel(lower_32_bits(val), gpu->mmio + (reg << 2));
644	trace_msm_gpu_regaccess(reg+1);
645	writel(upper_32_bits(val), gpu->mmio + ((reg + 1) << 2));
646}
647
648int msm_gpu_pm_suspend(struct msm_gpu *gpu);
649int msm_gpu_pm_resume(struct msm_gpu *gpu);
650
651void msm_gpu_show_fdinfo(struct msm_gpu *gpu, struct msm_context *ctx,
652			 struct drm_printer *p);
653
654int msm_submitqueue_init(struct drm_device *drm, struct msm_context *ctx);
655struct msm_gpu_submitqueue *msm_submitqueue_get(struct msm_context *ctx,
656		u32 id);
657int msm_submitqueue_create(struct drm_device *drm,
658		struct msm_context *ctx,
659		u32 prio, u32 flags, u32 *id);
660int msm_submitqueue_query(struct drm_device *drm, struct msm_context *ctx,
661		struct drm_msm_submitqueue_query *args);
662int msm_submitqueue_remove(struct msm_context *ctx, u32 id);
663void msm_submitqueue_close(struct msm_context *ctx);
664
665void msm_submitqueue_destroy(struct kref *kref);
666
667int msm_context_set_sysprof(struct msm_context *ctx, struct msm_gpu *gpu, int sysprof);
668void __msm_context_destroy(struct kref *kref);
669
670static inline void msm_context_put(struct msm_context *ctx)
671{
672	kref_put(&ctx->ref, __msm_context_destroy);
673}
674
675static inline struct msm_context *msm_context_get(
676	struct msm_context *ctx)
677{
678	kref_get(&ctx->ref);
679	return ctx;
680}
681
682void msm_devfreq_init(struct msm_gpu *gpu);
683void msm_devfreq_cleanup(struct msm_gpu *gpu);
684void msm_devfreq_resume(struct msm_gpu *gpu);
685void msm_devfreq_suspend(struct msm_gpu *gpu);
686void msm_devfreq_boost(struct msm_gpu *gpu, unsigned factor);
687void msm_devfreq_active(struct msm_gpu *gpu);
688void msm_devfreq_idle(struct msm_gpu *gpu);
689
690int msm_gpu_hw_init(struct msm_gpu *gpu);
691
692void msm_gpu_perfcntr_start(struct msm_gpu *gpu);
693void msm_gpu_perfcntr_stop(struct msm_gpu *gpu);
694int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime,
695		uint32_t *totaltime, uint32_t ncntrs, uint32_t *cntrs);
696
697void msm_gpu_retire(struct msm_gpu *gpu);
698void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit);
699
700int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
701		struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
702		const char *name, struct msm_gpu_config *config);
703
704struct drm_gpuvm *
705msm_gpu_create_private_vm(struct msm_gpu *gpu, struct task_struct *task,
706			  bool kernel_managed);
707
708void msm_gpu_cleanup(struct msm_gpu *gpu);
709
710struct msm_gpu *adreno_load_gpu(struct drm_device *dev);
711bool adreno_has_gpu(struct device_node *node);
712void __init adreno_register(void);
713void __exit adreno_unregister(void);
714
715static inline void msm_submitqueue_put(struct msm_gpu_submitqueue *queue)
716{
717	if (queue)
718		kref_put(&queue->ref, msm_submitqueue_destroy);
719}
720
721static inline struct msm_gpu_state *msm_gpu_crashstate_get(struct msm_gpu *gpu)
722{
723	struct msm_gpu_state *state = NULL;
724
725	mutex_lock(&gpu->lock);
726
727	if (gpu->crashstate) {
728		kref_get(&gpu->crashstate->ref);
729		state = gpu->crashstate;
730	}
731
732	mutex_unlock(&gpu->lock);
733
734	return state;
735}
736
737static inline void msm_gpu_crashstate_put(struct msm_gpu *gpu)
738{
739	mutex_lock(&gpu->lock);
740
741	if (gpu->crashstate) {
742		if (gpu->funcs->gpu_state_put(gpu->crashstate))
743			gpu->crashstate = NULL;
744	}
745
746	mutex_unlock(&gpu->lock);
747}
748
749void msm_gpu_fault_crashstate_capture(struct msm_gpu *gpu, struct msm_gpu_fault_info *fault_info);
750
751/*
752 * Simple macro to semi-cleanly add the MAP_PRIV flag for targets that can
753 * support expanded privileges
754 */
755#define check_apriv(gpu, flags) \
756	(((gpu)->hw_apriv ? MSM_BO_MAP_PRIV : 0) | (flags))
757
758
759#endif /* __MSM_GPU_H__ */