a5xx_gpu.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
   3 */
   4
   5#include <linux/kernel.h>
   6#include <linux/types.h>
   7#include <linux/cpumask.h>
   8#include <linux/firmware/qcom/qcom_scm.h>
   9#include <linux/pm_opp.h>
  10#include <linux/nvmem-consumer.h>
  11#include <linux/slab.h>
  12#include "msm_gem.h"
  13#include "msm_mmu.h"
  14#include "a5xx_gpu.h"
  15
  16extern bool hang_debug;
  17static void a5xx_dump(struct msm_gpu *gpu);
  18
  19#define GPU_PAS_ID 13
  20
  21static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
  22{
  23	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
  24	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
  25
  26	if (a5xx_gpu->has_whereami) {
  27		OUT_PKT7(ring, CP_WHERE_AM_I, 2);
  28		OUT_RING(ring, lower_32_bits(shadowptr(a5xx_gpu, ring)));
  29		OUT_RING(ring, upper_32_bits(shadowptr(a5xx_gpu, ring)));
  30	}
  31}
  32
  33void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring,
  34		bool sync)
  35{
  36	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
  37	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
  38	uint32_t wptr;
  39	unsigned long flags;
  40
  41	/*
  42	 * Most flush operations need to issue a WHERE_AM_I opcode to sync up
  43	 * the rptr shadow
  44	 */
  45	if (sync)
  46		update_shadow_rptr(gpu, ring);
  47
  48	spin_lock_irqsave(&ring->preempt_lock, flags);
  49
  50	/* Copy the shadow to the actual register */
  51	ring->cur = ring->next;
  52
  53	/* Make sure to wrap wptr if we need to */
  54	wptr = get_wptr(ring);
  55
  56	spin_unlock_irqrestore(&ring->preempt_lock, flags);
  57
  58	/* Make sure everything is posted before making a decision */
  59	mb();
  60
  61	/* Update HW if this is the current ring and we are not in preempt */
  62	if (a5xx_gpu->cur_ring == ring && !a5xx_in_preempt(a5xx_gpu))
  63		gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
  64}
  65
  66static void a5xx_submit_in_rb(struct msm_gpu *gpu, struct msm_gem_submit *submit)
  67{
  68	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
  69	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
  70	struct msm_ringbuffer *ring = submit->ring;
  71	struct drm_gem_object *obj;
  72	uint32_t *ptr, dwords;
  73	unsigned int i;
  74
  75	for (i = 0; i < submit->nr_cmds; i++) {
  76		switch (submit->cmd[i].type) {
  77		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
  78			break;
  79		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
  80			if (ring->cur_ctx_seqno == submit->queue->ctx->seqno)
  81				break;
  82			fallthrough;
  83		case MSM_SUBMIT_CMD_BUF:
  84			/* copy commands into RB: */
  85			obj = submit->bos[submit->cmd[i].idx].obj;
  86			dwords = submit->cmd[i].size;
  87
  88			ptr = msm_gem_get_vaddr(obj);
  89
  90			/* _get_vaddr() shouldn't fail at this point,
  91			 * since we've already mapped it once in
  92			 * submit_reloc()
  93			 */
  94			if (WARN_ON(IS_ERR_OR_NULL(ptr)))
  95				return;
  96
  97			for (i = 0; i < dwords; i++) {
  98				/* normally the OUT_PKTn() would wait
  99				 * for space for the packet.  But since
 100				 * we just OUT_RING() the whole thing,
 101				 * need to call adreno_wait_ring()
 102				 * ourself:
 103				 */
 104				adreno_wait_ring(ring, 1);
 105				OUT_RING(ring, ptr[i]);
 106			}
 107
 108			msm_gem_put_vaddr(obj);
 109
 110			break;
 111		}
 112	}
 113
 114	a5xx_gpu->last_seqno[ring->id] = submit->seqno;
 115	a5xx_flush(gpu, ring, true);
 116	a5xx_preempt_trigger(gpu);
 117
 118	/* we might not necessarily have a cmd from userspace to
 119	 * trigger an event to know that submit has completed, so
 120	 * do this manually:
 121	 */
 122	a5xx_idle(gpu, ring);
 123	ring->memptrs->fence = submit->seqno;
 124	msm_gpu_retire(gpu);
 125}
 126
 127static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 128{
 129	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 130	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 131	struct msm_ringbuffer *ring = submit->ring;
 132	unsigned int i, ibs = 0;
 133
 134	adreno_check_and_reenable_stall(adreno_gpu);
 135
 136	if (IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) && submit->in_rb) {
 137		ring->cur_ctx_seqno = 0;
 138		a5xx_submit_in_rb(gpu, submit);
 139		return;
 140	}
 141
 142	OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
 143	OUT_RING(ring, 0x02);
 144
 145	/* Turn off protected mode to write to special registers */
 146	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
 147	OUT_RING(ring, 0);
 148
 149	/* Set the save preemption record for the ring/command */
 150	OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
 151	OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
 152	OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[submit->ring->id]));
 153
 154	/* Turn back on protected mode */
 155	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
 156	OUT_RING(ring, 1);
 157
 158	/*
 159	 * Disable local preemption by default because it requires
 160	 * user-space to be aware of it and provide additional handling
 161	 * to restore rendering state or do various flushes on switch.
 162	 */
 163	OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
 164	OUT_RING(ring, 0x0);
 165
 166	/* Allow CP_CONTEXT_SWITCH_YIELD packets in the IB2 */
 167	OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
 168	OUT_RING(ring, 0x02);
 169
 170	/* Submit the commands */
 171	for (i = 0; i < submit->nr_cmds; i++) {
 172		switch (submit->cmd[i].type) {
 173		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
 174			break;
 175		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
 176			if (ring->cur_ctx_seqno == submit->queue->ctx->seqno)
 177				break;
 178			fallthrough;
 179		case MSM_SUBMIT_CMD_BUF:
 180			OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
 181			OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
 182			OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
 183			OUT_RING(ring, submit->cmd[i].size);
 184			ibs++;
 185			break;
 186		}
 187
 188		/*
 189		 * Periodically update shadow-wptr if needed, so that we
 190		 * can see partial progress of submits with large # of
 191		 * cmds.. otherwise we could needlessly stall waiting for
 192		 * ringbuffer state, simply due to looking at a shadow
 193		 * rptr value that has not been updated
 194		 */
 195		if ((ibs % 32) == 0)
 196			update_shadow_rptr(gpu, ring);
 197	}
 198
 199	/*
 200	 * Write the render mode to NULL (0) to indicate to the CP that the IBs
 201	 * are done rendering - otherwise a lucky preemption would start
 202	 * replaying from the last checkpoint
 203	 */
 204	OUT_PKT7(ring, CP_SET_RENDER_MODE, 5);
 205	OUT_RING(ring, 0);
 206	OUT_RING(ring, 0);
 207	OUT_RING(ring, 0);
 208	OUT_RING(ring, 0);
 209	OUT_RING(ring, 0);
 210
 211	/* Turn off IB level preemptions */
 212	OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
 213	OUT_RING(ring, 0x01);
 214
 215	/* Write the fence to the scratch register */
 216	OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
 217	OUT_RING(ring, submit->seqno);
 218	a5xx_gpu->last_seqno[ring->id] = submit->seqno;
 219
 220	/*
 221	 * Execute a CACHE_FLUSH_TS event. This will ensure that the
 222	 * timestamp is written to the memory and then triggers the interrupt
 223	 */
 224	OUT_PKT7(ring, CP_EVENT_WRITE, 4);
 225	OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
 226		CP_EVENT_WRITE_0_IRQ);
 227	OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
 228	OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
 229	OUT_RING(ring, submit->seqno);
 230
 231	/* Yield the floor on command completion */
 232	OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
 233	/*
 234	 * If dword[2:1] are non zero, they specify an address for the CP to
 235	 * write the value of dword[3] to on preemption complete. Write 0 to
 236	 * skip the write
 237	 */
 238	OUT_RING(ring, 0x00);
 239	OUT_RING(ring, 0x00);
 240	/* Data value - not used if the address above is 0 */
 241	OUT_RING(ring, 0x01);
 242	/* Set bit 0 to trigger an interrupt on preempt complete */
 243	OUT_RING(ring, 0x01);
 244
 245	/* A WHERE_AM_I packet is not needed after a YIELD */
 246	a5xx_flush(gpu, ring, false);
 247
 248	/* Check to see if we need to start preemption */
 249	a5xx_preempt_trigger(gpu);
 250}
 251
 252static const struct adreno_five_hwcg_regs {
 253	u32 offset;
 254	u32 value;
 255} a5xx_hwcg[] = {
 256	{REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 257	{REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
 258	{REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
 259	{REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
 260	{REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 261	{REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
 262	{REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
 263	{REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
 264	{REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 265	{REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
 266	{REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
 267	{REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
 268	{REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 269	{REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
 270	{REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
 271	{REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
 272	{REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 273	{REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
 274	{REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
 275	{REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
 276	{REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 277	{REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
 278	{REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
 279	{REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
 280	{REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
 281	{REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
 282	{REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
 283	{REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
 284	{REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 285	{REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
 286	{REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
 287	{REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
 288	{REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 289	{REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
 290	{REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
 291	{REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
 292	{REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
 293	{REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
 294	{REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
 295	{REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
 296	{REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 297	{REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
 298	{REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
 299	{REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
 300	{REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 301	{REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
 302	{REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
 303	{REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
 304	{REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
 305	{REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
 306	{REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
 307	{REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
 308	{REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 309	{REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 310	{REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 311	{REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 312	{REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
 313	{REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 314	{REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 315	{REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
 316	{REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
 317	{REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
 318	{REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
 319	{REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
 320	{REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
 321	{REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
 322	{REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
 323	{REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
 324	{REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
 325	{REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
 326	{REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
 327	{REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
 328	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
 329	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
 330	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
 331	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
 332	{REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
 333	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
 334	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
 335	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
 336	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
 337	{REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
 338	{REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 339	{REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
 340	{REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 341	{REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 342	{REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 343	{REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 344	{REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 345	{REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 346	{REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 347	{REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
 348}, a50x_hwcg[] = {
 349	{REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 350	{REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 351	{REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 352	{REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 353	{REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 354	{REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 355	{REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
 356	{REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 357	{REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 358	{REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
 359	{REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 360	{REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 361	{REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
 362	{REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 363	{REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 364	{REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 365	{REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 366	{REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00FFFFF4},
 367	{REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 368	{REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 369	{REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
 370	{REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
 371	{REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
 372	{REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
 373	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
 374	{REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
 375	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
 376	{REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
 377	{REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 378	{REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
 379	{REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 380	{REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 381	{REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 382	{REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 383	{REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 384	{REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 385	{REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 386	{REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 387}, a512_hwcg[] = {
 388	{REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 389	{REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
 390	{REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 391	{REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
 392	{REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 393	{REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
 394	{REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 395	{REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
 396	{REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 397	{REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
 398	{REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 399	{REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
 400	{REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
 401	{REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
 402	{REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 403	{REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
 404	{REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 405	{REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
 406	{REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
 407	{REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
 408	{REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 409	{REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
 410	{REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 411	{REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
 412	{REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
 413	{REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
 414	{REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 415	{REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 416	{REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 417	{REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 418	{REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
 419	{REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 420	{REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 421	{REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
 422	{REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
 423	{REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
 424	{REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
 425	{REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
 426	{REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
 427	{REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
 428	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
 429	{REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
 430	{REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
 431	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
 432	{REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
 433	{REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
 434	{REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 435	{REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
 436	{REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 437	{REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 438	{REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 439	{REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 440	{REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 441	{REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 442	{REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 443	{REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 444};
 445
 446void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)
 447{
 448	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 449	const struct adreno_five_hwcg_regs *regs;
 450	unsigned int i, sz;
 451
 452	if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
 453	    adreno_is_a508(adreno_gpu)) {
 454		regs = a50x_hwcg;
 455		sz = ARRAY_SIZE(a50x_hwcg);
 456	} else if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu)) {
 457		regs = a512_hwcg;
 458		sz = ARRAY_SIZE(a512_hwcg);
 459	} else {
 460		regs = a5xx_hwcg;
 461		sz = ARRAY_SIZE(a5xx_hwcg);
 462	}
 463
 464	for (i = 0; i < sz; i++)
 465		gpu_write(gpu, regs[i].offset,
 466			  state ? regs[i].value : 0);
 467
 468	if (adreno_is_a540(adreno_gpu)) {
 469		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_DELAY_GPMU, state ? 0x00000770 : 0);
 470		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_HYST_GPMU, state ? 0x00000004 : 0);
 471	}
 472
 473	gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);
 474	gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);
 475}
 476
 477static int a5xx_me_init(struct msm_gpu *gpu)
 478{
 479	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 480	struct msm_ringbuffer *ring = gpu->rb[0];
 481
 482	OUT_PKT7(ring, CP_ME_INIT, 8);
 483
 484	OUT_RING(ring, 0x0000002F);
 485
 486	/* Enable multiple hardware contexts */
 487	OUT_RING(ring, 0x00000003);
 488
 489	/* Enable error detection */
 490	OUT_RING(ring, 0x20000000);
 491
 492	/* Don't enable header dump */
 493	OUT_RING(ring, 0x00000000);
 494	OUT_RING(ring, 0x00000000);
 495
 496	/* Specify workarounds for various microcode issues */
 497	if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
 498	    adreno_is_a530(adreno_gpu)) {
 499		/* Workaround for token end syncs
 500		 * Force a WFI after every direct-render 3D mode draw and every
 501		 * 2D mode 3 draw
 502		 */
 503		OUT_RING(ring, 0x0000000B);
 504	} else if (adreno_is_a510(adreno_gpu)) {
 505		/* Workaround for token and syncs */
 506		OUT_RING(ring, 0x00000001);
 507	} else {
 508		/* No workarounds enabled */
 509		OUT_RING(ring, 0x00000000);
 510	}
 511
 512	OUT_RING(ring, 0x00000000);
 513	OUT_RING(ring, 0x00000000);
 514
 515	a5xx_flush(gpu, ring, true);
 516	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
 517}
 518
 519static int a5xx_preempt_start(struct msm_gpu *gpu)
 520{
 521	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 522	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 523	struct msm_ringbuffer *ring = gpu->rb[0];
 524
 525	if (gpu->nr_rings == 1)
 526		return 0;
 527
 528	/* Turn off protected mode to write to special registers */
 529	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
 530	OUT_RING(ring, 0);
 531
 532	/* Set the save preemption record for the ring/command */
 533	OUT_PKT4(ring, REG_A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 2);
 534	OUT_RING(ring, lower_32_bits(a5xx_gpu->preempt_iova[ring->id]));
 535	OUT_RING(ring, upper_32_bits(a5xx_gpu->preempt_iova[ring->id]));
 536
 537	/* Turn back on protected mode */
 538	OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
 539	OUT_RING(ring, 1);
 540
 541	OUT_PKT7(ring, CP_PREEMPT_ENABLE_GLOBAL, 1);
 542	OUT_RING(ring, 0x00);
 543
 544	OUT_PKT7(ring, CP_PREEMPT_ENABLE_LOCAL, 1);
 545	OUT_RING(ring, 0x01);
 546
 547	OUT_PKT7(ring, CP_YIELD_ENABLE, 1);
 548	OUT_RING(ring, 0x01);
 549
 550	/* Yield the floor on command completion */
 551	OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
 552	OUT_RING(ring, 0x00);
 553	OUT_RING(ring, 0x00);
 554	OUT_RING(ring, 0x01);
 555	OUT_RING(ring, 0x01);
 556
 557	/* The WHERE_AMI_I packet is not needed after a YIELD is issued */
 558	a5xx_flush(gpu, ring, false);
 559
 560	return a5xx_idle(gpu, ring) ? 0 : -EINVAL;
 561}
 562
 563static void a5xx_ucode_check_version(struct a5xx_gpu *a5xx_gpu,
 564		struct drm_gem_object *obj)
 565{
 566	u32 *buf = msm_gem_get_vaddr(obj);
 567
 568	if (IS_ERR(buf))
 569		return;
 570
 571	/*
 572	 * If the lowest nibble is 0xa that is an indication that this microcode
 573	 * has been patched. The actual version is in dword [3] but we only care
 574	 * about the patchlevel which is the lowest nibble of dword [3]
 575	 */
 576	if (((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1)
 577		a5xx_gpu->has_whereami = true;
 578
 579	msm_gem_put_vaddr(obj);
 580}
 581
 582static int a5xx_ucode_load(struct msm_gpu *gpu)
 583{
 584	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 585	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 586	int ret;
 587
 588	if (!a5xx_gpu->pm4_bo) {
 589		a5xx_gpu->pm4_bo = adreno_fw_create_bo(gpu,
 590			adreno_gpu->fw[ADRENO_FW_PM4], &a5xx_gpu->pm4_iova);
 591
 592
 593		if (IS_ERR(a5xx_gpu->pm4_bo)) {
 594			ret = PTR_ERR(a5xx_gpu->pm4_bo);
 595			a5xx_gpu->pm4_bo = NULL;
 596			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PM4: %d\n",
 597				ret);
 598			return ret;
 599		}
 600
 601		msm_gem_object_set_name(a5xx_gpu->pm4_bo, "pm4fw");
 602	}
 603
 604	if (!a5xx_gpu->pfp_bo) {
 605		a5xx_gpu->pfp_bo = adreno_fw_create_bo(gpu,
 606			adreno_gpu->fw[ADRENO_FW_PFP], &a5xx_gpu->pfp_iova);
 607
 608		if (IS_ERR(a5xx_gpu->pfp_bo)) {
 609			ret = PTR_ERR(a5xx_gpu->pfp_bo);
 610			a5xx_gpu->pfp_bo = NULL;
 611			DRM_DEV_ERROR(gpu->dev->dev, "could not allocate PFP: %d\n",
 612				ret);
 613			return ret;
 614		}
 615
 616		msm_gem_object_set_name(a5xx_gpu->pfp_bo, "pfpfw");
 617		a5xx_ucode_check_version(a5xx_gpu, a5xx_gpu->pfp_bo);
 618	}
 619
 620	if (a5xx_gpu->has_whereami) {
 621		if (!a5xx_gpu->shadow_bo) {
 622			a5xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
 623				sizeof(u32) * gpu->nr_rings,
 624				MSM_BO_WC | MSM_BO_MAP_PRIV,
 625				gpu->vm, &a5xx_gpu->shadow_bo,
 626				&a5xx_gpu->shadow_iova);
 627
 628			if (IS_ERR(a5xx_gpu->shadow))
 629				return PTR_ERR(a5xx_gpu->shadow);
 630
 631			msm_gem_object_set_name(a5xx_gpu->shadow_bo, "shadow");
 632		}
 633	} else if (gpu->nr_rings > 1) {
 634		/* Disable preemption if WHERE_AM_I isn't available */
 635		a5xx_preempt_fini(gpu);
 636		gpu->nr_rings = 1;
 637	}
 638
 639	return 0;
 640}
 641
 642#define SCM_GPU_ZAP_SHADER_RESUME 0
 643
 644static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
 645{
 646	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 647	int ret;
 648
 649	/*
 650	 * Adreno 506 have CPZ Retention feature and doesn't require
 651	 * to resume zap shader
 652	 */
 653	if (adreno_is_a506(adreno_gpu))
 654		return 0;
 655
 656	ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
 657	if (ret)
 658		DRM_ERROR("%s: zap-shader resume failed: %d\n",
 659			gpu->name, ret);
 660
 661	return ret;
 662}
 663
 664static int a5xx_zap_shader_init(struct msm_gpu *gpu)
 665{
 666	static bool loaded;
 667	int ret;
 668
 669	/*
 670	 * If the zap shader is already loaded into memory we just need to kick
 671	 * the remote processor to reinitialize it
 672	 */
 673	if (loaded)
 674		return a5xx_zap_shader_resume(gpu);
 675
 676	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
 677
 678	loaded = !ret;
 679	return ret;
 680}
 681
 682#define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 683	  A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
 684	  A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
 685	  A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
 686	  A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 687	  A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
 688	  A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
 689	  A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \
 690	  A5XX_RBBM_INT_0_MASK_CP_SW | \
 691	  A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
 692	  A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
 693	  A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
 694
 695static int a5xx_hw_init(struct msm_gpu *gpu)
 696{
 697	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 698	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 699	u32 hbb;
 700	int ret;
 701
 702	gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
 703
 704	if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
 705	    adreno_is_a540(adreno_gpu))
 706		gpu_write(gpu, REG_A5XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
 707
 708	/* Make all blocks contribute to the GPU BUSY perf counter */
 709	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
 710
 711	/* Enable RBBM error reporting bits */
 712	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
 713
 714	if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
 715		/*
 716		 * Mask out the activity signals from RB1-3 to avoid false
 717		 * positives
 718		 */
 719
 720		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
 721			0xF0000000);
 722		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
 723			0xFFFFFFFF);
 724		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
 725			0xFFFFFFFF);
 726		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
 727			0xFFFFFFFF);
 728		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
 729			0xFFFFFFFF);
 730		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
 731			0xFFFFFFFF);
 732		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
 733			0xFFFFFFFF);
 734		gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
 735			0xFFFFFFFF);
 736	}
 737
 738	/* Enable fault detection */
 739	gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
 740		(1 << 30) | 0xFFFF);
 741
 742	/* Turn on performance counters */
 743	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
 744
 745	/* Select CP0 to always count cycles */
 746	gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
 747
 748	/* Select RBBM0 to countable 6 to get the busy status for devfreq */
 749	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);
 750
 751	/* Increase VFD cache access so LRZ and other data gets evicted less */
 752	gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
 753
 754	/* Disable L2 bypass in the UCHE */
 755	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, lower_32_bits(adreno_gpu->uche_trap_base));
 756	gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, upper_32_bits(adreno_gpu->uche_trap_base));
 757	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, lower_32_bits(adreno_gpu->uche_trap_base));
 758	gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, upper_32_bits(adreno_gpu->uche_trap_base));
 759
 760	/* Set the GMEM VA range (0 to gpu->gmem) */
 761	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
 762	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
 763	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
 764		0x00100000 + adreno_gpu->info->gmem - 1);
 765	gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
 766
 767	if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
 768	    adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu)) {
 769		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20);
 770		if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
 771			adreno_is_a508(adreno_gpu))
 772			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
 773		else
 774			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20);
 775		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);
 776		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);
 777	} else {
 778		gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
 779		if (adreno_is_a530(adreno_gpu))
 780			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
 781		else
 782			gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
 783		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
 784		gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
 785	}
 786
 787	if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
 788	    adreno_is_a508(adreno_gpu))
 789		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
 790			  (0x100 << 11 | 0x100 << 22));
 791	else if (adreno_is_a509(adreno_gpu) || adreno_is_a510(adreno_gpu) ||
 792		 adreno_is_a512(adreno_gpu))
 793		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
 794			  (0x200 << 11 | 0x200 << 22));
 795	else
 796		gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
 797			  (0x400 << 11 | 0x300 << 22));
 798
 799	if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
 800		gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
 801
 802	/*
 803	 * Disable the RB sampler datapath DP2 clock gating optimization
 804	 * for 1-SP GPUs, as it is enabled by default.
 805	 */
 806	if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
 807	    adreno_is_a508(adreno_gpu) || adreno_is_a509(adreno_gpu) ||
 808	    adreno_is_a512(adreno_gpu))
 809		gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, 0, (1 << 9));
 810
 811	/* Disable UCHE global filter as SP can invalidate/flush independently */
 812	gpu_write(gpu, REG_A5XX_UCHE_MODE_CNTL, BIT(29));
 813
 814	/* Enable USE_RETENTION_FLOPS */
 815	gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
 816
 817	/* Enable ME/PFP split notification */
 818	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
 819
 820	/*
 821	 *  In A5x, CCU can send context_done event of a particular context to
 822	 *  UCHE which ultimately reaches CP even when there is valid
 823	 *  transaction of that context inside CCU. This can let CP to program
 824	 *  config registers, which will make the "valid transaction" inside
 825	 *  CCU to be interpreted differently. This can cause gpu fault. This
 826	 *  bug is fixed in latest A510 revision. To enable this bug fix -
 827	 *  bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1
 828	 *  (disable). For older A510 version this bit is unused.
 829	 */
 830	if (adreno_is_a510(adreno_gpu))
 831		gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0);
 832
 833	/* Enable HWCG */
 834	a5xx_set_hwcg(gpu, true);
 835
 836	gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
 837
 838	BUG_ON(adreno_gpu->ubwc_config->highest_bank_bit < 13);
 839	hbb = adreno_gpu->ubwc_config->highest_bank_bit - 13;
 840
 841	gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, hbb << 7);
 842	gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, hbb << 1);
 843
 844	if (adreno_is_a509(adreno_gpu) || adreno_is_a512(adreno_gpu) ||
 845	    adreno_is_a540(adreno_gpu))
 846		gpu_write(gpu, REG_A5XX_UCHE_DBG_ECO_CNTL_2, hbb);
 847
 848	/* Disable All flat shading optimization (ALLFLATOPTDIS) */
 849	gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, (1 << 10));
 850
 851	/* Protect registers from the CP */
 852	gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
 853
 854	/* RBBM */
 855	gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
 856	gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
 857	gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
 858	gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
 859	gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
 860	gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
 861
 862	/* Content protect */
 863	gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
 864		ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
 865			16));
 866	gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
 867		ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
 868
 869	/* CP */
 870	gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
 871	gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
 872	gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
 873	gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
 874
 875	/* RB */
 876	gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
 877	gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
 878
 879	/* VPC */
 880	gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
 881	gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 16));
 882
 883	/* UCHE */
 884	gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
 885
 886	/* SMMU */
 887	gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
 888			ADRENO_PROTECT_RW(0x10000, 0x8000));
 889
 890	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
 891	/*
 892	 * Disable the trusted memory range - we don't actually supported secure
 893	 * memory rendering at this point in time and we don't want to block off
 894	 * part of the virtual memory space.
 895	 */
 896	gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 0x00000000);
 897	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
 898
 899	/* Put the GPU into 64 bit by default */
 900	gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1);
 901	gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1);
 902	gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1);
 903	gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1);
 904	gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1);
 905	gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);
 906	gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1);
 907	gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1);
 908	gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1);
 909	gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1);
 910	gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1);
 911	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
 912
 913	/*
 914	 * VPC corner case with local memory load kill leads to corrupt
 915	 * internal state. Normal Disable does not work for all a5x chips.
 916	 * So do the following setting to disable it.
 917	 */
 918	if (adreno_gpu->info->quirks & ADRENO_QUIRK_LMLOADKILL_DISABLE) {
 919		gpu_rmw(gpu, REG_A5XX_VPC_DBG_ECO_CNTL, 0, BIT(23));
 920		gpu_rmw(gpu, REG_A5XX_HLSQ_DBG_ECO_CNTL, BIT(18), 0);
 921	}
 922
 923	ret = adreno_hw_init(gpu);
 924	if (ret)
 925		return ret;
 926
 927	if (adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))
 928		a5xx_gpmu_ucode_init(gpu);
 929
 930	gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO, a5xx_gpu->pm4_iova);
 931	gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO, a5xx_gpu->pfp_iova);
 932
 933	/* Set the ringbuffer address */
 934	gpu_write64(gpu, REG_A5XX_CP_RB_BASE, gpu->rb[0]->iova);
 935
 936	/*
 937	 * If the microcode supports the WHERE_AM_I opcode then we can use that
 938	 * in lieu of the RPTR shadow and enable preemption. Otherwise, we
 939	 * can't safely use the RPTR shadow or preemption. In either case, the
 940	 * RPTR shadow should be disabled in hardware.
 941	 */
 942	gpu_write(gpu, REG_A5XX_CP_RB_CNTL,
 943		MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
 944
 945	/* Configure the RPTR shadow if needed: */
 946	if (a5xx_gpu->shadow_bo) {
 947		gpu_write64(gpu, REG_A5XX_CP_RB_RPTR_ADDR,
 948			    shadowptr(a5xx_gpu, gpu->rb[0]));
 949	}
 950
 951	a5xx_preempt_hw_init(gpu);
 952
 953	/* Disable the interrupts through the initial bringup stage */
 954	gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
 955
 956	/* Clear ME_HALT to start the micro engine */
 957	gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
 958	ret = a5xx_me_init(gpu);
 959	if (ret)
 960		return ret;
 961
 962	ret = a5xx_power_init(gpu);
 963	if (ret)
 964		return ret;
 965
 966	/*
 967	 * Send a pipeline event stat to get misbehaving counters to start
 968	 * ticking correctly
 969	 */
 970	if (adreno_is_a530(adreno_gpu)) {
 971		OUT_PKT7(gpu->rb[0], CP_EVENT_WRITE, 1);
 972		OUT_RING(gpu->rb[0], CP_EVENT_WRITE_0_EVENT(STAT_EVENT));
 973
 974		a5xx_flush(gpu, gpu->rb[0], true);
 975		if (!a5xx_idle(gpu, gpu->rb[0]))
 976			return -EINVAL;
 977	}
 978
 979	/*
 980	 * If the chip that we are using does support loading one, then
 981	 * try to load a zap shader into the secure world. If successful
 982	 * we can use the CP to switch out of secure mode. If not then we
 983	 * have no resource but to try to switch ourselves out manually. If we
 984	 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
 985	 * be blocked and a permissions violation will soon follow.
 986	 */
 987	ret = a5xx_zap_shader_init(gpu);
 988	if (!ret) {
 989		OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
 990		OUT_RING(gpu->rb[0], 0x00000000);
 991
 992		a5xx_flush(gpu, gpu->rb[0], true);
 993		if (!a5xx_idle(gpu, gpu->rb[0]))
 994			return -EINVAL;
 995	} else if (ret == -ENODEV) {
 996		/*
 997		 * This device does not use zap shader (but print a warning
 998		 * just in case someone got their dt wrong.. hopefully they
 999		 * have a debug UART to realize the error of their ways...
1000		 * if you mess this up you are about to crash horribly)
1001		 */
1002		dev_warn_once(gpu->dev->dev,
1003			"Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
1004		gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
1005	} else {
1006		return ret;
1007	}
1008
1009	/* Last step - yield the ringbuffer */
1010	a5xx_preempt_start(gpu);
1011
1012	return 0;
1013}
1014
1015static void a5xx_recover(struct msm_gpu *gpu)
1016{
1017	int i;
1018
1019	adreno_dump_info(gpu);
1020
1021	for (i = 0; i < 8; i++) {
1022		printk("CP_SCRATCH_REG%d: %u\n", i,
1023			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
1024	}
1025
1026	if (hang_debug)
1027		a5xx_dump(gpu);
1028
1029	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
1030	gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
1031	gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
1032	adreno_recover(gpu);
1033}
1034
1035static void a5xx_destroy(struct msm_gpu *gpu)
1036{
1037	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1038	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1039
1040	DBG("%s", gpu->name);
1041
1042	a5xx_preempt_fini(gpu);
1043
1044	if (a5xx_gpu->pm4_bo) {
1045		msm_gem_unpin_iova(a5xx_gpu->pm4_bo, gpu->vm);
1046		drm_gem_object_put(a5xx_gpu->pm4_bo);
1047	}
1048
1049	if (a5xx_gpu->pfp_bo) {
1050		msm_gem_unpin_iova(a5xx_gpu->pfp_bo, gpu->vm);
1051		drm_gem_object_put(a5xx_gpu->pfp_bo);
1052	}
1053
1054	if (a5xx_gpu->gpmu_bo) {
1055		msm_gem_unpin_iova(a5xx_gpu->gpmu_bo, gpu->vm);
1056		drm_gem_object_put(a5xx_gpu->gpmu_bo);
1057	}
1058
1059	if (a5xx_gpu->shadow_bo) {
1060		msm_gem_unpin_iova(a5xx_gpu->shadow_bo, gpu->vm);
1061		drm_gem_object_put(a5xx_gpu->shadow_bo);
1062	}
1063
1064	adreno_gpu_cleanup(adreno_gpu);
1065	kfree(a5xx_gpu);
1066}
1067
1068static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
1069{
1070	if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
1071		return false;
1072
1073	/*
1074	 * Nearly every abnormality ends up pausing the GPU and triggering a
1075	 * fault so we can safely just watch for this one interrupt to fire
1076	 */
1077	return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
1078		A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
1079}
1080
1081bool a5xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1082{
1083	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1084	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1085
1086	if (ring != a5xx_gpu->cur_ring) {
1087		WARN(1, "Tried to idle a non-current ringbuffer\n");
1088		return false;
1089	}
1090
1091	/* wait for CP to drain ringbuffer: */
1092	if (!adreno_idle(gpu, ring))
1093		return false;
1094
1095	if (spin_until(_a5xx_check_idle(gpu))) {
1096		DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
1097			gpu->name, __builtin_return_address(0),
1098			gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1099			gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS),
1100			gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1101			gpu_read(gpu, REG_A5XX_CP_RB_WPTR));
1102		return false;
1103	}
1104
1105	return true;
1106}
1107
1108static int a5xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
1109{
1110	struct msm_gpu *gpu = arg;
1111	struct adreno_smmu_fault_info *info = data;
1112	char block[12] = "unknown";
1113	u32 scratch[] = {
1114			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
1115			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
1116			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
1117			gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)),
1118	};
1119
1120	if (info)
1121		snprintf(block, sizeof(block), "%x", info->fsynr1);
1122
1123	return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
1124}
1125
1126static void a5xx_cp_err_irq(struct msm_gpu *gpu)
1127{
1128	u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
1129
1130	if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
1131		u32 val;
1132
1133		gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
1134
1135		/*
1136		 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
1137		 * read it twice
1138		 */
1139
1140		gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1141		val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
1142
1143		dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
1144			val);
1145	}
1146
1147	if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
1148		dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
1149			gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
1150
1151	if (status & A5XX_CP_INT_CP_DMA_ERROR)
1152		dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
1153
1154	if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
1155		u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
1156
1157		dev_err_ratelimited(gpu->dev->dev,
1158			"CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
1159			val & (1 << 24) ? "WRITE" : "READ",
1160			(val & 0xFFFFF) >> 2, val);
1161	}
1162
1163	if (status & A5XX_CP_INT_CP_AHB_ERROR) {
1164		u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
1165		const char *access[16] = { "reserved", "reserved",
1166			"timestamp lo", "timestamp hi", "pfp read", "pfp write",
1167			"", "", "me read", "me write", "", "", "crashdump read",
1168			"crashdump write" };
1169
1170		dev_err_ratelimited(gpu->dev->dev,
1171			"CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
1172			status & 0xFFFFF, access[(status >> 24) & 0xF],
1173			(status & (1 << 31)), status);
1174	}
1175}
1176
1177static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
1178{
1179	if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
1180		u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
1181
1182		dev_err_ratelimited(gpu->dev->dev,
1183			"RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
1184			val & (1 << 28) ? "WRITE" : "READ",
1185			(val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
1186			(val >> 24) & 0xF);
1187
1188		/* Clear the error */
1189		gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
1190
1191		/* Clear the interrupt */
1192		gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1193			A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1194	}
1195
1196	if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
1197		dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
1198
1199	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
1200		dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
1201			gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
1202
1203	if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
1204		dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
1205			gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
1206
1207	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
1208		dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
1209			gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
1210
1211	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1212		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
1213
1214	if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
1215		dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
1216}
1217
1218static void a5xx_uche_err_irq(struct msm_gpu *gpu)
1219{
1220	uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
1221
1222	addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
1223
1224	dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
1225		addr);
1226}
1227
1228static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
1229{
1230	dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
1231}
1232
1233static void a5xx_fault_detect_irq(struct msm_gpu *gpu)
1234{
1235	struct drm_device *dev = gpu->dev;
1236	struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
1237
1238	/*
1239	 * If stalled on SMMU fault, we could trip the GPU's hang detection,
1240	 * but the fault handler will trigger the devcore dump, and we want
1241	 * to otherwise resume normally rather than killing the submit, so
1242	 * just bail.
1243	 */
1244	if (gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24))
1245		return;
1246
1247	DRM_DEV_ERROR(dev->dev, "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
1248		ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,
1249		gpu_read(gpu, REG_A5XX_RBBM_STATUS),
1250		gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
1251		gpu_read(gpu, REG_A5XX_CP_RB_WPTR),
1252		gpu_read64(gpu, REG_A5XX_CP_IB1_BASE),
1253		gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),
1254		gpu_read64(gpu, REG_A5XX_CP_IB2_BASE),
1255		gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));
1256
1257	/* Turn off the hangcheck timer to keep it from bothering us */
1258	timer_delete(&gpu->hangcheck_timer);
1259
1260	kthread_queue_work(gpu->worker, &gpu->recover_work);
1261}
1262
1263#define RBBM_ERROR_MASK \
1264	(A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
1265	A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
1266	A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
1267	A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
1268	A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
1269	A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
1270
1271static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
1272{
1273	struct msm_drm_private *priv = gpu->dev->dev_private;
1274	u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
1275
1276	/*
1277	 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
1278	 * before the source is cleared the interrupt will storm.
1279	 */
1280	gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
1281		status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
1282
1283	if (priv->disable_err_irq) {
1284		status &= A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS |
1285			  A5XX_RBBM_INT_0_MASK_CP_SW;
1286	}
1287
1288	/* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
1289	if (status & RBBM_ERROR_MASK)
1290		a5xx_rbbm_err_irq(gpu, status);
1291
1292	if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
1293		a5xx_cp_err_irq(gpu);
1294
1295	if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT)
1296		a5xx_fault_detect_irq(gpu);
1297
1298	if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
1299		a5xx_uche_err_irq(gpu);
1300
1301	if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
1302		a5xx_gpmu_err_irq(gpu);
1303
1304	if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) {
1305		a5xx_preempt_trigger(gpu);
1306		msm_gpu_retire(gpu);
1307	}
1308
1309	if (status & A5XX_RBBM_INT_0_MASK_CP_SW)
1310		a5xx_preempt_irq(gpu);
1311
1312	return IRQ_HANDLED;
1313}
1314
1315static const u32 a5xx_registers[] = {
1316	0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
1317	0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
1318	0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
1319	0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841,
1320	0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28,
1321	0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53,
1322	0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98,
1323	0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585,
1324	0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8,
1325	0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E,
1326	0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545,
1327	0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0,
1328	0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57,
1329	0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8,
1330	0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9,
1331	0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201,
1332	0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A,
1333	0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F,
1334	0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0,
1335	0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947,
1336	0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7,
1337	0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68,
1338	0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
1339	0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
1340	0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
1341	0xEAA5, 0xEAC2, 0xA800, 0xA800, 0xA820, 0xA828, 0xA840, 0xA87D,
1342	0XA880, 0xA88D, 0xA890, 0xA8A3, 0xA8D0, 0xA8D8, 0xA8E0, 0xA8F5,
1343	0xAC60, 0xAC60, ~0,
1344};
1345
1346static void a5xx_dump(struct msm_gpu *gpu)
1347{
1348	DRM_DEV_INFO(gpu->dev->dev, "status:   %08x\n",
1349		gpu_read(gpu, REG_A5XX_RBBM_STATUS));
1350	adreno_dump(gpu);
1351}
1352
1353static int a5xx_pm_resume(struct msm_gpu *gpu)
1354{
1355	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1356	int ret;
1357
1358	/* Turn on the core power */
1359	ret = msm_gpu_pm_resume(gpu);
1360	if (ret)
1361		return ret;
1362
1363	/* Adreno 505, 506, 508, 509, 510, 512 needs manual RBBM sus/res control */
1364	if (!(adreno_is_a530(adreno_gpu) || adreno_is_a540(adreno_gpu))) {
1365		/* Halt the sp_input_clk at HM level */
1366		gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055);
1367		a5xx_set_hwcg(gpu, true);
1368		/* Turn on sp_input_clk at HM level */
1369		gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0);
1370		return 0;
1371	}
1372
1373	/* Turn the RBCCU domain first to limit the chances of voltage droop */
1374	gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
1375
1376	/* Wait 3 usecs before polling */
1377	udelay(3);
1378
1379	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
1380		(1 << 20), (1 << 20));
1381	if (ret) {
1382		DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
1383			gpu->name,
1384			gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
1385		return ret;
1386	}
1387
1388	/* Turn on the SP domain */
1389	gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
1390	ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
1391		(1 << 20), (1 << 20));
1392	if (ret)
1393		DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
1394			gpu->name);
1395
1396	return ret;
1397}
1398
1399static int a5xx_pm_suspend(struct msm_gpu *gpu)
1400{
1401	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1402	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1403	u32 mask = 0xf;
1404	int i, ret;
1405
1406	/* A505, A506, A508, A510 have 3 XIN ports in VBIF */
1407	if (adreno_is_a505(adreno_gpu) || adreno_is_a506(adreno_gpu) ||
1408	    adreno_is_a508(adreno_gpu) || adreno_is_a510(adreno_gpu))
1409		mask = 0x7;
1410
1411	/* Clear the VBIF pipe before shutting down */
1412	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask);
1413	spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) &
1414				mask) == mask);
1415
1416	gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
1417
1418	/*
1419	 * Reset the VBIF before power collapse to avoid issue with FIFO
1420	 * entries on Adreno A510 and A530 (the others will tend to lock up)
1421	 */
1422	if (adreno_is_a510(adreno_gpu) || adreno_is_a530(adreno_gpu)) {
1423		gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
1424		gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
1425	}
1426
1427	ret = msm_gpu_pm_suspend(gpu);
1428	if (ret)
1429		return ret;
1430
1431	if (a5xx_gpu->has_whereami)
1432		for (i = 0; i < gpu->nr_rings; i++)
1433			a5xx_gpu->shadow[i] = 0;
1434
1435	return 0;
1436}
1437
1438static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
1439{
1440	*value = gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO);
1441
1442	return 0;
1443}
1444
1445struct a5xx_crashdumper {
1446	void *ptr;
1447	struct drm_gem_object *bo;
1448	u64 iova;
1449};
1450
1451struct a5xx_gpu_state {
1452	struct msm_gpu_state base;
1453	u32 *hlsqregs;
1454};
1455
1456static int a5xx_crashdumper_init(struct msm_gpu *gpu,
1457		struct a5xx_crashdumper *dumper)
1458{
1459	dumper->ptr = msm_gem_kernel_new(gpu->dev,
1460		SZ_1M, MSM_BO_WC, gpu->vm,
1461		&dumper->bo, &dumper->iova);
1462
1463	if (!IS_ERR(dumper->ptr))
1464		msm_gem_object_set_name(dumper->bo, "crashdump");
1465
1466	return PTR_ERR_OR_ZERO(dumper->ptr);
1467}
1468
1469static int a5xx_crashdumper_run(struct msm_gpu *gpu,
1470		struct a5xx_crashdumper *dumper)
1471{
1472	u32 val;
1473
1474	if (IS_ERR_OR_NULL(dumper->ptr))
1475		return -EINVAL;
1476
1477	gpu_write64(gpu, REG_A5XX_CP_CRASH_SCRIPT_BASE_LO, dumper->iova);
1478
1479	gpu_write(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, 1);
1480
1481	return gpu_poll_timeout(gpu, REG_A5XX_CP_CRASH_DUMP_CNTL, val,
1482		val & 0x04, 100, 10000);
1483}
1484
1485/*
1486 * These are a list of the registers that need to be read through the HLSQ
1487 * aperture through the crashdumper.  These are not nominally accessible from
1488 * the CPU on a secure platform.
1489 */
1490static const struct {
1491	u32 type;
1492	u32 regoffset;
1493	u32 count;
1494} a5xx_hlsq_aperture_regs[] = {
1495	{ 0x35, 0xe00, 0x32 },   /* HSLQ non-context */
1496	{ 0x31, 0x2080, 0x1 },   /* HLSQ 2D context 0 */
1497	{ 0x33, 0x2480, 0x1 },   /* HLSQ 2D context 1 */
1498	{ 0x32, 0xe780, 0x62 },  /* HLSQ 3D context 0 */
1499	{ 0x34, 0xef80, 0x62 },  /* HLSQ 3D context 1 */
1500	{ 0x3f, 0x0ec0, 0x40 },  /* SP non-context */
1501	{ 0x3d, 0x2040, 0x1 },   /* SP 2D context 0 */
1502	{ 0x3b, 0x2440, 0x1 },   /* SP 2D context 1 */
1503	{ 0x3e, 0xe580, 0x170 }, /* SP 3D context 0 */
1504	{ 0x3c, 0xed80, 0x170 }, /* SP 3D context 1 */
1505	{ 0x3a, 0x0f00, 0x1c },  /* TP non-context */
1506	{ 0x38, 0x2000, 0xa },   /* TP 2D context 0 */
1507	{ 0x36, 0x2400, 0xa },   /* TP 2D context 1 */
1508	{ 0x39, 0xe700, 0x80 },  /* TP 3D context 0 */
1509	{ 0x37, 0xef00, 0x80 },  /* TP 3D context 1 */
1510};
1511
1512static void a5xx_gpu_state_get_hlsq_regs(struct msm_gpu *gpu,
1513		struct a5xx_gpu_state *a5xx_state)
1514{
1515	struct a5xx_crashdumper dumper = { 0 };
1516	u32 offset, count = 0;
1517	u64 *ptr;
1518	int i;
1519
1520	if (a5xx_crashdumper_init(gpu, &dumper))
1521		return;
1522
1523	/* The script will be written at offset 0 */
1524	ptr = dumper.ptr;
1525
1526	/* Start writing the data at offset 256k */
1527	offset = dumper.iova + (256 * SZ_1K);
1528
1529	/* Count how many additional registers to get from the HLSQ aperture */
1530	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++)
1531		count += a5xx_hlsq_aperture_regs[i].count;
1532
1533	a5xx_state->hlsqregs = kcalloc(count, sizeof(u32), GFP_KERNEL);
1534	if (!a5xx_state->hlsqregs)
1535		return;
1536
1537	/* Build the crashdump script */
1538	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1539		u32 type = a5xx_hlsq_aperture_regs[i].type;
1540		u32 c = a5xx_hlsq_aperture_regs[i].count;
1541
1542		/* Write the register to select the desired bank */
1543		*ptr++ = ((u64) type << 8);
1544		*ptr++ = (((u64) REG_A5XX_HLSQ_DBG_READ_SEL) << 44) |
1545			(1 << 21) | 1;
1546
1547		*ptr++ = offset;
1548		*ptr++ = (((u64) REG_A5XX_HLSQ_DBG_AHB_READ_APERTURE) << 44)
1549			| c;
1550
1551		offset += c * sizeof(u32);
1552	}
1553
1554	/* Write two zeros to close off the script */
1555	*ptr++ = 0;
1556	*ptr++ = 0;
1557
1558	if (a5xx_crashdumper_run(gpu, &dumper)) {
1559		kfree(a5xx_state->hlsqregs);
1560		msm_gem_kernel_put(dumper.bo, gpu->vm);
1561		return;
1562	}
1563
1564	/* Copy the data from the crashdumper to the state */
1565	memcpy(a5xx_state->hlsqregs, dumper.ptr + (256 * SZ_1K),
1566		count * sizeof(u32));
1567
1568	msm_gem_kernel_put(dumper.bo, gpu->vm);
1569}
1570
1571static struct msm_gpu_state *a5xx_gpu_state_get(struct msm_gpu *gpu)
1572{
1573	struct a5xx_gpu_state *a5xx_state = kzalloc(sizeof(*a5xx_state),
1574			GFP_KERNEL);
1575	bool stalled = !!(gpu_read(gpu, REG_A5XX_RBBM_STATUS3) & BIT(24));
1576
1577	if (!a5xx_state)
1578		return ERR_PTR(-ENOMEM);
1579
1580	/* Temporarily disable hardware clock gating before reading the hw */
1581	a5xx_set_hwcg(gpu, false);
1582
1583	/* First get the generic state from the adreno core */
1584	adreno_gpu_state_get(gpu, &(a5xx_state->base));
1585
1586	a5xx_state->base.rbbm_status = gpu_read(gpu, REG_A5XX_RBBM_STATUS);
1587
1588	/*
1589	 * Get the HLSQ regs with the help of the crashdumper, but only if
1590	 * we are not stalled in an iommu fault (in which case the crashdumper
1591	 * would not have access to memory)
1592	 */
1593	if (!stalled)
1594		a5xx_gpu_state_get_hlsq_regs(gpu, a5xx_state);
1595
1596	a5xx_set_hwcg(gpu, true);
1597
1598	return &a5xx_state->base;
1599}
1600
1601static void a5xx_gpu_state_destroy(struct kref *kref)
1602{
1603	struct msm_gpu_state *state = container_of(kref,
1604		struct msm_gpu_state, ref);
1605	struct a5xx_gpu_state *a5xx_state = container_of(state,
1606		struct a5xx_gpu_state, base);
1607
1608	kfree(a5xx_state->hlsqregs);
1609
1610	adreno_gpu_state_destroy(state);
1611	kfree(a5xx_state);
1612}
1613
1614static int a5xx_gpu_state_put(struct msm_gpu_state *state)
1615{
1616	if (IS_ERR_OR_NULL(state))
1617		return 1;
1618
1619	return kref_put(&state->ref, a5xx_gpu_state_destroy);
1620}
1621
1622
1623#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1624static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
1625		      struct drm_printer *p)
1626{
1627	int i, j;
1628	u32 pos = 0;
1629	struct a5xx_gpu_state *a5xx_state = container_of(state,
1630		struct a5xx_gpu_state, base);
1631
1632	if (IS_ERR_OR_NULL(state))
1633		return;
1634
1635	adreno_show(gpu, state, p);
1636
1637	/* Dump the additional a5xx HLSQ registers */
1638	if (!a5xx_state->hlsqregs)
1639		return;
1640
1641	drm_printf(p, "registers-hlsq:\n");
1642
1643	for (i = 0; i < ARRAY_SIZE(a5xx_hlsq_aperture_regs); i++) {
1644		u32 o = a5xx_hlsq_aperture_regs[i].regoffset;
1645		u32 c = a5xx_hlsq_aperture_regs[i].count;
1646
1647		for (j = 0; j < c; j++, pos++, o++) {
1648			/*
1649			 * To keep the crashdump simple we pull the entire range
1650			 * for each register type but not all of the registers
1651			 * in the range are valid. Fortunately invalid registers
1652			 * stick out like a sore thumb with a value of
1653			 * 0xdeadbeef
1654			 */
1655			if (a5xx_state->hlsqregs[pos] == 0xdeadbeef)
1656				continue;
1657
1658			drm_printf(p, "  - { offset: 0x%04x, value: 0x%08x }\n",
1659				o << 2, a5xx_state->hlsqregs[pos]);
1660		}
1661	}
1662}
1663#endif
1664
1665static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)
1666{
1667	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1668	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1669
1670	return a5xx_gpu->cur_ring;
1671}
1672
1673static u64 a5xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
1674{
1675	u64 busy_cycles;
1676
1677	busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO);
1678	*out_sample_rate = clk_get_rate(gpu->core_clk);
1679
1680	return busy_cycles;
1681}
1682
1683static uint32_t a5xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
1684{
1685	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1686	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
1687
1688	if (a5xx_gpu->has_whereami)
1689		return a5xx_gpu->shadow[ring->id];
1690
1691	return ring->memptrs->rptr = gpu_read(gpu, REG_A5XX_CP_RB_RPTR);
1692}
1693
1694static void check_speed_bin(struct device *dev)
1695{
1696	struct nvmem_cell *cell;
1697	u32 val;
1698
1699	/*
1700	 * If the OPP table specifies a opp-supported-hw property then we have
1701	 * to set something with dev_pm_opp_set_supported_hw() or the table
1702	 * doesn't get populated so pick an arbitrary value that should
1703	 * ensure the default frequencies are selected but not conflict with any
1704	 * actual bins
1705	 */
1706	val = 0x80;
1707
1708	cell = nvmem_cell_get(dev, "speed_bin");
1709
1710	if (!IS_ERR(cell)) {
1711		void *buf = nvmem_cell_read(cell, NULL);
1712
1713		if (!IS_ERR(buf)) {
1714			u8 bin = *((u8 *) buf);
1715
1716			val = (1 << bin);
1717			kfree(buf);
1718		}
1719
1720		nvmem_cell_put(cell);
1721	}
1722
1723	devm_pm_opp_set_supported_hw(dev, &val, 1);
1724}
1725
1726static struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
1727{
1728	struct msm_drm_private *priv = dev->dev_private;
1729	struct platform_device *pdev = priv->gpu_pdev;
1730	struct adreno_platform_config *config = pdev->dev.platform_data;
1731	const struct qcom_ubwc_cfg_data *common_cfg;
1732	struct a5xx_gpu *a5xx_gpu = NULL;
1733	struct adreno_gpu *adreno_gpu;
1734	struct msm_gpu *gpu;
1735	unsigned int nr_rings;
1736	int ret;
1737
1738	a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1739	if (!a5xx_gpu)
1740		return ERR_PTR(-ENOMEM);
1741
1742	adreno_gpu = &a5xx_gpu->base;
1743	gpu = &adreno_gpu->base;
1744
1745	adreno_gpu->registers = a5xx_registers;
1746
1747	a5xx_gpu->lm_leakage = 0x4E001A;
1748
1749	check_speed_bin(&pdev->dev);
1750
1751	nr_rings = 4;
1752
1753	if (config->info->revn == 510)
1754		nr_rings = 1;
1755
1756	ret = adreno_gpu_init(dev, pdev, adreno_gpu, config->info->funcs, nr_rings);
1757	if (ret) {
1758		a5xx_destroy(&(a5xx_gpu->base.base));
1759		return ERR_PTR(ret);
1760	}
1761
1762	msm_mmu_set_fault_handler(to_msm_vm(gpu->vm)->mmu, gpu,
1763				  a5xx_fault_handler);
1764
1765	/* Set up the preemption specific bits and pieces for each ringbuffer */
1766	a5xx_preempt_init(gpu);
1767
1768	/* Inherit the common config and make some necessary fixups */
1769	common_cfg = qcom_ubwc_config_get_data();
1770	if (IS_ERR(common_cfg))
1771		return ERR_CAST(common_cfg);
1772
1773	/* Copy the data into the internal struct to drop the const qualifier (temporarily) */
1774	adreno_gpu->_ubwc_config = *common_cfg;
1775	adreno_gpu->ubwc_config = &adreno_gpu->_ubwc_config;
1776
1777	adreno_gpu->uche_trap_base = 0x0001ffffffff0000ull;
1778
1779	return gpu;
1780}
1781
1782const struct adreno_gpu_funcs a5xx_gpu_funcs = {
1783	.base = {
1784		.get_param = adreno_get_param,
1785		.set_param = adreno_set_param,
1786		.hw_init = a5xx_hw_init,
1787		.ucode_load = a5xx_ucode_load,
1788		.pm_suspend = a5xx_pm_suspend,
1789		.pm_resume = a5xx_pm_resume,
1790		.recover = a5xx_recover,
1791		.submit = a5xx_submit,
1792		.active_ring = a5xx_active_ring,
1793		.irq = a5xx_irq,
1794		.destroy = a5xx_destroy,
1795#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
1796		.show = a5xx_show,
1797#endif
1798#if defined(CONFIG_DEBUG_FS)
1799		.debugfs_init = a5xx_debugfs_init,
1800#endif
1801		.gpu_busy = a5xx_gpu_busy,
1802		.gpu_state_get = a5xx_gpu_state_get,
1803		.gpu_state_put = a5xx_gpu_state_put,
1804		.create_vm = adreno_create_vm,
1805		.get_rptr = a5xx_get_rptr,
1806	},
1807	.init = a5xx_gpu_init,
1808	.get_timestamp = a5xx_get_timestamp,
1809};