amdgpu_doorbell_mgr.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright 2022 Advanced Micro Devices, Inc.
  4 *
  5 * Permission is hereby granted, free of charge, to any person obtaining a
  6 * copy of this software and associated documentation files (the "Software"),
  7 * to deal in the Software without restriction, including without limitation
  8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9 * and/or sell copies of the Software, and to permit persons to whom the
 10 * Software is furnished to do so, subject to the following conditions:
 11 *
 12 * The above copyright notice and this permission notice shall be included in
 13 * all copies or substantial portions of the Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 21 * OTHER DEALINGS IN THE SOFTWARE.
 22 *
 23 */
 24
 25#include "amdgpu.h"
 26
 27/**
 28 * amdgpu_mm_rdoorbell - read a doorbell dword
 29 *
 30 * @adev: amdgpu_device pointer
 31 * @index: doorbell index
 32 *
 33 * Returns the value in the doorbell aperture at the
 34 * requested doorbell index (CIK).
 35 */
 36u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 37{
 38	if (amdgpu_device_skip_hw_access(adev))
 39		return 0;
 40
 41	if (index < adev->doorbell.num_kernel_doorbells)
 42		return readl(adev->doorbell.cpu_addr + index);
 43
 44	dev_err(adev->dev, "reading beyond doorbell aperture: 0x%08x!\n",
 45		index);
 46	return 0;
 47}
 48
 49/**
 50 * amdgpu_mm_wdoorbell - write a doorbell dword
 51 *
 52 * @adev: amdgpu_device pointer
 53 * @index: doorbell index
 54 * @v: value to write
 55 *
 56 * Writes @v to the doorbell aperture at the
 57 * requested doorbell index (CIK).
 58 */
 59void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 60{
 61	if (amdgpu_device_skip_hw_access(adev))
 62		return;
 63
 64	if (index < adev->doorbell.num_kernel_doorbells)
 65		writel(v, adev->doorbell.cpu_addr + index);
 66	else
 67		dev_err(adev->dev,
 68			"writing beyond doorbell aperture: 0x%08x!\n", index);
 69}
 70
 71/**
 72 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 73 *
 74 * @adev: amdgpu_device pointer
 75 * @index: doorbell index
 76 *
 77 * Returns the value in the doorbell aperture at the
 78 * requested doorbell index (VEGA10+).
 79 */
 80u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 81{
 82	if (amdgpu_device_skip_hw_access(adev))
 83		return 0;
 84
 85	if (index < adev->doorbell.num_kernel_doorbells)
 86		return atomic64_read((atomic64_t *)(adev->doorbell.cpu_addr + index));
 87
 88	dev_err(adev->dev, "reading beyond doorbell aperture: 0x%08x!\n",
 89		index);
 90	return 0;
 91}
 92
 93/**
 94 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 95 *
 96 * @adev: amdgpu_device pointer
 97 * @index: doorbell index
 98 * @v: value to write
 99 *
100 * Writes @v to the doorbell aperture at the
101 * requested doorbell index (VEGA10+).
102 */
103void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
104{
105	if (amdgpu_device_skip_hw_access(adev))
106		return;
107
108	if (index < adev->doorbell.num_kernel_doorbells)
109		atomic64_set((atomic64_t *)(adev->doorbell.cpu_addr + index), v);
110	else
111		dev_err(adev->dev,
112			"writing beyond doorbell aperture: 0x%08x!\n", index);
113}
114
115/**
116 * amdgpu_doorbell_index_on_bar - Find doorbell's absolute offset in BAR
117 *
118 * @adev: amdgpu_device pointer
119 * @db_bo: doorbell object's bo
120 * @doorbell_index: doorbell relative index in this doorbell object
121 * @db_size: doorbell size is in byte
122 *
123 * returns doorbell's absolute index in BAR
124 */
125uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev,
126				      struct amdgpu_bo *db_bo,
127				      uint32_t doorbell_index,
128				      uint32_t db_size)
129{
130	int db_bo_offset;
131
132	db_bo_offset = amdgpu_bo_gpu_offset_no_check(db_bo);
133
134	/* doorbell index is 32 bit but doorbell's size can be 32 bit
135	 * or 64 bit, so *db_size(in byte)/4 for alignment.
136	 */
137	return db_bo_offset / sizeof(u32) + doorbell_index *
138	       DIV_ROUND_UP(db_size, 4);
139}
140
141/**
142 * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells for graphics
143 *
144 * @adev: amdgpu_device pointer
145 *
146 * Creates doorbells for graphics driver usages.
147 * returns 0 on success, error otherwise.
148 */
149int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev)
150{
151	int r;
152	int size;
153
154	/* SI HW does not have doorbells, skip allocation */
155	if (adev->doorbell.num_kernel_doorbells == 0)
156		return 0;
157
158	/* Reserve first num_kernel_doorbells (page-aligned) for kernel ops */
159	size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32), PAGE_SIZE);
160
161	/* Allocate an extra page for MES kernel usages (ring test) */
162	adev->mes.db_start_dw_offset = size / sizeof(u32);
163	size += PAGE_SIZE;
164
165	r = amdgpu_bo_create_kernel(adev,
166				    size,
167				    PAGE_SIZE,
168				    AMDGPU_GEM_DOMAIN_DOORBELL,
169				    &adev->doorbell.kernel_doorbells,
170				    NULL,
171				    (void **)&adev->doorbell.cpu_addr);
172	if (r) {
173		dev_err(adev->dev,
174			"Failed to allocate kernel doorbells, err=%d\n", r);
175		return r;
176	}
177
178	adev->doorbell.num_kernel_doorbells = size / sizeof(u32);
179	return 0;
180}
181
182/*
183 * GPU doorbell aperture helpers function.
184 */
185/**
186 * amdgpu_doorbell_init - Init doorbell driver information.
187 *
188 * @adev: amdgpu_device pointer
189 *
190 * Init doorbell driver information (CIK)
191 * Returns 0 on success, error on failure.
192 */
193int amdgpu_doorbell_init(struct amdgpu_device *adev)
194{
195
196	/* No doorbell on SI hardware generation */
197	if (adev->asic_type < CHIP_BONAIRE) {
198		adev->doorbell.base = 0;
199		adev->doorbell.size = 0;
200		adev->doorbell.num_kernel_doorbells = 0;
201		return 0;
202	}
203
204	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
205		return -EINVAL;
206
207	amdgpu_asic_init_doorbell_index(adev);
208
209	/* doorbell bar mapping */
210	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
211	adev->doorbell.size = pci_resource_len(adev->pdev, 2);
212
213	adev->doorbell.num_kernel_doorbells =
214		min_t(u32, adev->doorbell.size / sizeof(u32),
215		      adev->doorbell_index.max_assignment + 1);
216	if (adev->doorbell.num_kernel_doorbells == 0)
217		return -EINVAL;
218
219	/*
220	 * For Vega, reserve and map two pages on doorbell BAR since SDMA
221	 * paging queue doorbell use the second page. The
222	 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
223	 * doorbells are in the first page. So with paging queue enabled,
224	 * the max num_kernel_doorbells should + 1 page (0x400 in dword)
225	 */
226	if (adev->asic_type >= CHIP_VEGA10)
227		adev->doorbell.num_kernel_doorbells += 0x400;
228
229	return 0;
230}
231
232/**
233 * amdgpu_doorbell_fini - Tear down doorbell driver information.
234 *
235 * @adev: amdgpu_device pointer
236 *
237 * Tear down doorbell driver information (CIK)
238 */
239void amdgpu_doorbell_fini(struct amdgpu_device *adev)
240{
241	amdgpu_bo_free_kernel(&adev->doorbell.kernel_doorbells,
242			      NULL,
243			      (void **)&adev->doorbell.cpu_addr);
244}