amdgpu_sdma.c

  1/*
  2 * Copyright 2018 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23
 24#include <linux/firmware.h>
 25#include "amdgpu.h"
 26#include "amdgpu_sdma.h"
 27#include "amdgpu_ras.h"
 28#include "amdgpu_reset.h"
 29#include "gc/gc_10_1_0_offset.h"
 30#include "gc/gc_10_3_0_sh_mask.h"
 31
 32#define AMDGPU_CSA_SDMA_SIZE 64
 33/* SDMA CSA reside in the 3rd page of CSA */
 34#define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
 35
 36/*
 37 * GPU SDMA IP block helpers function.
 38 */
 39
 40struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
 41{
 42	struct amdgpu_device *adev = ring->adev;
 43	int i;
 44
 45	for (i = 0; i < adev->sdma.num_instances; i++)
 46		if (ring == &adev->sdma.instance[i].ring ||
 47		    ring == &adev->sdma.instance[i].page)
 48			return &adev->sdma.instance[i];
 49
 50	return NULL;
 51}
 52
 53int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
 54{
 55	struct amdgpu_device *adev = ring->adev;
 56	int i;
 57
 58	for (i = 0; i < adev->sdma.num_instances; i++) {
 59		if (ring == &adev->sdma.instance[i].ring ||
 60			ring == &adev->sdma.instance[i].page) {
 61			*index = i;
 62			return 0;
 63		}
 64	}
 65
 66	return -EINVAL;
 67}
 68
 69uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
 70				     unsigned int vmid)
 71{
 72	struct amdgpu_device *adev = ring->adev;
 73	uint64_t csa_mc_addr;
 74	uint32_t index = 0;
 75	int r;
 76
 77	/* don't enable OS preemption on SDMA under SRIOV */
 78	if (amdgpu_sriov_vf(adev) || vmid == 0 || !adev->gfx.mcbp)
 79		return 0;
 80
 81	r = amdgpu_sdma_get_index_from_ring(ring, &index);
 82
 83	if (r || index > 31)
 84		csa_mc_addr = 0;
 85	else
 86		csa_mc_addr = amdgpu_csa_vaddr(adev) +
 87			AMDGPU_CSA_SDMA_OFFSET +
 88			index * AMDGPU_CSA_SDMA_SIZE;
 89
 90	return csa_mc_addr;
 91}
 92
 93int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
 94			      struct ras_common_if *ras_block)
 95{
 96	int r, i;
 97
 98	r = amdgpu_ras_block_late_init(adev, ras_block);
 99	if (r)
100		return r;
101
102	if (amdgpu_ras_is_supported(adev, ras_block->block)) {
103		for (i = 0; i < adev->sdma.num_instances; i++) {
104			r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,
105				AMDGPU_SDMA_IRQ_INSTANCE0 + i);
106			if (r)
107				goto late_fini;
108		}
109	}
110
111	return 0;
112
113late_fini:
114	amdgpu_ras_block_late_fini(adev, ras_block);
115	return r;
116}
117
118int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
119		void *err_data,
120		struct amdgpu_iv_entry *entry)
121{
122	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
123
124	if (amdgpu_sriov_vf(adev))
125		return AMDGPU_RAS_SUCCESS;
126
127	amdgpu_ras_reset_gpu(adev);
128
129	return AMDGPU_RAS_SUCCESS;
130}
131
132int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
133				      struct amdgpu_irq_src *source,
134				      struct amdgpu_iv_entry *entry)
135{
136	struct ras_common_if *ras_if = adev->sdma.ras_if;
137	struct ras_dispatch_if ih_data = {
138		.entry = entry,
139	};
140
141	if (!ras_if)
142		return 0;
143
144	ih_data.head = *ras_if;
145
146	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
147	return 0;
148}
149
150static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
151{
152	uint16_t version_major;
153	const struct common_firmware_header *header = NULL;
154	const struct sdma_firmware_header_v1_0 *hdr;
155	const struct sdma_firmware_header_v2_0 *hdr_v2;
156	const struct sdma_firmware_header_v3_0 *hdr_v3;
157
158	header = (const struct common_firmware_header *)
159		sdma_inst->fw->data;
160	version_major = le16_to_cpu(header->header_version_major);
161
162	switch (version_major) {
163	case 1:
164		hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
165		sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
166		sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
167		break;
168	case 2:
169		hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
170		sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
171		sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
172		break;
173	case 3:
174		hdr_v3 = (const struct sdma_firmware_header_v3_0 *)sdma_inst->fw->data;
175		sdma_inst->fw_version = le32_to_cpu(hdr_v3->header.ucode_version);
176		sdma_inst->feature_version = le32_to_cpu(hdr_v3->ucode_feature_version);
177		break;
178	default:
179		return -EINVAL;
180	}
181
182	if (sdma_inst->feature_version >= 20)
183		sdma_inst->burst_nop = true;
184
185	return 0;
186}
187
188void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
189				  bool duplicate)
190{
191	int i;
192
193	for (i = 0; i < adev->sdma.num_instances; i++) {
194		amdgpu_ucode_release(&adev->sdma.instance[i].fw);
195		if (duplicate)
196			break;
197	}
198
199	memset((void *)adev->sdma.instance, 0,
200	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
201}
202
203int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
204			       u32 instance, bool duplicate)
205{
206	struct amdgpu_firmware_info *info = NULL;
207	const struct common_firmware_header *header = NULL;
208	int err, i;
209	const struct sdma_firmware_header_v2_0 *sdma_hdr;
210	const struct sdma_firmware_header_v3_0 *sdma_hv3;
211	uint16_t version_major;
212	char ucode_prefix[30];
213
214	amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix));
215	if (instance == 0)
216		err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
217					   AMDGPU_UCODE_REQUIRED,
218					   "amdgpu/%s.bin", ucode_prefix);
219	else
220		err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,
221					   AMDGPU_UCODE_REQUIRED,
222					   "amdgpu/%s%d.bin", ucode_prefix, instance);
223	if (err)
224		goto out;
225
226	header = (const struct common_firmware_header *)
227		adev->sdma.instance[instance].fw->data;
228	version_major = le16_to_cpu(header->header_version_major);
229
230	if ((duplicate && instance) || (!duplicate && version_major > 1)) {
231		err = -EINVAL;
232		goto out;
233	}
234
235	err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
236	if (err)
237		goto out;
238
239	if (duplicate) {
240		for (i = 1; i < adev->sdma.num_instances; i++)
241			memcpy((void *)&adev->sdma.instance[i],
242			       (void *)&adev->sdma.instance[0],
243			       sizeof(struct amdgpu_sdma_instance));
244	}
245
246	DRM_DEBUG("psp_load == '%s'\n",
247		  adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
248
249	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
250		switch (version_major) {
251		case 1:
252			for (i = 0; i < adev->sdma.num_instances; i++) {
253				if (!duplicate && (instance != i))
254					continue;
255				else {
256					/* Use a single copy per SDMA firmware type. PSP uses the same instance for all
257					 * groups of SDMAs */
258					if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
259						IP_VERSION(4, 4, 2) ||
260					     amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
261						IP_VERSION(4, 4, 4) ||
262					     amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==
263						IP_VERSION(4, 4, 5)) &&
264					    adev->firmware.load_type ==
265						AMDGPU_FW_LOAD_PSP &&
266					    adev->sdma.num_inst_per_aid == i) {
267						break;
268					}
269					info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
270					info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
271					info->fw = adev->sdma.instance[i].fw;
272					adev->firmware.fw_size +=
273						ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
274				}
275			}
276			break;
277		case 2:
278			sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
279				adev->sdma.instance[0].fw->data;
280			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
281			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
282			info->fw = adev->sdma.instance[0].fw;
283			adev->firmware.fw_size +=
284				ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
285			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
286			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
287			info->fw = adev->sdma.instance[0].fw;
288			adev->firmware.fw_size +=
289				ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
290			break;
291		case 3:
292			sdma_hv3 = (const struct sdma_firmware_header_v3_0 *)
293				adev->sdma.instance[0].fw->data;
294			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_RS64];
295			info->ucode_id = AMDGPU_UCODE_ID_SDMA_RS64;
296			info->fw = adev->sdma.instance[0].fw;
297			adev->firmware.fw_size +=
298				ALIGN(le32_to_cpu(sdma_hv3->ucode_size_bytes), PAGE_SIZE);
299			break;
300		default:
301			err = -EINVAL;
302		}
303	}
304
305out:
306	if (err)
307		amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
308	return err;
309}
310
311int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev)
312{
313	int err = 0;
314	struct amdgpu_sdma_ras *ras = NULL;
315
316	/* adev->sdma.ras is NULL, which means sdma does not
317	 * support ras function, then do nothing here.
318	 */
319	if (!adev->sdma.ras)
320		return 0;
321
322	ras = adev->sdma.ras;
323
324	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
325	if (err) {
326		dev_err(adev->dev, "Failed to register sdma ras block!\n");
327		return err;
328	}
329
330	strcpy(ras->ras_block.ras_comm.name, "sdma");
331	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA;
332	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
333	adev->sdma.ras_if = &ras->ras_block.ras_comm;
334
335	/* If not define special ras_late_init function, use default ras_late_init */
336	if (!ras->ras_block.ras_late_init)
337		ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init;
338
339	/* If not defined special ras_cb function, use default ras_cb */
340	if (!ras->ras_block.ras_cb)
341		ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb;
342
343	return 0;
344}
345
346/*
347 * debugfs for to enable/disable sdma job submission to specific core.
348 */
349#if defined(CONFIG_DEBUG_FS)
350static int amdgpu_debugfs_sdma_sched_mask_set(void *data, u64 val)
351{
352	struct amdgpu_device *adev = (struct amdgpu_device *)data;
353	u64 i, num_ring;
354	u64 mask = 0;
355	struct amdgpu_ring *ring, *page = NULL;
356
357	if (!adev)
358		return -ENODEV;
359
360	/* Determine the number of rings per SDMA instance
361	 * (1 for sdma gfx ring, 2 if page queue exists)
362	 */
363	if (adev->sdma.has_page_queue)
364		num_ring = 2;
365	else
366		num_ring = 1;
367
368	/* Calculate the maximum possible mask value
369	 * based on the number of SDMA instances and rings
370	*/
371	mask = BIT_ULL(adev->sdma.num_instances * num_ring) - 1;
372
373	if ((val & mask) == 0)
374		return -EINVAL;
375
376	for (i = 0; i < adev->sdma.num_instances; ++i) {
377		ring = &adev->sdma.instance[i].ring;
378		if (adev->sdma.has_page_queue)
379			page = &adev->sdma.instance[i].page;
380		if (val & BIT_ULL(i * num_ring))
381			ring->sched.ready = true;
382		else
383			ring->sched.ready = false;
384
385		if (page) {
386			if (val & BIT_ULL(i * num_ring + 1))
387				page->sched.ready = true;
388			else
389				page->sched.ready = false;
390		}
391	}
392	/* publish sched.ready flag update effective immediately across smp */
393	smp_rmb();
394	return 0;
395}
396
397static int amdgpu_debugfs_sdma_sched_mask_get(void *data, u64 *val)
398{
399	struct amdgpu_device *adev = (struct amdgpu_device *)data;
400	u64 i, num_ring;
401	u64 mask = 0;
402	struct amdgpu_ring *ring, *page = NULL;
403
404	if (!adev)
405		return -ENODEV;
406
407	/* Determine the number of rings per SDMA instance
408	 * (1 for sdma gfx ring, 2 if page queue exists)
409	 */
410	if (adev->sdma.has_page_queue)
411		num_ring = 2;
412	else
413		num_ring = 1;
414
415	for (i = 0; i < adev->sdma.num_instances; ++i) {
416		ring = &adev->sdma.instance[i].ring;
417		if (adev->sdma.has_page_queue)
418			page = &adev->sdma.instance[i].page;
419
420		if (ring->sched.ready)
421			mask |= BIT_ULL(i * num_ring);
422		else
423			mask &= ~BIT_ULL(i * num_ring);
424
425		if (page) {
426			if (page->sched.ready)
427				mask |= BIT_ULL(i * num_ring + 1);
428			else
429				mask &= ~BIT_ULL(i * num_ring + 1);
430		}
431	}
432
433	*val = mask;
434	return 0;
435}
436
437DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_sdma_sched_mask_fops,
438			 amdgpu_debugfs_sdma_sched_mask_get,
439			 amdgpu_debugfs_sdma_sched_mask_set, "%llx\n");
440
441#endif
442
443void amdgpu_debugfs_sdma_sched_mask_init(struct amdgpu_device *adev)
444{
445#if defined(CONFIG_DEBUG_FS)
446	struct drm_minor *minor = adev_to_drm(adev)->primary;
447	struct dentry *root = minor->debugfs_root;
448	char name[32];
449
450	if (!(adev->sdma.num_instances > 1))
451		return;
452	sprintf(name, "amdgpu_sdma_sched_mask");
453	debugfs_create_file(name, 0600, root, adev,
454			    &amdgpu_debugfs_sdma_sched_mask_fops);
455#endif
456}
457
458static ssize_t amdgpu_get_sdma_reset_mask(struct device *dev,
459						struct device_attribute *attr,
460						char *buf)
461{
462	struct drm_device *ddev = dev_get_drvdata(dev);
463	struct amdgpu_device *adev = drm_to_adev(ddev);
464
465	if (!adev)
466		return -ENODEV;
467
468	return amdgpu_show_reset_mask(buf, adev->sdma.supported_reset);
469}
470
471static DEVICE_ATTR(sdma_reset_mask, 0444,
472		   amdgpu_get_sdma_reset_mask, NULL);
473
474int amdgpu_sdma_sysfs_reset_mask_init(struct amdgpu_device *adev)
475{
476	int r = 0;
477
478	if (!amdgpu_gpu_recovery)
479		return r;
480
481	if (adev->sdma.num_instances) {
482		r = device_create_file(adev->dev, &dev_attr_sdma_reset_mask);
483		if (r)
484			return r;
485	}
486
487	return r;
488}
489
490void amdgpu_sdma_sysfs_reset_mask_fini(struct amdgpu_device *adev)
491{
492	if (!amdgpu_gpu_recovery)
493		return;
494
495	if (adev->dev->kobj.sd) {
496		if (adev->sdma.num_instances)
497			device_remove_file(adev->dev, &dev_attr_sdma_reset_mask);
498	}
499}
500
501struct amdgpu_ring *amdgpu_sdma_get_shared_ring(struct amdgpu_device *adev, struct amdgpu_ring *ring)
502{
503	if (adev->sdma.has_page_queue &&
504	    (ring->me < adev->sdma.num_instances) &&
505	    (ring == &adev->sdma.instance[ring->me].ring))
506		return &adev->sdma.instance[ring->me].page;
507	else
508		return NULL;
509}
510
511/**
512* amdgpu_sdma_is_shared_inv_eng - Check if a ring is an SDMA ring that shares a VM invalidation engine
513* @adev: Pointer to the AMDGPU device structure
514* @ring: Pointer to the ring structure to check
515*
516* This function checks if the given ring is an SDMA ring that shares a VM invalidation engine.
517* It returns true if the ring is such an SDMA ring, false otherwise.
518*/
519bool amdgpu_sdma_is_shared_inv_eng(struct amdgpu_device *adev, struct amdgpu_ring *ring)
520{
521	int i = ring->me;
522
523	if (!adev->sdma.has_page_queue || i >= adev->sdma.num_instances)
524		return false;
525
526	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
527	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
528	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0))
529		return (ring == &adev->sdma.instance[i].page);
530	else
531		return false;
532}
533
534static int amdgpu_sdma_soft_reset(struct amdgpu_device *adev, u32 instance_id)
535{
536	struct amdgpu_sdma_instance *sdma_instance = &adev->sdma.instance[instance_id];
537
538	if (sdma_instance->funcs->soft_reset_kernel_queue)
539		return sdma_instance->funcs->soft_reset_kernel_queue(adev, instance_id);
540
541	return -EOPNOTSUPP;
542}
543
544/**
545 * amdgpu_sdma_reset_engine - Reset a specific SDMA engine
546 * @adev: Pointer to the AMDGPU device
547 * @instance_id: Logical ID of the SDMA engine instance to reset
548 * @caller_handles_kernel_queues: Skip kernel queue processing. Caller
549 * will handle it.
550 *
551 * Returns: 0 on success, or a negative error code on failure.
552 */
553int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
554			     bool caller_handles_kernel_queues)
555{
556	int ret = 0;
557	struct amdgpu_sdma_instance *sdma_instance = &adev->sdma.instance[instance_id];
558	struct amdgpu_ring *gfx_ring = &sdma_instance->ring;
559	struct amdgpu_ring *page_ring = &sdma_instance->page;
560
561	if (amdgpu_sriov_vf(adev))
562		return -EOPNOTSUPP;
563
564	mutex_lock(&sdma_instance->engine_reset_mutex);
565
566	if (!caller_handles_kernel_queues) {
567		/* Stop the scheduler's work queue for the GFX and page rings if they are running.
568		 * This ensures that no new tasks are submitted to the queues while
569		 * the reset is in progress.
570		 */
571		drm_sched_wqueue_stop(&gfx_ring->sched);
572
573		if (adev->sdma.has_page_queue)
574			drm_sched_wqueue_stop(&page_ring->sched);
575	}
576
577	if (sdma_instance->funcs->stop_kernel_queue) {
578		sdma_instance->funcs->stop_kernel_queue(gfx_ring);
579		if (adev->sdma.has_page_queue)
580			sdma_instance->funcs->stop_kernel_queue(page_ring);
581	}
582
583	/* Perform the SDMA reset for the specified instance */
584	ret = amdgpu_sdma_soft_reset(adev, instance_id);
585	if (ret) {
586		dev_err(adev->dev, "Failed to reset SDMA logical instance %u\n", instance_id);
587		goto exit;
588	}
589
590	if (sdma_instance->funcs->start_kernel_queue) {
591		sdma_instance->funcs->start_kernel_queue(gfx_ring);
592		if (adev->sdma.has_page_queue)
593			sdma_instance->funcs->start_kernel_queue(page_ring);
594	}
595
596exit:
597	if (!caller_handles_kernel_queues) {
598		/* Restart the scheduler's work queue for the GFX and page rings
599		 * if they were stopped by this function. This allows new tasks
600		 * to be submitted to the queues after the reset is complete.
601		 */
602		if (!ret) {
603			amdgpu_fence_driver_force_completion(gfx_ring);
604			drm_sched_wqueue_start(&gfx_ring->sched);
605			if (adev->sdma.has_page_queue) {
606				amdgpu_fence_driver_force_completion(page_ring);
607				drm_sched_wqueue_start(&page_ring->sched);
608			}
609		}
610	}
611	mutex_unlock(&sdma_instance->engine_reset_mutex);
612
613	return ret;
614}