amdgpu_job.c

  1/*
  2 * Copyright 2015 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 *
 23 */
 24#include <linux/kthread.h>
 25#include <linux/wait.h>
 26#include <linux/sched.h>
 27
 28#include <drm/drm_drv.h>
 29
 30#include "amdgpu.h"
 31#include "amdgpu_trace.h"
 32#include "amdgpu_reset.h"
 33#include "amdgpu_dev_coredump.h"
 34#include "amdgpu_xgmi.h"
 35
 36static void amdgpu_job_do_core_dump(struct amdgpu_device *adev,
 37				    struct amdgpu_job *job)
 38{
 39	int i;
 40
 41	dev_info(adev->dev, "Dumping IP State\n");
 42	for (i = 0; i < adev->num_ip_blocks; i++)
 43		if (adev->ip_blocks[i].version->funcs->dump_ip_state)
 44			adev->ip_blocks[i].version->funcs
 45				->dump_ip_state((void *)&adev->ip_blocks[i]);
 46	dev_info(adev->dev, "Dumping IP State Completed\n");
 47
 48	amdgpu_coredump(adev, true, false, job);
 49}
 50
 51static void amdgpu_job_core_dump(struct amdgpu_device *adev,
 52				 struct amdgpu_job *job)
 53{
 54	struct list_head device_list, *device_list_handle =  NULL;
 55	struct amdgpu_device *tmp_adev = NULL;
 56	struct amdgpu_hive_info *hive = NULL;
 57
 58	if (!amdgpu_sriov_vf(adev))
 59		hive = amdgpu_get_xgmi_hive(adev);
 60	if (hive)
 61		mutex_lock(&hive->hive_lock);
 62	/*
 63	 * Reuse the logic in amdgpu_device_gpu_recover() to build list of
 64	 * devices for code dump
 65	 */
 66	INIT_LIST_HEAD(&device_list);
 67	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) {
 68		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
 69			list_add_tail(&tmp_adev->reset_list, &device_list);
 70		if (!list_is_first(&adev->reset_list, &device_list))
 71			list_rotate_to_front(&adev->reset_list, &device_list);
 72		device_list_handle = &device_list;
 73	} else {
 74		list_add_tail(&adev->reset_list, &device_list);
 75		device_list_handle = &device_list;
 76	}
 77
 78	/* Do the coredump for each device */
 79	list_for_each_entry(tmp_adev, device_list_handle, reset_list)
 80		amdgpu_job_do_core_dump(tmp_adev, job);
 81
 82	if (hive) {
 83		mutex_unlock(&hive->hive_lock);
 84		amdgpu_put_xgmi_hive(hive);
 85	}
 86}
 87
 88static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 89{
 90	struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched);
 91	struct amdgpu_job *job = to_amdgpu_job(s_job);
 92	struct drm_wedge_task_info *info = NULL;
 93	struct amdgpu_task_info *ti = NULL;
 94	struct amdgpu_device *adev = ring->adev;
 95	int idx, r;
 96
 97	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
 98		dev_info(adev->dev, "%s - device unplugged skipping recovery on scheduler:%s",
 99			 __func__, s_job->sched->name);
100
101		/* Effectively the job is aborted as the device is gone */
102		return DRM_GPU_SCHED_STAT_ENODEV;
103	}
104
105	/*
106	 * Do the coredump immediately after a job timeout to get a very
107	 * close dump/snapshot/representation of GPU's current error status
108	 * Skip it for SRIOV, since VF FLR will be triggered by host driver
109	 * before job timeout
110	 */
111	if (!amdgpu_sriov_vf(adev))
112		amdgpu_job_core_dump(adev, job);
113
114	if (amdgpu_gpu_recovery &&
115	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_SOFT_RESET) &&
116	    amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) {
117		dev_err(adev->dev, "ring %s timeout, but soft recovered\n",
118			s_job->sched->name);
119		goto exit;
120	}
121
122	dev_err(adev->dev, "ring %s timeout, signaled seq=%u, emitted seq=%u\n",
123		job->base.sched->name, atomic_read(&ring->fence_drv.last_seq),
124		ring->fence_drv.sync_seq);
125
126	ti = amdgpu_vm_get_task_info_pasid(ring->adev, job->pasid);
127	if (ti) {
128		amdgpu_vm_print_task_info(adev, ti);
129		info = &ti->task;
130	}
131
132	/* attempt a per ring reset */
133	if (amdgpu_gpu_recovery &&
134	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_PER_QUEUE) &&
135	    ring->funcs->reset) {
136		dev_err(adev->dev, "Starting %s ring reset\n",
137			s_job->sched->name);
138		r = amdgpu_ring_reset(ring, job->vmid, job->hw_fence);
139		if (!r) {
140			atomic_inc(&ring->adev->gpu_reset_counter);
141			dev_err(adev->dev, "Ring %s reset succeeded\n",
142				ring->sched.name);
143			drm_dev_wedged_event(adev_to_drm(adev),
144					     DRM_WEDGE_RECOVERY_NONE, info);
145			goto exit;
146		}
147		dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name);
148	}
149
150	if (dma_fence_get_status(&s_job->s_fence->finished) == 0)
151		dma_fence_set_error(&s_job->s_fence->finished, -ETIME);
152
153	if (amdgpu_device_should_recover_gpu(ring->adev)) {
154		struct amdgpu_reset_context reset_context;
155		memset(&reset_context, 0, sizeof(reset_context));
156
157		reset_context.method = AMD_RESET_METHOD_NONE;
158		reset_context.reset_req_dev = adev;
159		reset_context.src = AMDGPU_RESET_SRC_JOB;
160		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
161
162		/*
163		 * To avoid an unnecessary extra coredump, as we have already
164		 * got the very close representation of GPU's error status
165		 */
166		set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
167
168		r = amdgpu_device_gpu_recover(ring->adev, job, &reset_context);
169		if (r)
170			dev_err(adev->dev, "GPU Recovery Failed: %d\n", r);
171	} else {
172		drm_sched_suspend_timeout(&ring->sched);
173		if (amdgpu_sriov_vf(adev))
174			adev->virt.tdr_debug = true;
175	}
176
177exit:
178	amdgpu_vm_put_task_info(ti);
179	drm_dev_exit(idx);
180	return DRM_GPU_SCHED_STAT_RESET;
181}
182
183int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
184		     struct drm_sched_entity *entity, void *owner,
185		     unsigned int num_ibs, struct amdgpu_job **job,
186		     u64 drm_client_id)
187{
188	struct amdgpu_fence *af;
189	int r;
190
191	if (num_ibs == 0)
192		return -EINVAL;
193
194	*job = kzalloc(struct_size(*job, ibs, num_ibs), GFP_KERNEL);
195	if (!*job)
196		return -ENOMEM;
197
198	af = kzalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);
199	if (!af) {
200		r = -ENOMEM;
201		goto err_job;
202	}
203	(*job)->hw_fence = af;
204
205	af = kzalloc(sizeof(struct amdgpu_fence), GFP_KERNEL);
206	if (!af) {
207		r = -ENOMEM;
208		goto err_fence;
209	}
210	(*job)->hw_vm_fence = af;
211
212	(*job)->vm = vm;
213
214	amdgpu_sync_create(&(*job)->explicit_sync);
215	(*job)->generation = amdgpu_vm_generation(adev, vm);
216	(*job)->vm_pd_addr = AMDGPU_BO_INVALID_OFFSET;
217
218	if (!entity)
219		return 0;
220
221	r = drm_sched_job_init(&(*job)->base, entity, 1, owner, drm_client_id);
222	if (!r)
223		return 0;
224
225	kfree((*job)->hw_vm_fence);
226
227err_fence:
228	kfree((*job)->hw_fence);
229err_job:
230	kfree(*job);
231	*job = NULL;
232
233	return r;
234}
235
236int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
237			     struct drm_sched_entity *entity, void *owner,
238			     size_t size, enum amdgpu_ib_pool_type pool_type,
239			     struct amdgpu_job **job, u64 k_job_id)
240{
241	int r;
242
243	r = amdgpu_job_alloc(adev, NULL, entity, owner, 1, job,
244			     k_job_id);
245	if (r)
246		return r;
247
248	(*job)->num_ibs = 1;
249	r = amdgpu_ib_get(adev, NULL, size, pool_type, &(*job)->ibs[0]);
250	if (r) {
251		if (entity)
252			drm_sched_job_cleanup(&(*job)->base);
253		kfree((*job)->hw_vm_fence);
254		kfree((*job)->hw_fence);
255		kfree(*job);
256		*job = NULL;
257	}
258
259	return r;
260}
261
262void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds,
263			      struct amdgpu_bo *gws, struct amdgpu_bo *oa)
264{
265	if (gds) {
266		job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT;
267		job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT;
268	}
269	if (gws) {
270		job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT;
271		job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT;
272	}
273	if (oa) {
274		job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT;
275		job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT;
276	}
277}
278
279void amdgpu_job_free_resources(struct amdgpu_job *job)
280{
281	struct dma_fence *f;
282	unsigned i;
283
284	/* Check if any fences were initialized */
285	if (job->base.s_fence && job->base.s_fence->finished.ops)
286		f = &job->base.s_fence->finished;
287	else if (job->hw_fence && job->hw_fence->base.ops)
288		f = &job->hw_fence->base;
289	else
290		f = NULL;
291
292	for (i = 0; i < job->num_ibs; ++i)
293		amdgpu_ib_free(&job->ibs[i], f);
294}
295
296static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
297{
298	struct amdgpu_job *job = to_amdgpu_job(s_job);
299
300	drm_sched_job_cleanup(s_job);
301
302	amdgpu_sync_free(&job->explicit_sync);
303
304	if (job->hw_fence->base.ops)
305		dma_fence_put(&job->hw_fence->base);
306	else
307		kfree(job->hw_fence);
308	if (job->hw_vm_fence->base.ops)
309		dma_fence_put(&job->hw_vm_fence->base);
310	else
311		kfree(job->hw_vm_fence);
312
313	kfree(job);
314}
315
316void amdgpu_job_set_gang_leader(struct amdgpu_job *job,
317				struct amdgpu_job *leader)
318{
319	struct dma_fence *fence = &leader->base.s_fence->scheduled;
320
321	WARN_ON(job->gang_submit);
322
323	/*
324	 * Don't add a reference when we are the gang leader to avoid circle
325	 * dependency.
326	 */
327	if (job != leader)
328		dma_fence_get(fence);
329	job->gang_submit = fence;
330}
331
332void amdgpu_job_free(struct amdgpu_job *job)
333{
334	if (job->base.entity)
335		drm_sched_job_cleanup(&job->base);
336
337	amdgpu_job_free_resources(job);
338	amdgpu_sync_free(&job->explicit_sync);
339	if (job->gang_submit != &job->base.s_fence->scheduled)
340		dma_fence_put(job->gang_submit);
341
342	if (job->hw_fence->base.ops)
343		dma_fence_put(&job->hw_fence->base);
344	else
345		kfree(job->hw_fence);
346	if (job->hw_vm_fence->base.ops)
347		dma_fence_put(&job->hw_vm_fence->base);
348	else
349		kfree(job->hw_vm_fence);
350
351	kfree(job);
352}
353
354struct dma_fence *amdgpu_job_submit(struct amdgpu_job *job)
355{
356	struct dma_fence *f;
357
358	drm_sched_job_arm(&job->base);
359	f = dma_fence_get(&job->base.s_fence->finished);
360	amdgpu_job_free_resources(job);
361	drm_sched_entity_push_job(&job->base);
362
363	return f;
364}
365
366int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
367			     struct dma_fence **fence)
368{
369	int r;
370
371	job->base.sched = &ring->sched;
372	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job, fence);
373
374	if (r)
375		return r;
376
377	amdgpu_job_free(job);
378	return 0;
379}
380
381static struct dma_fence *
382amdgpu_job_prepare_job(struct drm_sched_job *sched_job,
383		      struct drm_sched_entity *s_entity)
384{
385	struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched);
386	struct amdgpu_job *job = to_amdgpu_job(sched_job);
387	struct dma_fence *fence;
388	int r;
389
390	r = drm_sched_entity_error(s_entity);
391	if (r)
392		goto error;
393
394	if (job->gang_submit) {
395		fence = amdgpu_device_switch_gang(ring->adev, job->gang_submit);
396		if (fence)
397			return fence;
398	}
399
400	fence = amdgpu_device_enforce_isolation(ring->adev, ring, job);
401	if (fence)
402		return fence;
403
404	if (job->vm && !job->vmid) {
405		r = amdgpu_vmid_grab(job->vm, ring, job, &fence);
406		if (r) {
407			dev_err(ring->adev->dev, "Error getting VM ID (%d)\n", r);
408			goto error;
409		}
410		return fence;
411	}
412
413	return NULL;
414
415error:
416	dma_fence_set_error(&job->base.s_fence->finished, r);
417	return NULL;
418}
419
420static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
421{
422	struct amdgpu_ring *ring = to_amdgpu_ring(sched_job->sched);
423	struct amdgpu_device *adev = ring->adev;
424	struct dma_fence *fence = NULL, *finished;
425	struct amdgpu_job *job;
426	int r = 0;
427
428	job = to_amdgpu_job(sched_job);
429	finished = &job->base.s_fence->finished;
430
431	trace_amdgpu_sched_run_job(job);
432
433	/* Skip job if VRAM is lost and never resubmit gangs */
434	if (job->generation != amdgpu_vm_generation(adev, job->vm) ||
435	    (job->job_run_counter && job->gang_submit))
436		dma_fence_set_error(finished, -ECANCELED);
437
438	if (finished->error < 0) {
439		dev_dbg(adev->dev, "Skip scheduling IBs in ring(%s)",
440			ring->name);
441	} else {
442		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job,
443				       &fence);
444		if (r)
445			dev_err(adev->dev,
446				"Error scheduling IBs (%d) in ring(%s)", r,
447				ring->name);
448	}
449
450	job->job_run_counter++;
451	amdgpu_job_free_resources(job);
452
453	fence = r ? ERR_PTR(r) : fence;
454	return fence;
455}
456
457/*
458 * This is a duplicate function from DRM scheduler sched_internal.h.
459 * Plan is to remove it when amdgpu_job_stop_all_jobs_on_sched is removed, due
460 * latter being incorrect and racy.
461 *
462 * See https://lore.kernel.org/amd-gfx/44edde63-7181-44fb-a4f7-94e50514f539@amd.com/
463 */
464static struct drm_sched_job *
465drm_sched_entity_queue_pop(struct drm_sched_entity *entity)
466{
467	struct spsc_node *node;
468
469	node = spsc_queue_pop(&entity->job_queue);
470	if (!node)
471		return NULL;
472
473	return container_of(node, struct drm_sched_job, queue_node);
474}
475
476void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched)
477{
478	struct drm_sched_job *s_job;
479	struct drm_sched_entity *s_entity = NULL;
480	int i;
481
482	/* Signal all jobs not yet scheduled */
483	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) {
484		struct drm_sched_rq *rq = sched->sched_rq[i];
485		spin_lock(&rq->lock);
486		list_for_each_entry(s_entity, &rq->entities, list) {
487			while ((s_job = drm_sched_entity_queue_pop(s_entity))) {
488				struct drm_sched_fence *s_fence = s_job->s_fence;
489
490				dma_fence_signal(&s_fence->scheduled);
491				dma_fence_set_error(&s_fence->finished, -EHWPOISON);
492				dma_fence_signal(&s_fence->finished);
493			}
494		}
495		spin_unlock(&rq->lock);
496	}
497
498	/* Signal all jobs already scheduled to HW */
499	list_for_each_entry(s_job, &sched->pending_list, list) {
500		struct drm_sched_fence *s_fence = s_job->s_fence;
501
502		dma_fence_set_error(&s_fence->finished, -EHWPOISON);
503		dma_fence_signal(&s_fence->finished);
504	}
505}
506
507const struct drm_sched_backend_ops amdgpu_sched_ops = {
508	.prepare_job = amdgpu_job_prepare_job,
509	.run_job = amdgpu_job_run,
510	.timedout_job = amdgpu_job_timedout,
511	.free_job = amdgpu_job_free_cb
512};