amdgpu_ring_mux.c

  1/*
  2 * Copyright 2022 Advanced Micro Devices, Inc.
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 20 * OTHER DEALINGS IN THE SOFTWARE.
 21 *
 22 */
 23#include <linux/slab.h>
 24#include <drm/drm_print.h>
 25
 26#include "amdgpu_ring_mux.h"
 27#include "amdgpu_ring.h"
 28#include "amdgpu.h"
 29
 30#define AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT (HZ / 2)
 31#define AMDGPU_MAX_LAST_UNSIGNALED_THRESHOLD_US 10000
 32
 33static const struct ring_info {
 34	unsigned int hw_pio;
 35	const char *ring_name;
 36} sw_ring_info[] = {
 37	{ AMDGPU_RING_PRIO_DEFAULT, "gfx_low"},
 38	{ AMDGPU_RING_PRIO_2, "gfx_high"},
 39};
 40
 41static struct kmem_cache *amdgpu_mux_chunk_slab;
 42
 43static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux,
 44								struct amdgpu_ring *ring)
 45{
 46	return ring->entry_index < mux->ring_entry_size ?
 47			&mux->ring_entry[ring->entry_index] : NULL;
 48}
 49
 50/* copy packages on sw ring range[begin, end) */
 51static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux,
 52						  struct amdgpu_ring *ring,
 53						  u64 s_start, u64 s_end)
 54{
 55	u64 start, end;
 56	struct amdgpu_ring *real_ring = mux->real_ring;
 57
 58	start = s_start & ring->buf_mask;
 59	end = s_end & ring->buf_mask;
 60
 61	if (start == end) {
 62		DRM_ERROR("no more data copied from sw ring\n");
 63		return;
 64	}
 65	if (start > end) {
 66		amdgpu_ring_alloc(real_ring, (ring->ring_size >> 2) + end - start);
 67		amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start],
 68					   (ring->ring_size >> 2) - start);
 69		amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[0], end);
 70	} else {
 71		amdgpu_ring_alloc(real_ring, end - start);
 72		amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], end - start);
 73	}
 74}
 75
 76static void amdgpu_mux_resubmit_chunks(struct amdgpu_ring_mux *mux)
 77{
 78	struct amdgpu_mux_entry *e = NULL;
 79	struct amdgpu_mux_chunk *chunk;
 80	uint32_t seq, last_seq;
 81	int i;
 82
 83	/*find low priority entries:*/
 84	if (!mux->s_resubmit)
 85		return;
 86
 87	for (i = 0; i < mux->num_ring_entries; i++) {
 88		if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) {
 89			e = &mux->ring_entry[i];
 90			break;
 91		}
 92	}
 93
 94	if (!e) {
 95		DRM_ERROR("%s no low priority ring found\n", __func__);
 96		return;
 97	}
 98
 99	last_seq = atomic_read(&e->ring->fence_drv.last_seq);
100	seq = mux->seqno_to_resubmit;
101	if (last_seq < seq) {
102		/*resubmit all the fences between (last_seq, seq]*/
103		list_for_each_entry(chunk, &e->list, entry) {
104			if (chunk->sync_seq > last_seq && chunk->sync_seq <= seq) {
105				amdgpu_fence_update_start_timestamp(e->ring,
106								    chunk->sync_seq,
107								    ktime_get());
108				if (chunk->sync_seq ==
109					le32_to_cpu(*(e->ring->fence_drv.cpu_addr + 2))) {
110					if (chunk->cntl_offset <= e->ring->buf_mask)
111						amdgpu_ring_patch_cntl(e->ring,
112								       chunk->cntl_offset);
113					if (chunk->ce_offset <= e->ring->buf_mask)
114						amdgpu_ring_patch_ce(e->ring, chunk->ce_offset);
115					if (chunk->de_offset <= e->ring->buf_mask)
116						amdgpu_ring_patch_de(e->ring, chunk->de_offset);
117				}
118				amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring,
119								      chunk->start,
120								      chunk->end);
121				mux->wptr_resubmit = chunk->end;
122				amdgpu_ring_commit(mux->real_ring);
123			}
124		}
125	}
126
127	timer_delete(&mux->resubmit_timer);
128	mux->s_resubmit = false;
129}
130
131static void amdgpu_ring_mux_schedule_resubmit(struct amdgpu_ring_mux *mux)
132{
133	mod_timer(&mux->resubmit_timer, jiffies + AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT);
134}
135
136static void amdgpu_mux_resubmit_fallback(struct timer_list *t)
137{
138	struct amdgpu_ring_mux *mux = timer_container_of(mux, t,
139							 resubmit_timer);
140
141	if (!spin_trylock(&mux->lock)) {
142		amdgpu_ring_mux_schedule_resubmit(mux);
143		DRM_ERROR("reschedule resubmit\n");
144		return;
145	}
146	amdgpu_mux_resubmit_chunks(mux);
147	spin_unlock(&mux->lock);
148}
149
150int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring,
151			 unsigned int entry_size)
152{
153	mux->real_ring = ring;
154	mux->num_ring_entries = 0;
155
156	mux->ring_entry = kcalloc(entry_size, sizeof(struct amdgpu_mux_entry), GFP_KERNEL);
157	if (!mux->ring_entry)
158		return -ENOMEM;
159
160	mux->ring_entry_size = entry_size;
161	mux->s_resubmit = false;
162
163	amdgpu_mux_chunk_slab = KMEM_CACHE(amdgpu_mux_chunk, SLAB_HWCACHE_ALIGN);
164	if (!amdgpu_mux_chunk_slab) {
165		DRM_ERROR("create amdgpu_mux_chunk cache failed\n");
166		return -ENOMEM;
167	}
168
169	spin_lock_init(&mux->lock);
170	timer_setup(&mux->resubmit_timer, amdgpu_mux_resubmit_fallback, 0);
171
172	return 0;
173}
174
175void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux)
176{
177	struct amdgpu_mux_entry *e;
178	struct amdgpu_mux_chunk *chunk, *chunk2;
179	int i;
180
181	for (i = 0; i < mux->num_ring_entries; i++) {
182		e = &mux->ring_entry[i];
183		list_for_each_entry_safe(chunk, chunk2, &e->list, entry) {
184			list_del(&chunk->entry);
185			kmem_cache_free(amdgpu_mux_chunk_slab, chunk);
186		}
187	}
188	kmem_cache_destroy(amdgpu_mux_chunk_slab);
189	kfree(mux->ring_entry);
190	mux->ring_entry = NULL;
191	mux->num_ring_entries = 0;
192	mux->ring_entry_size = 0;
193}
194
195int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
196{
197	struct amdgpu_mux_entry *e;
198
199	if (mux->num_ring_entries >= mux->ring_entry_size) {
200		DRM_ERROR("add sw ring exceeding max entry size\n");
201		return -ENOENT;
202	}
203
204	e = &mux->ring_entry[mux->num_ring_entries];
205	ring->entry_index = mux->num_ring_entries;
206	e->ring = ring;
207
208	INIT_LIST_HEAD(&e->list);
209	mux->num_ring_entries += 1;
210	return 0;
211}
212
213void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr)
214{
215	struct amdgpu_mux_entry *e;
216
217	spin_lock(&mux->lock);
218
219	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT)
220		amdgpu_mux_resubmit_chunks(mux);
221
222	e = amdgpu_ring_mux_sw_entry(mux, ring);
223	if (!e) {
224		DRM_ERROR("cannot find entry for sw ring\n");
225		spin_unlock(&mux->lock);
226		return;
227	}
228
229	/* We could skip this set wptr as preemption in process. */
230	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) {
231		spin_unlock(&mux->lock);
232		return;
233	}
234
235	e->sw_cptr = e->sw_wptr;
236	/* Update cptr if the package already copied in resubmit functions */
237	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit)
238		e->sw_cptr = mux->wptr_resubmit;
239	e->sw_wptr = wptr;
240	e->start_ptr_in_hw_ring = mux->real_ring->wptr;
241
242	/* Skip copying for the packages already resubmitted.*/
243	if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) {
244		amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr);
245		e->end_ptr_in_hw_ring = mux->real_ring->wptr;
246		amdgpu_ring_commit(mux->real_ring);
247	} else {
248		e->end_ptr_in_hw_ring = mux->real_ring->wptr;
249	}
250	spin_unlock(&mux->lock);
251}
252
253u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
254{
255	struct amdgpu_mux_entry *e;
256
257	e = amdgpu_ring_mux_sw_entry(mux, ring);
258	if (!e) {
259		DRM_ERROR("cannot find entry for sw ring\n");
260		return 0;
261	}
262
263	return e->sw_wptr;
264}
265
266/**
267 * amdgpu_ring_mux_get_rptr - get the readptr of the software ring
268 * @mux: the multiplexer the software rings attach to
269 * @ring: the software ring of which we calculate the readptr
270 *
271 * The return value of the readptr is not precise while the other rings could
272 * write data onto the real ring buffer.After overwriting on the real ring, we
273 * can not decide if our packages have been excuted or not read yet. However,
274 * this function is only called by the tools such as umr to collect the latest
275 * packages for the hang analysis. We assume the hang happens near our latest
276 * submit. Thus we could use the following logic to give the clue:
277 * If the readptr is between start and end, then we return the copy pointer
278 * plus the distance from start to readptr. If the readptr is before start, we
279 * return the copy pointer. Lastly, if the readptr is past end, we return the
280 * write pointer.
281 */
282u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
283{
284	struct amdgpu_mux_entry *e;
285	u64 readp, offset, start, end;
286
287	e = amdgpu_ring_mux_sw_entry(mux, ring);
288	if (!e) {
289		DRM_ERROR("no sw entry found!\n");
290		return 0;
291	}
292
293	readp = amdgpu_ring_get_rptr(mux->real_ring);
294
295	start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask;
296	end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask;
297	if (start > end) {
298		if (readp <= end)
299			readp += mux->real_ring->ring_size >> 2;
300		end += mux->real_ring->ring_size >> 2;
301	}
302
303	if (start <= readp && readp <= end) {
304		offset = readp - start;
305		e->sw_rptr = (e->sw_cptr + offset) & ring->buf_mask;
306	} else if (readp < start) {
307		e->sw_rptr = e->sw_cptr;
308	} else {
309		/* end < readptr */
310		e->sw_rptr = e->sw_wptr;
311	}
312
313	return e->sw_rptr;
314}
315
316u64 amdgpu_sw_ring_get_rptr_gfx(struct amdgpu_ring *ring)
317{
318	struct amdgpu_device *adev = ring->adev;
319	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
320
321	WARN_ON(!ring->is_sw_ring);
322	return amdgpu_ring_mux_get_rptr(mux, ring);
323}
324
325u64 amdgpu_sw_ring_get_wptr_gfx(struct amdgpu_ring *ring)
326{
327	struct amdgpu_device *adev = ring->adev;
328	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
329
330	WARN_ON(!ring->is_sw_ring);
331	return amdgpu_ring_mux_get_wptr(mux, ring);
332}
333
334void amdgpu_sw_ring_set_wptr_gfx(struct amdgpu_ring *ring)
335{
336	struct amdgpu_device *adev = ring->adev;
337	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
338
339	WARN_ON(!ring->is_sw_ring);
340	amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr);
341}
342
343/* Override insert_nop to prevent emitting nops to the software rings */
344void amdgpu_sw_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
345{
346	WARN_ON(!ring->is_sw_ring);
347}
348
349const char *amdgpu_sw_ring_name(int idx)
350{
351	return idx < ARRAY_SIZE(sw_ring_info) ?
352		sw_ring_info[idx].ring_name : NULL;
353}
354
355unsigned int amdgpu_sw_ring_priority(int idx)
356{
357	return idx < ARRAY_SIZE(sw_ring_info) ?
358		sw_ring_info[idx].hw_pio : AMDGPU_RING_PRIO_DEFAULT;
359}
360
361/*Scan on low prio rings to have unsignaled fence and high ring has no fence.*/
362static int amdgpu_mcbp_scan(struct amdgpu_ring_mux *mux)
363{
364	struct amdgpu_ring *ring;
365	int i, need_preempt;
366
367	need_preempt = 0;
368	for (i = 0; i < mux->num_ring_entries; i++) {
369		ring = mux->ring_entry[i].ring;
370		if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT &&
371		    amdgpu_fence_count_emitted(ring) > 0)
372			return 0;
373		if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT &&
374		    amdgpu_fence_last_unsignaled_time_us(ring) >
375		    AMDGPU_MAX_LAST_UNSIGNALED_THRESHOLD_US)
376			need_preempt = 1;
377	}
378	return need_preempt && !mux->s_resubmit;
379}
380
381/* Trigger Mid-Command Buffer Preemption (MCBP) and find if we need to resubmit. */
382static int amdgpu_mcbp_trigger_preempt(struct amdgpu_ring_mux *mux)
383{
384	int r;
385
386	spin_lock(&mux->lock);
387	mux->pending_trailing_fence_signaled = true;
388	r = amdgpu_ring_preempt_ib(mux->real_ring);
389	spin_unlock(&mux->lock);
390	return r;
391}
392
393void amdgpu_sw_ring_ib_begin(struct amdgpu_ring *ring)
394{
395	struct amdgpu_device *adev = ring->adev;
396	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
397
398	WARN_ON(!ring->is_sw_ring);
399	if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) {
400		if (amdgpu_mcbp_scan(mux) > 0)
401			amdgpu_mcbp_trigger_preempt(mux);
402		return;
403	}
404
405	amdgpu_ring_mux_start_ib(mux, ring);
406}
407
408void amdgpu_sw_ring_ib_end(struct amdgpu_ring *ring)
409{
410	struct amdgpu_device *adev = ring->adev;
411	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
412
413	WARN_ON(!ring->is_sw_ring);
414	if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT)
415		return;
416	amdgpu_ring_mux_end_ib(mux, ring);
417}
418
419void amdgpu_sw_ring_ib_mark_offset(struct amdgpu_ring *ring, enum amdgpu_ring_mux_offset_type type)
420{
421	struct amdgpu_device *adev = ring->adev;
422	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
423	unsigned offset;
424
425	if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT)
426		return;
427
428	offset = ring->wptr & ring->buf_mask;
429
430	amdgpu_ring_mux_ib_mark_offset(mux, ring, offset, type);
431}
432
433void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
434{
435	struct amdgpu_mux_entry *e;
436	struct amdgpu_mux_chunk *chunk;
437
438	spin_lock(&mux->lock);
439	amdgpu_mux_resubmit_chunks(mux);
440	spin_unlock(&mux->lock);
441
442	e = amdgpu_ring_mux_sw_entry(mux, ring);
443	if (!e) {
444		DRM_ERROR("cannot find entry!\n");
445		return;
446	}
447
448	chunk = kmem_cache_alloc(amdgpu_mux_chunk_slab, GFP_KERNEL);
449	if (!chunk) {
450		DRM_ERROR("alloc amdgpu_mux_chunk_slab failed\n");
451		return;
452	}
453
454	chunk->start = ring->wptr;
455	/* the initialized value used to check if they are set by the ib submission*/
456	chunk->cntl_offset = ring->buf_mask + 1;
457	chunk->de_offset = ring->buf_mask + 1;
458	chunk->ce_offset = ring->buf_mask + 1;
459	list_add_tail(&chunk->entry, &e->list);
460}
461
462static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
463{
464	uint32_t last_seq = 0;
465	struct amdgpu_mux_entry *e;
466	struct amdgpu_mux_chunk *chunk, *tmp;
467
468	e = amdgpu_ring_mux_sw_entry(mux, ring);
469	if (!e) {
470		DRM_ERROR("cannot find entry!\n");
471		return;
472	}
473
474	last_seq = atomic_read(&ring->fence_drv.last_seq);
475
476	list_for_each_entry_safe(chunk, tmp, &e->list, entry) {
477		if (chunk->sync_seq <= last_seq) {
478			list_del(&chunk->entry);
479			kmem_cache_free(amdgpu_mux_chunk_slab, chunk);
480		}
481	}
482}
483
484void amdgpu_ring_mux_ib_mark_offset(struct amdgpu_ring_mux *mux,
485				    struct amdgpu_ring *ring, u64 offset,
486				    enum amdgpu_ring_mux_offset_type type)
487{
488	struct amdgpu_mux_entry *e;
489	struct amdgpu_mux_chunk *chunk;
490
491	e = amdgpu_ring_mux_sw_entry(mux, ring);
492	if (!e) {
493		DRM_ERROR("cannot find entry!\n");
494		return;
495	}
496
497	chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry);
498	if (!chunk) {
499		DRM_ERROR("cannot find chunk!\n");
500		return;
501	}
502
503	switch (type) {
504	case AMDGPU_MUX_OFFSET_TYPE_CONTROL:
505		chunk->cntl_offset = offset;
506		break;
507	case AMDGPU_MUX_OFFSET_TYPE_DE:
508		chunk->de_offset = offset;
509		break;
510	case AMDGPU_MUX_OFFSET_TYPE_CE:
511		chunk->ce_offset = offset;
512		break;
513	default:
514		DRM_ERROR("invalid type (%d)\n", type);
515		break;
516	}
517}
518
519void amdgpu_ring_mux_end_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
520{
521	struct amdgpu_mux_entry *e;
522	struct amdgpu_mux_chunk *chunk;
523
524	e = amdgpu_ring_mux_sw_entry(mux, ring);
525	if (!e) {
526		DRM_ERROR("cannot find entry!\n");
527		return;
528	}
529
530	chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry);
531	if (!chunk) {
532		DRM_ERROR("cannot find chunk!\n");
533		return;
534	}
535
536	chunk->end = ring->wptr;
537	chunk->sync_seq = READ_ONCE(ring->fence_drv.sync_seq);
538
539	scan_and_remove_signaled_chunk(mux, ring);
540}
541
542bool amdgpu_mcbp_handle_trailing_fence_irq(struct amdgpu_ring_mux *mux)
543{
544	struct amdgpu_mux_entry *e;
545	struct amdgpu_ring *ring = NULL;
546	int i;
547
548	if (!mux->pending_trailing_fence_signaled)
549		return false;
550
551	if (mux->real_ring->trail_seq != le32_to_cpu(*mux->real_ring->trail_fence_cpu_addr))
552		return false;
553
554	for (i = 0; i < mux->num_ring_entries; i++) {
555		e = &mux->ring_entry[i];
556		if (e->ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) {
557			ring = e->ring;
558			break;
559		}
560	}
561
562	if (!ring) {
563		DRM_ERROR("cannot find low priority ring\n");
564		return false;
565	}
566
567	amdgpu_fence_process(ring);
568	if (amdgpu_fence_count_emitted(ring) > 0) {
569		mux->s_resubmit = true;
570		mux->seqno_to_resubmit = ring->fence_drv.sync_seq;
571		amdgpu_ring_mux_schedule_resubmit(mux);
572	}
573
574	mux->pending_trailing_fence_signaled = false;
575	return true;
576}