amdgpu_eviction_fence.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright 2024 Advanced Micro Devices, Inc.
  4 *
  5 * Permission is hereby granted, free of charge, to any person obtaining a
  6 * copy of this software and associated documentation files (the "Software"),
  7 * to deal in the Software without restriction, including without limitation
  8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9 * and/or sell copies of the Software, and to permit persons to whom the
 10 * Software is furnished to do so, subject to the following conditions:
 11 *
 12 * The above copyright notice and this permission notice shall be included in
 13 * all copies or substantial portions of the Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 21 * OTHER DEALINGS IN THE SOFTWARE.
 22 *
 23 */
 24#include <linux/sched.h>
 25#include <drm/drm_exec.h>
 26#include "amdgpu.h"
 27
 28#define work_to_evf_mgr(w, name) container_of(w, struct amdgpu_eviction_fence_mgr, name)
 29#define evf_mgr_to_fpriv(e) container_of(e, struct amdgpu_fpriv, evf_mgr)
 30
 31static const char *
 32amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence)
 33{
 34	return "amdgpu_eviction_fence";
 35}
 36
 37static const char *
 38amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f)
 39{
 40	struct amdgpu_eviction_fence *ef;
 41
 42	ef = container_of(f, struct amdgpu_eviction_fence, base);
 43	return ef->timeline_name;
 44}
 45
 46int
 47amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
 48				    struct drm_exec *exec)
 49{
 50	struct amdgpu_eviction_fence *old_ef, *new_ef;
 51	struct drm_gem_object *obj;
 52	unsigned long index;
 53	int ret;
 54
 55	if (evf_mgr->ev_fence &&
 56	    !dma_fence_is_signaled(&evf_mgr->ev_fence->base))
 57		return 0;
 58	/*
 59	 * Steps to replace eviction fence:
 60	 * * lock all objects in exec (caller)
 61	 * * create a new eviction fence
 62	 * * update new eviction fence in evf_mgr
 63	 * * attach the new eviction fence to BOs
 64	 * * release the old fence
 65	 * * unlock the objects (caller)
 66	 */
 67	new_ef = amdgpu_eviction_fence_create(evf_mgr);
 68	if (!new_ef) {
 69		DRM_ERROR("Failed to create new eviction fence\n");
 70		return -ENOMEM;
 71	}
 72
 73	/* Update the eviction fence now */
 74	spin_lock(&evf_mgr->ev_fence_lock);
 75	old_ef = evf_mgr->ev_fence;
 76	evf_mgr->ev_fence = new_ef;
 77	spin_unlock(&evf_mgr->ev_fence_lock);
 78
 79	/* Attach the new fence */
 80	drm_exec_for_each_locked_object(exec, index, obj) {
 81		struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
 82
 83		if (!bo)
 84			continue;
 85		ret = amdgpu_eviction_fence_attach(evf_mgr, bo);
 86		if (ret) {
 87			DRM_ERROR("Failed to attch new eviction fence\n");
 88			goto free_err;
 89		}
 90	}
 91
 92	/* Free old fence */
 93	if (old_ef)
 94		dma_fence_put(&old_ef->base);
 95	return 0;
 96
 97free_err:
 98	kfree(new_ef);
 99	return ret;
100}
101
102static void
103amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
104{
105	struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work);
106	struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr);
107	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
108	struct amdgpu_eviction_fence *ev_fence;
109
110	mutex_lock(&uq_mgr->userq_mutex);
111	spin_lock(&evf_mgr->ev_fence_lock);
112	ev_fence = evf_mgr->ev_fence;
113	if (ev_fence)
114		dma_fence_get(&ev_fence->base);
115	else
116		goto unlock;
117	spin_unlock(&evf_mgr->ev_fence_lock);
118
119	amdgpu_userq_evict(uq_mgr, ev_fence);
120
121	mutex_unlock(&uq_mgr->userq_mutex);
122	dma_fence_put(&ev_fence->base);
123	return;
124
125unlock:
126	spin_unlock(&evf_mgr->ev_fence_lock);
127	mutex_unlock(&uq_mgr->userq_mutex);
128}
129
130static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f)
131{
132	struct amdgpu_eviction_fence_mgr *evf_mgr;
133	struct amdgpu_eviction_fence *ev_fence;
134
135	if (!f)
136		return true;
137
138	ev_fence = to_ev_fence(f);
139	evf_mgr = ev_fence->evf_mgr;
140
141	schedule_delayed_work(&evf_mgr->suspend_work, 0);
142	return true;
143}
144
145static const struct dma_fence_ops amdgpu_eviction_fence_ops = {
146	.get_driver_name = amdgpu_eviction_fence_get_driver_name,
147	.get_timeline_name = amdgpu_eviction_fence_get_timeline_name,
148	.enable_signaling = amdgpu_eviction_fence_enable_signaling,
149};
150
151void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr,
152				  struct amdgpu_eviction_fence *ev_fence)
153{
154	spin_lock(&evf_mgr->ev_fence_lock);
155	dma_fence_signal(&ev_fence->base);
156	spin_unlock(&evf_mgr->ev_fence_lock);
157}
158
159struct amdgpu_eviction_fence *
160amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr)
161{
162	struct amdgpu_eviction_fence *ev_fence;
163
164	ev_fence = kzalloc(sizeof(*ev_fence), GFP_KERNEL);
165	if (!ev_fence)
166		return NULL;
167
168	ev_fence->evf_mgr = evf_mgr;
169	get_task_comm(ev_fence->timeline_name, current);
170	spin_lock_init(&ev_fence->lock);
171	dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops,
172			 &ev_fence->lock, evf_mgr->ev_fence_ctx,
173			 atomic_inc_return(&evf_mgr->ev_fence_seq));
174	return ev_fence;
175}
176
177void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr)
178{
179	struct amdgpu_eviction_fence *ev_fence;
180
181	/* Wait for any pending work to execute */
182	flush_delayed_work(&evf_mgr->suspend_work);
183
184	spin_lock(&evf_mgr->ev_fence_lock);
185	ev_fence = evf_mgr->ev_fence;
186	spin_unlock(&evf_mgr->ev_fence_lock);
187
188	if (!ev_fence)
189		return;
190
191	dma_fence_wait(&ev_fence->base, false);
192
193	/* Last unref of ev_fence */
194	dma_fence_put(&ev_fence->base);
195}
196
197int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr,
198				 struct amdgpu_bo *bo)
199{
200	struct amdgpu_eviction_fence *ev_fence;
201	struct dma_resv *resv = bo->tbo.base.resv;
202	int ret;
203
204	if (!resv)
205		return 0;
206
207	ret = dma_resv_reserve_fences(resv, 1);
208	if (ret) {
209		DRM_DEBUG_DRIVER("Failed to resv fence space\n");
210		return ret;
211	}
212
213	spin_lock(&evf_mgr->ev_fence_lock);
214	ev_fence = evf_mgr->ev_fence;
215	if (ev_fence)
216		dma_resv_add_fence(resv, &ev_fence->base, DMA_RESV_USAGE_BOOKKEEP);
217	spin_unlock(&evf_mgr->ev_fence_lock);
218
219	return 0;
220}
221
222void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr,
223				  struct amdgpu_bo *bo)
224{
225	struct dma_fence *stub = dma_fence_get_stub();
226
227	dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx,
228				stub, DMA_RESV_USAGE_BOOKKEEP);
229	dma_fence_put(stub);
230}
231
232int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
233{
234	/* This needs to be done one time per open */
235	atomic_set(&evf_mgr->ev_fence_seq, 0);
236	evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1);
237	spin_lock_init(&evf_mgr->ev_fence_lock);
238
239	INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
240	return 0;
241}