dma-fence-unwrap.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * dma-fence-util: misc functions for dma_fence objects
  4 *
  5 * Copyright (C) 2022 Advanced Micro Devices, Inc.
  6 * Authors:
  7 *	Christian König <christian.koenig@amd.com>
  8 */
  9
 10#include <linux/dma-fence.h>
 11#include <linux/dma-fence-array.h>
 12#include <linux/dma-fence-chain.h>
 13#include <linux/dma-fence-unwrap.h>
 14#include <linux/slab.h>
 15#include <linux/sort.h>
 16
 17/* Internal helper to start new array iteration, don't use directly */
 18static struct dma_fence *
 19__dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
 20{
 21	cursor->array = dma_fence_chain_contained(cursor->chain);
 22	cursor->index = 0;
 23	return dma_fence_array_first(cursor->array);
 24}
 25
 26/**
 27 * dma_fence_unwrap_first - return the first fence from fence containers
 28 * @head: the entrypoint into the containers
 29 * @cursor: current position inside the containers
 30 *
 31 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
 32 * first fence.
 33 */
 34struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
 35					 struct dma_fence_unwrap *cursor)
 36{
 37	cursor->chain = dma_fence_get(head);
 38	return __dma_fence_unwrap_array(cursor);
 39}
 40EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
 41
 42/**
 43 * dma_fence_unwrap_next - return the next fence from a fence containers
 44 * @cursor: current position inside the containers
 45 *
 46 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
 47 * the next fence from them.
 48 */
 49struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
 50{
 51	struct dma_fence *tmp;
 52
 53	++cursor->index;
 54	tmp = dma_fence_array_next(cursor->array, cursor->index);
 55	if (tmp)
 56		return tmp;
 57
 58	cursor->chain = dma_fence_chain_walk(cursor->chain);
 59	return __dma_fence_unwrap_array(cursor);
 60}
 61EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
 62
 63
 64static int fence_cmp(const void *_a, const void *_b)
 65{
 66	struct dma_fence *a = *(struct dma_fence **)_a;
 67	struct dma_fence *b = *(struct dma_fence **)_b;
 68
 69	if (a->context < b->context)
 70		return -1;
 71	else if (a->context > b->context)
 72		return 1;
 73
 74	if (dma_fence_is_later(b, a))
 75		return 1;
 76	else if (dma_fence_is_later(a, b))
 77		return -1;
 78
 79	return 0;
 80}
 81
 82/**
 83 * dma_fence_dedup_array - Sort and deduplicate an array of dma_fence pointers
 84 * @fences:     Array of dma_fence pointers to be deduplicated
 85 * @num_fences: Number of entries in the @fences array
 86 *
 87 * Sorts the input array by context, then removes duplicate
 88 * fences with the same context, keeping only the most recent one.
 89 *
 90 * The array is modified in-place and unreferenced duplicate fences are released
 91 * via dma_fence_put(). The function returns the new number of fences after
 92 * deduplication.
 93 *
 94 * Return: Number of unique fences remaining in the array.
 95 */
 96int dma_fence_dedup_array(struct dma_fence **fences, int num_fences)
 97{
 98	int i, j;
 99
100	sort(fences, num_fences, sizeof(*fences), fence_cmp, NULL);
101
102	/*
103	 * Only keep the most recent fence for each context.
104	 */
105	j = 0;
106	for (i = 1; i < num_fences; i++) {
107		if (fences[i]->context == fences[j]->context)
108			dma_fence_put(fences[i]);
109		else
110			fences[++j] = fences[i];
111	}
112
113	return ++j;
114}
115EXPORT_SYMBOL_GPL(dma_fence_dedup_array);
116
117/* Implementation for the dma_fence_merge() marco, don't use directly */
118struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
119					   struct dma_fence **fences,
120					   struct dma_fence_unwrap *iter)
121{
122	struct dma_fence *tmp, *unsignaled = NULL, **array;
123	struct dma_fence_array *result;
124	ktime_t timestamp;
125	int i, count;
126
127	count = 0;
128	timestamp = ns_to_ktime(0);
129	for (i = 0; i < num_fences; ++i) {
130		dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
131			if (!dma_fence_is_signaled(tmp)) {
132				dma_fence_put(unsignaled);
133				unsignaled = dma_fence_get(tmp);
134				++count;
135			} else {
136				ktime_t t = dma_fence_timestamp(tmp);
137
138				if (ktime_after(t, timestamp))
139					timestamp = t;
140			}
141		}
142	}
143
144	/*
145	 * If we couldn't find a pending fence just return a private signaled
146	 * fence with the timestamp of the last signaled one.
147	 *
148	 * Or if there was a single unsignaled fence left we can return it
149	 * directly and early since that is a major path on many workloads.
150	 */
151	if (count == 0)
152		return dma_fence_allocate_private_stub(timestamp);
153	else if (count == 1)
154		return unsignaled;
155
156	dma_fence_put(unsignaled);
157
158	array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
159	if (!array)
160		return NULL;
161
162	count = 0;
163	for (i = 0; i < num_fences; ++i) {
164		dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
165			if (!dma_fence_is_signaled(tmp)) {
166				array[count++] = dma_fence_get(tmp);
167			} else {
168				ktime_t t = dma_fence_timestamp(tmp);
169
170				if (ktime_after(t, timestamp))
171					timestamp = t;
172			}
173		}
174	}
175
176	if (count == 0 || count == 1)
177		goto return_fastpath;
178
179	count = dma_fence_dedup_array(array, count);
180
181	if (count > 1) {
182		result = dma_fence_array_create(count, array,
183						dma_fence_context_alloc(1),
184						1, false);
185		if (!result) {
186			for (i = 0; i < count; i++)
187				dma_fence_put(array[i]);
188			tmp = NULL;
189			goto return_tmp;
190		}
191		return &result->base;
192	}
193
194return_fastpath:
195	if (count == 0)
196		tmp = dma_fence_allocate_private_stub(timestamp);
197	else
198		tmp = array[0];
199
200return_tmp:
201	kfree(array);
202	return tmp;
203}
204EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);