memory_mgr.c

  1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Copyright 2022 HabanaLabs, Ltd.
  5 * All Rights Reserved.
  6 */
  7
  8#include "habanalabs.h"
  9
 10/**
 11 * hl_mmap_mem_buf_get - increase the buffer refcount and return a pointer to
 12 *                        the buffer descriptor.
 13 *
 14 * @mmg: parent unified memory manager
 15 * @handle: requested buffer handle
 16 *
 17 * Find the buffer in the store and return a pointer to its descriptor.
 18 * Increase buffer refcount. If not found - return NULL.
 19 */
 20struct hl_mmap_mem_buf *hl_mmap_mem_buf_get(struct hl_mem_mgr *mmg, u64 handle)
 21{
 22	struct hl_mmap_mem_buf *buf;
 23
 24	spin_lock(&mmg->lock);
 25	buf = idr_find(&mmg->handles, lower_32_bits(handle >> PAGE_SHIFT));
 26	if (!buf) {
 27		spin_unlock(&mmg->lock);
 28		dev_dbg(mmg->dev, "Buff get failed, no match to handle %#llx\n", handle);
 29		return NULL;
 30	}
 31	kref_get(&buf->refcount);
 32	spin_unlock(&mmg->lock);
 33	return buf;
 34}
 35
 36/**
 37 * hl_mmap_mem_buf_destroy - destroy the unused buffer
 38 *
 39 * @buf: memory manager buffer descriptor
 40 *
 41 * Internal function, used as a final step of buffer release. Shall be invoked
 42 * only when the buffer is no longer in use (removed from idr). Will call the
 43 * release callback (if applicable), and free the memory.
 44 */
 45static void hl_mmap_mem_buf_destroy(struct hl_mmap_mem_buf *buf)
 46{
 47	if (buf->behavior->release)
 48		buf->behavior->release(buf);
 49
 50	kfree(buf);
 51}
 52
 53/**
 54 * hl_mmap_mem_buf_release - release buffer
 55 *
 56 * @kref: kref that reached 0.
 57 *
 58 * Internal function, used as a kref release callback, when the last user of
 59 * the buffer is released. Shall be called from an interrupt context.
 60 */
 61static void hl_mmap_mem_buf_release(struct kref *kref)
 62{
 63	struct hl_mmap_mem_buf *buf =
 64		container_of(kref, struct hl_mmap_mem_buf, refcount);
 65
 66	spin_lock(&buf->mmg->lock);
 67	idr_remove(&buf->mmg->handles, lower_32_bits(buf->handle >> PAGE_SHIFT));
 68	spin_unlock(&buf->mmg->lock);
 69
 70	hl_mmap_mem_buf_destroy(buf);
 71}
 72
 73/**
 74 * hl_mmap_mem_buf_remove_idr_locked - remove handle from idr
 75 *
 76 * @kref: kref that reached 0.
 77 *
 78 * Internal function, used for kref put by handle. Assumes mmg lock is taken.
 79 * Will remove the buffer from idr, without destroying it.
 80 */
 81static void hl_mmap_mem_buf_remove_idr_locked(struct kref *kref)
 82{
 83	struct hl_mmap_mem_buf *buf =
 84		container_of(kref, struct hl_mmap_mem_buf, refcount);
 85
 86	idr_remove(&buf->mmg->handles, lower_32_bits(buf->handle >> PAGE_SHIFT));
 87}
 88
 89/**
 90 * hl_mmap_mem_buf_put - decrease the reference to the buffer
 91 *
 92 * @buf: memory manager buffer descriptor
 93 *
 94 * Decrease the reference to the buffer, and release it if it was the last one.
 95 * Shall be called from an interrupt context.
 96 */
 97int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf)
 98{
 99	return kref_put(&buf->refcount, hl_mmap_mem_buf_release);
100}
101
102/**
103 * hl_mmap_mem_buf_put_handle - decrease the reference to the buffer with the
104 *                              given handle.
105 *
106 * @mmg: parent unified memory manager
107 * @handle: requested buffer handle
108 *
109 * Decrease the reference to the buffer, and release it if it was the last one.
110 * Shall not be called from an interrupt context. Return -EINVAL if handle was
111 * not found, else return the put outcome (0 or 1).
112 */
113int hl_mmap_mem_buf_put_handle(struct hl_mem_mgr *mmg, u64 handle)
114{
115	struct hl_mmap_mem_buf *buf;
116
117	spin_lock(&mmg->lock);
118	buf = idr_find(&mmg->handles, lower_32_bits(handle >> PAGE_SHIFT));
119	if (!buf) {
120		spin_unlock(&mmg->lock);
121		dev_dbg(mmg->dev,
122			 "Buff put failed, no match to handle %#llx\n", handle);
123		return -EINVAL;
124	}
125
126	if (kref_put(&buf->refcount, hl_mmap_mem_buf_remove_idr_locked)) {
127		spin_unlock(&mmg->lock);
128		hl_mmap_mem_buf_destroy(buf);
129		return 1;
130	}
131
132	spin_unlock(&mmg->lock);
133	return 0;
134}
135
136/**
137 * hl_mmap_mem_buf_alloc - allocate a new mappable buffer
138 *
139 * @mmg: parent unified memory manager
140 * @behavior: behavior object describing this buffer polymorphic behavior
141 * @gfp: gfp flags to use for the memory allocations
142 * @args: additional args passed to behavior->alloc
143 *
144 * Allocate and register a new memory buffer inside the give memory manager.
145 * Return the pointer to the new buffer on success or NULL on failure.
146 */
147struct hl_mmap_mem_buf *
148hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg,
149		      struct hl_mmap_mem_buf_behavior *behavior, gfp_t gfp,
150		      void *args)
151{
152	struct hl_mmap_mem_buf *buf;
153	int rc;
154
155	buf = kzalloc(sizeof(*buf), gfp);
156	if (!buf)
157		return NULL;
158
159	spin_lock(&mmg->lock);
160	rc = idr_alloc(&mmg->handles, buf, 1, 0, GFP_ATOMIC);
161	spin_unlock(&mmg->lock);
162	if (rc < 0) {
163		dev_err(mmg->dev,
164			"%s: Failed to allocate IDR for a new buffer, rc=%d\n",
165			behavior->topic, rc);
166		goto free_buf;
167	}
168
169	buf->mmg = mmg;
170	buf->behavior = behavior;
171	buf->handle = (((u64)rc | buf->behavior->mem_id) << PAGE_SHIFT);
172	kref_init(&buf->refcount);
173
174	rc = buf->behavior->alloc(buf, gfp, args);
175	if (rc) {
176		dev_err(mmg->dev, "%s: Failure in buffer alloc callback %d\n",
177			behavior->topic, rc);
178		goto remove_idr;
179	}
180
181	return buf;
182
183remove_idr:
184	spin_lock(&mmg->lock);
185	idr_remove(&mmg->handles, lower_32_bits(buf->handle >> PAGE_SHIFT));
186	spin_unlock(&mmg->lock);
187free_buf:
188	kfree(buf);
189	return NULL;
190}
191
192/**
193 * hl_mmap_mem_buf_vm_close - handle mmap close
194 *
195 * @vma: the vma object for which mmap was closed.
196 *
197 * Put the memory buffer if it is no longer mapped.
198 */
199static void hl_mmap_mem_buf_vm_close(struct vm_area_struct *vma)
200{
201	struct hl_mmap_mem_buf *buf =
202		(struct hl_mmap_mem_buf *)vma->vm_private_data;
203	long new_mmap_size;
204
205	new_mmap_size = buf->real_mapped_size - (vma->vm_end - vma->vm_start);
206
207	if (new_mmap_size > 0) {
208		buf->real_mapped_size = new_mmap_size;
209		return;
210	}
211
212	atomic_set(&buf->mmap, 0);
213	hl_mmap_mem_buf_put(buf);
214	vma->vm_private_data = NULL;
215}
216
217static const struct vm_operations_struct hl_mmap_mem_buf_vm_ops = {
218	.close = hl_mmap_mem_buf_vm_close
219};
220
221/**
222 * hl_mem_mgr_mmap - map the given buffer to the user
223 *
224 * @mmg: unified memory manager
225 * @vma: the vma object for which mmap was closed.
226 * @args: additional args passed to behavior->mmap
227 *
228 * Map the buffer specified by the vma->vm_pgoff to the given vma.
229 */
230int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma,
231		    void *args)
232{
233	struct hl_mmap_mem_buf *buf;
234	u64 user_mem_size;
235	u64 handle;
236	int rc;
237
238	/* We use the page offset to hold the idr and thus we need to clear
239	 * it before doing the mmap itself
240	 */
241	handle = vma->vm_pgoff << PAGE_SHIFT;
242	vma->vm_pgoff = 0;
243
244	/* Reference was taken here */
245	buf = hl_mmap_mem_buf_get(mmg, handle);
246	if (!buf) {
247		dev_err(mmg->dev,
248			"Memory mmap failed, no match to handle %#llx\n", handle);
249		return -EINVAL;
250	}
251
252	/* Validation check */
253	user_mem_size = vma->vm_end - vma->vm_start;
254	if (user_mem_size != ALIGN(buf->mappable_size, PAGE_SIZE)) {
255		dev_err(mmg->dev,
256			"%s: Memory mmap failed, mmap VM size 0x%llx != 0x%llx allocated physical mem size\n",
257			buf->behavior->topic, user_mem_size, buf->mappable_size);
258		rc = -EINVAL;
259		goto put_mem;
260	}
261
262	if (!access_ok((void __user *)(uintptr_t)vma->vm_start,
263		       user_mem_size)) {
264		dev_err(mmg->dev, "%s: User pointer is invalid - 0x%lx\n",
265			buf->behavior->topic, vma->vm_start);
266
267		rc = -EINVAL;
268		goto put_mem;
269	}
270
271	if (atomic_cmpxchg(&buf->mmap, 0, 1)) {
272		dev_err(mmg->dev,
273			"%s, Memory mmap failed, already mapped to user\n",
274			buf->behavior->topic);
275		rc = -EINVAL;
276		goto put_mem;
277	}
278
279	vma->vm_ops = &hl_mmap_mem_buf_vm_ops;
280
281	/* Note: We're transferring the memory reference to vma->vm_private_data here. */
282
283	vma->vm_private_data = buf;
284
285	rc = buf->behavior->mmap(buf, vma, args);
286	if (rc) {
287		atomic_set(&buf->mmap, 0);
288		goto put_mem;
289	}
290
291	buf->real_mapped_size = buf->mappable_size;
292	vma->vm_pgoff = handle >> PAGE_SHIFT;
293
294	return 0;
295
296put_mem:
297	hl_mmap_mem_buf_put(buf);
298	return rc;
299}
300
301/**
302 * hl_mem_mgr_init - initialize unified memory manager
303 *
304 * @dev: owner device pointer
305 * @mmg: structure to initialize
306 *
307 * Initialize an instance of unified memory manager
308 */
309void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg)
310{
311	mmg->dev = dev;
312	spin_lock_init(&mmg->lock);
313	idr_init(&mmg->handles);
314}
315
316static void hl_mem_mgr_fini_stats_reset(struct hl_mem_mgr_fini_stats *stats)
317{
318	if (!stats)
319		return;
320
321	memset(stats, 0, sizeof(*stats));
322}
323
324static void hl_mem_mgr_fini_stats_inc(u64 mem_id, struct hl_mem_mgr_fini_stats *stats)
325{
326	if (!stats)
327		return;
328
329	switch (mem_id) {
330	case HL_MMAP_TYPE_CB:
331		++stats->n_busy_cb;
332		break;
333	case HL_MMAP_TYPE_TS_BUFF:
334		++stats->n_busy_ts;
335		break;
336	default:
337		/* we currently store only CB/TS so this shouldn't happen */
338		++stats->n_busy_other;
339	}
340}
341
342/**
343 * hl_mem_mgr_fini - release unified memory manager
344 *
345 * @mmg: parent unified memory manager
346 * @stats: if non-NULL, will return some counters for handles that could not be removed.
347 *
348 * Release the unified memory manager. Shall be called from an interrupt context.
349 */
350void hl_mem_mgr_fini(struct hl_mem_mgr *mmg, struct hl_mem_mgr_fini_stats *stats)
351{
352	struct hl_mmap_mem_buf *buf;
353	struct idr *idp;
354	const char *topic;
355	u64 mem_id;
356	u32 id;
357
358	hl_mem_mgr_fini_stats_reset(stats);
359
360	idp = &mmg->handles;
361
362	idr_for_each_entry(idp, buf, id) {
363		topic = buf->behavior->topic;
364		mem_id = buf->behavior->mem_id;
365		if (hl_mmap_mem_buf_put(buf) != 1) {
366			dev_err(mmg->dev,
367				"%s: Buff handle %u for CTX is still alive\n",
368				topic, id);
369			hl_mem_mgr_fini_stats_inc(mem_id, stats);
370		}
371	}
372}
373
374/**
375 * hl_mem_mgr_idr_destroy() - destroy memory manager IDR.
376 * @mmg: parent unified memory manager
377 *
378 * Destroy the memory manager IDR.
379 * Shall be called when IDR is empty and no memory buffers are in use.
380 */
381void hl_mem_mgr_idr_destroy(struct hl_mem_mgr *mmg)
382{
383	if (!idr_is_empty(&mmg->handles))
384		dev_crit(mmg->dev, "memory manager IDR is destroyed while it is not empty!\n");
385
386	idr_destroy(&mmg->handles);
387}