hns_roce_db.c

  1/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
  2/*
  3 * Copyright (c) 2017 Hisilicon Limited.
  4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
  5 */
  6
  7#include <rdma/ib_umem.h>
  8#include "hns_roce_device.h"
  9
 10int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
 11			 struct hns_roce_db *db)
 12{
 13	unsigned long page_addr = virt & PAGE_MASK;
 14	struct hns_roce_user_db_page *page;
 15	unsigned int offset;
 16	int ret = 0;
 17
 18	mutex_lock(&context->page_mutex);
 19
 20	list_for_each_entry(page, &context->page_list, list)
 21		if (page->user_virt == page_addr)
 22			goto found;
 23
 24	page = kmalloc(sizeof(*page), GFP_KERNEL);
 25	if (!page) {
 26		ret = -ENOMEM;
 27		goto out;
 28	}
 29
 30	refcount_set(&page->refcount, 1);
 31	page->user_virt = page_addr;
 32	page->umem = ib_umem_get(context->ibucontext.device, page_addr,
 33				 PAGE_SIZE, 0);
 34	if (IS_ERR(page->umem)) {
 35		ret = PTR_ERR(page->umem);
 36		kfree(page);
 37		goto out;
 38	}
 39
 40	list_add(&page->list, &context->page_list);
 41
 42found:
 43	offset = virt - page_addr;
 44	db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset;
 45	db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset;
 46	db->u.user_page = page;
 47	refcount_inc(&page->refcount);
 48
 49out:
 50	mutex_unlock(&context->page_mutex);
 51
 52	return ret;
 53}
 54
 55void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
 56			    struct hns_roce_db *db)
 57{
 58	mutex_lock(&context->page_mutex);
 59
 60	refcount_dec(&db->u.user_page->refcount);
 61	if (refcount_dec_if_one(&db->u.user_page->refcount)) {
 62		list_del(&db->u.user_page->list);
 63		ib_umem_release(db->u.user_page->umem);
 64		kfree(db->u.user_page);
 65	}
 66
 67	mutex_unlock(&context->page_mutex);
 68}
 69
 70static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(
 71					struct device *dma_device)
 72{
 73	struct hns_roce_db_pgdir *pgdir;
 74
 75	pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);
 76	if (!pgdir)
 77		return NULL;
 78
 79	bitmap_fill(pgdir->order1,
 80		    HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
 81	pgdir->bits[0] = pgdir->order0;
 82	pgdir->bits[1] = pgdir->order1;
 83	pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE,
 84					 &pgdir->db_dma, GFP_KERNEL);
 85	if (!pgdir->page) {
 86		kfree(pgdir);
 87		return NULL;
 88	}
 89
 90	return pgdir;
 91}
 92
 93static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,
 94					struct hns_roce_db *db, int order)
 95{
 96	unsigned long o;
 97	unsigned long i;
 98
 99	for (o = order; o <= 1; ++o) {
100		i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o);
101		if (i < HNS_ROCE_DB_PER_PAGE >> o)
102			goto found;
103	}
104
105	return -ENOMEM;
106
107found:
108	clear_bit(i, pgdir->bits[o]);
109
110	i <<= o;
111
112	if (o > order)
113		set_bit(i ^ 1, pgdir->bits[order]);
114
115	db->u.pgdir	= pgdir;
116	db->index	= i;
117	db->db_record	= pgdir->page + db->index;
118	db->dma		= pgdir->db_dma  + db->index * HNS_ROCE_DB_UNIT_SIZE;
119	db->order	= order;
120
121	return 0;
122}
123
124int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
125		      int order)
126{
127	struct hns_roce_db_pgdir *pgdir;
128	int ret = 0;
129
130	mutex_lock(&hr_dev->pgdir_mutex);
131
132	list_for_each_entry(pgdir, &hr_dev->pgdir_list, list)
133		if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order))
134			goto out;
135
136	pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev);
137	if (!pgdir) {
138		ret = -ENOMEM;
139		goto out;
140	}
141
142	list_add(&pgdir->list, &hr_dev->pgdir_list);
143
144	/* This should never fail -- we just allocated an empty page: */
145	WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order));
146
147out:
148	mutex_unlock(&hr_dev->pgdir_mutex);
149
150	return ret;
151}
152
153void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
154{
155	unsigned long o;
156	unsigned long i;
157
158	mutex_lock(&hr_dev->pgdir_mutex);
159
160	o = db->order;
161	i = db->index;
162
163	if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
164		clear_bit(i ^ 1, db->u.pgdir->order0);
165		++o;
166	}
167
168	i >>= o;
169	set_bit(i, db->u.pgdir->bits[o]);
170
171	if (bitmap_full(db->u.pgdir->order1,
172			HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {
173		dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page,
174				  db->u.pgdir->db_dma);
175		list_del(&db->u.pgdir->list);
176		kfree(db->u.pgdir);
177	}
178
179	mutex_unlock(&hr_dev->pgdir_mutex);
180}