pool.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2024 NVIDIA Corporation & Affiliates */
  3
  4#include "internal.h"
  5#include "buddy.h"
  6
  7static void hws_pool_free_one_resource(struct mlx5hws_pool_resource *resource)
  8{
  9	switch (resource->pool->type) {
 10	case MLX5HWS_POOL_TYPE_STE:
 11		mlx5hws_cmd_ste_destroy(resource->pool->ctx->mdev, resource->base_id);
 12		break;
 13	case MLX5HWS_POOL_TYPE_STC:
 14		mlx5hws_cmd_stc_destroy(resource->pool->ctx->mdev, resource->base_id);
 15		break;
 16	default:
 17		break;
 18	}
 19
 20	kfree(resource);
 21}
 22
 23static void hws_pool_resource_free(struct mlx5hws_pool *pool)
 24{
 25	hws_pool_free_one_resource(pool->resource);
 26	pool->resource = NULL;
 27
 28	if (pool->tbl_type == MLX5HWS_TABLE_TYPE_FDB) {
 29		hws_pool_free_one_resource(pool->mirror_resource);
 30		pool->mirror_resource = NULL;
 31	}
 32}
 33
 34static struct mlx5hws_pool_resource *
 35hws_pool_create_one_resource(struct mlx5hws_pool *pool, u32 log_range,
 36			     u32 fw_ft_type)
 37{
 38	struct mlx5hws_cmd_ste_create_attr ste_attr;
 39	struct mlx5hws_cmd_stc_create_attr stc_attr;
 40	struct mlx5hws_pool_resource *resource;
 41	u32 obj_id = 0;
 42	int ret;
 43
 44	resource = kzalloc(sizeof(*resource), GFP_KERNEL);
 45	if (!resource)
 46		return NULL;
 47
 48	switch (pool->type) {
 49	case MLX5HWS_POOL_TYPE_STE:
 50		ste_attr.log_obj_range = log_range;
 51		ste_attr.table_type = fw_ft_type;
 52		ret = mlx5hws_cmd_ste_create(pool->ctx->mdev, &ste_attr, &obj_id);
 53		break;
 54	case MLX5HWS_POOL_TYPE_STC:
 55		stc_attr.log_obj_range = log_range;
 56		stc_attr.table_type = fw_ft_type;
 57		ret = mlx5hws_cmd_stc_create(pool->ctx->mdev, &stc_attr, &obj_id);
 58		break;
 59	default:
 60		ret = -EINVAL;
 61	}
 62
 63	if (ret)
 64		goto free_resource;
 65
 66	resource->pool = pool;
 67	resource->range = 1 << log_range;
 68	resource->base_id = obj_id;
 69
 70	return resource;
 71
 72free_resource:
 73	kfree(resource);
 74	return NULL;
 75}
 76
 77static int hws_pool_resource_alloc(struct mlx5hws_pool *pool)
 78{
 79	struct mlx5hws_pool_resource *resource;
 80	u32 fw_ft_type, opt_log_range;
 81
 82	fw_ft_type = mlx5hws_table_get_res_fw_ft_type(pool->tbl_type, false);
 83	opt_log_range = pool->opt_type == MLX5HWS_POOL_OPTIMIZE_MIRROR ?
 84				0 : pool->alloc_log_sz;
 85	resource = hws_pool_create_one_resource(pool, opt_log_range, fw_ft_type);
 86	if (!resource) {
 87		mlx5hws_err(pool->ctx, "Failed to allocate resource\n");
 88		return -EINVAL;
 89	}
 90
 91	pool->resource = resource;
 92
 93	if (pool->tbl_type == MLX5HWS_TABLE_TYPE_FDB) {
 94		struct mlx5hws_pool_resource *mirror_resource;
 95
 96		fw_ft_type = mlx5hws_table_get_res_fw_ft_type(pool->tbl_type, true);
 97		opt_log_range = pool->opt_type == MLX5HWS_POOL_OPTIMIZE_ORIG ?
 98					0 : pool->alloc_log_sz;
 99		mirror_resource = hws_pool_create_one_resource(pool, opt_log_range, fw_ft_type);
100		if (!mirror_resource) {
101			mlx5hws_err(pool->ctx, "Failed to allocate mirrored resource\n");
102			hws_pool_free_one_resource(resource);
103			pool->resource = NULL;
104			return -EINVAL;
105		}
106		pool->mirror_resource = mirror_resource;
107	}
108
109	return 0;
110}
111
112static int hws_pool_buddy_init(struct mlx5hws_pool *pool)
113{
114	struct mlx5hws_buddy_mem *buddy;
115
116	buddy = mlx5hws_buddy_create(pool->alloc_log_sz);
117	if (!buddy) {
118		mlx5hws_err(pool->ctx, "Failed to create buddy order: %zu\n",
119			    pool->alloc_log_sz);
120		return -ENOMEM;
121	}
122
123	if (hws_pool_resource_alloc(pool) != 0) {
124		mlx5hws_err(pool->ctx, "Failed to create resource type: %d size %zu\n",
125			    pool->type, pool->alloc_log_sz);
126		mlx5hws_buddy_cleanup(buddy);
127		kfree(buddy);
128		return -ENOMEM;
129	}
130
131	pool->db.buddy = buddy;
132
133	return 0;
134}
135
136static int hws_pool_buddy_db_get_chunk(struct mlx5hws_pool *pool,
137				       struct mlx5hws_pool_chunk *chunk)
138{
139	struct mlx5hws_buddy_mem *buddy = pool->db.buddy;
140
141	if (!buddy) {
142		mlx5hws_err(pool->ctx, "Bad buddy state\n");
143		return -EINVAL;
144	}
145
146	chunk->offset = mlx5hws_buddy_alloc_mem(buddy, chunk->order);
147	if (chunk->offset >= 0)
148		return 0;
149
150	return -ENOMEM;
151}
152
153static void hws_pool_buddy_db_put_chunk(struct mlx5hws_pool *pool,
154					struct mlx5hws_pool_chunk *chunk)
155{
156	struct mlx5hws_buddy_mem *buddy;
157
158	buddy = pool->db.buddy;
159	if (!buddy) {
160		mlx5hws_err(pool->ctx, "Bad buddy state\n");
161		return;
162	}
163
164	mlx5hws_buddy_free_mem(buddy, chunk->offset, chunk->order);
165}
166
167static void hws_pool_buddy_db_uninit(struct mlx5hws_pool *pool)
168{
169	struct mlx5hws_buddy_mem *buddy;
170
171	buddy = pool->db.buddy;
172	if (buddy) {
173		mlx5hws_buddy_cleanup(buddy);
174		kfree(buddy);
175		pool->db.buddy = NULL;
176	}
177}
178
179static int hws_pool_buddy_db_init(struct mlx5hws_pool *pool)
180{
181	int ret;
182
183	ret = hws_pool_buddy_init(pool);
184	if (ret)
185		return ret;
186
187	pool->p_db_uninit = &hws_pool_buddy_db_uninit;
188	pool->p_get_chunk = &hws_pool_buddy_db_get_chunk;
189	pool->p_put_chunk = &hws_pool_buddy_db_put_chunk;
190
191	return 0;
192}
193
194static unsigned long *hws_pool_create_and_init_bitmap(u32 log_range)
195{
196	unsigned long *bitmap;
197
198	bitmap = bitmap_zalloc(1 << log_range, GFP_KERNEL);
199	if (!bitmap)
200		return NULL;
201
202	bitmap_fill(bitmap, 1 << log_range);
203
204	return bitmap;
205}
206
207static int hws_pool_bitmap_init(struct mlx5hws_pool *pool)
208{
209	unsigned long *bitmap;
210
211	bitmap = hws_pool_create_and_init_bitmap(pool->alloc_log_sz);
212	if (!bitmap) {
213		mlx5hws_err(pool->ctx, "Failed to create bitmap order: %zu\n",
214			    pool->alloc_log_sz);
215		return -ENOMEM;
216	}
217
218	if (hws_pool_resource_alloc(pool) != 0) {
219		mlx5hws_err(pool->ctx, "Failed to create resource type: %d: size %zu\n",
220			    pool->type, pool->alloc_log_sz);
221		bitmap_free(bitmap);
222		return -ENOMEM;
223	}
224
225	pool->db.bitmap = bitmap;
226
227	return 0;
228}
229
230static int hws_pool_bitmap_db_get_chunk(struct mlx5hws_pool *pool,
231					struct mlx5hws_pool_chunk *chunk)
232{
233	unsigned long *bitmap, size;
234
235	if (chunk->order != 0) {
236		mlx5hws_err(pool->ctx, "Pool only supports order 0 allocs\n");
237		return -EINVAL;
238	}
239
240	bitmap = pool->db.bitmap;
241	if (!bitmap) {
242		mlx5hws_err(pool->ctx, "Bad bitmap state\n");
243		return -EINVAL;
244	}
245
246	size = 1 << pool->alloc_log_sz;
247
248	chunk->offset = find_first_bit(bitmap, size);
249	if (chunk->offset >= size)
250		return -ENOMEM;
251
252	bitmap_clear(bitmap, chunk->offset, 1);
253
254	return 0;
255}
256
257static void hws_pool_bitmap_db_put_chunk(struct mlx5hws_pool *pool,
258					 struct mlx5hws_pool_chunk *chunk)
259{
260	unsigned long *bitmap;
261
262	bitmap = pool->db.bitmap;
263	if (!bitmap) {
264		mlx5hws_err(pool->ctx, "Bad bitmap state\n");
265		return;
266	}
267
268	bitmap_set(bitmap, chunk->offset, 1);
269}
270
271static void hws_pool_bitmap_db_uninit(struct mlx5hws_pool *pool)
272{
273	unsigned long *bitmap;
274
275	bitmap = pool->db.bitmap;
276	if (bitmap) {
277		bitmap_free(bitmap);
278		pool->db.bitmap = NULL;
279	}
280}
281
282static int hws_pool_bitmap_db_init(struct mlx5hws_pool *pool)
283{
284	int ret;
285
286	ret = hws_pool_bitmap_init(pool);
287	if (ret)
288		return ret;
289
290	pool->p_db_uninit = &hws_pool_bitmap_db_uninit;
291	pool->p_get_chunk = &hws_pool_bitmap_db_get_chunk;
292	pool->p_put_chunk = &hws_pool_bitmap_db_put_chunk;
293
294	return 0;
295}
296
297static int hws_pool_db_init(struct mlx5hws_pool *pool,
298			    enum mlx5hws_db_type db_type)
299{
300	int ret;
301
302	if (db_type == MLX5HWS_POOL_DB_TYPE_BITMAP)
303		ret = hws_pool_bitmap_db_init(pool);
304	else
305		ret = hws_pool_buddy_db_init(pool);
306
307	if (ret) {
308		mlx5hws_err(pool->ctx, "Failed to init pool type: %d (ret: %d)\n",
309			    db_type, ret);
310		return ret;
311	}
312
313	return 0;
314}
315
316static void hws_pool_db_unint(struct mlx5hws_pool *pool)
317{
318	pool->p_db_uninit(pool);
319}
320
321int mlx5hws_pool_chunk_alloc(struct mlx5hws_pool *pool,
322			     struct mlx5hws_pool_chunk *chunk)
323{
324	int ret;
325
326	mutex_lock(&pool->lock);
327	ret = pool->p_get_chunk(pool, chunk);
328	if (ret == 0)
329		pool->available_elems -= 1 << chunk->order;
330	mutex_unlock(&pool->lock);
331
332	return ret;
333}
334
335void mlx5hws_pool_chunk_free(struct mlx5hws_pool *pool,
336			     struct mlx5hws_pool_chunk *chunk)
337{
338	mutex_lock(&pool->lock);
339	pool->p_put_chunk(pool, chunk);
340	pool->available_elems += 1 << chunk->order;
341	mutex_unlock(&pool->lock);
342}
343
344struct mlx5hws_pool *
345mlx5hws_pool_create(struct mlx5hws_context *ctx, struct mlx5hws_pool_attr *pool_attr)
346{
347	enum mlx5hws_db_type res_db_type;
348	struct mlx5hws_pool *pool;
349
350	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
351	if (!pool)
352		return NULL;
353
354	pool->ctx = ctx;
355	pool->type = pool_attr->pool_type;
356	pool->alloc_log_sz = pool_attr->alloc_log_sz;
357	pool->flags = pool_attr->flags;
358	pool->tbl_type = pool_attr->table_type;
359	pool->opt_type = pool_attr->opt_type;
360
361	if (pool->flags & MLX5HWS_POOL_FLAG_BUDDY)
362		res_db_type = MLX5HWS_POOL_DB_TYPE_BUDDY;
363	else
364		res_db_type = MLX5HWS_POOL_DB_TYPE_BITMAP;
365
366	pool->alloc_log_sz = pool_attr->alloc_log_sz;
367	pool->available_elems = 1 << pool_attr->alloc_log_sz;
368
369	if (hws_pool_db_init(pool, res_db_type))
370		goto free_pool;
371
372	mutex_init(&pool->lock);
373
374	return pool;
375
376free_pool:
377	kfree(pool);
378	return NULL;
379}
380
381void mlx5hws_pool_destroy(struct mlx5hws_pool *pool)
382{
383	mutex_destroy(&pool->lock);
384
385	if (pool->available_elems != 1 << pool->alloc_log_sz)
386		mlx5hws_err(pool->ctx, "Attempting to destroy non-empty pool\n");
387
388	if (pool->resource)
389		hws_pool_resource_free(pool);
390
391	hws_pool_db_unint(pool);
392
393	kfree(pool);
394}