dr_domain.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2019 Mellanox Technologies. */
  3
  4#include <linux/mlx5/eswitch.h>
  5#include <linux/err.h>
  6#include "dr_types.h"
  7
  8#define DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, dmn_type)	\
  9	((dmn)->info.caps.dmn_type##_sw_owner ||	\
 10	 ((dmn)->info.caps.dmn_type##_sw_owner_v2 &&	\
 11	  (dmn)->info.caps.sw_format_ver <= MLX5_STEERING_FORMAT_CONNECTX_8))
 12
 13bool mlx5dr_domain_is_support_ptrn_arg(struct mlx5dr_domain *dmn)
 14{
 15	return dmn->info.caps.sw_format_ver >= MLX5_STEERING_FORMAT_CONNECTX_6DX &&
 16	       dmn->info.caps.support_modify_argument;
 17}
 18
 19static int dr_domain_init_modify_header_resources(struct mlx5dr_domain *dmn)
 20{
 21	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
 22		return 0;
 23
 24	dmn->ptrn_mgr = mlx5dr_ptrn_mgr_create(dmn);
 25	if (!dmn->ptrn_mgr) {
 26		mlx5dr_err(dmn, "Couldn't create ptrn_mgr\n");
 27		return -ENOMEM;
 28	}
 29
 30	/* create argument pool */
 31	dmn->arg_mgr = mlx5dr_arg_mgr_create(dmn);
 32	if (!dmn->arg_mgr) {
 33		mlx5dr_err(dmn, "Couldn't create arg_mgr\n");
 34		goto free_modify_header_pattern;
 35	}
 36
 37	return 0;
 38
 39free_modify_header_pattern:
 40	mlx5dr_ptrn_mgr_destroy(dmn->ptrn_mgr);
 41	return -ENOMEM;
 42}
 43
 44static void dr_domain_destroy_modify_header_resources(struct mlx5dr_domain *dmn)
 45{
 46	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
 47		return;
 48
 49	mlx5dr_arg_mgr_destroy(dmn->arg_mgr);
 50	mlx5dr_ptrn_mgr_destroy(dmn->ptrn_mgr);
 51}
 52
 53static void dr_domain_init_csum_recalc_fts(struct mlx5dr_domain *dmn)
 54{
 55	/* Per vport cached FW FT for checksum recalculation, this
 56	 * recalculation is needed due to a HW bug in STEv0.
 57	 */
 58	xa_init(&dmn->csum_fts_xa);
 59}
 60
 61static void dr_domain_uninit_csum_recalc_fts(struct mlx5dr_domain *dmn)
 62{
 63	struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft;
 64	unsigned long i;
 65
 66	xa_for_each(&dmn->csum_fts_xa, i, recalc_cs_ft) {
 67		if (recalc_cs_ft)
 68			mlx5dr_fw_destroy_recalc_cs_ft(dmn, recalc_cs_ft);
 69	}
 70
 71	xa_destroy(&dmn->csum_fts_xa);
 72}
 73
 74int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn,
 75					u16 vport_num,
 76					u64 *rx_icm_addr)
 77{
 78	struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft;
 79	int ret;
 80
 81	recalc_cs_ft = xa_load(&dmn->csum_fts_xa, vport_num);
 82	if (!recalc_cs_ft) {
 83		/* Table hasn't been created yet */
 84		recalc_cs_ft = mlx5dr_fw_create_recalc_cs_ft(dmn, vport_num);
 85		if (!recalc_cs_ft)
 86			return -EINVAL;
 87
 88		ret = xa_err(xa_store(&dmn->csum_fts_xa, vport_num,
 89				      recalc_cs_ft, GFP_KERNEL));
 90		if (ret)
 91			return ret;
 92	}
 93
 94	*rx_icm_addr = recalc_cs_ft->rx_icm_addr;
 95
 96	return 0;
 97}
 98
 99static int dr_domain_init_mem_resources(struct mlx5dr_domain *dmn)
100{
101	int ret;
102
103	dmn->chunks_kmem_cache = kmem_cache_create("mlx5_dr_chunks",
104						   sizeof(struct mlx5dr_icm_chunk), 0,
105						   SLAB_HWCACHE_ALIGN, NULL);
106	if (!dmn->chunks_kmem_cache) {
107		mlx5dr_err(dmn, "Couldn't create chunks kmem_cache\n");
108		return -ENOMEM;
109	}
110
111	dmn->htbls_kmem_cache = kmem_cache_create("mlx5_dr_htbls",
112						  sizeof(struct mlx5dr_ste_htbl), 0,
113						  SLAB_HWCACHE_ALIGN, NULL);
114	if (!dmn->htbls_kmem_cache) {
115		mlx5dr_err(dmn, "Couldn't create hash tables kmem_cache\n");
116		ret = -ENOMEM;
117		goto free_chunks_kmem_cache;
118	}
119
120	dmn->ste_icm_pool = mlx5dr_icm_pool_create(dmn, DR_ICM_TYPE_STE);
121	if (!dmn->ste_icm_pool) {
122		mlx5dr_err(dmn, "Couldn't get icm memory\n");
123		ret = -ENOMEM;
124		goto free_htbls_kmem_cache;
125	}
126
127	dmn->action_icm_pool = mlx5dr_icm_pool_create(dmn, DR_ICM_TYPE_MODIFY_ACTION);
128	if (!dmn->action_icm_pool) {
129		mlx5dr_err(dmn, "Couldn't get action icm memory\n");
130		ret = -ENOMEM;
131		goto free_ste_icm_pool;
132	}
133
134	ret = mlx5dr_send_info_pool_create(dmn);
135	if (ret) {
136		mlx5dr_err(dmn, "Couldn't create send info pool\n");
137		goto free_action_icm_pool;
138	}
139
140	return 0;
141
142free_action_icm_pool:
143	mlx5dr_icm_pool_destroy(dmn->action_icm_pool);
144free_ste_icm_pool:
145	mlx5dr_icm_pool_destroy(dmn->ste_icm_pool);
146free_htbls_kmem_cache:
147	kmem_cache_destroy(dmn->htbls_kmem_cache);
148free_chunks_kmem_cache:
149	kmem_cache_destroy(dmn->chunks_kmem_cache);
150
151	return ret;
152}
153
154static void dr_domain_uninit_mem_resources(struct mlx5dr_domain *dmn)
155{
156	mlx5dr_send_info_pool_destroy(dmn);
157	mlx5dr_icm_pool_destroy(dmn->action_icm_pool);
158	mlx5dr_icm_pool_destroy(dmn->ste_icm_pool);
159	kmem_cache_destroy(dmn->htbls_kmem_cache);
160	kmem_cache_destroy(dmn->chunks_kmem_cache);
161}
162
163static int dr_domain_init_resources(struct mlx5dr_domain *dmn)
164{
165	int ret;
166
167	dmn->ste_ctx = mlx5dr_ste_get_ctx(dmn->info.caps.sw_format_ver);
168	if (!dmn->ste_ctx) {
169		mlx5dr_err(dmn, "SW Steering on this device is unsupported\n");
170		return -EOPNOTSUPP;
171	}
172
173	ret = mlx5_core_alloc_pd(dmn->mdev, &dmn->pdn);
174	if (ret) {
175		mlx5dr_err(dmn, "Couldn't allocate PD, ret: %d", ret);
176		return ret;
177	}
178
179	dmn->uar = mlx5_get_uars_page(dmn->mdev);
180	if (IS_ERR(dmn->uar)) {
181		mlx5dr_err(dmn, "Couldn't allocate UAR\n");
182		ret = PTR_ERR(dmn->uar);
183		goto clean_pd;
184	}
185
186	ret = dr_domain_init_mem_resources(dmn);
187	if (ret) {
188		mlx5dr_err(dmn, "Couldn't create domain memory resources\n");
189		goto clean_uar;
190	}
191
192	ret = dr_domain_init_modify_header_resources(dmn);
193	if (ret) {
194		mlx5dr_err(dmn, "Couldn't create modify-header-resources\n");
195		goto clean_mem_resources;
196	}
197
198	ret = mlx5dr_send_ring_alloc(dmn);
199	if (ret) {
200		mlx5dr_err(dmn, "Couldn't create send-ring\n");
201		goto clean_modify_hdr;
202	}
203
204	return 0;
205
206clean_modify_hdr:
207	dr_domain_destroy_modify_header_resources(dmn);
208clean_mem_resources:
209	dr_domain_uninit_mem_resources(dmn);
210clean_uar:
211	mlx5_put_uars_page(dmn->mdev, dmn->uar);
212clean_pd:
213	mlx5_core_dealloc_pd(dmn->mdev, dmn->pdn);
214
215	return ret;
216}
217
218static void dr_domain_uninit_resources(struct mlx5dr_domain *dmn)
219{
220	mlx5dr_send_ring_free(dmn, dmn->send_ring);
221	dr_domain_destroy_modify_header_resources(dmn);
222	dr_domain_uninit_mem_resources(dmn);
223	mlx5_put_uars_page(dmn->mdev, dmn->uar);
224	mlx5_core_dealloc_pd(dmn->mdev, dmn->pdn);
225}
226
227static void dr_domain_fill_uplink_caps(struct mlx5dr_domain *dmn,
228				       struct mlx5dr_cmd_vport_cap *uplink_vport)
229{
230	struct mlx5dr_esw_caps *esw_caps = &dmn->info.caps.esw_caps;
231
232	uplink_vport->num = MLX5_VPORT_UPLINK;
233	uplink_vport->icm_address_rx = esw_caps->uplink_icm_address_rx;
234	uplink_vport->icm_address_tx = esw_caps->uplink_icm_address_tx;
235	uplink_vport->vport_gvmi = 0;
236	uplink_vport->vhca_gvmi = dmn->info.caps.gvmi;
237}
238
239static int dr_domain_query_vport(struct mlx5dr_domain *dmn,
240				 u16 vport_number,
241				 bool other_vport,
242				 struct mlx5dr_cmd_vport_cap *vport_caps)
243{
244	int ret;
245
246	ret = mlx5dr_cmd_query_esw_vport_context(dmn->mdev,
247						 other_vport,
248						 vport_number,
249						 &vport_caps->icm_address_rx,
250						 &vport_caps->icm_address_tx);
251	if (ret)
252		return ret;
253
254	ret = mlx5dr_cmd_query_gvmi(dmn->mdev,
255				    other_vport,
256				    vport_number,
257				    &vport_caps->vport_gvmi);
258	if (ret)
259		return ret;
260
261	vport_caps->num = vport_number;
262	vport_caps->vhca_gvmi = dmn->info.caps.gvmi;
263
264	return 0;
265}
266
267static int dr_domain_query_esw_mgr(struct mlx5dr_domain *dmn)
268{
269	return dr_domain_query_vport(dmn, 0, false,
270				     &dmn->info.caps.vports.esw_manager_caps);
271}
272
273static void dr_domain_query_uplink(struct mlx5dr_domain *dmn)
274{
275	dr_domain_fill_uplink_caps(dmn, &dmn->info.caps.vports.uplink_caps);
276}
277
278static struct mlx5dr_cmd_vport_cap *
279dr_domain_add_vport_cap(struct mlx5dr_domain *dmn, u16 vport)
280{
281	struct mlx5dr_cmd_caps *caps = &dmn->info.caps;
282	struct mlx5dr_cmd_vport_cap *vport_caps;
283	int ret;
284
285	vport_caps = kvzalloc(sizeof(*vport_caps), GFP_KERNEL);
286	if (!vport_caps)
287		return NULL;
288
289	ret = dr_domain_query_vport(dmn, vport, true, vport_caps);
290	if (ret) {
291		kvfree(vport_caps);
292		return NULL;
293	}
294
295	ret = xa_insert(&caps->vports.vports_caps_xa, vport,
296			vport_caps, GFP_KERNEL);
297	if (ret) {
298		mlx5dr_dbg(dmn, "Couldn't insert new vport into xarray (%d)\n", ret);
299		kvfree(vport_caps);
300		if (ret == -EBUSY)
301			return ERR_PTR(-EBUSY);
302		return NULL;
303	}
304
305	return vport_caps;
306}
307
308static bool dr_domain_is_esw_mgr_vport(struct mlx5dr_domain *dmn, u16 vport)
309{
310	struct mlx5dr_cmd_caps *caps = &dmn->info.caps;
311
312	return (caps->is_ecpf && vport == MLX5_VPORT_ECPF) ||
313	       (!caps->is_ecpf && vport == 0);
314}
315
316struct mlx5dr_cmd_vport_cap *
317mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport)
318{
319	struct mlx5dr_cmd_caps *caps = &dmn->info.caps;
320	struct mlx5dr_cmd_vport_cap *vport_caps;
321
322	if (dr_domain_is_esw_mgr_vport(dmn, vport))
323		return &caps->vports.esw_manager_caps;
324
325	if (vport == MLX5_VPORT_UPLINK)
326		return &caps->vports.uplink_caps;
327
328vport_load:
329	vport_caps = xa_load(&caps->vports.vports_caps_xa, vport);
330	if (vport_caps)
331		return vport_caps;
332
333	vport_caps = dr_domain_add_vport_cap(dmn, vport);
334	if (PTR_ERR(vport_caps) == -EBUSY)
335		/* caps were already stored by another thread */
336		goto vport_load;
337
338	return vport_caps;
339}
340
341static void dr_domain_clear_vports(struct mlx5dr_domain *dmn)
342{
343	struct mlx5dr_cmd_vport_cap *vport_caps;
344	unsigned long i;
345
346	xa_for_each(&dmn->info.caps.vports.vports_caps_xa, i, vport_caps) {
347		vport_caps = xa_erase(&dmn->info.caps.vports.vports_caps_xa, i);
348		kvfree(vport_caps);
349	}
350}
351
352static int dr_domain_query_fdb_caps(struct mlx5_core_dev *mdev,
353				    struct mlx5dr_domain *dmn)
354{
355	int ret;
356
357	if (!dmn->info.caps.eswitch_manager)
358		return -EOPNOTSUPP;
359
360	ret = mlx5dr_cmd_query_esw_caps(mdev, &dmn->info.caps.esw_caps);
361	if (ret)
362		return ret;
363
364	dmn->info.caps.fdb_sw_owner = dmn->info.caps.esw_caps.sw_owner;
365	dmn->info.caps.fdb_sw_owner_v2 = dmn->info.caps.esw_caps.sw_owner_v2;
366	dmn->info.caps.esw_rx_drop_address = dmn->info.caps.esw_caps.drop_icm_address_rx;
367	dmn->info.caps.esw_tx_drop_address = dmn->info.caps.esw_caps.drop_icm_address_tx;
368
369	xa_init(&dmn->info.caps.vports.vports_caps_xa);
370
371	/* Query eswitch manager and uplink vports only. Rest of the
372	 * vports (vport 0, VFs and SFs) will be queried dynamically.
373	 */
374
375	ret = dr_domain_query_esw_mgr(dmn);
376	if (ret) {
377		mlx5dr_err(dmn, "Failed to query eswitch manager vport caps (err: %d)", ret);
378		goto free_vports_caps_xa;
379	}
380
381	dr_domain_query_uplink(dmn);
382
383	return 0;
384
385free_vports_caps_xa:
386	xa_destroy(&dmn->info.caps.vports.vports_caps_xa);
387
388	return ret;
389}
390
391static int dr_domain_caps_init(struct mlx5_core_dev *mdev,
392			       struct mlx5dr_domain *dmn)
393{
394	struct mlx5dr_cmd_vport_cap *vport_cap;
395	int ret;
396
397	if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) {
398		mlx5dr_err(dmn, "Failed to allocate domain, bad link type\n");
399		return -EOPNOTSUPP;
400	}
401
402	ret = mlx5dr_cmd_query_device(mdev, &dmn->info.caps);
403	if (ret)
404		return ret;
405
406	ret = dr_domain_query_fdb_caps(mdev, dmn);
407	if (ret)
408		return ret;
409
410	switch (dmn->type) {
411	case MLX5DR_DOMAIN_TYPE_NIC_RX:
412		if (!DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, rx))
413			return -EOPNOTSUPP;
414
415		dmn->info.supp_sw_steering = true;
416		dmn->info.rx.type = DR_DOMAIN_NIC_TYPE_RX;
417		dmn->info.rx.default_icm_addr = dmn->info.caps.nic_rx_drop_address;
418		dmn->info.rx.drop_icm_addr = dmn->info.caps.nic_rx_drop_address;
419		break;
420	case MLX5DR_DOMAIN_TYPE_NIC_TX:
421		if (!DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, tx))
422			return -EOPNOTSUPP;
423
424		dmn->info.supp_sw_steering = true;
425		dmn->info.tx.type = DR_DOMAIN_NIC_TYPE_TX;
426		dmn->info.tx.default_icm_addr = dmn->info.caps.nic_tx_allow_address;
427		dmn->info.tx.drop_icm_addr = dmn->info.caps.nic_tx_drop_address;
428		break;
429	case MLX5DR_DOMAIN_TYPE_FDB:
430		if (!dmn->info.caps.eswitch_manager)
431			return -EOPNOTSUPP;
432
433		if (!DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, fdb))
434			return -EOPNOTSUPP;
435
436		dmn->info.rx.type = DR_DOMAIN_NIC_TYPE_RX;
437		dmn->info.tx.type = DR_DOMAIN_NIC_TYPE_TX;
438		vport_cap = &dmn->info.caps.vports.esw_manager_caps;
439
440		dmn->info.supp_sw_steering = true;
441		dmn->info.tx.default_icm_addr = vport_cap->icm_address_tx;
442		dmn->info.rx.default_icm_addr = vport_cap->icm_address_rx;
443		dmn->info.rx.drop_icm_addr = dmn->info.caps.esw_rx_drop_address;
444		dmn->info.tx.drop_icm_addr = dmn->info.caps.esw_tx_drop_address;
445		break;
446	default:
447		mlx5dr_err(dmn, "Invalid domain\n");
448		ret = -EINVAL;
449		break;
450	}
451
452	return ret;
453}
454
455static void dr_domain_caps_uninit(struct mlx5dr_domain *dmn)
456{
457	dr_domain_clear_vports(dmn);
458	xa_destroy(&dmn->info.caps.vports.vports_caps_xa);
459}
460
461struct mlx5dr_domain *
462mlx5dr_domain_create(struct mlx5_core_dev *mdev, enum mlx5dr_domain_type type)
463{
464	struct mlx5dr_domain *dmn;
465	int ret;
466
467	if (type > MLX5DR_DOMAIN_TYPE_FDB)
468		return NULL;
469
470	dmn = kzalloc(sizeof(*dmn), GFP_KERNEL);
471	if (!dmn)
472		return NULL;
473
474	dmn->mdev = mdev;
475	dmn->type = type;
476	refcount_set(&dmn->refcount, 1);
477	mutex_init(&dmn->info.rx.mutex);
478	mutex_init(&dmn->info.tx.mutex);
479	xa_init(&dmn->definers_xa);
480	xa_init(&dmn->peer_dmn_xa);
481
482	if (dr_domain_caps_init(mdev, dmn)) {
483		mlx5dr_err(dmn, "Failed init domain, no caps\n");
484		goto def_xa_destroy;
485	}
486
487	dmn->info.max_log_action_icm_sz = DR_CHUNK_SIZE_4K;
488	dmn->info.max_log_sw_icm_sz = min_t(u32, DR_CHUNK_SIZE_1024K,
489					    dmn->info.caps.log_icm_size);
490	dmn->info.max_log_modify_hdr_pattern_icm_sz =
491		min_t(u32, DR_CHUNK_SIZE_4K,
492		      dmn->info.caps.log_modify_pattern_icm_size);
493
494	if (!dmn->info.supp_sw_steering) {
495		mlx5dr_err(dmn, "SW steering is not supported\n");
496		goto uninit_caps;
497	}
498
499	/* Allocate resources */
500	ret = dr_domain_init_resources(dmn);
501	if (ret) {
502		mlx5dr_err(dmn, "Failed init domain resources\n");
503		goto uninit_caps;
504	}
505
506	dr_domain_init_csum_recalc_fts(dmn);
507	mlx5dr_dbg_init_dump(dmn);
508	return dmn;
509
510uninit_caps:
511	dr_domain_caps_uninit(dmn);
512def_xa_destroy:
513	xa_destroy(&dmn->peer_dmn_xa);
514	xa_destroy(&dmn->definers_xa);
515	kfree(dmn);
516	return NULL;
517}
518
519int mlx5dr_domain_destroy(struct mlx5dr_domain *dmn)
520{
521	if (WARN_ON_ONCE(refcount_read(&dmn->refcount) > 1))
522		return -EBUSY;
523
524	/* make sure resources are not used by the hardware */
525	mlx5dr_cmd_sync_steering(dmn->mdev);
526	mlx5dr_dbg_uninit_dump(dmn);
527	dr_domain_uninit_csum_recalc_fts(dmn);
528	dr_domain_uninit_resources(dmn);
529	dr_domain_caps_uninit(dmn);
530	xa_destroy(&dmn->peer_dmn_xa);
531	xa_destroy(&dmn->definers_xa);
532	mutex_destroy(&dmn->info.tx.mutex);
533	mutex_destroy(&dmn->info.rx.mutex);
534	kfree(dmn);
535	return 0;
536}
537
538void mlx5dr_domain_set_peer(struct mlx5dr_domain *dmn,
539			    struct mlx5dr_domain *peer_dmn,
540			    u16 peer_vhca_id)
541{
542	struct mlx5dr_domain *peer;
543
544	mlx5dr_domain_lock(dmn);
545
546	peer = xa_load(&dmn->peer_dmn_xa, peer_vhca_id);
547	if (peer)
548		refcount_dec(&peer->refcount);
549
550	WARN_ON(xa_err(xa_store(&dmn->peer_dmn_xa, peer_vhca_id, peer_dmn, GFP_KERNEL)));
551
552	peer = xa_load(&dmn->peer_dmn_xa, peer_vhca_id);
553	if (peer)
554		refcount_inc(&peer->refcount);
555
556	mlx5dr_domain_unlock(dmn);
557}