sd.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
  3
  4#include "lib/sd.h"
  5#include "mlx5_core.h"
  6#include "lib/mlx5.h"
  7#include "fs_cmd.h"
  8#include <linux/mlx5/vport.h>
  9#include <linux/debugfs.h>
 10
 11#define sd_info(__dev, format, ...) \
 12	dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)
 13#define sd_warn(__dev, format, ...) \
 14	dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)
 15
 16struct mlx5_sd {
 17	u32 group_id;
 18	u8 host_buses;
 19	struct mlx5_devcom_comp_dev *devcom;
 20	struct dentry *dfs;
 21	bool primary;
 22	union {
 23		struct { /* primary */
 24			struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1];
 25			struct mlx5_flow_table *tx_ft;
 26		};
 27		struct { /* secondary */
 28			struct mlx5_core_dev *primary_dev;
 29			u32 alias_obj_id;
 30		};
 31	};
 32};
 33
 34static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev)
 35{
 36	struct mlx5_sd *sd = mlx5_get_sd(dev);
 37
 38	if (!sd)
 39		return 1;
 40
 41	return sd->host_buses;
 42}
 43
 44static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev)
 45{
 46	struct mlx5_sd *sd = mlx5_get_sd(dev);
 47
 48	if (!sd)
 49		return dev;
 50
 51	return sd->primary ? dev : sd->primary_dev;
 52}
 53
 54struct mlx5_core_dev *
 55mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx)
 56{
 57	struct mlx5_sd *sd;
 58
 59	if (idx == 0)
 60		return primary;
 61
 62	if (idx >= mlx5_sd_get_host_buses(primary))
 63		return NULL;
 64
 65	sd = mlx5_get_sd(primary);
 66	return sd->secondaries[idx - 1];
 67}
 68
 69int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix)
 70{
 71	return ch_ix % mlx5_sd_get_host_buses(dev);
 72}
 73
 74int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix)
 75{
 76	return ch_ix / mlx5_sd_get_host_buses(dev);
 77}
 78
 79struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix)
 80{
 81	int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix);
 82
 83	return mlx5_sd_primary_get_peer(primary, mdev_idx);
 84}
 85
 86static bool ft_create_alias_supported(struct mlx5_core_dev *dev)
 87{
 88	u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access);
 89	u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported);
 90
 91	if (!(obj_supp &
 92	    MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE))
 93		return false;
 94
 95	if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE))
 96		return false;
 97
 98	return true;
 99}
100
101static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses)
102{
103	/* Honor the SW implementation limit */
104	if (host_buses > MLX5_SD_MAX_GROUP_SZ)
105		return false;
106
107	/* Disconnect secondaries from the network */
108	if (!MLX5_CAP_GEN(dev, eswitch_manager))
109		return false;
110	if (!MLX5_CAP_GEN(dev, silent_mode))
111		return false;
112
113	/* RX steering from primary to secondaries */
114	if (!MLX5_CAP_GEN(dev, cross_vhca_rqt))
115		return false;
116	if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id))
117		return false;
118
119	/* TX steering from secondaries to primary */
120	if (!ft_create_alias_supported(dev))
121		return false;
122	if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default))
123		return false;
124
125	return true;
126}
127
128static int mlx5_query_sd(struct mlx5_core_dev *dev, bool *sdm,
129			 u8 *host_buses)
130{
131	u32 out[MLX5_ST_SZ_DW(mpir_reg)];
132	int err;
133
134	err = mlx5_query_mpir_reg(dev, out);
135	if (err)
136		return err;
137
138	*sdm = MLX5_GET(mpir_reg, out, sdm);
139	*host_buses = MLX5_GET(mpir_reg, out, host_buses);
140
141	return 0;
142}
143
144static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group)
145{
146	return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) | sd_group);
147}
148
149static int sd_init(struct mlx5_core_dev *dev)
150{
151	u8 host_buses, sd_group;
152	struct mlx5_sd *sd;
153	u32 group_id;
154	bool sdm;
155	int err;
156
157	/* Feature is currently implemented for PFs only */
158	if (!mlx5_core_is_pf(dev))
159		return 0;
160
161	/* Block on embedded CPU PFs */
162	if (mlx5_core_is_ecpf(dev))
163		return 0;
164
165	err = mlx5_query_nic_vport_sd_group(dev, &sd_group);
166	if (err)
167		return err;
168
169	if (!sd_group)
170		return 0;
171
172	if (!MLX5_CAP_MCAM_REG(dev, mpir))
173		return 0;
174
175	err = mlx5_query_sd(dev, &sdm, &host_buses);
176	if (err)
177		return err;
178
179	if (!sdm)
180		return 0;
181
182	group_id = mlx5_sd_group_id(dev, sd_group);
183
184	if (!mlx5_sd_is_supported(dev, host_buses)) {
185		sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n",
186			group_id);
187		return 0;
188	}
189
190	sd = kzalloc(sizeof(*sd), GFP_KERNEL);
191	if (!sd)
192		return -ENOMEM;
193
194	sd->host_buses = host_buses;
195	sd->group_id = group_id;
196
197	mlx5_set_sd(dev, sd);
198
199	return 0;
200}
201
202static void sd_cleanup(struct mlx5_core_dev *dev)
203{
204	struct mlx5_sd *sd = mlx5_get_sd(dev);
205
206	mlx5_set_sd(dev, NULL);
207	kfree(sd);
208}
209
210static int sd_register(struct mlx5_core_dev *dev)
211{
212	struct mlx5_devcom_comp_dev *devcom, *pos;
213	struct mlx5_devcom_match_attr attr = {};
214	struct mlx5_core_dev *peer, *primary;
215	struct mlx5_sd *sd, *primary_sd;
216	int err, i;
217
218	sd = mlx5_get_sd(dev);
219	attr.key.val = sd->group_id;
220	attr.flags = MLX5_DEVCOM_MATCH_FLAGS_NS;
221	attr.net = mlx5_core_net(dev);
222	devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP,
223						&attr, NULL, dev);
224	if (!devcom)
225		return -EINVAL;
226
227	sd->devcom = devcom;
228
229	if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses)
230		return 0;
231
232	mlx5_devcom_comp_lock(devcom);
233	mlx5_devcom_comp_set_ready(devcom, true);
234	mlx5_devcom_comp_unlock(devcom);
235
236	if (!mlx5_devcom_for_each_peer_begin(devcom)) {
237		err = -ENODEV;
238		goto err_devcom_unreg;
239	}
240
241	primary = dev;
242	mlx5_devcom_for_each_peer_entry(devcom, peer, pos)
243		if (peer->pdev->bus->number < primary->pdev->bus->number)
244			primary = peer;
245
246	primary_sd = mlx5_get_sd(primary);
247	primary_sd->primary = true;
248	i = 0;
249	/* loop the secondaries */
250	mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) {
251		struct mlx5_sd *peer_sd = mlx5_get_sd(peer);
252
253		primary_sd->secondaries[i++] = peer;
254		peer_sd->primary = false;
255		peer_sd->primary_dev = primary;
256	}
257
258	mlx5_devcom_for_each_peer_end(devcom);
259	return 0;
260
261err_devcom_unreg:
262	mlx5_devcom_comp_lock(sd->devcom);
263	mlx5_devcom_comp_set_ready(sd->devcom, false);
264	mlx5_devcom_comp_unlock(sd->devcom);
265	mlx5_devcom_unregister_component(sd->devcom);
266	return err;
267}
268
269static void sd_unregister(struct mlx5_core_dev *dev)
270{
271	struct mlx5_sd *sd = mlx5_get_sd(dev);
272
273	mlx5_devcom_comp_lock(sd->devcom);
274	mlx5_devcom_comp_set_ready(sd->devcom, false);
275	mlx5_devcom_comp_unlock(sd->devcom);
276	mlx5_devcom_unregister_component(sd->devcom);
277}
278
279static int sd_cmd_set_primary(struct mlx5_core_dev *primary, u8 *alias_key)
280{
281	struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {};
282	struct mlx5_sd *sd = mlx5_get_sd(primary);
283	struct mlx5_flow_table_attr ft_attr = {};
284	struct mlx5_flow_namespace *nic_ns;
285	struct mlx5_flow_table *ft;
286	int err;
287
288	nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS);
289	if (!nic_ns)
290		return -EOPNOTSUPP;
291
292	ft = mlx5_create_flow_table(nic_ns, &ft_attr);
293	if (IS_ERR(ft)) {
294		err = PTR_ERR(ft);
295		return err;
296	}
297	sd->tx_ft = ft;
298	memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN);
299	allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
300	allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
301
302	err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr);
303	if (err) {
304		mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n",
305			      err);
306		mlx5_destroy_flow_table(ft);
307		return err;
308	}
309
310	return 0;
311}
312
313static void sd_cmd_unset_primary(struct mlx5_core_dev *primary)
314{
315	struct mlx5_sd *sd = mlx5_get_sd(primary);
316
317	mlx5_destroy_flow_table(sd->tx_ft);
318}
319
320static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary,
321					struct mlx5_core_dev *primary,
322					struct mlx5_flow_table *ft,
323					u32 *obj_id, u8 *alias_key)
324{
325	u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
326	u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id);
327	struct mlx5_cmd_alias_obj_create_attr alias_attr = {};
328	int ret;
329
330	memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN);
331	alias_attr.obj_id = aliased_object_id;
332	alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
333	alias_attr.vhca_id = vhca_id_to_be_accessed;
334	ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id);
335	if (ret) {
336		mlx5_core_err(secondary, "Failed to create alias object err=%d\n",
337			      ret);
338		return ret;
339	}
340
341	return 0;
342}
343
344static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary)
345{
346	struct mlx5_sd *sd = mlx5_get_sd(secondary);
347
348	mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id,
349				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
350}
351
352static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary,
353				struct mlx5_core_dev *primary,
354				u8 *alias_key)
355{
356	struct mlx5_sd *primary_sd = mlx5_get_sd(primary);
357	struct mlx5_sd *sd = mlx5_get_sd(secondary);
358	int err;
359
360	err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1);
361	if (err)
362		return err;
363
364	err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft,
365					   &sd->alias_obj_id, alias_key);
366	if (err)
367		goto err_unset_silent;
368
369	err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false);
370	if (err)
371		goto err_destroy_alias_ft;
372
373	return 0;
374
375err_destroy_alias_ft:
376	sd_secondary_destroy_alias_ft(secondary);
377err_unset_silent:
378	mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
379	return err;
380}
381
382static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary)
383{
384	mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true);
385	sd_secondary_destroy_alias_ft(secondary);
386	mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
387}
388
389static void sd_print_group(struct mlx5_core_dev *primary)
390{
391	struct mlx5_sd *sd = mlx5_get_sd(primary);
392	struct mlx5_core_dev *pos;
393	int i;
394
395	sd_info(primary, "group id %#x, primary %s, vhca %#x\n",
396		sd->group_id, pci_name(primary->pdev),
397		MLX5_CAP_GEN(primary, vhca_id));
398	mlx5_sd_for_each_secondary(i, primary, pos)
399		sd_info(primary, "group id %#x, secondary_%d %s, vhca %#x\n",
400			sd->group_id, i - 1, pci_name(pos->pdev),
401			MLX5_CAP_GEN(pos, vhca_id));
402}
403
404static ssize_t dev_read(struct file *filp, char __user *buf, size_t count,
405			loff_t *pos)
406{
407	struct mlx5_core_dev *dev;
408	char tbuf[32];
409	int ret;
410
411	dev = filp->private_data;
412	ret = snprintf(tbuf, sizeof(tbuf), "%s vhca %#x\n", pci_name(dev->pdev),
413		       MLX5_CAP_GEN(dev, vhca_id));
414
415	return simple_read_from_buffer(buf, count, pos, tbuf, ret);
416}
417
418static const struct file_operations dev_fops = {
419	.owner	= THIS_MODULE,
420	.open	= simple_open,
421	.read	= dev_read,
422};
423
424int mlx5_sd_init(struct mlx5_core_dev *dev)
425{
426	struct mlx5_core_dev *primary, *pos, *to;
427	struct mlx5_sd *sd = mlx5_get_sd(dev);
428	u8 alias_key[ACCESS_KEY_LEN];
429	int err, i;
430
431	err = sd_init(dev);
432	if (err)
433		return err;
434
435	sd = mlx5_get_sd(dev);
436	if (!sd)
437		return 0;
438
439	err = sd_register(dev);
440	if (err)
441		goto err_sd_cleanup;
442
443	if (!mlx5_devcom_comp_is_ready(sd->devcom))
444		return 0;
445
446	primary = mlx5_sd_get_primary(dev);
447
448	for (i = 0; i < ACCESS_KEY_LEN; i++)
449		alias_key[i] = get_random_u8();
450
451	err = sd_cmd_set_primary(primary, alias_key);
452	if (err)
453		goto err_sd_unregister;
454
455	sd->dfs = debugfs_create_dir("multi-pf", mlx5_debugfs_get_dev_root(primary));
456	debugfs_create_x32("group_id", 0400, sd->dfs, &sd->group_id);
457	debugfs_create_file("primary", 0400, sd->dfs, primary, &dev_fops);
458
459	mlx5_sd_for_each_secondary(i, primary, pos) {
460		char name[32];
461
462		err = sd_cmd_set_secondary(pos, primary, alias_key);
463		if (err)
464			goto err_unset_secondaries;
465
466		snprintf(name, sizeof(name), "secondary_%d", i - 1);
467		debugfs_create_file(name, 0400, sd->dfs, pos, &dev_fops);
468
469	}
470
471	sd_info(primary, "group id %#x, size %d, combined\n",
472		sd->group_id, mlx5_devcom_comp_get_size(sd->devcom));
473	sd_print_group(primary);
474
475	return 0;
476
477err_unset_secondaries:
478	to = pos;
479	mlx5_sd_for_each_secondary_to(i, primary, to, pos)
480		sd_cmd_unset_secondary(pos);
481	sd_cmd_unset_primary(primary);
482	debugfs_remove_recursive(sd->dfs);
483err_sd_unregister:
484	sd_unregister(dev);
485err_sd_cleanup:
486	sd_cleanup(dev);
487	return err;
488}
489
490void mlx5_sd_cleanup(struct mlx5_core_dev *dev)
491{
492	struct mlx5_sd *sd = mlx5_get_sd(dev);
493	struct mlx5_core_dev *primary, *pos;
494	int i;
495
496	if (!sd)
497		return;
498
499	if (!mlx5_devcom_comp_is_ready(sd->devcom))
500		goto out;
501
502	primary = mlx5_sd_get_primary(dev);
503	mlx5_sd_for_each_secondary(i, primary, pos)
504		sd_cmd_unset_secondary(pos);
505	sd_cmd_unset_primary(primary);
506	debugfs_remove_recursive(sd->dfs);
507
508	sd_info(primary, "group id %#x, uncombined\n", sd->group_id);
509out:
510	sd_unregister(dev);
511	sd_cleanup(dev);
512}
513
514struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev,
515					  struct auxiliary_device *adev,
516					  int idx)
517{
518	struct mlx5_sd *sd = mlx5_get_sd(dev);
519	struct mlx5_core_dev *primary;
520
521	if (!sd)
522		return adev;
523
524	if (!mlx5_devcom_comp_is_ready(sd->devcom))
525		return NULL;
526
527	primary = mlx5_sd_get_primary(dev);
528	if (dev == primary)
529		return adev;
530
531	return &primary->priv.adev[idx]->adev;
532}