dev.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2020 Mellanox Technologies Ltd */
  3
  4#include <linux/mlx5/driver.h>
  5#include <linux/mlx5/device.h>
  6#include "mlx5_core.h"
  7#include "dev.h"
  8#include "sf/vhca_event.h"
  9#include "sf/sf.h"
 10#include "sf/mlx5_ifc_vhca_event.h"
 11#include "ecpf.h"
 12#define CREATE_TRACE_POINTS
 13#include "diag/dev_tracepoint.h"
 14
 15struct mlx5_sf_dev_table {
 16	struct xarray devices;
 17	phys_addr_t base_address;
 18	u64 sf_bar_length;
 19	struct workqueue_struct *active_wq;
 20	struct work_struct work;
 21	u8 stop_active_wq:1;
 22	struct mlx5_core_dev *dev;
 23};
 24
 25struct mlx5_sf_dev_active_work_ctx {
 26	struct work_struct work;
 27	struct mlx5_vhca_state_event event;
 28	struct mlx5_sf_dev_table *table;
 29	int sf_index;
 30};
 31
 32static bool mlx5_sf_dev_supported(const struct mlx5_core_dev *dev)
 33{
 34	return MLX5_CAP_GEN(dev, sf) && mlx5_vhca_event_supported(dev);
 35}
 36
 37bool mlx5_sf_dev_allocated(const struct mlx5_core_dev *dev)
 38{
 39	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
 40
 41	return table && !xa_empty(&table->devices);
 42}
 43
 44static ssize_t sfnum_show(struct device *dev, struct device_attribute *attr, char *buf)
 45{
 46	struct auxiliary_device *adev = container_of(dev, struct auxiliary_device, dev);
 47	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
 48
 49	return sysfs_emit(buf, "%u\n", sf_dev->sfnum);
 50}
 51static DEVICE_ATTR_RO(sfnum);
 52
 53static struct attribute *sf_device_attrs[] = {
 54	&dev_attr_sfnum.attr,
 55	NULL,
 56};
 57
 58static const struct attribute_group sf_attr_group = {
 59	.attrs = sf_device_attrs,
 60};
 61
 62static const struct attribute_group *sf_attr_groups[2] = {
 63	&sf_attr_group,
 64	NULL
 65};
 66
 67static void mlx5_sf_dev_release(struct device *device)
 68{
 69	struct auxiliary_device *adev = container_of(device, struct auxiliary_device, dev);
 70	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
 71
 72	mlx5_adev_idx_free(adev->id);
 73	kfree(sf_dev);
 74}
 75
 76static void mlx5_sf_dev_remove_aux(struct mlx5_core_dev *dev,
 77				   struct mlx5_sf_dev *sf_dev)
 78{
 79	int id;
 80
 81	id = sf_dev->adev.id;
 82	trace_mlx5_sf_dev_del(dev, sf_dev, id);
 83
 84	auxiliary_device_delete(&sf_dev->adev);
 85	auxiliary_device_uninit(&sf_dev->adev);
 86}
 87
 88static void mlx5_sf_dev_add(struct mlx5_core_dev *dev, u16 sf_index, u16 fn_id, u32 sfnum)
 89{
 90	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
 91	struct mlx5_sf_dev *sf_dev;
 92	struct pci_dev *pdev;
 93	int err;
 94	int id;
 95
 96	id = mlx5_adev_idx_alloc();
 97	if (id < 0) {
 98		err = id;
 99		goto add_err;
100	}
101
102	sf_dev = kzalloc(sizeof(*sf_dev), GFP_KERNEL);
103	if (!sf_dev) {
104		mlx5_adev_idx_free(id);
105		err = -ENOMEM;
106		goto add_err;
107	}
108	pdev = dev->pdev;
109	sf_dev->adev.id = id;
110	sf_dev->adev.name = MLX5_SF_DEV_ID_NAME;
111	sf_dev->adev.dev.release = mlx5_sf_dev_release;
112	sf_dev->adev.dev.parent = &pdev->dev;
113	sf_dev->adev.dev.groups = sf_attr_groups;
114	sf_dev->sfnum = sfnum;
115	sf_dev->parent_mdev = dev;
116	sf_dev->fn_id = fn_id;
117
118	sf_dev->bar_base_addr = table->base_address + (sf_index * table->sf_bar_length);
119
120	trace_mlx5_sf_dev_add(dev, sf_dev, id);
121
122	err = auxiliary_device_init(&sf_dev->adev);
123	if (err) {
124		mlx5_adev_idx_free(id);
125		kfree(sf_dev);
126		goto add_err;
127	}
128
129	err = auxiliary_device_add(&sf_dev->adev);
130	if (err) {
131		auxiliary_device_uninit(&sf_dev->adev);
132		goto add_err;
133	}
134
135	err = xa_insert(&table->devices, sf_index, sf_dev, GFP_KERNEL);
136	if (err)
137		goto xa_err;
138	return;
139
140xa_err:
141	mlx5_sf_dev_remove_aux(dev, sf_dev);
142add_err:
143	mlx5_core_err(dev, "SF DEV: fail device add for index=%d sfnum=%d err=%d\n",
144		      sf_index, sfnum, err);
145}
146
147static void mlx5_sf_dev_del(struct mlx5_core_dev *dev, struct mlx5_sf_dev *sf_dev, u16 sf_index)
148{
149	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
150
151	xa_erase(&table->devices, sf_index);
152	mlx5_sf_dev_remove_aux(dev, sf_dev);
153}
154
155static int
156mlx5_sf_dev_state_change_handler(struct notifier_block *nb, unsigned long event_code, void *data)
157{
158	struct mlx5_core_dev *dev = container_of(nb, struct mlx5_core_dev,
159						 priv.sf_dev_nb);
160	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
161	const struct mlx5_vhca_state_event *event = data;
162	struct mlx5_sf_dev *sf_dev;
163	u16 max_functions;
164	u16 sf_index;
165	u16 base_id;
166
167	if (!table)
168		return 0;
169
170	max_functions = mlx5_sf_max_functions(dev);
171	if (!max_functions)
172		return 0;
173
174	base_id = mlx5_sf_start_function_id(dev);
175	if (event->function_id < base_id || event->function_id >= (base_id + max_functions))
176		return 0;
177
178	sf_index = event->function_id - base_id;
179	sf_dev = xa_load(&table->devices, sf_index);
180	switch (event->new_vhca_state) {
181	case MLX5_VHCA_STATE_INVALID:
182	case MLX5_VHCA_STATE_ALLOCATED:
183		if (sf_dev)
184			mlx5_sf_dev_del(dev, sf_dev, sf_index);
185		break;
186	case MLX5_VHCA_STATE_TEARDOWN_REQUEST:
187		if (sf_dev)
188			mlx5_sf_dev_del(dev, sf_dev, sf_index);
189		else
190			mlx5_core_err(dev,
191				      "SF DEV: teardown state for invalid dev index=%d sfnum=0x%x\n",
192				      sf_index, event->sw_function_id);
193		break;
194	case MLX5_VHCA_STATE_ACTIVE:
195		if (!sf_dev)
196			mlx5_sf_dev_add(dev, sf_index, event->function_id,
197					event->sw_function_id);
198		break;
199	default:
200		break;
201	}
202	return 0;
203}
204
205static int mlx5_sf_dev_vhca_arm_all(struct mlx5_sf_dev_table *table)
206{
207	struct mlx5_core_dev *dev = table->dev;
208	u16 max_functions;
209	u16 function_id;
210	int err = 0;
211	int i;
212
213	max_functions = mlx5_sf_max_functions(dev);
214	function_id = mlx5_sf_start_function_id(dev);
215	/* Arm the vhca context as the vhca event notifier */
216	for (i = 0; i < max_functions; i++) {
217		err = mlx5_vhca_event_arm(dev, function_id);
218		if (err)
219			return err;
220
221		function_id++;
222	}
223	return 0;
224}
225
226static void mlx5_sf_dev_add_active_work(struct work_struct *_work)
227{
228	struct mlx5_sf_dev_active_work_ctx *work_ctx;
229
230	work_ctx = container_of(_work, struct mlx5_sf_dev_active_work_ctx, work);
231	if (work_ctx->table->stop_active_wq)
232		goto out;
233	/* Don't probe device which is already probe */
234	if (!xa_load(&work_ctx->table->devices, work_ctx->sf_index))
235		mlx5_sf_dev_add(work_ctx->table->dev, work_ctx->sf_index,
236				work_ctx->event.function_id, work_ctx->event.sw_function_id);
237	/* There is a race where SF got inactive after the query
238	 * above. e.g.: the query returns that the state of the
239	 * SF is active, and after that the eswitch manager set it to
240	 * inactive.
241	 * This case cannot be managed in SW, since the probing of the
242	 * SF is on one system, and the inactivation is on a different
243	 * system.
244	 * If the inactive is done after the SF perform init_hca(),
245	 * the SF will fully probe and then removed. If it was
246	 * done before init_hca(), the SF probe will fail.
247	 */
248out:
249	kfree(work_ctx);
250}
251
252/* In case SFs are generated externally, probe active SFs */
253static void mlx5_sf_dev_queue_active_works(struct work_struct *_work)
254{
255	struct mlx5_sf_dev_table *table = container_of(_work, struct mlx5_sf_dev_table, work);
256	u32 out[MLX5_ST_SZ_DW(query_vhca_state_out)] = {};
257	struct mlx5_sf_dev_active_work_ctx *work_ctx;
258	struct mlx5_core_dev *dev = table->dev;
259	u16 max_functions;
260	u16 function_id;
261	u16 sw_func_id;
262	int err = 0;
263	int wq_idx;
264	u8 state;
265	int i;
266
267	max_functions = mlx5_sf_max_functions(dev);
268	function_id = mlx5_sf_start_function_id(dev);
269	for (i = 0; i < max_functions; i++, function_id++) {
270		if (table->stop_active_wq)
271			return;
272		err = mlx5_cmd_query_vhca_state(dev, function_id, out, sizeof(out));
273		if (err)
274			/* A failure of specific vhca doesn't mean others will
275			 * fail as well.
276			 */
277			continue;
278		state = MLX5_GET(query_vhca_state_out, out, vhca_state_context.vhca_state);
279		if (state != MLX5_VHCA_STATE_ACTIVE)
280			continue;
281
282		sw_func_id = MLX5_GET(query_vhca_state_out, out, vhca_state_context.sw_function_id);
283		work_ctx = kzalloc(sizeof(*work_ctx), GFP_KERNEL);
284		if (!work_ctx)
285			return;
286
287		INIT_WORK(&work_ctx->work, &mlx5_sf_dev_add_active_work);
288		work_ctx->event.function_id = function_id;
289		work_ctx->event.sw_function_id = sw_func_id;
290		work_ctx->table = table;
291		work_ctx->sf_index = i;
292		wq_idx = work_ctx->event.function_id % MLX5_DEV_MAX_WQS;
293		mlx5_vhca_events_work_enqueue(dev, wq_idx, &work_ctx->work);
294	}
295}
296
297/* In case SFs are generated externally, probe active SFs */
298static int mlx5_sf_dev_create_active_works(struct mlx5_sf_dev_table *table)
299{
300	if (MLX5_CAP_GEN(table->dev, eswitch_manager))
301		return 0; /* the table is local */
302
303	/* Use a workqueue to probe active SFs, which are in large
304	 * quantity and may take up to minutes to probe.
305	 */
306	table->active_wq = create_singlethread_workqueue("mlx5_active_sf");
307	if (!table->active_wq)
308		return -ENOMEM;
309	INIT_WORK(&table->work, &mlx5_sf_dev_queue_active_works);
310	queue_work(table->active_wq, &table->work);
311	return 0;
312}
313
314static void mlx5_sf_dev_destroy_active_works(struct mlx5_sf_dev_table *table)
315{
316	if (table->active_wq) {
317		table->stop_active_wq = true;
318		destroy_workqueue(table->active_wq);
319	}
320}
321
322int mlx5_sf_dev_notifier_init(struct mlx5_core_dev *dev)
323{
324	if (mlx5_core_is_sf(dev))
325		return 0;
326
327	dev->priv.sf_dev_nb.notifier_call = mlx5_sf_dev_state_change_handler;
328	return mlx5_vhca_event_notifier_register(dev, &dev->priv.sf_dev_nb);
329}
330
331void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev)
332{
333	struct mlx5_sf_dev_table *table;
334	int err;
335
336	if (!mlx5_sf_dev_supported(dev))
337		return;
338
339	table = kzalloc(sizeof(*table), GFP_KERNEL);
340	if (!table) {
341		err = -ENOMEM;
342		goto table_err;
343	}
344
345	table->dev = dev;
346	table->sf_bar_length = 1 << (MLX5_CAP_GEN(dev, log_min_sf_size) + 12);
347	table->base_address = pci_resource_start(dev->pdev, 2);
348	xa_init(&table->devices);
349	dev->priv.sf_dev_table = table;
350
351	err = mlx5_sf_dev_create_active_works(table);
352	if (err)
353		goto add_active_err;
354
355	err = mlx5_sf_dev_vhca_arm_all(table);
356	if (err)
357		goto arm_err;
358	return;
359
360arm_err:
361	mlx5_sf_dev_destroy_active_works(table);
362	mlx5_vhca_event_work_queues_flush(dev);
363add_active_err:
364	kfree(table);
365	dev->priv.sf_dev_table = NULL;
366table_err:
367	mlx5_core_err(dev, "SF DEV table create err = %d\n", err);
368}
369
370static void mlx5_sf_dev_destroy_all(struct mlx5_sf_dev_table *table)
371{
372	struct mlx5_sf_dev *sf_dev;
373	unsigned long index;
374
375	xa_for_each(&table->devices, index, sf_dev) {
376		xa_erase(&table->devices, index);
377		mlx5_sf_dev_remove_aux(table->dev, sf_dev);
378	}
379}
380
381void mlx5_sf_dev_notifier_cleanup(struct mlx5_core_dev *dev)
382{
383	if (mlx5_core_is_sf(dev))
384		return;
385
386	mlx5_vhca_event_notifier_unregister(dev, &dev->priv.sf_dev_nb);
387}
388
389void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev)
390{
391	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
392
393	if (!table)
394		return;
395
396	mlx5_sf_dev_destroy_active_works(table);
397
398	/* Now that event handler is not running, it is safe to destroy
399	 * the sf device without race.
400	 */
401	mlx5_sf_dev_destroy_all(table);
402
403	WARN_ON(!xa_empty(&table->devices));
404	kfree(table);
405	dev->priv.sf_dev_table = NULL;
406}