eq.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/*
   3 * Copyright (c) 2013-2021, Mellanox Technologies inc.  All rights reserved.
   4 */
   5
   6#include <linux/interrupt.h>
   7#include <linux/notifier.h>
   8#include <linux/mlx5/driver.h>
   9#include <linux/mlx5/vport.h>
  10#include <linux/mlx5/eq.h>
  11#ifdef CONFIG_RFS_ACCEL
  12#include <linux/cpu_rmap.h>
  13#endif
  14#include "mlx5_core.h"
  15#include "lib/eq.h"
  16#include "fpga/core.h"
  17#include "eswitch.h"
  18#include "lib/clock.h"
  19#include "diag/fw_tracer.h"
  20#include "mlx5_irq.h"
  21#include "pci_irq.h"
  22#include "devlink.h"
  23#include "en_accel/ipsec.h"
  24
  25enum {
  26	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
  27};
  28
  29enum {
  30	MLX5_EQ_STATE_ARMED		= 0x9,
  31	MLX5_EQ_STATE_FIRED		= 0xa,
  32	MLX5_EQ_STATE_ALWAYS_ARMED	= 0xb,
  33};
  34
  35#define MLX5_EQ_DOORBELL_OFFSET 0x40
  36
  37/* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update
  38 * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is
  39 * used to set the EQ size, budget must be smaller than the EQ size.
  40 */
  41enum {
  42	MLX5_EQ_POLLING_BUDGET	= 128,
  43};
  44
  45static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE);
  46
  47struct mlx5_eq_table {
  48	struct xarray           comp_eqs;
  49	struct mlx5_eq_async    pages_eq;
  50	struct mlx5_eq_async    cmd_eq;
  51	struct mlx5_eq_async    async_eq;
  52
  53	struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX];
  54
  55	/* Since CQ DB is stored in async_eq */
  56	struct mlx5_nb          cq_err_nb;
  57
  58	struct mutex            lock; /* sync async eqs creations */
  59	struct mutex            comp_lock; /* sync comp eqs creations */
  60	int			curr_comp_eqs;
  61	int			max_comp_eqs;
  62	struct mlx5_irq_table	*irq_table;
  63	struct xarray           comp_irqs;
  64	struct mlx5_irq         *ctrl_irq;
  65	struct cpu_rmap		*rmap;
  66	struct cpumask          used_cpus;
  67};
  68
  69#define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG)	    | \
  70			       (1ull << MLX5_EVENT_TYPE_COMM_EST)	    | \
  71			       (1ull << MLX5_EVENT_TYPE_SQ_DRAINED)	    | \
  72			       (1ull << MLX5_EVENT_TYPE_CQ_ERROR)	    | \
  73			       (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR)	    | \
  74			       (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED)    | \
  75			       (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
  76			       (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
  77			       (1ull << MLX5_EVENT_TYPE_PORT_CHANGE)	    | \
  78			       (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
  79			       (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE)	    | \
  80			       (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
  81
  82static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
  83{
  84	u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {};
  85
  86	MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
  87	MLX5_SET(destroy_eq_in, in, eq_number, eqn);
  88	return mlx5_cmd_exec_in(dev, destroy_eq, in);
  89}
  90
  91/* caller must eventually call mlx5_cq_put on the returned cq */
  92static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn)
  93{
  94	struct mlx5_cq_table *table = &eq->cq_table;
  95	struct mlx5_core_cq *cq = NULL;
  96
  97	rcu_read_lock();
  98	cq = radix_tree_lookup(&table->tree, cqn);
  99	if (likely(cq))
 100		mlx5_cq_hold(cq);
 101	rcu_read_unlock();
 102
 103	return cq;
 104}
 105
 106static int mlx5_eq_comp_int(struct notifier_block *nb,
 107			    __always_unused unsigned long action,
 108			    __always_unused void *data)
 109{
 110	struct mlx5_eq_comp *eq_comp =
 111		container_of(nb, struct mlx5_eq_comp, irq_nb);
 112	struct mlx5_eq *eq = &eq_comp->core;
 113	struct mlx5_eqe *eqe;
 114	int num_eqes = 0;
 115
 116	while ((eqe = next_eqe_sw(eq))) {
 117		struct mlx5_core_cq *cq;
 118		u32 cqn;
 119
 120		/* Make sure we read EQ entry contents after we've
 121		 * checked the ownership bit.
 122		 */
 123		dma_rmb();
 124		/* Assume (eqe->type) is always MLX5_EVENT_TYPE_COMP */
 125		cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
 126
 127		cq = mlx5_eq_cq_get(eq, cqn);
 128		if (likely(cq)) {
 129			++cq->arm_sn;
 130			cq->comp(cq, eqe);
 131			mlx5_cq_put(cq);
 132		} else {
 133			dev_dbg_ratelimited(eq->dev->device,
 134					    "Completion event for bogus CQ 0x%x\n", cqn);
 135		}
 136
 137		++eq->cons_index;
 138
 139		if (++num_eqes >= MLX5_EQ_POLLING_BUDGET)
 140			break;
 141	}
 142
 143	eq_update_ci(eq, 1);
 144
 145	return 0;
 146}
 147
 148/* Some architectures don't latch interrupts when they are disabled, so using
 149 * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to
 150 * avoid losing them.  It is not recommended to use it, unless this is the last
 151 * resort.
 152 */
 153u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq)
 154{
 155	u32 count_eqe;
 156
 157	disable_irq(eq->core.irqn);
 158	count_eqe = eq->core.cons_index;
 159	mlx5_eq_comp_int(&eq->irq_nb, 0, NULL);
 160	count_eqe = eq->core.cons_index - count_eqe;
 161	enable_irq(eq->core.irqn);
 162
 163	return count_eqe;
 164}
 165
 166static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, bool recovery,
 167				   unsigned long *flags)
 168	__acquires(&eq->lock)
 169{
 170	if (!recovery)
 171		spin_lock(&eq->lock);
 172	else
 173		spin_lock_irqsave(&eq->lock, *flags);
 174}
 175
 176static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, bool recovery,
 177				     unsigned long *flags)
 178	__releases(&eq->lock)
 179{
 180	if (!recovery)
 181		spin_unlock(&eq->lock);
 182	else
 183		spin_unlock_irqrestore(&eq->lock, *flags);
 184}
 185
 186enum async_eq_nb_action {
 187	ASYNC_EQ_IRQ_HANDLER = 0,
 188	ASYNC_EQ_RECOVER = 1,
 189};
 190
 191static int mlx5_eq_async_int(struct notifier_block *nb,
 192			     unsigned long action, void *data)
 193{
 194	struct mlx5_eq_async *eq_async =
 195		container_of(nb, struct mlx5_eq_async, irq_nb);
 196	struct mlx5_eq *eq = &eq_async->core;
 197	struct mlx5_eq_table *eqt;
 198	struct mlx5_core_dev *dev;
 199	struct mlx5_eqe *eqe;
 200	unsigned long flags;
 201	int num_eqes = 0;
 202	bool recovery;
 203
 204	dev = eq->dev;
 205	eqt = dev->priv.eq_table;
 206
 207	recovery = action == ASYNC_EQ_RECOVER;
 208	mlx5_eq_async_int_lock(eq_async, recovery, &flags);
 209
 210	while ((eqe = next_eqe_sw(eq))) {
 211		/*
 212		 * Make sure we read EQ entry contents after we've
 213		 * checked the ownership bit.
 214		 */
 215		dma_rmb();
 216
 217		atomic_notifier_call_chain(&eqt->nh[eqe->type], eqe->type, eqe);
 218		atomic_notifier_call_chain(&eqt->nh[MLX5_EVENT_TYPE_NOTIFY_ANY], eqe->type, eqe);
 219
 220		++eq->cons_index;
 221
 222		if (++num_eqes >= MLX5_EQ_POLLING_BUDGET)
 223			break;
 224	}
 225
 226	eq_update_ci(eq, 1);
 227	mlx5_eq_async_int_unlock(eq_async, recovery, &flags);
 228
 229	return unlikely(recovery) ? num_eqes : 0;
 230}
 231
 232void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev)
 233{
 234	struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq;
 235	int eqes;
 236
 237	eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL);
 238	if (eqes)
 239		mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes);
 240}
 241
 242static void init_eq_buf(struct mlx5_eq *eq)
 243{
 244	struct mlx5_eqe *eqe;
 245	int i;
 246
 247	for (i = 0; i < eq_get_size(eq); i++) {
 248		eqe = get_eqe(eq, i);
 249		eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
 250	}
 251}
 252
 253static int
 254create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
 255	      struct mlx5_eq_param *param)
 256{
 257	u8 log_eq_size = order_base_2(param->nent + MLX5_NUM_SPARE_EQE);
 258	struct mlx5_cq_table *cq_table = &eq->cq_table;
 259	u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
 260	u8 log_eq_stride = ilog2(MLX5_EQE_SIZE);
 261	struct mlx5_priv *priv = &dev->priv;
 262	__be64 *pas;
 263	u16 vecidx;
 264	void *eqc;
 265	int inlen;
 266	u32 *in;
 267	int err;
 268	int i;
 269
 270	/* Init CQ table */
 271	memset(cq_table, 0, sizeof(*cq_table));
 272	spin_lock_init(&cq_table->lock);
 273	INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
 274
 275	eq->cons_index = 0;
 276
 277	err = mlx5_frag_buf_alloc_node(dev, wq_get_byte_sz(log_eq_size, log_eq_stride),
 278				       &eq->frag_buf, dev->priv.numa_node);
 279	if (err)
 280		return err;
 281
 282	mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc);
 283	init_eq_buf(eq);
 284
 285	eq->irq = param->irq;
 286	vecidx = mlx5_irq_get_index(eq->irq);
 287
 288	inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
 289		MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages;
 290
 291	in = kvzalloc(inlen, GFP_KERNEL);
 292	if (!in) {
 293		err = -ENOMEM;
 294		goto err_buf;
 295	}
 296
 297	pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
 298	mlx5_fill_page_frag_array(&eq->frag_buf, pas);
 299
 300	MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
 301	if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx))
 302		MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID);
 303
 304	for (i = 0; i < 4; i++)
 305		MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i,
 306				 param->mask[i]);
 307
 308	eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
 309	MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz);
 310	MLX5_SET(eqc, eqc, uar_page, priv->bfreg.up->index);
 311	MLX5_SET(eqc, eqc, intr, vecidx);
 312	MLX5_SET(eqc, eqc, log_page_size,
 313		 eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
 314
 315	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 316	if (err)
 317		goto err_in;
 318
 319	eq->vecidx = vecidx;
 320	eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
 321	eq->irqn = pci_irq_vector(dev->pdev, vecidx);
 322	eq->dev = dev;
 323	eq->doorbell = priv->bfreg.up->map + MLX5_EQ_DOORBELL_OFFSET;
 324
 325	err = mlx5_debug_eq_add(dev, eq);
 326	if (err)
 327		goto err_eq;
 328
 329	kvfree(in);
 330	return 0;
 331
 332err_eq:
 333	mlx5_cmd_destroy_eq(dev, eq->eqn);
 334
 335err_in:
 336	kvfree(in);
 337
 338err_buf:
 339	mlx5_frag_buf_free(dev, &eq->frag_buf);
 340	return err;
 341}
 342
 343/**
 344 * mlx5_eq_enable - Enable EQ for receiving EQEs
 345 * @dev : Device which owns the eq
 346 * @eq  : EQ to enable
 347 * @nb  : Notifier call block
 348 *
 349 * Must be called after EQ is created in device.
 350 *
 351 * @return: 0 if no error
 352 */
 353int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
 354		   struct notifier_block *nb)
 355{
 356	int err;
 357
 358	err = mlx5_irq_attach_nb(eq->irq, nb);
 359	if (!err)
 360		eq_update_ci(eq, 1);
 361
 362	return err;
 363}
 364EXPORT_SYMBOL(mlx5_eq_enable);
 365
 366/**
 367 * mlx5_eq_disable - Disable EQ for receiving EQEs
 368 * @dev : Device which owns the eq
 369 * @eq  : EQ to disable
 370 * @nb  : Notifier call block
 371 *
 372 * Must be called before EQ is destroyed.
 373 */
 374void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
 375		     struct notifier_block *nb)
 376{
 377	mlx5_irq_detach_nb(eq->irq, nb);
 378}
 379EXPORT_SYMBOL(mlx5_eq_disable);
 380
 381static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
 382{
 383	int err;
 384
 385	mlx5_debug_eq_remove(dev, eq);
 386
 387	err = mlx5_cmd_destroy_eq(dev, eq->eqn);
 388	if (err)
 389		mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
 390			       eq->eqn);
 391
 392	mlx5_frag_buf_free(dev, &eq->frag_buf);
 393	return err;
 394}
 395
 396int mlx5_eq_add_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
 397{
 398	struct mlx5_cq_table *table = &eq->cq_table;
 399	int err;
 400
 401	spin_lock(&table->lock);
 402	err = radix_tree_insert(&table->tree, cq->cqn, cq);
 403	spin_unlock(&table->lock);
 404
 405	return err;
 406}
 407
 408void mlx5_eq_del_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
 409{
 410	struct mlx5_cq_table *table = &eq->cq_table;
 411	struct mlx5_core_cq *tmp;
 412
 413	spin_lock(&table->lock);
 414	tmp = radix_tree_delete(&table->tree, cq->cqn);
 415	spin_unlock(&table->lock);
 416
 417	if (!tmp) {
 418		mlx5_core_dbg(eq->dev, "cq 0x%x not found in eq 0x%x tree\n",
 419			      eq->eqn, cq->cqn);
 420		return;
 421	}
 422
 423	if (tmp != cq)
 424		mlx5_core_dbg(eq->dev, "corruption on cqn 0x%x in eq 0x%x\n",
 425			      eq->eqn, cq->cqn);
 426}
 427
 428int mlx5_eq_table_init(struct mlx5_core_dev *dev)
 429{
 430	struct mlx5_eq_table *eq_table;
 431	int i;
 432
 433	eq_table = kvzalloc_node(sizeof(*eq_table), GFP_KERNEL,
 434				 dev->priv.numa_node);
 435	if (!eq_table)
 436		return -ENOMEM;
 437
 438	dev->priv.eq_table = eq_table;
 439
 440	mlx5_eq_debugfs_init(dev);
 441
 442	mutex_init(&eq_table->lock);
 443	for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++)
 444		ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]);
 445
 446	eq_table->irq_table = mlx5_irq_table_get(dev);
 447	cpumask_clear(&eq_table->used_cpus);
 448	xa_init(&eq_table->comp_eqs);
 449	xa_init(&eq_table->comp_irqs);
 450	mutex_init(&eq_table->comp_lock);
 451	eq_table->curr_comp_eqs = 0;
 452	return 0;
 453}
 454
 455void mlx5_eq_table_cleanup(struct mlx5_core_dev *dev)
 456{
 457	struct mlx5_eq_table *table = dev->priv.eq_table;
 458
 459	mlx5_eq_debugfs_cleanup(dev);
 460	xa_destroy(&table->comp_irqs);
 461	xa_destroy(&table->comp_eqs);
 462	kvfree(table);
 463}
 464
 465/* Async EQs */
 466
 467static int create_async_eq(struct mlx5_core_dev *dev,
 468			   struct mlx5_eq *eq, struct mlx5_eq_param *param)
 469{
 470	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
 471	int err;
 472
 473	mutex_lock(&eq_table->lock);
 474	err = create_map_eq(dev, eq, param);
 475	mutex_unlock(&eq_table->lock);
 476	return err;
 477}
 478
 479static int destroy_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
 480{
 481	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
 482	int err;
 483
 484	mutex_lock(&eq_table->lock);
 485	err = destroy_unmap_eq(dev, eq);
 486	mutex_unlock(&eq_table->lock);
 487	return err;
 488}
 489
 490static int cq_err_event_notifier(struct notifier_block *nb,
 491				 unsigned long type, void *data)
 492{
 493	struct mlx5_eq_table *eqt;
 494	struct mlx5_core_cq *cq;
 495	struct mlx5_eqe *eqe;
 496	struct mlx5_eq *eq;
 497	u32 cqn;
 498
 499	/* type == MLX5_EVENT_TYPE_CQ_ERROR */
 500
 501	eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb);
 502	eq  = &eqt->async_eq.core;
 503	eqe = data;
 504
 505	cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
 506	mlx5_core_warn(eq->dev, "CQ error on CQN 0x%x, syndrome 0x%x\n",
 507		       cqn, eqe->data.cq_err.syndrome);
 508
 509	cq = mlx5_eq_cq_get(eq, cqn);
 510	if (unlikely(!cq)) {
 511		mlx5_core_warn(eq->dev, "Async event for bogus CQ 0x%x\n", cqn);
 512		return NOTIFY_OK;
 513	}
 514
 515	if (cq->event)
 516		cq->event(cq, type);
 517
 518	mlx5_cq_put(cq);
 519
 520	return NOTIFY_OK;
 521}
 522
 523static void gather_user_async_events(struct mlx5_core_dev *dev, u64 mask[4])
 524{
 525	__be64 *user_unaffiliated_events;
 526	__be64 *user_affiliated_events;
 527	int i;
 528
 529	user_affiliated_events =
 530		MLX5_CAP_DEV_EVENT(dev, user_affiliated_events);
 531	user_unaffiliated_events =
 532		MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events);
 533
 534	for (i = 0; i < 4; i++)
 535		mask[i] |= be64_to_cpu(user_affiliated_events[i] |
 536				       user_unaffiliated_events[i]);
 537}
 538
 539static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4])
 540{
 541	u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
 542
 543	if (MLX5_VPORT_MANAGER(dev))
 544		async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
 545
 546	if (MLX5_CAP_GEN(dev, general_notification_event))
 547		async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT);
 548
 549	if (MLX5_CAP_GEN(dev, port_module_event))
 550		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
 551	else
 552		mlx5_core_dbg(dev, "port_module_event is not set\n");
 553
 554	if (MLX5_PPS_CAP(dev))
 555		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PPS_EVENT);
 556
 557	if (MLX5_CAP_GEN(dev, fpga))
 558		async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR) |
 559				    (1ull << MLX5_EVENT_TYPE_FPGA_QP_ERROR);
 560	if (MLX5_CAP_GEN_MAX(dev, dct))
 561		async_event_mask |= (1ull << MLX5_EVENT_TYPE_DCT_DRAINED);
 562
 563	if (MLX5_CAP_GEN(dev, temp_warn_event))
 564		async_event_mask |= (1ull << MLX5_EVENT_TYPE_TEMP_WARN_EVENT);
 565
 566	if (MLX5_CAP_MCAM_REG(dev, tracer_registers))
 567		async_event_mask |= (1ull << MLX5_EVENT_TYPE_DEVICE_TRACER);
 568
 569	if (MLX5_CAP_GEN(dev, max_num_of_monitor_counters))
 570		async_event_mask |= (1ull << MLX5_EVENT_TYPE_MONITOR_COUNTER);
 571
 572	if (mlx5_eswitch_is_funcs_handler(dev))
 573		async_event_mask |=
 574			(1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);
 575
 576	if (MLX5_CAP_GEN_MAX(dev, vhca_state))
 577		async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE);
 578
 579	if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload))
 580		async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
 581
 582	if (mlx5_ipsec_device_caps(dev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
 583		async_event_mask |=
 584			(1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
 585
 586	if (mlx5_pcie_cong_event_supported(dev))
 587		async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
 588
 589	mask[0] = async_event_mask;
 590
 591	if (MLX5_CAP_GEN(dev, event_cap))
 592		gather_user_async_events(dev, mask);
 593}
 594
 595static int
 596setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq,
 597	       struct mlx5_eq_param *param, const char *name)
 598{
 599	int err;
 600
 601	eq->irq_nb.notifier_call = mlx5_eq_async_int;
 602	spin_lock_init(&eq->lock);
 603
 604	err = create_async_eq(dev, &eq->core, param);
 605	if (err) {
 606		mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err);
 607		return err;
 608	}
 609	err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
 610	if (err) {
 611		mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err);
 612		destroy_async_eq(dev, &eq->core);
 613	}
 614	return err;
 615}
 616
 617static void cleanup_async_eq(struct mlx5_core_dev *dev,
 618			     struct mlx5_eq_async *eq, const char *name)
 619{
 620	int err;
 621
 622	mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
 623	err = destroy_async_eq(dev, &eq->core);
 624	if (err)
 625		mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n",
 626			      name, err);
 627}
 628
 629static u16 async_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
 630{
 631	struct devlink *devlink = priv_to_devlink(dev);
 632	union devlink_param_value val;
 633	int err;
 634
 635	err = devl_param_driverinit_value_get(devlink,
 636					      DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
 637					      &val);
 638	if (!err)
 639		return val.vu32;
 640	mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
 641	return MLX5_NUM_ASYNC_EQE;
 642}
 643
 644static int create_async_eqs(struct mlx5_core_dev *dev)
 645{
 646	struct mlx5_eq_table *table = dev->priv.eq_table;
 647	struct mlx5_eq_param param = {};
 648	int err;
 649
 650	/* All the async_eqs are using single IRQ, request one IRQ and share its
 651	 * index among all the async_eqs of this device.
 652	 */
 653	table->ctrl_irq = mlx5_ctrl_irq_request(dev);
 654	if (IS_ERR(table->ctrl_irq))
 655		return PTR_ERR(table->ctrl_irq);
 656
 657	MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR);
 658	mlx5_eq_notifier_register(dev, &table->cq_err_nb);
 659
 660	param = (struct mlx5_eq_param) {
 661		.irq = table->ctrl_irq,
 662		.nent = MLX5_NUM_CMD_EQE,
 663		.mask[0] = 1ull << MLX5_EVENT_TYPE_CMD,
 664	};
 665	mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_CREATE_EQ);
 666	err = setup_async_eq(dev, &table->cmd_eq, &param, "cmd");
 667	if (err)
 668		goto err1;
 669
 670	mlx5_cmd_use_events(dev);
 671	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
 672
 673	param = (struct mlx5_eq_param) {
 674		.irq = table->ctrl_irq,
 675		.nent = async_eq_depth_devlink_param_get(dev),
 676	};
 677
 678	gather_async_events_mask(dev, param.mask);
 679	err = setup_async_eq(dev, &table->async_eq, &param, "async");
 680	if (err)
 681		goto err2;
 682
 683	/* Skip page eq creation when the device does not request for page requests */
 684	if (MLX5_CAP_GEN(dev, page_request_disable)) {
 685		mlx5_core_dbg(dev, "Skip page EQ creation\n");
 686		return 0;
 687	}
 688
 689	param = (struct mlx5_eq_param) {
 690		.irq = table->ctrl_irq,
 691		.nent = /* TODO: sriov max_vf + */ 1,
 692		.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST,
 693	};
 694
 695	err = setup_async_eq(dev, &table->pages_eq, &param, "pages");
 696	if (err)
 697		goto err3;
 698
 699	return 0;
 700
 701err3:
 702	cleanup_async_eq(dev, &table->async_eq, "async");
 703err2:
 704	mlx5_cmd_use_polling(dev);
 705	cleanup_async_eq(dev, &table->cmd_eq, "cmd");
 706err1:
 707	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
 708	mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
 709	mlx5_ctrl_irq_release(dev, table->ctrl_irq);
 710	return err;
 711}
 712
 713static void destroy_async_eqs(struct mlx5_core_dev *dev)
 714{
 715	struct mlx5_eq_table *table = dev->priv.eq_table;
 716
 717	if (!MLX5_CAP_GEN(dev, page_request_disable))
 718		cleanup_async_eq(dev, &table->pages_eq, "pages");
 719	cleanup_async_eq(dev, &table->async_eq, "async");
 720	mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_DESTROY_EQ);
 721	mlx5_cmd_use_polling(dev);
 722	cleanup_async_eq(dev, &table->cmd_eq, "cmd");
 723	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
 724	mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
 725	mlx5_ctrl_irq_release(dev, table->ctrl_irq);
 726}
 727
 728struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev)
 729{
 730	return &dev->priv.eq_table->async_eq.core;
 731}
 732
 733void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev)
 734{
 735	synchronize_irq(dev->priv.eq_table->async_eq.core.irqn);
 736}
 737
 738void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev)
 739{
 740	synchronize_irq(dev->priv.eq_table->cmd_eq.core.irqn);
 741}
 742
 743/* Generic EQ API for mlx5_core consumers
 744 * Needed For RDMA ODP EQ for now
 745 */
 746struct mlx5_eq *
 747mlx5_eq_create_generic(struct mlx5_core_dev *dev,
 748		       struct mlx5_eq_param *param)
 749{
 750	struct mlx5_eq *eq = kvzalloc_node(sizeof(*eq), GFP_KERNEL,
 751					   dev->priv.numa_node);
 752	int err;
 753
 754	if (!eq)
 755		return ERR_PTR(-ENOMEM);
 756
 757	param->irq = dev->priv.eq_table->ctrl_irq;
 758	err = create_async_eq(dev, eq, param);
 759	if (err) {
 760		kvfree(eq);
 761		eq = ERR_PTR(err);
 762	}
 763
 764	return eq;
 765}
 766EXPORT_SYMBOL(mlx5_eq_create_generic);
 767
 768int mlx5_eq_destroy_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
 769{
 770	int err;
 771
 772	if (IS_ERR(eq))
 773		return -EINVAL;
 774
 775	err = destroy_async_eq(dev, eq);
 776	if (err)
 777		goto out;
 778
 779	kvfree(eq);
 780out:
 781	return err;
 782}
 783EXPORT_SYMBOL(mlx5_eq_destroy_generic);
 784
 785struct mlx5_eqe *mlx5_eq_get_eqe(struct mlx5_eq *eq, u32 cc)
 786{
 787	u32 ci = eq->cons_index + cc;
 788	u32 nent = eq_get_size(eq);
 789	struct mlx5_eqe *eqe;
 790
 791	eqe = get_eqe(eq, ci & (nent - 1));
 792	eqe = ((eqe->owner & 1) ^ !!(ci & nent)) ? NULL : eqe;
 793	/* Make sure we read EQ entry contents after we've
 794	 * checked the ownership bit.
 795	 */
 796	if (eqe)
 797		dma_rmb();
 798
 799	return eqe;
 800}
 801EXPORT_SYMBOL(mlx5_eq_get_eqe);
 802
 803void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm)
 804{
 805	eq->cons_index += cc;
 806	eq_update_ci(eq, arm);
 807}
 808EXPORT_SYMBOL(mlx5_eq_update_ci);
 809
 810static void comp_irq_release_pci(struct mlx5_core_dev *dev, u16 vecidx)
 811{
 812	struct mlx5_eq_table *table = dev->priv.eq_table;
 813	struct mlx5_irq *irq;
 814
 815	irq = xa_load(&table->comp_irqs, vecidx);
 816	if (!irq)
 817		return;
 818
 819	xa_erase(&table->comp_irqs, vecidx);
 820	mlx5_irq_release_vector(irq);
 821}
 822
 823static int mlx5_cpumask_default_spread(struct mlx5_core_dev *dev, int index)
 824{
 825	return cpumask_local_spread(index, dev->priv.numa_node);
 826}
 827
 828static struct cpu_rmap *mlx5_eq_table_get_pci_rmap(struct mlx5_core_dev *dev)
 829{
 830#ifdef CONFIG_RFS_ACCEL
 831#ifdef CONFIG_MLX5_SF
 832	if (mlx5_core_is_sf(dev))
 833		return dev->priv.parent_mdev->priv.eq_table->rmap;
 834#endif
 835	return dev->priv.eq_table->rmap;
 836#else
 837	return NULL;
 838#endif
 839}
 840
 841static int comp_irq_request_pci(struct mlx5_core_dev *dev, u16 vecidx)
 842{
 843	struct mlx5_eq_table *table = dev->priv.eq_table;
 844	struct cpu_rmap *rmap;
 845	struct mlx5_irq *irq;
 846	int cpu;
 847
 848	rmap = mlx5_eq_table_get_pci_rmap(dev);
 849	cpu = mlx5_cpumask_default_spread(dev, vecidx);
 850	irq = mlx5_irq_request_vector(dev, cpu, vecidx, &rmap);
 851	if (IS_ERR(irq))
 852		return PTR_ERR(irq);
 853
 854	return xa_err(xa_store(&table->comp_irqs, vecidx, irq, GFP_KERNEL));
 855}
 856
 857static void comp_irq_release_sf(struct mlx5_core_dev *dev, u16 vecidx)
 858{
 859	struct mlx5_eq_table *table = dev->priv.eq_table;
 860	struct mlx5_irq *irq;
 861	int cpu;
 862
 863	irq = xa_load(&table->comp_irqs, vecidx);
 864	if (!irq)
 865		return;
 866
 867	cpu = cpumask_first(mlx5_irq_get_affinity_mask(irq));
 868	cpumask_clear_cpu(cpu, &table->used_cpus);
 869	xa_erase(&table->comp_irqs, vecidx);
 870	mlx5_irq_affinity_irq_release(dev, irq);
 871}
 872
 873static int comp_irq_request_sf(struct mlx5_core_dev *dev, u16 vecidx)
 874{
 875	struct mlx5_irq_pool *pool = mlx5_irq_table_get_comp_irq_pool(dev);
 876	struct mlx5_eq_table *table = dev->priv.eq_table;
 877	struct irq_affinity_desc *af_desc;
 878	struct mlx5_irq *irq;
 879
 880	/* In case SF irq pool does not exist, fallback to the PF irqs */
 881	if (!mlx5_irq_pool_is_sf_pool(pool))
 882		return comp_irq_request_pci(dev, vecidx);
 883
 884	af_desc = kvzalloc(sizeof(*af_desc), GFP_KERNEL);
 885	if (!af_desc)
 886		return -ENOMEM;
 887
 888	af_desc->is_managed = false;
 889	cpumask_copy(&af_desc->mask, cpu_online_mask);
 890	cpumask_andnot(&af_desc->mask, &af_desc->mask, &table->used_cpus);
 891	irq = mlx5_irq_affinity_request(dev, pool, af_desc);
 892	if (IS_ERR(irq)) {
 893		kvfree(af_desc);
 894		return PTR_ERR(irq);
 895	}
 896
 897	cpumask_or(&table->used_cpus, &table->used_cpus, mlx5_irq_get_affinity_mask(irq));
 898	mlx5_core_dbg(pool->dev, "IRQ %u mapped to cpu %*pbl, %u EQs on this irq\n",
 899		      pci_irq_vector(dev->pdev, mlx5_irq_get_index(irq)),
 900		      cpumask_pr_args(mlx5_irq_get_affinity_mask(irq)),
 901		      mlx5_irq_read_locked(irq) / MLX5_EQ_REFS_PER_IRQ);
 902
 903	kvfree(af_desc);
 904
 905	return xa_err(xa_store(&table->comp_irqs, vecidx, irq, GFP_KERNEL));
 906}
 907
 908static void comp_irq_release(struct mlx5_core_dev *dev, u16 vecidx)
 909{
 910	mlx5_core_is_sf(dev) ? comp_irq_release_sf(dev, vecidx) :
 911			       comp_irq_release_pci(dev, vecidx);
 912}
 913
 914static int comp_irq_request(struct mlx5_core_dev *dev, u16 vecidx)
 915{
 916	return mlx5_core_is_sf(dev) ? comp_irq_request_sf(dev, vecidx) :
 917				      comp_irq_request_pci(dev, vecidx);
 918}
 919
 920#ifdef CONFIG_RFS_ACCEL
 921static int alloc_rmap(struct mlx5_core_dev *mdev)
 922{
 923	struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
 924
 925	/* rmap is a mapping between irq number and queue number.
 926	 * Each irq can be assigned only to a single rmap.
 927	 * Since SFs share IRQs, rmap mapping cannot function correctly
 928	 * for irqs that are shared between different core/netdev RX rings.
 929	 * Hence we don't allow netdev rmap for SFs.
 930	 */
 931	if (mlx5_core_is_sf(mdev))
 932		return 0;
 933
 934	eq_table->rmap = alloc_irq_cpu_rmap(eq_table->max_comp_eqs);
 935	if (!eq_table->rmap)
 936		return -ENOMEM;
 937	return 0;
 938}
 939
 940static void free_rmap(struct mlx5_core_dev *mdev)
 941{
 942	struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
 943
 944	if (eq_table->rmap) {
 945		free_irq_cpu_rmap(eq_table->rmap);
 946		eq_table->rmap = NULL;
 947	}
 948}
 949#else
 950static int alloc_rmap(struct mlx5_core_dev *mdev) { return 0; }
 951static void free_rmap(struct mlx5_core_dev *mdev) {}
 952#endif
 953
 954static void destroy_comp_eq(struct mlx5_core_dev *dev, struct mlx5_eq_comp *eq, u16 vecidx)
 955{
 956	struct mlx5_eq_table *table = dev->priv.eq_table;
 957
 958	xa_erase(&table->comp_eqs, vecidx);
 959	mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
 960	if (destroy_unmap_eq(dev, &eq->core))
 961		mlx5_core_warn(dev, "failed to destroy comp EQ 0x%x\n",
 962			       eq->core.eqn);
 963	tasklet_disable(&eq->tasklet_ctx.task);
 964	kfree(eq);
 965	comp_irq_release(dev, vecidx);
 966	table->curr_comp_eqs--;
 967}
 968
 969static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
 970{
 971	struct devlink *devlink = priv_to_devlink(dev);
 972	union devlink_param_value val;
 973	int err;
 974
 975	err = devl_param_driverinit_value_get(devlink,
 976					      DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
 977					      &val);
 978	if (!err)
 979		return val.vu32;
 980	mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
 981	return MLX5_COMP_EQ_SIZE;
 982}
 983
 984/* Must be called with EQ table comp_lock held */
 985static int create_comp_eq(struct mlx5_core_dev *dev, u16 vecidx)
 986{
 987	struct mlx5_eq_table *table = dev->priv.eq_table;
 988	struct mlx5_eq_param param = {};
 989	struct mlx5_eq_comp *eq;
 990	struct mlx5_irq *irq;
 991	int nent;
 992	int err;
 993
 994	lockdep_assert_held(&table->comp_lock);
 995	if (table->curr_comp_eqs == table->max_comp_eqs) {
 996		mlx5_core_err(dev, "maximum number of vectors is allocated, %d\n",
 997			      table->max_comp_eqs);
 998		return -ENOMEM;
 999	}
1000
1001	err = comp_irq_request(dev, vecidx);
1002	if (err)
1003		return err;
1004
1005	nent = comp_eq_depth_devlink_param_get(dev);
1006
1007	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, dev->priv.numa_node);
1008	if (!eq) {
1009		err = -ENOMEM;
1010		goto clean_irq;
1011	}
1012
1013	INIT_LIST_HEAD(&eq->tasklet_ctx.list);
1014	INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
1015	spin_lock_init(&eq->tasklet_ctx.lock);
1016	tasklet_setup(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb);
1017
1018	irq = xa_load(&table->comp_irqs, vecidx);
1019	eq->irq_nb.notifier_call = mlx5_eq_comp_int;
1020	param = (struct mlx5_eq_param) {
1021		.irq = irq,
1022		.nent = nent,
1023	};
1024
1025	err = create_map_eq(dev, &eq->core, &param);
1026	if (err)
1027		goto clean_eq;
1028	err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
1029	if (err) {
1030		destroy_unmap_eq(dev, &eq->core);
1031		goto clean_eq;
1032	}
1033
1034	mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn);
1035	err = xa_err(xa_store(&table->comp_eqs, vecidx, eq, GFP_KERNEL));
1036	if (err)
1037		goto disable_eq;
1038
1039	table->curr_comp_eqs++;
1040	return eq->core.eqn;
1041
1042disable_eq:
1043	mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
1044clean_eq:
1045	kfree(eq);
1046clean_irq:
1047	comp_irq_release(dev, vecidx);
1048	return err;
1049}
1050
1051int mlx5_comp_eqn_get(struct mlx5_core_dev *dev, u16 vecidx, int *eqn)
1052{
1053	struct mlx5_eq_table *table = dev->priv.eq_table;
1054	struct mlx5_eq_comp *eq;
1055	int ret = 0;
1056
1057	if (vecidx >= table->max_comp_eqs) {
1058		mlx5_core_dbg(dev, "Requested vector index %u should be less than %u",
1059			      vecidx, table->max_comp_eqs);
1060		return -EINVAL;
1061	}
1062
1063	mutex_lock(&table->comp_lock);
1064	eq = xa_load(&table->comp_eqs, vecidx);
1065	if (eq) {
1066		*eqn = eq->core.eqn;
1067		goto out;
1068	}
1069
1070	ret = create_comp_eq(dev, vecidx);
1071	if (ret < 0) {
1072		mutex_unlock(&table->comp_lock);
1073		return ret;
1074	}
1075
1076	*eqn = ret;
1077out:
1078	mutex_unlock(&table->comp_lock);
1079	return 0;
1080}
1081EXPORT_SYMBOL(mlx5_comp_eqn_get);
1082
1083int mlx5_comp_irqn_get(struct mlx5_core_dev *dev, int vector, unsigned int *irqn)
1084{
1085	struct mlx5_eq_table *table = dev->priv.eq_table;
1086	struct mlx5_eq_comp *eq;
1087	int eqn;
1088	int err;
1089
1090	/* Allocate the EQ if not allocated yet */
1091	err = mlx5_comp_eqn_get(dev, vector, &eqn);
1092	if (err)
1093		return err;
1094
1095	eq = xa_load(&table->comp_eqs, vector);
1096	*irqn = eq->core.irqn;
1097	return 0;
1098}
1099
1100unsigned int mlx5_comp_vectors_max(struct mlx5_core_dev *dev)
1101{
1102	return dev->priv.eq_table->max_comp_eqs;
1103}
1104EXPORT_SYMBOL(mlx5_comp_vectors_max);
1105
1106static struct cpumask *
1107mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector)
1108{
1109	struct mlx5_eq_table *table = dev->priv.eq_table;
1110	struct mlx5_eq_comp *eq;
1111
1112	eq = xa_load(&table->comp_eqs, vector);
1113	if (eq)
1114		return mlx5_irq_get_affinity_mask(eq->core.irq);
1115
1116	return NULL;
1117}
1118
1119int mlx5_comp_vector_get_cpu(struct mlx5_core_dev *dev, int vector)
1120{
1121	struct cpumask *mask;
1122	int cpu;
1123
1124	mask = mlx5_comp_irq_get_affinity_mask(dev, vector);
1125	if (mask)
1126		cpu = cpumask_first(mask);
1127	else
1128		cpu = mlx5_cpumask_default_spread(dev, vector);
1129
1130	return cpu;
1131}
1132EXPORT_SYMBOL(mlx5_comp_vector_get_cpu);
1133
1134#ifdef CONFIG_RFS_ACCEL
1135struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev)
1136{
1137	return dev->priv.eq_table->rmap;
1138}
1139#endif
1140
1141struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn)
1142{
1143	struct mlx5_eq_table *table = dev->priv.eq_table;
1144	struct mlx5_eq_comp *eq;
1145	unsigned long index;
1146
1147	xa_for_each(&table->comp_eqs, index, eq)
1148		if (eq->core.eqn == eqn)
1149			return eq;
1150
1151	return ERR_PTR(-ENOENT);
1152}
1153
1154/* This function should only be called after mlx5_cmd_force_teardown_hca */
1155void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev)
1156{
1157	mlx5_irq_table_free_irqs(dev);
1158}
1159
1160#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1161#define MLX5_MAX_ASYNC_EQS 4
1162#else
1163#define MLX5_MAX_ASYNC_EQS 3
1164#endif
1165
1166static int get_num_eqs(struct mlx5_core_dev *dev)
1167{
1168	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
1169	int max_dev_eqs;
1170	int num_eqs;
1171
1172	/* If ethernet is disabled we use just a single completion vector to
1173	 * have the other vectors available for other drivers using mlx5_core. For
1174	 * example, mlx5_vdpa
1175	 */
1176	if (!mlx5_core_is_eth_enabled(dev) && mlx5_eth_supported(dev))
1177		return 1;
1178
1179	max_dev_eqs = mlx5_max_eq_cap_get(dev);
1180
1181	num_eqs = min_t(int, mlx5_irq_table_get_num_comp(eq_table->irq_table),
1182			max_dev_eqs - MLX5_MAX_ASYNC_EQS);
1183	if (mlx5_core_is_sf(dev)) {
1184		int max_eqs_sf = MLX5_CAP_GEN_2(dev, sf_eq_usage) ?
1185				 MLX5_CAP_GEN_2(dev, max_num_eqs_24b) :
1186				 MLX5_COMP_EQS_PER_SF;
1187
1188		max_eqs_sf = min_t(int, max_eqs_sf,
1189				   mlx5_irq_table_get_sfs_vec(eq_table->irq_table));
1190		num_eqs = min_t(int, num_eqs, max_eqs_sf);
1191	}
1192
1193	return num_eqs;
1194}
1195
1196int mlx5_eq_table_create(struct mlx5_core_dev *dev)
1197{
1198	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
1199	int err;
1200
1201	eq_table->max_comp_eqs = get_num_eqs(dev);
1202	err = create_async_eqs(dev);
1203	if (err) {
1204		mlx5_core_err(dev, "Failed to create async EQs\n");
1205		goto err_async_eqs;
1206	}
1207
1208	err = alloc_rmap(dev);
1209	if (err) {
1210		mlx5_core_err(dev, "Failed to allocate rmap\n");
1211		goto err_rmap;
1212	}
1213
1214	return 0;
1215
1216err_rmap:
1217	destroy_async_eqs(dev);
1218err_async_eqs:
1219	return err;
1220}
1221
1222void mlx5_eq_table_destroy(struct mlx5_core_dev *dev)
1223{
1224	struct mlx5_eq_table *table = dev->priv.eq_table;
1225	struct mlx5_eq_comp *eq;
1226	unsigned long index;
1227
1228	xa_for_each(&table->comp_eqs, index, eq)
1229		destroy_comp_eq(dev, eq, index);
1230
1231	free_rmap(dev);
1232	destroy_async_eqs(dev);
1233}
1234
1235int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1236{
1237	struct mlx5_eq_table *eqt = dev->priv.eq_table;
1238
1239	return atomic_notifier_chain_register(&eqt->nh[nb->event_type], &nb->nb);
1240}
1241EXPORT_SYMBOL(mlx5_eq_notifier_register);
1242
1243int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1244{
1245	struct mlx5_eq_table *eqt = dev->priv.eq_table;
1246
1247	return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb);
1248}
1249EXPORT_SYMBOL(mlx5_eq_notifier_unregister);