irq_affinity.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
  3
  4#include "mlx5_core.h"
  5#include "mlx5_irq.h"
  6#include "pci_irq.h"
  7
  8static void cpu_put(struct mlx5_irq_pool *pool, int cpu)
  9{
 10	pool->irqs_per_cpu[cpu]--;
 11}
 12
 13static void cpu_get(struct mlx5_irq_pool *pool, int cpu)
 14{
 15	pool->irqs_per_cpu[cpu]++;
 16}
 17
 18/* Gets the least loaded CPU. e.g.: the CPU with least IRQs bound to it */
 19static int cpu_get_least_loaded(struct mlx5_irq_pool *pool,
 20				const struct cpumask *req_mask)
 21{
 22	int best_cpu = -1;
 23	int cpu;
 24
 25	for_each_cpu_and(cpu, req_mask, cpu_online_mask) {
 26		/* CPU has zero IRQs on it. No need to search any more CPUs. */
 27		if (!pool->irqs_per_cpu[cpu]) {
 28			best_cpu = cpu;
 29			break;
 30		}
 31		if (best_cpu < 0)
 32			best_cpu = cpu;
 33		if (pool->irqs_per_cpu[cpu] < pool->irqs_per_cpu[best_cpu])
 34			best_cpu = cpu;
 35	}
 36	if (best_cpu == -1) {
 37		/* There isn't online CPUs in req_mask */
 38		mlx5_core_err(pool->dev, "NO online CPUs in req_mask (%*pbl)\n",
 39			      cpumask_pr_args(req_mask));
 40		best_cpu = cpumask_first(cpu_online_mask);
 41	}
 42	pool->irqs_per_cpu[best_cpu]++;
 43	return best_cpu;
 44}
 45
 46/* Creating an IRQ from irq_pool */
 47static struct mlx5_irq *
 48irq_pool_request_irq(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc)
 49{
 50	struct irq_affinity_desc *auto_desc;
 51	struct mlx5_irq *irq;
 52	u32 irq_index;
 53	int err;
 54
 55	auto_desc = kvzalloc(sizeof(*auto_desc), GFP_KERNEL);
 56	if (!auto_desc)
 57		return ERR_PTR(-ENOMEM);
 58
 59	err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs, GFP_KERNEL);
 60	if (err) {
 61		kvfree(auto_desc);
 62		return ERR_PTR(err);
 63	}
 64
 65	if (pool->irqs_per_cpu) {
 66		if (cpumask_weight(&af_desc->mask) > 1)
 67			/* if req_mask contain more then one CPU, set the least loadad CPU
 68			 * of req_mask
 69			 */
 70			cpumask_set_cpu(cpu_get_least_loaded(pool, &af_desc->mask),
 71					&auto_desc->mask);
 72		else
 73			cpu_get(pool, cpumask_first(&af_desc->mask));
 74	}
 75
 76	irq = mlx5_irq_alloc(pool, irq_index,
 77			     cpumask_empty(&auto_desc->mask) ? af_desc : auto_desc,
 78			     NULL);
 79	if (IS_ERR(irq))
 80		xa_erase(&pool->irqs, irq_index);
 81
 82	kvfree(auto_desc);
 83
 84	return irq;
 85}
 86
 87/* Looking for the IRQ with the smallest refcount that fits req_mask.
 88 * If pool is sf_comp_pool, then we are looking for an IRQ with any of the
 89 * requested CPUs in req_mask.
 90 * for example: req_mask = 0xf, irq0_mask = 0x10, irq1_mask = 0x1. irq0_mask
 91 * isn't subset of req_mask, so we will skip it. irq1_mask is subset of req_mask,
 92 * we don't skip it.
 93 * If pool is sf_ctrl_pool, then all IRQs have the same mask, so any IRQ will
 94 * fit. And since mask is subset of itself, we will pass the first if bellow.
 95 */
 96static struct mlx5_irq *
 97irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, const struct cpumask *req_mask)
 98{
 99	int start = pool->xa_num_irqs.min;
100	int end = pool->xa_num_irqs.max;
101	struct mlx5_irq *irq = NULL;
102	struct mlx5_irq *iter;
103	int irq_refcount = 0;
104	unsigned long index;
105
106	lockdep_assert_held(&pool->lock);
107	xa_for_each_range(&pool->irqs, index, iter, start, end) {
108		struct cpumask *iter_mask = mlx5_irq_get_affinity_mask(iter);
109		int iter_refcount = mlx5_irq_read_locked(iter);
110
111		if (!cpumask_subset(iter_mask, req_mask))
112			/* skip IRQs with a mask which is not subset of req_mask */
113			continue;
114		if (iter_refcount < pool->min_threshold)
115			/* If we found an IRQ with less than min_thres, return it */
116			return iter;
117		if (!irq || iter_refcount < irq_refcount) {
118			/* In case we won't find an IRQ with less than min_thres,
119			 * keep a pointer to the least used IRQ
120			 */
121			irq_refcount = iter_refcount;
122			irq = iter;
123		}
124	}
125	return irq;
126}
127
128/**
129 * mlx5_irq_affinity_request - request an IRQ according to the given mask.
130 * @dev: mlx5 core device which is requesting the IRQ.
131 * @pool: IRQ pool to request from.
132 * @af_desc: affinity descriptor for this IRQ.
133 *
134 * This function returns a pointer to IRQ, or ERR_PTR in case of error.
135 */
136struct mlx5_irq *
137mlx5_irq_affinity_request(struct mlx5_core_dev *dev, struct mlx5_irq_pool *pool,
138			  struct irq_affinity_desc *af_desc)
139{
140	struct mlx5_irq *least_loaded_irq, *new_irq;
141	int ret;
142
143	mutex_lock(&pool->lock);
144	least_loaded_irq = irq_pool_find_least_loaded(pool, &af_desc->mask);
145	if (least_loaded_irq &&
146	    mlx5_irq_read_locked(least_loaded_irq) < pool->min_threshold)
147		goto out;
148	/* We didn't find an IRQ with less than min_thres, try to allocate a new IRQ */
149	new_irq = irq_pool_request_irq(pool, af_desc);
150	if (IS_ERR(new_irq)) {
151		if (!least_loaded_irq) {
152			/* We failed to create an IRQ and we didn't find an IRQ */
153			mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %pe\n",
154				      new_irq);
155			mutex_unlock(&pool->lock);
156			return new_irq;
157		}
158		/* We failed to create a new IRQ for the requested affinity,
159		 * sharing existing IRQ.
160		 */
161		goto out;
162	}
163	least_loaded_irq = new_irq;
164	goto unlock;
165out:
166	mlx5_irq_get_locked(least_loaded_irq);
167	if (mlx5_irq_read_locked(least_loaded_irq) > pool->max_threshold)
168		mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n",
169			      pci_irq_vector(pool->dev->pdev,
170					     mlx5_irq_get_index(least_loaded_irq)), pool->name,
171			      mlx5_irq_read_locked(least_loaded_irq) / MLX5_EQ_REFS_PER_IRQ);
172unlock:
173	mutex_unlock(&pool->lock);
174	if (mlx5_irq_pool_is_sf_pool(pool)) {
175		ret = auxiliary_device_sysfs_irq_add(mlx5_sf_coredev_to_adev(dev),
176						     mlx5_irq_get_irq(least_loaded_irq));
177		if (ret) {
178			mlx5_core_err(dev, "Failed to create sysfs entry for irq %d, ret = %d\n",
179				      mlx5_irq_get_irq(least_loaded_irq), ret);
180			mlx5_irq_put(least_loaded_irq);
181			least_loaded_irq = ERR_PTR(ret);
182		}
183	}
184	return least_loaded_irq;
185}
186
187void mlx5_irq_affinity_irq_release(struct mlx5_core_dev *dev, struct mlx5_irq *irq)
188{
189	struct mlx5_irq_pool *pool = mlx5_irq_get_pool(irq);
190	int cpu;
191
192	cpu = cpumask_first(mlx5_irq_get_affinity_mask(irq));
193	synchronize_irq(pci_irq_vector(pool->dev->pdev,
194				       mlx5_irq_get_index(irq)));
195	if (mlx5_irq_pool_is_sf_pool(pool))
196		auxiliary_device_sysfs_irq_remove(mlx5_sf_coredev_to_adev(dev),
197						  mlx5_irq_get_irq(irq));
198	if (mlx5_irq_put(irq))
199		if (pool->irqs_per_cpu)
200			cpu_put(pool, cpu);
201}