ionic_hw_stats.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
  3
  4#include <linux/dma-mapping.h>
  5
  6#include "ionic_fw.h"
  7#include "ionic_ibdev.h"
  8
  9static int ionic_v1_stat_normalize(struct ionic_v1_stat *hw_stats,
 10				   int hw_stats_count)
 11{
 12	int hw_stat_i;
 13
 14	for (hw_stat_i = 0; hw_stat_i < hw_stats_count; ++hw_stat_i) {
 15		struct ionic_v1_stat *stat = &hw_stats[hw_stat_i];
 16
 17		stat->type_off = be32_to_cpu(stat->be_type_off);
 18		stat->name[sizeof(stat->name) - 1] = 0;
 19		if (ionic_v1_stat_type(stat) == IONIC_V1_STAT_TYPE_NONE)
 20			break;
 21	}
 22
 23	return hw_stat_i;
 24}
 25
 26static void ionic_fill_stats_desc(struct rdma_stat_desc *hw_stats_hdrs,
 27				  struct ionic_v1_stat *hw_stats,
 28				  int hw_stats_count)
 29{
 30	int hw_stat_i;
 31
 32	for (hw_stat_i = 0; hw_stat_i < hw_stats_count; ++hw_stat_i) {
 33		struct ionic_v1_stat *stat = &hw_stats[hw_stat_i];
 34
 35		hw_stats_hdrs[hw_stat_i].name = stat->name;
 36	}
 37}
 38
 39static u64 ionic_v1_stat_val(struct ionic_v1_stat *stat,
 40			     void *vals_buf, size_t vals_len)
 41{
 42	unsigned int off = ionic_v1_stat_off(stat);
 43	int type = ionic_v1_stat_type(stat);
 44
 45#define __ionic_v1_stat_validate(__type)		\
 46	((off + sizeof(__type) <= vals_len) &&		\
 47	 (IS_ALIGNED(off, sizeof(__type))))
 48
 49	switch (type) {
 50	case IONIC_V1_STAT_TYPE_8:
 51		if (__ionic_v1_stat_validate(u8))
 52			return *(u8 *)(vals_buf + off);
 53		break;
 54	case IONIC_V1_STAT_TYPE_LE16:
 55		if (__ionic_v1_stat_validate(__le16))
 56			return le16_to_cpu(*(__le16 *)(vals_buf + off));
 57		break;
 58	case IONIC_V1_STAT_TYPE_LE32:
 59		if (__ionic_v1_stat_validate(__le32))
 60			return le32_to_cpu(*(__le32 *)(vals_buf + off));
 61		break;
 62	case IONIC_V1_STAT_TYPE_LE64:
 63		if (__ionic_v1_stat_validate(__le64))
 64			return le64_to_cpu(*(__le64 *)(vals_buf + off));
 65		break;
 66	case IONIC_V1_STAT_TYPE_BE16:
 67		if (__ionic_v1_stat_validate(__be16))
 68			return be16_to_cpu(*(__be16 *)(vals_buf + off));
 69		break;
 70	case IONIC_V1_STAT_TYPE_BE32:
 71		if (__ionic_v1_stat_validate(__be32))
 72			return be32_to_cpu(*(__be32 *)(vals_buf + off));
 73		break;
 74	case IONIC_V1_STAT_TYPE_BE64:
 75		if (__ionic_v1_stat_validate(__be64))
 76			return be64_to_cpu(*(__be64 *)(vals_buf + off));
 77		break;
 78	}
 79
 80	return ~0ull;
 81#undef __ionic_v1_stat_validate
 82}
 83
 84static int ionic_hw_stats_cmd(struct ionic_ibdev *dev,
 85			      dma_addr_t dma, size_t len, int qid, int op)
 86{
 87	struct ionic_admin_wr wr = {
 88		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
 89		.wqe = {
 90			.op = op,
 91			.len = cpu_to_le16(IONIC_ADMIN_STATS_HDRS_IN_V1_LEN),
 92			.cmd.stats = {
 93				.dma_addr = cpu_to_le64(dma),
 94				.length = cpu_to_le32(len),
 95				.id_ver = cpu_to_le32(qid),
 96			},
 97		}
 98	};
 99
100	if (dev->lif_cfg.admin_opcodes <= op)
101		return -EBADRQC;
102
103	ionic_admin_post(dev, &wr);
104
105	return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_INTERRUPT);
106}
107
108static int ionic_init_hw_stats(struct ionic_ibdev *dev)
109{
110	dma_addr_t hw_stats_dma;
111	int rc, hw_stats_count;
112
113	if (dev->hw_stats_hdrs)
114		return 0;
115
116	dev->hw_stats_count = 0;
117
118	/* buffer for current values from the device */
119	dev->hw_stats_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
120	if (!dev->hw_stats_buf) {
121		rc = -ENOMEM;
122		goto err_buf;
123	}
124
125	/* buffer for names, sizes, offsets of values */
126	dev->hw_stats = kzalloc(PAGE_SIZE, GFP_KERNEL);
127	if (!dev->hw_stats) {
128		rc = -ENOMEM;
129		goto err_hw_stats;
130	}
131
132	/* request the names, sizes, offsets */
133	hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, dev->hw_stats,
134				      PAGE_SIZE, DMA_FROM_DEVICE);
135	rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma);
136	if (rc)
137		goto err_dma;
138
139	rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE, 0,
140				IONIC_V1_ADMIN_STATS_HDRS);
141	if (rc)
142		goto err_cmd;
143
144	dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);
145
146	/* normalize and count the number of hw_stats */
147	hw_stats_count =
148		ionic_v1_stat_normalize(dev->hw_stats,
149					PAGE_SIZE / sizeof(*dev->hw_stats));
150	if (!hw_stats_count) {
151		rc = -ENODATA;
152		goto err_dma;
153	}
154
155	dev->hw_stats_count = hw_stats_count;
156
157	/* alloc and init array of names, for alloc_hw_stats */
158	dev->hw_stats_hdrs = kcalloc(hw_stats_count,
159				     sizeof(*dev->hw_stats_hdrs),
160				     GFP_KERNEL);
161	if (!dev->hw_stats_hdrs) {
162		rc = -ENOMEM;
163		goto err_dma;
164	}
165
166	ionic_fill_stats_desc(dev->hw_stats_hdrs, dev->hw_stats,
167			      hw_stats_count);
168
169	return 0;
170
171err_cmd:
172	dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);
173err_dma:
174	kfree(dev->hw_stats);
175err_hw_stats:
176	kfree(dev->hw_stats_buf);
177err_buf:
178	dev->hw_stats_count = 0;
179	dev->hw_stats = NULL;
180	dev->hw_stats_buf = NULL;
181	dev->hw_stats_hdrs = NULL;
182	return rc;
183}
184
185static struct rdma_hw_stats *ionic_alloc_hw_stats(struct ib_device *ibdev,
186						  u32 port)
187{
188	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);
189
190	if (port != 1)
191		return NULL;
192
193	return rdma_alloc_hw_stats_struct(dev->hw_stats_hdrs,
194					  dev->hw_stats_count,
195					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
196}
197
198static int ionic_get_hw_stats(struct ib_device *ibdev,
199			      struct rdma_hw_stats *hw_stats,
200			      u32 port, int index)
201{
202	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);
203	dma_addr_t hw_stats_dma;
204	int rc, hw_stat_i;
205
206	if (port != 1)
207		return -EINVAL;
208
209	hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, dev->hw_stats_buf,
210				      PAGE_SIZE, DMA_FROM_DEVICE);
211	rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma);
212	if (rc)
213		goto err_dma;
214
215	rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE,
216				0, IONIC_V1_ADMIN_STATS_VALS);
217	if (rc)
218		goto err_cmd;
219
220	dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma,
221			 PAGE_SIZE, DMA_FROM_DEVICE);
222
223	for (hw_stat_i = 0; hw_stat_i < dev->hw_stats_count; ++hw_stat_i)
224		hw_stats->value[hw_stat_i] =
225			ionic_v1_stat_val(&dev->hw_stats[hw_stat_i],
226					  dev->hw_stats_buf, PAGE_SIZE);
227
228	return hw_stat_i;
229
230err_cmd:
231	dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma,
232			 PAGE_SIZE, DMA_FROM_DEVICE);
233err_dma:
234	return rc;
235}
236
237static struct rdma_hw_stats *
238ionic_counter_alloc_stats(struct rdma_counter *counter)
239{
240	struct ionic_ibdev *dev = to_ionic_ibdev(counter->device);
241	struct ionic_counter *cntr;
242	int err;
243
244	cntr = kzalloc(sizeof(*cntr), GFP_KERNEL);
245	if (!cntr)
246		return NULL;
247
248	/* buffer for current values from the device */
249	cntr->vals = kzalloc(PAGE_SIZE, GFP_KERNEL);
250	if (!cntr->vals)
251		goto err_vals;
252
253	err = xa_alloc(&dev->counter_stats->xa_counters, &counter->id,
254		       cntr,
255		       XA_LIMIT(0, IONIC_MAX_QPID),
256		       GFP_KERNEL);
257	if (err)
258		goto err_xa;
259
260	INIT_LIST_HEAD(&cntr->qp_list);
261
262	return rdma_alloc_hw_stats_struct(dev->counter_stats->stats_hdrs,
263					 dev->counter_stats->queue_stats_count,
264					 RDMA_HW_STATS_DEFAULT_LIFESPAN);
265err_xa:
266	kfree(cntr->vals);
267err_vals:
268	kfree(cntr);
269
270	return NULL;
271}
272
273static int ionic_counter_dealloc(struct rdma_counter *counter)
274{
275	struct ionic_ibdev *dev = to_ionic_ibdev(counter->device);
276	struct ionic_counter *cntr;
277
278	cntr = xa_erase(&dev->counter_stats->xa_counters, counter->id);
279	if (!cntr)
280		return -EINVAL;
281
282	kfree(cntr->vals);
283	kfree(cntr);
284
285	return 0;
286}
287
288static int ionic_counter_bind_qp(struct rdma_counter *counter,
289				 struct ib_qp *ibqp,
290				 u32 port)
291{
292	struct ionic_ibdev *dev = to_ionic_ibdev(counter->device);
293	struct ionic_qp *qp = to_ionic_qp(ibqp);
294	struct ionic_counter *cntr;
295
296	cntr = xa_load(&dev->counter_stats->xa_counters, counter->id);
297	if (!cntr)
298		return -EINVAL;
299
300	list_add_tail(&qp->qp_list_counter, &cntr->qp_list);
301	ibqp->counter = counter;
302
303	return 0;
304}
305
306static int ionic_counter_unbind_qp(struct ib_qp *ibqp, u32 port)
307{
308	struct ionic_qp *qp = to_ionic_qp(ibqp);
309
310	if (ibqp->counter) {
311		list_del(&qp->qp_list_counter);
312		ibqp->counter = NULL;
313	}
314
315	return 0;
316}
317
318static int ionic_get_qp_stats(struct ib_device *ibdev,
319			      struct rdma_hw_stats *hw_stats,
320			      u32 counter_id)
321{
322	struct ionic_ibdev *dev = to_ionic_ibdev(ibdev);
323	struct ionic_counter_stats *cs;
324	struct ionic_counter *cntr;
325	dma_addr_t hw_stats_dma;
326	struct ionic_qp *qp;
327	int rc, stat_i = 0;
328
329	cs = dev->counter_stats;
330	cntr = xa_load(&cs->xa_counters, counter_id);
331	if (!cntr)
332		return -EINVAL;
333
334	hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, cntr->vals,
335				      PAGE_SIZE, DMA_FROM_DEVICE);
336	rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma);
337	if (rc)
338		return rc;
339
340	memset(hw_stats->value, 0, sizeof(u64) * hw_stats->num_counters);
341
342	list_for_each_entry(qp, &cntr->qp_list, qp_list_counter) {
343		rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE,
344					qp->qpid,
345					IONIC_V1_ADMIN_QP_STATS_VALS);
346		if (rc)
347			goto err_cmd;
348
349		for (stat_i = 0; stat_i < cs->queue_stats_count; ++stat_i)
350			hw_stats->value[stat_i] +=
351				ionic_v1_stat_val(&cs->hdr[stat_i],
352						  cntr->vals,
353						  PAGE_SIZE);
354	}
355
356	dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);
357	return stat_i;
358
359err_cmd:
360	dma_unmap_single(dev->lif_cfg.hwdev, hw_stats_dma, PAGE_SIZE, DMA_FROM_DEVICE);
361
362	return rc;
363}
364
365static int ionic_counter_update_stats(struct rdma_counter *counter)
366{
367	return ionic_get_qp_stats(counter->device, counter->stats, counter->id);
368}
369
370static int ionic_alloc_counters(struct ionic_ibdev *dev)
371{
372	struct ionic_counter_stats *cs = dev->counter_stats;
373	int rc, hw_stats_count;
374	dma_addr_t hdr_dma;
375
376	/* buffer for names, sizes, offsets of values */
377	cs->hdr = kzalloc(PAGE_SIZE, GFP_KERNEL);
378	if (!cs->hdr)
379		return -ENOMEM;
380
381	hdr_dma = dma_map_single(dev->lif_cfg.hwdev, cs->hdr,
382				 PAGE_SIZE, DMA_FROM_DEVICE);
383	rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma);
384	if (rc)
385		goto err_dma;
386
387	rc = ionic_hw_stats_cmd(dev, hdr_dma, PAGE_SIZE, 0,
388				IONIC_V1_ADMIN_QP_STATS_HDRS);
389	if (rc)
390		goto err_cmd;
391
392	dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, PAGE_SIZE, DMA_FROM_DEVICE);
393
394	/* normalize and count the number of hw_stats */
395	hw_stats_count = ionic_v1_stat_normalize(cs->hdr,
396						 PAGE_SIZE / sizeof(*cs->hdr));
397	if (!hw_stats_count) {
398		rc = -ENODATA;
399		goto err_dma;
400	}
401
402	cs->queue_stats_count = hw_stats_count;
403
404	/* alloc and init array of names */
405	cs->stats_hdrs = kcalloc(hw_stats_count, sizeof(*cs->stats_hdrs),
406				 GFP_KERNEL);
407	if (!cs->stats_hdrs) {
408		rc = -ENOMEM;
409		goto err_dma;
410	}
411
412	ionic_fill_stats_desc(cs->stats_hdrs, cs->hdr, hw_stats_count);
413
414	return 0;
415
416err_cmd:
417	dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, PAGE_SIZE, DMA_FROM_DEVICE);
418err_dma:
419	kfree(cs->hdr);
420
421	return rc;
422}
423
424static const struct ib_device_ops ionic_hw_stats_ops = {
425	.driver_id = RDMA_DRIVER_IONIC,
426	.alloc_hw_port_stats = ionic_alloc_hw_stats,
427	.get_hw_stats = ionic_get_hw_stats,
428};
429
430static const struct ib_device_ops ionic_counter_stats_ops = {
431	.counter_alloc_stats = ionic_counter_alloc_stats,
432	.counter_dealloc = ionic_counter_dealloc,
433	.counter_bind_qp = ionic_counter_bind_qp,
434	.counter_unbind_qp = ionic_counter_unbind_qp,
435	.counter_update_stats = ionic_counter_update_stats,
436};
437
438void ionic_stats_init(struct ionic_ibdev *dev)
439{
440	u16 stats_type = dev->lif_cfg.stats_type;
441	int rc;
442
443	if (stats_type & IONIC_LIF_RDMA_STAT_GLOBAL) {
444		rc = ionic_init_hw_stats(dev);
445		if (rc)
446			ibdev_dbg(&dev->ibdev, "Failed to init hw stats\n");
447		else
448			ib_set_device_ops(&dev->ibdev, &ionic_hw_stats_ops);
449	}
450
451	if (stats_type & IONIC_LIF_RDMA_STAT_QP) {
452		dev->counter_stats = kzalloc(sizeof(*dev->counter_stats),
453					     GFP_KERNEL);
454		if (!dev->counter_stats)
455			return;
456
457		rc = ionic_alloc_counters(dev);
458		if (rc) {
459			ibdev_dbg(&dev->ibdev, "Failed to init counter stats\n");
460			kfree(dev->counter_stats);
461			dev->counter_stats = NULL;
462			return;
463		}
464
465		xa_init_flags(&dev->counter_stats->xa_counters, XA_FLAGS_ALLOC);
466
467		ib_set_device_ops(&dev->ibdev, &ionic_counter_stats_ops);
468	}
469}
470
471void ionic_stats_cleanup(struct ionic_ibdev *dev)
472{
473	if (dev->counter_stats) {
474		xa_destroy(&dev->counter_stats->xa_counters);
475		kfree(dev->counter_stats->hdr);
476		kfree(dev->counter_stats->stats_hdrs);
477		kfree(dev->counter_stats);
478		dev->counter_stats = NULL;
479	}
480
481	kfree(dev->hw_stats);
482	kfree(dev->hw_stats_buf);
483	kfree(dev->hw_stats_hdrs);
484}