fw_tracer.c

   1/*
   2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32#define CREATE_TRACE_POINTS
  33#include "lib/eq.h"
  34#include "fw_tracer.h"
  35#include "fw_tracer_tracepoint.h"
  36#include <linux/ctype.h>
  37
  38static int mlx5_query_mtrc_caps(struct mlx5_fw_tracer *tracer)
  39{
  40	u32 *string_db_base_address_out = tracer->str_db.base_address_out;
  41	u32 *string_db_size_out = tracer->str_db.size_out;
  42	struct mlx5_core_dev *dev = tracer->dev;
  43	u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  44	u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  45	void *mtrc_cap_sp;
  46	int err, i;
  47
  48	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
  49				   MLX5_REG_MTRC_CAP, 0, 0);
  50	if (err) {
  51		mlx5_core_warn(dev, "FWTracer: Error reading tracer caps %d\n",
  52			       err);
  53		return err;
  54	}
  55
  56	if (!MLX5_GET(mtrc_cap, out, trace_to_memory)) {
  57		mlx5_core_dbg(dev, "FWTracer: Device does not support logging traces to memory\n");
  58		return -EOPNOTSUPP;
  59	}
  60
  61	tracer->trc_ver = MLX5_GET(mtrc_cap, out, trc_ver);
  62	tracer->str_db.first_string_trace =
  63			MLX5_GET(mtrc_cap, out, first_string_trace);
  64	tracer->str_db.num_string_trace =
  65			MLX5_GET(mtrc_cap, out, num_string_trace);
  66	tracer->str_db.num_string_db = MLX5_GET(mtrc_cap, out, num_string_db);
  67	tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
  68	tracer->str_db.loaded = false;
  69
  70	for (i = 0; i < tracer->str_db.num_string_db; i++) {
  71		mtrc_cap_sp = MLX5_ADDR_OF(mtrc_cap, out, string_db_param[i]);
  72		string_db_base_address_out[i] = MLX5_GET(mtrc_string_db_param,
  73							 mtrc_cap_sp,
  74							 string_db_base_address);
  75		string_db_size_out[i] = MLX5_GET(mtrc_string_db_param,
  76						 mtrc_cap_sp, string_db_size);
  77	}
  78
  79	return err;
  80}
  81
  82static int mlx5_set_mtrc_caps_trace_owner(struct mlx5_fw_tracer *tracer,
  83					  u32 *out, u32 out_size,
  84					  u8 trace_owner)
  85{
  86	struct mlx5_core_dev *dev = tracer->dev;
  87	u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  88
  89	MLX5_SET(mtrc_cap, in, trace_owner, trace_owner);
  90
  91	return mlx5_core_access_reg(dev, in, sizeof(in), out, out_size,
  92				    MLX5_REG_MTRC_CAP, 0, 1);
  93}
  94
  95static int mlx5_fw_tracer_ownership_acquire(struct mlx5_fw_tracer *tracer)
  96{
  97	struct mlx5_core_dev *dev = tracer->dev;
  98	u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
  99	int err;
 100
 101	err = mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
 102					     MLX5_FW_TRACER_ACQUIRE_OWNERSHIP);
 103	if (err) {
 104		mlx5_core_warn(dev, "FWTracer: Acquire tracer ownership failed %d\n",
 105			       err);
 106		return err;
 107	}
 108
 109	tracer->owner = !!MLX5_GET(mtrc_cap, out, trace_owner);
 110
 111	if (!tracer->owner)
 112		return -EBUSY;
 113
 114	return 0;
 115}
 116
 117static void mlx5_fw_tracer_ownership_release(struct mlx5_fw_tracer *tracer)
 118{
 119	u32 out[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
 120
 121	mlx5_set_mtrc_caps_trace_owner(tracer, out, sizeof(out),
 122				       MLX5_FW_TRACER_RELEASE_OWNERSHIP);
 123	tracer->owner = false;
 124}
 125
 126static int mlx5_fw_tracer_create_log_buf(struct mlx5_fw_tracer *tracer)
 127{
 128	struct mlx5_core_dev *dev = tracer->dev;
 129	struct device *ddev;
 130	dma_addr_t dma;
 131	void *buff;
 132	gfp_t gfp;
 133	int err;
 134
 135	tracer->buff.size = TRACE_BUFFER_SIZE_BYTE;
 136
 137	gfp = GFP_KERNEL | __GFP_ZERO;
 138	buff = (void *)__get_free_pages(gfp,
 139					get_order(tracer->buff.size));
 140	if (!buff) {
 141		err = -ENOMEM;
 142		mlx5_core_warn(dev, "FWTracer: Failed to allocate pages, %d\n", err);
 143		return err;
 144	}
 145	tracer->buff.log_buf = buff;
 146
 147	ddev = mlx5_core_dma_dev(dev);
 148	dma = dma_map_single(ddev, buff, tracer->buff.size, DMA_FROM_DEVICE);
 149	if (dma_mapping_error(ddev, dma)) {
 150		mlx5_core_warn(dev, "FWTracer: Unable to map DMA: %d\n",
 151			       dma_mapping_error(ddev, dma));
 152		err = -ENOMEM;
 153		goto free_pages;
 154	}
 155	tracer->buff.dma = dma;
 156
 157	return 0;
 158
 159free_pages:
 160	free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
 161
 162	return err;
 163}
 164
 165static void mlx5_fw_tracer_destroy_log_buf(struct mlx5_fw_tracer *tracer)
 166{
 167	struct mlx5_core_dev *dev = tracer->dev;
 168	struct device *ddev;
 169
 170	if (!tracer->buff.log_buf)
 171		return;
 172
 173	ddev = mlx5_core_dma_dev(dev);
 174	dma_unmap_single(ddev, tracer->buff.dma, tracer->buff.size, DMA_FROM_DEVICE);
 175	free_pages((unsigned long)tracer->buff.log_buf, get_order(tracer->buff.size));
 176}
 177
 178static int mlx5_fw_tracer_create_mkey(struct mlx5_fw_tracer *tracer)
 179{
 180	struct mlx5_core_dev *dev = tracer->dev;
 181	int err, inlen, i;
 182	__be64 *mtt;
 183	void *mkc;
 184	u32 *in;
 185
 186	inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
 187			sizeof(*mtt) * round_up(TRACER_BUFFER_PAGE_NUM, 2);
 188
 189	in = kvzalloc(inlen, GFP_KERNEL);
 190	if (!in)
 191		return -ENOMEM;
 192
 193	MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
 194		 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
 195	mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 196	for (i = 0 ; i < TRACER_BUFFER_PAGE_NUM ; i++)
 197		mtt[i] = cpu_to_be64(tracer->buff.dma + i * PAGE_SIZE);
 198
 199	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 200	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
 201	MLX5_SET(mkc, mkc, lr, 1);
 202	MLX5_SET(mkc, mkc, lw, 1);
 203	MLX5_SET(mkc, mkc, pd, tracer->buff.pdn);
 204	MLX5_SET(mkc, mkc, bsf_octword_size, 0);
 205	MLX5_SET(mkc, mkc, qpn, 0xffffff);
 206	MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
 207	MLX5_SET(mkc, mkc, translations_octword_size,
 208		 DIV_ROUND_UP(TRACER_BUFFER_PAGE_NUM, 2));
 209	MLX5_SET64(mkc, mkc, start_addr, tracer->buff.dma);
 210	MLX5_SET64(mkc, mkc, len, tracer->buff.size);
 211	err = mlx5_core_create_mkey(dev, &tracer->buff.mkey, in, inlen);
 212	if (err)
 213		mlx5_core_warn(dev, "FWTracer: Failed to create mkey, %d\n", err);
 214
 215	kvfree(in);
 216
 217	return err;
 218}
 219
 220static void mlx5_fw_tracer_free_strings_db(struct mlx5_fw_tracer *tracer)
 221{
 222	u32 num_string_db = tracer->str_db.num_string_db;
 223	int i;
 224
 225	for (i = 0; i < num_string_db; i++) {
 226		kfree(tracer->str_db.buffer[i]);
 227		tracer->str_db.buffer[i] = NULL;
 228	}
 229}
 230
 231static int mlx5_fw_tracer_allocate_strings_db(struct mlx5_fw_tracer *tracer)
 232{
 233	u32 *string_db_size_out = tracer->str_db.size_out;
 234	u32 num_string_db = tracer->str_db.num_string_db;
 235	int i;
 236
 237	for (i = 0; i < num_string_db; i++) {
 238		if (!string_db_size_out[i])
 239			continue;
 240		tracer->str_db.buffer[i] = kzalloc(string_db_size_out[i], GFP_KERNEL);
 241		if (!tracer->str_db.buffer[i])
 242			goto free_strings_db;
 243	}
 244
 245	return 0;
 246
 247free_strings_db:
 248	mlx5_fw_tracer_free_strings_db(tracer);
 249	return -ENOMEM;
 250}
 251
 252static void
 253mlx5_fw_tracer_init_saved_traces_array(struct mlx5_fw_tracer *tracer)
 254{
 255	tracer->st_arr.saved_traces_index = 0;
 256	mutex_init(&tracer->st_arr.lock);
 257}
 258
 259static void
 260mlx5_fw_tracer_clean_saved_traces_array(struct mlx5_fw_tracer *tracer)
 261{
 262	mutex_destroy(&tracer->st_arr.lock);
 263}
 264
 265static void mlx5_tracer_read_strings_db(struct work_struct *work)
 266{
 267	struct mlx5_fw_tracer *tracer = container_of(work, struct mlx5_fw_tracer,
 268						     read_fw_strings_work);
 269	u32 num_of_reads, num_string_db = tracer->str_db.num_string_db;
 270	struct mlx5_core_dev *dev = tracer->dev;
 271	u32 in[MLX5_ST_SZ_DW(mtrc_cap)] = {0};
 272	u32 leftovers, offset;
 273	int err = 0, i, j;
 274	u32 *out, outlen;
 275	void *out_value;
 276
 277	outlen = MLX5_ST_SZ_BYTES(mtrc_stdb) + STRINGS_DB_READ_SIZE_BYTES;
 278	out = kzalloc(outlen, GFP_KERNEL);
 279	if (!out) {
 280		err = -ENOMEM;
 281		goto out;
 282	}
 283
 284	for (i = 0; i < num_string_db; i++) {
 285		if (!tracer->str_db.size_out[i])
 286			continue;
 287		offset = 0;
 288		MLX5_SET(mtrc_stdb, in, string_db_index, i);
 289		num_of_reads = tracer->str_db.size_out[i] /
 290				STRINGS_DB_READ_SIZE_BYTES;
 291		leftovers = (tracer->str_db.size_out[i] %
 292				STRINGS_DB_READ_SIZE_BYTES) /
 293					STRINGS_DB_LEFTOVER_SIZE_BYTES;
 294
 295		MLX5_SET(mtrc_stdb, in, read_size, STRINGS_DB_READ_SIZE_BYTES);
 296		for (j = 0; j < num_of_reads; j++) {
 297			MLX5_SET(mtrc_stdb, in, start_offset, offset);
 298
 299			err = mlx5_core_access_reg(dev, in, sizeof(in), out,
 300						   outlen, MLX5_REG_MTRC_STDB,
 301						   0, 1);
 302			if (err) {
 303				mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
 304					      err);
 305				goto out_free;
 306			}
 307
 308			out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
 309			memcpy(tracer->str_db.buffer[i] + offset, out_value,
 310			       STRINGS_DB_READ_SIZE_BYTES);
 311			offset += STRINGS_DB_READ_SIZE_BYTES;
 312		}
 313
 314		/* Strings database is aligned to 64, need to read leftovers*/
 315		MLX5_SET(mtrc_stdb, in, read_size,
 316			 STRINGS_DB_LEFTOVER_SIZE_BYTES);
 317		for (j = 0; j < leftovers; j++) {
 318			MLX5_SET(mtrc_stdb, in, start_offset, offset);
 319
 320			err = mlx5_core_access_reg(dev, in, sizeof(in), out,
 321						   outlen, MLX5_REG_MTRC_STDB,
 322						   0, 1);
 323			if (err) {
 324				mlx5_core_dbg(dev, "FWTracer: Failed to read strings DB %d\n",
 325					      err);
 326				goto out_free;
 327			}
 328
 329			out_value = MLX5_ADDR_OF(mtrc_stdb, out, string_db_data);
 330			memcpy(tracer->str_db.buffer[i] + offset, out_value,
 331			       STRINGS_DB_LEFTOVER_SIZE_BYTES);
 332			offset += STRINGS_DB_LEFTOVER_SIZE_BYTES;
 333		}
 334	}
 335
 336	tracer->str_db.loaded = true;
 337
 338out_free:
 339	kfree(out);
 340out:
 341	return;
 342}
 343
 344static void mlx5_fw_tracer_arm(struct mlx5_core_dev *dev)
 345{
 346	u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 347	u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 348	int err;
 349
 350	MLX5_SET(mtrc_ctrl, in, arm_event, 1);
 351
 352	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 353				   MLX5_REG_MTRC_CTRL, 0, 1);
 354	if (err)
 355		mlx5_core_warn(dev, "FWTracer: Failed to arm tracer event %d\n", err);
 356}
 357
 358static const char *VAL_PARM		= "%llx";
 359static const char *REPLACE_64_VAL_PARM	= "%x%x";
 360static const char *PARAM_CHAR		= "%";
 361
 362static bool mlx5_is_valid_spec(const char *str)
 363{
 364	/* Parse format specifiers to find the actual type.
 365	 * Structure: %[flags][width][.precision][length]type
 366	 * Skip flags, width, precision & length.
 367	 */
 368	while (isdigit(*str) || *str == '#' || *str == '.' || *str == 'l')
 369		str++;
 370
 371	/* Check if it's a valid integer/hex specifier or %%:
 372	 * Valid formats: %x, %d, %i, %u, etc.
 373	 */
 374	if (*str != 'x' && *str != 'X' && *str != 'd' && *str != 'i' &&
 375	    *str != 'u' && *str != 'c' && *str != '%')
 376		return false;
 377
 378	return true;
 379}
 380
 381static bool mlx5_tracer_validate_params(const char *str)
 382{
 383	const char *substr = str;
 384
 385	if (!str)
 386		return false;
 387
 388	substr = strstr(substr, PARAM_CHAR);
 389	while (substr) {
 390		if (!mlx5_is_valid_spec(substr + 1))
 391			return false;
 392
 393		if (*(substr + 1) == '%')
 394			substr = strstr(substr + 2, PARAM_CHAR);
 395		else
 396			substr = strstr(substr + 1, PARAM_CHAR);
 397
 398	}
 399
 400	return true;
 401}
 402
 403static int mlx5_tracer_message_hash(u32 message_id)
 404{
 405	return jhash_1word(message_id, 0) & (MESSAGE_HASH_SIZE - 1);
 406}
 407
 408static struct tracer_string_format *mlx5_tracer_message_insert(struct mlx5_fw_tracer *tracer,
 409							       struct tracer_event *tracer_event)
 410{
 411	struct hlist_head *head =
 412		&tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
 413	struct tracer_string_format *cur_string;
 414
 415	cur_string = kzalloc(sizeof(*cur_string), GFP_KERNEL);
 416	if (!cur_string)
 417		return NULL;
 418
 419	hlist_add_head(&cur_string->hlist, head);
 420
 421	return cur_string;
 422}
 423
 424static struct tracer_string_format *mlx5_tracer_get_string(struct mlx5_fw_tracer *tracer,
 425							   struct tracer_event *tracer_event)
 426{
 427	struct tracer_string_format *cur_string;
 428	u32 str_ptr, offset;
 429	int i;
 430
 431	str_ptr = tracer_event->string_event.string_param;
 432
 433	for (i = 0; i < tracer->str_db.num_string_db; i++) {
 434		if (!tracer->str_db.size_out[i])
 435			continue;
 436		if (str_ptr > tracer->str_db.base_address_out[i] &&
 437		    str_ptr < tracer->str_db.base_address_out[i] +
 438		    tracer->str_db.size_out[i]) {
 439			offset = str_ptr - tracer->str_db.base_address_out[i];
 440			/* add it to the hash */
 441			cur_string = mlx5_tracer_message_insert(tracer, tracer_event);
 442			if (!cur_string)
 443				return NULL;
 444			cur_string->string = (char *)(tracer->str_db.buffer[i] +
 445							offset);
 446			return cur_string;
 447		}
 448	}
 449
 450	return NULL;
 451}
 452
 453static void mlx5_tracer_clean_message(struct tracer_string_format *str_frmt)
 454{
 455	hlist_del(&str_frmt->hlist);
 456	kfree(str_frmt);
 457}
 458
 459static int mlx5_tracer_get_num_of_params(char *str)
 460{
 461	char *substr, *pstr = str;
 462	int num_of_params = 0;
 463
 464	/* Validate that all parameters are valid before processing */
 465	if (!mlx5_tracer_validate_params(str))
 466		return -EINVAL;
 467
 468	/* replace %llx with %x%x */
 469	substr = strstr(pstr, VAL_PARM);
 470	while (substr) {
 471		memcpy(substr, REPLACE_64_VAL_PARM, 4);
 472		pstr = substr;
 473		substr = strstr(pstr, VAL_PARM);
 474	}
 475
 476	/* count all the % characters, but skip %% (escaped percent) */
 477	substr = strstr(str, PARAM_CHAR);
 478	while (substr) {
 479		if (*(substr + 1) != '%') {
 480			num_of_params += 1;
 481			str = substr + 1;
 482		} else {
 483			str = substr + 2;
 484		}
 485		substr = strstr(str, PARAM_CHAR);
 486	}
 487
 488	return num_of_params;
 489}
 490
 491static struct tracer_string_format *mlx5_tracer_message_find(struct hlist_head *head,
 492							     u8 event_id, u32 tmsn)
 493{
 494	struct tracer_string_format *message;
 495
 496	hlist_for_each_entry(message, head, hlist)
 497		if (message->event_id == event_id && message->tmsn == tmsn)
 498			return message;
 499
 500	return NULL;
 501}
 502
 503static struct tracer_string_format *mlx5_tracer_message_get(struct mlx5_fw_tracer *tracer,
 504							    struct tracer_event *tracer_event)
 505{
 506	struct hlist_head *head =
 507		&tracer->hash[mlx5_tracer_message_hash(tracer_event->string_event.tmsn)];
 508
 509	return mlx5_tracer_message_find(head, tracer_event->event_id, tracer_event->string_event.tmsn);
 510}
 511
 512static void poll_trace(struct mlx5_fw_tracer *tracer,
 513		       struct tracer_event *tracer_event, u64 *trace)
 514{
 515	u32 timestamp_low, timestamp_mid, timestamp_high, urts;
 516
 517	tracer_event->event_id = MLX5_GET(tracer_event, trace, event_id);
 518	tracer_event->lost_event = MLX5_GET(tracer_event, trace, lost);
 519	tracer_event->out = trace;
 520
 521	switch (tracer_event->event_id) {
 522	case TRACER_EVENT_TYPE_TIMESTAMP:
 523		tracer_event->type = TRACER_EVENT_TYPE_TIMESTAMP;
 524		urts = MLX5_GET(tracer_timestamp_event, trace, urts);
 525		if (tracer->trc_ver == 0)
 526			tracer_event->timestamp_event.unreliable = !!(urts >> 2);
 527		else
 528			tracer_event->timestamp_event.unreliable = !!(urts & 1);
 529
 530		timestamp_low = MLX5_GET(tracer_timestamp_event,
 531					 trace, timestamp7_0);
 532		timestamp_mid = MLX5_GET(tracer_timestamp_event,
 533					 trace, timestamp39_8);
 534		timestamp_high = MLX5_GET(tracer_timestamp_event,
 535					  trace, timestamp52_40);
 536
 537		tracer_event->timestamp_event.timestamp =
 538				((u64)timestamp_high << 40) |
 539				((u64)timestamp_mid << 8) |
 540				(u64)timestamp_low;
 541		break;
 542	default:
 543		if (tracer_event->event_id >= tracer->str_db.first_string_trace &&
 544		    tracer_event->event_id <= tracer->str_db.first_string_trace +
 545					      tracer->str_db.num_string_trace) {
 546			tracer_event->type = TRACER_EVENT_TYPE_STRING;
 547			tracer_event->string_event.timestamp =
 548				MLX5_GET(tracer_string_event, trace, timestamp);
 549			tracer_event->string_event.string_param =
 550				MLX5_GET(tracer_string_event, trace, string_param);
 551			tracer_event->string_event.tmsn =
 552				MLX5_GET(tracer_string_event, trace, tmsn);
 553			tracer_event->string_event.tdsn =
 554				MLX5_GET(tracer_string_event, trace, tdsn);
 555		} else {
 556			tracer_event->type = TRACER_EVENT_TYPE_UNRECOGNIZED;
 557		}
 558		break;
 559	}
 560}
 561
 562static u64 get_block_timestamp(struct mlx5_fw_tracer *tracer, u64 *ts_event)
 563{
 564	struct tracer_event tracer_event;
 565	u8 event_id;
 566
 567	event_id = MLX5_GET(tracer_event, ts_event, event_id);
 568
 569	if (event_id == TRACER_EVENT_TYPE_TIMESTAMP)
 570		poll_trace(tracer, &tracer_event, ts_event);
 571	else
 572		tracer_event.timestamp_event.timestamp = 0;
 573
 574	return tracer_event.timestamp_event.timestamp;
 575}
 576
 577static void mlx5_fw_tracer_clean_print_hash(struct mlx5_fw_tracer *tracer)
 578{
 579	struct tracer_string_format *str_frmt;
 580	struct hlist_node *n;
 581	int i;
 582
 583	for (i = 0; i < MESSAGE_HASH_SIZE; i++) {
 584		hlist_for_each_entry_safe(str_frmt, n, &tracer->hash[i], hlist)
 585			mlx5_tracer_clean_message(str_frmt);
 586	}
 587}
 588
 589static void mlx5_fw_tracer_clean_ready_list(struct mlx5_fw_tracer *tracer)
 590{
 591	struct tracer_string_format *str_frmt, *tmp_str;
 592
 593	list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list,
 594				 list)
 595		list_del(&str_frmt->list);
 596}
 597
 598static void mlx5_fw_tracer_save_trace(struct mlx5_fw_tracer *tracer,
 599				      u64 timestamp, bool lost,
 600				      u8 event_id, char *msg)
 601{
 602	struct mlx5_fw_trace_data *trace_data;
 603
 604	mutex_lock(&tracer->st_arr.lock);
 605	trace_data = &tracer->st_arr.straces[tracer->st_arr.saved_traces_index];
 606	trace_data->timestamp = timestamp;
 607	trace_data->lost = lost;
 608	trace_data->event_id = event_id;
 609	strscpy_pad(trace_data->msg, msg, TRACE_STR_MSG);
 610
 611	tracer->st_arr.saved_traces_index =
 612		(tracer->st_arr.saved_traces_index + 1) & (SAVED_TRACES_NUM - 1);
 613	mutex_unlock(&tracer->st_arr.lock);
 614}
 615
 616static noinline
 617void mlx5_tracer_print_trace(struct tracer_string_format *str_frmt,
 618			     struct mlx5_core_dev *dev,
 619			     u64 trace_timestamp)
 620{
 621	char	tmp[512];
 622
 623	if (str_frmt->invalid_string)
 624		snprintf(tmp, sizeof(tmp), "BAD_FORMAT: %s", str_frmt->string);
 625	else
 626		snprintf(tmp, sizeof(tmp), str_frmt->string,
 627			 str_frmt->params[0],
 628			 str_frmt->params[1],
 629			 str_frmt->params[2],
 630			 str_frmt->params[3],
 631			 str_frmt->params[4],
 632			 str_frmt->params[5],
 633			 str_frmt->params[6]);
 634
 635	trace_mlx5_fw(dev->tracer, trace_timestamp, str_frmt->lost,
 636		      str_frmt->event_id, tmp);
 637
 638	mlx5_fw_tracer_save_trace(dev->tracer, trace_timestamp,
 639				  str_frmt->lost, str_frmt->event_id, tmp);
 640
 641	/* remove it from hash */
 642	mlx5_tracer_clean_message(str_frmt);
 643}
 644
 645static int mlx5_tracer_handle_raw_string(struct mlx5_fw_tracer *tracer,
 646					 struct tracer_event *tracer_event)
 647{
 648	struct tracer_string_format *cur_string;
 649
 650	cur_string = mlx5_tracer_message_insert(tracer, tracer_event);
 651	if (!cur_string)
 652		return -1;
 653
 654	cur_string->event_id = tracer_event->event_id;
 655	cur_string->timestamp = tracer_event->string_event.timestamp;
 656	cur_string->lost = tracer_event->lost_event;
 657	cur_string->string = "0x%08x%08x";
 658	cur_string->num_of_params = 2;
 659	cur_string->params[0] = upper_32_bits(*tracer_event->out);
 660	cur_string->params[1] = lower_32_bits(*tracer_event->out);
 661	list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 662	return 0;
 663}
 664
 665static void mlx5_tracer_handle_bad_format_string(struct mlx5_fw_tracer *tracer,
 666						 struct tracer_string_format *cur_string)
 667{
 668	cur_string->invalid_string = true;
 669	list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 670}
 671
 672static int mlx5_tracer_handle_string_trace(struct mlx5_fw_tracer *tracer,
 673					   struct tracer_event *tracer_event)
 674{
 675	struct tracer_string_format *cur_string;
 676
 677	if (tracer_event->string_event.tdsn == 0) {
 678		cur_string = mlx5_tracer_get_string(tracer, tracer_event);
 679		if (!cur_string)
 680			return mlx5_tracer_handle_raw_string(tracer, tracer_event);
 681
 682		cur_string->event_id = tracer_event->event_id;
 683		cur_string->tmsn = tracer_event->string_event.tmsn;
 684		cur_string->timestamp = tracer_event->string_event.timestamp;
 685		cur_string->lost = tracer_event->lost_event;
 686		cur_string->last_param_num = 0;
 687		cur_string->num_of_params = mlx5_tracer_get_num_of_params(cur_string->string);
 688		if (cur_string->num_of_params < 0) {
 689			pr_debug("%s Invalid format string parameters\n",
 690				 __func__);
 691			mlx5_tracer_handle_bad_format_string(tracer, cur_string);
 692			return 0;
 693		}
 694		if (cur_string->num_of_params == 0) /* trace with no params */
 695			list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 696	} else {
 697		cur_string = mlx5_tracer_message_get(tracer, tracer_event);
 698		if (!cur_string) {
 699			pr_debug("%s Got string event for unknown string tmsn: %d\n",
 700				 __func__, tracer_event->string_event.tmsn);
 701			return mlx5_tracer_handle_raw_string(tracer, tracer_event);
 702		}
 703		if (cur_string->num_of_params < 0) {
 704			pr_debug("%s string parameter of invalid string, dumping\n",
 705				 __func__);
 706			return 0;
 707		}
 708		cur_string->last_param_num += 1;
 709		if (cur_string->last_param_num > TRACER_MAX_PARAMS) {
 710			pr_debug("%s Number of params exceeds the max (%d)\n",
 711				 __func__, TRACER_MAX_PARAMS);
 712			list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 713			return 0;
 714		}
 715		/* keep the new parameter */
 716		cur_string->params[cur_string->last_param_num - 1] =
 717			tracer_event->string_event.string_param;
 718		if (cur_string->last_param_num == cur_string->num_of_params)
 719			list_add_tail(&cur_string->list, &tracer->ready_strings_list);
 720	}
 721
 722	return 0;
 723}
 724
 725static void mlx5_tracer_handle_timestamp_trace(struct mlx5_fw_tracer *tracer,
 726					       struct tracer_event *tracer_event)
 727{
 728	struct tracer_timestamp_event timestamp_event =
 729						tracer_event->timestamp_event;
 730	struct tracer_string_format *str_frmt, *tmp_str;
 731	struct mlx5_core_dev *dev = tracer->dev;
 732	u64 trace_timestamp;
 733
 734	list_for_each_entry_safe(str_frmt, tmp_str, &tracer->ready_strings_list, list) {
 735		list_del(&str_frmt->list);
 736		if (str_frmt->timestamp < (timestamp_event.timestamp & MASK_6_0))
 737			trace_timestamp = (timestamp_event.timestamp & MASK_52_7) |
 738					  (str_frmt->timestamp & MASK_6_0);
 739		else
 740			trace_timestamp = ((timestamp_event.timestamp - 1) & MASK_52_7) |
 741					  (str_frmt->timestamp & MASK_6_0);
 742
 743		mlx5_tracer_print_trace(str_frmt, dev, trace_timestamp);
 744	}
 745}
 746
 747static int mlx5_tracer_handle_trace(struct mlx5_fw_tracer *tracer,
 748				    struct tracer_event *tracer_event)
 749{
 750	if (tracer_event->type == TRACER_EVENT_TYPE_STRING) {
 751		mlx5_tracer_handle_string_trace(tracer, tracer_event);
 752	} else if (tracer_event->type == TRACER_EVENT_TYPE_TIMESTAMP) {
 753		if (!tracer_event->timestamp_event.unreliable)
 754			mlx5_tracer_handle_timestamp_trace(tracer, tracer_event);
 755	} else {
 756		pr_debug("%s Got unrecognised type %d for parsing, exiting..\n",
 757			 __func__, tracer_event->type);
 758	}
 759	return 0;
 760}
 761
 762static void mlx5_fw_tracer_handle_traces(struct work_struct *work)
 763{
 764	struct mlx5_fw_tracer *tracer =
 765			container_of(work, struct mlx5_fw_tracer, handle_traces_work);
 766	u64 block_timestamp, last_block_timestamp, tmp_trace_block[TRACES_PER_BLOCK];
 767	u32 block_count, start_offset, prev_start_offset, prev_consumer_index;
 768	u32 trace_event_size = MLX5_ST_SZ_BYTES(tracer_event);
 769	struct mlx5_core_dev *dev = tracer->dev;
 770	struct tracer_event tracer_event;
 771	int i;
 772
 773	mlx5_core_dbg(dev, "FWTracer: Handle Trace event, owner=(%d)\n", tracer->owner);
 774	if (!tracer->owner)
 775		return;
 776
 777	if (unlikely(!tracer->str_db.loaded))
 778		goto arm;
 779
 780	block_count = tracer->buff.size / TRACER_BLOCK_SIZE_BYTE;
 781	start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
 782
 783	/* Copy the block to local buffer to avoid HW override while being processed */
 784	memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
 785	       TRACER_BLOCK_SIZE_BYTE);
 786
 787	block_timestamp =
 788		get_block_timestamp(tracer, &tmp_trace_block[TRACES_PER_BLOCK - 1]);
 789
 790	while (block_timestamp > tracer->last_timestamp) {
 791		/* Check block override if it's not the first block */
 792		if (tracer->last_timestamp) {
 793			u64 *ts_event;
 794			/* To avoid block override be the HW in case of buffer
 795			 * wraparound, the time stamp of the previous block
 796			 * should be compared to the last timestamp handled
 797			 * by the driver.
 798			 */
 799			prev_consumer_index =
 800				(tracer->buff.consumer_index - 1) & (block_count - 1);
 801			prev_start_offset = prev_consumer_index * TRACER_BLOCK_SIZE_BYTE;
 802
 803			ts_event = tracer->buff.log_buf + prev_start_offset +
 804				   (TRACES_PER_BLOCK - 1) * trace_event_size;
 805			last_block_timestamp = get_block_timestamp(tracer, ts_event);
 806			/* If previous timestamp different from last stored
 807			 * timestamp then there is a good chance that the
 808			 * current buffer is overwritten and therefore should
 809			 * not be parsed.
 810			 */
 811			if (tracer->last_timestamp != last_block_timestamp) {
 812				mlx5_core_warn(dev, "FWTracer: Events were lost\n");
 813				tracer->last_timestamp = block_timestamp;
 814				tracer->buff.consumer_index =
 815					(tracer->buff.consumer_index + 1) & (block_count - 1);
 816				break;
 817			}
 818		}
 819
 820		/* Parse events */
 821		for (i = 0; i < TRACES_PER_BLOCK ; i++) {
 822			poll_trace(tracer, &tracer_event, &tmp_trace_block[i]);
 823			mlx5_tracer_handle_trace(tracer, &tracer_event);
 824		}
 825
 826		tracer->buff.consumer_index =
 827			(tracer->buff.consumer_index + 1) & (block_count - 1);
 828
 829		tracer->last_timestamp = block_timestamp;
 830		start_offset = tracer->buff.consumer_index * TRACER_BLOCK_SIZE_BYTE;
 831		memcpy(tmp_trace_block, tracer->buff.log_buf + start_offset,
 832		       TRACER_BLOCK_SIZE_BYTE);
 833		block_timestamp = get_block_timestamp(tracer,
 834						      &tmp_trace_block[TRACES_PER_BLOCK - 1]);
 835	}
 836
 837arm:
 838	mlx5_fw_tracer_arm(dev);
 839}
 840
 841static int mlx5_fw_tracer_set_mtrc_conf(struct mlx5_fw_tracer *tracer)
 842{
 843	struct mlx5_core_dev *dev = tracer->dev;
 844	u32 out[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
 845	u32 in[MLX5_ST_SZ_DW(mtrc_conf)] = {0};
 846	int err;
 847
 848	MLX5_SET(mtrc_conf, in, trace_mode, TRACE_TO_MEMORY);
 849	MLX5_SET(mtrc_conf, in, log_trace_buffer_size,
 850		 ilog2(TRACER_BUFFER_PAGE_NUM));
 851	MLX5_SET(mtrc_conf, in, trace_mkey, tracer->buff.mkey);
 852
 853	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 854				   MLX5_REG_MTRC_CONF, 0, 1);
 855	if (err)
 856		mlx5_core_warn(dev, "FWTracer: Failed to set tracer configurations %d\n", err);
 857
 858	tracer->buff.consumer_index = 0;
 859	return err;
 860}
 861
 862static int mlx5_fw_tracer_set_mtrc_ctrl(struct mlx5_fw_tracer *tracer, u8 status, u8 arm)
 863{
 864	struct mlx5_core_dev *dev = tracer->dev;
 865	u32 out[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 866	u32 in[MLX5_ST_SZ_DW(mtrc_ctrl)] = {0};
 867	int err;
 868
 869	MLX5_SET(mtrc_ctrl, in, modify_field_select, TRACE_STATUS);
 870	MLX5_SET(mtrc_ctrl, in, trace_status, status);
 871	MLX5_SET(mtrc_ctrl, in, arm_event, arm);
 872
 873	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out),
 874				   MLX5_REG_MTRC_CTRL, 0, 1);
 875
 876	if (!err && status)
 877		tracer->last_timestamp = 0;
 878
 879	return err;
 880}
 881
 882static int mlx5_fw_tracer_start(struct mlx5_fw_tracer *tracer)
 883{
 884	struct mlx5_core_dev *dev = tracer->dev;
 885	int err;
 886
 887	err = mlx5_fw_tracer_ownership_acquire(tracer);
 888	if (err) {
 889		mlx5_core_dbg(dev, "FWTracer: Ownership was not granted %d\n", err);
 890		/* Don't fail since ownership can be acquired on a later FW event */
 891		return 0;
 892	}
 893
 894	err = mlx5_fw_tracer_set_mtrc_conf(tracer);
 895	if (err) {
 896		mlx5_core_warn(dev, "FWTracer: Failed to set tracer configuration %d\n", err);
 897		goto release_ownership;
 898	}
 899
 900	/* enable tracer & trace events */
 901	err = mlx5_fw_tracer_set_mtrc_ctrl(tracer, 1, 1);
 902	if (err) {
 903		mlx5_core_warn(dev, "FWTracer: Failed to enable tracer %d\n", err);
 904		goto release_ownership;
 905	}
 906
 907	mlx5_core_dbg(dev, "FWTracer: Ownership granted and active\n");
 908	return 0;
 909
 910release_ownership:
 911	mlx5_fw_tracer_ownership_release(tracer);
 912	return err;
 913}
 914
 915static void mlx5_fw_tracer_ownership_change(struct work_struct *work)
 916{
 917	struct mlx5_fw_tracer *tracer =
 918		container_of(work, struct mlx5_fw_tracer, ownership_change_work);
 919
 920	mlx5_core_dbg(tracer->dev, "FWTracer: ownership changed, current=(%d)\n", tracer->owner);
 921	if (tracer->owner) {
 922		mlx5_fw_tracer_ownership_acquire(tracer);
 923		return;
 924	}
 925
 926	mlx5_fw_tracer_start(tracer);
 927}
 928
 929static int mlx5_fw_tracer_set_core_dump_reg(struct mlx5_core_dev *dev,
 930					    u32 *in, int size_in)
 931{
 932	u32 out[MLX5_ST_SZ_DW(core_dump_reg)] = {};
 933
 934	if (!MLX5_CAP_DEBUG(dev, core_dump_general) &&
 935	    !MLX5_CAP_DEBUG(dev, core_dump_qp))
 936		return -EOPNOTSUPP;
 937
 938	return mlx5_core_access_reg(dev, in, size_in, out, sizeof(out),
 939				    MLX5_REG_CORE_DUMP, 0, 1);
 940}
 941
 942int mlx5_fw_tracer_trigger_core_dump_general(struct mlx5_core_dev *dev)
 943{
 944	struct mlx5_fw_tracer *tracer = dev->tracer;
 945	u32 in[MLX5_ST_SZ_DW(core_dump_reg)] = {};
 946	int err;
 947
 948	if (!MLX5_CAP_DEBUG(dev, core_dump_general) || !tracer)
 949		return -EOPNOTSUPP;
 950	if (!tracer->owner)
 951		return -EPERM;
 952
 953	MLX5_SET(core_dump_reg, in, core_dump_type, 0x0);
 954
 955	err =  mlx5_fw_tracer_set_core_dump_reg(dev, in, sizeof(in));
 956	if (err)
 957		return err;
 958	queue_work(tracer->work_queue, &tracer->handle_traces_work);
 959	flush_workqueue(tracer->work_queue);
 960	return 0;
 961}
 962
 963static void
 964mlx5_devlink_fmsg_fill_trace(struct devlink_fmsg *fmsg,
 965			     struct mlx5_fw_trace_data *trace_data)
 966{
 967	devlink_fmsg_obj_nest_start(fmsg);
 968	devlink_fmsg_u64_pair_put(fmsg, "timestamp", trace_data->timestamp);
 969	devlink_fmsg_bool_pair_put(fmsg, "lost", trace_data->lost);
 970	devlink_fmsg_u8_pair_put(fmsg, "event_id", trace_data->event_id);
 971	devlink_fmsg_string_pair_put(fmsg, "msg", trace_data->msg);
 972	devlink_fmsg_obj_nest_end(fmsg);
 973}
 974
 975int mlx5_fw_tracer_get_saved_traces_objects(struct mlx5_fw_tracer *tracer,
 976					    struct devlink_fmsg *fmsg)
 977{
 978	struct mlx5_fw_trace_data *straces = tracer->st_arr.straces;
 979	u32 index, start_index, end_index;
 980	u32 saved_traces_index;
 981
 982	if (!straces[0].timestamp)
 983		return -ENOMSG;
 984
 985	mutex_lock(&tracer->st_arr.lock);
 986	saved_traces_index = tracer->st_arr.saved_traces_index;
 987	if (straces[saved_traces_index].timestamp)
 988		start_index = saved_traces_index;
 989	else
 990		start_index = 0;
 991	end_index = (saved_traces_index - 1) & (SAVED_TRACES_NUM - 1);
 992
 993	devlink_fmsg_arr_pair_nest_start(fmsg, "dump fw traces");
 994	index = start_index;
 995	while (index != end_index) {
 996		mlx5_devlink_fmsg_fill_trace(fmsg, &straces[index]);
 997
 998		index = (index + 1) & (SAVED_TRACES_NUM - 1);
 999	}
1000
1001	devlink_fmsg_arr_pair_nest_end(fmsg);
1002	mutex_unlock(&tracer->st_arr.lock);
1003
1004	return 0;
1005}
1006
1007static void mlx5_fw_tracer_update_db(struct work_struct *work)
1008{
1009	struct mlx5_fw_tracer *tracer =
1010			container_of(work, struct mlx5_fw_tracer, update_db_work);
1011
1012	mlx5_fw_tracer_reload(tracer);
1013}
1014
1015/* Create software resources (Buffers, etc ..) */
1016struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev)
1017{
1018	struct mlx5_fw_tracer *tracer = NULL;
1019	int err;
1020
1021	if (!MLX5_CAP_MCAM_REG(dev, tracer_registers)) {
1022		mlx5_core_dbg(dev, "FWTracer: Tracer capability not present\n");
1023		return NULL;
1024	}
1025
1026	tracer = kvzalloc(sizeof(*tracer), GFP_KERNEL);
1027	if (!tracer)
1028		return ERR_PTR(-ENOMEM);
1029
1030	tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer");
1031	if (!tracer->work_queue) {
1032		err = -ENOMEM;
1033		goto free_tracer;
1034	}
1035
1036	tracer->dev = dev;
1037
1038	INIT_LIST_HEAD(&tracer->ready_strings_list);
1039	INIT_WORK(&tracer->ownership_change_work, mlx5_fw_tracer_ownership_change);
1040	INIT_WORK(&tracer->read_fw_strings_work, mlx5_tracer_read_strings_db);
1041	INIT_WORK(&tracer->handle_traces_work, mlx5_fw_tracer_handle_traces);
1042	INIT_WORK(&tracer->update_db_work, mlx5_fw_tracer_update_db);
1043	mutex_init(&tracer->state_lock);
1044
1045
1046	err = mlx5_query_mtrc_caps(tracer);
1047	if (err) {
1048		mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err);
1049		goto destroy_workqueue;
1050	}
1051
1052	err = mlx5_fw_tracer_create_log_buf(tracer);
1053	if (err) {
1054		mlx5_core_warn(dev, "FWTracer: Create log buffer failed %d\n", err);
1055		goto destroy_workqueue;
1056	}
1057
1058	err = mlx5_fw_tracer_allocate_strings_db(tracer);
1059	if (err) {
1060		mlx5_core_warn(dev, "FWTracer: Allocate strings database failed %d\n", err);
1061		goto free_log_buf;
1062	}
1063
1064	mlx5_fw_tracer_init_saved_traces_array(tracer);
1065	mlx5_core_dbg(dev, "FWTracer: Tracer created\n");
1066
1067	return tracer;
1068
1069free_log_buf:
1070	mlx5_fw_tracer_destroy_log_buf(tracer);
1071destroy_workqueue:
1072	tracer->dev = NULL;
1073	destroy_workqueue(tracer->work_queue);
1074free_tracer:
1075	kvfree(tracer);
1076	return ERR_PTR(err);
1077}
1078
1079static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data);
1080
1081/* Create HW resources + start tracer */
1082int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer)
1083{
1084	struct mlx5_core_dev *dev;
1085	int err;
1086
1087	if (IS_ERR_OR_NULL(tracer))
1088		return 0;
1089
1090	if (!tracer->str_db.loaded)
1091		queue_work(tracer->work_queue, &tracer->read_fw_strings_work);
1092
1093	mutex_lock(&tracer->state_lock);
1094	if (test_and_set_bit(MLX5_TRACER_STATE_UP, &tracer->state))
1095		goto unlock;
1096
1097	dev = tracer->dev;
1098
1099	err = mlx5_core_alloc_pd(dev, &tracer->buff.pdn);
1100	if (err) {
1101		mlx5_core_warn(dev, "FWTracer: Failed to allocate PD %d\n", err);
1102		goto err_cancel_work;
1103	}
1104
1105	err = mlx5_fw_tracer_create_mkey(tracer);
1106	if (err) {
1107		mlx5_core_warn(dev, "FWTracer: Failed to create mkey %d\n", err);
1108		goto err_dealloc_pd;
1109	}
1110
1111	MLX5_NB_INIT(&tracer->nb, fw_tracer_event, DEVICE_TRACER);
1112	mlx5_eq_notifier_register(dev, &tracer->nb);
1113
1114	err = mlx5_fw_tracer_start(tracer);
1115	if (err) {
1116		mlx5_core_warn(dev, "FWTracer: Failed to start tracer %d\n", err);
1117		goto err_notifier_unregister;
1118	}
1119unlock:
1120	mutex_unlock(&tracer->state_lock);
1121	return 0;
1122
1123err_notifier_unregister:
1124	mlx5_eq_notifier_unregister(dev, &tracer->nb);
1125	mlx5_core_destroy_mkey(dev, tracer->buff.mkey);
1126err_dealloc_pd:
1127	mlx5_core_dealloc_pd(dev, tracer->buff.pdn);
1128err_cancel_work:
1129	cancel_work_sync(&tracer->read_fw_strings_work);
1130	mutex_unlock(&tracer->state_lock);
1131	return err;
1132}
1133
1134/* Stop tracer + Cleanup HW resources */
1135void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer)
1136{
1137	if (IS_ERR_OR_NULL(tracer))
1138		return;
1139
1140	mutex_lock(&tracer->state_lock);
1141	if (!test_and_clear_bit(MLX5_TRACER_STATE_UP, &tracer->state))
1142		goto unlock;
1143
1144	mlx5_core_dbg(tracer->dev, "FWTracer: Cleanup, is owner ? (%d)\n",
1145		      tracer->owner);
1146	mlx5_eq_notifier_unregister(tracer->dev, &tracer->nb);
1147	cancel_work_sync(&tracer->ownership_change_work);
1148	cancel_work_sync(&tracer->handle_traces_work);
1149	/* It is valid to get here from update_db_work. Hence, don't wait for
1150	 * update_db_work to finished.
1151	 */
1152	cancel_work(&tracer->update_db_work);
1153
1154	if (tracer->owner)
1155		mlx5_fw_tracer_ownership_release(tracer);
1156
1157	mlx5_core_destroy_mkey(tracer->dev, tracer->buff.mkey);
1158	mlx5_core_dealloc_pd(tracer->dev, tracer->buff.pdn);
1159unlock:
1160	mutex_unlock(&tracer->state_lock);
1161}
1162
1163/* Free software resources (Buffers, etc ..) */
1164void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer)
1165{
1166	if (IS_ERR_OR_NULL(tracer))
1167		return;
1168
1169	mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n");
1170
1171	cancel_work_sync(&tracer->read_fw_strings_work);
1172	mlx5_fw_tracer_clean_ready_list(tracer);
1173	mlx5_fw_tracer_clean_print_hash(tracer);
1174	mlx5_fw_tracer_clean_saved_traces_array(tracer);
1175	mlx5_fw_tracer_free_strings_db(tracer);
1176	mlx5_fw_tracer_destroy_log_buf(tracer);
1177	mutex_destroy(&tracer->state_lock);
1178	destroy_workqueue(tracer->work_queue);
1179	kvfree(tracer);
1180}
1181
1182static int mlx5_fw_tracer_recreate_strings_db(struct mlx5_fw_tracer *tracer)
1183{
1184	struct mlx5_core_dev *dev;
1185	int err;
1186
1187	if (test_and_set_bit(MLX5_TRACER_RECREATE_DB, &tracer->state))
1188		return 0;
1189	cancel_work_sync(&tracer->read_fw_strings_work);
1190	mlx5_fw_tracer_clean_ready_list(tracer);
1191	mlx5_fw_tracer_clean_print_hash(tracer);
1192	mlx5_fw_tracer_clean_saved_traces_array(tracer);
1193	mlx5_fw_tracer_free_strings_db(tracer);
1194
1195	dev = tracer->dev;
1196	err = mlx5_query_mtrc_caps(tracer);
1197	if (err) {
1198		mlx5_core_dbg(dev, "FWTracer: Failed to query capabilities %d\n", err);
1199		goto out;
1200	}
1201
1202	err = mlx5_fw_tracer_allocate_strings_db(tracer);
1203	if (err) {
1204		mlx5_core_warn(dev, "FWTracer: Allocate strings DB failed %d\n", err);
1205		goto out;
1206	}
1207	mlx5_fw_tracer_init_saved_traces_array(tracer);
1208out:
1209	clear_bit(MLX5_TRACER_RECREATE_DB, &tracer->state);
1210	return err;
1211}
1212
1213int mlx5_fw_tracer_reload(struct mlx5_fw_tracer *tracer)
1214{
1215	struct mlx5_core_dev *dev;
1216	int err;
1217
1218	if (IS_ERR_OR_NULL(tracer))
1219		return 0;
1220
1221	dev = tracer->dev;
1222	mlx5_fw_tracer_cleanup(tracer);
1223	err = mlx5_fw_tracer_recreate_strings_db(tracer);
1224	if (err) {
1225		mlx5_core_warn(dev, "Failed to recreate FW tracer strings DB\n");
1226		return err;
1227	}
1228	err = mlx5_fw_tracer_init(tracer);
1229	if (err) {
1230		mlx5_core_warn(dev, "Failed to re-initialize FW tracer\n");
1231		return err;
1232	}
1233
1234	return 0;
1235}
1236
1237static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data)
1238{
1239	struct mlx5_fw_tracer *tracer = mlx5_nb_cof(nb, struct mlx5_fw_tracer, nb);
1240	struct mlx5_core_dev *dev = tracer->dev;
1241	struct mlx5_eqe *eqe = data;
1242
1243	switch (eqe->sub_type) {
1244	case MLX5_TRACER_SUBTYPE_OWNERSHIP_CHANGE:
1245		queue_work(tracer->work_queue, &tracer->ownership_change_work);
1246		break;
1247	case MLX5_TRACER_SUBTYPE_TRACES_AVAILABLE:
1248		queue_work(tracer->work_queue, &tracer->handle_traces_work);
1249		break;
1250	case MLX5_TRACER_SUBTYPE_STRINGS_DB_UPDATE:
1251		queue_work(tracer->work_queue, &tracer->update_db_work);
1252		break;
1253	default:
1254		mlx5_core_dbg(dev, "FWTracer: Event with unrecognized subtype: sub_type %d\n",
1255			      eqe->sub_type);
1256	}
1257
1258	return NOTIFY_OK;
1259}
1260
1261EXPORT_TRACEPOINT_SYMBOL(mlx5_fw);