main.c

   1/*
   2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/highmem.h>
  34#include <linux/module.h>
  35#include <linux/init.h>
  36#include <linux/errno.h>
  37#include <linux/pci.h>
  38#include <linux/dma-mapping.h>
  39#include <linux/slab.h>
  40#include <linux/interrupt.h>
  41#include <linux/delay.h>
  42#include <linux/mlx5/driver.h>
  43#include <linux/mlx5/cq.h>
  44#include <linux/mlx5/qp.h>
  45#include <linux/debugfs.h>
  46#include <linux/kmod.h>
  47#include <linux/mlx5/mlx5_ifc.h>
  48#include <linux/mlx5/vport.h>
  49#include <linux/version.h>
  50#include <net/devlink.h>
  51#include "mlx5_core.h"
  52#include "lib/eq.h"
  53#include "fs_core.h"
  54#include "lib/mpfs.h"
  55#include "eswitch.h"
  56#include "devlink.h"
  57#include "fw_reset.h"
  58#include "lib/mlx5.h"
  59#include "lib/tout.h"
  60#include "fpga/core.h"
  61#include "en_accel/ipsec.h"
  62#include "lib/clock.h"
  63#include "lib/vxlan.h"
  64#include "lib/geneve.h"
  65#include "lib/devcom.h"
  66#include "lib/pci_vsc.h"
  67#include "diag/fw_tracer.h"
  68#include "ecpf.h"
  69#include "lib/hv_vhca.h"
  70#include "diag/rsc_dump.h"
  71#include "sf/vhca_event.h"
  72#include "sf/dev/dev.h"
  73#include "sf/sf.h"
  74#include "mlx5_irq.h"
  75#include "hwmon.h"
  76#include "lag/lag.h"
  77
  78MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
  79MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver");
  80MODULE_LICENSE("Dual BSD/GPL");
  81
  82unsigned int mlx5_core_debug_mask;
  83module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644);
  84MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");
  85
  86static unsigned int prof_sel = MLX5_DEFAULT_PROF;
  87module_param_named(prof_sel, prof_sel, uint, 0444);
  88MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
  89
  90static u32 sw_owner_id[4];
  91#define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1)
  92static DEFINE_IDA(sw_vhca_ida);
  93
  94enum {
  95	MLX5_ATOMIC_REQ_MODE_BE = 0x0,
  96	MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
  97};
  98
  99#define LOG_MAX_SUPPORTED_QPS 0xff
 100
 101static struct mlx5_profile profile[] = {
 102	[0] = {
 103		.mask           = 0,
 104		.num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
 105	},
 106	[1] = {
 107		.mask		= MLX5_PROF_MASK_QP_SIZE,
 108		.log_max_qp	= 12,
 109		.num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
 110
 111	},
 112	[2] = {
 113		.mask		= MLX5_PROF_MASK_QP_SIZE |
 114				  MLX5_PROF_MASK_MR_CACHE,
 115		.log_max_qp	= LOG_MAX_SUPPORTED_QPS,
 116		.num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
 117		.mr_cache[0]	= {
 118			.size	= 500,
 119			.limit	= 250
 120		},
 121		.mr_cache[1]	= {
 122			.size	= 500,
 123			.limit	= 250
 124		},
 125		.mr_cache[2]	= {
 126			.size	= 500,
 127			.limit	= 250
 128		},
 129		.mr_cache[3]	= {
 130			.size	= 500,
 131			.limit	= 250
 132		},
 133		.mr_cache[4]	= {
 134			.size	= 500,
 135			.limit	= 250
 136		},
 137		.mr_cache[5]	= {
 138			.size	= 500,
 139			.limit	= 250
 140		},
 141		.mr_cache[6]	= {
 142			.size	= 500,
 143			.limit	= 250
 144		},
 145		.mr_cache[7]	= {
 146			.size	= 500,
 147			.limit	= 250
 148		},
 149		.mr_cache[8]	= {
 150			.size	= 500,
 151			.limit	= 250
 152		},
 153		.mr_cache[9]	= {
 154			.size	= 500,
 155			.limit	= 250
 156		},
 157		.mr_cache[10]	= {
 158			.size	= 500,
 159			.limit	= 250
 160		},
 161		.mr_cache[11]	= {
 162			.size	= 500,
 163			.limit	= 250
 164		},
 165		.mr_cache[12]	= {
 166			.size	= 64,
 167			.limit	= 32
 168		},
 169		.mr_cache[13]	= {
 170			.size	= 32,
 171			.limit	= 16
 172		},
 173		.mr_cache[14]	= {
 174			.size	= 16,
 175			.limit	= 8
 176		},
 177		.mr_cache[15]	= {
 178			.size	= 8,
 179			.limit	= 4
 180		},
 181	},
 182	[3] = {
 183		.mask		= MLX5_PROF_MASK_QP_SIZE,
 184		.log_max_qp	= LOG_MAX_SUPPORTED_QPS,
 185		.num_cmd_caches = 0,
 186	},
 187};
 188
 189static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
 190			u32 warn_time_mili, const char *init_state)
 191{
 192	unsigned long warn = jiffies + msecs_to_jiffies(warn_time_mili);
 193	unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili);
 194	u32 fw_initializing;
 195
 196	do {
 197		fw_initializing = ioread32be(&dev->iseg->initializing);
 198		if (!(fw_initializing >> 31))
 199			break;
 200		if (time_after(jiffies, end)) {
 201			mlx5_core_err(dev, "Firmware over %u MS in %s state, aborting\n",
 202				      max_wait_mili, init_state);
 203			return -ETIMEDOUT;
 204		}
 205		if (test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
 206			mlx5_core_warn(dev, "device is being removed, stop waiting for FW %s\n",
 207				       init_state);
 208			return -ENODEV;
 209		}
 210		if (warn_time_mili && time_after(jiffies, warn)) {
 211			mlx5_core_warn(dev, "Waiting for FW %s, timeout abort in %ds (0x%x)\n",
 212				       init_state, jiffies_to_msecs(end - warn) / 1000,
 213				       fw_initializing);
 214			warn = jiffies + msecs_to_jiffies(warn_time_mili);
 215		}
 216		msleep(mlx5_tout_ms(dev, FW_PRE_INIT_WAIT));
 217	} while (true);
 218
 219	return 0;
 220}
 221
 222static void mlx5_set_driver_version(struct mlx5_core_dev *dev)
 223{
 224	int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in,
 225					      driver_version);
 226	u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {};
 227	char *string;
 228
 229	if (!MLX5_CAP_GEN(dev, driver_version))
 230		return;
 231
 232	string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version);
 233
 234	snprintf(string, driver_ver_sz, "Linux,%s,%u.%u.%u",
 235		 KBUILD_MODNAME, LINUX_VERSION_MAJOR,
 236		 LINUX_VERSION_PATCHLEVEL, LINUX_VERSION_SUBLEVEL);
 237
 238	/*Send the command*/
 239	MLX5_SET(set_driver_version_in, in, opcode,
 240		 MLX5_CMD_OP_SET_DRIVER_VERSION);
 241
 242	mlx5_cmd_exec_in(dev, set_driver_version, in);
 243}
 244
 245static int set_dma_caps(struct pci_dev *pdev)
 246{
 247	int err;
 248
 249	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 250	if (err) {
 251		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
 252		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 253		if (err) {
 254			dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
 255			return err;
 256		}
 257	}
 258
 259	dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
 260	return err;
 261}
 262
 263static int mlx5_pci_enable_device(struct mlx5_core_dev *dev)
 264{
 265	struct pci_dev *pdev = dev->pdev;
 266	int err = 0;
 267
 268	mutex_lock(&dev->pci_status_mutex);
 269	if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) {
 270		err = pci_enable_device(pdev);
 271		if (!err)
 272			dev->pci_status = MLX5_PCI_STATUS_ENABLED;
 273	}
 274	mutex_unlock(&dev->pci_status_mutex);
 275
 276	return err;
 277}
 278
 279static void mlx5_pci_disable_device(struct mlx5_core_dev *dev)
 280{
 281	struct pci_dev *pdev = dev->pdev;
 282
 283	mutex_lock(&dev->pci_status_mutex);
 284	if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) {
 285		pci_disable_device(pdev);
 286		dev->pci_status = MLX5_PCI_STATUS_DISABLED;
 287	}
 288	mutex_unlock(&dev->pci_status_mutex);
 289}
 290
 291static int request_bar(struct pci_dev *pdev)
 292{
 293	int err = 0;
 294
 295	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
 296		dev_err(&pdev->dev, "Missing registers BAR, aborting\n");
 297		return -ENODEV;
 298	}
 299
 300	err = pci_request_regions(pdev, KBUILD_MODNAME);
 301	if (err)
 302		dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
 303
 304	return err;
 305}
 306
 307static void release_bar(struct pci_dev *pdev)
 308{
 309	pci_release_regions(pdev);
 310}
 311
 312struct mlx5_reg_host_endianness {
 313	u8	he;
 314	u8      rsvd[15];
 315};
 316
 317static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size)
 318{
 319	switch (size) {
 320	case 128:
 321		return 0;
 322	case 256:
 323		return 1;
 324	case 512:
 325		return 2;
 326	case 1024:
 327		return 3;
 328	case 2048:
 329		return 4;
 330	case 4096:
 331		return 5;
 332	default:
 333		mlx5_core_warn(dev, "invalid pkey table size %d\n", size);
 334		return 0;
 335	}
 336}
 337
 338void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *dev, struct net_device *netdev)
 339{
 340	mutex_lock(&dev->mlx5e_res.uplink_netdev_lock);
 341	dev->mlx5e_res.uplink_netdev = netdev;
 342	mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV,
 343					  netdev);
 344	mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock);
 345}
 346
 347void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *dev)
 348{
 349	mutex_lock(&dev->mlx5e_res.uplink_netdev_lock);
 350	mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV,
 351					  dev->mlx5e_res.uplink_netdev);
 352	mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock);
 353}
 354EXPORT_SYMBOL(mlx5_core_uplink_netdev_event_replay);
 355
 356void mlx5_core_mp_event_replay(struct mlx5_core_dev *dev, u32 event, void *data)
 357{
 358	mlx5_blocking_notifier_call_chain(dev, event, data);
 359}
 360EXPORT_SYMBOL(mlx5_core_mp_event_replay);
 361
 362int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type,
 363			    enum mlx5_cap_mode cap_mode)
 364{
 365	u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)];
 366	int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
 367	void *out, *hca_caps;
 368	u16 opmod = (cap_type << 1) | (cap_mode & 0x01);
 369	int err;
 370
 371	if (WARN_ON(!dev->caps.hca[cap_type]))
 372		/* this cap_type must be added to mlx5_hca_caps_alloc() */
 373		return -EINVAL;
 374
 375	memset(in, 0, sizeof(in));
 376	out = kzalloc(out_sz, GFP_KERNEL);
 377	if (!out)
 378		return -ENOMEM;
 379
 380	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 381	MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
 382	err = mlx5_cmd_exec_inout(dev, query_hca_cap, in, out);
 383	if (err) {
 384		mlx5_core_warn(dev,
 385			       "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n",
 386			       cap_type, cap_mode, err);
 387		goto query_ex;
 388	}
 389
 390	hca_caps =  MLX5_ADDR_OF(query_hca_cap_out, out, capability);
 391
 392	switch (cap_mode) {
 393	case HCA_CAP_OPMOD_GET_MAX:
 394		memcpy(dev->caps.hca[cap_type]->max, hca_caps,
 395		       MLX5_UN_SZ_BYTES(hca_cap_union));
 396		break;
 397	case HCA_CAP_OPMOD_GET_CUR:
 398		memcpy(dev->caps.hca[cap_type]->cur, hca_caps,
 399		       MLX5_UN_SZ_BYTES(hca_cap_union));
 400		break;
 401	default:
 402		mlx5_core_warn(dev,
 403			       "Tried to query dev cap type(%x) with wrong opmode(%x)\n",
 404			       cap_type, cap_mode);
 405		err = -EINVAL;
 406		break;
 407	}
 408query_ex:
 409	kfree(out);
 410	return err;
 411}
 412
 413int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type)
 414{
 415	int ret;
 416
 417	ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR);
 418	if (ret)
 419		return ret;
 420	return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX);
 421}
 422
 423static int set_caps(struct mlx5_core_dev *dev, void *in, int opmod)
 424{
 425	MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
 426	MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1);
 427	return mlx5_cmd_exec_in(dev, set_hca_cap, in);
 428}
 429
 430static int handle_hca_cap_atomic(struct mlx5_core_dev *dev, void *set_ctx)
 431{
 432	void *set_hca_cap;
 433	int req_endianness;
 434	int err;
 435
 436	if (!MLX5_CAP_GEN(dev, atomic))
 437		return 0;
 438
 439	err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC);
 440	if (err)
 441		return err;
 442
 443	req_endianness =
 444		MLX5_CAP_ATOMIC(dev,
 445				supported_atomic_req_8B_endianness_mode_1);
 446
 447	if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS)
 448		return 0;
 449
 450	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
 451
 452	/* Set requestor to host endianness */
 453	MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianness_mode,
 454		 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS);
 455
 456	return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC);
 457}
 458
 459static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx)
 460{
 461	bool do_set = false, mem_page_fault = false;
 462	void *set_hca_cap;
 463	int err;
 464
 465	if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) ||
 466	    !MLX5_CAP_GEN(dev, pg))
 467		return 0;
 468
 469	err = mlx5_core_get_caps(dev, MLX5_CAP_ODP);
 470	if (err)
 471		return err;
 472
 473	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
 474	memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ODP]->cur,
 475	       MLX5_ST_SZ_BYTES(odp_cap));
 476
 477	/* For best performance, enable memory scheme ODP only when
 478	 * it has page prefetch enabled.
 479	 */
 480	if (MLX5_CAP_ODP_MAX(dev, mem_page_fault) &&
 481	    MLX5_CAP_ODP_MAX(dev, memory_page_fault_scheme_cap.page_prefetch)) {
 482		mem_page_fault = true;
 483		do_set = true;
 484		MLX5_SET(odp_cap, set_hca_cap, mem_page_fault, mem_page_fault);
 485		goto set;
 486	}
 487
 488#define ODP_CAP_SET_MAX(dev, field)                                            \
 489	do {                                                                   \
 490		u32 _res = MLX5_CAP_ODP_MAX(dev, field);                       \
 491		if (_res) {                                                    \
 492			do_set = true;                                         \
 493			MLX5_SET(odp_cap, set_hca_cap, field, _res);           \
 494		}                                                              \
 495	} while (0)
 496
 497	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.ud_odp_caps.srq_receive);
 498	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.rc_odp_caps.srq_receive);
 499	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.srq_receive);
 500	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.send);
 501	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.receive);
 502	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.write);
 503	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.read);
 504	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.xrc_odp_caps.atomic);
 505	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.srq_receive);
 506	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.send);
 507	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.receive);
 508	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.write);
 509	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.read);
 510	ODP_CAP_SET_MAX(dev, transport_page_fault_scheme_cap.dc_odp_caps.atomic);
 511
 512set:
 513	if (do_set)
 514		err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ODP);
 515
 516	mlx5_core_dbg(dev, "Using ODP %s scheme\n",
 517		      mem_page_fault ? "memory" : "transport");
 518	return err;
 519}
 520
 521static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev)
 522{
 523	struct devlink *devlink = priv_to_devlink(dev);
 524	union devlink_param_value val;
 525	int err;
 526
 527	err = devl_param_driverinit_value_get(devlink,
 528					      DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
 529					      &val);
 530	if (!err)
 531		return val.vu32;
 532	mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err);
 533	return err;
 534}
 535
 536bool mlx5_is_roce_on(struct mlx5_core_dev *dev)
 537{
 538	struct devlink *devlink = priv_to_devlink(dev);
 539	union devlink_param_value val;
 540	int err;
 541
 542	err = devl_param_driverinit_value_get(devlink,
 543					      DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
 544					      &val);
 545
 546	if (!err)
 547		return val.vbool;
 548
 549	mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err);
 550	return MLX5_CAP_GEN(dev, roce);
 551}
 552EXPORT_SYMBOL(mlx5_is_roce_on);
 553
 554static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx)
 555{
 556	bool do_set = false;
 557	void *set_hca_cap;
 558	int err;
 559
 560	if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2))
 561		return 0;
 562
 563	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2);
 564	if (err)
 565		return err;
 566
 567	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
 568				   capability);
 569	memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur,
 570	       MLX5_ST_SZ_BYTES(cmd_hca_cap_2));
 571
 572	if (MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) &&
 573	    dev->priv.sw_vhca_id > 0) {
 574		MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1);
 575		do_set = true;
 576	}
 577
 578	if (MLX5_CAP_GEN_2_MAX(dev, lag_per_mp_group)) {
 579		MLX5_SET(cmd_hca_cap_2, set_hca_cap, lag_per_mp_group, 1);
 580		do_set = true;
 581	}
 582
 583	/* some FW versions that support querying MLX5_CAP_GENERAL_2
 584	 * capabilities but don't support setting them.
 585	 * Skip unnecessary update to hca_cap_2 when no changes were introduced
 586	 */
 587	return do_set ? set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2) : 0;
 588}
 589
 590static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
 591{
 592	struct mlx5_profile *prof = &dev->profile;
 593	void *set_hca_cap;
 594	int max_uc_list;
 595	int err;
 596
 597	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
 598	if (err)
 599		return err;
 600
 601	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
 602				   capability);
 603	memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL]->cur,
 604	       MLX5_ST_SZ_BYTES(cmd_hca_cap));
 605
 606	mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n",
 607		      mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)),
 608		      128);
 609	/* we limit the size of the pkey table to 128 entries for now */
 610	MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size,
 611		 to_fw_pkey_sz(dev, 128));
 612
 613	/* Check log_max_qp from HCA caps to set in current profile */
 614	if (prof->log_max_qp == LOG_MAX_SUPPORTED_QPS) {
 615		prof->log_max_qp = min_t(u8, 18, MLX5_CAP_GEN_MAX(dev, log_max_qp));
 616	} else if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) {
 617		mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n",
 618			       prof->log_max_qp,
 619			       MLX5_CAP_GEN_MAX(dev, log_max_qp));
 620		prof->log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp);
 621	}
 622	if (prof->mask & MLX5_PROF_MASK_QP_SIZE)
 623		MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp,
 624			 prof->log_max_qp);
 625
 626	/* disable cmdif checksum */
 627	MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
 628
 629	/* Enable 4K UAR only when HCA supports it and page size is bigger
 630	 * than 4K.
 631	 */
 632	if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096)
 633		MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1);
 634
 635	MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
 636
 637	if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte))
 638		MLX5_SET(cmd_hca_cap,
 639			 set_hca_cap,
 640			 cache_line_128byte,
 641			 cache_line_size() >= 128 ? 1 : 0);
 642
 643	if (MLX5_CAP_GEN_MAX(dev, dct))
 644		MLX5_SET(cmd_hca_cap, set_hca_cap, dct, 1);
 645
 646	if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_event))
 647		MLX5_SET(cmd_hca_cap, set_hca_cap, pci_sync_for_fw_update_event, 1);
 648	if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_with_driver_unload))
 649		MLX5_SET(cmd_hca_cap, set_hca_cap,
 650			 pci_sync_for_fw_update_with_driver_unload, 1);
 651	if (MLX5_CAP_GEN_MAX(dev, pcie_reset_using_hotreset_method))
 652		MLX5_SET(cmd_hca_cap, set_hca_cap,
 653			 pcie_reset_using_hotreset_method, 1);
 654
 655	if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports))
 656		MLX5_SET(cmd_hca_cap,
 657			 set_hca_cap,
 658			 num_vhca_ports,
 659			 MLX5_CAP_GEN_MAX(dev, num_vhca_ports));
 660
 661	if (MLX5_CAP_GEN_MAX(dev, release_all_pages))
 662		MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1);
 663
 664	if (MLX5_CAP_GEN_MAX(dev, mkey_by_name))
 665		MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1);
 666
 667	mlx5_vhca_state_cap_handle(dev, set_hca_cap);
 668
 669	if (MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix))
 670		MLX5_SET(cmd_hca_cap, set_hca_cap, num_total_dynamic_vf_msix,
 671			 MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix));
 672
 673	if (MLX5_CAP_GEN(dev, roce_rw_supported) && MLX5_CAP_GEN_MAX(dev, roce))
 674		MLX5_SET(cmd_hca_cap, set_hca_cap, roce,
 675			 mlx5_is_roce_on(dev));
 676
 677	max_uc_list = max_uc_list_get_devlink_param(dev);
 678	if (max_uc_list > 0)
 679		MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_current_uc_list,
 680			 ilog2(max_uc_list));
 681
 682	/* enable absolute native port num */
 683	if (MLX5_CAP_GEN_MAX(dev, abs_native_port_num))
 684		MLX5_SET(cmd_hca_cap, set_hca_cap, abs_native_port_num, 1);
 685
 686	return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
 687}
 688
 689/* Cached MLX5_CAP_GEN(dev, roce) can be out of sync this early in the
 690 * boot process.
 691 * In case RoCE cap is writable in FW and user/devlink requested to change the
 692 * cap, we are yet to query the final state of the above cap.
 693 * Hence, the need for this function.
 694 *
 695 * Returns
 696 * True:
 697 * 1) RoCE cap is read only in FW and already disabled
 698 * OR:
 699 * 2) RoCE cap is writable in FW and user/devlink requested it off.
 700 *
 701 * In any other case, return False.
 702 */
 703static bool is_roce_fw_disabled(struct mlx5_core_dev *dev)
 704{
 705	return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_on(dev)) ||
 706		(!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce));
 707}
 708
 709static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx)
 710{
 711	void *set_hca_cap;
 712	int err;
 713
 714	if (is_roce_fw_disabled(dev))
 715		return 0;
 716
 717	err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE);
 718	if (err)
 719		return err;
 720
 721	if (MLX5_CAP_ROCE(dev, sw_r_roce_src_udp_port) ||
 722	    !MLX5_CAP_ROCE_MAX(dev, sw_r_roce_src_udp_port))
 723		return 0;
 724
 725	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
 726	memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ROCE]->cur,
 727	       MLX5_ST_SZ_BYTES(roce_cap));
 728	MLX5_SET(roce_cap, set_hca_cap, sw_r_roce_src_udp_port, 1);
 729
 730	if (MLX5_CAP_ROCE_MAX(dev, qp_ooo_transmit_default))
 731		MLX5_SET(roce_cap, set_hca_cap, qp_ooo_transmit_default, 1);
 732
 733	err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ROCE);
 734	return err;
 735}
 736
 737static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev,
 738					 void *set_ctx)
 739{
 740	void *set_hca_cap;
 741	int err;
 742
 743	if (!MLX5_CAP_GEN(dev, port_selection_cap))
 744		return 0;
 745
 746	err = mlx5_core_get_caps(dev, MLX5_CAP_PORT_SELECTION);
 747	if (err)
 748		return err;
 749
 750	if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) ||
 751	    !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass))
 752		return 0;
 753
 754	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
 755	memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur,
 756	       MLX5_ST_SZ_BYTES(port_selection_cap));
 757	MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1);
 758
 759	err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_PORT_SELECTION);
 760
 761	return err;
 762}
 763
 764static int set_hca_cap(struct mlx5_core_dev *dev)
 765{
 766	int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
 767	void *set_ctx;
 768	int err;
 769
 770	set_ctx = kzalloc(set_sz, GFP_KERNEL);
 771	if (!set_ctx)
 772		return -ENOMEM;
 773
 774	err = handle_hca_cap(dev, set_ctx);
 775	if (err) {
 776		mlx5_core_err(dev, "handle_hca_cap failed\n");
 777		goto out;
 778	}
 779
 780	memset(set_ctx, 0, set_sz);
 781	err = handle_hca_cap_atomic(dev, set_ctx);
 782	if (err) {
 783		mlx5_core_err(dev, "handle_hca_cap_atomic failed\n");
 784		goto out;
 785	}
 786
 787	memset(set_ctx, 0, set_sz);
 788	err = handle_hca_cap_odp(dev, set_ctx);
 789	if (err) {
 790		mlx5_core_err(dev, "handle_hca_cap_odp failed\n");
 791		goto out;
 792	}
 793
 794	memset(set_ctx, 0, set_sz);
 795	err = handle_hca_cap_roce(dev, set_ctx);
 796	if (err) {
 797		mlx5_core_err(dev, "handle_hca_cap_roce failed\n");
 798		goto out;
 799	}
 800
 801	memset(set_ctx, 0, set_sz);
 802	err = handle_hca_cap_2(dev, set_ctx);
 803	if (err) {
 804		mlx5_core_err(dev, "handle_hca_cap_2 failed\n");
 805		goto out;
 806	}
 807
 808	memset(set_ctx, 0, set_sz);
 809	err = handle_hca_cap_port_selection(dev, set_ctx);
 810	if (err) {
 811		mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n");
 812		goto out;
 813	}
 814
 815out:
 816	kfree(set_ctx);
 817	return err;
 818}
 819
 820static int set_hca_ctrl(struct mlx5_core_dev *dev)
 821{
 822	struct mlx5_reg_host_endianness he_in;
 823	struct mlx5_reg_host_endianness he_out;
 824	int err;
 825
 826	if (!mlx5_core_is_pf(dev))
 827		return 0;
 828
 829	memset(&he_in, 0, sizeof(he_in));
 830	he_in.he = MLX5_SET_HOST_ENDIANNESS;
 831	err = mlx5_core_access_reg(dev, &he_in,  sizeof(he_in),
 832					&he_out, sizeof(he_out),
 833					MLX5_REG_HOST_ENDIANNESS, 0, 1);
 834	return err;
 835}
 836
 837static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev)
 838{
 839	int ret = 0;
 840
 841	/* Disable local_lb by default */
 842	if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH)
 843		ret = mlx5_nic_vport_update_local_lb(dev, false);
 844
 845	return ret;
 846}
 847
 848int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
 849{
 850	u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {};
 851
 852	MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
 853	MLX5_SET(enable_hca_in, in, function_id, func_id);
 854	MLX5_SET(enable_hca_in, in, embedded_cpu_function,
 855		 dev->caps.embedded_cpu);
 856	return mlx5_cmd_exec_in(dev, enable_hca, in);
 857}
 858
 859int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id)
 860{
 861	u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {};
 862
 863	MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
 864	MLX5_SET(disable_hca_in, in, function_id, func_id);
 865	MLX5_SET(enable_hca_in, in, embedded_cpu_function,
 866		 dev->caps.embedded_cpu);
 867	return mlx5_cmd_exec_in(dev, disable_hca, in);
 868}
 869
 870static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
 871{
 872	u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {};
 873	u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {};
 874	u32 sup_issi;
 875	int err;
 876
 877	MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI);
 878	err = mlx5_cmd_exec_inout(dev, query_issi, query_in, query_out);
 879	if (err) {
 880		u32 syndrome = MLX5_GET(query_issi_out, query_out, syndrome);
 881		u8 status = MLX5_GET(query_issi_out, query_out, status);
 882
 883		if (!status || syndrome == MLX5_DRIVER_SYND) {
 884			mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n",
 885				      err, status, syndrome);
 886			return err;
 887		}
 888
 889		mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n");
 890		dev->issi = 0;
 891		return 0;
 892	}
 893
 894	sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0);
 895
 896	if (sup_issi & (1 << 1)) {
 897		u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {};
 898
 899		MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI);
 900		MLX5_SET(set_issi_in, set_in, current_issi, 1);
 901		err = mlx5_cmd_exec_in(dev, set_issi, set_in);
 902		if (err) {
 903			mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n",
 904				      err);
 905			return err;
 906		}
 907
 908		dev->issi = 1;
 909
 910		return 0;
 911	} else if (sup_issi & (1 << 0) || !sup_issi) {
 912		return 0;
 913	}
 914
 915	return -EOPNOTSUPP;
 916}
 917
 918static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev,
 919			 const struct pci_device_id *id)
 920{
 921	int err = 0;
 922
 923	mutex_init(&dev->pci_status_mutex);
 924	pci_set_drvdata(dev->pdev, dev);
 925
 926	dev->bar_addr = pci_resource_start(pdev, 0);
 927
 928	err = mlx5_pci_enable_device(dev);
 929	if (err) {
 930		mlx5_core_err(dev, "Cannot enable PCI device, aborting\n");
 931		return err;
 932	}
 933
 934	err = request_bar(pdev);
 935	if (err) {
 936		mlx5_core_err(dev, "error requesting BARs, aborting\n");
 937		goto err_disable;
 938	}
 939
 940	pci_set_master(pdev);
 941
 942	err = set_dma_caps(pdev);
 943	if (err) {
 944		mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n");
 945		goto err_clr_master;
 946	}
 947
 948	if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
 949	    pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
 950	    pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128))
 951		mlx5_core_dbg(dev, "Enabling pci atomics failed\n");
 952
 953	dev->iseg_base = dev->bar_addr;
 954	dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
 955	if (!dev->iseg) {
 956		err = -ENOMEM;
 957		mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n");
 958		goto err_clr_master;
 959	}
 960
 961	mlx5_pci_vsc_init(dev);
 962
 963	pci_enable_ptm(pdev, NULL);
 964
 965	return 0;
 966
 967err_clr_master:
 968	release_bar(dev->pdev);
 969err_disable:
 970	mlx5_pci_disable_device(dev);
 971	return err;
 972}
 973
 974static void mlx5_pci_close(struct mlx5_core_dev *dev)
 975{
 976	/* health work might still be active, and it needs pci bar in
 977	 * order to know the NIC state. Therefore, drain the health WQ
 978	 * before removing the pci bars
 979	 */
 980	mlx5_drain_health_wq(dev);
 981	pci_disable_ptm(dev->pdev);
 982	iounmap(dev->iseg);
 983	release_bar(dev->pdev);
 984	mlx5_pci_disable_device(dev);
 985}
 986
 987static int mlx5_init_once(struct mlx5_core_dev *dev)
 988{
 989	int err;
 990
 991	dev->priv.devc = mlx5_devcom_register_device(dev);
 992	if (!dev->priv.devc)
 993		mlx5_core_warn(dev, "failed to register devcom device\n");
 994
 995	err = mlx5_query_board_id(dev);
 996	if (err) {
 997		mlx5_core_err(dev, "query board id failed\n");
 998		goto err_devcom;
 999	}
1000
1001	err = mlx5_irq_table_init(dev);
1002	if (err) {
1003		mlx5_core_err(dev, "failed to initialize irq table\n");
1004		goto err_devcom;
1005	}
1006
1007	err = mlx5_eq_table_init(dev);
1008	if (err) {
1009		mlx5_core_err(dev, "failed to initialize eq\n");
1010		goto err_irq_cleanup;
1011	}
1012
1013	err = mlx5_fw_reset_init(dev);
1014	if (err) {
1015		mlx5_core_err(dev, "failed to initialize fw reset events\n");
1016		goto err_eq_cleanup;
1017	}
1018
1019	mlx5_cq_debugfs_init(dev);
1020
1021	mlx5_init_reserved_gids(dev);
1022
1023	err = mlx5_init_clock(dev);
1024	if (err) {
1025		mlx5_core_err(dev, "failed to initialize hardware clock\n");
1026		goto err_tables_cleanup;
1027	}
1028
1029	dev->vxlan = mlx5_vxlan_create(dev);
1030	dev->geneve = mlx5_geneve_create(dev);
1031
1032	err = mlx5_init_rl_table(dev);
1033	if (err) {
1034		mlx5_core_err(dev, "Failed to init rate limiting\n");
1035		goto err_clock_cleanup;
1036	}
1037
1038	err = mlx5_mpfs_init(dev);
1039	if (err) {
1040		mlx5_core_err(dev, "Failed to init l2 table %d\n", err);
1041		goto err_rl_cleanup;
1042	}
1043
1044	err = mlx5_sriov_init(dev);
1045	if (err) {
1046		mlx5_core_err(dev, "Failed to init sriov %d\n", err);
1047		goto err_mpfs_cleanup;
1048	}
1049
1050	err = mlx5_eswitch_init(dev);
1051	if (err) {
1052		mlx5_core_err(dev, "Failed to init eswitch %d\n", err);
1053		goto err_sriov_cleanup;
1054	}
1055
1056	err = mlx5_fpga_init(dev);
1057	if (err) {
1058		mlx5_core_err(dev, "Failed to init fpga device %d\n", err);
1059		goto err_eswitch_cleanup;
1060	}
1061
1062	err = mlx5_vhca_event_init(dev);
1063	if (err) {
1064		mlx5_core_err(dev, "Failed to init vhca event notifier %d\n", err);
1065		goto err_fpga_cleanup;
1066	}
1067
1068	err = mlx5_sf_hw_table_init(dev);
1069	if (err) {
1070		mlx5_core_err(dev, "Failed to init SF HW table %d\n", err);
1071		goto err_sf_hw_table_cleanup;
1072	}
1073
1074	err = mlx5_sf_table_init(dev);
1075	if (err) {
1076		mlx5_core_err(dev, "Failed to init SF table %d\n", err);
1077		goto err_sf_table_cleanup;
1078	}
1079
1080	err = mlx5_fs_core_alloc(dev);
1081	if (err) {
1082		mlx5_core_err(dev, "Failed to alloc flow steering\n");
1083		goto err_fs;
1084	}
1085
1086	dev->dm = mlx5_dm_create(dev);
1087	dev->st = mlx5_st_create(dev);
1088	dev->tracer = mlx5_fw_tracer_create(dev);
1089	dev->hv_vhca = mlx5_hv_vhca_create(dev);
1090	dev->rsc_dump = mlx5_rsc_dump_create(dev);
1091
1092	return 0;
1093
1094err_fs:
1095	mlx5_sf_table_cleanup(dev);
1096err_sf_table_cleanup:
1097	mlx5_sf_hw_table_cleanup(dev);
1098err_sf_hw_table_cleanup:
1099	mlx5_vhca_event_cleanup(dev);
1100err_fpga_cleanup:
1101	mlx5_fpga_cleanup(dev);
1102err_eswitch_cleanup:
1103	mlx5_eswitch_cleanup(dev->priv.eswitch);
1104err_sriov_cleanup:
1105	mlx5_sriov_cleanup(dev);
1106err_mpfs_cleanup:
1107	mlx5_mpfs_cleanup(dev);
1108err_rl_cleanup:
1109	mlx5_cleanup_rl_table(dev);
1110err_clock_cleanup:
1111	mlx5_geneve_destroy(dev->geneve);
1112	mlx5_vxlan_destroy(dev->vxlan);
1113	mlx5_cleanup_clock(dev);
1114err_tables_cleanup:
1115	mlx5_cleanup_reserved_gids(dev);
1116	mlx5_cq_debugfs_cleanup(dev);
1117	mlx5_fw_reset_cleanup(dev);
1118err_eq_cleanup:
1119	mlx5_eq_table_cleanup(dev);
1120err_irq_cleanup:
1121	mlx5_irq_table_cleanup(dev);
1122err_devcom:
1123	mlx5_devcom_unregister_device(dev->priv.devc);
1124
1125	return err;
1126}
1127
1128static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
1129{
1130	mlx5_rsc_dump_destroy(dev);
1131	mlx5_hv_vhca_destroy(dev->hv_vhca);
1132	mlx5_fw_tracer_destroy(dev->tracer);
1133	mlx5_st_destroy(dev);
1134	mlx5_dm_cleanup(dev);
1135	mlx5_fs_core_free(dev);
1136	mlx5_sf_table_cleanup(dev);
1137	mlx5_sf_hw_table_cleanup(dev);
1138	mlx5_vhca_event_cleanup(dev);
1139	mlx5_fpga_cleanup(dev);
1140	mlx5_eswitch_cleanup(dev->priv.eswitch);
1141	mlx5_sriov_cleanup(dev);
1142	mlx5_mpfs_cleanup(dev);
1143	mlx5_cleanup_rl_table(dev);
1144	mlx5_geneve_destroy(dev->geneve);
1145	mlx5_vxlan_destroy(dev->vxlan);
1146	mlx5_cleanup_clock(dev);
1147	mlx5_cleanup_reserved_gids(dev);
1148	mlx5_cq_debugfs_cleanup(dev);
1149	mlx5_fw_reset_cleanup(dev);
1150	mlx5_eq_table_cleanup(dev);
1151	mlx5_irq_table_cleanup(dev);
1152	mlx5_devcom_unregister_device(dev->priv.devc);
1153}
1154
1155static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeout)
1156{
1157	int err;
1158
1159	mlx5_core_info(dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev),
1160		       fw_rev_min(dev), fw_rev_sub(dev));
1161
1162	/* Only PFs hold the relevant PCIe information for this query */
1163	if (mlx5_core_is_pf(dev))
1164		pcie_print_link_status(dev->pdev);
1165
1166	/* wait for firmware to accept initialization segments configurations
1167	 */
1168	err = wait_fw_init(dev, timeout,
1169			   mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL),
1170			   "pre-initializing");
1171	if (err)
1172		return err;
1173
1174	err = mlx5_cmd_enable(dev);
1175	if (err) {
1176		mlx5_core_err(dev, "Failed initializing command interface, aborting\n");
1177		return err;
1178	}
1179
1180	mlx5_tout_query_iseg(dev);
1181
1182	err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), 0, "initializing");
1183	if (err)
1184		goto err_cmd_cleanup;
1185
1186	dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev);
1187	mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP);
1188
1189	err = mlx5_core_enable_hca(dev, 0);
1190	if (err) {
1191		mlx5_core_err(dev, "enable hca failed\n");
1192		goto err_cmd_cleanup;
1193	}
1194
1195	mlx5_start_health_poll(dev);
1196
1197	err = mlx5_core_set_issi(dev);
1198	if (err) {
1199		mlx5_core_err(dev, "failed to set issi\n");
1200		goto stop_health_poll;
1201	}
1202
1203	err = mlx5_satisfy_startup_pages(dev, 1);
1204	if (err) {
1205		mlx5_core_err(dev, "failed to allocate boot pages\n");
1206		goto stop_health_poll;
1207	}
1208
1209	err = mlx5_tout_query_dtor(dev);
1210	if (err) {
1211		mlx5_core_err(dev, "failed to read dtor\n");
1212		goto reclaim_boot_pages;
1213	}
1214
1215	return 0;
1216
1217reclaim_boot_pages:
1218	mlx5_reclaim_startup_pages(dev);
1219stop_health_poll:
1220	mlx5_stop_health_poll(dev, boot);
1221	mlx5_core_disable_hca(dev, 0);
1222err_cmd_cleanup:
1223	mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
1224	mlx5_cmd_disable(dev);
1225
1226	return err;
1227}
1228
1229static void mlx5_function_disable(struct mlx5_core_dev *dev, bool boot)
1230{
1231	mlx5_reclaim_startup_pages(dev);
1232	mlx5_stop_health_poll(dev, boot);
1233	mlx5_core_disable_hca(dev, 0);
1234	mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
1235	mlx5_cmd_disable(dev);
1236}
1237
1238static int mlx5_function_open(struct mlx5_core_dev *dev)
1239{
1240	int err;
1241
1242	err = set_hca_ctrl(dev);
1243	if (err) {
1244		mlx5_core_err(dev, "set_hca_ctrl failed\n");
1245		return err;
1246	}
1247
1248	err = set_hca_cap(dev);
1249	if (err) {
1250		mlx5_core_err(dev, "set_hca_cap failed\n");
1251		return err;
1252	}
1253
1254	err = mlx5_satisfy_startup_pages(dev, 0);
1255	if (err) {
1256		mlx5_core_err(dev, "failed to allocate init pages\n");
1257		return err;
1258	}
1259
1260	err = mlx5_cmd_init_hca(dev, sw_owner_id);
1261	if (err) {
1262		mlx5_core_err(dev, "init hca failed\n");
1263		return err;
1264	}
1265
1266	mlx5_set_driver_version(dev);
1267
1268	err = mlx5_query_hca_caps(dev);
1269	if (err) {
1270		mlx5_core_err(dev, "query hca failed\n");
1271		return err;
1272	}
1273	mlx5_start_health_fw_log_up(dev);
1274	return 0;
1275}
1276
1277static int mlx5_function_close(struct mlx5_core_dev *dev)
1278{
1279	int err;
1280
1281	err = mlx5_cmd_teardown_hca(dev);
1282	if (err) {
1283		mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n");
1284		return err;
1285	}
1286
1287	return 0;
1288}
1289
1290static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout)
1291{
1292	int err;
1293
1294	err = mlx5_function_enable(dev, boot, timeout);
1295	if (err)
1296		return err;
1297
1298	err = mlx5_function_open(dev);
1299	if (err)
1300		mlx5_function_disable(dev, boot);
1301	return err;
1302}
1303
1304static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
1305{
1306	int err = mlx5_function_close(dev);
1307
1308	if (!err)
1309		mlx5_function_disable(dev, boot);
1310	else
1311		mlx5_stop_health_poll(dev, boot);
1312
1313	return err;
1314}
1315
1316static int mlx5_load(struct mlx5_core_dev *dev)
1317{
1318	int err;
1319
1320	err = mlx5_alloc_bfreg(dev, &dev->priv.bfreg, false, false);
1321	if (err) {
1322		mlx5_core_err(dev, "Failed allocating bfreg, %d\n", err);
1323		return err;
1324	}
1325
1326	mlx5_events_start(dev);
1327	mlx5_pagealloc_start(dev);
1328
1329	err = mlx5_irq_table_create(dev);
1330	if (err) {
1331		mlx5_core_err(dev, "Failed to alloc IRQs\n");
1332		goto err_irq_table;
1333	}
1334
1335	err = mlx5_eq_table_create(dev);
1336	if (err) {
1337		mlx5_core_err(dev, "Failed to create EQs\n");
1338		goto err_eq_table;
1339	}
1340
1341	mlx5_clock_load(dev);
1342
1343	err = mlx5_fw_tracer_init(dev->tracer);
1344	if (err) {
1345		mlx5_core_err(dev, "Failed to init FW tracer %d\n", err);
1346		mlx5_fw_tracer_destroy(dev->tracer);
1347		dev->tracer = NULL;
1348	}
1349
1350	mlx5_fw_reset_events_start(dev);
1351	mlx5_hv_vhca_init(dev->hv_vhca);
1352
1353	err = mlx5_rsc_dump_init(dev);
1354	if (err) {
1355		mlx5_core_err(dev, "Failed to init Resource dump %d\n", err);
1356		mlx5_rsc_dump_destroy(dev);
1357		dev->rsc_dump = NULL;
1358	}
1359
1360	err = mlx5_fpga_device_start(dev);
1361	if (err) {
1362		mlx5_core_err(dev, "fpga device start failed %d\n", err);
1363		goto err_fpga_start;
1364	}
1365
1366	err = mlx5_fs_core_init(dev);
1367	if (err) {
1368		mlx5_core_err(dev, "Failed to init flow steering\n");
1369		goto err_fs;
1370	}
1371
1372	err = mlx5_core_set_hca_defaults(dev);
1373	if (err) {
1374		mlx5_core_err(dev, "Failed to set hca defaults\n");
1375		goto err_set_hca;
1376	}
1377
1378	mlx5_vhca_event_start(dev);
1379
1380	err = mlx5_ec_init(dev);
1381	if (err) {
1382		mlx5_core_err(dev, "Failed to init embedded CPU\n");
1383		goto err_ec;
1384	}
1385
1386	mlx5_lag_add_mdev(dev);
1387	err = mlx5_sriov_attach(dev);
1388	if (err) {
1389		mlx5_core_err(dev, "sriov init failed %d\n", err);
1390		goto err_sriov;
1391	}
1392
1393	mlx5_sf_dev_table_create(dev);
1394
1395	err = mlx5_devlink_traps_register(priv_to_devlink(dev));
1396	if (err)
1397		goto err_traps_reg;
1398
1399	return 0;
1400
1401err_traps_reg:
1402	mlx5_sf_dev_table_destroy(dev);
1403	mlx5_sriov_detach(dev);
1404err_sriov:
1405	mlx5_lag_remove_mdev(dev);
1406	mlx5_ec_cleanup(dev);
1407err_ec:
1408	mlx5_vhca_event_stop(dev);
1409err_set_hca:
1410	mlx5_fs_core_cleanup(dev);
1411err_fs:
1412	mlx5_fpga_device_stop(dev);
1413err_fpga_start:
1414	mlx5_rsc_dump_cleanup(dev);
1415	mlx5_hv_vhca_cleanup(dev->hv_vhca);
1416	mlx5_fw_reset_events_stop(dev);
1417	mlx5_fw_tracer_cleanup(dev->tracer);
1418	mlx5_clock_unload(dev);
1419	mlx5_eq_table_destroy(dev);
1420err_eq_table:
1421	mlx5_irq_table_destroy(dev);
1422err_irq_table:
1423	mlx5_pagealloc_stop(dev);
1424	mlx5_events_stop(dev);
1425	mlx5_free_bfreg(dev, &dev->priv.bfreg);
1426	return err;
1427}
1428
1429static void mlx5_unload(struct mlx5_core_dev *dev)
1430{
1431	mlx5_eswitch_disable(dev->priv.eswitch);
1432	mlx5_devlink_traps_unregister(priv_to_devlink(dev));
1433	mlx5_vhca_event_stop(dev);
1434	mlx5_sf_dev_table_destroy(dev);
1435	mlx5_sriov_detach(dev);
1436	mlx5_lag_remove_mdev(dev);
1437	mlx5_ec_cleanup(dev);
1438	mlx5_sf_hw_table_destroy(dev);
1439	mlx5_fs_core_cleanup(dev);
1440	mlx5_fpga_device_stop(dev);
1441	mlx5_rsc_dump_cleanup(dev);
1442	mlx5_hv_vhca_cleanup(dev->hv_vhca);
1443	mlx5_fw_reset_events_stop(dev);
1444	mlx5_fw_tracer_cleanup(dev->tracer);
1445	mlx5_clock_unload(dev);
1446	mlx5_eq_table_destroy(dev);
1447	mlx5_irq_table_destroy(dev);
1448	mlx5_pagealloc_stop(dev);
1449	mlx5_events_stop(dev);
1450	mlx5_free_bfreg(dev, &dev->priv.bfreg);
1451}
1452
1453int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev)
1454{
1455	bool light_probe = mlx5_dev_is_lightweight(dev);
1456	int err = 0;
1457
1458	mutex_lock(&dev->intf_state_mutex);
1459	dev->state = MLX5_DEVICE_STATE_UP;
1460
1461	err = mlx5_function_setup(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
1462	if (err)
1463		goto err_function;
1464
1465	err = mlx5_init_once(dev);
1466	if (err) {
1467		mlx5_core_err(dev, "sw objs init failed\n");
1468		goto function_teardown;
1469	}
1470
1471	/* In case of light_probe, mlx5_devlink is already registered.
1472	 * Hence, don't register devlink again.
1473	 */
1474	if (!light_probe) {
1475		err = mlx5_devlink_params_register(priv_to_devlink(dev));
1476		if (err)
1477			goto err_devlink_params_reg;
1478	}
1479
1480	err = mlx5_load(dev);
1481	if (err)
1482		goto err_load;
1483
1484	set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1485
1486	err = mlx5_register_device(dev);
1487	if (err)
1488		goto err_register;
1489
1490	err = mlx5_crdump_enable(dev);
1491	if (err)
1492		mlx5_core_err(dev, "mlx5_crdump_enable failed with error code %d\n", err);
1493
1494	err = mlx5_hwmon_dev_register(dev);
1495	if (err)
1496		mlx5_core_err(dev, "mlx5_hwmon_dev_register failed with error code %d\n", err);
1497
1498	mutex_unlock(&dev->intf_state_mutex);
1499	return 0;
1500
1501err_register:
1502	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1503	mlx5_unload(dev);
1504err_load:
1505	if (!light_probe)
1506		mlx5_devlink_params_unregister(priv_to_devlink(dev));
1507err_devlink_params_reg:
1508	mlx5_cleanup_once(dev);
1509function_teardown:
1510	mlx5_function_teardown(dev, true);
1511err_function:
1512	dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1513	mutex_unlock(&dev->intf_state_mutex);
1514	return err;
1515}
1516
1517int mlx5_init_one(struct mlx5_core_dev *dev)
1518{
1519	struct devlink *devlink = priv_to_devlink(dev);
1520	int err;
1521
1522	devl_lock(devlink);
1523	devl_register(devlink);
1524	err = mlx5_init_one_devl_locked(dev);
1525	if (err)
1526		devl_unregister(devlink);
1527	devl_unlock(devlink);
1528	return err;
1529}
1530
1531void mlx5_uninit_one(struct mlx5_core_dev *dev)
1532{
1533	struct devlink *devlink = priv_to_devlink(dev);
1534
1535	devl_lock(devlink);
1536	mutex_lock(&dev->intf_state_mutex);
1537
1538	mlx5_hwmon_dev_unregister(dev);
1539	mlx5_crdump_disable(dev);
1540	mlx5_unregister_device(dev);
1541
1542	if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1543		mlx5_core_warn(dev, "%s: interface is down, NOP\n",
1544			       __func__);
1545		mlx5_devlink_params_unregister(priv_to_devlink(dev));
1546		mlx5_cleanup_once(dev);
1547		goto out;
1548	}
1549
1550	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1551	mlx5_unload(dev);
1552	mlx5_devlink_params_unregister(priv_to_devlink(dev));
1553	mlx5_cleanup_once(dev);
1554	mlx5_function_teardown(dev, true);
1555out:
1556	mutex_unlock(&dev->intf_state_mutex);
1557	devl_unregister(devlink);
1558	devl_unlock(devlink);
1559}
1560
1561int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery)
1562{
1563	int err = 0;
1564	u64 timeout;
1565
1566	devl_assert_locked(priv_to_devlink(dev));
1567	mutex_lock(&dev->intf_state_mutex);
1568	if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1569		mlx5_core_warn(dev, "interface is up, NOP\n");
1570		goto out;
1571	}
1572	/* remove any previous indication of internal error */
1573	dev->state = MLX5_DEVICE_STATE_UP;
1574
1575	if (recovery)
1576		timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT);
1577	else
1578		timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT);
1579	err = mlx5_function_setup(dev, false, timeout);
1580	if (err)
1581		goto err_function;
1582
1583	err = mlx5_load(dev);
1584	if (err)
1585		goto err_load;
1586
1587	set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1588
1589	err = mlx5_attach_device(dev);
1590	if (err)
1591		goto err_attach;
1592
1593	mutex_unlock(&dev->intf_state_mutex);
1594	return 0;
1595
1596err_attach:
1597	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1598	mlx5_unload(dev);
1599err_load:
1600	mlx5_function_teardown(dev, false);
1601err_function:
1602	dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1603out:
1604	mutex_unlock(&dev->intf_state_mutex);
1605	return err;
1606}
1607
1608int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery)
1609{
1610	struct devlink *devlink = priv_to_devlink(dev);
1611	int ret;
1612
1613	devl_lock(devlink);
1614	ret = mlx5_load_one_devl_locked(dev, recovery);
1615	devl_unlock(devlink);
1616	return ret;
1617}
1618
1619void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend)
1620{
1621	devl_assert_locked(priv_to_devlink(dev));
1622	mutex_lock(&dev->intf_state_mutex);
1623
1624	mlx5_detach_device(dev, suspend);
1625
1626	if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1627		mlx5_core_warn(dev, "%s: interface is down, NOP\n",
1628			       __func__);
1629		goto out;
1630	}
1631
1632	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1633	mlx5_unload(dev);
1634	mlx5_function_teardown(dev, false);
1635out:
1636	mutex_unlock(&dev->intf_state_mutex);
1637}
1638
1639void mlx5_unload_one(struct mlx5_core_dev *dev, bool suspend)
1640{
1641	struct devlink *devlink = priv_to_devlink(dev);
1642
1643	devl_lock(devlink);
1644	mlx5_unload_one_devl_locked(dev, suspend);
1645	devl_unlock(devlink);
1646}
1647
1648/* In case of light probe, we don't need a full query of hca_caps, but only the bellow caps.
1649 * A full query of hca_caps will be done when the device will reload.
1650 */
1651static int mlx5_query_hca_caps_light(struct mlx5_core_dev *dev)
1652{
1653	int err;
1654
1655	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
1656	if (err)
1657		return err;
1658
1659	if (MLX5_CAP_GEN(dev, eth_net_offloads)) {
1660		err = mlx5_core_get_caps_mode(dev, MLX5_CAP_ETHERNET_OFFLOADS,
1661					      HCA_CAP_OPMOD_GET_CUR);
1662		if (err)
1663			return err;
1664	}
1665
1666	if (MLX5_CAP_GEN(dev, nic_flow_table) ||
1667	    MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) {
1668		err = mlx5_core_get_caps_mode(dev, MLX5_CAP_FLOW_TABLE,
1669					      HCA_CAP_OPMOD_GET_CUR);
1670		if (err)
1671			return err;
1672	}
1673
1674	if (MLX5_CAP_GEN_64(dev, general_obj_types) &
1675		MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
1676		err = mlx5_core_get_caps_mode(dev, MLX5_CAP_VDPA_EMULATION,
1677					      HCA_CAP_OPMOD_GET_CUR);
1678		if (err)
1679			return err;
1680	}
1681
1682	return 0;
1683}
1684
1685int mlx5_init_one_light(struct mlx5_core_dev *dev)
1686{
1687	struct devlink *devlink = priv_to_devlink(dev);
1688	int err;
1689
1690	devl_lock(devlink);
1691	devl_register(devlink);
1692	dev->state = MLX5_DEVICE_STATE_UP;
1693	err = mlx5_function_enable(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
1694	if (err) {
1695		mlx5_core_warn(dev, "mlx5_function_enable err=%d\n", err);
1696		goto out;
1697	}
1698
1699	err = mlx5_query_hca_caps_light(dev);
1700	if (err) {
1701		mlx5_core_warn(dev, "mlx5_query_hca_caps_light err=%d\n", err);
1702		goto query_hca_caps_err;
1703	}
1704
1705	err = mlx5_devlink_params_register(priv_to_devlink(dev));
1706	if (err) {
1707		mlx5_core_warn(dev, "mlx5_devlink_param_reg err = %d\n", err);
1708		goto query_hca_caps_err;
1709	}
1710
1711	devl_unlock(devlink);
1712	return 0;
1713
1714query_hca_caps_err:
1715	mlx5_function_disable(dev, true);
1716out:
1717	dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1718	devl_unregister(devlink);
1719	devl_unlock(devlink);
1720	return err;
1721}
1722
1723void mlx5_uninit_one_light(struct mlx5_core_dev *dev)
1724{
1725	struct devlink *devlink = priv_to_devlink(dev);
1726
1727	devl_lock(devlink);
1728	mlx5_devlink_params_unregister(priv_to_devlink(dev));
1729	devl_unregister(devlink);
1730	devl_unlock(devlink);
1731	if (dev->state != MLX5_DEVICE_STATE_UP)
1732		return;
1733	mlx5_function_disable(dev, true);
1734}
1735
1736/* xxx_light() function are used in order to configure the device without full
1737 * init (light init). e.g.: There isn't a point in reload a device to light state.
1738 * Hence, mlx5_load_one_light() isn't needed.
1739 */
1740
1741void mlx5_unload_one_light(struct mlx5_core_dev *dev)
1742{
1743	if (dev->state != MLX5_DEVICE_STATE_UP)
1744		return;
1745	mlx5_function_disable(dev, false);
1746	dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1747}
1748
1749static const int types[] = {
1750	MLX5_CAP_GENERAL,
1751	MLX5_CAP_GENERAL_2,
1752	MLX5_CAP_ETHERNET_OFFLOADS,
1753	MLX5_CAP_IPOIB_ENHANCED_OFFLOADS,
1754	MLX5_CAP_ODP,
1755	MLX5_CAP_ATOMIC,
1756	MLX5_CAP_ROCE,
1757	MLX5_CAP_IPOIB_OFFLOADS,
1758	MLX5_CAP_FLOW_TABLE,
1759	MLX5_CAP_ESWITCH_FLOW_TABLE,
1760	MLX5_CAP_ESWITCH,
1761	MLX5_CAP_QOS,
1762	MLX5_CAP_DEBUG,
1763	MLX5_CAP_DEV_MEM,
1764	MLX5_CAP_DEV_EVENT,
1765	MLX5_CAP_TLS,
1766	MLX5_CAP_VDPA_EMULATION,
1767	MLX5_CAP_IPSEC,
1768	MLX5_CAP_PORT_SELECTION,
1769	MLX5_CAP_PSP,
1770	MLX5_CAP_MACSEC,
1771	MLX5_CAP_ADV_VIRTUALIZATION,
1772	MLX5_CAP_CRYPTO,
1773	MLX5_CAP_SHAMPO,
1774	MLX5_CAP_ADV_RDMA,
1775};
1776
1777static void mlx5_hca_caps_free(struct mlx5_core_dev *dev)
1778{
1779	int type;
1780	int i;
1781
1782	for (i = 0; i < ARRAY_SIZE(types); i++) {
1783		type = types[i];
1784		kfree(dev->caps.hca[type]);
1785	}
1786}
1787
1788static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev)
1789{
1790	struct mlx5_hca_cap *cap;
1791	int type;
1792	int i;
1793
1794	for (i = 0; i < ARRAY_SIZE(types); i++) {
1795		cap = kzalloc(sizeof(*cap), GFP_KERNEL);
1796		if (!cap)
1797			goto err;
1798		type = types[i];
1799		dev->caps.hca[type] = cap;
1800	}
1801
1802	return 0;
1803
1804err:
1805	mlx5_hca_caps_free(dev);
1806	return -ENOMEM;
1807}
1808
1809static int mlx5_notifiers_init(struct mlx5_core_dev *dev)
1810{
1811	int err;
1812
1813	err = mlx5_events_init(dev);
1814	if (err) {
1815		mlx5_core_err(dev, "failed to initialize events\n");
1816		return err;
1817	}
1818
1819	BLOCKING_INIT_NOTIFIER_HEAD(&dev->priv.esw_n_head);
1820	mlx5_vhca_state_notifier_init(dev);
1821
1822	err = mlx5_sf_hw_notifier_init(dev);
1823	if (err)
1824		goto err_sf_hw_notifier;
1825
1826	err = mlx5_sf_notifiers_init(dev);
1827	if (err)
1828		goto err_sf_notifiers;
1829
1830	err = mlx5_sf_dev_notifier_init(dev);
1831	if (err)
1832		goto err_sf_dev_notifier;
1833
1834	return 0;
1835
1836err_sf_dev_notifier:
1837	mlx5_sf_notifiers_cleanup(dev);
1838err_sf_notifiers:
1839	mlx5_sf_hw_notifier_cleanup(dev);
1840err_sf_hw_notifier:
1841	mlx5_events_cleanup(dev);
1842	return err;
1843}
1844
1845static void mlx5_notifiers_cleanup(struct mlx5_core_dev *dev)
1846{
1847	mlx5_sf_dev_notifier_cleanup(dev);
1848	mlx5_sf_notifiers_cleanup(dev);
1849	mlx5_sf_hw_notifier_cleanup(dev);
1850	mlx5_events_cleanup(dev);
1851}
1852
1853int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
1854{
1855	struct mlx5_priv *priv = &dev->priv;
1856	int err;
1857
1858	memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile));
1859	lockdep_register_key(&dev->lock_key);
1860	mutex_init(&dev->intf_state_mutex);
1861	lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key);
1862	mutex_init(&dev->mlx5e_res.uplink_netdev_lock);
1863	mutex_init(&dev->wc_state_lock);
1864
1865	mutex_init(&priv->bfregs.reg_head.lock);
1866	mutex_init(&priv->bfregs.wc_head.lock);
1867	INIT_LIST_HEAD(&priv->bfregs.reg_head.list);
1868	INIT_LIST_HEAD(&priv->bfregs.wc_head.list);
1869
1870	mutex_init(&priv->alloc_mutex);
1871	mutex_init(&priv->pgdir_mutex);
1872	INIT_LIST_HEAD(&priv->pgdir_list);
1873
1874	priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev));
1875	priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device),
1876						mlx5_debugfs_root);
1877
1878	INIT_LIST_HEAD(&priv->traps);
1879
1880	err = mlx5_cmd_init(dev);
1881	if (err) {
1882		mlx5_core_err(dev, "Failed initializing cmdif SW structs, aborting\n");
1883		goto err_cmd_init;
1884	}
1885
1886	err = mlx5_tout_init(dev);
1887	if (err) {
1888		mlx5_core_err(dev, "Failed initializing timeouts, aborting\n");
1889		goto err_timeout_init;
1890	}
1891
1892	err = mlx5_health_init(dev);
1893	if (err)
1894		goto err_health_init;
1895
1896	err = mlx5_pagealloc_init(dev);
1897	if (err)
1898		goto err_pagealloc_init;
1899
1900	err = mlx5_adev_init(dev);
1901	if (err)
1902		goto err_adev_init;
1903
1904	err = mlx5_hca_caps_alloc(dev);
1905	if (err)
1906		goto err_hca_caps;
1907
1908	err = mlx5_notifiers_init(dev);
1909	if (err)
1910		goto err_hca_caps;
1911
1912	/* The conjunction of sw_vhca_id with sw_owner_id will be a global
1913	 * unique id per function which uses mlx5_core.
1914	 * Those values are supplied to FW as part of the init HCA command to
1915	 * be used by both driver and FW when it's applicable.
1916	 */
1917	dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1,
1918					       MAX_SW_VHCA_ID,
1919					       GFP_KERNEL);
1920	if (dev->priv.sw_vhca_id < 0)
1921		mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n",
1922			      dev->priv.sw_vhca_id);
1923
1924	return 0;
1925
1926err_hca_caps:
1927	mlx5_adev_cleanup(dev);
1928err_adev_init:
1929	mlx5_pagealloc_cleanup(dev);
1930err_pagealloc_init:
1931	mlx5_health_cleanup(dev);
1932err_health_init:
1933	mlx5_tout_cleanup(dev);
1934err_timeout_init:
1935	mlx5_cmd_cleanup(dev);
1936err_cmd_init:
1937	debugfs_remove(dev->priv.dbg.dbg_root);
1938	mutex_destroy(&priv->pgdir_mutex);
1939	mutex_destroy(&priv->alloc_mutex);
1940	mutex_destroy(&priv->bfregs.wc_head.lock);
1941	mutex_destroy(&priv->bfregs.reg_head.lock);
1942	mutex_destroy(&dev->intf_state_mutex);
1943	lockdep_unregister_key(&dev->lock_key);
1944	return err;
1945}
1946
1947void mlx5_mdev_uninit(struct mlx5_core_dev *dev)
1948{
1949	struct mlx5_priv *priv = &dev->priv;
1950
1951	if (priv->sw_vhca_id > 0)
1952		ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id);
1953
1954	mlx5_notifiers_cleanup(dev);
1955	mlx5_hca_caps_free(dev);
1956	mlx5_adev_cleanup(dev);
1957	mlx5_pagealloc_cleanup(dev);
1958	mlx5_health_cleanup(dev);
1959	mlx5_tout_cleanup(dev);
1960	mlx5_cmd_cleanup(dev);
1961	debugfs_remove_recursive(dev->priv.dbg.dbg_root);
1962	mutex_destroy(&priv->pgdir_mutex);
1963	mutex_destroy(&priv->alloc_mutex);
1964	mutex_destroy(&priv->bfregs.wc_head.lock);
1965	mutex_destroy(&priv->bfregs.reg_head.lock);
1966	mutex_destroy(&dev->wc_state_lock);
1967	mutex_destroy(&dev->mlx5e_res.uplink_netdev_lock);
1968	mutex_destroy(&dev->intf_state_mutex);
1969	lockdep_unregister_key(&dev->lock_key);
1970}
1971
1972static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
1973{
1974	struct mlx5_core_dev *dev;
1975	struct devlink *devlink;
1976	int err;
1977
1978	devlink = mlx5_devlink_alloc(&pdev->dev);
1979	if (!devlink) {
1980		dev_err(&pdev->dev, "devlink alloc failed\n");
1981		return -ENOMEM;
1982	}
1983
1984	dev = devlink_priv(devlink);
1985	dev->device = &pdev->dev;
1986	dev->pdev = pdev;
1987
1988	dev->coredev_type = id->driver_data & MLX5_PCI_DEV_IS_VF ?
1989			 MLX5_COREDEV_VF : MLX5_COREDEV_PF;
1990
1991	dev->priv.adev_idx = mlx5_adev_idx_alloc();
1992	if (dev->priv.adev_idx < 0) {
1993		err = dev->priv.adev_idx;
1994		goto adev_init_err;
1995	}
1996
1997	err = mlx5_mdev_init(dev, prof_sel);
1998	if (err)
1999		goto mdev_init_err;
2000
2001	err = mlx5_pci_init(dev, pdev, id);
2002	if (err) {
2003		mlx5_core_err(dev, "mlx5_pci_init failed with error code %d\n",
2004			      err);
2005		goto pci_init_err;
2006	}
2007
2008	err = mlx5_init_one(dev);
2009	if (err) {
2010		mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n",
2011			      err);
2012		goto err_init_one;
2013	}
2014
2015	mlx5_vhca_debugfs_init(dev);
2016
2017	pci_save_state(pdev);
2018	return 0;
2019
2020err_init_one:
2021	mlx5_pci_close(dev);
2022pci_init_err:
2023	mlx5_mdev_uninit(dev);
2024mdev_init_err:
2025	mlx5_adev_idx_free(dev->priv.adev_idx);
2026adev_init_err:
2027	mlx5_devlink_free(devlink);
2028
2029	return err;
2030}
2031
2032static void remove_one(struct pci_dev *pdev)
2033{
2034	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
2035	struct devlink *devlink = priv_to_devlink(dev);
2036
2037	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
2038	mlx5_drain_fw_reset(dev);
2039	mlx5_drain_health_wq(dev);
2040	mlx5_sriov_disable(pdev, false);
2041	mlx5_uninit_one(dev);
2042	mlx5_pci_close(dev);
2043	mlx5_mdev_uninit(dev);
2044	mlx5_adev_idx_free(dev->priv.adev_idx);
2045	mlx5_devlink_free(devlink);
2046}
2047
2048#define mlx5_pci_trace(dev, fmt, ...) ({ \
2049	struct mlx5_core_dev *__dev = (dev); \
2050	mlx5_core_info(__dev, "%s Device state = %d health sensors: %d pci_status: %d. " fmt, \
2051		       __func__, __dev->state, mlx5_health_check_fatal_sensors(__dev), \
2052		       __dev->pci_status, ##__VA_ARGS__); \
2053})
2054
2055static const char *result2str(enum pci_ers_result result)
2056{
2057	return  result == PCI_ERS_RESULT_NEED_RESET ? "need reset" :
2058		result == PCI_ERS_RESULT_DISCONNECT ? "disconnect" :
2059		result == PCI_ERS_RESULT_RECOVERED  ? "recovered" :
2060		"unknown";
2061}
2062
2063static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
2064					      pci_channel_state_t state)
2065{
2066	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2067	enum pci_ers_result res;
2068
2069	mlx5_pci_trace(dev, "Enter, pci channel state = %d\n", state);
2070
2071	mlx5_enter_error_state(dev, false);
2072	mlx5_error_sw_reset(dev);
2073	mlx5_unload_one(dev, false);
2074	mlx5_drain_health_wq(dev);
2075	mlx5_pci_disable_device(dev);
2076
2077	res = state == pci_channel_io_perm_failure ?
2078		PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
2079
2080	mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, result = %d, %s\n",
2081		       __func__, dev->state, dev->pci_status, res, result2str(res));
2082	return res;
2083}
2084
2085/* wait for the device to show vital signs by waiting
2086 * for the health counter to start counting.
2087 */
2088static int wait_vital(struct pci_dev *pdev)
2089{
2090	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2091	struct mlx5_core_health *health = &dev->priv.health;
2092	const int niter = 100;
2093	u32 last_count = 0;
2094	u32 count;
2095	int i;
2096
2097	for (i = 0; i < niter; i++) {
2098		count = ioread32be(health->health_counter);
2099		if (count && count != 0xffffffff) {
2100			if (last_count && last_count != count) {
2101				mlx5_core_info(dev,
2102					       "wait vital counter value 0x%x after %d iterations\n",
2103					       count, i);
2104				return 0;
2105			}
2106			last_count = count;
2107		}
2108		msleep(50);
2109	}
2110
2111	return -ETIMEDOUT;
2112}
2113
2114static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
2115{
2116	enum pci_ers_result res = PCI_ERS_RESULT_DISCONNECT;
2117	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2118	int err;
2119
2120	mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Enter\n",
2121		       __func__, dev->state, dev->pci_status);
2122
2123	err = mlx5_pci_enable_device(dev);
2124	if (err) {
2125		mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n",
2126			      __func__, err);
2127		goto out;
2128	}
2129
2130	pci_set_master(pdev);
2131	pci_restore_state(pdev);
2132
2133	err = wait_vital(pdev);
2134	if (err) {
2135		mlx5_core_err(dev, "%s: wait vital failed with error code: %d\n",
2136			      __func__, err);
2137		goto out;
2138	}
2139
2140	res = PCI_ERS_RESULT_RECOVERED;
2141out:
2142	mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, err = %d, result = %d, %s\n",
2143		       __func__, dev->state, dev->pci_status, err, res, result2str(res));
2144	return res;
2145}
2146
2147static void mlx5_pci_resume(struct pci_dev *pdev)
2148{
2149	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2150	int err;
2151
2152	mlx5_pci_trace(dev, "Enter, loading driver..\n");
2153
2154	err = mlx5_load_one(dev, false);
2155
2156	if (!err)
2157		devlink_health_reporter_state_update(dev->priv.health.fw_fatal_reporter,
2158						     DEVLINK_HEALTH_REPORTER_STATE_HEALTHY);
2159
2160	mlx5_pci_trace(dev, "Done, err = %d, device %s\n", err,
2161		       !err ? "recovered" : "Failed");
2162}
2163
2164static const struct pci_error_handlers mlx5_err_handler = {
2165	.error_detected = mlx5_pci_err_detected,
2166	.slot_reset	= mlx5_pci_slot_reset,
2167	.resume		= mlx5_pci_resume
2168};
2169
2170static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
2171{
2172	bool fast_teardown = false, force_teardown = false;
2173	int ret = 1;
2174
2175	fast_teardown = MLX5_CAP_GEN(dev, fast_teardown);
2176	force_teardown = MLX5_CAP_GEN(dev, force_teardown);
2177
2178	mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown);
2179	mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown);
2180
2181	if (!fast_teardown && !force_teardown)
2182		return -EOPNOTSUPP;
2183
2184	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
2185		mlx5_core_dbg(dev, "Device in internal error state, giving up\n");
2186		return -EAGAIN;
2187	}
2188
2189	/* Panic tear down fw command will stop the PCI bus communication
2190	 * with the HCA, so the health poll is no longer needed.
2191	 */
2192	mlx5_stop_health_poll(dev, false);
2193
2194	ret = mlx5_cmd_fast_teardown_hca(dev);
2195	if (!ret)
2196		goto succeed;
2197
2198	ret = mlx5_cmd_force_teardown_hca(dev);
2199	if (!ret)
2200		goto succeed;
2201
2202	mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", ret);
2203	mlx5_start_health_poll(dev);
2204	return ret;
2205
2206succeed:
2207	mlx5_enter_error_state(dev, true);
2208
2209	/* Some platforms requiring freeing the IRQ's in the shutdown
2210	 * flow. If they aren't freed they can't be allocated after
2211	 * kexec. There is no need to cleanup the mlx5_core software
2212	 * contexts.
2213	 */
2214	mlx5_core_eq_free_irqs(dev);
2215
2216	return 0;
2217}
2218
2219static void shutdown(struct pci_dev *pdev)
2220{
2221	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
2222	int err;
2223
2224	mlx5_core_info(dev, "Shutdown was called\n");
2225	set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
2226	mlx5_drain_fw_reset(dev);
2227	mlx5_drain_health_wq(dev);
2228	err = mlx5_try_fast_unload(dev);
2229	if (err)
2230		mlx5_unload_one(dev, false);
2231	mlx5_pci_disable_device(dev);
2232}
2233
2234static int mlx5_suspend(struct pci_dev *pdev, pm_message_t state)
2235{
2236	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2237
2238	mlx5_unload_one(dev, true);
2239
2240	return 0;
2241}
2242
2243static int mlx5_resume(struct pci_dev *pdev)
2244{
2245	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2246
2247	return mlx5_load_one(dev, false);
2248}
2249
2250static const struct pci_device_id mlx5_core_pci_table[] = {
2251	{ PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) },
2252	{ PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF},	/* Connect-IB VF */
2253	{ PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) },
2254	{ PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF},	/* ConnectX-4 VF */
2255	{ PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) },
2256	{ PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF},	/* ConnectX-4LX VF */
2257	{ PCI_VDEVICE(MELLANOX, 0x1017) },			/* ConnectX-5, PCIe 3.0 */
2258	{ PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF},	/* ConnectX-5 VF */
2259	{ PCI_VDEVICE(MELLANOX, 0x1019) },			/* ConnectX-5 Ex */
2260	{ PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF},	/* ConnectX-5 Ex VF */
2261	{ PCI_VDEVICE(MELLANOX, 0x101b) },			/* ConnectX-6 */
2262	{ PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF},	/* ConnectX-6 VF */
2263	{ PCI_VDEVICE(MELLANOX, 0x101d) },			/* ConnectX-6 Dx */
2264	{ PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF},	/* ConnectX Family mlx5Gen Virtual Function */
2265	{ PCI_VDEVICE(MELLANOX, 0x101f) },			/* ConnectX-6 LX */
2266	{ PCI_VDEVICE(MELLANOX, 0x1021) },			/* ConnectX-7 */
2267	{ PCI_VDEVICE(MELLANOX, 0x1023) },			/* ConnectX-8 */
2268	{ PCI_VDEVICE(MELLANOX, 0x1025) },			/* ConnectX-9 */
2269	{ PCI_VDEVICE(MELLANOX, 0x1027) },			/* ConnectX-10 */
2270	{ PCI_VDEVICE(MELLANOX, 0x2101) },			/* ConnectX-10 NVLink-C2C */
2271	{ PCI_VDEVICE(MELLANOX, 0xa2d2) },			/* BlueField integrated ConnectX-5 network controller */
2272	{ PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF},	/* BlueField integrated ConnectX-5 network controller VF */
2273	{ PCI_VDEVICE(MELLANOX, 0xa2d6) },			/* BlueField-2 integrated ConnectX-6 Dx network controller */
2274	{ PCI_VDEVICE(MELLANOX, 0xa2dc) },			/* BlueField-3 integrated ConnectX-7 network controller */
2275	{ PCI_VDEVICE(MELLANOX, 0xa2df) },			/* BlueField-4 integrated ConnectX-8 network controller */
2276	{ 0, }
2277};
2278
2279MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
2280
2281void mlx5_disable_device(struct mlx5_core_dev *dev)
2282{
2283	mlx5_error_sw_reset(dev);
2284	mlx5_unload_one_devl_locked(dev, false);
2285}
2286
2287int mlx5_recover_device(struct mlx5_core_dev *dev)
2288{
2289	if (!mlx5_core_is_sf(dev)) {
2290		mlx5_pci_disable_device(dev);
2291		if (mlx5_pci_slot_reset(dev->pdev) != PCI_ERS_RESULT_RECOVERED)
2292			return -EIO;
2293	}
2294
2295	return mlx5_load_one_devl_locked(dev, true);
2296}
2297
2298static struct pci_driver mlx5_core_driver = {
2299	.name           = KBUILD_MODNAME,
2300	.id_table       = mlx5_core_pci_table,
2301	.probe          = probe_one,
2302	.remove         = remove_one,
2303	.suspend        = mlx5_suspend,
2304	.resume         = mlx5_resume,
2305	.shutdown	= shutdown,
2306	.err_handler	= &mlx5_err_handler,
2307	.sriov_configure   = mlx5_core_sriov_configure,
2308	.sriov_get_vf_total_msix = mlx5_sriov_get_vf_total_msix,
2309	.sriov_set_msix_vec_count = mlx5_core_sriov_set_msix_vec_count,
2310};
2311
2312/**
2313 * mlx5_vf_get_core_dev - Get the mlx5 core device from a given VF PCI device if
2314 *                     mlx5_core is its driver.
2315 * @pdev: The associated PCI device.
2316 *
2317 * Upon return the interface state lock stay held to let caller uses it safely.
2318 * Caller must ensure to use the returned mlx5 device for a narrow window
2319 * and put it back with mlx5_vf_put_core_dev() immediately once usage was over.
2320 *
2321 * Return: Pointer to the associated mlx5_core_dev or NULL.
2322 */
2323struct mlx5_core_dev *mlx5_vf_get_core_dev(struct pci_dev *pdev)
2324{
2325	struct mlx5_core_dev *mdev;
2326
2327	mdev = pci_iov_get_pf_drvdata(pdev, &mlx5_core_driver);
2328	if (IS_ERR(mdev))
2329		return NULL;
2330
2331	mutex_lock(&mdev->intf_state_mutex);
2332	if (!test_bit(MLX5_INTERFACE_STATE_UP, &mdev->intf_state)) {
2333		mutex_unlock(&mdev->intf_state_mutex);
2334		return NULL;
2335	}
2336
2337	return mdev;
2338}
2339EXPORT_SYMBOL(mlx5_vf_get_core_dev);
2340
2341/**
2342 * mlx5_vf_put_core_dev - Put the mlx5 core device back.
2343 * @mdev: The mlx5 core device.
2344 *
2345 * Upon return the interface state lock is unlocked and caller should not
2346 * access the mdev any more.
2347 */
2348void mlx5_vf_put_core_dev(struct mlx5_core_dev *mdev)
2349{
2350	mutex_unlock(&mdev->intf_state_mutex);
2351}
2352EXPORT_SYMBOL(mlx5_vf_put_core_dev);
2353
2354static void mlx5_core_verify_params(void)
2355{
2356	if (prof_sel >= ARRAY_SIZE(profile)) {
2357		pr_warn("mlx5_core: WARNING: Invalid module parameter prof_sel %d, valid range 0-%zu, changing back to default(%d)\n",
2358			prof_sel,
2359			ARRAY_SIZE(profile) - 1,
2360			MLX5_DEFAULT_PROF);
2361		prof_sel = MLX5_DEFAULT_PROF;
2362	}
2363}
2364
2365static int __init mlx5_init(void)
2366{
2367	int err;
2368
2369	WARN_ONCE(strcmp(MLX5_ADEV_NAME, KBUILD_MODNAME),
2370		  "mlx5_core name not in sync with kernel module name");
2371
2372	get_random_bytes(&sw_owner_id, sizeof(sw_owner_id));
2373
2374	mlx5_core_verify_params();
2375	mlx5_register_debugfs();
2376
2377	err = mlx5e_init();
2378	if (err)
2379		goto err_debug;
2380
2381	err = mlx5_sf_driver_register();
2382	if (err)
2383		goto err_sf;
2384
2385	err = pci_register_driver(&mlx5_core_driver);
2386	if (err)
2387		goto err_pci;
2388
2389	return 0;
2390
2391err_pci:
2392	mlx5_sf_driver_unregister();
2393err_sf:
2394	mlx5e_cleanup();
2395err_debug:
2396	mlx5_unregister_debugfs();
2397	return err;
2398}
2399
2400static void __exit mlx5_cleanup(void)
2401{
2402	pci_unregister_driver(&mlx5_core_driver);
2403	mlx5_sf_driver_unregister();
2404	mlx5e_cleanup();
2405	mlx5_unregister_debugfs();
2406}
2407
2408module_init(mlx5_init);
2409module_exit(mlx5_cleanup);