devlink.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2020, Intel Corporation. */
   3
   4#include <linux/vmalloc.h>
   5
   6#include "ice.h"
   7#include "ice_lib.h"
   8#include "devlink.h"
   9#include "port.h"
  10#include "ice_eswitch.h"
  11#include "ice_fw_update.h"
  12#include "ice_dcb_lib.h"
  13#include "ice_sf_eth.h"
  14
  15/* context for devlink info version reporting */
  16struct ice_info_ctx {
  17	char buf[128];
  18	struct ice_orom_info pending_orom;
  19	struct ice_nvm_info pending_nvm;
  20	struct ice_netlist_info pending_netlist;
  21	struct ice_hw_dev_caps dev_caps;
  22};
  23
  24/* The following functions are used to format specific strings for various
  25 * devlink info versions. The ctx parameter is used to provide the storage
  26 * buffer, as well as any ancillary information calculated when the info
  27 * request was made.
  28 *
  29 * If a version does not exist, for example when attempting to get the
  30 * inactive version of flash when there is no pending update, the function
  31 * should leave the buffer in the ctx structure empty.
  32 */
  33
  34static void ice_info_get_dsn(struct ice_pf *pf, struct ice_info_ctx *ctx)
  35{
  36	u8 dsn[8];
  37
  38	/* Copy the DSN into an array in Big Endian format */
  39	put_unaligned_be64(pci_get_dsn(pf->pdev), dsn);
  40
  41	snprintf(ctx->buf, sizeof(ctx->buf), "%8phD", dsn);
  42}
  43
  44static void ice_info_pba(struct ice_pf *pf, struct ice_info_ctx *ctx)
  45{
  46	struct ice_hw *hw = &pf->hw;
  47	int status;
  48
  49	status = ice_read_pba_string(hw, (u8 *)ctx->buf, sizeof(ctx->buf));
  50	if (status)
  51		/* We failed to locate the PBA, so just skip this entry */
  52		dev_dbg(ice_pf_to_dev(pf), "Failed to read Product Board Assembly string, status %d\n",
  53			status);
  54}
  55
  56static void ice_info_fw_mgmt(struct ice_pf *pf, struct ice_info_ctx *ctx)
  57{
  58	struct ice_hw *hw = &pf->hw;
  59
  60	snprintf(ctx->buf, sizeof(ctx->buf), "%u.%u.%u",
  61		 hw->fw_maj_ver, hw->fw_min_ver, hw->fw_patch);
  62}
  63
  64static void ice_info_fw_api(struct ice_pf *pf, struct ice_info_ctx *ctx)
  65{
  66	struct ice_hw *hw = &pf->hw;
  67
  68	snprintf(ctx->buf, sizeof(ctx->buf), "%u.%u.%u", hw->api_maj_ver,
  69		 hw->api_min_ver, hw->api_patch);
  70}
  71
  72static void ice_info_fw_build(struct ice_pf *pf, struct ice_info_ctx *ctx)
  73{
  74	struct ice_hw *hw = &pf->hw;
  75
  76	snprintf(ctx->buf, sizeof(ctx->buf), "0x%08x", hw->fw_build);
  77}
  78
  79static void ice_info_orom_ver(struct ice_pf *pf, struct ice_info_ctx *ctx)
  80{
  81	struct ice_orom_info *orom = &pf->hw.flash.orom;
  82
  83	snprintf(ctx->buf, sizeof(ctx->buf), "%u.%u.%u",
  84		 orom->major, orom->build, orom->patch);
  85}
  86
  87static void
  88ice_info_pending_orom_ver(struct ice_pf __always_unused *pf,
  89			  struct ice_info_ctx *ctx)
  90{
  91	struct ice_orom_info *orom = &ctx->pending_orom;
  92
  93	if (ctx->dev_caps.common_cap.nvm_update_pending_orom)
  94		snprintf(ctx->buf, sizeof(ctx->buf), "%u.%u.%u",
  95			 orom->major, orom->build, orom->patch);
  96}
  97
  98static void ice_info_nvm_ver(struct ice_pf *pf, struct ice_info_ctx *ctx)
  99{
 100	struct ice_nvm_info *nvm = &pf->hw.flash.nvm;
 101
 102	snprintf(ctx->buf, sizeof(ctx->buf), "%x.%02x", nvm->major, nvm->minor);
 103}
 104
 105static void
 106ice_info_pending_nvm_ver(struct ice_pf __always_unused *pf,
 107			 struct ice_info_ctx *ctx)
 108{
 109	struct ice_nvm_info *nvm = &ctx->pending_nvm;
 110
 111	if (ctx->dev_caps.common_cap.nvm_update_pending_nvm)
 112		snprintf(ctx->buf, sizeof(ctx->buf), "%x.%02x",
 113			 nvm->major, nvm->minor);
 114}
 115
 116static void ice_info_eetrack(struct ice_pf *pf, struct ice_info_ctx *ctx)
 117{
 118	struct ice_nvm_info *nvm = &pf->hw.flash.nvm;
 119
 120	snprintf(ctx->buf, sizeof(ctx->buf), "0x%08x", nvm->eetrack);
 121}
 122
 123static void
 124ice_info_pending_eetrack(struct ice_pf *pf, struct ice_info_ctx *ctx)
 125{
 126	struct ice_nvm_info *nvm = &ctx->pending_nvm;
 127
 128	if (ctx->dev_caps.common_cap.nvm_update_pending_nvm)
 129		snprintf(ctx->buf, sizeof(ctx->buf), "0x%08x", nvm->eetrack);
 130}
 131
 132static void ice_info_ddp_pkg_name(struct ice_pf *pf, struct ice_info_ctx *ctx)
 133{
 134	struct ice_hw *hw = &pf->hw;
 135
 136	snprintf(ctx->buf, sizeof(ctx->buf), "%s", hw->active_pkg_name);
 137}
 138
 139static void
 140ice_info_ddp_pkg_version(struct ice_pf *pf, struct ice_info_ctx *ctx)
 141{
 142	struct ice_pkg_ver *pkg = &pf->hw.active_pkg_ver;
 143
 144	snprintf(ctx->buf, sizeof(ctx->buf), "%u.%u.%u.%u",
 145		 pkg->major, pkg->minor, pkg->update, pkg->draft);
 146}
 147
 148static void
 149ice_info_ddp_pkg_bundle_id(struct ice_pf *pf, struct ice_info_ctx *ctx)
 150{
 151	snprintf(ctx->buf, sizeof(ctx->buf), "0x%08x", pf->hw.active_track_id);
 152}
 153
 154static void ice_info_netlist_ver(struct ice_pf *pf, struct ice_info_ctx *ctx)
 155{
 156	struct ice_netlist_info *netlist = &pf->hw.flash.netlist;
 157
 158	/* The netlist version fields are BCD formatted */
 159	snprintf(ctx->buf, sizeof(ctx->buf), "%x.%x.%x-%x.%x.%x",
 160		 netlist->major, netlist->minor,
 161		 netlist->type >> 16, netlist->type & 0xFFFF,
 162		 netlist->rev, netlist->cust_ver);
 163}
 164
 165static void ice_info_netlist_build(struct ice_pf *pf, struct ice_info_ctx *ctx)
 166{
 167	struct ice_netlist_info *netlist = &pf->hw.flash.netlist;
 168
 169	snprintf(ctx->buf, sizeof(ctx->buf), "0x%08x", netlist->hash);
 170}
 171
 172static void
 173ice_info_pending_netlist_ver(struct ice_pf __always_unused *pf,
 174			     struct ice_info_ctx *ctx)
 175{
 176	struct ice_netlist_info *netlist = &ctx->pending_netlist;
 177
 178	/* The netlist version fields are BCD formatted */
 179	if (ctx->dev_caps.common_cap.nvm_update_pending_netlist)
 180		snprintf(ctx->buf, sizeof(ctx->buf), "%x.%x.%x-%x.%x.%x",
 181			 netlist->major, netlist->minor,
 182			 netlist->type >> 16, netlist->type & 0xFFFF,
 183			 netlist->rev, netlist->cust_ver);
 184}
 185
 186static void
 187ice_info_pending_netlist_build(struct ice_pf __always_unused *pf,
 188			       struct ice_info_ctx *ctx)
 189{
 190	struct ice_netlist_info *netlist = &ctx->pending_netlist;
 191
 192	if (ctx->dev_caps.common_cap.nvm_update_pending_netlist)
 193		snprintf(ctx->buf, sizeof(ctx->buf), "0x%08x", netlist->hash);
 194}
 195
 196static void ice_info_cgu_fw_build(struct ice_pf *pf, struct ice_info_ctx *ctx)
 197{
 198	u32 id, cfg_ver, fw_ver;
 199
 200	if (!ice_is_feature_supported(pf, ICE_F_CGU))
 201		return;
 202	if (ice_aq_get_cgu_info(&pf->hw, &id, &cfg_ver, &fw_ver))
 203		return;
 204	snprintf(ctx->buf, sizeof(ctx->buf), "%u.%u.%u", id, cfg_ver, fw_ver);
 205}
 206
 207static void ice_info_cgu_id(struct ice_pf *pf, struct ice_info_ctx *ctx)
 208{
 209	if (!ice_is_feature_supported(pf, ICE_F_CGU))
 210		return;
 211	snprintf(ctx->buf, sizeof(ctx->buf), "%u", pf->hw.cgu_part_number);
 212}
 213
 214#define fixed(key, getter) { ICE_VERSION_FIXED, key, getter, NULL }
 215#define running(key, getter) { ICE_VERSION_RUNNING, key, getter, NULL }
 216#define stored(key, getter, fallback) { ICE_VERSION_STORED, key, getter, fallback }
 217
 218/* The combined() macro inserts both the running entry as well as a stored
 219 * entry. The running entry will always report the version from the active
 220 * handler. The stored entry will first try the pending handler, and fallback
 221 * to the active handler if the pending function does not report a version.
 222 * The pending handler should check the status of a pending update for the
 223 * relevant flash component. It should only fill in the buffer in the case
 224 * where a valid pending version is available. This ensures that the related
 225 * stored and running versions remain in sync, and that stored versions are
 226 * correctly reported as expected.
 227 */
 228#define combined(key, active, pending) \
 229	running(key, active), \
 230	stored(key, pending, active)
 231
 232enum ice_version_type {
 233	ICE_VERSION_FIXED,
 234	ICE_VERSION_RUNNING,
 235	ICE_VERSION_STORED,
 236};
 237
 238static const struct ice_devlink_version {
 239	enum ice_version_type type;
 240	const char *key;
 241	void (*getter)(struct ice_pf *pf, struct ice_info_ctx *ctx);
 242	void (*fallback)(struct ice_pf *pf, struct ice_info_ctx *ctx);
 243} ice_devlink_versions[] = {
 244	fixed(DEVLINK_INFO_VERSION_GENERIC_BOARD_ID, ice_info_pba),
 245	running(DEVLINK_INFO_VERSION_GENERIC_FW_MGMT, ice_info_fw_mgmt),
 246	running("fw.mgmt.api", ice_info_fw_api),
 247	running("fw.mgmt.build", ice_info_fw_build),
 248	combined(DEVLINK_INFO_VERSION_GENERIC_FW_UNDI, ice_info_orom_ver, ice_info_pending_orom_ver),
 249	combined("fw.psid.api", ice_info_nvm_ver, ice_info_pending_nvm_ver),
 250	combined(DEVLINK_INFO_VERSION_GENERIC_FW_BUNDLE_ID, ice_info_eetrack, ice_info_pending_eetrack),
 251	running("fw.app.name", ice_info_ddp_pkg_name),
 252	running(DEVLINK_INFO_VERSION_GENERIC_FW_APP, ice_info_ddp_pkg_version),
 253	running("fw.app.bundle_id", ice_info_ddp_pkg_bundle_id),
 254	combined("fw.netlist", ice_info_netlist_ver, ice_info_pending_netlist_ver),
 255	combined("fw.netlist.build", ice_info_netlist_build, ice_info_pending_netlist_build),
 256	fixed("cgu.id", ice_info_cgu_id),
 257	running("fw.cgu", ice_info_cgu_fw_build),
 258};
 259
 260/**
 261 * ice_devlink_info_get - .info_get devlink handler
 262 * @devlink: devlink instance structure
 263 * @req: the devlink info request
 264 * @extack: extended netdev ack structure
 265 *
 266 * Callback for the devlink .info_get operation. Reports information about the
 267 * device.
 268 *
 269 * Return: zero on success or an error code on failure.
 270 */
 271static int ice_devlink_info_get(struct devlink *devlink,
 272				struct devlink_info_req *req,
 273				struct netlink_ext_ack *extack)
 274{
 275	struct ice_pf *pf = devlink_priv(devlink);
 276	struct device *dev = ice_pf_to_dev(pf);
 277	struct ice_hw *hw = &pf->hw;
 278	struct ice_info_ctx *ctx;
 279	size_t i;
 280	int err;
 281
 282	err = ice_wait_for_reset(pf, 10 * HZ);
 283	if (err) {
 284		NL_SET_ERR_MSG_MOD(extack, "Device is busy resetting");
 285		return err;
 286	}
 287
 288	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 289	if (!ctx)
 290		return -ENOMEM;
 291
 292	/* discover capabilities first */
 293	err = ice_discover_dev_caps(hw, &ctx->dev_caps);
 294	if (err) {
 295		dev_dbg(dev, "Failed to discover device capabilities, status %d aq_err %s\n",
 296			err, libie_aq_str(hw->adminq.sq_last_status));
 297		NL_SET_ERR_MSG_MOD(extack, "Unable to discover device capabilities");
 298		goto out_free_ctx;
 299	}
 300
 301	if (ctx->dev_caps.common_cap.nvm_update_pending_orom) {
 302		err = ice_get_inactive_orom_ver(hw, &ctx->pending_orom);
 303		if (err) {
 304			dev_dbg(dev, "Unable to read inactive Option ROM version data, status %d aq_err %s\n",
 305				err, libie_aq_str(hw->adminq.sq_last_status));
 306
 307			/* disable display of pending Option ROM */
 308			ctx->dev_caps.common_cap.nvm_update_pending_orom = false;
 309		}
 310	}
 311
 312	if (ctx->dev_caps.common_cap.nvm_update_pending_nvm) {
 313		err = ice_get_inactive_nvm_ver(hw, &ctx->pending_nvm);
 314		if (err) {
 315			dev_dbg(dev, "Unable to read inactive NVM version data, status %d aq_err %s\n",
 316				err, libie_aq_str(hw->adminq.sq_last_status));
 317
 318			/* disable display of pending Option ROM */
 319			ctx->dev_caps.common_cap.nvm_update_pending_nvm = false;
 320		}
 321	}
 322
 323	if (ctx->dev_caps.common_cap.nvm_update_pending_netlist) {
 324		err = ice_get_inactive_netlist_ver(hw, &ctx->pending_netlist);
 325		if (err) {
 326			dev_dbg(dev, "Unable to read inactive Netlist version data, status %d aq_err %s\n",
 327				err, libie_aq_str(hw->adminq.sq_last_status));
 328
 329			/* disable display of pending Option ROM */
 330			ctx->dev_caps.common_cap.nvm_update_pending_netlist = false;
 331		}
 332	}
 333
 334	ice_info_get_dsn(pf, ctx);
 335
 336	err = devlink_info_serial_number_put(req, ctx->buf);
 337	if (err) {
 338		NL_SET_ERR_MSG_MOD(extack, "Unable to set serial number");
 339		goto out_free_ctx;
 340	}
 341
 342	for (i = 0; i < ARRAY_SIZE(ice_devlink_versions); i++) {
 343		enum ice_version_type type = ice_devlink_versions[i].type;
 344		const char *key = ice_devlink_versions[i].key;
 345
 346		memset(ctx->buf, 0, sizeof(ctx->buf));
 347
 348		ice_devlink_versions[i].getter(pf, ctx);
 349
 350		/* If the default getter doesn't report a version, use the
 351		 * fallback function. This is primarily useful in the case of
 352		 * "stored" versions that want to report the same value as the
 353		 * running version in the normal case of no pending update.
 354		 */
 355		if (ctx->buf[0] == '\0' && ice_devlink_versions[i].fallback)
 356			ice_devlink_versions[i].fallback(pf, ctx);
 357
 358		/* Do not report missing versions */
 359		if (ctx->buf[0] == '\0')
 360			continue;
 361
 362		switch (type) {
 363		case ICE_VERSION_FIXED:
 364			err = devlink_info_version_fixed_put(req, key, ctx->buf);
 365			if (err) {
 366				NL_SET_ERR_MSG_MOD(extack, "Unable to set fixed version");
 367				goto out_free_ctx;
 368			}
 369			break;
 370		case ICE_VERSION_RUNNING:
 371			err = devlink_info_version_running_put_ext(req, key,
 372								   ctx->buf,
 373								   DEVLINK_INFO_VERSION_TYPE_COMPONENT);
 374			if (err) {
 375				NL_SET_ERR_MSG_MOD(extack, "Unable to set running version");
 376				goto out_free_ctx;
 377			}
 378			break;
 379		case ICE_VERSION_STORED:
 380			err = devlink_info_version_stored_put_ext(req, key,
 381								  ctx->buf,
 382								  DEVLINK_INFO_VERSION_TYPE_COMPONENT);
 383			if (err) {
 384				NL_SET_ERR_MSG_MOD(extack, "Unable to set stored version");
 385				goto out_free_ctx;
 386			}
 387			break;
 388		}
 389	}
 390
 391out_free_ctx:
 392	kfree(ctx);
 393	return err;
 394}
 395
 396/**
 397 * ice_devlink_reload_empr_start - Start EMP reset to activate new firmware
 398 * @pf: pointer to the pf instance
 399 * @extack: netlink extended ACK structure
 400 *
 401 * Allow user to activate new Embedded Management Processor firmware by
 402 * issuing device specific EMP reset. Called in response to
 403 * a DEVLINK_CMD_RELOAD with the DEVLINK_RELOAD_ACTION_FW_ACTIVATE.
 404 *
 405 * Note that teardown and rebuild of the driver state happens automatically as
 406 * part of an interrupt and watchdog task. This is because all physical
 407 * functions on the device must be able to reset when an EMP reset occurs from
 408 * any source.
 409 */
 410static int
 411ice_devlink_reload_empr_start(struct ice_pf *pf,
 412			      struct netlink_ext_ack *extack)
 413{
 414	struct device *dev = ice_pf_to_dev(pf);
 415	struct ice_hw *hw = &pf->hw;
 416	u8 pending;
 417	int err;
 418
 419	err = ice_get_pending_updates(pf, &pending, extack);
 420	if (err)
 421		return err;
 422
 423	/* pending is a bitmask of which flash banks have a pending update,
 424	 * including the main NVM bank, the Option ROM bank, and the netlist
 425	 * bank. If any of these bits are set, then there is a pending update
 426	 * waiting to be activated.
 427	 */
 428	if (!pending) {
 429		NL_SET_ERR_MSG_MOD(extack, "No pending firmware update");
 430		return -ECANCELED;
 431	}
 432
 433	if (pf->fw_emp_reset_disabled) {
 434		NL_SET_ERR_MSG_MOD(extack, "EMP reset is not available. To activate firmware, a reboot or power cycle is needed");
 435		return -ECANCELED;
 436	}
 437
 438	dev_dbg(dev, "Issuing device EMP reset to activate firmware\n");
 439
 440	err = ice_aq_nvm_update_empr(hw);
 441	if (err) {
 442		dev_err(dev, "Failed to trigger EMP device reset to reload firmware, err %d aq_err %s\n",
 443			err, libie_aq_str(hw->adminq.sq_last_status));
 444		NL_SET_ERR_MSG_MOD(extack, "Failed to trigger EMP device reset to reload firmware");
 445		return err;
 446	}
 447
 448	return 0;
 449}
 450
 451/**
 452 * ice_devlink_reinit_down - unload given PF
 453 * @pf: pointer to the PF struct
 454 */
 455static void ice_devlink_reinit_down(struct ice_pf *pf)
 456{
 457	/* No need to take devl_lock, it's already taken by devlink API */
 458	ice_unload(pf);
 459	rtnl_lock();
 460	ice_vsi_decfg(ice_get_main_vsi(pf));
 461	rtnl_unlock();
 462	ice_deinit_pf(pf);
 463	ice_deinit_hw(&pf->hw);
 464	ice_deinit_dev(pf);
 465}
 466
 467/**
 468 * ice_devlink_reload_down - prepare for reload
 469 * @devlink: pointer to the devlink instance to reload
 470 * @netns_change: if true, the network namespace is changing
 471 * @action: the action to perform
 472 * @limit: limits on what reload should do, such as not resetting
 473 * @extack: netlink extended ACK structure
 474 */
 475static int
 476ice_devlink_reload_down(struct devlink *devlink, bool netns_change,
 477			enum devlink_reload_action action,
 478			enum devlink_reload_limit limit,
 479			struct netlink_ext_ack *extack)
 480{
 481	struct ice_pf *pf = devlink_priv(devlink);
 482
 483	switch (action) {
 484	case DEVLINK_RELOAD_ACTION_DRIVER_REINIT:
 485		if (ice_is_eswitch_mode_switchdev(pf)) {
 486			NL_SET_ERR_MSG_MOD(extack,
 487					   "Go to legacy mode before doing reinit");
 488			return -EOPNOTSUPP;
 489		}
 490		if (ice_is_adq_active(pf)) {
 491			NL_SET_ERR_MSG_MOD(extack,
 492					   "Turn off ADQ before doing reinit");
 493			return -EOPNOTSUPP;
 494		}
 495		if (ice_has_vfs(pf)) {
 496			NL_SET_ERR_MSG_MOD(extack,
 497					   "Remove all VFs before doing reinit");
 498			return -EOPNOTSUPP;
 499		}
 500		ice_devlink_reinit_down(pf);
 501		return 0;
 502	case DEVLINK_RELOAD_ACTION_FW_ACTIVATE:
 503		return ice_devlink_reload_empr_start(pf, extack);
 504	default:
 505		WARN_ON(1);
 506		return -EOPNOTSUPP;
 507	}
 508}
 509
 510/**
 511 * ice_devlink_reload_empr_finish - Wait for EMP reset to finish
 512 * @pf: pointer to the pf instance
 513 * @extack: netlink extended ACK structure
 514 *
 515 * Wait for driver to finish rebuilding after EMP reset is completed. This
 516 * includes time to wait for both the actual device reset as well as the time
 517 * for the driver's rebuild to complete.
 518 */
 519static int
 520ice_devlink_reload_empr_finish(struct ice_pf *pf,
 521			       struct netlink_ext_ack *extack)
 522{
 523	int err;
 524
 525	err = ice_wait_for_reset(pf, 60 * HZ);
 526	if (err) {
 527		NL_SET_ERR_MSG_MOD(extack, "Device still resetting after 1 minute");
 528		return err;
 529	}
 530
 531	return 0;
 532}
 533
 534/**
 535 * ice_get_tx_topo_user_sel - Read user's choice from flash
 536 * @pf: pointer to pf structure
 537 * @layers: value read from flash will be saved here
 538 *
 539 * Reads user's preference for Tx Scheduler Topology Tree from PFA TLV.
 540 *
 541 * Return: zero when read was successful, negative values otherwise.
 542 */
 543static int ice_get_tx_topo_user_sel(struct ice_pf *pf, uint8_t *layers)
 544{
 545	struct ice_aqc_nvm_tx_topo_user_sel usr_sel = {};
 546	struct ice_hw *hw = &pf->hw;
 547	int err;
 548
 549	err = ice_acquire_nvm(hw, ICE_RES_READ);
 550	if (err)
 551		return err;
 552
 553	err = ice_aq_read_nvm(hw, ICE_AQC_NVM_TX_TOPO_MOD_ID, 0,
 554			      sizeof(usr_sel), &usr_sel, true, true, NULL);
 555	if (err)
 556		goto exit_release_res;
 557
 558	if (usr_sel.data & ICE_AQC_NVM_TX_TOPO_USER_SEL)
 559		*layers = ICE_SCHED_5_LAYERS;
 560	else
 561		*layers = ICE_SCHED_9_LAYERS;
 562
 563exit_release_res:
 564	ice_release_nvm(hw);
 565
 566	return err;
 567}
 568
 569/**
 570 * ice_update_tx_topo_user_sel - Save user's preference in flash
 571 * @pf: pointer to pf structure
 572 * @layers: value to be saved in flash
 573 *
 574 * Variable "layers" defines user's preference about number of layers in Tx
 575 * Scheduler Topology Tree. This choice should be stored in PFA TLV field
 576 * and be picked up by driver, next time during init.
 577 *
 578 * Return: zero when save was successful, negative values otherwise.
 579 */
 580static int ice_update_tx_topo_user_sel(struct ice_pf *pf, int layers)
 581{
 582	struct ice_aqc_nvm_tx_topo_user_sel usr_sel = {};
 583	struct ice_hw *hw = &pf->hw;
 584	int err;
 585
 586	err = ice_acquire_nvm(hw, ICE_RES_WRITE);
 587	if (err)
 588		return err;
 589
 590	err = ice_aq_read_nvm(hw, ICE_AQC_NVM_TX_TOPO_MOD_ID, 0,
 591			      sizeof(usr_sel), &usr_sel, true, true, NULL);
 592	if (err)
 593		goto exit_release_res;
 594
 595	if (layers == ICE_SCHED_5_LAYERS)
 596		usr_sel.data |= ICE_AQC_NVM_TX_TOPO_USER_SEL;
 597	else
 598		usr_sel.data &= ~ICE_AQC_NVM_TX_TOPO_USER_SEL;
 599
 600	err = ice_write_one_nvm_block(pf, ICE_AQC_NVM_TX_TOPO_MOD_ID, 2,
 601				      sizeof(usr_sel.data), &usr_sel.data,
 602				      true, NULL, NULL);
 603exit_release_res:
 604	ice_release_nvm(hw);
 605
 606	return err;
 607}
 608
 609/**
 610 * ice_devlink_tx_sched_layers_get - Get tx_scheduling_layers parameter
 611 * @devlink: pointer to the devlink instance
 612 * @id: the parameter ID to set
 613 * @ctx: context to store the parameter value
 614 * @extack: netlink extended ACK structure
 615 *
 616 * Return: zero on success and negative value on failure.
 617 */
 618static int ice_devlink_tx_sched_layers_get(struct devlink *devlink, u32 id,
 619					   struct devlink_param_gset_ctx *ctx,
 620					   struct netlink_ext_ack *extack)
 621{
 622	struct ice_pf *pf = devlink_priv(devlink);
 623	int err;
 624
 625	err = ice_get_tx_topo_user_sel(pf, &ctx->val.vu8);
 626	if (err)
 627		return err;
 628
 629	return 0;
 630}
 631
 632/**
 633 * ice_devlink_tx_sched_layers_set - Set tx_scheduling_layers parameter
 634 * @devlink: pointer to the devlink instance
 635 * @id: the parameter ID to set
 636 * @ctx: context to get the parameter value
 637 * @extack: netlink extended ACK structure
 638 *
 639 * Return: zero on success and negative value on failure.
 640 */
 641static int ice_devlink_tx_sched_layers_set(struct devlink *devlink, u32 id,
 642					   struct devlink_param_gset_ctx *ctx,
 643					   struct netlink_ext_ack *extack)
 644{
 645	struct ice_pf *pf = devlink_priv(devlink);
 646	int err;
 647
 648	err = ice_update_tx_topo_user_sel(pf, ctx->val.vu8);
 649	if (err)
 650		return err;
 651
 652	NL_SET_ERR_MSG_MOD(extack,
 653			   "Tx scheduling layers have been changed on this device. You must do the PCI slot powercycle for the change to take effect.");
 654
 655	return 0;
 656}
 657
 658/**
 659 * ice_devlink_tx_sched_layers_validate - Validate passed tx_scheduling_layers
 660 *                                        parameter value
 661 * @devlink: unused pointer to devlink instance
 662 * @id: the parameter ID to validate
 663 * @val: value to validate
 664 * @extack: netlink extended ACK structure
 665 *
 666 * Supported values are:
 667 * - 5 - five layers Tx Scheduler Topology Tree
 668 * - 9 - nine layers Tx Scheduler Topology Tree
 669 *
 670 * Return: zero when passed parameter value is supported. Negative value on
 671 * error.
 672 */
 673static int ice_devlink_tx_sched_layers_validate(struct devlink *devlink, u32 id,
 674						union devlink_param_value val,
 675						struct netlink_ext_ack *extack)
 676{
 677	if (val.vu8 != ICE_SCHED_5_LAYERS && val.vu8 != ICE_SCHED_9_LAYERS) {
 678		NL_SET_ERR_MSG_MOD(extack,
 679				   "Wrong number of tx scheduler layers provided.");
 680		return -EINVAL;
 681	}
 682
 683	return 0;
 684}
 685
 686/**
 687 * ice_tear_down_devlink_rate_tree - removes devlink-rate exported tree
 688 * @pf: pf struct
 689 *
 690 * This function tears down tree exported during VF's creation.
 691 */
 692void ice_tear_down_devlink_rate_tree(struct ice_pf *pf)
 693{
 694	struct devlink *devlink;
 695	struct ice_vf *vf;
 696	unsigned int bkt;
 697
 698	devlink = priv_to_devlink(pf);
 699
 700	devl_lock(devlink);
 701	mutex_lock(&pf->vfs.table_lock);
 702	ice_for_each_vf(pf, bkt, vf) {
 703		if (vf->devlink_port.devlink_rate)
 704			devl_rate_leaf_destroy(&vf->devlink_port);
 705	}
 706	mutex_unlock(&pf->vfs.table_lock);
 707
 708	devl_rate_nodes_destroy(devlink);
 709	devl_unlock(devlink);
 710}
 711
 712/**
 713 * ice_enable_custom_tx - try to enable custom Tx feature
 714 * @pf: pf struct
 715 *
 716 * This function tries to enable custom Tx feature,
 717 * it's not possible to enable it, if DCB or ADQ is active.
 718 */
 719static bool ice_enable_custom_tx(struct ice_pf *pf)
 720{
 721	struct ice_port_info *pi = ice_get_main_vsi(pf)->port_info;
 722	struct device *dev = ice_pf_to_dev(pf);
 723
 724	if (pi->is_custom_tx_enabled)
 725		/* already enabled, return true */
 726		return true;
 727
 728	if (ice_is_adq_active(pf)) {
 729		dev_err(dev, "ADQ active, can't modify Tx scheduler tree\n");
 730		return false;
 731	}
 732
 733	if (ice_is_dcb_active(pf)) {
 734		dev_err(dev, "DCB active, can't modify Tx scheduler tree\n");
 735		return false;
 736	}
 737
 738	pi->is_custom_tx_enabled = true;
 739
 740	return true;
 741}
 742
 743/**
 744 * ice_traverse_tx_tree - traverse Tx scheduler tree
 745 * @devlink: devlink struct
 746 * @node: current node, used for recursion
 747 * @tc_node: tc_node struct, that is treated as a root
 748 * @pf: pf struct
 749 *
 750 * This function traverses Tx scheduler tree and exports
 751 * entire structure to the devlink-rate.
 752 */
 753static void ice_traverse_tx_tree(struct devlink *devlink, struct ice_sched_node *node,
 754				 struct ice_sched_node *tc_node, struct ice_pf *pf)
 755{
 756	struct devlink_rate *rate_node = NULL;
 757	struct ice_dynamic_port *sf;
 758	struct ice_vf *vf;
 759	int i;
 760
 761	if (node->rate_node)
 762		/* already added, skip to the next */
 763		goto traverse_children;
 764
 765	if (node->parent == tc_node) {
 766		/* create root node */
 767		rate_node = devl_rate_node_create(devlink, node, node->name, NULL);
 768	} else if (node->vsi_handle &&
 769		   pf->vsi[node->vsi_handle]->type == ICE_VSI_VF &&
 770		   pf->vsi[node->vsi_handle]->vf) {
 771		vf = pf->vsi[node->vsi_handle]->vf;
 772		if (!vf->devlink_port.devlink_rate)
 773			/* leaf nodes doesn't have children
 774			 * so we don't set rate_node
 775			 */
 776			devl_rate_leaf_create(&vf->devlink_port, node,
 777					      node->parent->rate_node);
 778	} else if (node->vsi_handle &&
 779		   pf->vsi[node->vsi_handle]->type == ICE_VSI_SF &&
 780		   pf->vsi[node->vsi_handle]->sf) {
 781		sf = pf->vsi[node->vsi_handle]->sf;
 782		if (!sf->devlink_port.devlink_rate)
 783			/* leaf nodes doesn't have children
 784			 * so we don't set rate_node
 785			 */
 786			devl_rate_leaf_create(&sf->devlink_port, node,
 787					      node->parent->rate_node);
 788	} else if (node->info.data.elem_type != ICE_AQC_ELEM_TYPE_LEAF &&
 789		   node->parent->rate_node) {
 790		rate_node = devl_rate_node_create(devlink, node, node->name,
 791						  node->parent->rate_node);
 792	}
 793
 794	if (rate_node && !IS_ERR(rate_node))
 795		node->rate_node = rate_node;
 796
 797traverse_children:
 798	for (i = 0; i < node->num_children; i++)
 799		ice_traverse_tx_tree(devlink, node->children[i], tc_node, pf);
 800}
 801
 802/**
 803 * ice_devlink_rate_init_tx_topology - export Tx scheduler tree to devlink rate
 804 * @devlink: devlink struct
 805 * @vsi: main vsi struct
 806 *
 807 * This function finds a root node, then calls ice_traverse_tx tree, which
 808 * traverses the tree and exports it's contents to devlink rate.
 809 */
 810int ice_devlink_rate_init_tx_topology(struct devlink *devlink, struct ice_vsi *vsi)
 811{
 812	struct ice_port_info *pi = vsi->port_info;
 813	struct ice_sched_node *tc_node;
 814	struct ice_pf *pf = vsi->back;
 815	int i;
 816
 817	tc_node = pi->root->children[0];
 818	mutex_lock(&pi->sched_lock);
 819	for (i = 0; i < tc_node->num_children; i++)
 820		ice_traverse_tx_tree(devlink, tc_node->children[i], tc_node, pf);
 821	mutex_unlock(&pi->sched_lock);
 822
 823	return 0;
 824}
 825
 826static void ice_clear_rate_nodes(struct ice_sched_node *node)
 827{
 828	node->rate_node = NULL;
 829
 830	for (int i = 0; i < node->num_children; i++)
 831		ice_clear_rate_nodes(node->children[i]);
 832}
 833
 834/**
 835 * ice_devlink_rate_clear_tx_topology - clear node->rate_node
 836 * @vsi: main vsi struct
 837 *
 838 * Clear rate_node to cleanup creation of Tx topology.
 839 *
 840 */
 841void ice_devlink_rate_clear_tx_topology(struct ice_vsi *vsi)
 842{
 843	struct ice_port_info *pi = vsi->port_info;
 844
 845	mutex_lock(&pi->sched_lock);
 846	ice_clear_rate_nodes(pi->root->children[0]);
 847	mutex_unlock(&pi->sched_lock);
 848}
 849
 850/**
 851 * ice_set_object_tx_share - sets node scheduling parameter
 852 * @pi: devlink struct instance
 853 * @node: node struct instance
 854 * @bw: bandwidth in bytes per second
 855 * @extack: extended netdev ack structure
 856 *
 857 * This function sets ICE_MIN_BW scheduling BW limit.
 858 */
 859static int ice_set_object_tx_share(struct ice_port_info *pi, struct ice_sched_node *node,
 860				   u64 bw, struct netlink_ext_ack *extack)
 861{
 862	int status;
 863
 864	mutex_lock(&pi->sched_lock);
 865	/* converts bytes per second to kilo bits per second */
 866	node->tx_share = div_u64(bw, 125);
 867	status = ice_sched_set_node_bw_lmt(pi, node, ICE_MIN_BW, node->tx_share);
 868	mutex_unlock(&pi->sched_lock);
 869
 870	if (status)
 871		NL_SET_ERR_MSG_MOD(extack, "Can't set scheduling node tx_share");
 872
 873	return status;
 874}
 875
 876/**
 877 * ice_set_object_tx_max - sets node scheduling parameter
 878 * @pi: devlink struct instance
 879 * @node: node struct instance
 880 * @bw: bandwidth in bytes per second
 881 * @extack: extended netdev ack structure
 882 *
 883 * This function sets ICE_MAX_BW scheduling BW limit.
 884 */
 885static int ice_set_object_tx_max(struct ice_port_info *pi, struct ice_sched_node *node,
 886				 u64 bw, struct netlink_ext_ack *extack)
 887{
 888	int status;
 889
 890	mutex_lock(&pi->sched_lock);
 891	/* converts bytes per second value to kilo bits per second */
 892	node->tx_max = div_u64(bw, 125);
 893	status = ice_sched_set_node_bw_lmt(pi, node, ICE_MAX_BW, node->tx_max);
 894	mutex_unlock(&pi->sched_lock);
 895
 896	if (status)
 897		NL_SET_ERR_MSG_MOD(extack, "Can't set scheduling node tx_max");
 898
 899	return status;
 900}
 901
 902/**
 903 * ice_set_object_tx_priority - sets node scheduling parameter
 904 * @pi: devlink struct instance
 905 * @node: node struct instance
 906 * @priority: value representing priority for strict priority arbitration
 907 * @extack: extended netdev ack structure
 908 *
 909 * This function sets priority of node among siblings.
 910 */
 911static int ice_set_object_tx_priority(struct ice_port_info *pi, struct ice_sched_node *node,
 912				      u32 priority, struct netlink_ext_ack *extack)
 913{
 914	int status;
 915
 916	if (priority >= 8) {
 917		NL_SET_ERR_MSG_MOD(extack, "Priority should be less than 8");
 918		return -EINVAL;
 919	}
 920
 921	mutex_lock(&pi->sched_lock);
 922	node->tx_priority = priority;
 923	status = ice_sched_set_node_priority(pi, node, node->tx_priority);
 924	mutex_unlock(&pi->sched_lock);
 925
 926	if (status)
 927		NL_SET_ERR_MSG_MOD(extack, "Can't set scheduling node tx_priority");
 928
 929	return status;
 930}
 931
 932/**
 933 * ice_set_object_tx_weight - sets node scheduling parameter
 934 * @pi: devlink struct instance
 935 * @node: node struct instance
 936 * @weight: value represeting relative weight for WFQ arbitration
 937 * @extack: extended netdev ack structure
 938 *
 939 * This function sets node weight for WFQ algorithm.
 940 */
 941static int ice_set_object_tx_weight(struct ice_port_info *pi, struct ice_sched_node *node,
 942				    u32 weight, struct netlink_ext_ack *extack)
 943{
 944	int status;
 945
 946	if (weight > 200 || weight < 1) {
 947		NL_SET_ERR_MSG_MOD(extack, "Weight must be between 1 and 200");
 948		return -EINVAL;
 949	}
 950
 951	mutex_lock(&pi->sched_lock);
 952	node->tx_weight = weight;
 953	status = ice_sched_set_node_weight(pi, node, node->tx_weight);
 954	mutex_unlock(&pi->sched_lock);
 955
 956	if (status)
 957		NL_SET_ERR_MSG_MOD(extack, "Can't set scheduling node tx_weight");
 958
 959	return status;
 960}
 961
 962/**
 963 * ice_get_pi_from_dev_rate - get port info from devlink_rate
 964 * @rate_node: devlink struct instance
 965 *
 966 * This function returns corresponding port_info struct of devlink_rate
 967 */
 968static struct ice_port_info *ice_get_pi_from_dev_rate(struct devlink_rate *rate_node)
 969{
 970	struct ice_pf *pf = devlink_priv(rate_node->devlink);
 971
 972	return ice_get_main_vsi(pf)->port_info;
 973}
 974
 975static int ice_devlink_rate_node_new(struct devlink_rate *rate_node, void **priv,
 976				     struct netlink_ext_ack *extack)
 977{
 978	struct ice_sched_node *node;
 979	struct ice_port_info *pi;
 980
 981	pi = ice_get_pi_from_dev_rate(rate_node);
 982
 983	if (!ice_enable_custom_tx(devlink_priv(rate_node->devlink)))
 984		return -EBUSY;
 985
 986	/* preallocate memory for ice_sched_node */
 987	node = devm_kzalloc(ice_hw_to_dev(pi->hw), sizeof(*node), GFP_KERNEL);
 988	if (!node)
 989		return -ENOMEM;
 990
 991	*priv = node;
 992
 993	return 0;
 994}
 995
 996static int ice_devlink_rate_node_del(struct devlink_rate *rate_node, void *priv,
 997				     struct netlink_ext_ack *extack)
 998{
 999	struct ice_sched_node *node, *tc_node;
1000	struct ice_port_info *pi;
1001
1002	pi = ice_get_pi_from_dev_rate(rate_node);
1003	tc_node = pi->root->children[0];
1004	node = priv;
1005
1006	if (!rate_node->parent || !node || tc_node == node || !extack)
1007		return 0;
1008
1009	if (!ice_enable_custom_tx(devlink_priv(rate_node->devlink)))
1010		return -EBUSY;
1011
1012	/* can't allow to delete a node with children */
1013	if (node->num_children)
1014		return -EINVAL;
1015
1016	mutex_lock(&pi->sched_lock);
1017	ice_free_sched_node(pi, node);
1018	mutex_unlock(&pi->sched_lock);
1019
1020	return 0;
1021}
1022
1023static int ice_devlink_rate_leaf_tx_max_set(struct devlink_rate *rate_leaf, void *priv,
1024					    u64 tx_max, struct netlink_ext_ack *extack)
1025{
1026	struct ice_sched_node *node = priv;
1027
1028	if (!ice_enable_custom_tx(devlink_priv(rate_leaf->devlink)))
1029		return -EBUSY;
1030
1031	if (!node)
1032		return 0;
1033
1034	return ice_set_object_tx_max(ice_get_pi_from_dev_rate(rate_leaf),
1035				     node, tx_max, extack);
1036}
1037
1038static int ice_devlink_rate_leaf_tx_share_set(struct devlink_rate *rate_leaf, void *priv,
1039					      u64 tx_share, struct netlink_ext_ack *extack)
1040{
1041	struct ice_sched_node *node = priv;
1042
1043	if (!ice_enable_custom_tx(devlink_priv(rate_leaf->devlink)))
1044		return -EBUSY;
1045
1046	if (!node)
1047		return 0;
1048
1049	return ice_set_object_tx_share(ice_get_pi_from_dev_rate(rate_leaf), node,
1050				       tx_share, extack);
1051}
1052
1053static int ice_devlink_rate_leaf_tx_priority_set(struct devlink_rate *rate_leaf, void *priv,
1054						 u32 tx_priority, struct netlink_ext_ack *extack)
1055{
1056	struct ice_sched_node *node = priv;
1057
1058	if (!ice_enable_custom_tx(devlink_priv(rate_leaf->devlink)))
1059		return -EBUSY;
1060
1061	if (!node)
1062		return 0;
1063
1064	return ice_set_object_tx_priority(ice_get_pi_from_dev_rate(rate_leaf), node,
1065					  tx_priority, extack);
1066}
1067
1068static int ice_devlink_rate_leaf_tx_weight_set(struct devlink_rate *rate_leaf, void *priv,
1069					       u32 tx_weight, struct netlink_ext_ack *extack)
1070{
1071	struct ice_sched_node *node = priv;
1072
1073	if (!ice_enable_custom_tx(devlink_priv(rate_leaf->devlink)))
1074		return -EBUSY;
1075
1076	if (!node)
1077		return 0;
1078
1079	return ice_set_object_tx_weight(ice_get_pi_from_dev_rate(rate_leaf), node,
1080					tx_weight, extack);
1081}
1082
1083static int ice_devlink_rate_node_tx_max_set(struct devlink_rate *rate_node, void *priv,
1084					    u64 tx_max, struct netlink_ext_ack *extack)
1085{
1086	struct ice_sched_node *node = priv;
1087
1088	if (!ice_enable_custom_tx(devlink_priv(rate_node->devlink)))
1089		return -EBUSY;
1090
1091	if (!node)
1092		return 0;
1093
1094	return ice_set_object_tx_max(ice_get_pi_from_dev_rate(rate_node),
1095				     node, tx_max, extack);
1096}
1097
1098static int ice_devlink_rate_node_tx_share_set(struct devlink_rate *rate_node, void *priv,
1099					      u64 tx_share, struct netlink_ext_ack *extack)
1100{
1101	struct ice_sched_node *node = priv;
1102
1103	if (!ice_enable_custom_tx(devlink_priv(rate_node->devlink)))
1104		return -EBUSY;
1105
1106	if (!node)
1107		return 0;
1108
1109	return ice_set_object_tx_share(ice_get_pi_from_dev_rate(rate_node),
1110				       node, tx_share, extack);
1111}
1112
1113static int ice_devlink_rate_node_tx_priority_set(struct devlink_rate *rate_node, void *priv,
1114						 u32 tx_priority, struct netlink_ext_ack *extack)
1115{
1116	struct ice_sched_node *node = priv;
1117
1118	if (!ice_enable_custom_tx(devlink_priv(rate_node->devlink)))
1119		return -EBUSY;
1120
1121	if (!node)
1122		return 0;
1123
1124	return ice_set_object_tx_priority(ice_get_pi_from_dev_rate(rate_node),
1125					  node, tx_priority, extack);
1126}
1127
1128static int ice_devlink_rate_node_tx_weight_set(struct devlink_rate *rate_node, void *priv,
1129					       u32 tx_weight, struct netlink_ext_ack *extack)
1130{
1131	struct ice_sched_node *node = priv;
1132
1133	if (!ice_enable_custom_tx(devlink_priv(rate_node->devlink)))
1134		return -EBUSY;
1135
1136	if (!node)
1137		return 0;
1138
1139	return ice_set_object_tx_weight(ice_get_pi_from_dev_rate(rate_node),
1140					node, tx_weight, extack);
1141}
1142
1143static int ice_devlink_set_parent(struct devlink_rate *devlink_rate,
1144				  struct devlink_rate *parent,
1145				  void *priv, void *parent_priv,
1146				  struct netlink_ext_ack *extack)
1147{
1148	struct ice_port_info *pi = ice_get_pi_from_dev_rate(devlink_rate);
1149	struct ice_sched_node *tc_node, *node, *parent_node;
1150	u16 num_nodes_added;
1151	u32 first_node_teid;
1152	u32 node_teid;
1153	int status;
1154
1155	tc_node = pi->root->children[0];
1156	node = priv;
1157
1158	if (!extack)
1159		return 0;
1160
1161	if (!ice_enable_custom_tx(devlink_priv(devlink_rate->devlink)))
1162		return -EBUSY;
1163
1164	if (!parent) {
1165		if (!node || tc_node == node || node->num_children)
1166			return -EINVAL;
1167
1168		mutex_lock(&pi->sched_lock);
1169		ice_free_sched_node(pi, node);
1170		mutex_unlock(&pi->sched_lock);
1171
1172		return 0;
1173	}
1174
1175	parent_node = parent_priv;
1176
1177	/* if the node doesn't exist, create it */
1178	if (!node->parent) {
1179		mutex_lock(&pi->sched_lock);
1180		status = ice_sched_add_elems(pi, tc_node, parent_node,
1181					     parent_node->tx_sched_layer + 1,
1182					     1, &num_nodes_added, &first_node_teid,
1183					     &node);
1184		mutex_unlock(&pi->sched_lock);
1185
1186		if (status) {
1187			NL_SET_ERR_MSG_MOD(extack, "Can't add a new node");
1188			return status;
1189		}
1190
1191		if (devlink_rate->tx_share)
1192			ice_set_object_tx_share(pi, node, devlink_rate->tx_share, extack);
1193		if (devlink_rate->tx_max)
1194			ice_set_object_tx_max(pi, node, devlink_rate->tx_max, extack);
1195		if (devlink_rate->tx_priority)
1196			ice_set_object_tx_priority(pi, node, devlink_rate->tx_priority, extack);
1197		if (devlink_rate->tx_weight)
1198			ice_set_object_tx_weight(pi, node, devlink_rate->tx_weight, extack);
1199	} else {
1200		node_teid = le32_to_cpu(node->info.node_teid);
1201		mutex_lock(&pi->sched_lock);
1202		status = ice_sched_move_nodes(pi, parent_node, 1, &node_teid);
1203		mutex_unlock(&pi->sched_lock);
1204
1205		if (status)
1206			NL_SET_ERR_MSG_MOD(extack, "Can't move existing node to a new parent");
1207	}
1208
1209	return status;
1210}
1211
1212static void ice_set_min_max_msix(struct ice_pf *pf)
1213{
1214	struct devlink *devlink = priv_to_devlink(pf);
1215	union devlink_param_value val;
1216	int err;
1217
1218	err = devl_param_driverinit_value_get(devlink,
1219					      DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
1220					      &val);
1221	if (!err)
1222		pf->msix.min = val.vu32;
1223
1224	err = devl_param_driverinit_value_get(devlink,
1225					      DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
1226					      &val);
1227	if (!err)
1228		pf->msix.max = val.vu32;
1229}
1230
1231/**
1232 * ice_devlink_reinit_up - do reinit of the given PF
1233 * @pf: pointer to the PF struct
1234 */
1235static int ice_devlink_reinit_up(struct ice_pf *pf)
1236{
1237	struct ice_vsi *vsi = ice_get_main_vsi(pf);
1238	struct device *dev = ice_pf_to_dev(pf);
1239	bool need_dev_deinit = false;
1240	int err;
1241
1242	err = ice_init_hw(&pf->hw);
1243	if (err) {
1244		dev_err(dev, "ice_init_hw failed: %d\n", err);
1245		return err;
1246	}
1247
1248	/* load MSI-X values */
1249	ice_set_min_max_msix(pf);
1250
1251	err = ice_init_dev(pf);
1252	if (err)
1253		goto unroll_hw_init;
1254
1255	err = ice_init_pf(pf);
1256	if (err) {
1257		dev_err(dev, "ice_init_pf failed: %d\n", err);
1258		goto unroll_dev_init;
1259	}
1260
1261	vsi->flags = ICE_VSI_FLAG_INIT;
1262
1263	rtnl_lock();
1264	err = ice_vsi_cfg(vsi);
1265	rtnl_unlock();
1266	if (err)
1267		goto unroll_pf_init;
1268
1269	/* No need to take devl_lock, it's already taken by devlink API */
1270	err = ice_load(pf);
1271	if (err)
1272		goto err_load;
1273
1274	return 0;
1275
1276err_load:
1277	rtnl_lock();
1278	ice_vsi_decfg(vsi);
1279	rtnl_unlock();
1280unroll_pf_init:
1281	ice_deinit_pf(pf);
1282unroll_dev_init:
1283	need_dev_deinit = true;
1284unroll_hw_init:
1285	ice_deinit_hw(&pf->hw);
1286	if (need_dev_deinit)
1287		ice_deinit_dev(pf);
1288	return err;
1289}
1290
1291/**
1292 * ice_devlink_reload_up - do reload up after reinit
1293 * @devlink: pointer to the devlink instance reloading
1294 * @action: the action requested
1295 * @limit: limits imposed by userspace, such as not resetting
1296 * @actions_performed: on return, indicate what actions actually performed
1297 * @extack: netlink extended ACK structure
1298 */
1299static int
1300ice_devlink_reload_up(struct devlink *devlink,
1301		      enum devlink_reload_action action,
1302		      enum devlink_reload_limit limit,
1303		      u32 *actions_performed,
1304		      struct netlink_ext_ack *extack)
1305{
1306	struct ice_pf *pf = devlink_priv(devlink);
1307
1308	switch (action) {
1309	case DEVLINK_RELOAD_ACTION_DRIVER_REINIT:
1310		*actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT);
1311		return ice_devlink_reinit_up(pf);
1312	case DEVLINK_RELOAD_ACTION_FW_ACTIVATE:
1313		*actions_performed = BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE);
1314		return ice_devlink_reload_empr_finish(pf, extack);
1315	default:
1316		WARN_ON(1);
1317		return -EOPNOTSUPP;
1318	}
1319}
1320
1321static const struct devlink_ops ice_devlink_ops = {
1322	.supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK,
1323	.reload_actions = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
1324			  BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE),
1325	.reload_down = ice_devlink_reload_down,
1326	.reload_up = ice_devlink_reload_up,
1327	.eswitch_mode_get = ice_eswitch_mode_get,
1328	.eswitch_mode_set = ice_eswitch_mode_set,
1329	.info_get = ice_devlink_info_get,
1330	.flash_update = ice_devlink_flash_update,
1331
1332	.rate_node_new = ice_devlink_rate_node_new,
1333	.rate_node_del = ice_devlink_rate_node_del,
1334
1335	.rate_leaf_tx_max_set = ice_devlink_rate_leaf_tx_max_set,
1336	.rate_leaf_tx_share_set = ice_devlink_rate_leaf_tx_share_set,
1337	.rate_leaf_tx_priority_set = ice_devlink_rate_leaf_tx_priority_set,
1338	.rate_leaf_tx_weight_set = ice_devlink_rate_leaf_tx_weight_set,
1339
1340	.rate_node_tx_max_set = ice_devlink_rate_node_tx_max_set,
1341	.rate_node_tx_share_set = ice_devlink_rate_node_tx_share_set,
1342	.rate_node_tx_priority_set = ice_devlink_rate_node_tx_priority_set,
1343	.rate_node_tx_weight_set = ice_devlink_rate_node_tx_weight_set,
1344
1345	.rate_leaf_parent_set = ice_devlink_set_parent,
1346	.rate_node_parent_set = ice_devlink_set_parent,
1347
1348	.port_new = ice_devlink_port_new,
1349};
1350
1351static const struct devlink_ops ice_sf_devlink_ops;
1352
1353static int
1354ice_devlink_enable_roce_get(struct devlink *devlink, u32 id,
1355			    struct devlink_param_gset_ctx *ctx,
1356			    struct netlink_ext_ack *extack)
1357{
1358	struct ice_pf *pf = devlink_priv(devlink);
1359	struct iidc_rdma_core_dev_info *cdev;
1360
1361	cdev = pf->cdev_info;
1362	if (!cdev)
1363		return -EOPNOTSUPP;
1364
1365	ctx->val.vbool = !!(cdev->rdma_protocol & IIDC_RDMA_PROTOCOL_ROCEV2);
1366
1367	return 0;
1368}
1369
1370static int ice_devlink_enable_roce_set(struct devlink *devlink, u32 id,
1371				       struct devlink_param_gset_ctx *ctx,
1372				       struct netlink_ext_ack *extack)
1373{
1374	struct ice_pf *pf = devlink_priv(devlink);
1375	struct iidc_rdma_core_dev_info *cdev;
1376	bool roce_ena = ctx->val.vbool;
1377	int ret;
1378
1379	cdev = pf->cdev_info;
1380	if (!cdev)
1381		return -ENODEV;
1382
1383	if (!roce_ena) {
1384		ice_unplug_aux_dev(pf);
1385		cdev->rdma_protocol &= ~IIDC_RDMA_PROTOCOL_ROCEV2;
1386		return 0;
1387	}
1388
1389	cdev->rdma_protocol |= IIDC_RDMA_PROTOCOL_ROCEV2;
1390	ret = ice_plug_aux_dev(pf);
1391	if (ret)
1392		cdev->rdma_protocol &= ~IIDC_RDMA_PROTOCOL_ROCEV2;
1393
1394	return ret;
1395}
1396
1397static int
1398ice_devlink_enable_roce_validate(struct devlink *devlink, u32 id,
1399				 union devlink_param_value val,
1400				 struct netlink_ext_ack *extack)
1401{
1402	struct ice_pf *pf = devlink_priv(devlink);
1403	struct iidc_rdma_core_dev_info *cdev;
1404
1405	cdev = pf->cdev_info;
1406	if (!cdev)
1407		return -ENODEV;
1408
1409	if (!test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
1410		return -EOPNOTSUPP;
1411
1412	if (cdev->rdma_protocol & IIDC_RDMA_PROTOCOL_IWARP) {
1413		NL_SET_ERR_MSG_MOD(extack, "iWARP is currently enabled. This device cannot enable iWARP and RoCEv2 simultaneously");
1414		return -EOPNOTSUPP;
1415	}
1416
1417	return 0;
1418}
1419
1420static int
1421ice_devlink_enable_iw_get(struct devlink *devlink, u32 id,
1422			  struct devlink_param_gset_ctx *ctx,
1423			  struct netlink_ext_ack *extack)
1424{
1425	struct ice_pf *pf = devlink_priv(devlink);
1426	struct iidc_rdma_core_dev_info *cdev;
1427
1428	cdev = pf->cdev_info;
1429	if (!cdev)
1430		return -EOPNOTSUPP;
1431
1432	ctx->val.vbool = !!(cdev->rdma_protocol & IIDC_RDMA_PROTOCOL_IWARP);
1433
1434	return 0;
1435}
1436
1437static int ice_devlink_enable_iw_set(struct devlink *devlink, u32 id,
1438				     struct devlink_param_gset_ctx *ctx,
1439				     struct netlink_ext_ack *extack)
1440{
1441	struct ice_pf *pf = devlink_priv(devlink);
1442	struct iidc_rdma_core_dev_info *cdev;
1443	bool iw_ena = ctx->val.vbool;
1444	int ret;
1445
1446	cdev = pf->cdev_info;
1447	if (!cdev)
1448		return -ENODEV;
1449
1450	if (!iw_ena) {
1451		ice_unplug_aux_dev(pf);
1452		cdev->rdma_protocol &= ~IIDC_RDMA_PROTOCOL_IWARP;
1453		return 0;
1454	}
1455
1456	cdev->rdma_protocol |= IIDC_RDMA_PROTOCOL_IWARP;
1457	ret = ice_plug_aux_dev(pf);
1458	if (ret)
1459		cdev->rdma_protocol &= ~IIDC_RDMA_PROTOCOL_IWARP;
1460
1461	return ret;
1462}
1463
1464static int
1465ice_devlink_enable_iw_validate(struct devlink *devlink, u32 id,
1466			       union devlink_param_value val,
1467			       struct netlink_ext_ack *extack)
1468{
1469	struct ice_pf *pf = devlink_priv(devlink);
1470
1471	if (!test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
1472		return -EOPNOTSUPP;
1473
1474	if (pf->cdev_info->rdma_protocol & IIDC_RDMA_PROTOCOL_ROCEV2) {
1475		NL_SET_ERR_MSG_MOD(extack, "RoCEv2 is currently enabled. This device cannot enable iWARP and RoCEv2 simultaneously");
1476		return -EOPNOTSUPP;
1477	}
1478
1479	return 0;
1480}
1481
1482#define DEVLINK_LOCAL_FWD_DISABLED_STR "disabled"
1483#define DEVLINK_LOCAL_FWD_ENABLED_STR "enabled"
1484#define DEVLINK_LOCAL_FWD_PRIORITIZED_STR "prioritized"
1485
1486/**
1487 * ice_devlink_local_fwd_mode_to_str - Get string for local_fwd mode.
1488 * @mode: local forwarding for mode used in port_info struct.
1489 *
1490 * Return: Mode respective string or "Invalid".
1491 */
1492static const char *
1493ice_devlink_local_fwd_mode_to_str(enum ice_local_fwd_mode mode)
1494{
1495	switch (mode) {
1496	case ICE_LOCAL_FWD_MODE_ENABLED:
1497		return DEVLINK_LOCAL_FWD_ENABLED_STR;
1498	case ICE_LOCAL_FWD_MODE_PRIORITIZED:
1499		return DEVLINK_LOCAL_FWD_PRIORITIZED_STR;
1500	case ICE_LOCAL_FWD_MODE_DISABLED:
1501		return DEVLINK_LOCAL_FWD_DISABLED_STR;
1502	}
1503
1504	return "Invalid";
1505}
1506
1507/**
1508 * ice_devlink_local_fwd_str_to_mode - Get local_fwd mode from string name.
1509 * @mode_str: local forwarding mode string.
1510 *
1511 * Return: Mode value or negative number if invalid.
1512 */
1513static int ice_devlink_local_fwd_str_to_mode(const char *mode_str)
1514{
1515	if (!strcmp(mode_str, DEVLINK_LOCAL_FWD_ENABLED_STR))
1516		return ICE_LOCAL_FWD_MODE_ENABLED;
1517	else if (!strcmp(mode_str, DEVLINK_LOCAL_FWD_PRIORITIZED_STR))
1518		return ICE_LOCAL_FWD_MODE_PRIORITIZED;
1519	else if (!strcmp(mode_str, DEVLINK_LOCAL_FWD_DISABLED_STR))
1520		return ICE_LOCAL_FWD_MODE_DISABLED;
1521
1522	return -EINVAL;
1523}
1524
1525/**
1526 * ice_devlink_local_fwd_get - Get local_fwd parameter.
1527 * @devlink: Pointer to the devlink instance.
1528 * @id: The parameter ID to set.
1529 * @ctx: Context to store the parameter value.
1530 * @extack: netlink extended ACK structure
1531 *
1532 * Return: Zero.
1533 */
1534static int ice_devlink_local_fwd_get(struct devlink *devlink, u32 id,
1535				     struct devlink_param_gset_ctx *ctx,
1536				     struct netlink_ext_ack *extack)
1537{
1538	struct ice_pf *pf = devlink_priv(devlink);
1539	struct ice_port_info *pi;
1540	const char *mode_str;
1541
1542	pi = pf->hw.port_info;
1543	mode_str = ice_devlink_local_fwd_mode_to_str(pi->local_fwd_mode);
1544	snprintf(ctx->val.vstr, sizeof(ctx->val.vstr), "%s", mode_str);
1545
1546	return 0;
1547}
1548
1549/**
1550 * ice_devlink_local_fwd_set - Set local_fwd parameter.
1551 * @devlink: Pointer to the devlink instance.
1552 * @id: The parameter ID to set.
1553 * @ctx: Context to get the parameter value.
1554 * @extack: Netlink extended ACK structure.
1555 *
1556 * Return: Zero.
1557 */
1558static int ice_devlink_local_fwd_set(struct devlink *devlink, u32 id,
1559				     struct devlink_param_gset_ctx *ctx,
1560				     struct netlink_ext_ack *extack)
1561{
1562	int new_local_fwd_mode = ice_devlink_local_fwd_str_to_mode(ctx->val.vstr);
1563	struct ice_pf *pf = devlink_priv(devlink);
1564	struct device *dev = ice_pf_to_dev(pf);
1565	struct ice_port_info *pi;
1566
1567	pi = pf->hw.port_info;
1568	if (pi->local_fwd_mode != new_local_fwd_mode) {
1569		pi->local_fwd_mode = new_local_fwd_mode;
1570		dev_info(dev, "Setting local_fwd to %s\n", ctx->val.vstr);
1571		ice_schedule_reset(pf, ICE_RESET_CORER);
1572	}
1573
1574	return 0;
1575}
1576
1577/**
1578 * ice_devlink_local_fwd_validate - Validate passed local_fwd parameter value.
1579 * @devlink: Unused pointer to devlink instance.
1580 * @id: The parameter ID to validate.
1581 * @val: Value to validate.
1582 * @extack: Netlink extended ACK structure.
1583 *
1584 * Supported values are:
1585 * "enabled" - local_fwd is enabled, "disabled" - local_fwd is disabled
1586 * "prioritized" - local_fwd traffic is prioritized in scheduling.
1587 *
1588 * Return: Zero when passed parameter value is supported. Negative value on
1589 * error.
1590 */
1591static int ice_devlink_local_fwd_validate(struct devlink *devlink, u32 id,
1592					  union devlink_param_value val,
1593					  struct netlink_ext_ack *extack)
1594{
1595	if (ice_devlink_local_fwd_str_to_mode(val.vstr) < 0) {
1596		NL_SET_ERR_MSG_MOD(extack, "Error: Requested value is not supported.");
1597		return -EINVAL;
1598	}
1599
1600	return 0;
1601}
1602
1603static int
1604ice_devlink_msix_max_pf_validate(struct devlink *devlink, u32 id,
1605				 union devlink_param_value val,
1606				 struct netlink_ext_ack *extack)
1607{
1608	struct ice_pf *pf = devlink_priv(devlink);
1609
1610	if (val.vu32 > pf->hw.func_caps.common_cap.num_msix_vectors)
1611		return -EINVAL;
1612
1613	return 0;
1614}
1615
1616static int
1617ice_devlink_msix_min_pf_validate(struct devlink *devlink, u32 id,
1618				 union devlink_param_value val,
1619				 struct netlink_ext_ack *extack)
1620{
1621	if (val.vu32 < ICE_MIN_MSIX)
1622		return -EINVAL;
1623
1624	return 0;
1625}
1626
1627static int ice_devlink_enable_rdma_validate(struct devlink *devlink, u32 id,
1628					    union devlink_param_value val,
1629					    struct netlink_ext_ack *extack)
1630{
1631	struct ice_pf *pf = devlink_priv(devlink);
1632	bool new_state = val.vbool;
1633
1634	if (new_state && !test_bit(ICE_FLAG_RDMA_ENA, pf->flags))
1635		return -EOPNOTSUPP;
1636
1637	return 0;
1638}
1639
1640enum ice_param_id {
1641	ICE_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
1642	ICE_DEVLINK_PARAM_ID_TX_SCHED_LAYERS,
1643	ICE_DEVLINK_PARAM_ID_LOCAL_FWD,
1644};
1645
1646static const struct devlink_param ice_dvl_rdma_params[] = {
1647	DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
1648			      ice_devlink_enable_roce_get,
1649			      ice_devlink_enable_roce_set,
1650			      ice_devlink_enable_roce_validate),
1651	DEVLINK_PARAM_GENERIC(ENABLE_IWARP, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
1652			      ice_devlink_enable_iw_get,
1653			      ice_devlink_enable_iw_set,
1654			      ice_devlink_enable_iw_validate),
1655	DEVLINK_PARAM_GENERIC(ENABLE_RDMA, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
1656			      NULL, NULL, ice_devlink_enable_rdma_validate),
1657};
1658
1659static const struct devlink_param ice_dvl_msix_params[] = {
1660	DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MAX,
1661			      BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
1662			      NULL, NULL, ice_devlink_msix_max_pf_validate),
1663	DEVLINK_PARAM_GENERIC(MSIX_VEC_PER_PF_MIN,
1664			      BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
1665			      NULL, NULL, ice_devlink_msix_min_pf_validate),
1666};
1667
1668static const struct devlink_param ice_dvl_sched_params[] = {
1669	DEVLINK_PARAM_DRIVER(ICE_DEVLINK_PARAM_ID_TX_SCHED_LAYERS,
1670			     "tx_scheduling_layers",
1671			     DEVLINK_PARAM_TYPE_U8,
1672			     BIT(DEVLINK_PARAM_CMODE_PERMANENT),
1673			     ice_devlink_tx_sched_layers_get,
1674			     ice_devlink_tx_sched_layers_set,
1675			     ice_devlink_tx_sched_layers_validate),
1676	DEVLINK_PARAM_DRIVER(ICE_DEVLINK_PARAM_ID_LOCAL_FWD,
1677			     "local_forwarding", DEVLINK_PARAM_TYPE_STRING,
1678			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),
1679			     ice_devlink_local_fwd_get,
1680			     ice_devlink_local_fwd_set,
1681			     ice_devlink_local_fwd_validate),
1682};
1683
1684static void ice_devlink_free(void *devlink_ptr)
1685{
1686	devlink_free((struct devlink *)devlink_ptr);
1687}
1688
1689/**
1690 * ice_allocate_pf - Allocate devlink and return PF structure pointer
1691 * @dev: the device to allocate for
1692 *
1693 * Allocate a devlink instance for this device and return the private area as
1694 * the PF structure. The devlink memory is kept track of through devres by
1695 * adding an action to remove it when unwinding.
1696 */
1697struct ice_pf *ice_allocate_pf(struct device *dev)
1698{
1699	struct devlink *devlink;
1700
1701	devlink = devlink_alloc(&ice_devlink_ops, sizeof(struct ice_pf), dev);
1702	if (!devlink)
1703		return NULL;
1704
1705	/* Add an action to teardown the devlink when unwinding the driver */
1706	if (devm_add_action_or_reset(dev, ice_devlink_free, devlink))
1707		return NULL;
1708
1709	return devlink_priv(devlink);
1710}
1711
1712/**
1713 * ice_allocate_sf - Allocate devlink and return SF structure pointer
1714 * @dev: the device to allocate for
1715 * @pf: pointer to the PF structure
1716 *
1717 * Allocate a devlink instance for SF.
1718 *
1719 * Return: ice_sf_priv pointer to allocated memory or ERR_PTR in case of error
1720 */
1721struct ice_sf_priv *ice_allocate_sf(struct device *dev, struct ice_pf *pf)
1722{
1723	struct devlink *devlink;
1724	int err;
1725
1726	devlink = devlink_alloc(&ice_sf_devlink_ops, sizeof(struct ice_sf_priv),
1727				dev);
1728	if (!devlink)
1729		return ERR_PTR(-ENOMEM);
1730
1731	err = devl_nested_devlink_set(priv_to_devlink(pf), devlink);
1732	if (err) {
1733		devlink_free(devlink);
1734		return ERR_PTR(err);
1735	}
1736
1737	return devlink_priv(devlink);
1738}
1739
1740/**
1741 * ice_devlink_register - Register devlink interface for this PF
1742 * @pf: the PF to register the devlink for.
1743 *
1744 * Register the devlink instance associated with this physical function.
1745 *
1746 * Return: zero on success or an error code on failure.
1747 */
1748void ice_devlink_register(struct ice_pf *pf)
1749{
1750	struct devlink *devlink = priv_to_devlink(pf);
1751
1752	devl_register(devlink);
1753}
1754
1755/**
1756 * ice_devlink_unregister - Unregister devlink resources for this PF.
1757 * @pf: the PF structure to cleanup
1758 *
1759 * Releases resources used by devlink and cleans up associated memory.
1760 */
1761void ice_devlink_unregister(struct ice_pf *pf)
1762{
1763	devl_unregister(priv_to_devlink(pf));
1764}
1765
1766int ice_devlink_register_params(struct ice_pf *pf)
1767{
1768	struct devlink *devlink = priv_to_devlink(pf);
1769	union devlink_param_value value;
1770	struct ice_hw *hw = &pf->hw;
1771	int status;
1772
1773	status = devl_params_register(devlink, ice_dvl_rdma_params,
1774				      ARRAY_SIZE(ice_dvl_rdma_params));
1775	if (status)
1776		return status;
1777
1778	status = devl_params_register(devlink, ice_dvl_msix_params,
1779				      ARRAY_SIZE(ice_dvl_msix_params));
1780	if (status)
1781		goto unregister_rdma_params;
1782
1783	if (hw->func_caps.common_cap.tx_sched_topo_comp_mode_en)
1784		status = devl_params_register(devlink, ice_dvl_sched_params,
1785					      ARRAY_SIZE(ice_dvl_sched_params));
1786	if (status)
1787		goto unregister_msix_params;
1788
1789	value.vu32 = pf->msix.max;
1790	devl_param_driverinit_value_set(devlink,
1791					DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
1792					value);
1793	value.vu32 = pf->msix.min;
1794	devl_param_driverinit_value_set(devlink,
1795					DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
1796					value);
1797
1798	value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags);
1799	devl_param_driverinit_value_set(devlink,
1800					DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
1801					value);
1802
1803	return 0;
1804
1805unregister_msix_params:
1806	devl_params_unregister(devlink, ice_dvl_msix_params,
1807			       ARRAY_SIZE(ice_dvl_msix_params));
1808unregister_rdma_params:
1809	devl_params_unregister(devlink, ice_dvl_rdma_params,
1810			       ARRAY_SIZE(ice_dvl_rdma_params));
1811	return status;
1812}
1813
1814void ice_devlink_unregister_params(struct ice_pf *pf)
1815{
1816	struct devlink *devlink = priv_to_devlink(pf);
1817	struct ice_hw *hw = &pf->hw;
1818
1819	devl_params_unregister(devlink, ice_dvl_rdma_params,
1820			       ARRAY_SIZE(ice_dvl_rdma_params));
1821	devl_params_unregister(devlink, ice_dvl_msix_params,
1822			       ARRAY_SIZE(ice_dvl_msix_params));
1823
1824	if (hw->func_caps.common_cap.tx_sched_topo_comp_mode_en)
1825		devl_params_unregister(devlink, ice_dvl_sched_params,
1826				       ARRAY_SIZE(ice_dvl_sched_params));
1827}
1828
1829#define ICE_DEVLINK_READ_BLK_SIZE (1024 * 1024)
1830
1831static const struct devlink_region_ops ice_nvm_region_ops;
1832static const struct devlink_region_ops ice_sram_region_ops;
1833
1834/**
1835 * ice_devlink_nvm_snapshot - Capture a snapshot of the NVM flash contents
1836 * @devlink: the devlink instance
1837 * @ops: the devlink region to snapshot
1838 * @extack: extended ACK response structure
1839 * @data: on exit points to snapshot data buffer
1840 *
1841 * This function is called in response to a DEVLINK_CMD_REGION_NEW for either
1842 * the nvm-flash or shadow-ram region.
1843 *
1844 * It captures a snapshot of the NVM or Shadow RAM flash contents. This
1845 * snapshot can then later be viewed via the DEVLINK_CMD_REGION_READ netlink
1846 * interface.
1847 *
1848 * @returns zero on success, and updates the data pointer. Returns a non-zero
1849 * error code on failure.
1850 */
1851static int ice_devlink_nvm_snapshot(struct devlink *devlink,
1852				    const struct devlink_region_ops *ops,
1853				    struct netlink_ext_ack *extack, u8 **data)
1854{
1855	struct ice_pf *pf = devlink_priv(devlink);
1856	struct device *dev = ice_pf_to_dev(pf);
1857	struct ice_hw *hw = &pf->hw;
1858	bool read_shadow_ram;
1859	u8 *nvm_data, *tmp, i;
1860	u32 nvm_size, left;
1861	s8 num_blks;
1862	int status;
1863
1864	if (ops == &ice_nvm_region_ops) {
1865		read_shadow_ram = false;
1866		nvm_size = hw->flash.flash_size;
1867	} else if (ops == &ice_sram_region_ops) {
1868		read_shadow_ram = true;
1869		nvm_size = hw->flash.sr_words * 2u;
1870	} else {
1871		NL_SET_ERR_MSG_MOD(extack, "Unexpected region in snapshot function");
1872		return -EOPNOTSUPP;
1873	}
1874
1875	nvm_data = vzalloc(nvm_size);
1876	if (!nvm_data)
1877		return -ENOMEM;
1878
1879	num_blks = DIV_ROUND_UP(nvm_size, ICE_DEVLINK_READ_BLK_SIZE);
1880	tmp = nvm_data;
1881	left = nvm_size;
1882
1883	/* Some systems take longer to read the NVM than others which causes the
1884	 * FW to reclaim the NVM lock before the entire NVM has been read. Fix
1885	 * this by breaking the reads of the NVM into smaller chunks that will
1886	 * probably not take as long. This has some overhead since we are
1887	 * increasing the number of AQ commands, but it should always work
1888	 */
1889	for (i = 0; i < num_blks; i++) {
1890		u32 read_sz = min_t(u32, ICE_DEVLINK_READ_BLK_SIZE, left);
1891
1892		status = ice_acquire_nvm(hw, ICE_RES_READ);
1893		if (status) {
1894			dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n",
1895				status, hw->adminq.sq_last_status);
1896			NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore");
1897			vfree(nvm_data);
1898			return -EIO;
1899		}
1900
1901		status = ice_read_flat_nvm(hw, i * ICE_DEVLINK_READ_BLK_SIZE,
1902					   &read_sz, tmp, read_shadow_ram);
1903		if (status) {
1904			dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n",
1905				read_sz, status, hw->adminq.sq_last_status);
1906			NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents");
1907			ice_release_nvm(hw);
1908			vfree(nvm_data);
1909			return -EIO;
1910		}
1911		ice_release_nvm(hw);
1912
1913		tmp += read_sz;
1914		left -= read_sz;
1915	}
1916
1917	*data = nvm_data;
1918
1919	return 0;
1920}
1921
1922/**
1923 * ice_devlink_nvm_read - Read a portion of NVM flash contents
1924 * @devlink: the devlink instance
1925 * @ops: the devlink region to snapshot
1926 * @extack: extended ACK response structure
1927 * @offset: the offset to start at
1928 * @size: the amount to read
1929 * @data: the data buffer to read into
1930 *
1931 * This function is called in response to DEVLINK_CMD_REGION_READ to directly
1932 * read a section of the NVM contents.
1933 *
1934 * It reads from either the nvm-flash or shadow-ram region contents.
1935 *
1936 * @returns zero on success, and updates the data pointer. Returns a non-zero
1937 * error code on failure.
1938 */
1939static int ice_devlink_nvm_read(struct devlink *devlink,
1940				const struct devlink_region_ops *ops,
1941				struct netlink_ext_ack *extack,
1942				u64 offset, u32 size, u8 *data)
1943{
1944	struct ice_pf *pf = devlink_priv(devlink);
1945	struct device *dev = ice_pf_to_dev(pf);
1946	struct ice_hw *hw = &pf->hw;
1947	bool read_shadow_ram;
1948	u64 nvm_size;
1949	int status;
1950
1951	if (ops == &ice_nvm_region_ops) {
1952		read_shadow_ram = false;
1953		nvm_size = hw->flash.flash_size;
1954	} else if (ops == &ice_sram_region_ops) {
1955		read_shadow_ram = true;
1956		nvm_size = hw->flash.sr_words * 2u;
1957	} else {
1958		NL_SET_ERR_MSG_MOD(extack, "Unexpected region in snapshot function");
1959		return -EOPNOTSUPP;
1960	}
1961
1962	if (offset + size >= nvm_size) {
1963		NL_SET_ERR_MSG_MOD(extack, "Cannot read beyond the region size");
1964		return -ERANGE;
1965	}
1966
1967	status = ice_acquire_nvm(hw, ICE_RES_READ);
1968	if (status) {
1969		dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n",
1970			status, hw->adminq.sq_last_status);
1971		NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore");
1972		return -EIO;
1973	}
1974
1975	status = ice_read_flat_nvm(hw, (u32)offset, &size, data,
1976				   read_shadow_ram);
1977	if (status) {
1978		dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n",
1979			size, status, hw->adminq.sq_last_status);
1980		NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents");
1981		ice_release_nvm(hw);
1982		return -EIO;
1983	}
1984	ice_release_nvm(hw);
1985
1986	return 0;
1987}
1988
1989/**
1990 * ice_devlink_devcaps_snapshot - Capture snapshot of device capabilities
1991 * @devlink: the devlink instance
1992 * @ops: the devlink region being snapshotted
1993 * @extack: extended ACK response structure
1994 * @data: on exit points to snapshot data buffer
1995 *
1996 * This function is called in response to the DEVLINK_CMD_REGION_TRIGGER for
1997 * the device-caps devlink region. It captures a snapshot of the device
1998 * capabilities reported by firmware.
1999 *
2000 * @returns zero on success, and updates the data pointer. Returns a non-zero
2001 * error code on failure.
2002 */
2003static int
2004ice_devlink_devcaps_snapshot(struct devlink *devlink,
2005			     const struct devlink_region_ops *ops,
2006			     struct netlink_ext_ack *extack, u8 **data)
2007{
2008	struct ice_pf *pf = devlink_priv(devlink);
2009	struct device *dev = ice_pf_to_dev(pf);
2010	struct ice_hw *hw = &pf->hw;
2011	void *devcaps;
2012	int status;
2013
2014	devcaps = vzalloc(ICE_AQ_MAX_BUF_LEN);
2015	if (!devcaps)
2016		return -ENOMEM;
2017
2018	status = ice_aq_list_caps(hw, devcaps, ICE_AQ_MAX_BUF_LEN, NULL,
2019				  ice_aqc_opc_list_dev_caps, NULL);
2020	if (status) {
2021		dev_dbg(dev, "ice_aq_list_caps: failed to read device capabilities, err %d aq_err %d\n",
2022			status, hw->adminq.sq_last_status);
2023		NL_SET_ERR_MSG_MOD(extack, "Failed to read device capabilities");
2024		vfree(devcaps);
2025		return status;
2026	}
2027
2028	*data = (u8 *)devcaps;
2029
2030	return 0;
2031}
2032
2033static const struct devlink_region_ops ice_nvm_region_ops = {
2034	.name = "nvm-flash",
2035	.destructor = vfree,
2036	.snapshot = ice_devlink_nvm_snapshot,
2037	.read = ice_devlink_nvm_read,
2038};
2039
2040static const struct devlink_region_ops ice_sram_region_ops = {
2041	.name = "shadow-ram",
2042	.destructor = vfree,
2043	.snapshot = ice_devlink_nvm_snapshot,
2044	.read = ice_devlink_nvm_read,
2045};
2046
2047static const struct devlink_region_ops ice_devcaps_region_ops = {
2048	.name = "device-caps",
2049	.destructor = vfree,
2050	.snapshot = ice_devlink_devcaps_snapshot,
2051};
2052
2053/**
2054 * ice_devlink_init_regions - Initialize devlink regions
2055 * @pf: the PF device structure
2056 *
2057 * Create devlink regions used to enable access to dump the contents of the
2058 * flash memory on the device.
2059 */
2060void ice_devlink_init_regions(struct ice_pf *pf)
2061{
2062	struct devlink *devlink = priv_to_devlink(pf);
2063	struct device *dev = ice_pf_to_dev(pf);
2064	u64 nvm_size, sram_size;
2065
2066	nvm_size = pf->hw.flash.flash_size;
2067	pf->nvm_region = devl_region_create(devlink, &ice_nvm_region_ops, 1,
2068					    nvm_size);
2069	if (IS_ERR(pf->nvm_region)) {
2070		dev_err(dev, "failed to create NVM devlink region, err %ld\n",
2071			PTR_ERR(pf->nvm_region));
2072		pf->nvm_region = NULL;
2073	}
2074
2075	sram_size = pf->hw.flash.sr_words * 2u;
2076	pf->sram_region = devl_region_create(devlink, &ice_sram_region_ops,
2077					     1, sram_size);
2078	if (IS_ERR(pf->sram_region)) {
2079		dev_err(dev, "failed to create shadow-ram devlink region, err %ld\n",
2080			PTR_ERR(pf->sram_region));
2081		pf->sram_region = NULL;
2082	}
2083
2084	pf->devcaps_region = devl_region_create(devlink,
2085						&ice_devcaps_region_ops, 10,
2086						ICE_AQ_MAX_BUF_LEN);
2087	if (IS_ERR(pf->devcaps_region)) {
2088		dev_err(dev, "failed to create device-caps devlink region, err %ld\n",
2089			PTR_ERR(pf->devcaps_region));
2090		pf->devcaps_region = NULL;
2091	}
2092}
2093
2094/**
2095 * ice_devlink_destroy_regions - Destroy devlink regions
2096 * @pf: the PF device structure
2097 *
2098 * Remove previously created regions for this PF.
2099 */
2100void ice_devlink_destroy_regions(struct ice_pf *pf)
2101{
2102	if (pf->nvm_region)
2103		devl_region_destroy(pf->nvm_region);
2104
2105	if (pf->sram_region)
2106		devl_region_destroy(pf->sram_region);
2107
2108	if (pf->devcaps_region)
2109		devl_region_destroy(pf->devcaps_region);
2110}