spectrum_buffers.c

   1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2/* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
   3
   4#include <linux/kernel.h>
   5#include <linux/types.h>
   6#include <linux/dcbnl.h>
   7#include <linux/if_ether.h>
   8#include <linux/list.h>
   9#include <linux/netlink.h>
  10
  11#include "spectrum.h"
  12#include "core.h"
  13#include "port.h"
  14#include "reg.h"
  15
  16struct mlxsw_sp_sb_pr {
  17	enum mlxsw_reg_sbpr_mode mode;
  18	u32 size;
  19	u8 freeze_mode:1,
  20	   freeze_size:1;
  21};
  22
  23struct mlxsw_cp_sb_occ {
  24	u32 cur;
  25	u32 max;
  26};
  27
  28struct mlxsw_sp_sb_cm {
  29	u32 min_buff;
  30	u32 max_buff;
  31	u16 pool_index;
  32	struct mlxsw_cp_sb_occ occ;
  33	u8 freeze_pool:1,
  34	   freeze_thresh:1;
  35};
  36
  37#define MLXSW_SP_SB_INFI -1U
  38#define MLXSW_SP_SB_REST -2U
  39
  40struct mlxsw_sp_sb_pm {
  41	u32 min_buff;
  42	u32 max_buff;
  43	struct mlxsw_cp_sb_occ occ;
  44};
  45
  46struct mlxsw_sp_sb_mm {
  47	u32 min_buff;
  48	u32 max_buff;
  49	u16 pool_index;
  50};
  51
  52struct mlxsw_sp_sb_pool_des {
  53	enum mlxsw_reg_sbxx_dir dir;
  54	u8 pool;
  55};
  56
  57#define MLXSW_SP_SB_POOL_ING		0
  58#define MLXSW_SP_SB_POOL_EGR		4
  59#define MLXSW_SP_SB_POOL_EGR_MC		8
  60#define MLXSW_SP_SB_POOL_ING_CPU	9
  61#define MLXSW_SP_SB_POOL_EGR_CPU	10
  62
  63static const struct mlxsw_sp_sb_pool_des mlxsw_sp1_sb_pool_dess[] = {
  64	{MLXSW_REG_SBXX_DIR_INGRESS, 0},
  65	{MLXSW_REG_SBXX_DIR_INGRESS, 1},
  66	{MLXSW_REG_SBXX_DIR_INGRESS, 2},
  67	{MLXSW_REG_SBXX_DIR_INGRESS, 3},
  68	{MLXSW_REG_SBXX_DIR_EGRESS, 0},
  69	{MLXSW_REG_SBXX_DIR_EGRESS, 1},
  70	{MLXSW_REG_SBXX_DIR_EGRESS, 2},
  71	{MLXSW_REG_SBXX_DIR_EGRESS, 3},
  72	{MLXSW_REG_SBXX_DIR_EGRESS, 15},
  73	{MLXSW_REG_SBXX_DIR_INGRESS, 4},
  74	{MLXSW_REG_SBXX_DIR_EGRESS, 4},
  75};
  76
  77static const struct mlxsw_sp_sb_pool_des mlxsw_sp2_sb_pool_dess[] = {
  78	{MLXSW_REG_SBXX_DIR_INGRESS, 0},
  79	{MLXSW_REG_SBXX_DIR_INGRESS, 1},
  80	{MLXSW_REG_SBXX_DIR_INGRESS, 2},
  81	{MLXSW_REG_SBXX_DIR_INGRESS, 3},
  82	{MLXSW_REG_SBXX_DIR_EGRESS, 0},
  83	{MLXSW_REG_SBXX_DIR_EGRESS, 1},
  84	{MLXSW_REG_SBXX_DIR_EGRESS, 2},
  85	{MLXSW_REG_SBXX_DIR_EGRESS, 3},
  86	{MLXSW_REG_SBXX_DIR_EGRESS, 15},
  87	{MLXSW_REG_SBXX_DIR_INGRESS, 4},
  88	{MLXSW_REG_SBXX_DIR_EGRESS, 4},
  89};
  90
  91#define MLXSW_SP_SB_ING_TC_COUNT 8
  92#define MLXSW_SP_SB_EG_TC_COUNT 16
  93
  94struct mlxsw_sp_sb_port {
  95	struct mlxsw_sp_sb_cm ing_cms[MLXSW_SP_SB_ING_TC_COUNT];
  96	struct mlxsw_sp_sb_cm eg_cms[MLXSW_SP_SB_EG_TC_COUNT];
  97	struct mlxsw_sp_sb_pm *pms;
  98};
  99
 100struct mlxsw_sp_sb {
 101	struct mlxsw_sp_sb_pr *prs;
 102	struct mlxsw_sp_sb_port *ports;
 103	u32 cell_size;
 104	u32 max_headroom_cells;
 105	u64 sb_size;
 106};
 107
 108struct mlxsw_sp_sb_vals {
 109	unsigned int pool_count;
 110	const struct mlxsw_sp_sb_pool_des *pool_dess;
 111	const struct mlxsw_sp_sb_pm *pms;
 112	const struct mlxsw_sp_sb_pm *pms_cpu;
 113	const struct mlxsw_sp_sb_pr *prs;
 114	const struct mlxsw_sp_sb_mm *mms;
 115	const struct mlxsw_sp_sb_cm *cms_ingress;
 116	const struct mlxsw_sp_sb_cm *cms_egress;
 117	const struct mlxsw_sp_sb_cm *cms_cpu;
 118	unsigned int mms_count;
 119	unsigned int cms_ingress_count;
 120	unsigned int cms_egress_count;
 121	unsigned int cms_cpu_count;
 122};
 123
 124struct mlxsw_sp_sb_ops {
 125	u32 (*int_buf_size_get)(int mtu, u32 speed);
 126};
 127
 128u32 mlxsw_sp_cells_bytes(const struct mlxsw_sp *mlxsw_sp, u32 cells)
 129{
 130	return mlxsw_sp->sb->cell_size * cells;
 131}
 132
 133u32 mlxsw_sp_bytes_cells(const struct mlxsw_sp *mlxsw_sp, u32 bytes)
 134{
 135	return DIV_ROUND_UP(bytes, mlxsw_sp->sb->cell_size);
 136}
 137
 138static u32 mlxsw_sp_port_headroom_8x_adjust(const struct mlxsw_sp_port *mlxsw_sp_port,
 139					    u32 size_cells)
 140{
 141	/* Ports with eight lanes use two headroom buffers between which the
 142	 * configured headroom size is split. Therefore, multiply the calculated
 143	 * headroom size by two.
 144	 */
 145	return mlxsw_sp_port->mapping.width == 8 ? 2 * size_cells : size_cells;
 146}
 147
 148static struct mlxsw_sp_sb_pr *mlxsw_sp_sb_pr_get(struct mlxsw_sp *mlxsw_sp,
 149						 u16 pool_index)
 150{
 151	return &mlxsw_sp->sb->prs[pool_index];
 152}
 153
 154static bool mlxsw_sp_sb_cm_exists(u8 pg_buff, enum mlxsw_reg_sbxx_dir dir)
 155{
 156	if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
 157		return pg_buff < MLXSW_SP_SB_ING_TC_COUNT;
 158	else
 159		return pg_buff < MLXSW_SP_SB_EG_TC_COUNT;
 160}
 161
 162static struct mlxsw_sp_sb_cm *mlxsw_sp_sb_cm_get(struct mlxsw_sp *mlxsw_sp,
 163						 u16 local_port, u8 pg_buff,
 164						 enum mlxsw_reg_sbxx_dir dir)
 165{
 166	struct mlxsw_sp_sb_port *sb_port = &mlxsw_sp->sb->ports[local_port];
 167
 168	WARN_ON(!mlxsw_sp_sb_cm_exists(pg_buff, dir));
 169	if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
 170		return &sb_port->ing_cms[pg_buff];
 171	else
 172		return &sb_port->eg_cms[pg_buff];
 173}
 174
 175static struct mlxsw_sp_sb_pm *mlxsw_sp_sb_pm_get(struct mlxsw_sp *mlxsw_sp,
 176						 u16 local_port, u16 pool_index)
 177{
 178	return &mlxsw_sp->sb->ports[local_port].pms[pool_index];
 179}
 180
 181static int mlxsw_sp_sb_pr_write(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
 182				enum mlxsw_reg_sbpr_mode mode,
 183				u32 size, bool infi_size)
 184{
 185	const struct mlxsw_sp_sb_pool_des *des =
 186		&mlxsw_sp->sb_vals->pool_dess[pool_index];
 187	char sbpr_pl[MLXSW_REG_SBPR_LEN];
 188	struct mlxsw_sp_sb_pr *pr;
 189	int err;
 190
 191	mlxsw_reg_sbpr_pack(sbpr_pl, des->pool, des->dir, mode,
 192			    size, infi_size);
 193	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpr), sbpr_pl);
 194	if (err)
 195		return err;
 196
 197	if (infi_size)
 198		size = mlxsw_sp_bytes_cells(mlxsw_sp, mlxsw_sp->sb->sb_size);
 199	pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
 200	pr->mode = mode;
 201	pr->size = size;
 202	return 0;
 203}
 204
 205static int mlxsw_sp_sb_pr_desc_write(struct mlxsw_sp *mlxsw_sp,
 206				     enum mlxsw_reg_sbxx_dir dir,
 207				     enum mlxsw_reg_sbpr_mode mode,
 208				     u32 size, bool infi_size)
 209{
 210	char sbpr_pl[MLXSW_REG_SBPR_LEN];
 211
 212	/* The FW default descriptor buffer configuration uses only pool 14 for
 213	 * descriptors.
 214	 */
 215	mlxsw_reg_sbpr_pack(sbpr_pl, 14, dir, mode, size, infi_size);
 216	mlxsw_reg_sbpr_desc_set(sbpr_pl, true);
 217	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpr), sbpr_pl);
 218}
 219
 220static int mlxsw_sp_sb_cm_write(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 221				u8 pg_buff, u32 min_buff, u32 max_buff,
 222				bool infi_max, u16 pool_index)
 223{
 224	const struct mlxsw_sp_sb_pool_des *des =
 225		&mlxsw_sp->sb_vals->pool_dess[pool_index];
 226	char sbcm_pl[MLXSW_REG_SBCM_LEN];
 227	struct mlxsw_sp_sb_cm *cm;
 228	int err;
 229
 230	mlxsw_reg_sbcm_pack(sbcm_pl, local_port, pg_buff, des->dir,
 231			    min_buff, max_buff, infi_max, des->pool);
 232	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbcm), sbcm_pl);
 233	if (err)
 234		return err;
 235
 236	if (mlxsw_sp_sb_cm_exists(pg_buff, des->dir)) {
 237		if (infi_max)
 238			max_buff = mlxsw_sp_bytes_cells(mlxsw_sp,
 239							mlxsw_sp->sb->sb_size);
 240
 241		cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, pg_buff,
 242					des->dir);
 243		cm->min_buff = min_buff;
 244		cm->max_buff = max_buff;
 245		cm->pool_index = pool_index;
 246	}
 247	return 0;
 248}
 249
 250static int mlxsw_sp_sb_pm_write(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 251				u16 pool_index, u32 min_buff, u32 max_buff)
 252{
 253	const struct mlxsw_sp_sb_pool_des *des =
 254		&mlxsw_sp->sb_vals->pool_dess[pool_index];
 255	char sbpm_pl[MLXSW_REG_SBPM_LEN];
 256	struct mlxsw_sp_sb_pm *pm;
 257	int err;
 258
 259	mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir, false,
 260			    min_buff, max_buff);
 261	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl);
 262	if (err)
 263		return err;
 264
 265	pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index);
 266	pm->min_buff = min_buff;
 267	pm->max_buff = max_buff;
 268	return 0;
 269}
 270
 271static int mlxsw_sp_sb_pm_occ_clear(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 272				    u16 pool_index, struct list_head *bulk_list)
 273{
 274	const struct mlxsw_sp_sb_pool_des *des =
 275		&mlxsw_sp->sb_vals->pool_dess[pool_index];
 276	char sbpm_pl[MLXSW_REG_SBPM_LEN];
 277
 278	if (local_port == MLXSW_PORT_CPU_PORT &&
 279	    des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
 280		return 0;
 281
 282	mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir,
 283			    true, 0, 0);
 284	return mlxsw_reg_trans_query(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl,
 285				     bulk_list, NULL, 0);
 286}
 287
 288static void mlxsw_sp_sb_pm_occ_query_cb(struct mlxsw_core *mlxsw_core,
 289					char *sbpm_pl, size_t sbpm_pl_len,
 290					unsigned long cb_priv)
 291{
 292	struct mlxsw_sp_sb_pm *pm = (struct mlxsw_sp_sb_pm *) cb_priv;
 293
 294	mlxsw_reg_sbpm_unpack(sbpm_pl, &pm->occ.cur, &pm->occ.max);
 295}
 296
 297static int mlxsw_sp_sb_pm_occ_query(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 298				    u16 pool_index, struct list_head *bulk_list)
 299{
 300	const struct mlxsw_sp_sb_pool_des *des =
 301		&mlxsw_sp->sb_vals->pool_dess[pool_index];
 302	char sbpm_pl[MLXSW_REG_SBPM_LEN];
 303	struct mlxsw_sp_sb_pm *pm;
 304
 305	if (local_port == MLXSW_PORT_CPU_PORT &&
 306	    des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
 307		return 0;
 308
 309	pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index);
 310	mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir,
 311			    false, 0, 0);
 312	return mlxsw_reg_trans_query(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl,
 313				     bulk_list,
 314				     mlxsw_sp_sb_pm_occ_query_cb,
 315				     (unsigned long) pm);
 316}
 317
 318void mlxsw_sp_hdroom_prios_reset_buf_idx(struct mlxsw_sp_hdroom *hdroom)
 319{
 320	int prio;
 321
 322	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++) {
 323		switch (hdroom->mode) {
 324		case MLXSW_SP_HDROOM_MODE_DCB:
 325			hdroom->prios.prio[prio].buf_idx = hdroom->prios.prio[prio].ets_buf_idx;
 326			break;
 327		case MLXSW_SP_HDROOM_MODE_TC:
 328			hdroom->prios.prio[prio].buf_idx = hdroom->prios.prio[prio].set_buf_idx;
 329			break;
 330		}
 331	}
 332}
 333
 334void mlxsw_sp_hdroom_bufs_reset_lossiness(struct mlxsw_sp_hdroom *hdroom)
 335{
 336	int prio;
 337	int i;
 338
 339	for (i = 0; i < DCBX_MAX_BUFFERS; i++)
 340		hdroom->bufs.buf[i].lossy = true;
 341
 342	for (prio = 0; prio < IEEE_8021Q_MAX_PRIORITIES; prio++) {
 343		if (!hdroom->prios.prio[prio].lossy)
 344			hdroom->bufs.buf[hdroom->prios.prio[prio].buf_idx].lossy = false;
 345	}
 346}
 347
 348static u16 mlxsw_sp_hdroom_buf_threshold_get(const struct mlxsw_sp *mlxsw_sp, int mtu)
 349{
 350	return 2 * mlxsw_sp_bytes_cells(mlxsw_sp, mtu);
 351}
 352
 353static void mlxsw_sp_hdroom_buf_pack(char *pbmc_pl, int index, u16 size, u16 thres, bool lossy)
 354{
 355	if (lossy)
 356		mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl, index, size);
 357	else
 358		mlxsw_reg_pbmc_lossless_buffer_pack(pbmc_pl, index, size,
 359						    thres);
 360}
 361
 362static u16 mlxsw_sp_hdroom_buf_delay_get(const struct mlxsw_sp *mlxsw_sp,
 363					 const struct mlxsw_sp_hdroom *hdroom)
 364{
 365	u16 delay_cells;
 366
 367	delay_cells = mlxsw_sp_bytes_cells(mlxsw_sp, hdroom->delay_bytes);
 368
 369	/* In the worst case scenario the delay will be made up of packets that
 370	 * are all of size CELL_SIZE + 1, which means each packet will require
 371	 * almost twice its true size when buffered in the switch. We therefore
 372	 * multiply this value by the "cell factor", which is close to 2.
 373	 *
 374	 * Another MTU is added in case the transmitting host already started
 375	 * transmitting a maximum length frame when the PFC packet was received.
 376	 */
 377	return 2 * delay_cells + mlxsw_sp_bytes_cells(mlxsw_sp, hdroom->mtu);
 378}
 379
 380static u32 mlxsw_sp_hdroom_int_buf_size_get(struct mlxsw_sp *mlxsw_sp, int mtu, u32 speed)
 381{
 382	u32 buffsize = mlxsw_sp->sb_ops->int_buf_size_get(mtu, speed);
 383
 384	return mlxsw_sp_bytes_cells(mlxsw_sp, buffsize) + 1;
 385}
 386
 387static bool mlxsw_sp_hdroom_buf_is_used(const struct mlxsw_sp_hdroom *hdroom, int buf)
 388{
 389	int prio;
 390
 391	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++) {
 392		if (hdroom->prios.prio[prio].buf_idx == buf)
 393			return true;
 394	}
 395	return false;
 396}
 397
 398void mlxsw_sp_hdroom_bufs_reset_sizes(struct mlxsw_sp_port *mlxsw_sp_port,
 399				      struct mlxsw_sp_hdroom *hdroom)
 400{
 401	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 402	unsigned int max_mtu = mlxsw_sp_port->dev->max_mtu;
 403	u16 reserve_cells;
 404	int i;
 405
 406	max_mtu += MLXSW_PORT_ETH_FRAME_HDR;
 407	/* Internal buffer. */
 408	reserve_cells = mlxsw_sp_hdroom_int_buf_size_get(mlxsw_sp, max_mtu,
 409							 mlxsw_sp_port->max_speed);
 410	reserve_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, reserve_cells);
 411	hdroom->int_buf.reserve_cells = reserve_cells;
 412
 413	if (hdroom->int_buf.enable)
 414		hdroom->int_buf.size_cells = reserve_cells;
 415	else
 416		hdroom->int_buf.size_cells = 0;
 417
 418	/* PG buffers. */
 419	for (i = 0; i < DCBX_MAX_BUFFERS; i++) {
 420		struct mlxsw_sp_hdroom_buf *buf = &hdroom->bufs.buf[i];
 421		u16 thres_cells;
 422		u16 delay_cells;
 423
 424		if (!mlxsw_sp_hdroom_buf_is_used(hdroom, i)) {
 425			thres_cells = 0;
 426			delay_cells = 0;
 427		} else if (buf->lossy) {
 428			thres_cells = mlxsw_sp_hdroom_buf_threshold_get(mlxsw_sp, hdroom->mtu);
 429			delay_cells = 0;
 430		} else {
 431			thres_cells = mlxsw_sp_hdroom_buf_threshold_get(mlxsw_sp, hdroom->mtu);
 432			delay_cells = mlxsw_sp_hdroom_buf_delay_get(mlxsw_sp, hdroom);
 433		}
 434
 435		thres_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, thres_cells);
 436		delay_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, delay_cells);
 437
 438		buf->thres_cells = thres_cells;
 439		if (hdroom->mode == MLXSW_SP_HDROOM_MODE_DCB) {
 440			buf->size_cells = thres_cells + delay_cells;
 441		} else {
 442			/* Do not allow going below the minimum size, even if
 443			 * the user requested it.
 444			 */
 445			buf->size_cells = max(buf->set_size_cells, buf->thres_cells);
 446		}
 447	}
 448}
 449
 450#define MLXSW_SP_PB_UNUSED 8
 451
 452static int mlxsw_sp_hdroom_configure_buffers(struct mlxsw_sp_port *mlxsw_sp_port,
 453					     const struct mlxsw_sp_hdroom *hdroom, bool force)
 454{
 455	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 456	char pbmc_pl[MLXSW_REG_PBMC_LEN];
 457	bool dirty;
 458	int err;
 459	int i;
 460
 461	dirty = memcmp(&mlxsw_sp_port->hdroom->bufs, &hdroom->bufs, sizeof(hdroom->bufs));
 462	if (!dirty && !force)
 463		return 0;
 464
 465	mlxsw_reg_pbmc_pack(pbmc_pl, mlxsw_sp_port->local_port, 0xffff, 0xffff / 2);
 466	for (i = 0; i < MLXSW_SP_PB_COUNT; i++) {
 467		const struct mlxsw_sp_hdroom_buf *buf = &hdroom->bufs.buf[i];
 468
 469		if (i == MLXSW_SP_PB_UNUSED)
 470			continue;
 471
 472		mlxsw_sp_hdroom_buf_pack(pbmc_pl, i, buf->size_cells, buf->thres_cells, buf->lossy);
 473	}
 474
 475	mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl, MLXSW_REG_PBMC_PORT_SHARED_BUF_IDX, 0);
 476	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pbmc), pbmc_pl);
 477	if (err)
 478		return err;
 479
 480	mlxsw_sp_port->hdroom->bufs = hdroom->bufs;
 481	return 0;
 482}
 483
 484static int mlxsw_sp_hdroom_configure_priomap(struct mlxsw_sp_port *mlxsw_sp_port,
 485					     const struct mlxsw_sp_hdroom *hdroom, bool force)
 486{
 487	char pptb_pl[MLXSW_REG_PPTB_LEN];
 488	bool dirty;
 489	int prio;
 490	int err;
 491
 492	dirty = memcmp(&mlxsw_sp_port->hdroom->prios, &hdroom->prios, sizeof(hdroom->prios));
 493	if (!dirty && !force)
 494		return 0;
 495
 496	mlxsw_reg_pptb_pack(pptb_pl, mlxsw_sp_port->local_port);
 497	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++)
 498		mlxsw_reg_pptb_prio_to_buff_pack(pptb_pl, prio, hdroom->prios.prio[prio].buf_idx);
 499
 500	err = mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pptb), pptb_pl);
 501	if (err)
 502		return err;
 503
 504	mlxsw_sp_port->hdroom->prios = hdroom->prios;
 505	return 0;
 506}
 507
 508static int mlxsw_sp_hdroom_configure_int_buf(struct mlxsw_sp_port *mlxsw_sp_port,
 509					     const struct mlxsw_sp_hdroom *hdroom, bool force)
 510{
 511	char sbib_pl[MLXSW_REG_SBIB_LEN];
 512	bool dirty;
 513	int err;
 514
 515	dirty = memcmp(&mlxsw_sp_port->hdroom->int_buf, &hdroom->int_buf, sizeof(hdroom->int_buf));
 516	if (!dirty && !force)
 517		return 0;
 518
 519	mlxsw_reg_sbib_pack(sbib_pl, mlxsw_sp_port->local_port, hdroom->int_buf.size_cells);
 520	err = mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(sbib), sbib_pl);
 521	if (err)
 522		return err;
 523
 524	mlxsw_sp_port->hdroom->int_buf = hdroom->int_buf;
 525	return 0;
 526}
 527
 528static bool mlxsw_sp_hdroom_bufs_fit(struct mlxsw_sp *mlxsw_sp,
 529				     const struct mlxsw_sp_hdroom *hdroom)
 530{
 531	u32 taken_headroom_cells = 0;
 532	int i;
 533
 534	for (i = 0; i < MLXSW_SP_PB_COUNT; i++)
 535		taken_headroom_cells += hdroom->bufs.buf[i].size_cells;
 536
 537	taken_headroom_cells += hdroom->int_buf.reserve_cells;
 538	return taken_headroom_cells <= mlxsw_sp->sb->max_headroom_cells;
 539}
 540
 541static int __mlxsw_sp_hdroom_configure(struct mlxsw_sp_port *mlxsw_sp_port,
 542				       const struct mlxsw_sp_hdroom *hdroom, bool force)
 543{
 544	struct mlxsw_sp_hdroom orig_hdroom;
 545	struct mlxsw_sp_hdroom tmp_hdroom;
 546	int err;
 547	int i;
 548
 549	/* Port buffers need to be configured in three steps. First, all buffers
 550	 * with non-zero size are configured. Then, prio-to-buffer map is
 551	 * updated, allowing traffic to flow to the now non-zero buffers.
 552	 * Finally, zero-sized buffers are configured, because now no traffic
 553	 * should be directed to them anymore. This way, in a non-congested
 554	 * system, no packet drops are introduced by the reconfiguration.
 555	 */
 556
 557	orig_hdroom = *mlxsw_sp_port->hdroom;
 558	tmp_hdroom = orig_hdroom;
 559	for (i = 0; i < MLXSW_SP_PB_COUNT; i++) {
 560		if (hdroom->bufs.buf[i].size_cells)
 561			tmp_hdroom.bufs.buf[i] = hdroom->bufs.buf[i];
 562	}
 563
 564	if (!mlxsw_sp_hdroom_bufs_fit(mlxsw_sp_port->mlxsw_sp, &tmp_hdroom) ||
 565	    !mlxsw_sp_hdroom_bufs_fit(mlxsw_sp_port->mlxsw_sp, hdroom))
 566		return -ENOBUFS;
 567
 568	err = mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &tmp_hdroom, force);
 569	if (err)
 570		return err;
 571
 572	err = mlxsw_sp_hdroom_configure_priomap(mlxsw_sp_port, hdroom, force);
 573	if (err)
 574		goto err_configure_priomap;
 575
 576	err = mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, hdroom, false);
 577	if (err)
 578		goto err_configure_buffers;
 579
 580	err = mlxsw_sp_hdroom_configure_int_buf(mlxsw_sp_port, hdroom, false);
 581	if (err)
 582		goto err_configure_int_buf;
 583
 584	*mlxsw_sp_port->hdroom = *hdroom;
 585	return 0;
 586
 587err_configure_int_buf:
 588	mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &tmp_hdroom, false);
 589err_configure_buffers:
 590	mlxsw_sp_hdroom_configure_priomap(mlxsw_sp_port, &tmp_hdroom, false);
 591err_configure_priomap:
 592	mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &orig_hdroom, false);
 593	return err;
 594}
 595
 596int mlxsw_sp_hdroom_configure(struct mlxsw_sp_port *mlxsw_sp_port,
 597			      const struct mlxsw_sp_hdroom *hdroom)
 598{
 599	return __mlxsw_sp_hdroom_configure(mlxsw_sp_port, hdroom, false);
 600}
 601
 602static int mlxsw_sp_port_headroom_init(struct mlxsw_sp_port *mlxsw_sp_port)
 603{
 604	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 605	struct mlxsw_sp_hdroom hdroom = {};
 606	u32 size9;
 607	int prio;
 608
 609	hdroom.mtu = mlxsw_sp_port->dev->mtu;
 610	hdroom.mode = MLXSW_SP_HDROOM_MODE_DCB;
 611	for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++)
 612		hdroom.prios.prio[prio].lossy = true;
 613
 614	mlxsw_sp_hdroom_bufs_reset_lossiness(&hdroom);
 615	mlxsw_sp_hdroom_bufs_reset_sizes(mlxsw_sp_port, &hdroom);
 616
 617	/* Buffer 9 is used for control traffic. */
 618	size9 = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port,
 619						 mlxsw_sp_port->dev->max_mtu +
 620						 MLXSW_PORT_ETH_FRAME_HDR);
 621	hdroom.bufs.buf[9].size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size9);
 622
 623	return __mlxsw_sp_hdroom_configure(mlxsw_sp_port, &hdroom, true);
 624}
 625
 626static int mlxsw_sp_sb_port_init(struct mlxsw_sp *mlxsw_sp,
 627				 struct mlxsw_sp_sb_port *sb_port)
 628{
 629	struct mlxsw_sp_sb_pm *pms;
 630
 631	pms = kcalloc(mlxsw_sp->sb_vals->pool_count, sizeof(*pms),
 632		      GFP_KERNEL);
 633	if (!pms)
 634		return -ENOMEM;
 635	sb_port->pms = pms;
 636	return 0;
 637}
 638
 639static void mlxsw_sp_sb_port_fini(struct mlxsw_sp_sb_port *sb_port)
 640{
 641	kfree(sb_port->pms);
 642}
 643
 644static int mlxsw_sp_sb_ports_init(struct mlxsw_sp *mlxsw_sp)
 645{
 646	unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sp->core);
 647	struct mlxsw_sp_sb_pr *prs;
 648	int i;
 649	int err;
 650
 651	mlxsw_sp->sb->ports = kcalloc(max_ports,
 652				      sizeof(struct mlxsw_sp_sb_port),
 653				      GFP_KERNEL);
 654	if (!mlxsw_sp->sb->ports)
 655		return -ENOMEM;
 656
 657	prs = kcalloc(mlxsw_sp->sb_vals->pool_count, sizeof(*prs),
 658		      GFP_KERNEL);
 659	if (!prs) {
 660		err = -ENOMEM;
 661		goto err_alloc_prs;
 662	}
 663	mlxsw_sp->sb->prs = prs;
 664
 665	for (i = 0; i < max_ports; i++) {
 666		err = mlxsw_sp_sb_port_init(mlxsw_sp, &mlxsw_sp->sb->ports[i]);
 667		if (err)
 668			goto err_sb_port_init;
 669	}
 670
 671	return 0;
 672
 673err_sb_port_init:
 674	for (i--; i >= 0; i--)
 675		mlxsw_sp_sb_port_fini(&mlxsw_sp->sb->ports[i]);
 676	kfree(mlxsw_sp->sb->prs);
 677err_alloc_prs:
 678	kfree(mlxsw_sp->sb->ports);
 679	return err;
 680}
 681
 682static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp)
 683{
 684	int max_ports = mlxsw_core_max_ports(mlxsw_sp->core);
 685	int i;
 686
 687	for (i = max_ports - 1; i >= 0; i--)
 688		mlxsw_sp_sb_port_fini(&mlxsw_sp->sb->ports[i]);
 689	kfree(mlxsw_sp->sb->prs);
 690	kfree(mlxsw_sp->sb->ports);
 691}
 692
 693#define MLXSW_SP_SB_PR(_mode, _size)	\
 694	{				\
 695		.mode = _mode,		\
 696		.size = _size,		\
 697	}
 698
 699#define MLXSW_SP_SB_PR_EXT(_mode, _size, _freeze_mode, _freeze_size)	\
 700	{								\
 701		.mode = _mode,						\
 702		.size = _size,						\
 703		.freeze_mode = _freeze_mode,				\
 704		.freeze_size = _freeze_size,				\
 705	}
 706
 707#define MLXSW_SP1_SB_PR_CPU_SIZE	(256 * 1000)
 708
 709/* Order according to mlxsw_sp1_sb_pool_dess */
 710static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = {
 711	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
 712	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 713	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 714	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 715	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
 716			   true, false),
 717	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 718	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 719	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 720	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI,
 721			   true, true),
 722	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 723			   MLXSW_SP1_SB_PR_CPU_SIZE, true, false),
 724	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 725			   MLXSW_SP1_SB_PR_CPU_SIZE, true, false),
 726};
 727
 728#define MLXSW_SP2_SB_PR_CPU_SIZE	(256 * 1000)
 729
 730/* Order according to mlxsw_sp2_sb_pool_dess */
 731static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = {
 732	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
 733	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 734	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 735	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 736	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
 737			   true, false),
 738	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 739	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 740	MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 741	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI,
 742			   true, true),
 743	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 744			   MLXSW_SP2_SB_PR_CPU_SIZE, true, false),
 745	MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 746			   MLXSW_SP2_SB_PR_CPU_SIZE, true, false),
 747};
 748
 749static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp,
 750				const struct mlxsw_sp_sb_pr *prs,
 751				const struct mlxsw_sp_sb_pool_des *pool_dess,
 752				size_t prs_len)
 753{
 754	/* Round down, unlike mlxsw_sp_bytes_cells(). */
 755	u32 sb_cells = div_u64(mlxsw_sp->sb->sb_size, mlxsw_sp->sb->cell_size);
 756	u32 rest_cells[2] = {sb_cells, sb_cells};
 757	int i;
 758	int err;
 759
 760	/* Calculate how much space to give to the "REST" pools in either
 761	 * direction.
 762	 */
 763	for (i = 0; i < prs_len; i++) {
 764		enum mlxsw_reg_sbxx_dir dir = pool_dess[i].dir;
 765		u32 size = prs[i].size;
 766		u32 size_cells;
 767
 768		if (size == MLXSW_SP_SB_INFI || size == MLXSW_SP_SB_REST)
 769			continue;
 770
 771		size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
 772		if (WARN_ON_ONCE(size_cells > rest_cells[dir]))
 773			continue;
 774
 775		rest_cells[dir] -= size_cells;
 776	}
 777
 778	for (i = 0; i < prs_len; i++) {
 779		u32 size = prs[i].size;
 780		u32 size_cells;
 781
 782		if (size == MLXSW_SP_SB_INFI) {
 783			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
 784						   0, true);
 785		} else if (size == MLXSW_SP_SB_REST) {
 786			size_cells = rest_cells[pool_dess[i].dir];
 787			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
 788						   size_cells, false);
 789		} else {
 790			size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
 791			err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
 792						   size_cells, false);
 793		}
 794		if (err)
 795			return err;
 796	}
 797
 798	err = mlxsw_sp_sb_pr_desc_write(mlxsw_sp, MLXSW_REG_SBXX_DIR_INGRESS,
 799					MLXSW_REG_SBPR_MODE_DYNAMIC, 0, true);
 800	if (err)
 801		return err;
 802
 803	err = mlxsw_sp_sb_pr_desc_write(mlxsw_sp, MLXSW_REG_SBXX_DIR_EGRESS,
 804					MLXSW_REG_SBPR_MODE_DYNAMIC, 0, true);
 805	if (err)
 806		return err;
 807
 808	return 0;
 809}
 810
 811#define MLXSW_SP_SB_CM(_min_buff, _max_buff, _pool)	\
 812	{						\
 813		.min_buff = _min_buff,			\
 814		.max_buff = _max_buff,			\
 815		.pool_index = _pool,			\
 816	}
 817
 818#define MLXSW_SP_SB_CM_ING(_min_buff, _max_buff)	\
 819	{						\
 820		.min_buff = _min_buff,			\
 821		.max_buff = _max_buff,			\
 822		.pool_index = MLXSW_SP_SB_POOL_ING,	\
 823	}
 824
 825#define MLXSW_SP_SB_CM_EGR(_min_buff, _max_buff)	\
 826	{						\
 827		.min_buff = _min_buff,			\
 828		.max_buff = _max_buff,			\
 829		.pool_index = MLXSW_SP_SB_POOL_EGR,	\
 830	}
 831
 832#define MLXSW_SP_SB_CM_EGR_MC(_min_buff, _max_buff)	\
 833	{						\
 834		.min_buff = _min_buff,			\
 835		.max_buff = _max_buff,			\
 836		.pool_index = MLXSW_SP_SB_POOL_EGR_MC,	\
 837		.freeze_pool = true,			\
 838		.freeze_thresh = true,			\
 839	}
 840
 841static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_ingress[] = {
 842	MLXSW_SP_SB_CM_ING(10000, 8),
 843	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 844	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 845	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 846	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 847	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 848	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 849	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 850	MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */
 851	MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU),
 852};
 853
 854static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_ingress[] = {
 855	MLXSW_SP_SB_CM_ING(0, 7),
 856	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 857	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 858	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 859	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 860	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 861	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 862	MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 863	MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */
 864	MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU),
 865};
 866
 867static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_egress[] = {
 868	MLXSW_SP_SB_CM_EGR(1500, 9),
 869	MLXSW_SP_SB_CM_EGR(1500, 9),
 870	MLXSW_SP_SB_CM_EGR(1500, 9),
 871	MLXSW_SP_SB_CM_EGR(1500, 9),
 872	MLXSW_SP_SB_CM_EGR(1500, 9),
 873	MLXSW_SP_SB_CM_EGR(1500, 9),
 874	MLXSW_SP_SB_CM_EGR(1500, 9),
 875	MLXSW_SP_SB_CM_EGR(1500, 9),
 876	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 877	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 878	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 879	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 880	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 881	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 882	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 883	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 884	MLXSW_SP_SB_CM_EGR(1, 0xff),
 885};
 886
 887static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_egress[] = {
 888	MLXSW_SP_SB_CM_EGR(0, 7),
 889	MLXSW_SP_SB_CM_EGR(0, 7),
 890	MLXSW_SP_SB_CM_EGR(0, 7),
 891	MLXSW_SP_SB_CM_EGR(0, 7),
 892	MLXSW_SP_SB_CM_EGR(0, 7),
 893	MLXSW_SP_SB_CM_EGR(0, 7),
 894	MLXSW_SP_SB_CM_EGR(0, 7),
 895	MLXSW_SP_SB_CM_EGR(0, 7),
 896	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 897	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 898	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 899	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 900	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 901	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 902	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 903	MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 904	MLXSW_SP_SB_CM_EGR(1, 0xff),
 905};
 906
 907#define MLXSW_SP_CPU_PORT_SB_CM MLXSW_SP_SB_CM(0, 0, MLXSW_SP_SB_POOL_EGR_CPU)
 908
 909static const struct mlxsw_sp_sb_cm mlxsw_sp_cpu_port_sb_cms[] = {
 910	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 911	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 912	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 913	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 914	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 915	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 916	MLXSW_SP_CPU_PORT_SB_CM,
 917	MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 918	MLXSW_SP_CPU_PORT_SB_CM,
 919	MLXSW_SP_CPU_PORT_SB_CM,
 920	MLXSW_SP_CPU_PORT_SB_CM,
 921	MLXSW_SP_CPU_PORT_SB_CM,
 922	MLXSW_SP_CPU_PORT_SB_CM,
 923	MLXSW_SP_CPU_PORT_SB_CM,
 924	MLXSW_SP_CPU_PORT_SB_CM,
 925	MLXSW_SP_CPU_PORT_SB_CM,
 926	MLXSW_SP_CPU_PORT_SB_CM,
 927	MLXSW_SP_CPU_PORT_SB_CM,
 928	MLXSW_SP_CPU_PORT_SB_CM,
 929	MLXSW_SP_CPU_PORT_SB_CM,
 930	MLXSW_SP_CPU_PORT_SB_CM,
 931	MLXSW_SP_CPU_PORT_SB_CM,
 932	MLXSW_SP_CPU_PORT_SB_CM,
 933	MLXSW_SP_CPU_PORT_SB_CM,
 934	MLXSW_SP_CPU_PORT_SB_CM,
 935	MLXSW_SP_CPU_PORT_SB_CM,
 936	MLXSW_SP_CPU_PORT_SB_CM,
 937	MLXSW_SP_CPU_PORT_SB_CM,
 938	MLXSW_SP_CPU_PORT_SB_CM,
 939	MLXSW_SP_CPU_PORT_SB_CM,
 940	MLXSW_SP_CPU_PORT_SB_CM,
 941	MLXSW_SP_CPU_PORT_SB_CM,
 942};
 943
 944static bool
 945mlxsw_sp_sb_pool_is_static(struct mlxsw_sp *mlxsw_sp, u16 pool_index)
 946{
 947	struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
 948
 949	return pr->mode == MLXSW_REG_SBPR_MODE_STATIC;
 950}
 951
 952static int __mlxsw_sp_sb_cms_init(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 953				  enum mlxsw_reg_sbxx_dir dir,
 954				  const struct mlxsw_sp_sb_cm *cms,
 955				  size_t cms_len)
 956{
 957	const struct mlxsw_sp_sb_vals *sb_vals = mlxsw_sp->sb_vals;
 958	int i;
 959	int err;
 960
 961	for (i = 0; i < cms_len; i++) {
 962		const struct mlxsw_sp_sb_cm *cm;
 963		u32 min_buff;
 964		u32 max_buff;
 965
 966		if (i == 8 && dir == MLXSW_REG_SBXX_DIR_INGRESS)
 967			continue; /* PG number 8 does not exist, skip it */
 968		cm = &cms[i];
 969		if (WARN_ON(sb_vals->pool_dess[cm->pool_index].dir != dir))
 970			continue;
 971
 972		min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, cm->min_buff);
 973		max_buff = cm->max_buff;
 974		if (max_buff == MLXSW_SP_SB_INFI) {
 975			err = mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, i,
 976						   min_buff, 0,
 977						   true, cm->pool_index);
 978		} else {
 979			if (mlxsw_sp_sb_pool_is_static(mlxsw_sp,
 980						       cm->pool_index))
 981				max_buff = mlxsw_sp_bytes_cells(mlxsw_sp,
 982								max_buff);
 983			err = mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, i,
 984						   min_buff, max_buff,
 985						   false, cm->pool_index);
 986		}
 987		if (err)
 988			return err;
 989	}
 990	return 0;
 991}
 992
 993static int mlxsw_sp_port_sb_cms_init(struct mlxsw_sp_port *mlxsw_sp_port)
 994{
 995	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 996	int err;
 997
 998	err = __mlxsw_sp_sb_cms_init(mlxsw_sp,
 999				     mlxsw_sp_port->local_port,
1000				     MLXSW_REG_SBXX_DIR_INGRESS,
1001				     mlxsw_sp->sb_vals->cms_ingress,
1002				     mlxsw_sp->sb_vals->cms_ingress_count);
1003	if (err)
1004		return err;
1005	return __mlxsw_sp_sb_cms_init(mlxsw_sp_port->mlxsw_sp,
1006				      mlxsw_sp_port->local_port,
1007				      MLXSW_REG_SBXX_DIR_EGRESS,
1008				      mlxsw_sp->sb_vals->cms_egress,
1009				      mlxsw_sp->sb_vals->cms_egress_count);
1010}
1011
1012static int mlxsw_sp_cpu_port_sb_cms_init(struct mlxsw_sp *mlxsw_sp)
1013{
1014	return __mlxsw_sp_sb_cms_init(mlxsw_sp, 0, MLXSW_REG_SBXX_DIR_EGRESS,
1015				      mlxsw_sp->sb_vals->cms_cpu,
1016				      mlxsw_sp->sb_vals->cms_cpu_count);
1017}
1018
1019#define MLXSW_SP_SB_PM(_min_buff, _max_buff)	\
1020	{					\
1021		.min_buff = _min_buff,		\
1022		.max_buff = _max_buff,		\
1023	}
1024
1025/* Order according to mlxsw_sp1_sb_pool_dess */
1026static const struct mlxsw_sp_sb_pm mlxsw_sp1_sb_pms[] = {
1027	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX),
1028	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1029	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1030	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1031	MLXSW_SP_SB_PM(0, 7),
1032	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1033	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1034	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1035	MLXSW_SP_SB_PM(10000, 90000),
1036	MLXSW_SP_SB_PM(0, 8),	/* 50% occupancy */
1037	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1038};
1039
1040/* Order according to mlxsw_sp2_sb_pool_dess */
1041static const struct mlxsw_sp_sb_pm mlxsw_sp2_sb_pms[] = {
1042	MLXSW_SP_SB_PM(0, 7),
1043	MLXSW_SP_SB_PM(0, 0),
1044	MLXSW_SP_SB_PM(0, 0),
1045	MLXSW_SP_SB_PM(0, 0),
1046	MLXSW_SP_SB_PM(0, 7),
1047	MLXSW_SP_SB_PM(0, 0),
1048	MLXSW_SP_SB_PM(0, 0),
1049	MLXSW_SP_SB_PM(0, 0),
1050	MLXSW_SP_SB_PM(10000, 90000),
1051	MLXSW_SP_SB_PM(0, 8),	/* 50% occupancy */
1052	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1053};
1054
1055/* Order according to mlxsw_sp*_sb_pool_dess */
1056static const struct mlxsw_sp_sb_pm mlxsw_sp_cpu_port_sb_pms[] = {
1057	MLXSW_SP_SB_PM(0, 0),
1058	MLXSW_SP_SB_PM(0, 0),
1059	MLXSW_SP_SB_PM(0, 0),
1060	MLXSW_SP_SB_PM(0, 0),
1061	MLXSW_SP_SB_PM(0, 0),
1062	MLXSW_SP_SB_PM(0, 0),
1063	MLXSW_SP_SB_PM(0, 0),
1064	MLXSW_SP_SB_PM(0, 0),
1065	MLXSW_SP_SB_PM(0, 90000),
1066	MLXSW_SP_SB_PM(0, 0),
1067	MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX),
1068};
1069
1070static int mlxsw_sp_sb_pms_init(struct mlxsw_sp *mlxsw_sp, u16 local_port,
1071				const struct mlxsw_sp_sb_pm *pms,
1072				bool skip_ingress)
1073{
1074	int i, err;
1075
1076	for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1077		const struct mlxsw_sp_sb_pm *pm = &pms[i];
1078		const struct mlxsw_sp_sb_pool_des *des;
1079		u32 max_buff;
1080		u32 min_buff;
1081
1082		des = &mlxsw_sp->sb_vals->pool_dess[i];
1083		if (skip_ingress && des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
1084			continue;
1085
1086		min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, pm->min_buff);
1087		max_buff = pm->max_buff;
1088		if (mlxsw_sp_sb_pool_is_static(mlxsw_sp, i))
1089			max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, max_buff);
1090		err = mlxsw_sp_sb_pm_write(mlxsw_sp, local_port, i, min_buff,
1091					   max_buff);
1092		if (err)
1093			return err;
1094	}
1095	return 0;
1096}
1097
1098static int mlxsw_sp_port_sb_pms_init(struct mlxsw_sp_port *mlxsw_sp_port)
1099{
1100	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1101
1102	return mlxsw_sp_sb_pms_init(mlxsw_sp, mlxsw_sp_port->local_port,
1103				    mlxsw_sp->sb_vals->pms, false);
1104}
1105
1106static int mlxsw_sp_cpu_port_sb_pms_init(struct mlxsw_sp *mlxsw_sp)
1107{
1108	return mlxsw_sp_sb_pms_init(mlxsw_sp, 0, mlxsw_sp->sb_vals->pms_cpu,
1109				    true);
1110}
1111
1112#define MLXSW_SP_SB_MM(_min_buff, _max_buff)		\
1113	{						\
1114		.min_buff = _min_buff,			\
1115		.max_buff = _max_buff,			\
1116		.pool_index = MLXSW_SP_SB_POOL_EGR,	\
1117	}
1118
1119static const struct mlxsw_sp_sb_mm mlxsw_sp_sb_mms[] = {
1120	MLXSW_SP_SB_MM(0, 6),
1121	MLXSW_SP_SB_MM(0, 6),
1122	MLXSW_SP_SB_MM(0, 6),
1123	MLXSW_SP_SB_MM(0, 6),
1124	MLXSW_SP_SB_MM(0, 6),
1125	MLXSW_SP_SB_MM(0, 6),
1126	MLXSW_SP_SB_MM(0, 6),
1127	MLXSW_SP_SB_MM(0, 6),
1128	MLXSW_SP_SB_MM(0, 6),
1129	MLXSW_SP_SB_MM(0, 6),
1130	MLXSW_SP_SB_MM(0, 6),
1131	MLXSW_SP_SB_MM(0, 6),
1132	MLXSW_SP_SB_MM(0, 6),
1133	MLXSW_SP_SB_MM(0, 6),
1134	MLXSW_SP_SB_MM(0, 6),
1135};
1136
1137static int mlxsw_sp_sb_mms_init(struct mlxsw_sp *mlxsw_sp)
1138{
1139	char sbmm_pl[MLXSW_REG_SBMM_LEN];
1140	int i;
1141	int err;
1142
1143	for (i = 0; i < mlxsw_sp->sb_vals->mms_count; i++) {
1144		const struct mlxsw_sp_sb_pool_des *des;
1145		const struct mlxsw_sp_sb_mm *mc;
1146		u32 min_buff;
1147
1148		mc = &mlxsw_sp->sb_vals->mms[i];
1149		des = &mlxsw_sp->sb_vals->pool_dess[mc->pool_index];
1150		/* All pools used by sb_mm's are initialized using dynamic
1151		 * thresholds, therefore 'max_buff' isn't specified in cells.
1152		 */
1153		min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, mc->min_buff);
1154		mlxsw_reg_sbmm_pack(sbmm_pl, i, min_buff, mc->max_buff,
1155				    des->pool);
1156		err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbmm), sbmm_pl);
1157		if (err)
1158			return err;
1159	}
1160	return 0;
1161}
1162
1163static void mlxsw_sp_pool_count(struct mlxsw_sp *mlxsw_sp,
1164				u16 *p_ingress_len, u16 *p_egress_len)
1165{
1166	int i;
1167
1168	for (i = 0; i < mlxsw_sp->sb_vals->pool_count; ++i) {
1169		if (mlxsw_sp->sb_vals->pool_dess[i].dir ==
1170		    MLXSW_REG_SBXX_DIR_INGRESS)
1171			(*p_ingress_len)++;
1172		else
1173			(*p_egress_len)++;
1174	}
1175
1176	WARN(*p_egress_len == 0, "No egress pools\n");
1177}
1178
1179const struct mlxsw_sp_sb_vals mlxsw_sp1_sb_vals = {
1180	.pool_count = ARRAY_SIZE(mlxsw_sp1_sb_pool_dess),
1181	.pool_dess = mlxsw_sp1_sb_pool_dess,
1182	.pms = mlxsw_sp1_sb_pms,
1183	.pms_cpu = mlxsw_sp_cpu_port_sb_pms,
1184	.prs = mlxsw_sp1_sb_prs,
1185	.mms = mlxsw_sp_sb_mms,
1186	.cms_ingress = mlxsw_sp1_sb_cms_ingress,
1187	.cms_egress = mlxsw_sp1_sb_cms_egress,
1188	.cms_cpu = mlxsw_sp_cpu_port_sb_cms,
1189	.mms_count = ARRAY_SIZE(mlxsw_sp_sb_mms),
1190	.cms_ingress_count = ARRAY_SIZE(mlxsw_sp1_sb_cms_ingress),
1191	.cms_egress_count = ARRAY_SIZE(mlxsw_sp1_sb_cms_egress),
1192	.cms_cpu_count = ARRAY_SIZE(mlxsw_sp_cpu_port_sb_cms),
1193};
1194
1195const struct mlxsw_sp_sb_vals mlxsw_sp2_sb_vals = {
1196	.pool_count = ARRAY_SIZE(mlxsw_sp2_sb_pool_dess),
1197	.pool_dess = mlxsw_sp2_sb_pool_dess,
1198	.pms = mlxsw_sp2_sb_pms,
1199	.pms_cpu = mlxsw_sp_cpu_port_sb_pms,
1200	.prs = mlxsw_sp2_sb_prs,
1201	.mms = mlxsw_sp_sb_mms,
1202	.cms_ingress = mlxsw_sp2_sb_cms_ingress,
1203	.cms_egress = mlxsw_sp2_sb_cms_egress,
1204	.cms_cpu = mlxsw_sp_cpu_port_sb_cms,
1205	.mms_count = ARRAY_SIZE(mlxsw_sp_sb_mms),
1206	.cms_ingress_count = ARRAY_SIZE(mlxsw_sp2_sb_cms_ingress),
1207	.cms_egress_count = ARRAY_SIZE(mlxsw_sp2_sb_cms_egress),
1208	.cms_cpu_count = ARRAY_SIZE(mlxsw_sp_cpu_port_sb_cms),
1209};
1210
1211static u32 mlxsw_sp1_pb_int_buf_size_get(int mtu, u32 speed)
1212{
1213	return mtu * 5 / 2;
1214}
1215
1216static u32 __mlxsw_sp_pb_int_buf_size_get(int mtu, u32 speed, u32 buffer_factor)
1217{
1218	return 3 * mtu + buffer_factor * speed / 1000;
1219}
1220
1221#define MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR 38
1222
1223static u32 mlxsw_sp2_pb_int_buf_size_get(int mtu, u32 speed)
1224{
1225	int factor = MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR;
1226
1227	return __mlxsw_sp_pb_int_buf_size_get(mtu, speed, factor);
1228}
1229
1230#define MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR 50
1231
1232static u32 mlxsw_sp3_pb_int_buf_size_get(int mtu, u32 speed)
1233{
1234	int factor = MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR;
1235
1236	return __mlxsw_sp_pb_int_buf_size_get(mtu, speed, factor);
1237}
1238
1239const struct mlxsw_sp_sb_ops mlxsw_sp1_sb_ops = {
1240	.int_buf_size_get = mlxsw_sp1_pb_int_buf_size_get,
1241};
1242
1243const struct mlxsw_sp_sb_ops mlxsw_sp2_sb_ops = {
1244	.int_buf_size_get = mlxsw_sp2_pb_int_buf_size_get,
1245};
1246
1247const struct mlxsw_sp_sb_ops mlxsw_sp3_sb_ops = {
1248	.int_buf_size_get = mlxsw_sp3_pb_int_buf_size_get,
1249};
1250
1251int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp)
1252{
1253	u32 max_headroom_size;
1254	u16 ing_pool_count = 0;
1255	u16 eg_pool_count = 0;
1256	int err;
1257
1258	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE))
1259		return -EIO;
1260
1261	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, GUARANTEED_SHARED_BUFFER))
1262		return -EIO;
1263
1264	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_HEADROOM_SIZE))
1265		return -EIO;
1266
1267	mlxsw_sp->sb = kzalloc(sizeof(*mlxsw_sp->sb), GFP_KERNEL);
1268	if (!mlxsw_sp->sb)
1269		return -ENOMEM;
1270	mlxsw_sp->sb->cell_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, CELL_SIZE);
1271	mlxsw_sp->sb->sb_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1272						   GUARANTEED_SHARED_BUFFER);
1273	max_headroom_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1274					       MAX_HEADROOM_SIZE);
1275	/* Round down, because this limit must not be overstepped. */
1276	mlxsw_sp->sb->max_headroom_cells = max_headroom_size /
1277						mlxsw_sp->sb->cell_size;
1278
1279	err = mlxsw_sp_sb_ports_init(mlxsw_sp);
1280	if (err)
1281		goto err_sb_ports_init;
1282	err = mlxsw_sp_sb_prs_init(mlxsw_sp, mlxsw_sp->sb_vals->prs,
1283				   mlxsw_sp->sb_vals->pool_dess,
1284				   mlxsw_sp->sb_vals->pool_count);
1285	if (err)
1286		goto err_sb_prs_init;
1287	err = mlxsw_sp_cpu_port_sb_cms_init(mlxsw_sp);
1288	if (err)
1289		goto err_sb_cpu_port_sb_cms_init;
1290	err = mlxsw_sp_cpu_port_sb_pms_init(mlxsw_sp);
1291	if (err)
1292		goto err_sb_cpu_port_pms_init;
1293	err = mlxsw_sp_sb_mms_init(mlxsw_sp);
1294	if (err)
1295		goto err_sb_mms_init;
1296	mlxsw_sp_pool_count(mlxsw_sp, &ing_pool_count, &eg_pool_count);
1297	err = devl_sb_register(priv_to_devlink(mlxsw_sp->core), 0,
1298			       mlxsw_sp->sb->sb_size,
1299			       ing_pool_count,
1300			       eg_pool_count,
1301			       MLXSW_SP_SB_ING_TC_COUNT,
1302			       MLXSW_SP_SB_EG_TC_COUNT);
1303	if (err)
1304		goto err_devlink_sb_register;
1305
1306	return 0;
1307
1308err_devlink_sb_register:
1309err_sb_mms_init:
1310err_sb_cpu_port_pms_init:
1311err_sb_cpu_port_sb_cms_init:
1312err_sb_prs_init:
1313	mlxsw_sp_sb_ports_fini(mlxsw_sp);
1314err_sb_ports_init:
1315	kfree(mlxsw_sp->sb);
1316	return err;
1317}
1318
1319void mlxsw_sp_buffers_fini(struct mlxsw_sp *mlxsw_sp)
1320{
1321	devl_sb_unregister(priv_to_devlink(mlxsw_sp->core), 0);
1322	mlxsw_sp_sb_ports_fini(mlxsw_sp);
1323	kfree(mlxsw_sp->sb);
1324}
1325
1326int mlxsw_sp_port_buffers_init(struct mlxsw_sp_port *mlxsw_sp_port)
1327{
1328	int err;
1329
1330	mlxsw_sp_port->hdroom = kzalloc(sizeof(*mlxsw_sp_port->hdroom), GFP_KERNEL);
1331	if (!mlxsw_sp_port->hdroom)
1332		return -ENOMEM;
1333	mlxsw_sp_port->hdroom->mtu = mlxsw_sp_port->dev->mtu;
1334
1335	err = mlxsw_sp_port_headroom_init(mlxsw_sp_port);
1336	if (err)
1337		goto err_headroom_init;
1338	err = mlxsw_sp_port_sb_cms_init(mlxsw_sp_port);
1339	if (err)
1340		goto err_port_sb_cms_init;
1341	err = mlxsw_sp_port_sb_pms_init(mlxsw_sp_port);
1342	if (err)
1343		goto err_port_sb_pms_init;
1344	return 0;
1345
1346err_port_sb_pms_init:
1347err_port_sb_cms_init:
1348err_headroom_init:
1349	kfree(mlxsw_sp_port->hdroom);
1350	return err;
1351}
1352
1353void mlxsw_sp_port_buffers_fini(struct mlxsw_sp_port *mlxsw_sp_port)
1354{
1355	kfree(mlxsw_sp_port->hdroom);
1356}
1357
1358int mlxsw_sp_sb_pool_get(struct mlxsw_core *mlxsw_core,
1359			 unsigned int sb_index, u16 pool_index,
1360			 struct devlink_sb_pool_info *pool_info)
1361{
1362	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1363	enum mlxsw_reg_sbxx_dir dir;
1364	struct mlxsw_sp_sb_pr *pr;
1365
1366	dir = mlxsw_sp->sb_vals->pool_dess[pool_index].dir;
1367	pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1368	pool_info->pool_type = (enum devlink_sb_pool_type) dir;
1369	pool_info->size = mlxsw_sp_cells_bytes(mlxsw_sp, pr->size);
1370	pool_info->threshold_type = (enum devlink_sb_threshold_type) pr->mode;
1371	pool_info->cell_size = mlxsw_sp->sb->cell_size;
1372	return 0;
1373}
1374
1375int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core,
1376			 unsigned int sb_index, u16 pool_index, u32 size,
1377			 enum devlink_sb_threshold_type threshold_type,
1378			 struct netlink_ext_ack *extack)
1379{
1380	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1381	u32 pool_size = mlxsw_sp_bytes_cells(mlxsw_sp, size);
1382	const struct mlxsw_sp_sb_pr *pr;
1383	enum mlxsw_reg_sbpr_mode mode;
1384
1385	mode = (enum mlxsw_reg_sbpr_mode) threshold_type;
1386	pr = &mlxsw_sp->sb_vals->prs[pool_index];
1387
1388	if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core,
1389				      GUARANTEED_SHARED_BUFFER)) {
1390		NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size");
1391		return -EINVAL;
1392	}
1393
1394	if (pr->freeze_mode && pr->mode != mode) {
1395		NL_SET_ERR_MSG_MOD(extack, "Changing this pool's threshold type is forbidden");
1396		return -EINVAL;
1397	}
1398
1399	if (pr->freeze_size && pr->size != size) {
1400		NL_SET_ERR_MSG_MOD(extack, "Changing this pool's size is forbidden");
1401		return -EINVAL;
1402	}
1403
1404	return mlxsw_sp_sb_pr_write(mlxsw_sp, pool_index, mode,
1405				    pool_size, false);
1406}
1407
1408#define MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET (-2) /* 3->1, 16->14 */
1409
1410static u32 mlxsw_sp_sb_threshold_out(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
1411				     u32 max_buff)
1412{
1413	struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1414
1415	if (pr->mode == MLXSW_REG_SBPR_MODE_DYNAMIC)
1416		return max_buff - MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET;
1417	return mlxsw_sp_cells_bytes(mlxsw_sp, max_buff);
1418}
1419
1420static int mlxsw_sp_sb_threshold_in(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
1421				    u32 threshold, u32 *p_max_buff,
1422				    struct netlink_ext_ack *extack)
1423{
1424	struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1425
1426	if (pr->mode == MLXSW_REG_SBPR_MODE_DYNAMIC) {
1427		int val;
1428
1429		val = threshold + MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET;
1430		if (val < MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN ||
1431		    val > MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX) {
1432			NL_SET_ERR_MSG_MOD(extack, "Invalid dynamic threshold value");
1433			return -EINVAL;
1434		}
1435		*p_max_buff = val;
1436	} else {
1437		*p_max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, threshold);
1438	}
1439	return 0;
1440}
1441
1442int mlxsw_sp_sb_port_pool_get(struct mlxsw_core_port *mlxsw_core_port,
1443			      unsigned int sb_index, u16 pool_index,
1444			      u32 *p_threshold)
1445{
1446	struct mlxsw_sp_port *mlxsw_sp_port =
1447			mlxsw_core_port_driver_priv(mlxsw_core_port);
1448	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1449	u16 local_port = mlxsw_sp_port->local_port;
1450	struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port,
1451						       pool_index);
1452
1453	*p_threshold = mlxsw_sp_sb_threshold_out(mlxsw_sp, pool_index,
1454						 pm->max_buff);
1455	return 0;
1456}
1457
1458int mlxsw_sp_sb_port_pool_set(struct mlxsw_core_port *mlxsw_core_port,
1459			      unsigned int sb_index, u16 pool_index,
1460			      u32 threshold, struct netlink_ext_ack *extack)
1461{
1462	struct mlxsw_sp_port *mlxsw_sp_port =
1463			mlxsw_core_port_driver_priv(mlxsw_core_port);
1464	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1465	u16 local_port = mlxsw_sp_port->local_port;
1466	u32 max_buff;
1467	int err;
1468
1469	if (local_port == MLXSW_PORT_CPU_PORT) {
1470		NL_SET_ERR_MSG_MOD(extack, "Changing CPU port's threshold is forbidden");
1471		return -EINVAL;
1472	}
1473
1474	err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index,
1475				       threshold, &max_buff, extack);
1476	if (err)
1477		return err;
1478
1479	return mlxsw_sp_sb_pm_write(mlxsw_sp, local_port, pool_index,
1480				    0, max_buff);
1481}
1482
1483int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port,
1484				 unsigned int sb_index, u16 tc_index,
1485				 enum devlink_sb_pool_type pool_type,
1486				 u16 *p_pool_index, u32 *p_threshold)
1487{
1488	struct mlxsw_sp_port *mlxsw_sp_port =
1489			mlxsw_core_port_driver_priv(mlxsw_core_port);
1490	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1491	u16 local_port = mlxsw_sp_port->local_port;
1492	u8 pg_buff = tc_index;
1493	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1494	struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
1495						       pg_buff, dir);
1496
1497	*p_threshold = mlxsw_sp_sb_threshold_out(mlxsw_sp, cm->pool_index,
1498						 cm->max_buff);
1499	*p_pool_index = cm->pool_index;
1500	return 0;
1501}
1502
1503int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port,
1504				 unsigned int sb_index, u16 tc_index,
1505				 enum devlink_sb_pool_type pool_type,
1506				 u16 pool_index, u32 threshold,
1507				 struct netlink_ext_ack *extack)
1508{
1509	struct mlxsw_sp_port *mlxsw_sp_port =
1510			mlxsw_core_port_driver_priv(mlxsw_core_port);
1511	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1512	u16 local_port = mlxsw_sp_port->local_port;
1513	const struct mlxsw_sp_sb_cm *cm;
1514	u8 pg_buff = tc_index;
1515	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1516	u32 max_buff;
1517	int err;
1518
1519	if (local_port == MLXSW_PORT_CPU_PORT) {
1520		NL_SET_ERR_MSG_MOD(extack, "Changing CPU port's binding is forbidden");
1521		return -EINVAL;
1522	}
1523
1524	if (dir != mlxsw_sp->sb_vals->pool_dess[pool_index].dir) {
1525		NL_SET_ERR_MSG_MOD(extack, "Binding egress TC to ingress pool and vice versa is forbidden");
1526		return -EINVAL;
1527	}
1528
1529	if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
1530		cm = &mlxsw_sp->sb_vals->cms_ingress[tc_index];
1531	else
1532		cm = &mlxsw_sp->sb_vals->cms_egress[tc_index];
1533
1534	if (cm->freeze_pool && cm->pool_index != pool_index) {
1535		NL_SET_ERR_MSG_MOD(extack, "Binding this TC to a different pool is forbidden");
1536		return -EINVAL;
1537	}
1538
1539	if (cm->freeze_thresh && cm->max_buff != threshold) {
1540		NL_SET_ERR_MSG_MOD(extack, "Changing this TC's threshold is forbidden");
1541		return -EINVAL;
1542	}
1543
1544	err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index,
1545				       threshold, &max_buff, extack);
1546	if (err)
1547		return err;
1548
1549	return mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, pg_buff,
1550				    0, max_buff, false, pool_index);
1551}
1552
1553#define MASKED_COUNT_MAX \
1554	(MLXSW_REG_SBSR_REC_MAX_COUNT / \
1555	 (MLXSW_SP_SB_ING_TC_COUNT + MLXSW_SP_SB_EG_TC_COUNT))
1556
1557struct mlxsw_sp_sb_sr_occ_query_cb_ctx {
1558	u8 masked_count;
1559	u16 local_port_1;
1560};
1561
1562static void mlxsw_sp_sb_sr_occ_query_cb(struct mlxsw_core *mlxsw_core,
1563					char *sbsr_pl, size_t sbsr_pl_len,
1564					unsigned long cb_priv)
1565{
1566	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1567	struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx;
1568	u8 masked_count;
1569	u16 local_port;
1570	int rec_index = 0;
1571	struct mlxsw_sp_sb_cm *cm;
1572	int i;
1573
1574	memcpy(&cb_ctx, &cb_priv, sizeof(cb_ctx));
1575
1576	masked_count = 0;
1577	for (local_port = cb_ctx.local_port_1;
1578	     local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1579		if (!mlxsw_sp->ports[local_port])
1580			continue;
1581		if (local_port == MLXSW_PORT_CPU_PORT) {
1582			/* Ingress quotas are not supported for the CPU port */
1583			masked_count++;
1584			continue;
1585		}
1586		for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++) {
1587			cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, i,
1588						MLXSW_REG_SBXX_DIR_INGRESS);
1589			mlxsw_reg_sbsr_rec_unpack(sbsr_pl, rec_index++,
1590						  &cm->occ.cur, &cm->occ.max);
1591		}
1592		if (++masked_count == cb_ctx.masked_count)
1593			break;
1594	}
1595	masked_count = 0;
1596	for (local_port = cb_ctx.local_port_1;
1597	     local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1598		if (!mlxsw_sp->ports[local_port])
1599			continue;
1600		for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++) {
1601			cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, i,
1602						MLXSW_REG_SBXX_DIR_EGRESS);
1603			mlxsw_reg_sbsr_rec_unpack(sbsr_pl, rec_index++,
1604						  &cm->occ.cur, &cm->occ.max);
1605		}
1606		if (++masked_count == cb_ctx.masked_count)
1607			break;
1608	}
1609}
1610
1611int mlxsw_sp_sb_occ_snapshot(struct mlxsw_core *mlxsw_core,
1612			     unsigned int sb_index)
1613{
1614	u16 local_port, local_port_1, first_local_port, last_local_port;
1615	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1616	struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx;
1617	u8 masked_count, current_page = 0;
1618	unsigned long cb_priv = 0;
1619	LIST_HEAD(bulk_list);
1620	char *sbsr_pl;
1621	int i;
1622	int err;
1623	int err2;
1624
1625	sbsr_pl = kmalloc(MLXSW_REG_SBSR_LEN, GFP_KERNEL);
1626	if (!sbsr_pl)
1627		return -ENOMEM;
1628
1629	local_port = MLXSW_PORT_CPU_PORT;
1630next_batch:
1631	local_port_1 = local_port;
1632	masked_count = 0;
1633	mlxsw_reg_sbsr_pack(sbsr_pl, false);
1634	mlxsw_reg_sbsr_port_page_set(sbsr_pl, current_page);
1635	first_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE;
1636	last_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE +
1637			  MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE - 1;
1638
1639	for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++)
1640		mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1);
1641	for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++)
1642		mlxsw_reg_sbsr_tclass_mask_set(sbsr_pl, i, 1);
1643	for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1644		if (!mlxsw_sp->ports[local_port])
1645			continue;
1646		if (local_port > last_local_port) {
1647			current_page++;
1648			goto do_query;
1649		}
1650		if (local_port != MLXSW_PORT_CPU_PORT) {
1651			/* Ingress quotas are not supported for the CPU port */
1652			mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl,
1653							     local_port - first_local_port,
1654							     1);
1655		}
1656		mlxsw_reg_sbsr_egress_port_mask_set(sbsr_pl,
1657						    local_port - first_local_port,
1658						    1);
1659		for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1660			err = mlxsw_sp_sb_pm_occ_query(mlxsw_sp, local_port, i,
1661						       &bulk_list);
1662			if (err)
1663				goto out;
1664		}
1665		if (++masked_count == MASKED_COUNT_MAX)
1666			goto do_query;
1667	}
1668
1669do_query:
1670	cb_ctx.masked_count = masked_count;
1671	cb_ctx.local_port_1 = local_port_1;
1672	memcpy(&cb_priv, &cb_ctx, sizeof(cb_ctx));
1673	err = mlxsw_reg_trans_query(mlxsw_core, MLXSW_REG(sbsr), sbsr_pl,
1674				    &bulk_list, mlxsw_sp_sb_sr_occ_query_cb,
1675				    cb_priv);
1676	if (err)
1677		goto out;
1678	if (local_port < mlxsw_core_max_ports(mlxsw_core)) {
1679		local_port++;
1680		goto next_batch;
1681	}
1682
1683out:
1684	err2 = mlxsw_reg_trans_bulk_wait(&bulk_list);
1685	if (!err)
1686		err = err2;
1687	kfree(sbsr_pl);
1688	return err;
1689}
1690
1691int mlxsw_sp_sb_occ_max_clear(struct mlxsw_core *mlxsw_core,
1692			      unsigned int sb_index)
1693{
1694	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1695	u16 local_port, first_local_port, last_local_port;
1696	LIST_HEAD(bulk_list);
1697	unsigned int masked_count;
1698	u8 current_page = 0;
1699	char *sbsr_pl;
1700	int i;
1701	int err;
1702	int err2;
1703
1704	sbsr_pl = kmalloc(MLXSW_REG_SBSR_LEN, GFP_KERNEL);
1705	if (!sbsr_pl)
1706		return -ENOMEM;
1707
1708	local_port = MLXSW_PORT_CPU_PORT;
1709next_batch:
1710	masked_count = 0;
1711	mlxsw_reg_sbsr_pack(sbsr_pl, true);
1712	mlxsw_reg_sbsr_port_page_set(sbsr_pl, current_page);
1713	first_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE;
1714	last_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE +
1715			  MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE - 1;
1716
1717	for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++)
1718		mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1);
1719	for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++)
1720		mlxsw_reg_sbsr_tclass_mask_set(sbsr_pl, i, 1);
1721	for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1722		if (!mlxsw_sp->ports[local_port])
1723			continue;
1724		if (local_port > last_local_port) {
1725			current_page++;
1726			goto do_query;
1727		}
1728		if (local_port != MLXSW_PORT_CPU_PORT) {
1729			/* Ingress quotas are not supported for the CPU port */
1730			mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl,
1731							     local_port - first_local_port,
1732							     1);
1733		}
1734		mlxsw_reg_sbsr_egress_port_mask_set(sbsr_pl,
1735						    local_port - first_local_port,
1736						    1);
1737		for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1738			err = mlxsw_sp_sb_pm_occ_clear(mlxsw_sp, local_port, i,
1739						       &bulk_list);
1740			if (err)
1741				goto out;
1742		}
1743		if (++masked_count == MASKED_COUNT_MAX)
1744			goto do_query;
1745	}
1746
1747do_query:
1748	err = mlxsw_reg_trans_query(mlxsw_core, MLXSW_REG(sbsr), sbsr_pl,
1749				    &bulk_list, NULL, 0);
1750	if (err)
1751		goto out;
1752	if (local_port < mlxsw_core_max_ports(mlxsw_core)) {
1753		local_port++;
1754		goto next_batch;
1755	}
1756
1757out:
1758	err2 = mlxsw_reg_trans_bulk_wait(&bulk_list);
1759	if (!err)
1760		err = err2;
1761	kfree(sbsr_pl);
1762	return err;
1763}
1764
1765int mlxsw_sp_sb_occ_port_pool_get(struct mlxsw_core_port *mlxsw_core_port,
1766				  unsigned int sb_index, u16 pool_index,
1767				  u32 *p_cur, u32 *p_max)
1768{
1769	struct mlxsw_sp_port *mlxsw_sp_port =
1770			mlxsw_core_port_driver_priv(mlxsw_core_port);
1771	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1772	u16 local_port = mlxsw_sp_port->local_port;
1773	struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port,
1774						       pool_index);
1775
1776	*p_cur = mlxsw_sp_cells_bytes(mlxsw_sp, pm->occ.cur);
1777	*p_max = mlxsw_sp_cells_bytes(mlxsw_sp, pm->occ.max);
1778	return 0;
1779}
1780
1781int mlxsw_sp_sb_occ_tc_port_bind_get(struct mlxsw_core_port *mlxsw_core_port,
1782				     unsigned int sb_index, u16 tc_index,
1783				     enum devlink_sb_pool_type pool_type,
1784				     u32 *p_cur, u32 *p_max)
1785{
1786	struct mlxsw_sp_port *mlxsw_sp_port =
1787			mlxsw_core_port_driver_priv(mlxsw_core_port);
1788	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1789	u16 local_port = mlxsw_sp_port->local_port;
1790	u8 pg_buff = tc_index;
1791	enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1792	struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
1793						       pg_buff, dir);
1794
1795	*p_cur = mlxsw_sp_cells_bytes(mlxsw_sp, cm->occ.cur);
1796	*p_max = mlxsw_sp_cells_bytes(mlxsw_sp, cm->occ.max);
1797	return 0;
1798}