spectrum_acl_erp.c

   1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
   3
   4#include <linux/bitmap.h>
   5#include <linux/errno.h>
   6#include <linux/genalloc.h>
   7#include <linux/gfp.h>
   8#include <linux/kernel.h>
   9#include <linux/list.h>
  10#include <linux/mutex.h>
  11#include <linux/objagg.h>
  12#include <linux/rtnetlink.h>
  13#include <linux/slab.h>
  14
  15#include "core.h"
  16#include "reg.h"
  17#include "spectrum.h"
  18#include "spectrum_acl_tcam.h"
  19
  20/* gen_pool_alloc() returns 0 when allocation fails, so use an offset */
  21#define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100
  22#define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16
  23
  24struct mlxsw_sp_acl_erp_core {
  25	unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];
  26	struct gen_pool *erp_tables;
  27	struct mlxsw_sp *mlxsw_sp;
  28	struct mlxsw_sp_acl_bf *bf;
  29	unsigned int num_erp_banks;
  30};
  31
  32struct mlxsw_sp_acl_erp_key {
  33	char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];
  34#define __MASK_LEN 0x38
  35#define __MASK_IDX(i) (__MASK_LEN - (i) - 1)
  36	bool ctcam;
  37};
  38
  39struct mlxsw_sp_acl_erp {
  40	struct mlxsw_sp_acl_erp_key key;
  41	u8 id;
  42	u8 index;
  43	DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
  44	struct list_head list;
  45	struct mlxsw_sp_acl_erp_table *erp_table;
  46};
  47
  48struct mlxsw_sp_acl_erp_master_mask {
  49	DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
  50	unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN];
  51};
  52
  53struct mlxsw_sp_acl_erp_table {
  54	struct mlxsw_sp_acl_erp_master_mask master_mask;
  55	DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
  56	DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
  57	struct list_head atcam_erps_list;
  58	struct mlxsw_sp_acl_erp_core *erp_core;
  59	struct mlxsw_sp_acl_atcam_region *aregion;
  60	const struct mlxsw_sp_acl_erp_table_ops *ops;
  61	unsigned long base_index;
  62	unsigned int num_atcam_erps;
  63	unsigned int num_max_atcam_erps;
  64	unsigned int num_ctcam_erps;
  65	unsigned int num_deltas;
  66	struct objagg *objagg;
  67	struct mutex objagg_lock; /* guards objagg manipulation */
  68};
  69
  70struct mlxsw_sp_acl_erp_table_ops {
  71	struct mlxsw_sp_acl_erp *
  72		(*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table,
  73			      struct mlxsw_sp_acl_erp_key *key);
  74	void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table,
  75			    struct mlxsw_sp_acl_erp *erp);
  76};
  77
  78static struct mlxsw_sp_acl_erp *
  79mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
  80			     struct mlxsw_sp_acl_erp_key *key);
  81static void
  82mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
  83			      struct mlxsw_sp_acl_erp *erp);
  84static struct mlxsw_sp_acl_erp *
  85mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
  86				    struct mlxsw_sp_acl_erp_key *key);
  87static void
  88mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
  89				     struct mlxsw_sp_acl_erp *erp);
  90static struct mlxsw_sp_acl_erp *
  91mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
  92				   struct mlxsw_sp_acl_erp_key *key);
  93static void
  94mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
  95				    struct mlxsw_sp_acl_erp *erp);
  96static void
  97mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
  98				 struct mlxsw_sp_acl_erp *erp);
  99
 100static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = {
 101	.erp_create = mlxsw_sp_acl_erp_mask_create,
 102	.erp_destroy = mlxsw_sp_acl_erp_mask_destroy,
 103};
 104
 105static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = {
 106	.erp_create = mlxsw_sp_acl_erp_mask_create,
 107	.erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy,
 108};
 109
 110static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = {
 111	.erp_create = mlxsw_sp_acl_erp_second_mask_create,
 112	.erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy,
 113};
 114
 115static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = {
 116	.erp_create = mlxsw_sp_acl_erp_first_mask_create,
 117	.erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,
 118};
 119
 120static bool
 121mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table)
 122{
 123	return erp_table->ops != &erp_single_mask_ops &&
 124	       erp_table->ops != &erp_no_mask_ops;
 125}
 126
 127static unsigned int
 128mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp)
 129{
 130	return erp->index % erp->erp_table->erp_core->num_erp_banks;
 131}
 132
 133static unsigned int
 134mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table)
 135{
 136	struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
 137	struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
 138
 139	return erp_core->erpt_entries_size[aregion->type];
 140}
 141
 142static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table,
 143				   u8 *p_id)
 144{
 145	u8 id;
 146
 147	id = find_first_zero_bit(erp_table->erp_id_bitmap,
 148				 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
 149	if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) {
 150		__set_bit(id, erp_table->erp_id_bitmap);
 151		*p_id = id;
 152		return 0;
 153	}
 154
 155	return -ENOBUFS;
 156}
 157
 158static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table,
 159				    u8 id)
 160{
 161	__clear_bit(id, erp_table->erp_id_bitmap);
 162}
 163
 164static void
 165mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit,
 166				     struct mlxsw_sp_acl_erp_master_mask *mask)
 167{
 168	if (mask->count[bit]++ == 0)
 169		__set_bit(bit, mask->bitmap);
 170}
 171
 172static void
 173mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit,
 174				       struct mlxsw_sp_acl_erp_master_mask *mask)
 175{
 176	if (--mask->count[bit] == 0)
 177		__clear_bit(bit, mask->bitmap);
 178}
 179
 180static int
 181mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table)
 182{
 183	struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
 184	struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
 185	char percr_pl[MLXSW_REG_PERCR_LEN];
 186	char *master_mask;
 187
 188	mlxsw_reg_percr_pack(percr_pl, region->id);
 189	master_mask = mlxsw_reg_percr_master_mask_data(percr_pl);
 190	bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap,
 191			MLXSW_SP_ACL_TCAM_MASK_LEN);
 192
 193	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
 194}
 195
 196static int
 197mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,
 198				 struct mlxsw_sp_acl_erp_key *key)
 199{
 200	DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
 201	unsigned long bit;
 202	int err;
 203
 204	bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
 205			  MLXSW_SP_ACL_TCAM_MASK_LEN);
 206	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
 207		mlxsw_sp_acl_erp_master_mask_bit_set(bit,
 208						     &erp_table->master_mask);
 209
 210	err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
 211	if (err)
 212		goto err_master_mask_update;
 213
 214	return 0;
 215
 216err_master_mask_update:
 217	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
 218		mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
 219						       &erp_table->master_mask);
 220	return err;
 221}
 222
 223static int
 224mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,
 225				   struct mlxsw_sp_acl_erp_key *key)
 226{
 227	DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
 228	unsigned long bit;
 229	int err;
 230
 231	bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
 232			  MLXSW_SP_ACL_TCAM_MASK_LEN);
 233	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
 234		mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
 235						       &erp_table->master_mask);
 236
 237	err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
 238	if (err)
 239		goto err_master_mask_update;
 240
 241	return 0;
 242
 243err_master_mask_update:
 244	for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
 245		mlxsw_sp_acl_erp_master_mask_bit_set(bit,
 246						     &erp_table->master_mask);
 247	return err;
 248}
 249
 250static struct mlxsw_sp_acl_erp *
 251mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table,
 252				struct mlxsw_sp_acl_erp_key *key)
 253{
 254	struct mlxsw_sp_acl_erp *erp;
 255	int err;
 256
 257	erp = kzalloc(sizeof(*erp), GFP_KERNEL);
 258	if (!erp)
 259		return ERR_PTR(-ENOMEM);
 260
 261	err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id);
 262	if (err)
 263		goto err_erp_id_get;
 264
 265	memcpy(&erp->key, key, sizeof(*key));
 266	list_add(&erp->list, &erp_table->atcam_erps_list);
 267	erp_table->num_atcam_erps++;
 268	erp->erp_table = erp_table;
 269
 270	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
 271	if (err)
 272		goto err_master_mask_set;
 273
 274	return erp;
 275
 276err_master_mask_set:
 277	erp_table->num_atcam_erps--;
 278	list_del(&erp->list);
 279	mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
 280err_erp_id_get:
 281	kfree(erp);
 282	return ERR_PTR(err);
 283}
 284
 285static void
 286mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp)
 287{
 288	struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
 289
 290	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
 291	erp_table->num_atcam_erps--;
 292	list_del(&erp->list);
 293	mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
 294	kfree(erp);
 295}
 296
 297static int
 298mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core,
 299			     unsigned int num_erps,
 300			     enum mlxsw_sp_acl_atcam_region_type region_type,
 301			     unsigned long *p_index)
 302{
 303	unsigned int num_rows, entry_size;
 304	unsigned long index;
 305
 306	/* We only allow allocations of entire rows */
 307	if (num_erps % erp_core->num_erp_banks != 0)
 308		return -EINVAL;
 309
 310	entry_size = erp_core->erpt_entries_size[region_type];
 311	num_rows = num_erps / erp_core->num_erp_banks;
 312
 313	index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size);
 314	if (!index)
 315		return -ENOBUFS;
 316
 317	*p_index = index - MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
 318
 319	return 0;
 320}
 321
 322static void
 323mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core,
 324			    unsigned int num_erps,
 325			    enum mlxsw_sp_acl_atcam_region_type region_type,
 326			    unsigned long index)
 327{
 328	unsigned long base_index;
 329	unsigned int entry_size;
 330	size_t size;
 331
 332	entry_size = erp_core->erpt_entries_size[region_type];
 333	base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
 334	size = num_erps / erp_core->num_erp_banks * entry_size;
 335	gen_pool_free(erp_core->erp_tables, base_index, size);
 336}
 337
 338static struct mlxsw_sp_acl_erp *
 339mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table)
 340{
 341	if (!list_is_singular(&erp_table->atcam_erps_list))
 342		return NULL;
 343
 344	return list_first_entry(&erp_table->atcam_erps_list,
 345				struct mlxsw_sp_acl_erp, list);
 346}
 347
 348static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table,
 349				      u8 *p_index)
 350{
 351	u8 index;
 352
 353	index = find_first_zero_bit(erp_table->erp_index_bitmap,
 354				    erp_table->num_max_atcam_erps);
 355	if (index < erp_table->num_max_atcam_erps) {
 356		__set_bit(index, erp_table->erp_index_bitmap);
 357		*p_index = index;
 358		return 0;
 359	}
 360
 361	return -ENOBUFS;
 362}
 363
 364static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table,
 365				       u8 index)
 366{
 367	__clear_bit(index, erp_table->erp_index_bitmap);
 368}
 369
 370static void
 371mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table,
 372			      const struct mlxsw_sp_acl_erp *erp,
 373			      u8 *p_erpt_bank, u8 *p_erpt_index)
 374{
 375	unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table);
 376	struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
 377	unsigned int row;
 378
 379	*p_erpt_bank = erp->index % erp_core->num_erp_banks;
 380	row = erp->index / erp_core->num_erp_banks;
 381	*p_erpt_index = erp_table->base_index + row * entry_size;
 382}
 383
 384static int
 385mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
 386			       struct mlxsw_sp_acl_erp *erp)
 387{
 388	struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
 389	enum mlxsw_reg_perpt_key_size key_size;
 390	char perpt_pl[MLXSW_REG_PERPT_LEN];
 391	u8 erpt_bank, erpt_index;
 392
 393	mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
 394	key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
 395	mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
 396			     0, erp_table->base_index, erp->index,
 397			     erp->key.mask);
 398	mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
 399					MLXSW_SP_ACL_ERP_MAX_PER_REGION);
 400	mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true);
 401	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
 402}
 403
 404static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp)
 405{
 406	char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
 407	struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
 408	struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
 409	enum mlxsw_reg_perpt_key_size key_size;
 410	char perpt_pl[MLXSW_REG_PERPT_LEN];
 411	u8 erpt_bank, erpt_index;
 412
 413	mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
 414	key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
 415	mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
 416			     0, erp_table->base_index, erp->index, empty_mask);
 417	mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
 418					MLXSW_SP_ACL_ERP_MAX_PER_REGION);
 419	mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false);
 420	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
 421}
 422
 423static int
 424mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table,
 425			      bool ctcam_le)
 426{
 427	struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
 428	struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
 429	char pererp_pl[MLXSW_REG_PERERP_LEN];
 430
 431	mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
 432			      erp_table->base_index, 0);
 433	mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
 434					 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
 435
 436	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
 437}
 438
 439static void
 440mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table)
 441{
 442	struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
 443	struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
 444	char pererp_pl[MLXSW_REG_PERERP_LEN];
 445	struct mlxsw_sp_acl_erp *master_rp;
 446
 447	master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
 448	/* It is possible we do not have a master RP when we disable the
 449	 * table when there are no rules in the A-TCAM and the last C-TCAM
 450	 * rule is deleted
 451	 */
 452	mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0,
 453			      master_rp ? master_rp->id : 0);
 454	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
 455}
 456
 457static int
 458mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table)
 459{
 460	struct mlxsw_sp_acl_erp *erp;
 461	int err;
 462
 463	list_for_each_entry(erp, &erp_table->atcam_erps_list, list) {
 464		err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
 465		if (err)
 466			goto err_table_erp_add;
 467	}
 468
 469	return 0;
 470
 471err_table_erp_add:
 472	list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list,
 473					     list)
 474		mlxsw_sp_acl_erp_table_erp_del(erp);
 475	return err;
 476}
 477
 478static int
 479mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table)
 480{
 481	unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps;
 482	struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
 483	unsigned long old_base_index = erp_table->base_index;
 484	bool ctcam_le = erp_table->num_ctcam_erps > 0;
 485	int err;
 486
 487	if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps)
 488		return 0;
 489
 490	if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION)
 491		return -ENOBUFS;
 492
 493	num_erps = old_num_erps + erp_core->num_erp_banks;
 494	err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps,
 495					   erp_table->aregion->type,
 496					   &erp_table->base_index);
 497	if (err)
 498		return err;
 499	erp_table->num_max_atcam_erps = num_erps;
 500
 501	err = mlxsw_sp_acl_erp_table_relocate(erp_table);
 502	if (err)
 503		goto err_table_relocate;
 504
 505	err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le);
 506	if (err)
 507		goto err_table_enable;
 508
 509	mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps,
 510				    erp_table->aregion->type, old_base_index);
 511
 512	return 0;
 513
 514err_table_enable:
 515err_table_relocate:
 516	erp_table->num_max_atcam_erps = old_num_erps;
 517	mlxsw_sp_acl_erp_table_free(erp_core, num_erps,
 518				    erp_table->aregion->type,
 519				    erp_table->base_index);
 520	erp_table->base_index = old_base_index;
 521	return err;
 522}
 523
 524static int
 525mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table,
 526			   struct mlxsw_sp_acl_erp *erp)
 527{
 528	struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
 529	unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
 530	struct mlxsw_sp_acl_atcam_entry *aentry;
 531	int err;
 532
 533	list_for_each_entry(aentry, &aregion->entries_list, list) {
 534		err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp,
 535						erp_table->erp_core->bf,
 536						aregion, erp_bank, aentry);
 537		if (err)
 538			goto bf_entry_add_err;
 539	}
 540
 541	return 0;
 542
 543bf_entry_add_err:
 544	list_for_each_entry_continue_reverse(aentry, &aregion->entries_list,
 545					     list)
 546		mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
 547					  erp_table->erp_core->bf,
 548					  aregion, erp_bank, aentry);
 549	return err;
 550}
 551
 552static void
 553mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table,
 554			   struct mlxsw_sp_acl_erp *erp)
 555{
 556	struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
 557	unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
 558	struct mlxsw_sp_acl_atcam_entry *aentry;
 559
 560	list_for_each_entry_reverse(aentry, &aregion->entries_list, list)
 561		mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
 562					  erp_table->erp_core->bf,
 563					  aregion, erp_bank, aentry);
 564}
 565
 566static int
 567mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)
 568{
 569	struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
 570	struct mlxsw_sp_acl_erp *master_rp;
 571	int err;
 572
 573	/* Initially, allocate a single eRP row. Expand later as needed */
 574	err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks,
 575					   erp_table->aregion->type,
 576					   &erp_table->base_index);
 577	if (err)
 578		return err;
 579	erp_table->num_max_atcam_erps = erp_core->num_erp_banks;
 580
 581	/* Transition the sole RP currently configured (the master RP)
 582	 * to the eRP table
 583	 */
 584	master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
 585	if (!master_rp) {
 586		err = -EINVAL;
 587		goto err_table_master_rp;
 588	}
 589
 590	/* Make sure the master RP is using a valid index, as
 591	 * only a single eRP row is currently allocated.
 592	 */
 593	master_rp->index = 0;
 594	__set_bit(master_rp->index, erp_table->erp_index_bitmap);
 595
 596	err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);
 597	if (err)
 598		goto err_table_master_rp_add;
 599
 600	/* Update Bloom filter before enabling eRP table, as rules
 601	 * on the master RP were not set to Bloom filter up to this
 602	 * point.
 603	 */
 604	err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp);
 605	if (err)
 606		goto err_table_bf_add;
 607
 608	err = mlxsw_sp_acl_erp_table_enable(erp_table, false);
 609	if (err)
 610		goto err_table_enable;
 611
 612	return 0;
 613
 614err_table_enable:
 615	mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
 616err_table_bf_add:
 617	mlxsw_sp_acl_erp_table_erp_del(master_rp);
 618err_table_master_rp_add:
 619	__clear_bit(master_rp->index, erp_table->erp_index_bitmap);
 620err_table_master_rp:
 621	mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
 622				    erp_table->aregion->type,
 623				    erp_table->base_index);
 624	return err;
 625}
 626
 627static void
 628mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table)
 629{
 630	struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
 631	struct mlxsw_sp_acl_erp *master_rp;
 632
 633	mlxsw_sp_acl_erp_table_disable(erp_table);
 634	master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
 635	if (!master_rp)
 636		return;
 637	mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
 638	mlxsw_sp_acl_erp_table_erp_del(master_rp);
 639	__clear_bit(master_rp->index, erp_table->erp_index_bitmap);
 640	mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
 641				    erp_table->aregion->type,
 642				    erp_table->base_index);
 643}
 644
 645static int
 646mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
 647				struct mlxsw_sp_acl_erp *erp)
 648{
 649	struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
 650	struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
 651	bool ctcam_le = erp_table->num_ctcam_erps > 0;
 652	char pererp_pl[MLXSW_REG_PERERP_LEN];
 653
 654	mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
 655			      erp_table->base_index, 0);
 656	mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
 657					 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
 658	mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true);
 659
 660	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
 661}
 662
 663static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp)
 664{
 665	struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
 666	struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
 667	struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
 668	bool ctcam_le = erp_table->num_ctcam_erps > 0;
 669	char pererp_pl[MLXSW_REG_PERERP_LEN];
 670
 671	mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
 672			      erp_table->base_index, 0);
 673	mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
 674					 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
 675	mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false);
 676
 677	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
 678}
 679
 680static int
 681mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table)
 682{
 683	/* No need to re-enable lookup in the C-TCAM */
 684	if (erp_table->num_ctcam_erps > 1)
 685		return 0;
 686
 687	return mlxsw_sp_acl_erp_table_enable(erp_table, true);
 688}
 689
 690static void
 691mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table)
 692{
 693	/* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */
 694	if (erp_table->num_ctcam_erps > 1)
 695		return;
 696
 697	mlxsw_sp_acl_erp_table_enable(erp_table, false);
 698}
 699
 700static int
 701__mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table,
 702				   unsigned int *inc_num)
 703{
 704	int err;
 705
 706	/* If there are C-TCAM eRP or deltas in use we need to transition
 707	 * the region to use eRP table, if it is not already done
 708	 */
 709	if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) {
 710		err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
 711		if (err)
 712			return err;
 713	}
 714
 715	/* When C-TCAM or deltas are used, the eRP table must be used */
 716	if (erp_table->ops != &erp_multiple_masks_ops)
 717		erp_table->ops = &erp_multiple_masks_ops;
 718
 719	(*inc_num)++;
 720
 721	return 0;
 722}
 723
 724static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table)
 725{
 726	return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
 727						  &erp_table->num_ctcam_erps);
 728}
 729
 730static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table)
 731{
 732	return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
 733						  &erp_table->num_deltas);
 734}
 735
 736static void
 737__mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table,
 738				   unsigned int *dec_num)
 739{
 740	(*dec_num)--;
 741
 742	/* If there are no C-TCAM eRP or deltas in use, the state we
 743	 * transition to depends on the number of A-TCAM eRPs currently
 744	 * in use.
 745	 */
 746	if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0)
 747		return;
 748
 749	switch (erp_table->num_atcam_erps) {
 750	case 2:
 751		/* Keep using the eRP table, but correctly set the
 752		 * operations pointer so that when an A-TCAM eRP is
 753		 * deleted we will transition to use the master mask
 754		 */
 755		erp_table->ops = &erp_two_masks_ops;
 756		break;
 757	case 1:
 758		/* We only kept the eRP table because we had C-TCAM
 759		 * eRPs in use. Now that the last C-TCAM eRP is gone we
 760		 * can stop using the table and transition to use the
 761		 * master mask
 762		 */
 763		mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
 764		erp_table->ops = &erp_single_mask_ops;
 765		break;
 766	case 0:
 767		/* There are no more eRPs of any kind used by the region
 768		 * so free its eRP table and transition to initial state
 769		 */
 770		mlxsw_sp_acl_erp_table_disable(erp_table);
 771		mlxsw_sp_acl_erp_table_free(erp_table->erp_core,
 772					    erp_table->num_max_atcam_erps,
 773					    erp_table->aregion->type,
 774					    erp_table->base_index);
 775		erp_table->ops = &erp_no_mask_ops;
 776		break;
 777	default:
 778		break;
 779	}
 780}
 781
 782static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table)
 783{
 784	__mlxsw_sp_acl_erp_table_other_dec(erp_table,
 785					   &erp_table->num_ctcam_erps);
 786}
 787
 788static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table)
 789{
 790	__mlxsw_sp_acl_erp_table_other_dec(erp_table,
 791					   &erp_table->num_deltas);
 792}
 793
 794static struct mlxsw_sp_acl_erp *
 795mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
 796				   struct mlxsw_sp_acl_erp_key *key)
 797{
 798	struct mlxsw_sp_acl_erp *erp;
 799	int err;
 800
 801	erp = kzalloc(sizeof(*erp), GFP_KERNEL);
 802	if (!erp)
 803		return ERR_PTR(-ENOMEM);
 804
 805	memcpy(&erp->key, key, sizeof(*key));
 806	bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
 807			  MLXSW_SP_ACL_TCAM_MASK_LEN);
 808
 809	err = mlxsw_sp_acl_erp_ctcam_inc(erp_table);
 810	if (err)
 811		goto err_erp_ctcam_inc;
 812
 813	erp->erp_table = erp_table;
 814
 815	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
 816	if (err)
 817		goto err_master_mask_set;
 818
 819	err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);
 820	if (err)
 821		goto err_erp_region_ctcam_enable;
 822
 823	return erp;
 824
 825err_erp_region_ctcam_enable:
 826	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
 827err_master_mask_set:
 828	mlxsw_sp_acl_erp_ctcam_dec(erp_table);
 829err_erp_ctcam_inc:
 830	kfree(erp);
 831	return ERR_PTR(err);
 832}
 833
 834static void
 835mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp)
 836{
 837	struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
 838
 839	mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);
 840	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
 841	mlxsw_sp_acl_erp_ctcam_dec(erp_table);
 842	kfree(erp);
 843}
 844
 845static struct mlxsw_sp_acl_erp *
 846mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
 847			     struct mlxsw_sp_acl_erp_key *key)
 848{
 849	struct mlxsw_sp_acl_erp *erp;
 850	int err;
 851
 852	if (key->ctcam)
 853		return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
 854
 855	/* Expand the eRP table for the new eRP, if needed */
 856	err = mlxsw_sp_acl_erp_table_expand(erp_table);
 857	if (err)
 858		return ERR_PTR(err);
 859
 860	erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
 861	if (IS_ERR(erp))
 862		return erp;
 863
 864	err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
 865	if (err)
 866		goto err_erp_index_get;
 867
 868	err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
 869	if (err)
 870		goto err_table_erp_add;
 871
 872	err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
 873	if (err)
 874		goto err_region_erp_add;
 875
 876	erp_table->ops = &erp_multiple_masks_ops;
 877
 878	return erp;
 879
 880err_region_erp_add:
 881	mlxsw_sp_acl_erp_table_erp_del(erp);
 882err_table_erp_add:
 883	mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
 884err_erp_index_get:
 885	mlxsw_sp_acl_erp_generic_destroy(erp);
 886	return ERR_PTR(err);
 887}
 888
 889static void
 890mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
 891			      struct mlxsw_sp_acl_erp *erp)
 892{
 893	if (erp->key.ctcam)
 894		return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
 895
 896	mlxsw_sp_acl_erp_region_erp_del(erp);
 897	mlxsw_sp_acl_erp_table_erp_del(erp);
 898	mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
 899	mlxsw_sp_acl_erp_generic_destroy(erp);
 900
 901	if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 &&
 902	    erp_table->num_deltas == 0)
 903		erp_table->ops = &erp_two_masks_ops;
 904}
 905
 906static struct mlxsw_sp_acl_erp *
 907mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
 908				    struct mlxsw_sp_acl_erp_key *key)
 909{
 910	struct mlxsw_sp_acl_erp *erp;
 911	int err;
 912
 913	if (key->ctcam)
 914		return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
 915
 916	/* Transition to use eRP table instead of master mask */
 917	err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
 918	if (err)
 919		return ERR_PTR(err);
 920
 921	erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
 922	if (IS_ERR(erp)) {
 923		err = PTR_ERR(erp);
 924		goto err_erp_create;
 925	}
 926
 927	err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
 928	if (err)
 929		goto err_erp_index_get;
 930
 931	err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
 932	if (err)
 933		goto err_table_erp_add;
 934
 935	err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
 936	if (err)
 937		goto err_region_erp_add;
 938
 939	erp_table->ops = &erp_two_masks_ops;
 940
 941	return erp;
 942
 943err_region_erp_add:
 944	mlxsw_sp_acl_erp_table_erp_del(erp);
 945err_table_erp_add:
 946	mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
 947err_erp_index_get:
 948	mlxsw_sp_acl_erp_generic_destroy(erp);
 949err_erp_create:
 950	mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
 951	return ERR_PTR(err);
 952}
 953
 954static void
 955mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
 956				     struct mlxsw_sp_acl_erp *erp)
 957{
 958	if (erp->key.ctcam)
 959		return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
 960
 961	mlxsw_sp_acl_erp_region_erp_del(erp);
 962	mlxsw_sp_acl_erp_table_erp_del(erp);
 963	mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
 964	mlxsw_sp_acl_erp_generic_destroy(erp);
 965	/* Transition to use master mask instead of eRP table */
 966	mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
 967
 968	erp_table->ops = &erp_single_mask_ops;
 969}
 970
 971static struct mlxsw_sp_acl_erp *
 972mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
 973				   struct mlxsw_sp_acl_erp_key *key)
 974{
 975	struct mlxsw_sp_acl_erp *erp;
 976
 977	if (key->ctcam)
 978		return ERR_PTR(-EINVAL);
 979
 980	erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
 981	if (IS_ERR(erp))
 982		return erp;
 983
 984	erp_table->ops = &erp_single_mask_ops;
 985
 986	return erp;
 987}
 988
 989static void
 990mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
 991				    struct mlxsw_sp_acl_erp *erp)
 992{
 993	mlxsw_sp_acl_erp_generic_destroy(erp);
 994	erp_table->ops = &erp_no_mask_ops;
 995}
 996
 997static void
 998mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
 999				 struct mlxsw_sp_acl_erp *erp)
1000{
1001	WARN_ON(1);
1002}
1003
1004struct mlxsw_sp_acl_erp_mask *
1005mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion,
1006			  const char *mask, bool ctcam)
1007{
1008	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1009	struct mlxsw_sp_acl_erp_key key;
1010	struct objagg_obj *objagg_obj;
1011
1012	memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);
1013	key.ctcam = ctcam;
1014	mutex_lock(&erp_table->objagg_lock);
1015	objagg_obj = objagg_obj_get(erp_table->objagg, &key);
1016	mutex_unlock(&erp_table->objagg_lock);
1017	if (IS_ERR(objagg_obj))
1018		return ERR_CAST(objagg_obj);
1019	return (struct mlxsw_sp_acl_erp_mask *) objagg_obj;
1020}
1021
1022void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion,
1023			       struct mlxsw_sp_acl_erp_mask *erp_mask)
1024{
1025	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1026	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1027
1028	mutex_lock(&erp_table->objagg_lock);
1029	objagg_obj_put(erp_table->objagg, objagg_obj);
1030	mutex_unlock(&erp_table->objagg_lock);
1031}
1032
1033int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp,
1034			       struct mlxsw_sp_acl_atcam_region *aregion,
1035			       struct mlxsw_sp_acl_erp_mask *erp_mask,
1036			       struct mlxsw_sp_acl_atcam_entry *aentry)
1037{
1038	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1039	const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1040	unsigned int erp_bank;
1041
1042	if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1043		return 0;
1044
1045	erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1046	return mlxsw_sp_acl_bf_entry_add(mlxsw_sp,
1047					erp->erp_table->erp_core->bf,
1048					aregion, erp_bank, aentry);
1049}
1050
1051void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp,
1052				struct mlxsw_sp_acl_atcam_region *aregion,
1053				struct mlxsw_sp_acl_erp_mask *erp_mask,
1054				struct mlxsw_sp_acl_atcam_entry *aentry)
1055{
1056	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1057	const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1058	unsigned int erp_bank;
1059
1060	if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1061		return;
1062
1063	erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1064	mlxsw_sp_acl_bf_entry_del(mlxsw_sp,
1065				  erp->erp_table->erp_core->bf,
1066				  aregion, erp_bank, aentry);
1067}
1068
1069bool
1070mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1071{
1072	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1073	const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj);
1074
1075	return key->ctcam;
1076}
1077
1078u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1079{
1080	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1081	const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1082
1083	return erp->id;
1084}
1085
1086struct mlxsw_sp_acl_erp_delta {
1087	struct mlxsw_sp_acl_erp_key key;
1088	u16 start;
1089	u8 mask;
1090};
1091
1092u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta)
1093{
1094	return delta->start;
1095}
1096
1097u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta)
1098{
1099	return delta->mask;
1100}
1101
1102u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
1103				const char *enc_key)
1104{
1105	u16 start = delta->start;
1106	u8 mask = delta->mask;
1107	u16 tmp;
1108
1109	if (!mask)
1110		return 0;
1111
1112	tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)];
1113	if (start / 8 + 1 < __MASK_LEN)
1114		tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8;
1115	tmp >>= start % 8;
1116	tmp &= mask;
1117	return tmp;
1118}
1119
1120void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
1121				  const char *enc_key)
1122{
1123	u16 start = delta->start;
1124	u8 mask = delta->mask;
1125	unsigned char *byte;
1126	u16 tmp;
1127
1128	tmp = mask;
1129	tmp <<= start % 8;
1130	tmp = ~tmp;
1131
1132	byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)];
1133	*byte &= tmp & 0xff;
1134	if (start / 8 + 1 < __MASK_LEN) {
1135		byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)];
1136		*byte &= (tmp >> 8) & 0xff;
1137	}
1138}
1139
1140static const struct mlxsw_sp_acl_erp_delta
1141mlxsw_sp_acl_erp_delta_default = {};
1142
1143const struct mlxsw_sp_acl_erp_delta *
1144mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1145{
1146	struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1147	const struct mlxsw_sp_acl_erp_delta *delta;
1148
1149	delta = objagg_obj_delta_priv(objagg_obj);
1150	if (!delta)
1151		delta = &mlxsw_sp_acl_erp_delta_default;
1152	return delta;
1153}
1154
1155static int
1156mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key,
1157			    const struct mlxsw_sp_acl_erp_key *key,
1158			    u16 *delta_start, u8 *delta_mask)
1159{
1160	int offset = 0;
1161	int si = -1;
1162	u16 pmask;
1163	u16 mask;
1164	int i;
1165
1166	/* The difference between 2 masks can be up to 8 consecutive bits. */
1167	for (i = 0; i < __MASK_LEN; i++) {
1168		if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)])
1169			continue;
1170		if (si == -1)
1171			si = i;
1172		else if (si != i - 1)
1173			return -EINVAL;
1174	}
1175	if (si == -1) {
1176		/* The masks are the same, this can happen in case eRPs with
1177		 * the same mask were created in both A-TCAM and C-TCAM.
1178		 * The only possible condition under which this can happen
1179		 * is identical rule insertion. Delta is not possible here.
1180		 */
1181		return -EINVAL;
1182	}
1183	pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)];
1184	mask = (unsigned char) key->mask[__MASK_IDX(si)];
1185	if (si + 1 < __MASK_LEN) {
1186		pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8;
1187		mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8;
1188	}
1189
1190	if ((pmask ^ mask) & pmask)
1191		return -EINVAL;
1192	mask &= ~pmask;
1193	while (!(mask & (1 << offset)))
1194		offset++;
1195	while (!(mask & 1))
1196		mask >>= 1;
1197	if (mask & 0xff00)
1198		return -EINVAL;
1199
1200	*delta_start = si * 8 + offset;
1201	*delta_mask = mask;
1202
1203	return 0;
1204}
1205
1206static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj,
1207					 const void *obj)
1208{
1209	const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1210	const struct mlxsw_sp_acl_erp_key *key = obj;
1211	u16 delta_start;
1212	u8 delta_mask;
1213	int err;
1214
1215	err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1216					  &delta_start, &delta_mask);
1217	return err ? false : true;
1218}
1219
1220static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj,
1221					   void *obj)
1222{
1223	struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1224	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1225	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1226	struct mlxsw_sp_acl_erp_key *key = obj;
1227	struct mlxsw_sp_acl_erp_delta *delta;
1228	u16 delta_start;
1229	u8 delta_mask;
1230	int err;
1231
1232	if (parent_key->ctcam || key->ctcam)
1233		return ERR_PTR(-EINVAL);
1234	err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1235					  &delta_start, &delta_mask);
1236	if (err)
1237		return ERR_PTR(-EINVAL);
1238
1239	delta = kzalloc(sizeof(*delta), GFP_KERNEL);
1240	if (!delta)
1241		return ERR_PTR(-ENOMEM);
1242	delta->start = delta_start;
1243	delta->mask = delta_mask;
1244
1245	err = mlxsw_sp_acl_erp_delta_inc(erp_table);
1246	if (err)
1247		goto err_erp_delta_inc;
1248
1249	memcpy(&delta->key, key, sizeof(*key));
1250	err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key);
1251	if (err)
1252		goto err_master_mask_set;
1253
1254	return delta;
1255
1256err_master_mask_set:
1257	mlxsw_sp_acl_erp_delta_dec(erp_table);
1258err_erp_delta_inc:
1259	kfree(delta);
1260	return ERR_PTR(err);
1261}
1262
1263static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv)
1264{
1265	struct mlxsw_sp_acl_erp_delta *delta = delta_priv;
1266	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1267	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1268
1269	mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key);
1270	mlxsw_sp_acl_erp_delta_dec(erp_table);
1271	kfree(delta);
1272}
1273
1274static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj,
1275					  unsigned int root_id)
1276{
1277	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1278	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1279	struct mlxsw_sp_acl_erp_key *key = obj;
1280
1281	if (!key->ctcam &&
1282	    root_id != OBJAGG_OBJ_ROOT_ID_INVALID &&
1283	    root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION)
1284		return ERR_PTR(-ENOBUFS);
1285	return erp_table->ops->erp_create(erp_table, key);
1286}
1287
1288static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv)
1289{
1290	struct mlxsw_sp_acl_atcam_region *aregion = priv;
1291	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1292
1293	erp_table->ops->erp_destroy(erp_table, root_priv);
1294}
1295
1296static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = {
1297	.obj_size = sizeof(struct mlxsw_sp_acl_erp_key),
1298	.delta_check = mlxsw_sp_acl_erp_delta_check,
1299	.delta_create = mlxsw_sp_acl_erp_delta_create,
1300	.delta_destroy = mlxsw_sp_acl_erp_delta_destroy,
1301	.root_create = mlxsw_sp_acl_erp_root_create,
1302	.root_destroy = mlxsw_sp_acl_erp_root_destroy,
1303};
1304
1305static struct mlxsw_sp_acl_erp_table *
1306mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion,
1307			      struct objagg_hints *hints)
1308{
1309	struct mlxsw_sp_acl_erp_table *erp_table;
1310	int err;
1311
1312	erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL);
1313	if (!erp_table)
1314		return ERR_PTR(-ENOMEM);
1315
1316	erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops,
1317					  hints, aregion);
1318	if (IS_ERR(erp_table->objagg)) {
1319		err = PTR_ERR(erp_table->objagg);
1320		goto err_objagg_create;
1321	}
1322
1323	erp_table->erp_core = aregion->atcam->erp_core;
1324	erp_table->ops = &erp_no_mask_ops;
1325	INIT_LIST_HEAD(&erp_table->atcam_erps_list);
1326	erp_table->aregion = aregion;
1327	mutex_init(&erp_table->objagg_lock);
1328
1329	return erp_table;
1330
1331err_objagg_create:
1332	kfree(erp_table);
1333	return ERR_PTR(err);
1334}
1335
1336static void
1337mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)
1338{
1339	WARN_ON(!list_empty(&erp_table->atcam_erps_list));
1340	mutex_destroy(&erp_table->objagg_lock);
1341	objagg_destroy(erp_table->objagg);
1342	kfree(erp_table);
1343}
1344
1345static int
1346mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion)
1347{
1348	struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1349	char percr_pl[MLXSW_REG_PERCR_LEN];
1350
1351	mlxsw_reg_percr_pack(percr_pl, aregion->region->id);
1352	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
1353}
1354
1355static int
1356mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion)
1357{
1358	struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1359	char pererp_pl[MLXSW_REG_PERERP_LEN];
1360
1361	mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0,
1362			      0, 0);
1363	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
1364}
1365
1366static int
1367mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp,
1368			     struct mlxsw_sp_acl_atcam_region *aregion,
1369			     struct objagg_hints *hints, bool *p_rehash_needed)
1370{
1371	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1372	const struct objagg_stats *ostats;
1373	const struct objagg_stats *hstats;
1374	int err;
1375
1376	*p_rehash_needed = false;
1377
1378	mutex_lock(&erp_table->objagg_lock);
1379	ostats = objagg_stats_get(erp_table->objagg);
1380	mutex_unlock(&erp_table->objagg_lock);
1381	if (IS_ERR(ostats)) {
1382		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n");
1383		return PTR_ERR(ostats);
1384	}
1385
1386	hstats = objagg_hints_stats_get(hints);
1387	if (IS_ERR(hstats)) {
1388		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n");
1389		err = PTR_ERR(hstats);
1390		goto err_hints_stats_get;
1391	}
1392
1393	/* Very basic criterion for now. */
1394	if (hstats->root_count < ostats->root_count)
1395		*p_rehash_needed = true;
1396
1397	err = 0;
1398
1399	objagg_stats_put(hstats);
1400err_hints_stats_get:
1401	objagg_stats_put(ostats);
1402	return err;
1403}
1404
1405void *
1406mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
1407{
1408	struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1409	struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1410	struct objagg_hints *hints;
1411	bool rehash_needed;
1412	int err;
1413
1414	mutex_lock(&erp_table->objagg_lock);
1415	hints = objagg_hints_get(erp_table->objagg,
1416				 OBJAGG_OPT_ALGO_SIMPLE_GREEDY);
1417	mutex_unlock(&erp_table->objagg_lock);
1418	if (IS_ERR(hints)) {
1419		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n");
1420		return ERR_CAST(hints);
1421	}
1422	err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints,
1423					   &rehash_needed);
1424	if (err)
1425		goto errout;
1426
1427	if (!rehash_needed) {
1428		err = -EAGAIN;
1429		goto errout;
1430	}
1431	return hints;
1432
1433errout:
1434	objagg_hints_put(hints);
1435	return ERR_PTR(err);
1436}
1437
1438void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv)
1439{
1440	struct objagg_hints *hints = hints_priv;
1441
1442	objagg_hints_put(hints);
1443}
1444
1445int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion,
1446				 void *hints_priv)
1447{
1448	struct mlxsw_sp_acl_erp_table *erp_table;
1449	struct objagg_hints *hints = hints_priv;
1450	int err;
1451
1452	erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints);
1453	if (IS_ERR(erp_table))
1454		return PTR_ERR(erp_table);
1455	aregion->erp_table = erp_table;
1456
1457	/* Initialize the region's master mask to all zeroes */
1458	err = mlxsw_sp_acl_erp_master_mask_init(aregion);
1459	if (err)
1460		goto err_erp_master_mask_init;
1461
1462	/* Initialize the region to not use the eRP table */
1463	err = mlxsw_sp_acl_erp_region_param_init(aregion);
1464	if (err)
1465		goto err_erp_region_param_init;
1466
1467	return 0;
1468
1469err_erp_region_param_init:
1470err_erp_master_mask_init:
1471	mlxsw_sp_acl_erp_table_destroy(erp_table);
1472	return err;
1473}
1474
1475void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
1476{
1477	mlxsw_sp_acl_erp_table_destroy(aregion->erp_table);
1478}
1479
1480static int
1481mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp,
1482				    struct mlxsw_sp_acl_erp_core *erp_core)
1483{
1484	unsigned int size;
1485
1486	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) ||
1487	    !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) ||
1488	    !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) ||
1489	    !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB))
1490		return -EIO;
1491
1492	size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB);
1493	erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size;
1494
1495	size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB);
1496	erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size;
1497
1498	size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB);
1499	erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size;
1500
1501	size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB);
1502	erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size;
1503
1504	return 0;
1505}
1506
1507static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp,
1508					struct mlxsw_sp_acl_erp_core *erp_core)
1509{
1510	unsigned int erpt_bank_size;
1511	int err;
1512
1513	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) ||
1514	    !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS))
1515		return -EIO;
1516	erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1517					    ACL_MAX_ERPT_BANK_SIZE);
1518	erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1519						     ACL_MAX_ERPT_BANKS);
1520
1521	erp_core->erp_tables = gen_pool_create(0, -1);
1522	if (!erp_core->erp_tables)
1523		return -ENOMEM;
1524	gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL);
1525
1526	err = gen_pool_add(erp_core->erp_tables,
1527			   MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size,
1528			   -1);
1529	if (err)
1530		goto err_gen_pool_add;
1531
1532	erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks);
1533	if (IS_ERR(erp_core->bf)) {
1534		err = PTR_ERR(erp_core->bf);
1535		goto err_bf_init;
1536	}
1537
1538	/* Different regions require masks of different sizes */
1539	err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);
1540	if (err)
1541		goto err_erp_tables_sizes_query;
1542
1543	return 0;
1544
1545err_erp_tables_sizes_query:
1546	mlxsw_sp_acl_bf_fini(erp_core->bf);
1547err_bf_init:
1548err_gen_pool_add:
1549	gen_pool_destroy(erp_core->erp_tables);
1550	return err;
1551}
1552
1553static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,
1554					 struct mlxsw_sp_acl_erp_core *erp_core)
1555{
1556	mlxsw_sp_acl_bf_fini(erp_core->bf);
1557	gen_pool_destroy(erp_core->erp_tables);
1558}
1559
1560int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp,
1561			   struct mlxsw_sp_acl_atcam *atcam)
1562{
1563	struct mlxsw_sp_acl_erp_core *erp_core;
1564	int err;
1565
1566	erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL);
1567	if (!erp_core)
1568		return -ENOMEM;
1569	erp_core->mlxsw_sp = mlxsw_sp;
1570	atcam->erp_core = erp_core;
1571
1572	err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core);
1573	if (err)
1574		goto err_erp_tables_init;
1575
1576	return 0;
1577
1578err_erp_tables_init:
1579	kfree(erp_core);
1580	return err;
1581}
1582
1583void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,
1584			    struct mlxsw_sp_acl_atcam *atcam)
1585{
1586	mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core);
1587	kfree(atcam->erp_core);
1588}