spectrum_acl_atcam.c

  1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
  2/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
  3
  4#include <linux/kernel.h>
  5#include <linux/err.h>
  6#include <linux/errno.h>
  7#include <linux/gfp.h>
  8#include <linux/refcount.h>
  9#include <linux/rhashtable.h>
 10#define CREATE_TRACE_POINTS
 11#include <trace/events/mlxsw.h>
 12
 13#include "reg.h"
 14#include "core.h"
 15#include "spectrum.h"
 16#include "spectrum_acl_tcam.h"
 17#include "core_acl_flex_keys.h"
 18
 19#define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START	0
 20#define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END	5
 21
 22struct mlxsw_sp_acl_atcam_lkey_id_ht_key {
 23	char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */
 24	u8 erp_id;
 25};
 26
 27struct mlxsw_sp_acl_atcam_lkey_id {
 28	struct rhash_head ht_node;
 29	struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key;
 30	refcount_t refcnt;
 31	u32 id;
 32};
 33
 34struct mlxsw_sp_acl_atcam_region_ops {
 35	int (*init)(struct mlxsw_sp_acl_atcam_region *aregion);
 36	void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion);
 37	struct mlxsw_sp_acl_atcam_lkey_id *
 38		(*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion,
 39			       char *enc_key, u8 erp_id);
 40	void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion,
 41			    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id);
 42};
 43
 44struct mlxsw_sp_acl_atcam_region_generic {
 45	struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id;
 46};
 47
 48struct mlxsw_sp_acl_atcam_region_12kb {
 49	struct rhashtable lkey_ht;
 50	unsigned int max_lkey_id;
 51	unsigned long *used_lkey_id;
 52};
 53
 54static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = {
 55	.key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key),
 56	.key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key),
 57	.head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node),
 58};
 59
 60static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = {
 61	.key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key),
 62	.key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key),
 63	.head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node),
 64};
 65
 66static bool
 67mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry)
 68{
 69	return mlxsw_sp_acl_erp_mask_is_ctcam(aentry->erp_mask);
 70}
 71
 72static int
 73mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion)
 74{
 75	struct mlxsw_sp_acl_atcam_region_generic *region_generic;
 76
 77	region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL);
 78	if (!region_generic)
 79		return -ENOMEM;
 80
 81	refcount_set(&region_generic->dummy_lkey_id.refcnt, 1);
 82	aregion->priv = region_generic;
 83
 84	return 0;
 85}
 86
 87static void
 88mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion)
 89{
 90	kfree(aregion->priv);
 91}
 92
 93static struct mlxsw_sp_acl_atcam_lkey_id *
 94mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
 95				       char *enc_key, u8 erp_id)
 96{
 97	struct mlxsw_sp_acl_atcam_region_generic *region_generic;
 98
 99	region_generic = aregion->priv;
100	return &region_generic->dummy_lkey_id;
101}
102
103static void
104mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
105				       struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
106{
107}
108
109static const struct mlxsw_sp_acl_atcam_region_ops
110mlxsw_sp_acl_atcam_region_generic_ops = {
111	.init		= mlxsw_sp_acl_atcam_region_generic_init,
112	.fini		= mlxsw_sp_acl_atcam_region_generic_fini,
113	.lkey_id_get	= mlxsw_sp_acl_atcam_generic_lkey_id_get,
114	.lkey_id_put	= mlxsw_sp_acl_atcam_generic_lkey_id_put,
115};
116
117static int
118mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion)
119{
120	struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
121	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb;
122	u64 max_lkey_id;
123	int err;
124
125	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID))
126		return -EIO;
127
128	max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID);
129	region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL);
130	if (!region_12kb)
131		return -ENOMEM;
132
133	region_12kb->used_lkey_id = bitmap_zalloc(max_lkey_id, GFP_KERNEL);
134	if (!region_12kb->used_lkey_id) {
135		err = -ENOMEM;
136		goto err_used_lkey_id_alloc;
137	}
138
139	err = rhashtable_init(&region_12kb->lkey_ht,
140			      &mlxsw_sp_acl_atcam_lkey_id_ht_params);
141	if (err)
142		goto err_rhashtable_init;
143
144	region_12kb->max_lkey_id = max_lkey_id;
145	aregion->priv = region_12kb;
146
147	return 0;
148
149err_rhashtable_init:
150	bitmap_free(region_12kb->used_lkey_id);
151err_used_lkey_id_alloc:
152	kfree(region_12kb);
153	return err;
154}
155
156static void
157mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion)
158{
159	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
160
161	rhashtable_destroy(&region_12kb->lkey_ht);
162	bitmap_free(region_12kb->used_lkey_id);
163	kfree(region_12kb);
164}
165
166static struct mlxsw_sp_acl_atcam_lkey_id *
167mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion,
168				  struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key)
169{
170	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
171	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
172	u32 id;
173	int err;
174
175	id = find_first_zero_bit(region_12kb->used_lkey_id,
176				 region_12kb->max_lkey_id);
177	if (id < region_12kb->max_lkey_id)
178		__set_bit(id, region_12kb->used_lkey_id);
179	else
180		return ERR_PTR(-ENOBUFS);
181
182	lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL);
183	if (!lkey_id) {
184		err = -ENOMEM;
185		goto err_lkey_id_alloc;
186	}
187
188	lkey_id->id = id;
189	memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key));
190	refcount_set(&lkey_id->refcnt, 1);
191
192	err = rhashtable_insert_fast(&region_12kb->lkey_ht,
193				     &lkey_id->ht_node,
194				     mlxsw_sp_acl_atcam_lkey_id_ht_params);
195	if (err)
196		goto err_rhashtable_insert;
197
198	return lkey_id;
199
200err_rhashtable_insert:
201	kfree(lkey_id);
202err_lkey_id_alloc:
203	__clear_bit(id, region_12kb->used_lkey_id);
204	return ERR_PTR(err);
205}
206
207static void
208mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion,
209				   struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
210{
211	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
212	u32 id = lkey_id->id;
213
214	rhashtable_remove_fast(&region_12kb->lkey_ht, &lkey_id->ht_node,
215			       mlxsw_sp_acl_atcam_lkey_id_ht_params);
216	kfree(lkey_id);
217	__clear_bit(id, region_12kb->used_lkey_id);
218}
219
220static struct mlxsw_sp_acl_atcam_lkey_id *
221mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
222				    char *enc_key, u8 erp_id)
223{
224	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
225	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
226	struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } };
227	struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
228	struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
229	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
230
231	memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key));
232	mlxsw_afk_clear(afk, ht_key.enc_key,
233			MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START,
234			MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END);
235	ht_key.erp_id = erp_id;
236	lkey_id = rhashtable_lookup_fast(&region_12kb->lkey_ht, &ht_key,
237					 mlxsw_sp_acl_atcam_lkey_id_ht_params);
238	if (lkey_id) {
239		refcount_inc(&lkey_id->refcnt);
240		return lkey_id;
241	}
242
243	return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key);
244}
245
246static void
247mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
248				    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
249{
250	if (refcount_dec_and_test(&lkey_id->refcnt))
251		mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id);
252}
253
254static const struct mlxsw_sp_acl_atcam_region_ops
255mlxsw_sp_acl_atcam_region_12kb_ops = {
256	.init		= mlxsw_sp_acl_atcam_region_12kb_init,
257	.fini		= mlxsw_sp_acl_atcam_region_12kb_fini,
258	.lkey_id_get	= mlxsw_sp_acl_atcam_12kb_lkey_id_get,
259	.lkey_id_put	= mlxsw_sp_acl_atcam_12kb_lkey_id_put,
260};
261
262static const struct mlxsw_sp_acl_atcam_region_ops *
263mlxsw_sp_acl_atcam_region_ops_arr[] = {
264	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB]	=
265		&mlxsw_sp_acl_atcam_region_generic_ops,
266	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB]	=
267		&mlxsw_sp_acl_atcam_region_generic_ops,
268	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB]	=
269		&mlxsw_sp_acl_atcam_region_generic_ops,
270	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB]	=
271		&mlxsw_sp_acl_atcam_region_12kb_ops,
272};
273
274int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp,
275					u16 region_id)
276{
277	char perar_pl[MLXSW_REG_PERAR_LEN];
278	/* For now, just assume that every region has 12 key blocks */
279	u16 hw_region = region_id * 3;
280	u64 max_regions;
281
282	max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
283	if (hw_region >= max_regions)
284		return -ENOBUFS;
285
286	mlxsw_reg_perar_pack(perar_pl, region_id, hw_region);
287	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl);
288}
289
290static void
291mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion)
292{
293	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
294	enum mlxsw_sp_acl_atcam_region_type region_type;
295	unsigned int blocks_count;
296
297	/* We already know the blocks count can not exceed the maximum
298	 * blocks count.
299	 */
300	blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
301	if (blocks_count <= 2)
302		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB;
303	else if (blocks_count <= 4)
304		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB;
305	else if (blocks_count <= 8)
306		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB;
307	else
308		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB;
309
310	aregion->type = region_type;
311	aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type];
312}
313
314int
315mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp,
316			       struct mlxsw_sp_acl_atcam *atcam,
317			       struct mlxsw_sp_acl_atcam_region *aregion,
318			       struct mlxsw_sp_acl_tcam_region *region,
319			       void *hints_priv,
320			       const struct mlxsw_sp_acl_ctcam_region_ops *ops)
321{
322	int err;
323
324	aregion->region = region;
325	aregion->atcam = atcam;
326	mlxsw_sp_acl_atcam_region_type_init(aregion);
327	INIT_LIST_HEAD(&aregion->entries_list);
328
329	err = rhashtable_init(&aregion->entries_ht,
330			      &mlxsw_sp_acl_atcam_entries_ht_params);
331	if (err)
332		return err;
333	err = aregion->ops->init(aregion);
334	if (err)
335		goto err_ops_init;
336	err = mlxsw_sp_acl_erp_region_init(aregion, hints_priv);
337	if (err)
338		goto err_erp_region_init;
339	err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion,
340					     region, ops);
341	if (err)
342		goto err_ctcam_region_init;
343
344	return 0;
345
346err_ctcam_region_init:
347	mlxsw_sp_acl_erp_region_fini(aregion);
348err_erp_region_init:
349	aregion->ops->fini(aregion);
350err_ops_init:
351	rhashtable_destroy(&aregion->entries_ht);
352	return err;
353}
354
355void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
356{
357	mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion);
358	mlxsw_sp_acl_erp_region_fini(aregion);
359	aregion->ops->fini(aregion);
360	rhashtable_destroy(&aregion->entries_ht);
361	WARN_ON(!list_empty(&aregion->entries_list));
362}
363
364void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion,
365				   struct mlxsw_sp_acl_atcam_chunk *achunk,
366				   unsigned int priority)
367{
368	mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk,
369				      priority);
370}
371
372void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk)
373{
374	mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk);
375}
376
377static int
378mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
379				       struct mlxsw_sp_acl_atcam_region *aregion,
380				       struct mlxsw_sp_acl_atcam_entry *aentry,
381				       struct mlxsw_sp_acl_rule_info *rulei)
382{
383	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
384	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
385	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
386	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
387	u32 kvdl_index, priority;
388	int err;
389
390	err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
391	if (err)
392		return err;
393
394	lkey_id = aregion->ops->lkey_id_get(aregion, aentry->ht_key.enc_key,
395					    erp_id);
396	if (IS_ERR(lkey_id))
397		return PTR_ERR(lkey_id);
398	aentry->lkey_id = lkey_id;
399
400	kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
401	mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE,
402			     priority, region->tcam_region_info,
403			     aentry->ht_key.enc_key, erp_id,
404			     aentry->delta_info.start,
405			     aentry->delta_info.mask,
406			     aentry->delta_info.value,
407			     refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
408			     kvdl_index);
409	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
410	if (err)
411		goto err_ptce3_write;
412
413	return 0;
414
415err_ptce3_write:
416	aregion->ops->lkey_id_put(aregion, lkey_id);
417	return err;
418}
419
420static void
421mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
422				       struct mlxsw_sp_acl_atcam_region *aregion,
423				       struct mlxsw_sp_acl_atcam_entry *aentry)
424{
425	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
426	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
427	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
428	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
429
430	mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0,
431			     region->tcam_region_info,
432			     aentry->ht_key.enc_key, erp_id,
433			     aentry->delta_info.start,
434			     aentry->delta_info.mask,
435			     aentry->delta_info.value,
436			     refcount_read(&lkey_id->refcnt) != 1,
437			     lkey_id->id, 0);
438	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
439	aregion->ops->lkey_id_put(aregion, lkey_id);
440}
441
442static int
443mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
444					       struct mlxsw_sp_acl_atcam_region *aregion,
445					       struct mlxsw_sp_acl_atcam_entry *aentry,
446					       struct mlxsw_sp_acl_rule_info *rulei)
447{
448	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
449	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
450	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
451	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
452	u32 kvdl_index, priority;
453	int err;
454
455	err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
456	if (err)
457		return err;
458	kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
459	mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_UPDATE,
460			     priority, region->tcam_region_info,
461			     aentry->ht_key.enc_key, erp_id,
462			     aentry->delta_info.start,
463			     aentry->delta_info.mask,
464			     aentry->delta_info.value,
465			     refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
466			     kvdl_index);
467	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
468}
469
470static int
471__mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
472			       struct mlxsw_sp_acl_atcam_region *aregion,
473			       struct mlxsw_sp_acl_atcam_entry *aentry,
474			       struct mlxsw_sp_acl_rule_info *rulei)
475{
476	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
477	char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
478	struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
479	const struct mlxsw_sp_acl_erp_delta *delta;
480	struct mlxsw_sp_acl_erp_mask *erp_mask;
481	int err;
482
483	mlxsw_afk_encode(afk, region->key_info, &rulei->values,
484			 aentry->ht_key.enc_key, mask);
485
486	erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, false);
487	if (IS_ERR(erp_mask))
488		return PTR_ERR(erp_mask);
489	aentry->erp_mask = erp_mask;
490	aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask);
491
492	/* Compute all needed delta information and clear the delta bits
493	 * from the encoded key.
494	 */
495	delta = mlxsw_sp_acl_erp_delta(aentry->erp_mask);
496	aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta);
497	aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta);
498	aentry->delta_info.value =
499		mlxsw_sp_acl_erp_delta_value(delta, aentry->ht_key.enc_key);
500	mlxsw_sp_acl_erp_delta_clear(delta, aentry->ht_key.enc_key);
501
502	/* Add rule to the list of A-TCAM rules, assuming this
503	 * rule is intended to A-TCAM. In case this rule does
504	 * not fit into A-TCAM it will be removed from the list.
505	 */
506	list_add(&aentry->list, &aregion->entries_list);
507
508	/* We can't insert identical rules into the A-TCAM, so fail and
509	 * let the rule spill into C-TCAM
510	 */
511	err = rhashtable_lookup_insert_fast(&aregion->entries_ht,
512					    &aentry->ht_node,
513					    mlxsw_sp_acl_atcam_entries_ht_params);
514	if (err)
515		goto err_rhashtable_insert;
516
517	/* Bloom filter must be updated here, before inserting the rule into
518	 * the A-TCAM.
519	 */
520	err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry);
521	if (err)
522		goto err_bf_insert;
523
524	err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry,
525						     rulei);
526	if (err)
527		goto err_rule_insert;
528
529	return 0;
530
531err_rule_insert:
532	mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry);
533err_bf_insert:
534	rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
535			       mlxsw_sp_acl_atcam_entries_ht_params);
536err_rhashtable_insert:
537	list_del(&aentry->list);
538	mlxsw_sp_acl_erp_mask_put(aregion, erp_mask);
539	return err;
540}
541
542static void
543__mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
544			       struct mlxsw_sp_acl_atcam_region *aregion,
545			       struct mlxsw_sp_acl_atcam_entry *aentry)
546{
547	mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry);
548	mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, aentry->erp_mask, aentry);
549	rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
550			       mlxsw_sp_acl_atcam_entries_ht_params);
551	list_del(&aentry->list);
552	mlxsw_sp_acl_erp_mask_put(aregion, aentry->erp_mask);
553}
554
555static int
556__mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
557					  struct mlxsw_sp_acl_atcam_region *aregion,
558					  struct mlxsw_sp_acl_atcam_entry *aentry,
559					  struct mlxsw_sp_acl_rule_info *rulei)
560{
561	return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion,
562							      aentry, rulei);
563}
564
565int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
566				 struct mlxsw_sp_acl_atcam_region *aregion,
567				 struct mlxsw_sp_acl_atcam_chunk *achunk,
568				 struct mlxsw_sp_acl_atcam_entry *aentry,
569				 struct mlxsw_sp_acl_rule_info *rulei)
570{
571	int err;
572
573	err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei);
574	if (!err)
575		return 0;
576
577	/* It is possible we failed to add the rule to the A-TCAM due to
578	 * exceeded number of masks. Try to spill into C-TCAM.
579	 */
580	trace_mlxsw_sp_acl_atcam_entry_add_ctcam_spill(mlxsw_sp, aregion);
581	err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion,
582					   &achunk->cchunk, &aentry->centry,
583					   rulei, true);
584	if (!err)
585		return 0;
586
587	return err;
588}
589
590void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
591				  struct mlxsw_sp_acl_atcam_region *aregion,
592				  struct mlxsw_sp_acl_atcam_chunk *achunk,
593				  struct mlxsw_sp_acl_atcam_entry *aentry)
594{
595	if (mlxsw_sp_acl_atcam_is_centry(aentry))
596		mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion,
597					     &achunk->cchunk, &aentry->centry);
598	else
599		__mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry);
600}
601
602int
603mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
604					struct mlxsw_sp_acl_atcam_region *aregion,
605					struct mlxsw_sp_acl_atcam_entry *aentry,
606					struct mlxsw_sp_acl_rule_info *rulei)
607{
608	int err;
609
610	if (mlxsw_sp_acl_atcam_is_centry(aentry))
611		err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp,
612							      &aregion->cregion,
613							      &aentry->centry,
614							      rulei);
615	else
616		err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp,
617								aregion, aentry,
618								rulei);
619
620	return err;
621}
622
623int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp,
624			    struct mlxsw_sp_acl_atcam *atcam)
625{
626	return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam);
627}
628
629void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp,
630			     struct mlxsw_sp_acl_atcam *atcam)
631{
632	mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam);
633}
634
635void *
636mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
637{
638	return mlxsw_sp_acl_erp_rehash_hints_get(aregion);
639}
640
641void mlxsw_sp_acl_atcam_rehash_hints_put(void *hints_priv)
642{
643	mlxsw_sp_acl_erp_rehash_hints_put(hints_priv);
644}