dr_rule.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2019 Mellanox Technologies. */
   3
   4#include "dr_types.h"
   5
   6#if defined(CONFIG_FRAME_WARN) && (CONFIG_FRAME_WARN < 2048)
   7/* don't try to optimize STE allocation if the stack is too constaraining */
   8#define DR_RULE_MAX_STES_OPTIMIZED 0
   9#else
  10#define DR_RULE_MAX_STES_OPTIMIZED 2
  11#endif
  12#define DR_RULE_MAX_STE_CHAIN_OPTIMIZED (DR_RULE_MAX_STES_OPTIMIZED + DR_ACTION_MAX_STES)
  13
  14static int dr_rule_append_to_miss_list(struct mlx5dr_domain *dmn,
  15				       enum mlx5dr_domain_nic_type nic_type,
  16				       struct mlx5dr_ste *new_last_ste,
  17				       struct list_head *miss_list,
  18				       struct list_head *send_list)
  19{
  20	struct mlx5dr_ste_ctx *ste_ctx = dmn->ste_ctx;
  21	struct mlx5dr_ste_send_info *ste_info_last;
  22	struct mlx5dr_ste *last_ste;
  23
  24	/* The new entry will be inserted after the last */
  25	last_ste = list_last_entry(miss_list, struct mlx5dr_ste, miss_list_node);
  26	WARN_ON(!last_ste);
  27
  28	ste_info_last = mlx5dr_send_info_alloc(dmn, nic_type);
  29	if (!ste_info_last)
  30		return -ENOMEM;
  31
  32	mlx5dr_ste_set_miss_addr(ste_ctx, mlx5dr_ste_get_hw_ste(last_ste),
  33				 mlx5dr_ste_get_icm_addr(new_last_ste));
  34	list_add_tail(&new_last_ste->miss_list_node, miss_list);
  35
  36	mlx5dr_send_fill_and_append_ste_send_info(last_ste, DR_STE_SIZE_CTRL,
  37						  0, mlx5dr_ste_get_hw_ste(last_ste),
  38						  ste_info_last, send_list, true);
  39
  40	return 0;
  41}
  42
  43static void dr_rule_set_last_ste_miss_addr(struct mlx5dr_matcher *matcher,
  44					   struct mlx5dr_matcher_rx_tx *nic_matcher,
  45					   u8 *hw_ste)
  46{
  47	struct mlx5dr_ste_ctx *ste_ctx = matcher->tbl->dmn->ste_ctx;
  48	u64 icm_addr;
  49
  50	if (mlx5dr_ste_is_miss_addr_set(ste_ctx, hw_ste))
  51		return;
  52
  53	icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(nic_matcher->e_anchor->chunk);
  54	mlx5dr_ste_set_miss_addr(ste_ctx, hw_ste, icm_addr);
  55}
  56
  57static struct mlx5dr_ste *
  58dr_rule_create_collision_htbl(struct mlx5dr_matcher *matcher,
  59			      struct mlx5dr_matcher_rx_tx *nic_matcher,
  60			      u8 *hw_ste)
  61{
  62	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
  63	struct mlx5dr_ste_htbl *new_htbl;
  64	struct mlx5dr_ste *ste;
  65
  66	/* Create new table for miss entry */
  67	new_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool,
  68					 DR_CHUNK_SIZE_1,
  69					 MLX5DR_STE_LU_TYPE_DONT_CARE,
  70					 0);
  71	if (!new_htbl) {
  72		mlx5dr_dbg(dmn, "Failed allocating collision table\n");
  73		return NULL;
  74	}
  75
  76	/* One and only entry, never grows */
  77	ste = new_htbl->chunk->ste_arr;
  78	dr_rule_set_last_ste_miss_addr(matcher, nic_matcher, hw_ste);
  79	mlx5dr_htbl_get(new_htbl);
  80
  81	return ste;
  82}
  83
  84static struct mlx5dr_ste *
  85dr_rule_create_collision_entry(struct mlx5dr_matcher *matcher,
  86			       struct mlx5dr_matcher_rx_tx *nic_matcher,
  87			       u8 *hw_ste,
  88			       struct mlx5dr_ste *orig_ste)
  89{
  90	struct mlx5dr_ste *ste;
  91
  92	ste = dr_rule_create_collision_htbl(matcher, nic_matcher, hw_ste);
  93	if (!ste) {
  94		mlx5dr_dbg(matcher->tbl->dmn, "Failed creating collision entry\n");
  95		return NULL;
  96	}
  97
  98	ste->ste_chain_location = orig_ste->ste_chain_location;
  99	ste->htbl->pointing_ste = orig_ste->htbl->pointing_ste;
 100
 101	/* In collision entry, all members share the same miss_list_head */
 102	ste->htbl->chunk->miss_list = mlx5dr_ste_get_miss_list(orig_ste);
 103
 104	/* Next table */
 105	if (mlx5dr_ste_create_next_htbl(matcher, nic_matcher, ste, hw_ste,
 106					DR_CHUNK_SIZE_1)) {
 107		mlx5dr_dbg(matcher->tbl->dmn, "Failed allocating table\n");
 108		goto free_tbl;
 109	}
 110
 111	return ste;
 112
 113free_tbl:
 114	mlx5dr_ste_free(ste, matcher, nic_matcher);
 115	return NULL;
 116}
 117
 118static int
 119dr_rule_handle_one_ste_in_update_list(struct mlx5dr_ste_send_info *ste_info,
 120				      struct mlx5dr_domain *dmn)
 121{
 122	int ret;
 123
 124	list_del(&ste_info->send_list);
 125
 126	/* Copy data to ste, only reduced size or control, the last 16B (mask)
 127	 * is already written to the hw.
 128	 */
 129	if (ste_info->size == DR_STE_SIZE_CTRL)
 130		memcpy(mlx5dr_ste_get_hw_ste(ste_info->ste),
 131		       ste_info->data, DR_STE_SIZE_CTRL);
 132	else
 133		memcpy(mlx5dr_ste_get_hw_ste(ste_info->ste),
 134		       ste_info->data, DR_STE_SIZE_REDUCED);
 135
 136	ret = mlx5dr_send_postsend_ste(dmn, ste_info->ste, ste_info->data,
 137				       ste_info->size, ste_info->offset);
 138	if (ret)
 139		goto out;
 140
 141out:
 142	mlx5dr_send_info_free(ste_info);
 143	return ret;
 144}
 145
 146static int dr_rule_send_update_list(struct list_head *send_ste_list,
 147				    struct mlx5dr_domain *dmn,
 148				    bool is_reverse)
 149{
 150	struct mlx5dr_ste_send_info *ste_info, *tmp_ste_info;
 151	int ret;
 152
 153	if (is_reverse) {
 154		list_for_each_entry_safe_reverse(ste_info, tmp_ste_info,
 155						 send_ste_list, send_list) {
 156			ret = dr_rule_handle_one_ste_in_update_list(ste_info,
 157								    dmn);
 158			if (ret)
 159				return ret;
 160		}
 161	} else {
 162		list_for_each_entry_safe(ste_info, tmp_ste_info,
 163					 send_ste_list, send_list) {
 164			ret = dr_rule_handle_one_ste_in_update_list(ste_info,
 165								    dmn);
 166			if (ret)
 167				return ret;
 168		}
 169	}
 170
 171	return 0;
 172}
 173
 174static struct mlx5dr_ste *
 175dr_rule_find_ste_in_miss_list(struct list_head *miss_list, u8 *hw_ste)
 176{
 177	struct mlx5dr_ste *ste;
 178
 179	if (list_empty(miss_list))
 180		return NULL;
 181
 182	/* Check if hw_ste is present in the list */
 183	list_for_each_entry(ste, miss_list, miss_list_node) {
 184		if (mlx5dr_ste_equal_tag(mlx5dr_ste_get_hw_ste(ste), hw_ste))
 185			return ste;
 186	}
 187
 188	return NULL;
 189}
 190
 191static struct mlx5dr_ste *
 192dr_rule_rehash_handle_collision(struct mlx5dr_matcher *matcher,
 193				struct mlx5dr_matcher_rx_tx *nic_matcher,
 194				struct list_head *update_list,
 195				struct mlx5dr_ste *col_ste,
 196				u8 *hw_ste)
 197{
 198	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 199	struct mlx5dr_ste *new_ste;
 200	int ret;
 201
 202	new_ste = dr_rule_create_collision_htbl(matcher, nic_matcher, hw_ste);
 203	if (!new_ste)
 204		return NULL;
 205
 206	/* Update collision pointing STE */
 207	new_ste->htbl->pointing_ste = col_ste->htbl->pointing_ste;
 208
 209	/* In collision entry, all members share the same miss_list_head */
 210	new_ste->htbl->chunk->miss_list = mlx5dr_ste_get_miss_list(col_ste);
 211
 212	/* Update the previous from the list */
 213	ret = dr_rule_append_to_miss_list(dmn, nic_matcher->nic_tbl->nic_dmn->type,
 214					  new_ste, mlx5dr_ste_get_miss_list(col_ste),
 215					  update_list);
 216	if (ret) {
 217		mlx5dr_dbg(dmn, "Failed update dup entry\n");
 218		goto err_exit;
 219	}
 220
 221	return new_ste;
 222
 223err_exit:
 224	mlx5dr_ste_free(new_ste, matcher, nic_matcher);
 225	return NULL;
 226}
 227
 228static void dr_rule_rehash_copy_ste_ctrl(struct mlx5dr_matcher *matcher,
 229					 struct mlx5dr_matcher_rx_tx *nic_matcher,
 230					 struct mlx5dr_ste *cur_ste,
 231					 struct mlx5dr_ste *new_ste)
 232{
 233	new_ste->next_htbl = cur_ste->next_htbl;
 234	new_ste->ste_chain_location = cur_ste->ste_chain_location;
 235
 236	if (new_ste->next_htbl)
 237		new_ste->next_htbl->pointing_ste = new_ste;
 238
 239	/* We need to copy the refcount since this ste
 240	 * may have been traversed several times
 241	 */
 242	new_ste->refcount = cur_ste->refcount;
 243
 244	/* Link old STEs rule to the new ste */
 245	mlx5dr_rule_set_last_member(cur_ste->rule_rx_tx, new_ste, false);
 246}
 247
 248static struct mlx5dr_ste *
 249dr_rule_rehash_copy_ste(struct mlx5dr_matcher *matcher,
 250			struct mlx5dr_matcher_rx_tx *nic_matcher,
 251			struct mlx5dr_ste *cur_ste,
 252			struct mlx5dr_ste_htbl *new_htbl,
 253			struct list_head *update_list)
 254{
 255	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 256	struct mlx5dr_ste_send_info *ste_info;
 257	bool use_update_list = false;
 258	u8 hw_ste[DR_STE_SIZE] = {};
 259	struct mlx5dr_ste *new_ste;
 260	int new_idx;
 261	u8 sb_idx;
 262
 263	/* Copy STE mask from the matcher */
 264	sb_idx = cur_ste->ste_chain_location - 1;
 265	mlx5dr_ste_set_bit_mask(hw_ste, nic_matcher->ste_builder[sb_idx].bit_mask);
 266
 267	/* Copy STE control and tag */
 268	memcpy(hw_ste, mlx5dr_ste_get_hw_ste(cur_ste), DR_STE_SIZE_REDUCED);
 269	dr_rule_set_last_ste_miss_addr(matcher, nic_matcher, hw_ste);
 270
 271	new_idx = mlx5dr_ste_calc_hash_index(hw_ste, new_htbl);
 272	new_ste = &new_htbl->chunk->ste_arr[new_idx];
 273
 274	if (mlx5dr_ste_is_not_used(new_ste)) {
 275		mlx5dr_htbl_get(new_htbl);
 276		list_add_tail(&new_ste->miss_list_node,
 277			      mlx5dr_ste_get_miss_list(new_ste));
 278	} else {
 279		new_ste = dr_rule_rehash_handle_collision(matcher,
 280							  nic_matcher,
 281							  update_list,
 282							  new_ste,
 283							  hw_ste);
 284		if (!new_ste) {
 285			mlx5dr_dbg(dmn, "Failed adding collision entry, index: %d\n",
 286				   new_idx);
 287			return NULL;
 288		}
 289		new_htbl->ctrl.num_of_collisions++;
 290		use_update_list = true;
 291	}
 292
 293	memcpy(mlx5dr_ste_get_hw_ste(new_ste), hw_ste, DR_STE_SIZE_REDUCED);
 294
 295	new_htbl->ctrl.num_of_valid_entries++;
 296
 297	if (use_update_list) {
 298		ste_info = mlx5dr_send_info_alloc(dmn,
 299						  nic_matcher->nic_tbl->nic_dmn->type);
 300		if (!ste_info)
 301			goto err_exit;
 302
 303		mlx5dr_send_fill_and_append_ste_send_info(new_ste, DR_STE_SIZE, 0,
 304							  hw_ste, ste_info,
 305							  update_list, true);
 306	}
 307
 308	dr_rule_rehash_copy_ste_ctrl(matcher, nic_matcher, cur_ste, new_ste);
 309
 310	return new_ste;
 311
 312err_exit:
 313	mlx5dr_ste_free(new_ste, matcher, nic_matcher);
 314	return NULL;
 315}
 316
 317static int dr_rule_rehash_copy_miss_list(struct mlx5dr_matcher *matcher,
 318					 struct mlx5dr_matcher_rx_tx *nic_matcher,
 319					 struct list_head *cur_miss_list,
 320					 struct mlx5dr_ste_htbl *new_htbl,
 321					 struct list_head *update_list)
 322{
 323	struct mlx5dr_ste *tmp_ste, *cur_ste, *new_ste;
 324
 325	if (list_empty(cur_miss_list))
 326		return 0;
 327
 328	list_for_each_entry_safe(cur_ste, tmp_ste, cur_miss_list, miss_list_node) {
 329		new_ste = dr_rule_rehash_copy_ste(matcher,
 330						  nic_matcher,
 331						  cur_ste,
 332						  new_htbl,
 333						  update_list);
 334		if (!new_ste)
 335			goto err_insert;
 336
 337		list_del(&cur_ste->miss_list_node);
 338		mlx5dr_htbl_put(cur_ste->htbl);
 339	}
 340	return 0;
 341
 342err_insert:
 343	mlx5dr_err(matcher->tbl->dmn, "Fatal error during resize\n");
 344	WARN_ON(true);
 345	return -EINVAL;
 346}
 347
 348static int dr_rule_rehash_copy_htbl(struct mlx5dr_matcher *matcher,
 349				    struct mlx5dr_matcher_rx_tx *nic_matcher,
 350				    struct mlx5dr_ste_htbl *cur_htbl,
 351				    struct mlx5dr_ste_htbl *new_htbl,
 352				    struct list_head *update_list)
 353{
 354	struct mlx5dr_ste *cur_ste;
 355	int cur_entries;
 356	int err = 0;
 357	int i;
 358
 359	cur_entries = mlx5dr_icm_pool_chunk_size_to_entries(cur_htbl->chunk->size);
 360
 361	if (cur_entries < 1) {
 362		mlx5dr_dbg(matcher->tbl->dmn, "Invalid number of entries\n");
 363		return -EINVAL;
 364	}
 365
 366	for (i = 0; i < cur_entries; i++) {
 367		cur_ste = &cur_htbl->chunk->ste_arr[i];
 368		if (mlx5dr_ste_is_not_used(cur_ste)) /* Empty, nothing to copy */
 369			continue;
 370
 371		err = dr_rule_rehash_copy_miss_list(matcher,
 372						    nic_matcher,
 373						    mlx5dr_ste_get_miss_list(cur_ste),
 374						    new_htbl,
 375						    update_list);
 376		if (err)
 377			goto clean_copy;
 378
 379		/* In order to decrease the number of allocated ste_send_info
 380		 * structs, send the current table row now.
 381		 */
 382		err = dr_rule_send_update_list(update_list, matcher->tbl->dmn, false);
 383		if (err) {
 384			mlx5dr_dbg(matcher->tbl->dmn, "Failed updating table to HW\n");
 385			goto clean_copy;
 386		}
 387	}
 388
 389clean_copy:
 390	return err;
 391}
 392
 393static struct mlx5dr_ste_htbl *
 394dr_rule_rehash_htbl(struct mlx5dr_rule *rule,
 395		    struct mlx5dr_rule_rx_tx *nic_rule,
 396		    struct mlx5dr_ste_htbl *cur_htbl,
 397		    u8 ste_location,
 398		    struct list_head *update_list,
 399		    enum mlx5dr_icm_chunk_size new_size)
 400{
 401	struct mlx5dr_ste_send_info *del_ste_info, *tmp_ste_info;
 402	struct mlx5dr_matcher *matcher = rule->matcher;
 403	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 404	struct mlx5dr_matcher_rx_tx *nic_matcher;
 405	struct mlx5dr_ste_send_info *ste_info;
 406	struct mlx5dr_htbl_connect_info info;
 407	struct mlx5dr_domain_rx_tx *nic_dmn;
 408	u8 formatted_ste[DR_STE_SIZE] = {};
 409	LIST_HEAD(rehash_table_send_list);
 410	struct mlx5dr_ste *ste_to_update;
 411	struct mlx5dr_ste_htbl *new_htbl;
 412	int err;
 413
 414	nic_matcher = nic_rule->nic_matcher;
 415	nic_dmn = nic_matcher->nic_tbl->nic_dmn;
 416
 417	ste_info = mlx5dr_send_info_alloc(dmn,
 418					  nic_matcher->nic_tbl->nic_dmn->type);
 419	if (!ste_info)
 420		return NULL;
 421
 422	new_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool,
 423					 new_size,
 424					 cur_htbl->lu_type,
 425					 cur_htbl->byte_mask);
 426	if (!new_htbl) {
 427		mlx5dr_err(dmn, "Failed to allocate new hash table\n");
 428		goto free_ste_info;
 429	}
 430
 431	/* Write new table to HW */
 432	info.type = CONNECT_MISS;
 433	info.miss_icm_addr = mlx5dr_icm_pool_get_chunk_icm_addr(nic_matcher->e_anchor->chunk);
 434	mlx5dr_ste_set_formatted_ste(dmn->ste_ctx,
 435				     dmn->info.caps.gvmi,
 436				     nic_dmn->type,
 437				     new_htbl,
 438				     formatted_ste,
 439				     &info);
 440
 441	new_htbl->pointing_ste = cur_htbl->pointing_ste;
 442	new_htbl->pointing_ste->next_htbl = new_htbl;
 443	err = dr_rule_rehash_copy_htbl(matcher,
 444				       nic_matcher,
 445				       cur_htbl,
 446				       new_htbl,
 447				       &rehash_table_send_list);
 448	if (err)
 449		goto free_new_htbl;
 450
 451	if (mlx5dr_send_postsend_htbl(dmn, new_htbl, formatted_ste,
 452				      nic_matcher->ste_builder[ste_location - 1].bit_mask)) {
 453		mlx5dr_err(dmn, "Failed writing table to HW\n");
 454		goto free_new_htbl;
 455	}
 456
 457	/* Writing to the hw is done in regular order of rehash_table_send_list,
 458	 * in order to have the origin data written before the miss address of
 459	 * collision entries, if exists.
 460	 */
 461	if (dr_rule_send_update_list(&rehash_table_send_list, dmn, false)) {
 462		mlx5dr_err(dmn, "Failed updating table to HW\n");
 463		goto free_ste_list;
 464	}
 465
 466	/* Connect previous hash table to current */
 467	if (ste_location == 1) {
 468		/* The previous table is an anchor, anchors size is always one STE */
 469		struct mlx5dr_ste_htbl *prev_htbl = cur_htbl->pointing_ste->htbl;
 470
 471		/* On matcher s_anchor we keep an extra refcount */
 472		mlx5dr_htbl_get(new_htbl);
 473		mlx5dr_htbl_put(cur_htbl);
 474
 475		nic_matcher->s_htbl = new_htbl;
 476
 477		/* It is safe to operate dr_ste_set_hit_addr on the hw_ste here
 478		 * (48B len) which works only on first 32B
 479		 */
 480		mlx5dr_ste_set_hit_addr(dmn->ste_ctx,
 481					prev_htbl->chunk->hw_ste_arr,
 482					mlx5dr_icm_pool_get_chunk_icm_addr(new_htbl->chunk),
 483					mlx5dr_icm_pool_get_chunk_num_of_entries(new_htbl->chunk));
 484
 485		ste_to_update = &prev_htbl->chunk->ste_arr[0];
 486	} else {
 487		mlx5dr_ste_set_hit_addr_by_next_htbl(dmn->ste_ctx,
 488						     mlx5dr_ste_get_hw_ste(cur_htbl->pointing_ste),
 489						     new_htbl);
 490		ste_to_update = cur_htbl->pointing_ste;
 491	}
 492
 493	mlx5dr_send_fill_and_append_ste_send_info(ste_to_update, DR_STE_SIZE_CTRL,
 494						  0, mlx5dr_ste_get_hw_ste(ste_to_update),
 495						  ste_info, update_list, false);
 496
 497	return new_htbl;
 498
 499free_ste_list:
 500	/* Clean all ste_info's from the new table */
 501	list_for_each_entry_safe(del_ste_info, tmp_ste_info,
 502				 &rehash_table_send_list, send_list) {
 503		list_del(&del_ste_info->send_list);
 504		mlx5dr_send_info_free(del_ste_info);
 505	}
 506
 507free_new_htbl:
 508	mlx5dr_ste_htbl_free(new_htbl);
 509free_ste_info:
 510	mlx5dr_send_info_free(ste_info);
 511	mlx5dr_info(dmn, "Failed creating rehash table\n");
 512	return NULL;
 513}
 514
 515static struct mlx5dr_ste_htbl *dr_rule_rehash(struct mlx5dr_rule *rule,
 516					      struct mlx5dr_rule_rx_tx *nic_rule,
 517					      struct mlx5dr_ste_htbl *cur_htbl,
 518					      u8 ste_location,
 519					      struct list_head *update_list)
 520{
 521	struct mlx5dr_domain *dmn = rule->matcher->tbl->dmn;
 522	enum mlx5dr_icm_chunk_size new_size;
 523
 524	new_size = mlx5dr_icm_next_higher_chunk(cur_htbl->chunk->size);
 525	new_size = min_t(u32, new_size, dmn->info.max_log_sw_icm_sz);
 526
 527	if (new_size == cur_htbl->chunk->size)
 528		return NULL; /* Skip rehash, we already at the max size */
 529
 530	return dr_rule_rehash_htbl(rule, nic_rule, cur_htbl, ste_location,
 531				   update_list, new_size);
 532}
 533
 534static struct mlx5dr_ste *
 535dr_rule_handle_collision(struct mlx5dr_matcher *matcher,
 536			 struct mlx5dr_matcher_rx_tx *nic_matcher,
 537			 struct mlx5dr_ste *ste,
 538			 u8 *hw_ste,
 539			 struct list_head *miss_list,
 540			 struct list_head *send_list)
 541{
 542	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 543	struct mlx5dr_ste_send_info *ste_info;
 544	struct mlx5dr_ste *new_ste;
 545
 546	ste_info = mlx5dr_send_info_alloc(dmn,
 547					  nic_matcher->nic_tbl->nic_dmn->type);
 548	if (!ste_info)
 549		return NULL;
 550
 551	new_ste = dr_rule_create_collision_entry(matcher, nic_matcher, hw_ste, ste);
 552	if (!new_ste)
 553		goto free_send_info;
 554
 555	if (dr_rule_append_to_miss_list(dmn, nic_matcher->nic_tbl->nic_dmn->type,
 556					new_ste, miss_list, send_list)) {
 557		mlx5dr_dbg(dmn, "Failed to update prev miss_list\n");
 558		goto err_exit;
 559	}
 560
 561	mlx5dr_send_fill_and_append_ste_send_info(new_ste, DR_STE_SIZE, 0, hw_ste,
 562						  ste_info, send_list, false);
 563
 564	ste->htbl->ctrl.num_of_collisions++;
 565	ste->htbl->ctrl.num_of_valid_entries++;
 566
 567	return new_ste;
 568
 569err_exit:
 570	mlx5dr_ste_free(new_ste, matcher, nic_matcher);
 571free_send_info:
 572	mlx5dr_send_info_free(ste_info);
 573	return NULL;
 574}
 575
 576static void dr_rule_remove_action_members(struct mlx5dr_rule *rule)
 577{
 578	struct mlx5dr_rule_action_member *action_mem;
 579	struct mlx5dr_rule_action_member *tmp;
 580
 581	list_for_each_entry_safe(action_mem, tmp, &rule->rule_actions_list, list) {
 582		list_del(&action_mem->list);
 583		refcount_dec(&action_mem->action->refcount);
 584		kvfree(action_mem);
 585	}
 586}
 587
 588static int dr_rule_add_action_members(struct mlx5dr_rule *rule,
 589				      size_t num_actions,
 590				      struct mlx5dr_action *actions[])
 591{
 592	struct mlx5dr_rule_action_member *action_mem;
 593	int i;
 594
 595	for (i = 0; i < num_actions; i++) {
 596		action_mem = kvzalloc(sizeof(*action_mem), GFP_KERNEL);
 597		if (!action_mem)
 598			goto free_action_members;
 599
 600		action_mem->action = actions[i];
 601		INIT_LIST_HEAD(&action_mem->list);
 602		list_add_tail(&action_mem->list, &rule->rule_actions_list);
 603		refcount_inc(&action_mem->action->refcount);
 604	}
 605
 606	return 0;
 607
 608free_action_members:
 609	dr_rule_remove_action_members(rule);
 610	return -ENOMEM;
 611}
 612
 613void mlx5dr_rule_set_last_member(struct mlx5dr_rule_rx_tx *nic_rule,
 614				 struct mlx5dr_ste *ste,
 615				 bool force)
 616{
 617	/* Update rule member is usually done for the last STE or during rule
 618	 * creation to recover from mid-creation failure (for this peruse the
 619	 * force flag is used)
 620	 */
 621	if (ste->next_htbl && !force)
 622		return;
 623
 624	/* Update is required since each rule keeps track of its last STE */
 625	ste->rule_rx_tx = nic_rule;
 626	nic_rule->last_rule_ste = ste;
 627}
 628
 629static struct mlx5dr_ste *dr_rule_get_pointed_ste(struct mlx5dr_ste *curr_ste)
 630{
 631	struct mlx5dr_ste *first_ste;
 632
 633	first_ste = list_first_entry(mlx5dr_ste_get_miss_list(curr_ste),
 634				     struct mlx5dr_ste, miss_list_node);
 635
 636	return first_ste->htbl->pointing_ste;
 637}
 638
 639int mlx5dr_rule_get_reverse_rule_members(struct mlx5dr_ste **ste_arr,
 640					 struct mlx5dr_ste *curr_ste,
 641					 int *num_of_stes)
 642{
 643	bool first = false;
 644
 645	*num_of_stes = 0;
 646
 647	if (!curr_ste)
 648		return -ENOENT;
 649
 650	/* Iterate from last to first */
 651	while (!first) {
 652		first = curr_ste->ste_chain_location == 1;
 653		ste_arr[*num_of_stes] = curr_ste;
 654		*num_of_stes += 1;
 655		curr_ste = dr_rule_get_pointed_ste(curr_ste);
 656	}
 657
 658	return 0;
 659}
 660
 661static void dr_rule_clean_rule_members(struct mlx5dr_rule *rule,
 662				       struct mlx5dr_rule_rx_tx *nic_rule)
 663{
 664	struct mlx5dr_ste *ste_arr[DR_RULE_MAX_STES + DR_ACTION_MAX_STES];
 665	struct mlx5dr_ste *curr_ste = nic_rule->last_rule_ste;
 666	int i;
 667
 668	if (mlx5dr_rule_get_reverse_rule_members(ste_arr, curr_ste, &i))
 669		return;
 670
 671	while (i--)
 672		mlx5dr_ste_put(ste_arr[i], rule->matcher, nic_rule->nic_matcher);
 673}
 674
 675static u16 dr_get_bits_per_mask(u16 byte_mask)
 676{
 677	u16 bits = 0;
 678
 679	while (byte_mask) {
 680		byte_mask = byte_mask & (byte_mask - 1);
 681		bits++;
 682	}
 683
 684	return bits;
 685}
 686
 687static bool dr_rule_need_enlarge_hash(struct mlx5dr_ste_htbl *htbl,
 688				      struct mlx5dr_domain *dmn,
 689				      struct mlx5dr_domain_rx_tx *nic_dmn)
 690{
 691	struct mlx5dr_ste_htbl_ctrl *ctrl = &htbl->ctrl;
 692	int threshold;
 693
 694	if (dmn->info.max_log_sw_icm_sz <= htbl->chunk->size)
 695		return false;
 696
 697	if (!mlx5dr_ste_htbl_may_grow(htbl))
 698		return false;
 699
 700	if (dr_get_bits_per_mask(htbl->byte_mask) * BITS_PER_BYTE <= htbl->chunk->size)
 701		return false;
 702
 703	threshold = mlx5dr_ste_htbl_increase_threshold(htbl);
 704	if (ctrl->num_of_collisions >= threshold &&
 705	    (ctrl->num_of_valid_entries - ctrl->num_of_collisions) >= threshold)
 706		return true;
 707
 708	return false;
 709}
 710
 711static int dr_rule_handle_action_stes(struct mlx5dr_rule *rule,
 712				      struct mlx5dr_rule_rx_tx *nic_rule,
 713				      struct list_head *send_ste_list,
 714				      struct mlx5dr_ste *last_ste,
 715				      u8 *hw_ste_arr,
 716				      u32 new_hw_ste_arr_sz)
 717{
 718	struct mlx5dr_matcher_rx_tx *nic_matcher = nic_rule->nic_matcher;
 719	struct mlx5dr_ste_send_info *ste_info_arr[DR_ACTION_MAX_STES];
 720	u8 num_of_builders = nic_matcher->num_of_builders;
 721	struct mlx5dr_matcher *matcher = rule->matcher;
 722	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 723	u8 *curr_hw_ste, *prev_hw_ste;
 724	struct mlx5dr_ste *action_ste;
 725	int i, k;
 726
 727	/* Two cases:
 728	 * 1. num_of_builders is equal to new_hw_ste_arr_sz, the action in the ste
 729	 * 2. num_of_builders is less then new_hw_ste_arr_sz, new ste was added
 730	 *    to support the action.
 731	 */
 732
 733	for (i = num_of_builders, k = 0; i < new_hw_ste_arr_sz; i++, k++) {
 734		curr_hw_ste = hw_ste_arr + i * DR_STE_SIZE;
 735		prev_hw_ste = (i == 0) ? curr_hw_ste : hw_ste_arr + ((i - 1) * DR_STE_SIZE);
 736		action_ste = dr_rule_create_collision_htbl(matcher,
 737							   nic_matcher,
 738							   curr_hw_ste);
 739		if (!action_ste)
 740			return -ENOMEM;
 741
 742		mlx5dr_ste_get(action_ste);
 743
 744		action_ste->htbl->pointing_ste = last_ste;
 745		last_ste->next_htbl = action_ste->htbl;
 746		last_ste = action_ste;
 747
 748		/* While free ste we go over the miss list, so add this ste to the list */
 749		list_add_tail(&action_ste->miss_list_node,
 750			      mlx5dr_ste_get_miss_list(action_ste));
 751
 752		ste_info_arr[k] = mlx5dr_send_info_alloc(dmn,
 753							 nic_matcher->nic_tbl->nic_dmn->type);
 754		if (!ste_info_arr[k])
 755			goto err_exit;
 756
 757		/* Point current ste to the new action */
 758		mlx5dr_ste_set_hit_addr_by_next_htbl(dmn->ste_ctx,
 759						     prev_hw_ste,
 760						     action_ste->htbl);
 761
 762		mlx5dr_rule_set_last_member(nic_rule, action_ste, true);
 763
 764		mlx5dr_send_fill_and_append_ste_send_info(action_ste, DR_STE_SIZE, 0,
 765							  curr_hw_ste,
 766							  ste_info_arr[k],
 767							  send_ste_list, false);
 768	}
 769
 770	last_ste->next_htbl = NULL;
 771
 772	return 0;
 773
 774err_exit:
 775	mlx5dr_ste_put(action_ste, matcher, nic_matcher);
 776	return -ENOMEM;
 777}
 778
 779static int dr_rule_handle_empty_entry(struct mlx5dr_matcher *matcher,
 780				      struct mlx5dr_matcher_rx_tx *nic_matcher,
 781				      struct mlx5dr_ste_htbl *cur_htbl,
 782				      struct mlx5dr_ste *ste,
 783				      u8 ste_location,
 784				      u8 *hw_ste,
 785				      struct list_head *miss_list,
 786				      struct list_head *send_list)
 787{
 788	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 789	struct mlx5dr_ste_send_info *ste_info;
 790
 791	/* Take ref on table, only on first time this ste is used */
 792	mlx5dr_htbl_get(cur_htbl);
 793
 794	/* new entry -> new branch */
 795	list_add_tail(&ste->miss_list_node, miss_list);
 796
 797	dr_rule_set_last_ste_miss_addr(matcher, nic_matcher, hw_ste);
 798
 799	ste->ste_chain_location = ste_location;
 800
 801	ste_info = mlx5dr_send_info_alloc(dmn,
 802					  nic_matcher->nic_tbl->nic_dmn->type);
 803	if (!ste_info)
 804		goto clean_ste_setting;
 805
 806	if (mlx5dr_ste_create_next_htbl(matcher,
 807					nic_matcher,
 808					ste,
 809					hw_ste,
 810					DR_CHUNK_SIZE_1)) {
 811		mlx5dr_dbg(dmn, "Failed allocating table\n");
 812		goto clean_ste_info;
 813	}
 814
 815	cur_htbl->ctrl.num_of_valid_entries++;
 816
 817	mlx5dr_send_fill_and_append_ste_send_info(ste, DR_STE_SIZE, 0, hw_ste,
 818						  ste_info, send_list, false);
 819
 820	return 0;
 821
 822clean_ste_info:
 823	mlx5dr_send_info_free(ste_info);
 824clean_ste_setting:
 825	list_del_init(&ste->miss_list_node);
 826	mlx5dr_htbl_put(cur_htbl);
 827
 828	return -ENOMEM;
 829}
 830
 831static struct mlx5dr_ste *
 832dr_rule_handle_ste_branch(struct mlx5dr_rule *rule,
 833			  struct mlx5dr_rule_rx_tx *nic_rule,
 834			  struct list_head *send_ste_list,
 835			  struct mlx5dr_ste_htbl *cur_htbl,
 836			  u8 *hw_ste,
 837			  u8 ste_location,
 838			  struct mlx5dr_ste_htbl **put_htbl)
 839{
 840	struct mlx5dr_matcher *matcher = rule->matcher;
 841	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 842	struct mlx5dr_matcher_rx_tx *nic_matcher;
 843	struct mlx5dr_domain_rx_tx *nic_dmn;
 844	struct mlx5dr_ste_htbl *new_htbl;
 845	struct mlx5dr_ste *matched_ste;
 846	struct list_head *miss_list;
 847	bool skip_rehash = false;
 848	struct mlx5dr_ste *ste;
 849	int index;
 850
 851	nic_matcher = nic_rule->nic_matcher;
 852	nic_dmn = nic_matcher->nic_tbl->nic_dmn;
 853
 854again:
 855	index = mlx5dr_ste_calc_hash_index(hw_ste, cur_htbl);
 856	miss_list = &cur_htbl->chunk->miss_list[index];
 857	ste = &cur_htbl->chunk->ste_arr[index];
 858
 859	if (mlx5dr_ste_is_not_used(ste)) {
 860		if (dr_rule_handle_empty_entry(matcher, nic_matcher, cur_htbl,
 861					       ste, ste_location,
 862					       hw_ste, miss_list,
 863					       send_ste_list))
 864			return NULL;
 865	} else {
 866		/* Hash table index in use, check if this ste is in the miss list */
 867		matched_ste = dr_rule_find_ste_in_miss_list(miss_list, hw_ste);
 868		if (matched_ste) {
 869			/* If it is last STE in the chain, and has the same tag
 870			 * it means that all the previous stes are the same,
 871			 * if so, this rule is duplicated.
 872			 */
 873			if (!mlx5dr_ste_is_last_in_rule(nic_matcher, ste_location))
 874				return matched_ste;
 875
 876			mlx5dr_dbg(dmn, "Duplicate rule inserted\n");
 877		}
 878
 879		if (!skip_rehash && dr_rule_need_enlarge_hash(cur_htbl, dmn, nic_dmn)) {
 880			/* Hash table index in use, try to resize of the hash */
 881			skip_rehash = true;
 882
 883			/* Hold the table till we update.
 884			 * Release in dr_rule_create_rule()
 885			 */
 886			*put_htbl = cur_htbl;
 887			mlx5dr_htbl_get(cur_htbl);
 888
 889			new_htbl = dr_rule_rehash(rule, nic_rule, cur_htbl,
 890						  ste_location, send_ste_list);
 891			if (!new_htbl) {
 892				mlx5dr_err(dmn, "Failed creating rehash table, htbl-log_size: %d\n",
 893					   cur_htbl->chunk->size);
 894				mlx5dr_htbl_put(cur_htbl);
 895			} else {
 896				cur_htbl = new_htbl;
 897			}
 898			goto again;
 899		} else {
 900			/* Hash table index in use, add another collision (miss) */
 901			ste = dr_rule_handle_collision(matcher,
 902						       nic_matcher,
 903						       ste,
 904						       hw_ste,
 905						       miss_list,
 906						       send_ste_list);
 907			if (!ste) {
 908				mlx5dr_dbg(dmn, "failed adding collision entry, index: %d\n",
 909					   index);
 910				return NULL;
 911			}
 912		}
 913	}
 914	return ste;
 915}
 916
 917static bool dr_rule_cmp_value_to_mask(u8 *mask, u8 *value,
 918				      u32 s_idx, u32 e_idx)
 919{
 920	u32 i;
 921
 922	for (i = s_idx; i < e_idx; i++) {
 923		if (value[i] & ~mask[i]) {
 924			pr_info("Rule parameters contains a value not specified by mask\n");
 925			return false;
 926		}
 927	}
 928	return true;
 929}
 930
 931static bool dr_rule_verify(struct mlx5dr_matcher *matcher,
 932			   struct mlx5dr_match_parameters *value,
 933			   struct mlx5dr_match_param *param)
 934{
 935	u8 match_criteria = matcher->match_criteria;
 936	size_t value_size = value->match_sz;
 937	u8 *mask_p = (u8 *)&matcher->mask;
 938	u8 *param_p = (u8 *)param;
 939	u32 s_idx, e_idx;
 940
 941	if (!value_size ||
 942	    (value_size > DR_SZ_MATCH_PARAM || (value_size % sizeof(u32)))) {
 943		mlx5dr_err(matcher->tbl->dmn, "Rule parameters length is incorrect\n");
 944		return false;
 945	}
 946
 947	mlx5dr_ste_copy_param(matcher->match_criteria, param, value, false);
 948
 949	if (match_criteria & DR_MATCHER_CRITERIA_OUTER) {
 950		s_idx = offsetof(struct mlx5dr_match_param, outer);
 951		e_idx = min(s_idx + sizeof(param->outer), value_size);
 952
 953		if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) {
 954			mlx5dr_err(matcher->tbl->dmn, "Rule outer parameters contains a value not specified by mask\n");
 955			return false;
 956		}
 957	}
 958
 959	if (match_criteria & DR_MATCHER_CRITERIA_MISC) {
 960		s_idx = offsetof(struct mlx5dr_match_param, misc);
 961		e_idx = min(s_idx + sizeof(param->misc), value_size);
 962
 963		if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) {
 964			mlx5dr_err(matcher->tbl->dmn, "Rule misc parameters contains a value not specified by mask\n");
 965			return false;
 966		}
 967	}
 968
 969	if (match_criteria & DR_MATCHER_CRITERIA_INNER) {
 970		s_idx = offsetof(struct mlx5dr_match_param, inner);
 971		e_idx = min(s_idx + sizeof(param->inner), value_size);
 972
 973		if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) {
 974			mlx5dr_err(matcher->tbl->dmn, "Rule inner parameters contains a value not specified by mask\n");
 975			return false;
 976		}
 977	}
 978
 979	if (match_criteria & DR_MATCHER_CRITERIA_MISC2) {
 980		s_idx = offsetof(struct mlx5dr_match_param, misc2);
 981		e_idx = min(s_idx + sizeof(param->misc2), value_size);
 982
 983		if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) {
 984			mlx5dr_err(matcher->tbl->dmn, "Rule misc2 parameters contains a value not specified by mask\n");
 985			return false;
 986		}
 987	}
 988
 989	if (match_criteria & DR_MATCHER_CRITERIA_MISC3) {
 990		s_idx = offsetof(struct mlx5dr_match_param, misc3);
 991		e_idx = min(s_idx + sizeof(param->misc3), value_size);
 992
 993		if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) {
 994			mlx5dr_err(matcher->tbl->dmn, "Rule misc3 parameters contains a value not specified by mask\n");
 995			return false;
 996		}
 997	}
 998
 999	if (match_criteria & DR_MATCHER_CRITERIA_MISC4) {
1000		s_idx = offsetof(struct mlx5dr_match_param, misc4);
1001		e_idx = min(s_idx + sizeof(param->misc4), value_size);
1002
1003		if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) {
1004			mlx5dr_err(matcher->tbl->dmn,
1005				   "Rule misc4 parameters contains a value not specified by mask\n");
1006			return false;
1007		}
1008	}
1009
1010	if (match_criteria & DR_MATCHER_CRITERIA_MISC5) {
1011		s_idx = offsetof(struct mlx5dr_match_param, misc5);
1012		e_idx = min(s_idx + sizeof(param->misc5), value_size);
1013
1014		if (!dr_rule_cmp_value_to_mask(mask_p, param_p, s_idx, e_idx)) {
1015			mlx5dr_err(matcher->tbl->dmn, "Rule misc5 parameters contains a value not specified by mask\n");
1016			return false;
1017		}
1018	}
1019	return true;
1020}
1021
1022static int dr_rule_destroy_rule_nic(struct mlx5dr_rule *rule,
1023				    struct mlx5dr_rule_rx_tx *nic_rule)
1024{
1025	/* Check if this nic rule was actually created, or was it skipped
1026	 * and only the other type of the RX/TX nic rule was created.
1027	 */
1028	if (!nic_rule->last_rule_ste)
1029		return 0;
1030
1031	mlx5dr_domain_nic_lock(nic_rule->nic_matcher->nic_tbl->nic_dmn);
1032	dr_rule_clean_rule_members(rule, nic_rule);
1033
1034	nic_rule->nic_matcher->rules--;
1035	if (!nic_rule->nic_matcher->rules)
1036		mlx5dr_matcher_remove_from_tbl_nic(rule->matcher->tbl->dmn,
1037						   nic_rule->nic_matcher);
1038
1039	mlx5dr_domain_nic_unlock(nic_rule->nic_matcher->nic_tbl->nic_dmn);
1040
1041	return 0;
1042}
1043
1044static int dr_rule_destroy_rule_fdb(struct mlx5dr_rule *rule)
1045{
1046	dr_rule_destroy_rule_nic(rule, &rule->rx);
1047	dr_rule_destroy_rule_nic(rule, &rule->tx);
1048	return 0;
1049}
1050
1051static int dr_rule_destroy_rule(struct mlx5dr_rule *rule)
1052{
1053	struct mlx5dr_domain *dmn = rule->matcher->tbl->dmn;
1054
1055	mlx5dr_dbg_rule_del(rule);
1056
1057	switch (dmn->type) {
1058	case MLX5DR_DOMAIN_TYPE_NIC_RX:
1059		dr_rule_destroy_rule_nic(rule, &rule->rx);
1060		break;
1061	case MLX5DR_DOMAIN_TYPE_NIC_TX:
1062		dr_rule_destroy_rule_nic(rule, &rule->tx);
1063		break;
1064	case MLX5DR_DOMAIN_TYPE_FDB:
1065		dr_rule_destroy_rule_fdb(rule);
1066		break;
1067	default:
1068		return -EINVAL;
1069	}
1070
1071	dr_rule_remove_action_members(rule);
1072	kfree(rule);
1073	return 0;
1074}
1075
1076static enum mlx5dr_ipv dr_rule_get_ipv(struct mlx5dr_match_spec *spec)
1077{
1078	if (spec->ip_version == 6 || spec->ethertype == ETH_P_IPV6)
1079		return DR_RULE_IPV6;
1080
1081	return DR_RULE_IPV4;
1082}
1083
1084static bool dr_rule_skip(enum mlx5dr_domain_type domain,
1085			 enum mlx5dr_domain_nic_type nic_type,
1086			 struct mlx5dr_match_param *mask,
1087			 struct mlx5dr_match_param *value,
1088			 u32 flow_source)
1089{
1090	bool rx = nic_type == DR_DOMAIN_NIC_TYPE_RX;
1091
1092	if (domain != MLX5DR_DOMAIN_TYPE_FDB)
1093		return false;
1094
1095	if (mask->misc.source_port) {
1096		if (rx && value->misc.source_port != MLX5_VPORT_UPLINK)
1097			return true;
1098
1099		if (!rx && value->misc.source_port == MLX5_VPORT_UPLINK)
1100			return true;
1101	}
1102
1103	if (rx && flow_source == MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT)
1104		return true;
1105
1106	if (!rx && flow_source == MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK)
1107		return true;
1108
1109	return false;
1110}
1111
1112static int
1113dr_rule_create_rule_nic(struct mlx5dr_rule *rule,
1114			struct mlx5dr_rule_rx_tx *nic_rule,
1115			struct mlx5dr_match_param *param,
1116			size_t num_actions,
1117			struct mlx5dr_action *actions[])
1118{
1119	u8 hw_ste_arr_optimized[DR_RULE_MAX_STE_CHAIN_OPTIMIZED * DR_STE_SIZE] = {};
1120	struct mlx5dr_ste_send_info *ste_info, *tmp_ste_info;
1121	struct mlx5dr_matcher *matcher = rule->matcher;
1122	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
1123	struct mlx5dr_matcher_rx_tx *nic_matcher;
1124	struct mlx5dr_domain_rx_tx *nic_dmn;
1125	struct mlx5dr_ste_htbl *htbl = NULL;
1126	struct mlx5dr_ste_htbl *cur_htbl;
1127	struct mlx5dr_ste *ste = NULL;
1128	LIST_HEAD(send_ste_list);
1129	bool hw_ste_arr_is_opt;
1130	u8 *hw_ste_arr = NULL;
1131	u32 new_hw_ste_arr_sz;
1132	int ret, i;
1133
1134	nic_matcher = nic_rule->nic_matcher;
1135	nic_dmn = nic_matcher->nic_tbl->nic_dmn;
1136
1137	if (dr_rule_skip(dmn->type, nic_dmn->type, &matcher->mask, param,
1138			 rule->flow_source))
1139		return 0;
1140
1141	mlx5dr_domain_nic_lock(nic_dmn);
1142
1143	ret = mlx5dr_matcher_select_builders(matcher,
1144					     nic_matcher,
1145					     dr_rule_get_ipv(&param->outer),
1146					     dr_rule_get_ipv(&param->inner));
1147	if (ret)
1148		goto err_unlock;
1149
1150	hw_ste_arr_is_opt = nic_matcher->num_of_builders <= DR_RULE_MAX_STES_OPTIMIZED;
1151	if (likely(hw_ste_arr_is_opt)) {
1152		hw_ste_arr = hw_ste_arr_optimized;
1153	} else {
1154		hw_ste_arr = kzalloc((nic_matcher->num_of_builders + DR_ACTION_MAX_STES) *
1155				     DR_STE_SIZE, GFP_KERNEL);
1156
1157		if (!hw_ste_arr) {
1158			ret = -ENOMEM;
1159			goto err_unlock;
1160		}
1161	}
1162
1163	ret = mlx5dr_matcher_add_to_tbl_nic(dmn, nic_matcher);
1164	if (ret)
1165		goto free_hw_ste;
1166
1167	/* Set the tag values inside the ste array */
1168	ret = mlx5dr_ste_build_ste_arr(matcher, nic_matcher, param, hw_ste_arr);
1169	if (ret)
1170		goto remove_from_nic_tbl;
1171
1172	/* Set the actions values/addresses inside the ste array */
1173	ret = mlx5dr_actions_build_ste_arr(matcher, nic_matcher, actions,
1174					   num_actions, hw_ste_arr,
1175					   &new_hw_ste_arr_sz);
1176	if (ret)
1177		goto remove_from_nic_tbl;
1178
1179	cur_htbl = nic_matcher->s_htbl;
1180
1181	/* Go over the array of STEs, and build dr_ste accordingly.
1182	 * The loop is over only the builders which are equal or less to the
1183	 * number of stes, in case we have actions that lives in other stes.
1184	 */
1185	for (i = 0; i < nic_matcher->num_of_builders; i++) {
1186		/* Calculate CRC and keep new ste entry */
1187		u8 *cur_hw_ste_ent = hw_ste_arr + (i * DR_STE_SIZE);
1188
1189		ste = dr_rule_handle_ste_branch(rule,
1190						nic_rule,
1191						&send_ste_list,
1192						cur_htbl,
1193						cur_hw_ste_ent,
1194						i + 1,
1195						&htbl);
1196		if (!ste) {
1197			mlx5dr_err(dmn, "Failed creating next branch\n");
1198			ret = -ENOENT;
1199			goto free_rule;
1200		}
1201
1202		cur_htbl = ste->next_htbl;
1203
1204		mlx5dr_ste_get(ste);
1205		mlx5dr_rule_set_last_member(nic_rule, ste, true);
1206	}
1207
1208	/* Connect actions */
1209	ret = dr_rule_handle_action_stes(rule, nic_rule, &send_ste_list,
1210					 ste, hw_ste_arr, new_hw_ste_arr_sz);
1211	if (ret) {
1212		mlx5dr_dbg(dmn, "Failed apply actions\n");
1213		goto free_rule;
1214	}
1215	ret = dr_rule_send_update_list(&send_ste_list, dmn, true);
1216	if (ret) {
1217		mlx5dr_err(dmn, "Failed sending ste!\n");
1218		goto free_rule;
1219	}
1220
1221	if (htbl)
1222		mlx5dr_htbl_put(htbl);
1223
1224	nic_matcher->rules++;
1225
1226	mlx5dr_domain_nic_unlock(nic_dmn);
1227
1228	if (unlikely(!hw_ste_arr_is_opt))
1229		kfree(hw_ste_arr);
1230
1231	return 0;
1232
1233free_rule:
1234	dr_rule_clean_rule_members(rule, nic_rule);
1235	/* Clean all ste_info's */
1236	list_for_each_entry_safe(ste_info, tmp_ste_info, &send_ste_list, send_list) {
1237		list_del(&ste_info->send_list);
1238		mlx5dr_send_info_free(ste_info);
1239	}
1240
1241remove_from_nic_tbl:
1242	if (!nic_matcher->rules)
1243		mlx5dr_matcher_remove_from_tbl_nic(dmn, nic_matcher);
1244
1245free_hw_ste:
1246	if (!hw_ste_arr_is_opt)
1247		kfree(hw_ste_arr);
1248
1249err_unlock:
1250	mlx5dr_domain_nic_unlock(nic_dmn);
1251
1252	return ret;
1253}
1254
1255static int
1256dr_rule_create_rule_fdb(struct mlx5dr_rule *rule,
1257			struct mlx5dr_match_param *param,
1258			size_t num_actions,
1259			struct mlx5dr_action *actions[])
1260{
1261	struct mlx5dr_match_param copy_param = {};
1262	int ret;
1263
1264	/* Copy match_param since they will be consumed during the first
1265	 * nic_rule insertion.
1266	 */
1267	memcpy(&copy_param, param, sizeof(struct mlx5dr_match_param));
1268
1269	ret = dr_rule_create_rule_nic(rule, &rule->rx, param,
1270				      num_actions, actions);
1271	if (ret)
1272		return ret;
1273
1274	ret = dr_rule_create_rule_nic(rule, &rule->tx, &copy_param,
1275				      num_actions, actions);
1276	if (ret)
1277		goto destroy_rule_nic_rx;
1278
1279	return 0;
1280
1281destroy_rule_nic_rx:
1282	dr_rule_destroy_rule_nic(rule, &rule->rx);
1283	return ret;
1284}
1285
1286static struct mlx5dr_rule *
1287dr_rule_create_rule(struct mlx5dr_matcher *matcher,
1288		    struct mlx5dr_match_parameters *value,
1289		    size_t num_actions,
1290		    struct mlx5dr_action *actions[],
1291		    u32 flow_source)
1292{
1293	struct mlx5dr_domain *dmn = matcher->tbl->dmn;
1294	struct mlx5dr_match_param param = {};
1295	struct mlx5dr_rule *rule;
1296	int ret;
1297
1298	if (!dr_rule_verify(matcher, value, &param))
1299		return NULL;
1300
1301	rule = kzalloc(sizeof(*rule), GFP_KERNEL);
1302	if (!rule)
1303		return NULL;
1304
1305	rule->matcher = matcher;
1306	rule->flow_source = flow_source;
1307	INIT_LIST_HEAD(&rule->rule_actions_list);
1308
1309	ret = dr_rule_add_action_members(rule, num_actions, actions);
1310	if (ret)
1311		goto free_rule;
1312
1313	switch (dmn->type) {
1314	case MLX5DR_DOMAIN_TYPE_NIC_RX:
1315		rule->rx.nic_matcher = &matcher->rx;
1316		ret = dr_rule_create_rule_nic(rule, &rule->rx, &param,
1317					      num_actions, actions);
1318		break;
1319	case MLX5DR_DOMAIN_TYPE_NIC_TX:
1320		rule->tx.nic_matcher = &matcher->tx;
1321		ret = dr_rule_create_rule_nic(rule, &rule->tx, &param,
1322					      num_actions, actions);
1323		break;
1324	case MLX5DR_DOMAIN_TYPE_FDB:
1325		rule->rx.nic_matcher = &matcher->rx;
1326		rule->tx.nic_matcher = &matcher->tx;
1327		ret = dr_rule_create_rule_fdb(rule, &param,
1328					      num_actions, actions);
1329		break;
1330	default:
1331		ret = -EINVAL;
1332		break;
1333	}
1334
1335	if (ret)
1336		goto remove_action_members;
1337
1338	INIT_LIST_HEAD(&rule->dbg_node);
1339	mlx5dr_dbg_rule_add(rule);
1340	return rule;
1341
1342remove_action_members:
1343	dr_rule_remove_action_members(rule);
1344free_rule:
1345	kfree(rule);
1346	mlx5dr_err(dmn, "Failed creating rule\n");
1347	return NULL;
1348}
1349
1350struct mlx5dr_rule *mlx5dr_rule_create(struct mlx5dr_matcher *matcher,
1351				       struct mlx5dr_match_parameters *value,
1352				       size_t num_actions,
1353				       struct mlx5dr_action *actions[],
1354				       u32 flow_source)
1355{
1356	struct mlx5dr_rule *rule;
1357
1358	refcount_inc(&matcher->refcount);
1359
1360	rule = dr_rule_create_rule(matcher, value, num_actions, actions, flow_source);
1361	if (!rule)
1362		refcount_dec(&matcher->refcount);
1363
1364	return rule;
1365}
1366
1367int mlx5dr_rule_destroy(struct mlx5dr_rule *rule)
1368{
1369	struct mlx5dr_matcher *matcher = rule->matcher;
1370	int ret;
1371
1372	ret = dr_rule_destroy_rule(rule);
1373	if (!ret)
1374		refcount_dec(&matcher->refcount);
1375
1376	return ret;
1377}