matcher.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2024 NVIDIA Corporation & Affiliates */
   3
   4#include "internal.h"
   5
   6static bool hws_matcher_requires_col_tbl(u8 log_num_of_rules)
   7{
   8	/* Collision table concatenation is done only for large rule tables */
   9	return log_num_of_rules > MLX5HWS_MATCHER_ASSURED_RULES_TH;
  10}
  11
  12static u8 hws_matcher_rules_to_tbl_depth(u8 log_num_of_rules)
  13{
  14	if (hws_matcher_requires_col_tbl(log_num_of_rules))
  15		return MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH;
  16
  17	/* For small rule tables we use a single deep table to assure insertion */
  18	return min(log_num_of_rules, MLX5HWS_MATCHER_ASSURED_COL_TBL_DEPTH);
  19}
  20
  21static void hws_matcher_destroy_end_ft(struct mlx5hws_matcher *matcher)
  22{
  23	mlx5hws_table_destroy_default_ft(matcher->tbl, matcher->end_ft_id);
  24}
  25
  26int mlx5hws_matcher_update_end_ft_isolated(struct mlx5hws_table *tbl,
  27					   u32 miss_ft_id)
  28{
  29	struct mlx5hws_matcher *tmp_matcher;
  30
  31	if (list_empty(&tbl->matchers_list))
  32		return -EINVAL;
  33
  34	/* Update isolated_matcher_end_ft_id attribute for all
  35	 * the matchers in isolated table.
  36	 */
  37	list_for_each_entry(tmp_matcher, &tbl->matchers_list, list_node)
  38		tmp_matcher->attr.isolated_matcher_end_ft_id = miss_ft_id;
  39
  40	tmp_matcher = list_last_entry(&tbl->matchers_list,
  41				      struct mlx5hws_matcher,
  42				      list_node);
  43
  44	return mlx5hws_table_ft_set_next_ft(tbl->ctx,
  45					    tmp_matcher->end_ft_id,
  46					    tbl->fw_ft_type,
  47					    miss_ft_id);
  48}
  49
  50static int hws_matcher_connect_end_ft_isolated(struct mlx5hws_matcher *matcher)
  51{
  52	struct mlx5hws_table *tbl = matcher->tbl;
  53	u32 end_ft_id;
  54	int ret;
  55
  56	/* Reset end_ft next RTCs */
  57	ret = mlx5hws_table_ft_set_next_rtc(tbl->ctx,
  58					    matcher->end_ft_id,
  59					    matcher->tbl->fw_ft_type,
  60					    0, 0);
  61	if (ret) {
  62		mlx5hws_err(tbl->ctx, "Isolated matcher: failed to reset FT's next RTCs\n");
  63		return ret;
  64	}
  65
  66	/* Connect isolated matcher's end_ft to the complex matcher's end FT */
  67	end_ft_id = matcher->attr.isolated_matcher_end_ft_id;
  68	ret = mlx5hws_table_ft_set_next_ft(tbl->ctx,
  69					   matcher->end_ft_id,
  70					   matcher->tbl->fw_ft_type,
  71					   end_ft_id);
  72
  73	if (ret) {
  74		mlx5hws_err(tbl->ctx, "Isolated matcher: failed to set FT's miss_ft_id\n");
  75		return ret;
  76	}
  77
  78	return 0;
  79}
  80
  81static int hws_matcher_create_end_ft_isolated(struct mlx5hws_matcher *matcher)
  82{
  83	struct mlx5hws_table *tbl = matcher->tbl;
  84	int ret;
  85
  86	ret = mlx5hws_table_create_default_ft(tbl->ctx->mdev,
  87					      tbl,
  88					      0,
  89					      &matcher->end_ft_id);
  90	if (ret) {
  91		mlx5hws_err(tbl->ctx, "Isolated matcher: failed to create end flow table\n");
  92		return ret;
  93	}
  94
  95	ret = hws_matcher_connect_end_ft_isolated(matcher);
  96	if (ret) {
  97		mlx5hws_err(tbl->ctx, "Isolated matcher: failed to connect end FT\n");
  98		goto destroy_default_ft;
  99	}
 100
 101	return 0;
 102
 103destroy_default_ft:
 104	mlx5hws_table_destroy_default_ft(tbl, matcher->end_ft_id);
 105	return ret;
 106}
 107
 108static int hws_matcher_create_end_ft(struct mlx5hws_matcher *matcher)
 109{
 110	struct mlx5hws_table *tbl = matcher->tbl;
 111	int ret;
 112
 113	if (mlx5hws_matcher_is_isolated(matcher))
 114		ret = hws_matcher_create_end_ft_isolated(matcher);
 115	else
 116		ret = mlx5hws_table_create_default_ft(tbl->ctx->mdev,
 117						      tbl,
 118						      0,
 119						      &matcher->end_ft_id);
 120
 121	if (ret) {
 122		mlx5hws_err(tbl->ctx, "Failed to create matcher end flow table\n");
 123		return ret;
 124	}
 125
 126	return 0;
 127}
 128
 129static int hws_matcher_connect_isolated_first(struct mlx5hws_matcher *matcher)
 130{
 131	struct mlx5hws_table *tbl = matcher->tbl;
 132	struct mlx5hws_context *ctx = tbl->ctx;
 133	int ret;
 134
 135	/* Isolated matcher's end_ft is already pointing to the end_ft
 136	 * of the complex matcher - it was set at creation of end_ft,
 137	 * so no need to connect it.
 138	 * We still need to connect the isolated table's start FT to
 139	 * this matcher's RTC.
 140	 */
 141	ret = mlx5hws_table_ft_set_next_rtc(ctx,
 142					    tbl->ft_id,
 143					    tbl->fw_ft_type,
 144					    matcher->match_ste.rtc_0_id,
 145					    matcher->match_ste.rtc_1_id);
 146	if (ret) {
 147		mlx5hws_err(ctx, "Isolated matcher: failed to connect start FT to match RTC\n");
 148		return ret;
 149	}
 150
 151	/* Reset table's FT default miss (drop refcount) */
 152	ret = mlx5hws_table_ft_set_default_next_ft(tbl, tbl->ft_id);
 153	if (ret) {
 154		mlx5hws_err(ctx, "Isolated matcher: failed to reset table ft default miss\n");
 155		return ret;
 156	}
 157
 158	list_add(&matcher->list_node, &tbl->matchers_list);
 159
 160	return ret;
 161}
 162
 163static int hws_matcher_connect_isolated_last(struct mlx5hws_matcher *matcher)
 164{
 165	struct mlx5hws_table *tbl = matcher->tbl;
 166	struct mlx5hws_context *ctx = tbl->ctx;
 167	struct mlx5hws_matcher *last;
 168	int ret;
 169
 170	last = list_last_entry(&tbl->matchers_list,
 171			       struct mlx5hws_matcher,
 172			       list_node);
 173
 174	/* New matcher's end_ft is already pointing to the end_ft of
 175	 * the complex matcher.
 176	 * Connect previous matcher's end_ft to this new matcher RTC.
 177	 */
 178	ret = mlx5hws_table_ft_set_next_rtc(ctx,
 179					    last->end_ft_id,
 180					    tbl->fw_ft_type,
 181					    matcher->match_ste.rtc_0_id,
 182					    matcher->match_ste.rtc_1_id);
 183	if (ret) {
 184		mlx5hws_err(ctx,
 185			    "Isolated matcher: failed to connect matcher end_ft to new match RTC\n");
 186		return ret;
 187	}
 188
 189	/* Reset prev matcher FT default miss (drop refcount) */
 190	ret = mlx5hws_table_ft_set_default_next_ft(tbl, last->end_ft_id);
 191	if (ret) {
 192		mlx5hws_err(ctx, "Isolated matcher: failed to reset matcher ft default miss\n");
 193		return ret;
 194	}
 195
 196	/* Insert after the last matcher */
 197	list_add(&matcher->list_node, &last->list_node);
 198
 199	return 0;
 200}
 201
 202static int hws_matcher_connect_isolated(struct mlx5hws_matcher *matcher)
 203{
 204	/* Isolated matcher is expected to be the only one in its table.
 205	 * However, it can have a collision matcher, and it can go through
 206	 * rehash process, in which case we will temporary have both old and
 207	 * new matchers in the isolated table.
 208	 * Check if this is the first matcher in the isolated table.
 209	 */
 210	if (list_empty(&matcher->tbl->matchers_list))
 211		return hws_matcher_connect_isolated_first(matcher);
 212
 213	/* If this wasn't the first matcher, then we have 3 possible cases:
 214	 *  - this is a collision matcher for the first matcher
 215	 *  - this is a new rehash dest matcher
 216	 *  - this is a collision matcher for the new rehash dest matcher
 217	 * The logic to add new matcher is the same for all these cases.
 218	 */
 219	return hws_matcher_connect_isolated_last(matcher);
 220}
 221
 222static int hws_matcher_connect(struct mlx5hws_matcher *matcher)
 223{
 224	struct mlx5hws_table *tbl = matcher->tbl;
 225	struct mlx5hws_context *ctx = tbl->ctx;
 226	struct mlx5hws_matcher *prev = NULL;
 227	struct mlx5hws_matcher *next = NULL;
 228	struct mlx5hws_matcher *tmp_matcher;
 229	int ret;
 230
 231	if (mlx5hws_matcher_is_isolated(matcher))
 232		return hws_matcher_connect_isolated(matcher);
 233
 234	/* Find location in matcher list */
 235	if (list_empty(&tbl->matchers_list)) {
 236		list_add(&matcher->list_node, &tbl->matchers_list);
 237		goto connect;
 238	}
 239
 240	list_for_each_entry(tmp_matcher, &tbl->matchers_list, list_node) {
 241		if (tmp_matcher->attr.priority > matcher->attr.priority) {
 242			next = tmp_matcher;
 243			break;
 244		}
 245		prev = tmp_matcher;
 246	}
 247
 248	if (next)
 249		/* insert before next */
 250		list_add_tail(&matcher->list_node, &next->list_node);
 251	else
 252		/* insert after prev */
 253		list_add(&matcher->list_node, &prev->list_node);
 254
 255connect:
 256	if (next) {
 257		/* Connect to next RTC */
 258		ret = mlx5hws_table_ft_set_next_rtc(ctx,
 259						    matcher->end_ft_id,
 260						    tbl->fw_ft_type,
 261						    next->match_ste.rtc_0_id,
 262						    next->match_ste.rtc_1_id);
 263		if (ret) {
 264			mlx5hws_err(ctx, "Failed to connect new matcher to next RTC\n");
 265			goto remove_from_list;
 266		}
 267	} else {
 268		/* Connect last matcher to next miss_tbl if exists */
 269		ret = mlx5hws_table_connect_to_miss_table(tbl, tbl->default_miss.miss_tbl);
 270		if (ret) {
 271			mlx5hws_err(ctx, "Failed connect new matcher to miss_tbl\n");
 272			goto remove_from_list;
 273		}
 274	}
 275
 276	/* Connect to previous FT */
 277	ret = mlx5hws_table_ft_set_next_rtc(ctx,
 278					    prev ? prev->end_ft_id : tbl->ft_id,
 279					    tbl->fw_ft_type,
 280					    matcher->match_ste.rtc_0_id,
 281					    matcher->match_ste.rtc_1_id);
 282	if (ret) {
 283		mlx5hws_err(ctx, "Failed to connect new matcher to previous FT\n");
 284		goto remove_from_list;
 285	}
 286
 287	/* Reset prev matcher FT default miss (drop refcount) */
 288	ret = mlx5hws_table_ft_set_default_next_ft(tbl, prev ? prev->end_ft_id : tbl->ft_id);
 289	if (ret) {
 290		mlx5hws_err(ctx, "Failed to reset matcher ft default miss\n");
 291		goto remove_from_list;
 292	}
 293
 294	if (!prev) {
 295		/* Update tables missing to current matcher in the table */
 296		ret = mlx5hws_table_update_connected_miss_tables(tbl);
 297		if (ret) {
 298			mlx5hws_err(ctx, "Fatal error, failed to update connected miss table\n");
 299			goto remove_from_list;
 300		}
 301	}
 302
 303	return 0;
 304
 305remove_from_list:
 306	list_del_init(&matcher->list_node);
 307	return ret;
 308}
 309
 310static int hws_matcher_disconnect_isolated(struct mlx5hws_matcher *matcher)
 311{
 312	struct mlx5hws_matcher *first, *last, *prev, *next;
 313	struct mlx5hws_table *tbl = matcher->tbl;
 314	struct mlx5hws_context *ctx = tbl->ctx;
 315	u32 end_ft_id;
 316	int ret;
 317
 318	first = list_first_entry(&tbl->matchers_list,
 319				 struct mlx5hws_matcher,
 320				 list_node);
 321	last = list_last_entry(&tbl->matchers_list,
 322			       struct mlx5hws_matcher,
 323			       list_node);
 324	prev = list_prev_entry(matcher, list_node);
 325	next = list_next_entry(matcher, list_node);
 326
 327	list_del_init(&matcher->list_node);
 328
 329	if (first == last) {
 330		/* This was the only matcher in the list.
 331		 * Reset isolated table FT next RTCs and connect it
 332		 * to the whole complex matcher end FT instead.
 333		 */
 334		ret = mlx5hws_table_ft_set_next_rtc(ctx,
 335						    tbl->ft_id,
 336						    tbl->fw_ft_type,
 337						    0, 0);
 338		if (ret) {
 339			mlx5hws_err(tbl->ctx, "Isolated matcher: failed to reset FT's next RTCs\n");
 340			return ret;
 341		}
 342
 343		end_ft_id = matcher->attr.isolated_matcher_end_ft_id;
 344		ret = mlx5hws_table_ft_set_next_ft(tbl->ctx,
 345						   tbl->ft_id,
 346						   tbl->fw_ft_type,
 347						   end_ft_id);
 348		if (ret) {
 349			mlx5hws_err(tbl->ctx, "Isolated matcher: failed to set FT's miss_ft_id\n");
 350			return ret;
 351		}
 352
 353		return 0;
 354	}
 355
 356	/* At this point we know that there are more matchers in the list */
 357
 358	if (matcher == first) {
 359		/* We've disconnected the first matcher.
 360		 * Now update isolated table default FT.
 361		 */
 362		if (!next)
 363			return -EINVAL;
 364		return mlx5hws_table_ft_set_next_rtc(ctx,
 365						     tbl->ft_id,
 366						     tbl->fw_ft_type,
 367						     next->match_ste.rtc_0_id,
 368						     next->match_ste.rtc_1_id);
 369	}
 370
 371	if (matcher == last) {
 372		/* If we've disconnected the last matcher - update prev
 373		 * matcher's end_ft to point to the complex matcher end_ft.
 374		 */
 375		if (!prev)
 376			return -EINVAL;
 377		return hws_matcher_connect_end_ft_isolated(prev);
 378	}
 379
 380	/* This wasn't the first or the last matcher, which means that it has
 381	 * both prev and next matchers. Note that this only happens if we're
 382	 * disconnecting collision matcher of the old matcher during rehash.
 383	 */
 384	if (!prev || !next ||
 385	    !(matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION))
 386		return -EINVAL;
 387
 388	/* Update prev end FT to point to next match RTC */
 389	return mlx5hws_table_ft_set_next_rtc(ctx,
 390					     prev->end_ft_id,
 391					     tbl->fw_ft_type,
 392					     next->match_ste.rtc_0_id,
 393					     next->match_ste.rtc_1_id);
 394}
 395
 396static int hws_matcher_disconnect(struct mlx5hws_matcher *matcher)
 397{
 398	struct mlx5hws_matcher *next = NULL, *prev = NULL;
 399	struct mlx5hws_table *tbl = matcher->tbl;
 400	u32 prev_ft_id = tbl->ft_id;
 401	int ret;
 402
 403	if (mlx5hws_matcher_is_isolated(matcher))
 404		return hws_matcher_disconnect_isolated(matcher);
 405
 406	if (!list_is_first(&matcher->list_node, &tbl->matchers_list)) {
 407		prev = list_prev_entry(matcher, list_node);
 408		prev_ft_id = prev->end_ft_id;
 409	}
 410
 411	if (!list_is_last(&matcher->list_node, &tbl->matchers_list))
 412		next = list_next_entry(matcher, list_node);
 413
 414	list_del_init(&matcher->list_node);
 415
 416	if (next) {
 417		/* Connect previous end FT to next RTC */
 418		ret = mlx5hws_table_ft_set_next_rtc(tbl->ctx,
 419						    prev_ft_id,
 420						    tbl->fw_ft_type,
 421						    next->match_ste.rtc_0_id,
 422						    next->match_ste.rtc_1_id);
 423		if (ret) {
 424			mlx5hws_err(tbl->ctx, "Fatal error, failed to disconnect matcher\n");
 425			return ret;
 426		}
 427	} else {
 428		ret = mlx5hws_table_connect_to_miss_table(tbl, tbl->default_miss.miss_tbl);
 429		if (ret) {
 430			mlx5hws_err(tbl->ctx, "Fatal error, failed to disconnect last matcher\n");
 431			return ret;
 432		}
 433	}
 434
 435	/* Removing first matcher, update connected miss tables if exists */
 436	if (prev_ft_id == tbl->ft_id) {
 437		ret = mlx5hws_table_update_connected_miss_tables(tbl);
 438		if (ret) {
 439			mlx5hws_err(tbl->ctx,
 440				    "Fatal error, failed to update connected miss table\n");
 441			return ret;
 442		}
 443	}
 444
 445	ret = mlx5hws_table_ft_set_default_next_ft(tbl, prev_ft_id);
 446	if (ret) {
 447		mlx5hws_err(tbl->ctx, "Fatal error, failed to restore matcher ft default miss\n");
 448		return ret;
 449	}
 450
 451	return 0;
 452}
 453
 454static void hws_matcher_set_rtc_attr_sz(struct mlx5hws_matcher *matcher,
 455					struct mlx5hws_cmd_rtc_create_attr *rtc_attr,
 456					bool is_mirror)
 457{
 458	enum mlx5hws_matcher_flow_src flow_src = matcher->attr.optimize_flow_src;
 459
 460	if ((flow_src == MLX5HWS_MATCHER_FLOW_SRC_VPORT && !is_mirror) ||
 461	    (flow_src == MLX5HWS_MATCHER_FLOW_SRC_WIRE && is_mirror)) {
 462		/* Optimize FDB RTC */
 463		rtc_attr->log_size = 0;
 464		rtc_attr->log_depth = 0;
 465	}
 466}
 467
 468static int hws_matcher_create_rtc(struct mlx5hws_matcher *matcher)
 469{
 470	struct mlx5hws_matcher_attr *attr = &matcher->attr;
 471	struct mlx5hws_cmd_rtc_create_attr rtc_attr = {0};
 472	struct mlx5hws_match_template *mt = matcher->mt;
 473	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 474	union mlx5hws_matcher_size *size_rx, *size_tx;
 475	struct mlx5hws_table *tbl = matcher->tbl;
 476	u32 obj_id;
 477	int ret;
 478
 479	size_rx = &attr->size[MLX5HWS_MATCHER_SIZE_TYPE_RX];
 480	size_tx = &attr->size[MLX5HWS_MATCHER_SIZE_TYPE_TX];
 481
 482	rtc_attr.log_size = size_rx->table.sz_row_log;
 483	rtc_attr.log_depth = size_rx->table.sz_col_log;
 484	rtc_attr.is_frst_jumbo = mlx5hws_matcher_mt_is_jumbo(mt);
 485	rtc_attr.is_scnd_range = 0;
 486	rtc_attr.miss_ft_id = matcher->end_ft_id;
 487
 488	if (attr->insert_mode == MLX5HWS_MATCHER_INSERT_BY_HASH) {
 489		/* The usual Hash Table */
 490		rtc_attr.update_index_mode =
 491			MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH;
 492
 493		/* The first mt is used since all share the same definer */
 494		rtc_attr.match_definer_0 = mlx5hws_definer_get_id(mt->definer);
 495	} else if (attr->insert_mode == MLX5HWS_MATCHER_INSERT_BY_INDEX) {
 496		rtc_attr.update_index_mode =
 497			MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET;
 498		rtc_attr.num_hash_definer = 1;
 499
 500		if (attr->distribute_mode ==
 501		    MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH) {
 502			/* Hash Split Table */
 503			rtc_attr.access_index_mode =
 504				MLX5_IFC_RTC_STE_ACCESS_MODE_BY_HASH;
 505			rtc_attr.match_definer_0 =
 506				mlx5hws_definer_get_id(mt->definer);
 507		} else if (attr->distribute_mode ==
 508			   MLX5HWS_MATCHER_DISTRIBUTE_BY_LINEAR) {
 509			/* Linear Lookup Table */
 510			rtc_attr.access_index_mode =
 511				MLX5_IFC_RTC_STE_ACCESS_MODE_LINEAR;
 512			rtc_attr.match_definer_0 =
 513				ctx->caps->linear_match_definer;
 514		}
 515	}
 516
 517	rtc_attr.pd = ctx->pd_num;
 518	rtc_attr.ste_base = matcher->match_ste.ste_0_base;
 519	rtc_attr.reparse_mode = mlx5hws_context_get_reparse_mode(ctx);
 520	rtc_attr.table_type = mlx5hws_table_get_res_fw_ft_type(tbl->type, false);
 521	hws_matcher_set_rtc_attr_sz(matcher, &rtc_attr, false);
 522
 523	/* STC is a single resource (obj_id), use any STC for the ID */
 524	obj_id = mlx5hws_pool_get_base_id(ctx->stc_pool);
 525	rtc_attr.stc_base = obj_id;
 526
 527	ret = mlx5hws_cmd_rtc_create(ctx->mdev, &rtc_attr,
 528				     &matcher->match_ste.rtc_0_id);
 529	if (ret) {
 530		mlx5hws_err(ctx, "Failed to create matcher RTC\n");
 531		return ret;
 532	}
 533
 534	if (tbl->type == MLX5HWS_TABLE_TYPE_FDB) {
 535		rtc_attr.log_size = size_tx->table.sz_row_log;
 536		rtc_attr.log_depth = size_tx->table.sz_col_log;
 537		rtc_attr.ste_base = matcher->match_ste.ste_1_base;
 538		rtc_attr.table_type = mlx5hws_table_get_res_fw_ft_type(tbl->type, true);
 539
 540		obj_id = mlx5hws_pool_get_base_mirror_id(ctx->stc_pool);
 541		rtc_attr.stc_base = obj_id;
 542		hws_matcher_set_rtc_attr_sz(matcher, &rtc_attr, true);
 543
 544		ret = mlx5hws_cmd_rtc_create(ctx->mdev, &rtc_attr,
 545					     &matcher->match_ste.rtc_1_id);
 546		if (ret) {
 547			mlx5hws_err(ctx, "Failed to create mirror matcher RTC\n");
 548			goto destroy_rtc_0;
 549		}
 550	}
 551
 552	return 0;
 553
 554destroy_rtc_0:
 555	mlx5hws_cmd_rtc_destroy(ctx->mdev, matcher->match_ste.rtc_0_id);
 556	return ret;
 557}
 558
 559static void hws_matcher_destroy_rtc(struct mlx5hws_matcher *matcher)
 560{
 561	struct mlx5_core_dev *mdev = matcher->tbl->ctx->mdev;
 562
 563	if (matcher->tbl->type == MLX5HWS_TABLE_TYPE_FDB)
 564		mlx5hws_cmd_rtc_destroy(mdev, matcher->match_ste.rtc_1_id);
 565
 566	mlx5hws_cmd_rtc_destroy(mdev, matcher->match_ste.rtc_0_id);
 567}
 568
 569static int
 570hws_matcher_check_attr_sz(struct mlx5hws_cmd_query_caps *caps,
 571			  struct mlx5hws_matcher *matcher)
 572{
 573	struct mlx5hws_matcher_attr *attr = &matcher->attr;
 574	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 575	union mlx5hws_matcher_size *size;
 576	int i;
 577
 578	for (i = 0; i < 2; i++) {
 579		size = &attr->size[i];
 580
 581		if (size->table.sz_col_log > caps->rtc_log_depth_max) {
 582			mlx5hws_err(ctx, "Matcher depth exceeds limit %d\n",
 583				    caps->rtc_log_depth_max);
 584			return -EOPNOTSUPP;
 585		}
 586
 587		if (size->table.sz_col_log + size->table.sz_row_log >
 588		    caps->ste_alloc_log_max) {
 589			mlx5hws_err(ctx,
 590				    "Total matcher size exceeds limit %d\n",
 591				    caps->ste_alloc_log_max);
 592			return -EOPNOTSUPP;
 593		}
 594
 595		if (size->table.sz_col_log + size->table.sz_row_log <
 596		    caps->ste_alloc_log_gran) {
 597			mlx5hws_err(ctx, "Total matcher size below limit %d\n",
 598				    caps->ste_alloc_log_gran);
 599			return -EOPNOTSUPP;
 600		}
 601	}
 602
 603	return 0;
 604}
 605
 606static int hws_matcher_check_and_process_at(struct mlx5hws_matcher *matcher,
 607					    struct mlx5hws_action_template *at)
 608{
 609	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 610	bool valid;
 611	int ret;
 612
 613	valid = mlx5hws_action_check_combo(ctx, at->action_type_arr, matcher->tbl->type);
 614	if (!valid) {
 615		mlx5hws_err(ctx, "Invalid combination in action template\n");
 616		return -EINVAL;
 617	}
 618
 619	/* Process action template to setters */
 620	ret = mlx5hws_action_template_process(at);
 621	if (ret) {
 622		mlx5hws_err(ctx, "Failed to process action template\n");
 623		return ret;
 624	}
 625
 626	return 0;
 627}
 628
 629static int hws_matcher_bind_at(struct mlx5hws_matcher *matcher)
 630{
 631	bool is_jumbo = mlx5hws_matcher_mt_is_jumbo(matcher->mt);
 632	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 633	u8 required_stes, max_stes;
 634	int i, ret;
 635
 636	if (matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION)
 637		return 0;
 638
 639	max_stes = 0;
 640	for (i = 0; i < matcher->num_of_at; i++) {
 641		struct mlx5hws_action_template *at = &matcher->at[i];
 642
 643		ret = hws_matcher_check_and_process_at(matcher, at);
 644		if (ret) {
 645			mlx5hws_err(ctx, "Invalid at %d", i);
 646			return ret;
 647		}
 648
 649		required_stes = at->num_of_action_stes - (!is_jumbo || at->only_term);
 650		max_stes = max(max_stes, required_stes);
 651
 652		/* Future: Optimize reparse */
 653	}
 654
 655	matcher->num_of_action_stes = max_stes;
 656
 657	return 0;
 658}
 659
 660static void hws_matcher_set_ip_version_match(struct mlx5hws_matcher *matcher)
 661{
 662	int i;
 663
 664	for (i = 0; i < matcher->mt->fc_sz; i++) {
 665		switch (matcher->mt->fc[i].fname) {
 666		case MLX5HWS_DEFINER_FNAME_ETH_TYPE_O:
 667			matcher->matches_outer_ethertype = 1;
 668			break;
 669		case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_O:
 670			matcher->matches_outer_ip_version = 1;
 671			break;
 672		case MLX5HWS_DEFINER_FNAME_ETH_TYPE_I:
 673			matcher->matches_inner_ethertype = 1;
 674			break;
 675		case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_I:
 676			matcher->matches_inner_ip_version = 1;
 677			break;
 678		default:
 679			break;
 680		}
 681	}
 682}
 683
 684static int hws_matcher_bind_mt(struct mlx5hws_matcher *matcher)
 685{
 686	struct mlx5hws_cmd_ste_create_attr ste_attr = {};
 687	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 688	union mlx5hws_matcher_size *size;
 689	int ret;
 690
 691	/* Calculate match, range and hash definers */
 692	if (!(matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION)) {
 693		ret = mlx5hws_definer_mt_init(ctx, matcher->mt);
 694		if (ret) {
 695			if (ret == -E2BIG)
 696				mlx5hws_err(ctx, "Failed to set matcher templates with match definers\n");
 697			return ret;
 698		}
 699	}
 700
 701	hws_matcher_set_ip_version_match(matcher);
 702
 703	/* Create an STE range each for RX and TX. */
 704	ste_attr.table_type = FS_FT_FDB_RX;
 705	size = &matcher->attr.size[MLX5HWS_MATCHER_SIZE_TYPE_RX];
 706	ste_attr.log_obj_range =
 707		matcher->attr.optimize_flow_src ==
 708			MLX5HWS_MATCHER_FLOW_SRC_VPORT ?
 709			0 : size->table.sz_col_log + size->table.sz_row_log;
 710
 711	ret = mlx5hws_cmd_ste_create(ctx->mdev, &ste_attr,
 712				     &matcher->match_ste.ste_0_base);
 713	if (ret) {
 714		mlx5hws_err(ctx, "Failed to allocate RX STE range (%d)\n", ret);
 715		goto uninit_match_definer;
 716	}
 717
 718	ste_attr.table_type = FS_FT_FDB_TX;
 719	size = &matcher->attr.size[MLX5HWS_MATCHER_SIZE_TYPE_TX];
 720	ste_attr.log_obj_range =
 721		matcher->attr.optimize_flow_src ==
 722			MLX5HWS_MATCHER_FLOW_SRC_WIRE ?
 723			0 : size->table.sz_col_log + size->table.sz_row_log;
 724
 725	ret = mlx5hws_cmd_ste_create(ctx->mdev, &ste_attr,
 726				     &matcher->match_ste.ste_1_base);
 727	if (ret) {
 728		mlx5hws_err(ctx, "Failed to allocate TX STE range (%d)\n", ret);
 729		goto destroy_rx_ste_range;
 730	}
 731
 732	return 0;
 733
 734destroy_rx_ste_range:
 735	mlx5hws_cmd_ste_destroy(ctx->mdev, matcher->match_ste.ste_0_base);
 736uninit_match_definer:
 737	if (!(matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION))
 738		mlx5hws_definer_mt_uninit(ctx, matcher->mt);
 739	return ret;
 740}
 741
 742static void hws_matcher_unbind_mt(struct mlx5hws_matcher *matcher)
 743{
 744	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 745
 746	mlx5hws_cmd_ste_destroy(ctx->mdev, matcher->match_ste.ste_1_base);
 747	mlx5hws_cmd_ste_destroy(ctx->mdev, matcher->match_ste.ste_0_base);
 748	if (!(matcher->flags & MLX5HWS_MATCHER_FLAGS_COLLISION))
 749		mlx5hws_definer_mt_uninit(ctx, matcher->mt);
 750}
 751
 752static int
 753hws_matcher_validate_insert_mode(struct mlx5hws_cmd_query_caps *caps,
 754				 struct mlx5hws_matcher *matcher)
 755{
 756	struct mlx5hws_matcher_attr *attr = &matcher->attr;
 757	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 758	union mlx5hws_matcher_size *size_rx, *size_tx;
 759
 760	size_rx = &matcher->attr.size[MLX5HWS_MATCHER_SIZE_TYPE_RX];
 761	size_tx = &matcher->attr.size[MLX5HWS_MATCHER_SIZE_TYPE_TX];
 762
 763	switch (attr->insert_mode) {
 764	case MLX5HWS_MATCHER_INSERT_BY_HASH:
 765		if (matcher->attr.distribute_mode != MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH) {
 766			mlx5hws_err(ctx, "Invalid matcher distribute mode\n");
 767			return -EOPNOTSUPP;
 768		}
 769		break;
 770
 771	case MLX5HWS_MATCHER_INSERT_BY_INDEX:
 772		if (size_rx->table.sz_col_log || size_tx->table.sz_col_log) {
 773			mlx5hws_err(ctx, "Matcher with INSERT_BY_INDEX supports only Nx1 table size\n");
 774			return -EOPNOTSUPP;
 775		}
 776
 777		if (attr->distribute_mode == MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH) {
 778			/* Hash Split Table */
 779			if (!caps->rtc_hash_split_table) {
 780				mlx5hws_err(ctx, "FW doesn't support insert by index and hash distribute\n");
 781				return -EOPNOTSUPP;
 782			}
 783		} else if (attr->distribute_mode == MLX5HWS_MATCHER_DISTRIBUTE_BY_LINEAR) {
 784			/* Linear Lookup Table */
 785			if (!caps->rtc_linear_lookup_table ||
 786			    !IS_BIT_SET(caps->access_index_mode,
 787					MLX5_IFC_RTC_STE_ACCESS_MODE_LINEAR)) {
 788				mlx5hws_err(ctx, "FW doesn't support insert by index and linear distribute\n");
 789				return -EOPNOTSUPP;
 790			}
 791
 792			if (size_rx->table.sz_row_log >
 793				MLX5_IFC_RTC_LINEAR_LOOKUP_TBL_LOG_MAX ||
 794			    size_tx->table.sz_row_log >
 795				MLX5_IFC_RTC_LINEAR_LOOKUP_TBL_LOG_MAX) {
 796				mlx5hws_err(ctx, "Matcher with linear distribute: rows exceed limit %d",
 797					    MLX5_IFC_RTC_LINEAR_LOOKUP_TBL_LOG_MAX);
 798				return -EOPNOTSUPP;
 799			}
 800		} else {
 801			mlx5hws_err(ctx, "Matcher has unsupported distribute mode\n");
 802			return -EOPNOTSUPP;
 803		}
 804		break;
 805
 806	default:
 807		mlx5hws_err(ctx, "Matcher has unsupported insert mode\n");
 808		return -EOPNOTSUPP;
 809	}
 810
 811	return 0;
 812}
 813
 814static int
 815hws_matcher_process_attr(struct mlx5hws_cmd_query_caps *caps,
 816			 struct mlx5hws_matcher *matcher)
 817{
 818	struct mlx5hws_matcher_attr *attr = &matcher->attr;
 819	union mlx5hws_matcher_size *size_rx, *size_tx;
 820
 821	size_rx = &attr->size[MLX5HWS_MATCHER_SIZE_TYPE_RX];
 822	size_tx = &attr->size[MLX5HWS_MATCHER_SIZE_TYPE_TX];
 823
 824	if (hws_matcher_validate_insert_mode(caps, matcher))
 825		return -EOPNOTSUPP;
 826
 827	if (matcher->tbl->type != MLX5HWS_TABLE_TYPE_FDB && attr->optimize_flow_src) {
 828		mlx5hws_err(matcher->tbl->ctx, "NIC domain doesn't support flow_src\n");
 829		return -EOPNOTSUPP;
 830	}
 831
 832	/* Convert number of rules to the required depth */
 833	if (attr->mode == MLX5HWS_MATCHER_RESOURCE_MODE_RULE &&
 834	    attr->insert_mode == MLX5HWS_MATCHER_INSERT_BY_HASH) {
 835		size_rx->table.sz_col_log =
 836			hws_matcher_rules_to_tbl_depth(size_rx->rule.num_log);
 837		size_tx->table.sz_col_log =
 838			hws_matcher_rules_to_tbl_depth(size_tx->rule.num_log);
 839	}
 840
 841	matcher->flags |= attr->resizable ? MLX5HWS_MATCHER_FLAGS_RESIZABLE : 0;
 842	matcher->flags |= attr->isolated_matcher_end_ft_id ?
 843			  MLX5HWS_MATCHER_FLAGS_ISOLATED : 0;
 844
 845	return hws_matcher_check_attr_sz(caps, matcher);
 846}
 847
 848static int hws_matcher_create_and_connect(struct mlx5hws_matcher *matcher)
 849{
 850	int ret;
 851
 852	/* Select and create the definers for current matcher */
 853	ret = hws_matcher_bind_mt(matcher);
 854	if (ret)
 855		return ret;
 856
 857	/* Calculate and verify action combination */
 858	ret = hws_matcher_bind_at(matcher);
 859	if (ret)
 860		goto unbind_mt;
 861
 862	/* Create matcher end flow table anchor */
 863	ret = hws_matcher_create_end_ft(matcher);
 864	if (ret)
 865		goto unbind_mt;
 866
 867	/* Allocate the RTC for the new matcher */
 868	ret = hws_matcher_create_rtc(matcher);
 869	if (ret)
 870		goto destroy_end_ft;
 871
 872	/* Connect the matcher to the matcher list */
 873	ret = hws_matcher_connect(matcher);
 874	if (ret)
 875		goto destroy_rtc;
 876
 877	return 0;
 878
 879destroy_rtc:
 880	hws_matcher_destroy_rtc(matcher);
 881destroy_end_ft:
 882	hws_matcher_destroy_end_ft(matcher);
 883unbind_mt:
 884	hws_matcher_unbind_mt(matcher);
 885	return ret;
 886}
 887
 888static void hws_matcher_destroy_and_disconnect(struct mlx5hws_matcher *matcher)
 889{
 890	hws_matcher_disconnect(matcher);
 891	hws_matcher_destroy_rtc(matcher);
 892	hws_matcher_destroy_end_ft(matcher);
 893	hws_matcher_unbind_mt(matcher);
 894}
 895
 896static int
 897hws_matcher_create_col_matcher(struct mlx5hws_matcher *matcher)
 898{
 899	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 900	union mlx5hws_matcher_size *size_rx, *size_tx;
 901	struct mlx5hws_matcher *col_matcher;
 902	int i, ret;
 903
 904	size_rx = &matcher->attr.size[MLX5HWS_MATCHER_SIZE_TYPE_RX];
 905	size_tx = &matcher->attr.size[MLX5HWS_MATCHER_SIZE_TYPE_TX];
 906
 907	if (matcher->attr.mode != MLX5HWS_MATCHER_RESOURCE_MODE_RULE ||
 908	    matcher->attr.insert_mode == MLX5HWS_MATCHER_INSERT_BY_INDEX)
 909		return 0;
 910
 911	if (!hws_matcher_requires_col_tbl(size_rx->rule.num_log) &&
 912	    !hws_matcher_requires_col_tbl(size_tx->rule.num_log))
 913		return 0;
 914
 915	col_matcher = kzalloc(sizeof(*matcher), GFP_KERNEL);
 916	if (!col_matcher)
 917		return -ENOMEM;
 918
 919	col_matcher->tbl = matcher->tbl;
 920	col_matcher->mt = matcher->mt;
 921	col_matcher->at = matcher->at;
 922	col_matcher->num_of_at = matcher->num_of_at;
 923	col_matcher->num_of_mt = matcher->num_of_mt;
 924	col_matcher->attr.priority = matcher->attr.priority;
 925	col_matcher->flags = matcher->flags;
 926	col_matcher->flags |= MLX5HWS_MATCHER_FLAGS_COLLISION;
 927	col_matcher->attr.mode = MLX5HWS_MATCHER_RESOURCE_MODE_HTABLE;
 928	col_matcher->attr.optimize_flow_src = matcher->attr.optimize_flow_src;
 929	for (i = 0; i < 2; i++) {
 930		union mlx5hws_matcher_size *dst = &col_matcher->attr.size[i];
 931		union mlx5hws_matcher_size *src = &matcher->attr.size[i];
 932
 933		dst->table.sz_row_log = src->rule.num_log;
 934		dst->table.sz_col_log = MLX5HWS_MATCHER_ASSURED_COL_TBL_DEPTH;
 935		if (dst->table.sz_row_log > MLX5HWS_MATCHER_ASSURED_ROW_RATIO)
 936			dst->table.sz_row_log -=
 937				MLX5HWS_MATCHER_ASSURED_ROW_RATIO;
 938	}
 939
 940	col_matcher->attr.max_num_of_at_attach = matcher->attr.max_num_of_at_attach;
 941	col_matcher->attr.isolated_matcher_end_ft_id =
 942		matcher->attr.isolated_matcher_end_ft_id;
 943
 944	ret = hws_matcher_process_attr(ctx->caps, col_matcher);
 945	if (ret)
 946		goto free_col_matcher;
 947
 948	ret = hws_matcher_create_and_connect(col_matcher);
 949	if (ret)
 950		goto free_col_matcher;
 951
 952	matcher->col_matcher = col_matcher;
 953
 954	return 0;
 955
 956free_col_matcher:
 957	kfree(col_matcher);
 958	mlx5hws_err(ctx, "Failed to create assured collision matcher\n");
 959	return ret;
 960}
 961
 962static void
 963hws_matcher_destroy_col_matcher(struct mlx5hws_matcher *matcher)
 964{
 965	if (matcher->attr.mode != MLX5HWS_MATCHER_RESOURCE_MODE_RULE ||
 966	    matcher->attr.insert_mode == MLX5HWS_MATCHER_INSERT_BY_INDEX)
 967		return;
 968
 969	if (matcher->col_matcher) {
 970		hws_matcher_destroy_and_disconnect(matcher->col_matcher);
 971		kfree(matcher->col_matcher);
 972	}
 973}
 974
 975static int hws_matcher_init(struct mlx5hws_matcher *matcher)
 976{
 977	struct mlx5hws_context *ctx = matcher->tbl->ctx;
 978	int ret;
 979
 980	mutex_lock(&ctx->ctrl_lock);
 981
 982	/* Allocate matcher resource and connect to the packet pipe */
 983	ret = hws_matcher_create_and_connect(matcher);
 984	if (ret)
 985		goto unlock_err;
 986
 987	/* Create additional matcher for collision handling */
 988	ret = hws_matcher_create_col_matcher(matcher);
 989	if (ret)
 990		goto destory_and_disconnect;
 991	mutex_unlock(&ctx->ctrl_lock);
 992
 993	return 0;
 994
 995destory_and_disconnect:
 996	hws_matcher_destroy_and_disconnect(matcher);
 997unlock_err:
 998	mutex_unlock(&ctx->ctrl_lock);
 999	return ret;
1000}
1001
1002static int hws_matcher_uninit(struct mlx5hws_matcher *matcher)
1003{
1004	struct mlx5hws_context *ctx = matcher->tbl->ctx;
1005
1006	mutex_lock(&ctx->ctrl_lock);
1007	hws_matcher_destroy_col_matcher(matcher);
1008	hws_matcher_destroy_and_disconnect(matcher);
1009	mutex_unlock(&ctx->ctrl_lock);
1010
1011	return 0;
1012}
1013
1014static int hws_matcher_grow_at_array(struct mlx5hws_matcher *matcher)
1015{
1016	void *p;
1017
1018	if (matcher->size_of_at_array >= MLX5HWS_MATCHER_MAX_AT)
1019		return -ENOMEM;
1020
1021	matcher->size_of_at_array *= 2;
1022	p = krealloc(matcher->at,
1023		     matcher->size_of_at_array * sizeof(*matcher->at),
1024		     __GFP_ZERO | GFP_KERNEL);
1025	if (!p) {
1026		matcher->size_of_at_array /= 2;
1027		return -ENOMEM;
1028	}
1029
1030	matcher->at = p;
1031
1032	return 0;
1033}
1034
1035int mlx5hws_matcher_attach_at(struct mlx5hws_matcher *matcher,
1036			      struct mlx5hws_action_template *at)
1037{
1038	bool is_jumbo = mlx5hws_matcher_mt_is_jumbo(matcher->mt);
1039	u32 required_stes;
1040	int ret;
1041
1042	if (unlikely(matcher->num_of_at >= matcher->size_of_at_array)) {
1043		ret = hws_matcher_grow_at_array(matcher);
1044		if (ret)
1045			return ret;
1046
1047		if (matcher->col_matcher) {
1048			ret = hws_matcher_grow_at_array(matcher->col_matcher);
1049			if (ret)
1050				return ret;
1051		}
1052	}
1053
1054	ret = hws_matcher_check_and_process_at(matcher, at);
1055	if (ret)
1056		return ret;
1057
1058	required_stes = at->num_of_action_stes - (!is_jumbo || at->only_term);
1059	if (matcher->num_of_action_stes < required_stes)
1060		matcher->num_of_action_stes = required_stes;
1061
1062	matcher->at[matcher->num_of_at] = *at;
1063	matcher->num_of_at += 1;
1064
1065	if (matcher->col_matcher)
1066		matcher->col_matcher->num_of_at = matcher->num_of_at;
1067
1068	return 0;
1069}
1070
1071static int
1072hws_matcher_set_templates(struct mlx5hws_matcher *matcher,
1073			  struct mlx5hws_match_template *mt[],
1074			  u8 num_of_mt,
1075			  struct mlx5hws_action_template *at[],
1076			  u8 num_of_at)
1077{
1078	struct mlx5hws_context *ctx = matcher->tbl->ctx;
1079	int ret = 0;
1080	int i;
1081
1082	if (!num_of_mt || !num_of_at) {
1083		mlx5hws_err(ctx, "Number of action/match template cannot be zero\n");
1084		return -EOPNOTSUPP;
1085	}
1086
1087	matcher->mt = kcalloc(num_of_mt, sizeof(*matcher->mt), GFP_KERNEL);
1088	if (!matcher->mt)
1089		return -ENOMEM;
1090
1091	matcher->size_of_at_array =
1092		num_of_at + matcher->attr.max_num_of_at_attach;
1093	matcher->at = kvcalloc(matcher->size_of_at_array, sizeof(*matcher->at),
1094			       GFP_KERNEL);
1095	if (!matcher->at) {
1096		mlx5hws_err(ctx, "Failed to allocate action template array\n");
1097		ret = -ENOMEM;
1098		goto free_mt;
1099	}
1100
1101	for (i = 0; i < num_of_mt; i++)
1102		matcher->mt[i] = *mt[i];
1103
1104	for (i = 0; i < num_of_at; i++)
1105		matcher->at[i] = *at[i];
1106
1107	matcher->num_of_mt = num_of_mt;
1108	matcher->num_of_at = num_of_at;
1109
1110	return 0;
1111
1112free_mt:
1113	kfree(matcher->mt);
1114	return ret;
1115}
1116
1117static void
1118hws_matcher_unset_templates(struct mlx5hws_matcher *matcher)
1119{
1120	kvfree(matcher->at);
1121	kfree(matcher->mt);
1122}
1123
1124struct mlx5hws_matcher *
1125mlx5hws_matcher_create(struct mlx5hws_table *tbl,
1126		       struct mlx5hws_match_template *mt[],
1127		       u8 num_of_mt,
1128		       struct mlx5hws_action_template *at[],
1129		       u8 num_of_at,
1130		       struct mlx5hws_matcher_attr *attr)
1131{
1132	struct mlx5hws_context *ctx = tbl->ctx;
1133	struct mlx5hws_matcher *matcher;
1134	int ret;
1135
1136	matcher = kzalloc(sizeof(*matcher), GFP_KERNEL);
1137	if (!matcher)
1138		return NULL;
1139
1140	matcher->tbl = tbl;
1141	matcher->attr = *attr;
1142
1143	ret = hws_matcher_process_attr(tbl->ctx->caps, matcher);
1144	if (ret)
1145		goto free_matcher;
1146
1147	ret = hws_matcher_set_templates(matcher, mt, num_of_mt, at, num_of_at);
1148	if (ret)
1149		goto free_matcher;
1150
1151	ret = hws_matcher_init(matcher);
1152	if (ret) {
1153		mlx5hws_err(ctx, "Failed to initialise matcher: %d\n", ret);
1154		goto unset_templates;
1155	}
1156
1157	return matcher;
1158
1159unset_templates:
1160	hws_matcher_unset_templates(matcher);
1161free_matcher:
1162	kfree(matcher);
1163	return NULL;
1164}
1165
1166int mlx5hws_matcher_destroy(struct mlx5hws_matcher *matcher)
1167{
1168	hws_matcher_uninit(matcher);
1169	hws_matcher_unset_templates(matcher);
1170	kfree(matcher);
1171	return 0;
1172}
1173
1174struct mlx5hws_match_template *
1175mlx5hws_match_template_create(struct mlx5hws_context *ctx,
1176			      u32 *match_param,
1177			      u32 match_param_sz,
1178			      u8 match_criteria_enable)
1179{
1180	struct mlx5hws_match_template *mt;
1181
1182	mt = kzalloc(sizeof(*mt), GFP_KERNEL);
1183	if (!mt)
1184		return NULL;
1185
1186	mt->match_param = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
1187	if (!mt->match_param)
1188		goto free_template;
1189
1190	memcpy(mt->match_param, match_param, match_param_sz);
1191	mt->match_criteria_enable = match_criteria_enable;
1192
1193	return mt;
1194
1195free_template:
1196	kfree(mt);
1197	return NULL;
1198}
1199
1200int mlx5hws_match_template_destroy(struct mlx5hws_match_template *mt)
1201{
1202	kfree(mt->match_param);
1203	kfree(mt);
1204	return 0;
1205}
1206
1207static int hws_matcher_resize_precheck(struct mlx5hws_matcher *src_matcher,
1208				       struct mlx5hws_matcher *dst_matcher)
1209{
1210	struct mlx5hws_context *ctx = src_matcher->tbl->ctx;
1211	int i;
1212
1213	if (src_matcher->tbl->type != dst_matcher->tbl->type) {
1214		mlx5hws_err(ctx, "Table type mismatch for src/dst matchers\n");
1215		return -EINVAL;
1216	}
1217
1218	if (!mlx5hws_matcher_is_resizable(src_matcher) ||
1219	    !mlx5hws_matcher_is_resizable(dst_matcher)) {
1220		mlx5hws_err(ctx, "Src/dst matcher is not resizable\n");
1221		return -EINVAL;
1222	}
1223
1224	if (mlx5hws_matcher_is_insert_by_idx(src_matcher) !=
1225	    mlx5hws_matcher_is_insert_by_idx(dst_matcher)) {
1226		mlx5hws_err(ctx, "Src/dst matchers insert mode mismatch\n");
1227		return -EINVAL;
1228	}
1229
1230	if (mlx5hws_matcher_is_in_resize(src_matcher) ||
1231	    mlx5hws_matcher_is_in_resize(dst_matcher)) {
1232		mlx5hws_err(ctx, "Src/dst matcher is already in resize\n");
1233		return -EINVAL;
1234	}
1235
1236	/* Compare match templates - make sure the definers are equivalent */
1237	if (src_matcher->num_of_mt != dst_matcher->num_of_mt) {
1238		mlx5hws_err(ctx, "Src/dst matcher match templates mismatch\n");
1239		return -EINVAL;
1240	}
1241
1242	if (src_matcher->num_of_action_stes > dst_matcher->num_of_action_stes) {
1243		mlx5hws_err(ctx, "Src/dst matcher max STEs mismatch\n");
1244		return -EINVAL;
1245	}
1246
1247	for (i = 0; i < src_matcher->num_of_mt; i++) {
1248		if (mlx5hws_definer_compare(src_matcher->mt[i].definer,
1249					    dst_matcher->mt[i].definer)) {
1250			mlx5hws_err(ctx, "Src/dst matcher definers mismatch\n");
1251			return -EINVAL;
1252		}
1253	}
1254
1255	return 0;
1256}
1257
1258int mlx5hws_matcher_resize_set_target(struct mlx5hws_matcher *src_matcher,
1259				      struct mlx5hws_matcher *dst_matcher)
1260{
1261	int ret = 0;
1262
1263	mutex_lock(&src_matcher->tbl->ctx->ctrl_lock);
1264
1265	ret = hws_matcher_resize_precheck(src_matcher, dst_matcher);
1266	if (ret)
1267		goto out;
1268
1269	src_matcher->resize_dst = dst_matcher;
1270
1271out:
1272	mutex_unlock(&src_matcher->tbl->ctx->ctrl_lock);
1273	return ret;
1274}
1275
1276int mlx5hws_matcher_resize_rule_move(struct mlx5hws_matcher *src_matcher,
1277				     struct mlx5hws_rule *rule,
1278				     struct mlx5hws_rule_attr *attr)
1279{
1280	struct mlx5hws_context *ctx = src_matcher->tbl->ctx;
1281
1282	if (unlikely(!mlx5hws_matcher_is_in_resize(src_matcher))) {
1283		mlx5hws_err(ctx, "Matcher is not resizable or not in resize\n");
1284		return -EINVAL;
1285	}
1286
1287	if (unlikely(src_matcher != rule->matcher)) {
1288		mlx5hws_err(ctx, "Rule doesn't belong to src matcher\n");
1289		return -EINVAL;
1290	}
1291
1292	return mlx5hws_rule_move_hws_add(rule, attr);
1293}