bwc_complex.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2024 NVIDIA Corporation & Affiliates */
   3
   4#include "internal.h"
   5
   6/* We chain submatchers by applying three rules on a subrule: modify header (to
   7 * set register C6), jump to table (to the next submatcher) and the mandatory
   8 * last rule.
   9 */
  10#define HWS_NUM_CHAIN_ACTIONS 3
  11
  12static const struct rhashtable_params hws_rules_hash_params = {
  13	.key_len = sizeof_field(struct mlx5hws_bwc_complex_subrule_data,
  14				match_tag),
  15	.key_offset =
  16		offsetof(struct mlx5hws_bwc_complex_subrule_data, match_tag),
  17	.head_offset =
  18		offsetof(struct mlx5hws_bwc_complex_subrule_data, hash_node),
  19	.automatic_shrinking = true, .min_size = 1,
  20};
  21
  22static bool
  23hws_match_params_exceeds_definer(struct mlx5hws_context *ctx,
  24				 u8 match_criteria_enable,
  25				 struct mlx5hws_match_parameters *mask,
  26				 bool allow_jumbo)
  27{
  28	struct mlx5hws_definer match_layout = {0};
  29	struct mlx5hws_match_template *mt;
  30	bool is_complex = false;
  31	int ret;
  32
  33	if (!match_criteria_enable)
  34		return false; /* empty matcher */
  35
  36	mt = mlx5hws_match_template_create(ctx,
  37					   mask->match_buf,
  38					   mask->match_sz,
  39					   match_criteria_enable);
  40	if (!mt) {
  41		mlx5hws_err(ctx, "Complex matcher: failed creating match template\n");
  42		return false;
  43	}
  44
  45	ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, allow_jumbo);
  46	if (ret) {
  47		/* The only case that we're interested in is E2BIG,
  48		 * which means that the match parameters need to be
  49		 * split into complex martcher.
  50		 * For all other cases (good or bad) - just return true
  51		 * and let the usual match creation path handle it,
  52		 * both for good and bad flows.
  53		 */
  54		if (ret == -E2BIG) {
  55			is_complex = true;
  56			mlx5hws_dbg(ctx, "Matcher definer layout: need complex matcher\n");
  57		} else {
  58			mlx5hws_err(ctx, "Failed to calculate matcher definer layout\n");
  59		}
  60	} else {
  61		kfree(mt->fc);
  62	}
  63
  64	mlx5hws_match_template_destroy(mt);
  65
  66	return is_complex;
  67}
  68
  69bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx,
  70					 u8 match_criteria_enable,
  71					 struct mlx5hws_match_parameters *mask)
  72{
  73	return hws_match_params_exceeds_definer(ctx, match_criteria_enable,
  74						mask, true);
  75}
  76
  77static int
  78hws_get_last_set_dword_idx(const struct mlx5hws_match_parameters *mask)
  79{
  80	int i;
  81
  82	for (i = mask->match_sz / 4 - 1; i >= 0; i--)
  83		if (mask->match_buf[i])
  84			return i;
  85
  86	return -1;
  87}
  88
  89static bool hws_match_mask_is_empty(const struct mlx5hws_match_parameters *mask)
  90{
  91	return hws_get_last_set_dword_idx(mask) == -1;
  92}
  93
  94static bool hws_dword_is_inner_ipaddr_off(int dword_off)
  95{
  96	/* IPv4 and IPv6 addresses share the same entry via a union, and the
  97	 * source and dest addresses are contiguous in the fte_match_param. So
  98	 * we need to check 8 words.
  99	 */
 100	static const int inner_ip_dword_off =
 101		__mlx5_dw_off(fte_match_param, inner_headers.src_ipv4_src_ipv6);
 102
 103	return dword_off >= inner_ip_dword_off &&
 104	       dword_off < inner_ip_dword_off + 8;
 105}
 106
 107static bool hws_dword_is_outer_ipaddr_off(int dword_off)
 108{
 109	static const int outer_ip_dword_off =
 110		__mlx5_dw_off(fte_match_param, outer_headers.src_ipv4_src_ipv6);
 111
 112	return dword_off >= outer_ip_dword_off &&
 113	       dword_off < outer_ip_dword_off + 8;
 114}
 115
 116static void hws_add_dword_to_mask(struct mlx5hws_match_parameters *mask,
 117				  const struct mlx5hws_match_parameters *orig,
 118				  int dword_idx, bool *added_inner_ipv,
 119				  bool *added_outer_ipv)
 120{
 121	mask->match_buf[dword_idx] |= orig->match_buf[dword_idx];
 122
 123	*added_inner_ipv = false;
 124	*added_outer_ipv = false;
 125
 126	/* Any IP address fragment must be accompanied by a match on IP version.
 127	 * Use the `added_ipv` variables to keep track if we added IP versions
 128	 * specifically for this dword, so that we can roll them back if the
 129	 * match params become too large to fit into a definer.
 130	 */
 131	if (hws_dword_is_inner_ipaddr_off(dword_idx) &&
 132	    !MLX5_GET(fte_match_param, mask->match_buf,
 133		      inner_headers.ip_version)) {
 134		MLX5_SET(fte_match_param, mask->match_buf,
 135			 inner_headers.ip_version, 0xf);
 136		*added_inner_ipv = true;
 137	}
 138	if (hws_dword_is_outer_ipaddr_off(dword_idx) &&
 139	    !MLX5_GET(fte_match_param, mask->match_buf,
 140		      outer_headers.ip_version)) {
 141		MLX5_SET(fte_match_param, mask->match_buf,
 142			 outer_headers.ip_version, 0xf);
 143		*added_outer_ipv = true;
 144	}
 145}
 146
 147static void hws_remove_dword_from_mask(struct mlx5hws_match_parameters *mask,
 148				       int dword_idx, bool added_inner_ipv,
 149				       bool added_outer_ipv)
 150{
 151	mask->match_buf[dword_idx] = 0;
 152	if (added_inner_ipv)
 153		MLX5_SET(fte_match_param, mask->match_buf,
 154			 inner_headers.ip_version, 0);
 155	if (added_outer_ipv)
 156		MLX5_SET(fte_match_param, mask->match_buf,
 157			 outer_headers.ip_version, 0);
 158}
 159
 160/* Avoid leaving a single lower dword in `mask` if there are others present in
 161 * `orig`. Splitting IPv6 addresses like this causes them to be interpreted as
 162 * IPv4.
 163 */
 164static void hws_avoid_ipv6_split_of(struct mlx5hws_match_parameters *orig,
 165				    struct mlx5hws_match_parameters *mask,
 166				    int off)
 167{
 168	/* Masks are allocated to a full fte_match_param, but it can't hurt to
 169	 * double check.
 170	 */
 171	if (orig->match_sz <= off + 3 || mask->match_sz <= off + 3)
 172		return;
 173
 174	/* Lower dword is not set, nothing to do. */
 175	if (!mask->match_buf[off + 3])
 176		return;
 177
 178	/* Higher dwords also present in `mask`, no ambiguity. */
 179	if (mask->match_buf[off] || mask->match_buf[off + 1] ||
 180	    mask->match_buf[off + 2])
 181		return;
 182
 183	/* There are no higher dwords in `orig`, i.e. we match on IPv4. */
 184	if (!orig->match_buf[off] && !orig->match_buf[off + 1] &&
 185	    !orig->match_buf[off + 2])
 186		return;
 187
 188	/* Put the lower dword back in `orig`. It is always safe to do this, the
 189	 * dword will just be picked up in the next submask.
 190	 */
 191	orig->match_buf[off + 3] = mask->match_buf[off + 3];
 192	mask->match_buf[off + 3] = 0;
 193}
 194
 195static void hws_avoid_ipv6_split(struct mlx5hws_match_parameters *orig,
 196				 struct mlx5hws_match_parameters *mask)
 197{
 198	hws_avoid_ipv6_split_of(orig, mask,
 199				__mlx5_dw_off(fte_match_param,
 200					      outer_headers.src_ipv4_src_ipv6));
 201	hws_avoid_ipv6_split_of(orig, mask,
 202				__mlx5_dw_off(fte_match_param,
 203					      outer_headers.dst_ipv4_dst_ipv6));
 204	hws_avoid_ipv6_split_of(orig, mask,
 205				__mlx5_dw_off(fte_match_param,
 206					      inner_headers.src_ipv4_src_ipv6));
 207	hws_avoid_ipv6_split_of(orig, mask,
 208				__mlx5_dw_off(fte_match_param,
 209					      inner_headers.dst_ipv4_dst_ipv6));
 210}
 211
 212/* Build a subset of the `orig` match parameters into `mask`. This subset is
 213 * guaranteed to fit in a single definer an as such is a candidate for being a
 214 * part of a complex matcher. Upon successful execution, the match params that
 215 * go into `mask` are cleared from `orig`.
 216 */
 217static int hws_get_simple_params(struct mlx5hws_context *ctx, u8 match_criteria,
 218				 struct mlx5hws_match_parameters *orig,
 219				 struct mlx5hws_match_parameters *mask)
 220{
 221	bool added_inner_ipv, added_outer_ipv;
 222	int dword_idx;
 223	u32 *backup;
 224	int ret;
 225
 226	dword_idx = hws_get_last_set_dword_idx(orig);
 227	/* Nothing to do, we consumed all of the match params before. */
 228	if (dword_idx == -1)
 229		return 0;
 230
 231	backup = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
 232	if (!backup)
 233		return -ENOMEM;
 234
 235	while (1) {
 236		dword_idx = hws_get_last_set_dword_idx(orig);
 237		/* Nothing to do, we consumed all of the original match params
 238		 * into this subset, which still fits into a single matcher.
 239		 */
 240		if (dword_idx == -1) {
 241			ret = 0;
 242			goto free_backup;
 243		}
 244
 245		memcpy(backup, mask->match_buf, mask->match_sz);
 246
 247		/* Try to add this dword to the current subset. */
 248		hws_add_dword_to_mask(mask, orig, dword_idx, &added_inner_ipv,
 249				      &added_outer_ipv);
 250
 251		if (hws_match_params_exceeds_definer(ctx, match_criteria, mask,
 252						     false)) {
 253			/* We just added a match param that makes the definer
 254			 * too large. Revert and return what we had before.
 255			 * Note that we can't just zero out the affected fields,
 256			 * because it's possible that the dword we're looking at
 257			 * wasn't zero before (e.g. it included auto-added
 258			 * matches in IP version. This is why we employ the
 259			 * rather cumbersome memcpy for backing up.
 260			 */
 261			memcpy(mask->match_buf, backup, mask->match_sz);
 262			/* Possible future improvement: We can't add any more
 263			 * dwords, but it may be possible to squeeze in
 264			 * individual bytes, as definers have special slots for
 265			 * those.
 266			 *
 267			 * For now, keep the code simple. This results in an
 268			 * extra submatcher in some cases, but it's good enough.
 269			 */
 270			ret = 0;
 271			break;
 272		}
 273
 274		/* The current subset of match params still fits in a single
 275		 * definer. Remove the dword from the original mask.
 276		 *
 277		 * Also remove any explicit match on IP version if we just
 278		 * included one here. We will still automatically add it to
 279		 * accompany any IP address fragment, but do not need to
 280		 * consider it by itself.
 281		 */
 282		hws_remove_dword_from_mask(orig, dword_idx, added_inner_ipv,
 283					   added_outer_ipv);
 284	}
 285
 286	/* Make sure we have not picked up a single lower dword of an IPv6
 287	 * address, as the firmware will erroneously treat it as an IPv4
 288	 * address.
 289	 */
 290	hws_avoid_ipv6_split(orig, mask);
 291
 292free_backup:
 293	kfree(backup);
 294
 295	return ret;
 296}
 297
 298static int
 299hws_bwc_matcher_split_mask(struct mlx5hws_context *ctx, u8 match_criteria,
 300			   const struct mlx5hws_match_parameters *mask,
 301			   struct mlx5hws_match_parameters *submasks,
 302			   int *num_submasks)
 303{
 304	struct mlx5hws_match_parameters mask_copy;
 305	int ret, i = 0;
 306
 307	mask_copy.match_sz = MLX5_ST_SZ_BYTES(fte_match_param);
 308	mask_copy.match_buf = kzalloc(mask_copy.match_sz, GFP_KERNEL);
 309	if (!mask_copy.match_buf)
 310		return -ENOMEM;
 311
 312	memcpy(mask_copy.match_buf, mask->match_buf, mask->match_sz);
 313
 314	while (!hws_match_mask_is_empty(&mask_copy)) {
 315		if (i >= MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS) {
 316			mlx5hws_err(ctx,
 317				    "Complex matcher: mask too large for %d matchers\n",
 318				    MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS);
 319			ret = -E2BIG;
 320			goto free_copy;
 321		}
 322		/* All but the first matcher need to match on register C6 to
 323		 * connect pieces of the complex rule together.
 324		 */
 325		if (i > 0) {
 326			MLX5_SET(fte_match_param, submasks[i].match_buf,
 327				 misc_parameters_2.metadata_reg_c_6, -1);
 328			match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;
 329		}
 330		ret = hws_get_simple_params(ctx, match_criteria, &mask_copy,
 331					    &submasks[i]);
 332		if (ret < 0)
 333			goto free_copy;
 334		i++;
 335	}
 336
 337	*num_submasks = i;
 338	ret = 0;
 339
 340free_copy:
 341	kfree(mask_copy.match_buf);
 342
 343	return ret;
 344}
 345
 346static struct mlx5hws_table *
 347hws_isolated_table_create(const struct mlx5hws_bwc_matcher *cmatcher)
 348{
 349	struct mlx5hws_bwc_complex_submatcher *first_subm;
 350	struct mlx5hws_cmd_ft_modify_attr ft_attr = {0};
 351	struct mlx5hws_table_attr tbl_attr = {0};
 352	struct mlx5hws_table *orig_tbl;
 353	struct mlx5hws_context *ctx;
 354	struct mlx5hws_table *tbl;
 355	int ret;
 356
 357	first_subm = &cmatcher->complex->submatchers[0];
 358	orig_tbl = first_subm->tbl;
 359	ctx = orig_tbl->ctx;
 360
 361	tbl_attr.type = orig_tbl->type;
 362	tbl_attr.level = orig_tbl->level;
 363	tbl = mlx5hws_table_create(ctx, &tbl_attr);
 364	if (!tbl)
 365		return ERR_PTR(-EINVAL);
 366
 367	/* Set the default miss of the isolated table to point
 368	 * to the end anchor of the original matcher.
 369	 */
 370	mlx5hws_cmd_set_attr_connect_miss_tbl(ctx, tbl->fw_ft_type,
 371					      tbl->type, &ft_attr);
 372	ft_attr.table_miss_id = first_subm->bwc_matcher->matcher->end_ft_id;
 373
 374	ret = mlx5hws_cmd_flow_table_modify(ctx->mdev, &ft_attr, tbl->ft_id);
 375	if (ret) {
 376		mlx5hws_err(ctx, "Complex matcher: failed to set isolated tbl default miss\n");
 377		goto destroy_tbl;
 378	}
 379
 380	return tbl;
 381
 382destroy_tbl:
 383	mlx5hws_table_destroy(tbl);
 384
 385	return ERR_PTR(ret);
 386}
 387
 388static int hws_submatcher_init_first(struct mlx5hws_bwc_matcher *cmatcher,
 389				     struct mlx5hws_table *table, u32 priority,
 390				     u8 match_criteria,
 391				     struct mlx5hws_match_parameters *mask)
 392{
 393	enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS];
 394	struct mlx5hws_bwc_complex_submatcher *subm;
 395	int ret;
 396
 397	subm = &cmatcher->complex->submatchers[0];
 398
 399	/* The first submatcher lives in the original table and does not have an
 400	 * associated jump to table action. It also points to the outer complex
 401	 * matcher.
 402	 */
 403	subm->tbl = table;
 404	subm->action_tbl = NULL;
 405	subm->bwc_matcher = cmatcher;
 406
 407	action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
 408	action_types[1] = MLX5HWS_ACTION_TYP_TBL;
 409	action_types[2] = MLX5HWS_ACTION_TYP_LAST;
 410
 411	ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl,
 412						priority, match_criteria, mask,
 413						action_types);
 414	if (ret)
 415		return ret;
 416
 417	subm->bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_COMPLEX_FIRST;
 418
 419	ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params);
 420	if (ret)
 421		goto destroy_matcher;
 422	mutex_init(&subm->hash_lock);
 423	ida_init(&subm->chain_ida);
 424
 425	return 0;
 426
 427destroy_matcher:
 428	mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
 429
 430	return ret;
 431}
 432
 433static int hws_submatcher_init(struct mlx5hws_bwc_matcher *cmatcher, int idx,
 434			       struct mlx5hws_table *table, u32 priority,
 435			       u8 match_criteria,
 436			       struct mlx5hws_match_parameters *mask)
 437{
 438	enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS];
 439	struct mlx5hws_bwc_complex_submatcher *subm;
 440	bool is_last;
 441	int ret;
 442
 443	if (!idx)
 444		return hws_submatcher_init_first(cmatcher, table, priority,
 445						 match_criteria, mask);
 446
 447	subm = &cmatcher->complex->submatchers[idx];
 448	is_last = idx == cmatcher->complex->num_submatchers - 1;
 449
 450	subm->tbl = hws_isolated_table_create(cmatcher);
 451	if (IS_ERR(subm->tbl))
 452		return PTR_ERR(subm->tbl);
 453
 454	subm->action_tbl =
 455		mlx5hws_action_create_dest_table(subm->tbl->ctx, subm->tbl,
 456						 MLX5HWS_ACTION_FLAG_HWS_FDB);
 457	if (!subm->action_tbl) {
 458		ret = -EINVAL;
 459		goto destroy_tbl;
 460	}
 461
 462	subm->bwc_matcher = kzalloc(sizeof(*subm->bwc_matcher), GFP_KERNEL);
 463	if (!subm->bwc_matcher) {
 464		ret = -ENOMEM;
 465		goto destroy_action;
 466	}
 467
 468	/* Every matcher other than the first also matches of register C6 to
 469	 * bind subrules together in the complex rule using the chain ids.
 470	 */
 471	match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;
 472
 473	action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
 474	action_types[1] = MLX5HWS_ACTION_TYP_TBL;
 475	action_types[2] = MLX5HWS_ACTION_TYP_LAST;
 476
 477	/* Every matcher other than the last sets register C6 and jumps to the
 478	 * next submatcher's table. The final submatcher will use the
 479	 * user-supplied actions and will attach an action template at rule
 480	 * insertion time.
 481	 */
 482	ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl,
 483						priority, match_criteria, mask,
 484						is_last ? NULL : action_types);
 485	if (ret)
 486		goto free_matcher;
 487
 488	subm->bwc_matcher->matcher_type =
 489		MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER;
 490
 491	ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params);
 492	if (ret)
 493		goto destroy_matcher;
 494	mutex_init(&subm->hash_lock);
 495	ida_init(&subm->chain_ida);
 496
 497	return 0;
 498
 499destroy_matcher:
 500	mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
 501free_matcher:
 502	kfree(subm->bwc_matcher);
 503destroy_action:
 504	mlx5hws_action_destroy(subm->action_tbl);
 505destroy_tbl:
 506	mlx5hws_table_destroy(subm->tbl);
 507
 508	return ret;
 509}
 510
 511static void hws_submatcher_destroy(struct mlx5hws_bwc_matcher *cmatcher,
 512				   int idx)
 513{
 514	struct mlx5hws_bwc_complex_submatcher *subm;
 515
 516	subm = &cmatcher->complex->submatchers[idx];
 517
 518	ida_destroy(&subm->chain_ida);
 519	mutex_destroy(&subm->hash_lock);
 520	rhashtable_destroy(&subm->rules_hash);
 521
 522	if (subm->bwc_matcher) {
 523		mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
 524		if (idx)
 525			kfree(subm->bwc_matcher);
 526	}
 527
 528	/* We own all of the isolated tables, but not the original one. */
 529	if (idx) {
 530		mlx5hws_action_destroy(subm->action_tbl);
 531		mlx5hws_table_destroy(subm->tbl);
 532	}
 533}
 534
 535static int
 536hws_complex_data_actions_init(struct mlx5hws_bwc_matcher_complex_data *cdata)
 537{
 538	struct mlx5hws_context *ctx = cdata->submatchers[0].tbl->ctx;
 539	u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
 540	struct mlx5hws_action_mh_pattern ptrn;
 541	int ret = 0;
 542
 543	/* Create modify header action to set REG_C_6 */
 544	MLX5_SET(set_action_in, modify_hdr_action,
 545		 action_type, MLX5_MODIFICATION_TYPE_SET);
 546	MLX5_SET(set_action_in, modify_hdr_action,
 547		 field, MLX5_MODI_META_REG_C_6);
 548	MLX5_SET(set_action_in, modify_hdr_action,
 549		 length, 0); /* zero means length of 32 */
 550	MLX5_SET(set_action_in, modify_hdr_action, offset, 0);
 551	MLX5_SET(set_action_in, modify_hdr_action, data, 0);
 552
 553	ptrn.data = (void *)modify_hdr_action;
 554	ptrn.sz = MLX5HWS_ACTION_DOUBLE_SIZE;
 555
 556	cdata->action_metadata =
 557		mlx5hws_action_create_modify_header(ctx, 1, &ptrn, 0,
 558						    MLX5HWS_ACTION_FLAG_HWS_FDB);
 559	if (!cdata->action_metadata) {
 560		mlx5hws_err(ctx, "Complex matcher: failed to create set reg C6 action\n");
 561		return -EINVAL;
 562	}
 563
 564	/* Create last action */
 565	cdata->action_last =
 566		mlx5hws_action_create_last(ctx, MLX5HWS_ACTION_FLAG_HWS_FDB);
 567	if (!cdata->action_last) {
 568		mlx5hws_err(ctx, "Complex matcher: failed to create last action\n");
 569		ret = -EINVAL;
 570		goto destroy_action_metadata;
 571	}
 572
 573	return 0;
 574
 575destroy_action_metadata:
 576	mlx5hws_action_destroy(cdata->action_metadata);
 577
 578	return ret;
 579}
 580
 581static void
 582hws_complex_data_actions_destroy(struct mlx5hws_bwc_matcher_complex_data *cdata)
 583{
 584	mlx5hws_action_destroy(cdata->action_last);
 585	mlx5hws_action_destroy(cdata->action_metadata);
 586}
 587
 588int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher,
 589				       struct mlx5hws_table *table,
 590				       u32 priority, u8 match_criteria_enable,
 591				       struct mlx5hws_match_parameters *mask)
 592{
 593	struct mlx5hws_match_parameters
 594		submasks[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
 595	struct mlx5hws_bwc_matcher_complex_data *cdata;
 596	struct mlx5hws_context *ctx = table->ctx;
 597	int num_submatchers;
 598	int i, ret;
 599
 600	for (i = 0; i < ARRAY_SIZE(submasks); i++) {
 601		submasks[i].match_sz = MLX5_ST_SZ_BYTES(fte_match_param);
 602		submasks[i].match_buf = kzalloc(submasks[i].match_sz,
 603						GFP_KERNEL);
 604		if (!submasks[i].match_buf) {
 605			ret = -ENOMEM;
 606			goto free_submasks;
 607		}
 608	}
 609
 610	ret = hws_bwc_matcher_split_mask(ctx, match_criteria_enable, mask,
 611					 submasks, &num_submatchers);
 612	if (ret)
 613		goto free_submasks;
 614
 615	cdata = kzalloc(sizeof(*cdata), GFP_KERNEL);
 616	if (!cdata) {
 617		ret = -ENOMEM;
 618		goto free_submasks;
 619	}
 620
 621	bwc_matcher->complex = cdata;
 622	cdata->num_submatchers = num_submatchers;
 623
 624	for (i = 0; i < num_submatchers; i++) {
 625		ret = hws_submatcher_init(bwc_matcher, i, table, priority,
 626					  match_criteria_enable, &submasks[i]);
 627		if (ret)
 628			goto destroy_submatchers;
 629	}
 630
 631	ret = hws_complex_data_actions_init(cdata);
 632	if (ret)
 633		goto destroy_submatchers;
 634
 635	ret = 0;
 636	goto free_submasks;
 637
 638destroy_submatchers:
 639	while (i--)
 640		hws_submatcher_destroy(bwc_matcher, i);
 641	kfree(cdata);
 642	bwc_matcher->complex = NULL;
 643
 644free_submasks:
 645	for (i = 0; i < ARRAY_SIZE(submasks); i++)
 646		kfree(submasks[i].match_buf);
 647
 648	return ret;
 649}
 650
 651void
 652mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher)
 653{
 654	int i;
 655
 656	hws_complex_data_actions_destroy(bwc_matcher->complex);
 657	for (i = 0; i < bwc_matcher->complex->num_submatchers; i++)
 658		hws_submatcher_destroy(bwc_matcher, i);
 659	kfree(bwc_matcher->complex);
 660	bwc_matcher->complex = NULL;
 661}
 662
 663static int
 664hws_complex_get_subrule_data(struct mlx5hws_bwc_rule *bwc_rule,
 665			     struct mlx5hws_bwc_complex_submatcher *subm,
 666			     u32 *match_params)
 667__must_hold(&subm->hash_lock)
 668{
 669	struct mlx5hws_bwc_matcher *bwc_matcher = subm->bwc_matcher;
 670	struct mlx5hws_bwc_complex_subrule_data *sr_data, *old_data;
 671	struct mlx5hws_match_template *mt;
 672	int ret;
 673
 674	sr_data = kzalloc(sizeof(*sr_data), GFP_KERNEL);
 675	if (!sr_data)
 676		return -ENOMEM;
 677
 678	ret = ida_alloc(&subm->chain_ida, GFP_KERNEL);
 679	if (ret < 0)
 680		goto free_sr_data;
 681	sr_data->chain_id = ret;
 682
 683	refcount_set(&sr_data->refcount, 1);
 684
 685	mt  = bwc_matcher->matcher->mt;
 686	mlx5hws_definer_create_tag(match_params, mt->fc, mt->fc_sz,
 687				   (u8 *)&sr_data->match_tag);
 688
 689	old_data = rhashtable_lookup_get_insert_fast(&subm->rules_hash,
 690						     &sr_data->hash_node,
 691						     hws_rules_hash_params);
 692	if (IS_ERR(old_data)) {
 693		ret = PTR_ERR(old_data);
 694		goto free_ida;
 695	}
 696
 697	if (old_data) {
 698		/* Rule with the same tag already exists - update refcount */
 699		refcount_inc(&old_data->refcount);
 700		/* Let the new rule use the same tag as the existing rule.
 701		 * Note that we don't have any indication for the rule creation
 702		 * process that a rule with similar matching params already
 703		 * exists - no harm done when this rule is be overwritten by
 704		 * the same STE.
 705		 * There's some performance advantage in skipping such cases,
 706		 * so this is left for future optimizations.
 707		 */
 708		bwc_rule->subrule_data = old_data;
 709		ret = 0;
 710		goto free_ida;
 711	}
 712
 713	bwc_rule->subrule_data = sr_data;
 714	return 0;
 715
 716free_ida:
 717	ida_free(&subm->chain_ida, sr_data->chain_id);
 718free_sr_data:
 719	kfree(sr_data);
 720
 721	return ret;
 722}
 723
 724static void
 725hws_complex_put_subrule_data(struct mlx5hws_bwc_rule *bwc_rule,
 726			     struct mlx5hws_bwc_complex_submatcher *subm,
 727			     bool *is_last_rule)
 728__must_hold(&subm->hash_lock)
 729{
 730	struct mlx5hws_bwc_complex_subrule_data *sr_data;
 731
 732	if (is_last_rule)
 733		*is_last_rule = false;
 734
 735	sr_data = bwc_rule->subrule_data;
 736	if (refcount_dec_and_test(&sr_data->refcount)) {
 737		rhashtable_remove_fast(&subm->rules_hash,
 738				       &sr_data->hash_node,
 739				       hws_rules_hash_params);
 740		ida_free(&subm->chain_ida, sr_data->chain_id);
 741		kfree(sr_data);
 742		if (is_last_rule)
 743			*is_last_rule = true;
 744	}
 745
 746	bwc_rule->subrule_data = NULL;
 747}
 748
 749static int hws_complex_subrule_create(struct mlx5hws_bwc_matcher *cmatcher,
 750				      struct mlx5hws_bwc_rule *subrule,
 751				      u32 *match_params, u32 flow_source,
 752				      int bwc_queue_idx, int subm_idx,
 753				      struct mlx5hws_rule_action *actions,
 754				      u32 *chain_id)
 755{
 756	struct mlx5hws_rule_action chain_actions[HWS_NUM_CHAIN_ACTIONS] = {0};
 757	u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
 758	struct mlx5hws_bwc_matcher_complex_data *cdata;
 759	struct mlx5hws_bwc_complex_submatcher *subm;
 760	int ret;
 761
 762	cdata = cmatcher->complex;
 763	subm = &cdata->submatchers[subm_idx];
 764
 765	mutex_lock(&subm->hash_lock);
 766
 767	ret = hws_complex_get_subrule_data(subrule, subm, match_params);
 768	if (ret)
 769		goto unlock;
 770
 771	*chain_id = subrule->subrule_data->chain_id;
 772
 773	if (!actions) {
 774		MLX5_SET(set_action_in, modify_hdr_action, data, *chain_id);
 775		chain_actions[0].action = cdata->action_metadata;
 776		chain_actions[0].modify_header.data = modify_hdr_action;
 777		chain_actions[1].action =
 778			cdata->submatchers[subm_idx + 1].action_tbl;
 779		chain_actions[2].action = cdata->action_last;
 780		actions = chain_actions;
 781	}
 782
 783	ret = mlx5hws_bwc_rule_create_simple(subrule, match_params, actions,
 784					     flow_source, bwc_queue_idx);
 785	if (ret)
 786		goto put_subrule_data;
 787
 788	ret = 0;
 789	goto unlock;
 790
 791put_subrule_data:
 792	hws_complex_put_subrule_data(subrule, subm, NULL);
 793unlock:
 794	mutex_unlock(&subm->hash_lock);
 795
 796	return ret;
 797}
 798
 799static int hws_complex_subrule_destroy(struct mlx5hws_bwc_rule *bwc_rule,
 800				       struct mlx5hws_bwc_matcher *cmatcher,
 801				       int subm_idx)
 802{
 803	struct mlx5hws_bwc_matcher_complex_data *cdata;
 804	struct mlx5hws_bwc_complex_submatcher *subm;
 805	struct mlx5hws_context *ctx;
 806	bool is_last_rule;
 807	int ret = 0;
 808
 809	cdata = cmatcher->complex;
 810	subm = &cdata->submatchers[subm_idx];
 811	ctx = subm->tbl->ctx;
 812
 813	mutex_lock(&subm->hash_lock);
 814
 815	hws_complex_put_subrule_data(bwc_rule, subm, &is_last_rule);
 816	bwc_rule->rule->skip_delete = !is_last_rule;
 817	ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule);
 818	if (unlikely(ret))
 819		mlx5hws_err(ctx,
 820			    "Complex rule: failed to delete subrule %d (%d)\n",
 821			    subm_idx, ret);
 822
 823	if (subm_idx)
 824		mlx5hws_bwc_rule_free(bwc_rule);
 825
 826	mutex_unlock(&subm->hash_lock);
 827
 828	return ret;
 829}
 830
 831int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule,
 832				    struct mlx5hws_match_parameters *params,
 833				    u32 flow_source,
 834				    struct mlx5hws_rule_action rule_actions[],
 835				    u16 bwc_queue_idx)
 836{
 837	struct mlx5hws_bwc_rule
 838		*subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
 839	struct mlx5hws_bwc_matcher *cmatcher = bwc_rule->bwc_matcher;
 840	struct mlx5hws_bwc_matcher_complex_data *cdata;
 841	struct mlx5hws_rule_action *subrule_actions;
 842	struct mlx5hws_bwc_complex_submatcher *subm;
 843	struct mlx5hws_bwc_rule *subrule;
 844	u32 *match_params;
 845	u32 chain_id;
 846	int i, ret;
 847
 848	cdata = cmatcher->complex;
 849	if (!cdata)
 850		return -EINVAL;
 851
 852	/* Duplicate user data because we will modify it to set register C6
 853	 * values. For the same reason, make sure that we allocate a full
 854	 * match_param even if the user gave us fewer bytes. We need to ensure
 855	 * there is space for the match on C6.
 856	 */
 857	match_params = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
 858	if (!match_params)
 859		return -ENOMEM;
 860
 861	memcpy(match_params, params->match_buf, params->match_sz);
 862
 863	ret = hws_complex_subrule_create(cmatcher, bwc_rule, match_params,
 864					 flow_source, bwc_queue_idx, 0,
 865					 NULL, &chain_id);
 866	if (ret)
 867		goto free_match_params;
 868	subrules[0] = bwc_rule;
 869
 870	for (i = 1; i < cdata->num_submatchers; i++) {
 871		subm = &cdata->submatchers[i];
 872		subrule = mlx5hws_bwc_rule_alloc(subm->bwc_matcher);
 873		if (!subrule) {
 874			ret = -ENOMEM;
 875			goto destroy_subrules;
 876		}
 877
 878		/* Match on the previous subrule's chain_id. This is how
 879		 * subrules are connected in steering.
 880		 */
 881		MLX5_SET(fte_match_param, match_params,
 882			 misc_parameters_2.metadata_reg_c_6, chain_id);
 883
 884		/* The last subrule uses the complex rule's user-specified
 885		 * actions. Everything else uses the chaining rules based on the
 886		 * next table and chain_id.
 887		 */
 888		subrule_actions =
 889			i == cdata->num_submatchers - 1 ? rule_actions : NULL;
 890
 891		ret = hws_complex_subrule_create(cmatcher, subrule,
 892						 match_params, flow_source,
 893						 bwc_queue_idx, i,
 894						 subrule_actions, &chain_id);
 895		if (ret) {
 896			mlx5hws_bwc_rule_free(subrule);
 897			goto destroy_subrules;
 898		}
 899
 900		subrules[i] = subrule;
 901	}
 902
 903	for (i = 0; i < cdata->num_submatchers - 1; i++)
 904		subrules[i]->next_subrule = subrules[i + 1];
 905
 906	kfree(match_params);
 907
 908	return 0;
 909
 910destroy_subrules:
 911	while (i--)
 912		hws_complex_subrule_destroy(subrules[i], cmatcher, i);
 913free_match_params:
 914	kfree(match_params);
 915
 916	return ret;
 917}
 918
 919int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule)
 920{
 921	struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
 922	struct mlx5hws_bwc_rule
 923		*subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
 924	struct mlx5hws_bwc_matcher_complex_data *cdata;
 925	int i, err, ret_val;
 926
 927	cdata = bwc_matcher->complex;
 928
 929	/* Construct a list of all the subrules we need to destroy. */
 930	subrules[0] = bwc_rule;
 931	for (i = 1; i < cdata->num_submatchers; i++)
 932		subrules[i] = subrules[i - 1]->next_subrule;
 933
 934	ret_val = 0;
 935	for (i = 0; i < cdata->num_submatchers; i++) {
 936		err = hws_complex_subrule_destroy(subrules[i], bwc_matcher, i);
 937		/* If something goes wrong, plow along to destroy all of the
 938		 * subrules but return an error upstack.
 939		 */
 940		if (unlikely(err))
 941			ret_val = err;
 942	}
 943
 944	return ret_val;
 945}
 946
 947static void
 948hws_bwc_matcher_init_move(struct mlx5hws_bwc_matcher *bwc_matcher)
 949{
 950	struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
 951	u16 bwc_queues = mlx5hws_bwc_queues(ctx);
 952	struct mlx5hws_bwc_rule *bwc_rule;
 953	struct list_head *rules_list;
 954	int i;
 955
 956	for (i = 0; i < bwc_queues; i++) {
 957		rules_list = &bwc_matcher->rules[i];
 958		if (list_empty(rules_list))
 959			continue;
 960
 961		list_for_each_entry(bwc_rule, rules_list, list_node) {
 962			if (!bwc_rule->subrule_data)
 963				continue;
 964			bwc_rule->subrule_data->was_moved = false;
 965		}
 966	}
 967}
 968
 969int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher)
 970{
 971	struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
 972	struct mlx5hws_matcher *matcher = bwc_matcher->matcher;
 973	u16 bwc_queues = mlx5hws_bwc_queues(ctx);
 974	struct mlx5hws_bwc_rule *tmp_bwc_rule;
 975	struct mlx5hws_rule_attr rule_attr;
 976	int move_error = 0, poll_error = 0;
 977	struct mlx5hws_rule *tmp_rule;
 978	struct list_head *rules_list;
 979	u32 expected_completions = 1;
 980	int i, ret = 0;
 981
 982	hws_bwc_matcher_init_move(bwc_matcher);
 983
 984	mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr);
 985
 986	for (i = 0; i < bwc_queues; i++) {
 987		rules_list = &bwc_matcher->rules[i];
 988		if (list_empty(rules_list))
 989			continue;
 990
 991		rule_attr.queue_id = mlx5hws_bwc_get_queue_id(ctx, i);
 992
 993		list_for_each_entry(tmp_bwc_rule, rules_list, list_node) {
 994			/* Check if a rule with similar tag has already
 995			 * been moved.
 996			 */
 997			if (tmp_bwc_rule->subrule_data->was_moved) {
 998				/* This rule is a duplicate of rule with
 999				 * identical tag that has already been moved
1000				 * earlier. Just update this rule's RTCs.
1001				 */
1002				tmp_bwc_rule->rule->rtc_0 =
1003					tmp_bwc_rule->subrule_data->rtc_0;
1004				tmp_bwc_rule->rule->rtc_1 =
1005					tmp_bwc_rule->subrule_data->rtc_1;
1006				tmp_bwc_rule->rule->matcher =
1007					tmp_bwc_rule->rule->matcher->resize_dst;
1008				continue;
1009			}
1010
1011			/* First time we're moving rule with this tag.
1012			 * Move it for real.
1013			 */
1014			tmp_rule = tmp_bwc_rule->rule;
1015			tmp_rule->skip_delete = false;
1016			ret = mlx5hws_matcher_resize_rule_move(matcher,
1017							       tmp_rule,
1018							       &rule_attr);
1019			if (unlikely(ret)) {
1020				if (!move_error) {
1021					mlx5hws_err(ctx,
1022						    "Moving complex BWC rule: move failed (%d), attempting to move rest of the rules\n",
1023						    ret);
1024					move_error = ret;
1025				}
1026				/* Rule wasn't queued, no need to poll */
1027				continue;
1028			}
1029
1030			expected_completions = 1;
1031			ret = mlx5hws_bwc_queue_poll(ctx,
1032						     rule_attr.queue_id,
1033						     &expected_completions,
1034						     true);
1035			if (unlikely(ret)) {
1036				if (ret == -ETIMEDOUT) {
1037					mlx5hws_err(ctx,
1038						    "Moving complex BWC rule: timeout polling for completions (%d), aborting rehash\n",
1039						    ret);
1040					return ret;
1041				}
1042				if (!poll_error) {
1043					mlx5hws_err(ctx,
1044						    "Moving complex BWC rule: polling for completions failed (%d), attempting to move rest of the rules\n",
1045						    ret);
1046					poll_error = ret;
1047				}
1048			}
1049
1050			/* Done moving the rule to the new matcher,
1051			 * now update RTCs for all the duplicated rules.
1052			 */
1053			tmp_bwc_rule->subrule_data->rtc_0 =
1054				tmp_bwc_rule->rule->rtc_0;
1055			tmp_bwc_rule->subrule_data->rtc_1 =
1056				tmp_bwc_rule->rule->rtc_1;
1057
1058			tmp_bwc_rule->subrule_data->was_moved = true;
1059		}
1060	}
1061
1062	/* Return the first error that happened */
1063	if (unlikely(move_error))
1064		return move_error;
1065	if (unlikely(poll_error))
1066		return poll_error;
1067
1068	return ret;
1069}
1070
1071int
1072mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher)
1073{
1074	struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
1075	struct mlx5hws_bwc_matcher_complex_data *cdata;
1076	struct mlx5hws_table *isolated_tbl;
1077	u32 end_ft_id;
1078	int i, ret;
1079
1080	cdata = bwc_matcher->complex;
1081
1082	/* We are rehashing the first submatcher. We need to update the
1083	 * subsequent submatchers to point to the end_ft of this new matcher.
1084	 * This needs to be done before moving any rules to prevent possible
1085	 * steering loops.
1086	 */
1087	end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id;
1088	for (i = 1; i < cdata->num_submatchers; i++) {
1089		isolated_tbl = cdata->submatchers[i].tbl;
1090		ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl,
1091							     end_ft_id);
1092		if (ret) {
1093			mlx5hws_err(ctx,
1094				    "Complex matcher: failed updating end_ft of isolated matcher (%d)\n",
1095				    ret);
1096			return ret;
1097		}
1098	}
1099
1100	return mlx5hws_bwc_matcher_complex_move(bwc_matcher);
1101}