action.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2024 NVIDIA Corporation & Affiliates */
   3
   4#include "internal.h"
   5
   6#define MLX5HWS_ACTION_METER_INIT_COLOR_OFFSET 1
   7
   8/* Header removal size limited to 128B (64 words) */
   9#define MLX5HWS_ACTION_REMOVE_HEADER_MAX_SIZE 128
  10
  11/* This is the longest supported action sequence for FDB table:
  12 * DECAP, POP_VLAN, MODIFY, CTR, ASO, PUSH_VLAN, MODIFY, ENCAP, Term.
  13 */
  14static const u32 action_order_arr[MLX5HWS_ACTION_TYP_MAX] = {
  15	BIT(MLX5HWS_ACTION_TYP_REMOVE_HEADER) |
  16	BIT(MLX5HWS_ACTION_TYP_REFORMAT_TNL_L2_TO_L2) |
  17	BIT(MLX5HWS_ACTION_TYP_REFORMAT_TNL_L3_TO_L2),
  18	BIT(MLX5HWS_ACTION_TYP_POP_VLAN),
  19	BIT(MLX5HWS_ACTION_TYP_POP_VLAN),
  20	BIT(MLX5HWS_ACTION_TYP_MODIFY_HDR),
  21	BIT(MLX5HWS_ACTION_TYP_PUSH_VLAN),
  22	BIT(MLX5HWS_ACTION_TYP_PUSH_VLAN),
  23	BIT(MLX5HWS_ACTION_TYP_INSERT_HEADER) |
  24	BIT(MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L2) |
  25	BIT(MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L3),
  26	BIT(MLX5HWS_ACTION_TYP_CTR),
  27	BIT(MLX5HWS_ACTION_TYP_TAG),
  28	BIT(MLX5HWS_ACTION_TYP_ASO_METER),
  29	BIT(MLX5HWS_ACTION_TYP_MODIFY_HDR),
  30	BIT(MLX5HWS_ACTION_TYP_TBL) |
  31	BIT(MLX5HWS_ACTION_TYP_VPORT) |
  32	BIT(MLX5HWS_ACTION_TYP_DROP) |
  33	BIT(MLX5HWS_ACTION_TYP_SAMPLER) |
  34	BIT(MLX5HWS_ACTION_TYP_RANGE) |
  35	BIT(MLX5HWS_ACTION_TYP_DEST_ARRAY),
  36	BIT(MLX5HWS_ACTION_TYP_LAST),
  37};
  38
  39static const char * const mlx5hws_action_type_str[] = {
  40	[MLX5HWS_ACTION_TYP_LAST] = "LAST",
  41	[MLX5HWS_ACTION_TYP_REFORMAT_TNL_L2_TO_L2] = "TNL_L2_TO_L2",
  42	[MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L2] = "L2_TO_TNL_L2",
  43	[MLX5HWS_ACTION_TYP_REFORMAT_TNL_L3_TO_L2] = "TNL_L3_TO_L2",
  44	[MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L3] = "L2_TO_TNL_L3",
  45	[MLX5HWS_ACTION_TYP_DROP] = "DROP",
  46	[MLX5HWS_ACTION_TYP_TBL] = "TBL",
  47	[MLX5HWS_ACTION_TYP_CTR] = "CTR",
  48	[MLX5HWS_ACTION_TYP_TAG] = "TAG",
  49	[MLX5HWS_ACTION_TYP_MODIFY_HDR] = "MODIFY_HDR",
  50	[MLX5HWS_ACTION_TYP_VPORT] = "VPORT",
  51	[MLX5HWS_ACTION_TYP_MISS] = "DEFAULT_MISS",
  52	[MLX5HWS_ACTION_TYP_POP_VLAN] = "POP_VLAN",
  53	[MLX5HWS_ACTION_TYP_PUSH_VLAN] = "PUSH_VLAN",
  54	[MLX5HWS_ACTION_TYP_ASO_METER] = "ASO_METER",
  55	[MLX5HWS_ACTION_TYP_DEST_ARRAY] = "DEST_ARRAY",
  56	[MLX5HWS_ACTION_TYP_INSERT_HEADER] = "INSERT_HEADER",
  57	[MLX5HWS_ACTION_TYP_REMOVE_HEADER] = "REMOVE_HEADER",
  58	[MLX5HWS_ACTION_TYP_SAMPLER] = "SAMPLER",
  59	[MLX5HWS_ACTION_TYP_RANGE] = "RANGE",
  60};
  61
  62static_assert(ARRAY_SIZE(mlx5hws_action_type_str) == MLX5HWS_ACTION_TYP_MAX,
  63	      "Missing mlx5hws_action_type_str");
  64
  65const char *mlx5hws_action_type_to_str(enum mlx5hws_action_type action_type)
  66{
  67	return mlx5hws_action_type_str[action_type];
  68}
  69
  70enum mlx5hws_action_type mlx5hws_action_get_type(struct mlx5hws_action *action)
  71{
  72	return action->type;
  73}
  74
  75struct mlx5_core_dev *mlx5hws_action_get_dev(struct mlx5hws_action *action)
  76{
  77	return action->ctx->mdev;
  78}
  79
  80static int hws_action_get_shared_stc_nic(struct mlx5hws_context *ctx,
  81					 enum mlx5hws_context_shared_stc_type stc_type,
  82					 u8 tbl_type)
  83{
  84	struct mlx5hws_cmd_stc_modify_attr stc_attr = {0};
  85	struct mlx5hws_action_shared_stc *shared_stc;
  86	int ret;
  87
  88	mutex_lock(&ctx->ctrl_lock);
  89	if (ctx->common_res.shared_stc[stc_type]) {
  90		ctx->common_res.shared_stc[stc_type]->refcount++;
  91		mutex_unlock(&ctx->ctrl_lock);
  92		return 0;
  93	}
  94
  95	shared_stc = kzalloc(sizeof(*shared_stc), GFP_KERNEL);
  96	if (!shared_stc) {
  97		ret = -ENOMEM;
  98		goto unlock_and_out;
  99	}
 100	switch (stc_type) {
 101	case MLX5HWS_CONTEXT_SHARED_STC_DECAP_L3:
 102		stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_HEADER_REMOVE;
 103		stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_DW5;
 104		stc_attr.reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
 105		stc_attr.remove_header.decap = 0;
 106		stc_attr.remove_header.start_anchor = MLX5_HEADER_ANCHOR_PACKET_START;
 107		stc_attr.remove_header.end_anchor = MLX5_HEADER_ANCHOR_IPV6_IPV4;
 108		break;
 109	case MLX5HWS_CONTEXT_SHARED_STC_DOUBLE_POP:
 110		stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_REMOVE_WORDS;
 111		stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_DW5;
 112		stc_attr.reparse_mode = MLX5_IFC_STC_REPARSE_ALWAYS;
 113		stc_attr.remove_words.start_anchor = MLX5_HEADER_ANCHOR_FIRST_VLAN_START;
 114		stc_attr.remove_words.num_of_words = MLX5HWS_ACTION_HDR_LEN_L2_VLAN;
 115		break;
 116	default:
 117		mlx5hws_err(ctx, "No such stc_type: %d\n", stc_type);
 118		pr_warn("HWS: Invalid stc_type: %d\n", stc_type);
 119		ret = -EINVAL;
 120		goto free_shared_stc;
 121	}
 122
 123	ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, tbl_type,
 124					      &shared_stc->stc_chunk);
 125	if (ret) {
 126		mlx5hws_err(ctx, "Failed to allocate shared decap l2 STC\n");
 127		goto free_shared_stc;
 128	}
 129
 130	ctx->common_res.shared_stc[stc_type] = shared_stc;
 131	ctx->common_res.shared_stc[stc_type]->refcount = 1;
 132
 133	mutex_unlock(&ctx->ctrl_lock);
 134
 135	return 0;
 136
 137free_shared_stc:
 138	kfree(shared_stc);
 139unlock_and_out:
 140	mutex_unlock(&ctx->ctrl_lock);
 141	return ret;
 142}
 143
 144static int hws_action_get_shared_stc(struct mlx5hws_action *action,
 145				     enum mlx5hws_context_shared_stc_type stc_type)
 146{
 147	struct mlx5hws_context *ctx = action->ctx;
 148	int ret;
 149
 150	if (stc_type >= MLX5HWS_CONTEXT_SHARED_STC_MAX) {
 151		pr_warn("HWS: Invalid shared stc_type: %d\n", stc_type);
 152		return -EINVAL;
 153	}
 154
 155	if (unlikely(!(action->flags & MLX5HWS_ACTION_FLAG_HWS_FDB))) {
 156		pr_warn("HWS: Invalid action->flags: %d\n", action->flags);
 157		return -EINVAL;
 158	}
 159
 160	ret = hws_action_get_shared_stc_nic(ctx, stc_type, MLX5HWS_TABLE_TYPE_FDB);
 161	if (ret) {
 162		mlx5hws_err(ctx,
 163			    "Failed to allocate memory for FDB shared STCs (type: %d)\n",
 164			    stc_type);
 165		return ret;
 166	}
 167
 168	return 0;
 169}
 170
 171static void hws_action_put_shared_stc(struct mlx5hws_action *action,
 172				      enum mlx5hws_context_shared_stc_type stc_type)
 173{
 174	enum mlx5hws_table_type tbl_type = MLX5HWS_TABLE_TYPE_FDB;
 175	struct mlx5hws_action_shared_stc *shared_stc;
 176	struct mlx5hws_context *ctx = action->ctx;
 177
 178	if (stc_type >= MLX5HWS_CONTEXT_SHARED_STC_MAX) {
 179		pr_warn("HWS: Invalid shared stc_type: %d\n", stc_type);
 180		return;
 181	}
 182
 183	mutex_lock(&ctx->ctrl_lock);
 184	if (--ctx->common_res.shared_stc[stc_type]->refcount) {
 185		mutex_unlock(&ctx->ctrl_lock);
 186		return;
 187	}
 188
 189	shared_stc = ctx->common_res.shared_stc[stc_type];
 190
 191	mlx5hws_action_free_single_stc(ctx, tbl_type, &shared_stc->stc_chunk);
 192	kfree(shared_stc);
 193	ctx->common_res.shared_stc[stc_type] = NULL;
 194	mutex_unlock(&ctx->ctrl_lock);
 195}
 196
 197static void hws_action_print_combo(struct mlx5hws_context *ctx,
 198				   enum mlx5hws_action_type *user_actions)
 199{
 200	mlx5hws_err(ctx, "Invalid action_type sequence");
 201	while (*user_actions != MLX5HWS_ACTION_TYP_LAST) {
 202		mlx5hws_err(ctx, " %s", mlx5hws_action_type_to_str(*user_actions));
 203		user_actions++;
 204	}
 205	mlx5hws_err(ctx, "\n");
 206}
 207
 208bool mlx5hws_action_check_combo(struct mlx5hws_context *ctx,
 209				enum mlx5hws_action_type *user_actions,
 210				enum mlx5hws_table_type table_type)
 211{
 212	const u32 *order_arr = action_order_arr;
 213	bool valid_combo;
 214	u8 order_idx = 0;
 215	u8 user_idx = 0;
 216
 217	if (table_type >= MLX5HWS_TABLE_TYPE_MAX) {
 218		mlx5hws_err(ctx, "Invalid table_type %d", table_type);
 219		return false;
 220	}
 221
 222	while (order_arr[order_idx] != BIT(MLX5HWS_ACTION_TYP_LAST)) {
 223		/* User action order validated move to next user action */
 224		if (BIT(user_actions[user_idx]) & order_arr[order_idx])
 225			user_idx++;
 226
 227		/* Iterate to the next supported action in the order */
 228		order_idx++;
 229	}
 230
 231	/* Combination is valid if all user action were processed */
 232	valid_combo = user_actions[user_idx] == MLX5HWS_ACTION_TYP_LAST;
 233	if (!valid_combo)
 234		hws_action_print_combo(ctx, user_actions);
 235
 236	return valid_combo;
 237}
 238
 239static bool
 240hws_action_fixup_stc_attr(struct mlx5hws_context *ctx,
 241			  struct mlx5hws_cmd_stc_modify_attr *stc_attr,
 242			  struct mlx5hws_cmd_stc_modify_attr *fixup_stc_attr,
 243			  enum mlx5hws_table_type table_type,
 244			  bool is_mirror)
 245{
 246	struct mlx5hws_pool *pool;
 247	bool use_fixup = false;
 248	u32 fw_tbl_type;
 249	u32 base_id;
 250
 251	fw_tbl_type = mlx5hws_table_get_res_fw_ft_type(table_type, is_mirror);
 252
 253	switch (stc_attr->action_type) {
 254	case MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_STE_TABLE:
 255		if (is_mirror && stc_attr->ste_table.ignore_tx) {
 256			fixup_stc_attr->action_type = MLX5_IFC_STC_ACTION_TYPE_DROP;
 257			fixup_stc_attr->action_offset = MLX5HWS_ACTION_OFFSET_HIT;
 258			fixup_stc_attr->stc_offset = stc_attr->stc_offset;
 259			use_fixup = true;
 260			break;
 261		}
 262		pool = stc_attr->ste_table.ste_pool;
 263		if (!is_mirror)
 264			base_id = mlx5hws_pool_get_base_id(pool);
 265		else
 266			base_id = mlx5hws_pool_get_base_mirror_id(pool);
 267
 268		*fixup_stc_attr = *stc_attr;
 269		fixup_stc_attr->ste_table.ste_obj_id = base_id;
 270		use_fixup = true;
 271		break;
 272
 273	case MLX5_IFC_STC_ACTION_TYPE_TAG:
 274		if (fw_tbl_type == FS_FT_FDB_TX) {
 275			fixup_stc_attr->action_type = MLX5_IFC_STC_ACTION_TYPE_NOP;
 276			fixup_stc_attr->action_offset = MLX5HWS_ACTION_OFFSET_DW5;
 277			fixup_stc_attr->stc_offset = stc_attr->stc_offset;
 278			use_fixup = true;
 279		}
 280		break;
 281
 282	case MLX5_IFC_STC_ACTION_TYPE_ALLOW:
 283		if (fw_tbl_type == FS_FT_FDB_TX || fw_tbl_type == FS_FT_FDB_RX) {
 284			fixup_stc_attr->action_type = MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_VPORT;
 285			fixup_stc_attr->action_offset = stc_attr->action_offset;
 286			fixup_stc_attr->stc_offset = stc_attr->stc_offset;
 287			fixup_stc_attr->vport.esw_owner_vhca_id = ctx->caps->vhca_id;
 288			fixup_stc_attr->vport.vport_num = ctx->caps->eswitch_manager_vport_number;
 289			fixup_stc_attr->vport.eswitch_owner_vhca_id_valid =
 290				ctx->caps->merged_eswitch;
 291			use_fixup = true;
 292		}
 293		break;
 294
 295	case MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_VPORT:
 296		if (stc_attr->vport.vport_num != MLX5_VPORT_UPLINK)
 297			break;
 298
 299		if (fw_tbl_type == FS_FT_FDB_TX || fw_tbl_type == FS_FT_FDB_RX) {
 300			/* The FW doesn't allow to go to wire in the TX/RX by JUMP_TO_VPORT */
 301			fixup_stc_attr->action_type = MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_UPLINK;
 302			fixup_stc_attr->action_offset = stc_attr->action_offset;
 303			fixup_stc_attr->stc_offset = stc_attr->stc_offset;
 304			fixup_stc_attr->vport.vport_num = 0;
 305			fixup_stc_attr->vport.esw_owner_vhca_id = stc_attr->vport.esw_owner_vhca_id;
 306			fixup_stc_attr->vport.eswitch_owner_vhca_id_valid =
 307				stc_attr->vport.eswitch_owner_vhca_id_valid;
 308		}
 309		use_fixup = true;
 310		break;
 311
 312	default:
 313		break;
 314	}
 315
 316	return use_fixup;
 317}
 318
 319int mlx5hws_action_alloc_single_stc(struct mlx5hws_context *ctx,
 320				    struct mlx5hws_cmd_stc_modify_attr *stc_attr,
 321				    u32 table_type,
 322				    struct mlx5hws_pool_chunk *stc)
 323__must_hold(&ctx->ctrl_lock)
 324{
 325	struct mlx5hws_cmd_stc_modify_attr cleanup_stc_attr = {0};
 326	struct mlx5hws_cmd_stc_modify_attr fixup_stc_attr = {0};
 327	struct mlx5hws_pool *stc_pool = ctx->stc_pool;
 328	bool use_fixup;
 329	u32 obj_0_id;
 330	int ret;
 331
 332	ret = mlx5hws_pool_chunk_alloc(stc_pool, stc);
 333	if (ret) {
 334		mlx5hws_err(ctx, "Failed to allocate single action STC\n");
 335		return ret;
 336	}
 337
 338	stc_attr->stc_offset = stc->offset;
 339
 340	/* Dynamic reparse not supported, overwrite and use default */
 341	if (!mlx5hws_context_cap_dynamic_reparse(ctx))
 342		stc_attr->reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
 343
 344	obj_0_id = mlx5hws_pool_get_base_id(stc_pool);
 345
 346	/* According to table/action limitation change the stc_attr */
 347	use_fixup = hws_action_fixup_stc_attr(ctx, stc_attr, &fixup_stc_attr, table_type, false);
 348	ret = mlx5hws_cmd_stc_modify(ctx->mdev, obj_0_id,
 349				     use_fixup ? &fixup_stc_attr : stc_attr);
 350	if (ret) {
 351		mlx5hws_err(ctx, "Failed to modify STC action_type %d tbl_type %d\n",
 352			    stc_attr->action_type, table_type);
 353		goto free_chunk;
 354	}
 355
 356	/* Modify the FDB peer */
 357	if (table_type == MLX5HWS_TABLE_TYPE_FDB) {
 358		u32 obj_1_id;
 359
 360		obj_1_id = mlx5hws_pool_get_base_mirror_id(stc_pool);
 361
 362		use_fixup = hws_action_fixup_stc_attr(ctx, stc_attr,
 363						      &fixup_stc_attr,
 364						      table_type, true);
 365		ret = mlx5hws_cmd_stc_modify(ctx->mdev, obj_1_id,
 366					     use_fixup ? &fixup_stc_attr : stc_attr);
 367		if (ret) {
 368			mlx5hws_err(ctx,
 369				    "Failed to modify peer STC action_type %d tbl_type %d\n",
 370				    stc_attr->action_type, table_type);
 371			goto clean_obj_0;
 372		}
 373	}
 374
 375	return 0;
 376
 377clean_obj_0:
 378	cleanup_stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_DROP;
 379	cleanup_stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_HIT;
 380	cleanup_stc_attr.stc_offset = stc->offset;
 381	mlx5hws_cmd_stc_modify(ctx->mdev, obj_0_id, &cleanup_stc_attr);
 382free_chunk:
 383	mlx5hws_pool_chunk_free(stc_pool, stc);
 384	return ret;
 385}
 386
 387void mlx5hws_action_free_single_stc(struct mlx5hws_context *ctx,
 388				    u32 table_type,
 389				    struct mlx5hws_pool_chunk *stc)
 390__must_hold(&ctx->ctrl_lock)
 391{
 392	struct mlx5hws_cmd_stc_modify_attr stc_attr = {0};
 393	struct mlx5hws_pool *stc_pool = ctx->stc_pool;
 394	u32 obj_id;
 395
 396	/* Modify the STC not to point to an object */
 397	stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_DROP;
 398	stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_HIT;
 399	stc_attr.stc_offset = stc->offset;
 400	obj_id = mlx5hws_pool_get_base_id(stc_pool);
 401	mlx5hws_cmd_stc_modify(ctx->mdev, obj_id, &stc_attr);
 402
 403	if (table_type == MLX5HWS_TABLE_TYPE_FDB) {
 404		obj_id = mlx5hws_pool_get_base_mirror_id(stc_pool);
 405		mlx5hws_cmd_stc_modify(ctx->mdev, obj_id, &stc_attr);
 406	}
 407
 408	mlx5hws_pool_chunk_free(stc_pool, stc);
 409}
 410
 411static u32 hws_action_get_mh_stc_type(struct mlx5hws_context *ctx,
 412				      __be64 pattern)
 413{
 414	u8 action_type = MLX5_GET(set_action_in, &pattern, action_type);
 415
 416	switch (action_type) {
 417	case MLX5_MODIFICATION_TYPE_SET:
 418		return MLX5_IFC_STC_ACTION_TYPE_SET;
 419	case MLX5_MODIFICATION_TYPE_ADD:
 420		return MLX5_IFC_STC_ACTION_TYPE_ADD;
 421	case MLX5_MODIFICATION_TYPE_COPY:
 422		return MLX5_IFC_STC_ACTION_TYPE_COPY;
 423	case MLX5_MODIFICATION_TYPE_ADD_FIELD:
 424		return MLX5_IFC_STC_ACTION_TYPE_ADD_FIELD;
 425	default:
 426		mlx5hws_err(ctx, "Unsupported action type: 0x%x\n", action_type);
 427		return MLX5_IFC_STC_ACTION_TYPE_NOP;
 428	}
 429}
 430
 431static void hws_action_fill_stc_attr(struct mlx5hws_action *action,
 432				     u32 obj_id,
 433				     struct mlx5hws_cmd_stc_modify_attr *attr)
 434{
 435	attr->reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
 436
 437	switch (action->type) {
 438	case MLX5HWS_ACTION_TYP_TAG:
 439		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_TAG;
 440		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW5;
 441		break;
 442	case MLX5HWS_ACTION_TYP_DROP:
 443		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_DROP;
 444		attr->action_offset = MLX5HWS_ACTION_OFFSET_HIT;
 445		break;
 446	case MLX5HWS_ACTION_TYP_MISS:
 447		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_ALLOW;
 448		attr->action_offset = MLX5HWS_ACTION_OFFSET_HIT;
 449		break;
 450	case MLX5HWS_ACTION_TYP_CTR:
 451		attr->id = obj_id;
 452		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_COUNTER;
 453		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW0;
 454		break;
 455	case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L3_TO_L2:
 456	case MLX5HWS_ACTION_TYP_MODIFY_HDR:
 457		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW6;
 458		attr->reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
 459		if (action->modify_header.require_reparse)
 460			attr->reparse_mode = MLX5_IFC_STC_REPARSE_ALWAYS;
 461
 462		if (action->modify_header.num_of_actions == 1) {
 463			attr->modify_action.data = action->modify_header.single_action;
 464			attr->action_type = hws_action_get_mh_stc_type(action->ctx,
 465								       attr->modify_action.data);
 466
 467			if (attr->action_type == MLX5_IFC_STC_ACTION_TYPE_ADD ||
 468			    attr->action_type == MLX5_IFC_STC_ACTION_TYPE_SET)
 469				MLX5_SET(set_action_in, &attr->modify_action.data, data, 0);
 470		} else {
 471			attr->action_type = MLX5_IFC_STC_ACTION_TYPE_ACC_MODIFY_LIST;
 472			attr->modify_header.arg_id = action->modify_header.arg_id;
 473			attr->modify_header.pattern_id = action->modify_header.pat_id;
 474		}
 475		break;
 476	case MLX5HWS_ACTION_TYP_TBL:
 477	case MLX5HWS_ACTION_TYP_DEST_ARRAY:
 478	case MLX5HWS_ACTION_TYP_SAMPLER:
 479		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_FT;
 480		attr->action_offset = MLX5HWS_ACTION_OFFSET_HIT;
 481		attr->dest_table_id = obj_id;
 482		break;
 483	case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L2_TO_L2:
 484		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_HEADER_REMOVE;
 485		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW5;
 486		attr->reparse_mode = MLX5_IFC_STC_REPARSE_ALWAYS;
 487		attr->remove_header.decap = 1;
 488		attr->remove_header.start_anchor = MLX5_HEADER_ANCHOR_PACKET_START;
 489		attr->remove_header.end_anchor = MLX5_HEADER_ANCHOR_INNER_MAC;
 490		break;
 491	case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L2:
 492	case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L3:
 493	case MLX5HWS_ACTION_TYP_INSERT_HEADER:
 494		attr->reparse_mode = MLX5_IFC_STC_REPARSE_ALWAYS;
 495		if (!action->reformat.require_reparse)
 496			attr->reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
 497
 498		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_HEADER_INSERT;
 499		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW6;
 500		attr->insert_header.encap = action->reformat.encap;
 501		attr->insert_header.insert_anchor = action->reformat.anchor;
 502		attr->insert_header.arg_id = action->reformat.arg_id;
 503		attr->insert_header.header_size = action->reformat.header_size;
 504		attr->insert_header.insert_offset = action->reformat.offset;
 505		break;
 506	case MLX5HWS_ACTION_TYP_ASO_METER:
 507		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW6;
 508		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_ASO;
 509		attr->aso.aso_type = ASO_OPC_MOD_POLICER;
 510		attr->aso.devx_obj_id = obj_id;
 511		attr->aso.return_reg_id = action->aso.return_reg_id;
 512		break;
 513	case MLX5HWS_ACTION_TYP_VPORT:
 514		attr->action_offset = MLX5HWS_ACTION_OFFSET_HIT;
 515		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_VPORT;
 516		attr->vport.vport_num = action->vport.vport_num;
 517		attr->vport.esw_owner_vhca_id =	action->vport.esw_owner_vhca_id;
 518		attr->vport.eswitch_owner_vhca_id_valid = action->vport.esw_owner_vhca_id_valid;
 519		break;
 520	case MLX5HWS_ACTION_TYP_POP_VLAN:
 521		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_REMOVE_WORDS;
 522		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW5;
 523		attr->reparse_mode = MLX5_IFC_STC_REPARSE_ALWAYS;
 524		attr->remove_words.start_anchor = MLX5_HEADER_ANCHOR_FIRST_VLAN_START;
 525		attr->remove_words.num_of_words = MLX5HWS_ACTION_HDR_LEN_L2_VLAN / 2;
 526		break;
 527	case MLX5HWS_ACTION_TYP_PUSH_VLAN:
 528		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_HEADER_INSERT;
 529		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW6;
 530		attr->reparse_mode = MLX5_IFC_STC_REPARSE_ALWAYS;
 531		attr->insert_header.encap = 0;
 532		attr->insert_header.is_inline = 1;
 533		attr->insert_header.insert_anchor = MLX5_HEADER_ANCHOR_PACKET_START;
 534		attr->insert_header.insert_offset = MLX5HWS_ACTION_HDR_LEN_L2_MACS;
 535		attr->insert_header.header_size = MLX5HWS_ACTION_HDR_LEN_L2_VLAN;
 536		break;
 537	case MLX5HWS_ACTION_TYP_REMOVE_HEADER:
 538		attr->action_type = MLX5_IFC_STC_ACTION_TYPE_REMOVE_WORDS;
 539		attr->remove_header.decap = 0; /* the mode we support decap is 0 */
 540		attr->remove_words.start_anchor = action->remove_header.anchor;
 541		/* the size is in already in words */
 542		attr->remove_words.num_of_words = action->remove_header.size;
 543		attr->action_offset = MLX5HWS_ACTION_OFFSET_DW5;
 544		attr->reparse_mode = MLX5_IFC_STC_REPARSE_ALWAYS;
 545		break;
 546	default:
 547		mlx5hws_err(action->ctx, "Invalid action type %d\n", action->type);
 548	}
 549}
 550
 551static int
 552hws_action_create_stcs(struct mlx5hws_action *action, u32 obj_id)
 553{
 554	struct mlx5hws_cmd_stc_modify_attr stc_attr = {0};
 555	struct mlx5hws_context *ctx = action->ctx;
 556	int ret;
 557
 558	hws_action_fill_stc_attr(action, obj_id, &stc_attr);
 559
 560	/* Block unsupported parallel obj modify over the same base */
 561	mutex_lock(&ctx->ctrl_lock);
 562
 563	/* Allocate STC for FDB */
 564	if (action->flags & MLX5HWS_ACTION_FLAG_HWS_FDB) {
 565		ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr,
 566						      MLX5HWS_TABLE_TYPE_FDB,
 567						      &action->stc);
 568		if (ret)
 569			goto out_err;
 570	}
 571
 572	mutex_unlock(&ctx->ctrl_lock);
 573
 574	return 0;
 575
 576out_err:
 577	mutex_unlock(&ctx->ctrl_lock);
 578	return ret;
 579}
 580
 581static void
 582hws_action_destroy_stcs(struct mlx5hws_action *action)
 583{
 584	struct mlx5hws_context *ctx = action->ctx;
 585
 586	/* Block unsupported parallel obj modify over the same base */
 587	mutex_lock(&ctx->ctrl_lock);
 588
 589	if (action->flags & MLX5HWS_ACTION_FLAG_HWS_FDB)
 590		mlx5hws_action_free_single_stc(ctx, MLX5HWS_TABLE_TYPE_FDB,
 591					       &action->stc);
 592
 593	mutex_unlock(&ctx->ctrl_lock);
 594}
 595
 596static bool hws_action_is_flag_hws_fdb(u32 flags)
 597{
 598	return flags & MLX5HWS_ACTION_FLAG_HWS_FDB;
 599}
 600
 601static bool
 602hws_action_validate_hws_action(struct mlx5hws_context *ctx, u32 flags)
 603{
 604	if (!(ctx->flags & MLX5HWS_CONTEXT_FLAG_HWS_SUPPORT)) {
 605		mlx5hws_err(ctx, "Cannot create HWS action since HWS is not supported\n");
 606		return false;
 607	}
 608
 609	if ((flags & MLX5HWS_ACTION_FLAG_HWS_FDB) && !ctx->caps->eswitch_manager) {
 610		mlx5hws_err(ctx, "Cannot create HWS action for FDB for non-eswitch-manager\n");
 611		return false;
 612	}
 613
 614	return true;
 615}
 616
 617static struct mlx5hws_action *
 618hws_action_create_generic_bulk(struct mlx5hws_context *ctx,
 619			       u32 flags,
 620			       enum mlx5hws_action_type action_type,
 621			       u8 bulk_sz)
 622{
 623	struct mlx5hws_action *action;
 624	int i;
 625
 626	if (!hws_action_is_flag_hws_fdb(flags)) {
 627		mlx5hws_err(ctx,
 628			    "Action (type: %d) flags must specify only HWS FDB\n", action_type);
 629		return NULL;
 630	}
 631
 632	if (!hws_action_validate_hws_action(ctx, flags))
 633		return NULL;
 634
 635	action = kcalloc(bulk_sz, sizeof(*action), GFP_KERNEL);
 636	if (!action)
 637		return NULL;
 638
 639	for (i = 0; i < bulk_sz; i++) {
 640		action[i].ctx = ctx;
 641		action[i].flags = flags;
 642		action[i].type = action_type;
 643	}
 644
 645	return action;
 646}
 647
 648static struct mlx5hws_action *
 649hws_action_create_generic(struct mlx5hws_context *ctx,
 650			  u32 flags,
 651			  enum mlx5hws_action_type action_type)
 652{
 653	return hws_action_create_generic_bulk(ctx, flags, action_type, 1);
 654}
 655
 656struct mlx5hws_action *
 657mlx5hws_action_create_dest_table_num(struct mlx5hws_context *ctx,
 658				     u32 table_id,
 659				     u32 flags)
 660{
 661	struct mlx5hws_action *action;
 662	int ret;
 663
 664	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_TBL);
 665	if (!action)
 666		return NULL;
 667
 668	ret = hws_action_create_stcs(action, table_id);
 669	if (ret)
 670		goto free_action;
 671
 672	action->dest_obj.obj_id = table_id;
 673
 674	return action;
 675
 676free_action:
 677	kfree(action);
 678	return NULL;
 679}
 680
 681struct mlx5hws_action *
 682mlx5hws_action_create_dest_table(struct mlx5hws_context *ctx,
 683				 struct mlx5hws_table *tbl,
 684				 u32 flags)
 685{
 686	return mlx5hws_action_create_dest_table_num(ctx, tbl->ft_id, flags);
 687}
 688
 689struct mlx5hws_action *
 690mlx5hws_action_create_dest_drop(struct mlx5hws_context *ctx, u32 flags)
 691{
 692	struct mlx5hws_action *action;
 693	int ret;
 694
 695	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_DROP);
 696	if (!action)
 697		return NULL;
 698
 699	ret = hws_action_create_stcs(action, 0);
 700	if (ret)
 701		goto free_action;
 702
 703	return action;
 704
 705free_action:
 706	kfree(action);
 707	return NULL;
 708}
 709
 710struct mlx5hws_action *
 711mlx5hws_action_create_default_miss(struct mlx5hws_context *ctx, u32 flags)
 712{
 713	struct mlx5hws_action *action;
 714	int ret;
 715
 716	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_MISS);
 717	if (!action)
 718		return NULL;
 719
 720	ret = hws_action_create_stcs(action, 0);
 721	if (ret)
 722		goto free_action;
 723
 724	return action;
 725
 726free_action:
 727	kfree(action);
 728	return NULL;
 729}
 730
 731struct mlx5hws_action *
 732mlx5hws_action_create_tag(struct mlx5hws_context *ctx, u32 flags)
 733{
 734	struct mlx5hws_action *action;
 735	int ret;
 736
 737	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_TAG);
 738	if (!action)
 739		return NULL;
 740
 741	ret = hws_action_create_stcs(action, 0);
 742	if (ret)
 743		goto free_action;
 744
 745	return action;
 746
 747free_action:
 748	kfree(action);
 749	return NULL;
 750}
 751
 752static struct mlx5hws_action *
 753hws_action_create_aso(struct mlx5hws_context *ctx,
 754		      enum mlx5hws_action_type action_type,
 755		      u32 obj_id,
 756		      u8 return_reg_id,
 757		      u32 flags)
 758{
 759	struct mlx5hws_action *action;
 760	int ret;
 761
 762	action = hws_action_create_generic(ctx, flags, action_type);
 763	if (!action)
 764		return NULL;
 765
 766	action->aso.obj_id = obj_id;
 767	action->aso.return_reg_id = return_reg_id;
 768
 769	ret = hws_action_create_stcs(action, obj_id);
 770	if (ret)
 771		goto free_action;
 772
 773	return action;
 774
 775free_action:
 776	kfree(action);
 777	return NULL;
 778}
 779
 780struct mlx5hws_action *
 781mlx5hws_action_create_aso_meter(struct mlx5hws_context *ctx,
 782				u32 obj_id,
 783				u8 return_reg_id,
 784				u32 flags)
 785{
 786	return hws_action_create_aso(ctx, MLX5HWS_ACTION_TYP_ASO_METER,
 787				     obj_id, return_reg_id, flags);
 788}
 789
 790struct mlx5hws_action *
 791mlx5hws_action_create_counter(struct mlx5hws_context *ctx,
 792			      u32 obj_id,
 793			      u32 flags)
 794{
 795	struct mlx5hws_action *action;
 796	int ret;
 797
 798	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_CTR);
 799	if (!action)
 800		return NULL;
 801
 802	ret = hws_action_create_stcs(action, obj_id);
 803	if (ret)
 804		goto free_action;
 805
 806	return action;
 807
 808free_action:
 809	kfree(action);
 810	return NULL;
 811}
 812
 813struct mlx5hws_action *
 814mlx5hws_action_create_dest_vport(struct mlx5hws_context *ctx,
 815				 u16 vport_num,
 816				 bool vhca_id_valid,
 817				 u16 vhca_id,
 818				 u32 flags)
 819{
 820	struct mlx5hws_action *action;
 821	int ret;
 822
 823	if (!(flags & MLX5HWS_ACTION_FLAG_HWS_FDB)) {
 824		mlx5hws_err(ctx, "Vport action is supported for FDB only\n");
 825		return NULL;
 826	}
 827
 828	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_VPORT);
 829	if (!action)
 830		return NULL;
 831
 832	if (!ctx->caps->merged_eswitch && vhca_id_valid && vhca_id != ctx->caps->vhca_id) {
 833		mlx5hws_err(ctx, "Non merged eswitch cannot send to other vhca\n");
 834		goto free_action;
 835	}
 836
 837	action->vport.vport_num = vport_num;
 838	action->vport.esw_owner_vhca_id_valid = vhca_id_valid;
 839
 840	if (vhca_id_valid)
 841		action->vport.esw_owner_vhca_id = vhca_id;
 842
 843	ret = hws_action_create_stcs(action, 0);
 844	if (ret) {
 845		mlx5hws_err(ctx, "Failed creating stc for vport %d\n", vport_num);
 846		goto free_action;
 847	}
 848
 849	return action;
 850
 851free_action:
 852	kfree(action);
 853	return NULL;
 854}
 855
 856struct mlx5hws_action *
 857mlx5hws_action_create_push_vlan(struct mlx5hws_context *ctx, u32 flags)
 858{
 859	struct mlx5hws_action *action;
 860	int ret;
 861
 862	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_PUSH_VLAN);
 863	if (!action)
 864		return NULL;
 865
 866	ret = hws_action_create_stcs(action, 0);
 867	if (ret) {
 868		mlx5hws_err(ctx, "Failed creating stc for push vlan\n");
 869		goto free_action;
 870	}
 871
 872	return action;
 873
 874free_action:
 875	kfree(action);
 876	return NULL;
 877}
 878
 879struct mlx5hws_action *
 880mlx5hws_action_create_pop_vlan(struct mlx5hws_context *ctx, u32 flags)
 881{
 882	struct mlx5hws_action *action;
 883	int ret;
 884
 885	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_POP_VLAN);
 886	if (!action)
 887		return NULL;
 888
 889	ret = hws_action_get_shared_stc(action, MLX5HWS_CONTEXT_SHARED_STC_DOUBLE_POP);
 890	if (ret) {
 891		mlx5hws_err(ctx, "Failed to create remove stc for reformat\n");
 892		goto free_action;
 893	}
 894
 895	ret = hws_action_create_stcs(action, 0);
 896	if (ret) {
 897		mlx5hws_err(ctx, "Failed creating stc for pop vlan\n");
 898		goto free_shared;
 899	}
 900
 901	return action;
 902
 903free_shared:
 904	hws_action_put_shared_stc(action, MLX5HWS_CONTEXT_SHARED_STC_DOUBLE_POP);
 905free_action:
 906	kfree(action);
 907	return NULL;
 908}
 909
 910static int
 911hws_action_handle_insert_with_ptr(struct mlx5hws_action *action,
 912				  u8 num_of_hdrs,
 913				  struct mlx5hws_action_reformat_header *hdrs,
 914				  u32 log_bulk_sz)
 915{
 916	size_t max_sz = 0;
 917	u32 arg_id;
 918	int ret, i;
 919
 920	for (i = 0; i < num_of_hdrs; i++) {
 921		if (hdrs[i].sz % W_SIZE != 0) {
 922			mlx5hws_err(action->ctx,
 923				    "Header data size should be in WORD granularity\n");
 924			return -EINVAL;
 925		}
 926		max_sz = max(hdrs[i].sz, max_sz);
 927	}
 928
 929	/* Allocate single shared arg object for all headers */
 930	ret = mlx5hws_arg_create(action->ctx,
 931				 hdrs->data,
 932				 max_sz,
 933				 log_bulk_sz,
 934				 action->flags & MLX5HWS_ACTION_FLAG_SHARED,
 935				 &arg_id);
 936	if (ret)
 937		return ret;
 938
 939	for (i = 0; i < num_of_hdrs; i++) {
 940		action[i].reformat.arg_id = arg_id;
 941		action[i].reformat.header_size = hdrs[i].sz;
 942		action[i].reformat.num_of_hdrs = num_of_hdrs;
 943		action[i].reformat.max_hdr_sz = max_sz;
 944		action[i].reformat.require_reparse = true;
 945
 946		if (action[i].type == MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L2 ||
 947		    action[i].type == MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L3) {
 948			action[i].reformat.anchor = MLX5_HEADER_ANCHOR_PACKET_START;
 949			action[i].reformat.offset = 0;
 950			action[i].reformat.encap = 1;
 951		}
 952
 953		ret = hws_action_create_stcs(&action[i], 0);
 954		if (ret) {
 955			mlx5hws_err(action->ctx, "Failed to create stc for reformat\n");
 956			goto free_stc;
 957		}
 958	}
 959
 960	return 0;
 961
 962free_stc:
 963	while (i--)
 964		hws_action_destroy_stcs(&action[i]);
 965
 966	mlx5hws_arg_destroy(action->ctx, arg_id);
 967	return ret;
 968}
 969
 970static int
 971hws_action_handle_l2_to_tunnel_l3(struct mlx5hws_action *action,
 972				  u8 num_of_hdrs,
 973				  struct mlx5hws_action_reformat_header *hdrs,
 974				  u32 log_bulk_sz)
 975{
 976	int ret;
 977
 978	/* The action is remove-l2-header + insert-l3-header */
 979	ret = hws_action_get_shared_stc(action, MLX5HWS_CONTEXT_SHARED_STC_DECAP_L3);
 980	if (ret) {
 981		mlx5hws_err(action->ctx, "Failed to create remove stc for reformat\n");
 982		return ret;
 983	}
 984
 985	/* Reuse the insert with pointer for the L2L3 header */
 986	ret = hws_action_handle_insert_with_ptr(action,
 987						num_of_hdrs,
 988						hdrs,
 989						log_bulk_sz);
 990	if (ret)
 991		goto put_shared_stc;
 992
 993	return 0;
 994
 995put_shared_stc:
 996	hws_action_put_shared_stc(action, MLX5HWS_CONTEXT_SHARED_STC_DECAP_L3);
 997	return ret;
 998}
 999
1000static void hws_action_prepare_decap_l3_actions(size_t data_sz,
1001						u8 *mh_data,
1002						int *num_of_actions)
1003{
1004	int actions;
1005	u32 i;
1006
1007	/* Remove L2L3 outer headers */
1008	MLX5_SET(stc_ste_param_remove, mh_data, action_type,
1009		 MLX5_MODIFICATION_TYPE_REMOVE);
1010	MLX5_SET(stc_ste_param_remove, mh_data, decap, 0x1);
1011	MLX5_SET(stc_ste_param_remove, mh_data, remove_start_anchor,
1012		 MLX5_HEADER_ANCHOR_PACKET_START);
1013	MLX5_SET(stc_ste_param_remove, mh_data, remove_end_anchor,
1014		 MLX5_HEADER_ANCHOR_INNER_IPV6_IPV4);
1015	mh_data += MLX5HWS_ACTION_DOUBLE_SIZE; /* Assume every action is 2 dw */
1016	actions = 1;
1017
1018	/* Add the new header using inline action 4Byte at a time, the header
1019	 * is added in reversed order to the beginning of the packet to avoid
1020	 * incorrect parsing by the HW. Since header is 14B or 18B an extra
1021	 * two bytes are padded and later removed.
1022	 */
1023	for (i = 0; i < data_sz / MLX5HWS_ACTION_INLINE_DATA_SIZE + 1; i++) {
1024		MLX5_SET(stc_ste_param_insert, mh_data, action_type,
1025			 MLX5_MODIFICATION_TYPE_INSERT);
1026		MLX5_SET(stc_ste_param_insert, mh_data, inline_data, 0x1);
1027		MLX5_SET(stc_ste_param_insert, mh_data, insert_anchor,
1028			 MLX5_HEADER_ANCHOR_PACKET_START);
1029		MLX5_SET(stc_ste_param_insert, mh_data, insert_size, 2);
1030		mh_data += MLX5HWS_ACTION_DOUBLE_SIZE;
1031		actions++;
1032	}
1033
1034	/* Remove first 2 extra bytes */
1035	MLX5_SET(stc_ste_param_remove_words, mh_data, action_type,
1036		 MLX5_MODIFICATION_TYPE_REMOVE_WORDS);
1037	MLX5_SET(stc_ste_param_remove_words, mh_data, remove_start_anchor,
1038		 MLX5_HEADER_ANCHOR_PACKET_START);
1039	/* The hardware expects here size in words (2 bytes) */
1040	MLX5_SET(stc_ste_param_remove_words, mh_data, remove_size, 1);
1041	actions++;
1042
1043	*num_of_actions = actions;
1044}
1045
1046static int
1047hws_action_handle_tunnel_l3_to_l2(struct mlx5hws_action *action,
1048				  u8 num_of_hdrs,
1049				  struct mlx5hws_action_reformat_header *hdrs,
1050				  u32 log_bulk_sz)
1051{
1052	u8 mh_data[MLX5HWS_ACTION_REFORMAT_DATA_SIZE] = {0};
1053	struct mlx5hws_context *ctx = action->ctx;
1054	u32 arg_id, pat_id;
1055	int num_of_actions;
1056	int mh_data_size;
1057	int ret, i;
1058
1059	for (i = 0; i < num_of_hdrs; i++) {
1060		if (hdrs[i].sz != MLX5HWS_ACTION_HDR_LEN_L2 &&
1061		    hdrs[i].sz != MLX5HWS_ACTION_HDR_LEN_L2_W_VLAN) {
1062			mlx5hws_err(ctx, "Data size is not supported for decap-l3\n");
1063			return -EINVAL;
1064		}
1065	}
1066
1067	/* Create a full modify header action list in case shared */
1068	hws_action_prepare_decap_l3_actions(hdrs->sz, mh_data, &num_of_actions);
1069	if (action->flags & MLX5HWS_ACTION_FLAG_SHARED)
1070		mlx5hws_action_prepare_decap_l3_data(hdrs->data, mh_data, num_of_actions);
1071
1072	/* All DecapL3 cases require the same max arg size */
1073	ret = mlx5hws_arg_create_modify_header_arg(ctx,
1074						   (__be64 *)mh_data,
1075						   num_of_actions,
1076						   log_bulk_sz,
1077						   action->flags & MLX5HWS_ACTION_FLAG_SHARED,
1078						   &arg_id);
1079	if (ret)
1080		return ret;
1081
1082	for (i = 0; i < num_of_hdrs; i++) {
1083		memset(mh_data, 0, MLX5HWS_ACTION_REFORMAT_DATA_SIZE);
1084		hws_action_prepare_decap_l3_actions(hdrs[i].sz, mh_data, &num_of_actions);
1085		mh_data_size = num_of_actions * MLX5HWS_MODIFY_ACTION_SIZE;
1086
1087		ret = mlx5hws_pat_get_pattern(ctx, (__be64 *)mh_data, mh_data_size, &pat_id);
1088		if (ret) {
1089			mlx5hws_err(ctx, "Failed to allocate pattern for DecapL3\n");
1090			goto free_stc_and_pat;
1091		}
1092
1093		action[i].modify_header.max_num_of_actions = num_of_actions;
1094		action[i].modify_header.num_of_actions = num_of_actions;
1095		action[i].modify_header.num_of_patterns = num_of_hdrs;
1096		action[i].modify_header.arg_id = arg_id;
1097		action[i].modify_header.pat_id = pat_id;
1098		action[i].modify_header.require_reparse =
1099			mlx5hws_pat_require_reparse((__be64 *)mh_data, num_of_actions);
1100
1101		ret = hws_action_create_stcs(&action[i], 0);
1102		if (ret) {
1103			mlx5hws_pat_put_pattern(ctx, pat_id);
1104			goto free_stc_and_pat;
1105		}
1106	}
1107
1108	return 0;
1109
1110free_stc_and_pat:
1111	while (i--) {
1112		hws_action_destroy_stcs(&action[i]);
1113		mlx5hws_pat_put_pattern(ctx, action[i].modify_header.pat_id);
1114	}
1115
1116	mlx5hws_arg_destroy(action->ctx, arg_id);
1117	return ret;
1118}
1119
1120static int
1121hws_action_create_reformat_hws(struct mlx5hws_action *action,
1122			       u8 num_of_hdrs,
1123			       struct mlx5hws_action_reformat_header *hdrs,
1124			       u32 bulk_size)
1125{
1126	int ret;
1127
1128	switch (action->type) {
1129	case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L2_TO_L2:
1130		ret = hws_action_create_stcs(action, 0);
1131		break;
1132	case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L2:
1133		ret = hws_action_handle_insert_with_ptr(action, num_of_hdrs, hdrs, bulk_size);
1134		break;
1135	case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L3:
1136		ret = hws_action_handle_l2_to_tunnel_l3(action, num_of_hdrs, hdrs, bulk_size);
1137		break;
1138	case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L3_TO_L2:
1139		ret = hws_action_handle_tunnel_l3_to_l2(action, num_of_hdrs, hdrs, bulk_size);
1140		break;
1141	default:
1142		mlx5hws_err(action->ctx, "Invalid HWS reformat action type\n");
1143		return -EINVAL;
1144	}
1145
1146	return ret;
1147}
1148
1149struct mlx5hws_action *
1150mlx5hws_action_create_reformat(struct mlx5hws_context *ctx,
1151			       enum mlx5hws_action_type reformat_type,
1152			       u8 num_of_hdrs,
1153			       struct mlx5hws_action_reformat_header *hdrs,
1154			       u32 log_bulk_size,
1155			       u32 flags)
1156{
1157	struct mlx5hws_action *action;
1158	int ret;
1159
1160	if (!num_of_hdrs) {
1161		mlx5hws_err(ctx, "Reformat num_of_hdrs cannot be zero\n");
1162		return NULL;
1163	}
1164
1165	action = hws_action_create_generic_bulk(ctx, flags, reformat_type, num_of_hdrs);
1166	if (!action)
1167		return NULL;
1168
1169	if ((flags & MLX5HWS_ACTION_FLAG_SHARED) && (log_bulk_size || num_of_hdrs > 1)) {
1170		mlx5hws_err(ctx, "Reformat flags don't fit HWS (flags: 0x%x)\n", flags);
1171		goto free_action;
1172	}
1173
1174	ret = hws_action_create_reformat_hws(action, num_of_hdrs, hdrs, log_bulk_size);
1175	if (ret) {
1176		mlx5hws_err(ctx, "Failed to create HWS reformat action\n");
1177		goto free_action;
1178	}
1179
1180	return action;
1181
1182free_action:
1183	kfree(action);
1184	return NULL;
1185}
1186
1187static int
1188hws_action_create_modify_header_hws(struct mlx5hws_action *action,
1189				    u8 num_of_patterns,
1190				    struct mlx5hws_action_mh_pattern *pattern,
1191				    u32 log_bulk_size)
1192{
1193	u16 num_actions, max_mh_actions = 0, hw_max_actions;
1194	struct mlx5hws_context *ctx = action->ctx;
1195	int i, ret, size_in_bytes;
1196	u32 pat_id, arg_id = 0;
1197	__be64 *new_pattern;
1198	size_t pat_max_sz;
1199
1200	pat_max_sz = MLX5HWS_ARG_CHUNK_SIZE_MAX * MLX5HWS_ARG_DATA_SIZE;
1201	hw_max_actions = pat_max_sz / MLX5HWS_MODIFY_ACTION_SIZE;
1202	size_in_bytes = pat_max_sz * sizeof(__be64);
1203	new_pattern = kcalloc(num_of_patterns, size_in_bytes, GFP_KERNEL);
1204	if (!new_pattern)
1205		return -ENOMEM;
1206
1207	/* Calculate maximum number of mh actions for shared arg allocation */
1208	for (i = 0; i < num_of_patterns; i++) {
1209		size_t new_num_actions;
1210		size_t cur_num_actions;
1211		u32 nop_locations;
1212
1213		cur_num_actions = pattern[i].sz / MLX5HWS_MODIFY_ACTION_SIZE;
1214
1215		ret = mlx5hws_pat_calc_nop(pattern[i].data, cur_num_actions,
1216					   hw_max_actions, &new_num_actions,
1217					   &nop_locations,
1218					   &new_pattern[i * pat_max_sz]);
1219		if (ret) {
1220			mlx5hws_err(ctx, "Too many actions after nop insertion\n");
1221			goto free_new_pat;
1222		}
1223
1224		action[i].modify_header.nop_locations = nop_locations;
1225		action[i].modify_header.num_of_actions = new_num_actions;
1226
1227		max_mh_actions = max(max_mh_actions, new_num_actions);
1228	}
1229
1230	if (mlx5hws_arg_get_arg_log_size(max_mh_actions) >= MLX5HWS_ARG_CHUNK_SIZE_MAX) {
1231		mlx5hws_err(ctx, "Num of actions (%d) bigger than allowed\n",
1232			    max_mh_actions);
1233		ret = -EINVAL;
1234		goto free_new_pat;
1235	}
1236
1237	/* Allocate single shared arg for all patterns based on the max size */
1238	if (max_mh_actions > 1) {
1239		ret = mlx5hws_arg_create_modify_header_arg(ctx,
1240							   pattern->data,
1241							   max_mh_actions,
1242							   log_bulk_size,
1243							   action->flags &
1244							   MLX5HWS_ACTION_FLAG_SHARED,
1245							   &arg_id);
1246		if (ret)
1247			goto free_new_pat;
1248	}
1249
1250	for (i = 0; i < num_of_patterns; i++) {
1251		if (!mlx5hws_pat_verify_actions(ctx, pattern[i].data, pattern[i].sz)) {
1252			mlx5hws_err(ctx, "Fail to verify pattern modify actions\n");
1253			ret = -EINVAL;
1254			goto free_stc_and_pat;
1255		}
1256		num_actions = pattern[i].sz / MLX5HWS_MODIFY_ACTION_SIZE;
1257		action[i].modify_header.num_of_patterns = num_of_patterns;
1258		action[i].modify_header.max_num_of_actions = max_mh_actions;
1259
1260		action[i].modify_header.require_reparse =
1261			mlx5hws_pat_require_reparse(pattern[i].data, num_actions);
1262
1263		if (num_actions == 1) {
1264			pat_id = 0;
1265			/* Optimize single modify action to be used inline */
1266			action[i].modify_header.single_action = pattern[i].data[0];
1267			action[i].modify_header.single_action_type =
1268				MLX5_GET(set_action_in, pattern[i].data, action_type);
1269		} else {
1270			/* Multiple modify actions require a pattern */
1271			if (unlikely(action[i].modify_header.nop_locations)) {
1272				size_t pattern_sz;
1273
1274				pattern_sz = action[i].modify_header.num_of_actions *
1275					     MLX5HWS_MODIFY_ACTION_SIZE;
1276				ret =
1277				mlx5hws_pat_get_pattern(ctx,
1278							&new_pattern[i * pat_max_sz],
1279							pattern_sz, &pat_id);
1280			} else {
1281				ret = mlx5hws_pat_get_pattern(ctx,
1282							      pattern[i].data,
1283							      pattern[i].sz,
1284							      &pat_id);
1285			}
1286			if (ret) {
1287				mlx5hws_err(ctx,
1288					    "Failed to allocate pattern for modify header\n");
1289				goto free_stc_and_pat;
1290			}
1291
1292			action[i].modify_header.arg_id = arg_id;
1293			action[i].modify_header.pat_id = pat_id;
1294		}
1295		/* Allocate STC for each action representing a header */
1296		ret = hws_action_create_stcs(&action[i], 0);
1297		if (ret) {
1298			if (pat_id)
1299				mlx5hws_pat_put_pattern(ctx, pat_id);
1300			goto free_stc_and_pat;
1301		}
1302	}
1303
1304	kfree(new_pattern);
1305	return 0;
1306
1307free_stc_and_pat:
1308	while (i--) {
1309		hws_action_destroy_stcs(&action[i]);
1310		if (action[i].modify_header.pat_id)
1311			mlx5hws_pat_put_pattern(ctx, action[i].modify_header.pat_id);
1312	}
1313
1314	if (arg_id)
1315		mlx5hws_arg_destroy(ctx, arg_id);
1316free_new_pat:
1317	kfree(new_pattern);
1318	return ret;
1319}
1320
1321struct mlx5hws_action *
1322mlx5hws_action_create_modify_header(struct mlx5hws_context *ctx,
1323				    u8 num_of_patterns,
1324				    struct mlx5hws_action_mh_pattern *patterns,
1325				    u32 log_bulk_size,
1326				    u32 flags)
1327{
1328	struct mlx5hws_action *action;
1329	int ret;
1330
1331	if (!num_of_patterns) {
1332		mlx5hws_err(ctx, "Invalid number of patterns\n");
1333		return NULL;
1334	}
1335	action = hws_action_create_generic_bulk(ctx, flags,
1336						MLX5HWS_ACTION_TYP_MODIFY_HDR,
1337						num_of_patterns);
1338	if (!action)
1339		return NULL;
1340
1341	if ((flags & MLX5HWS_ACTION_FLAG_SHARED) && (log_bulk_size || num_of_patterns > 1)) {
1342		mlx5hws_err(ctx, "Action cannot be shared with requested pattern or size\n");
1343		goto free_action;
1344	}
1345
1346	ret = hws_action_create_modify_header_hws(action,
1347						  num_of_patterns,
1348						  patterns,
1349						  log_bulk_size);
1350	if (ret)
1351		goto free_action;
1352
1353	return action;
1354
1355free_action:
1356	kfree(action);
1357	return NULL;
1358}
1359
1360struct mlx5hws_action *
1361mlx5hws_action_create_dest_array(struct mlx5hws_context *ctx, size_t num_dest,
1362				 struct mlx5hws_action_dest_attr *dests,
1363				 u32 flags)
1364{
1365	struct mlx5hws_cmd_set_fte_dest *dest_list = NULL;
1366	struct mlx5hws_cmd_ft_create_attr ft_attr = {0};
1367	struct mlx5hws_cmd_set_fte_attr fte_attr = {0};
1368	struct mlx5hws_cmd_forward_tbl *fw_island;
1369	struct mlx5hws_action *action;
1370	int ret, last_dest_idx = -1;
1371	u32 i;
1372
1373	if (num_dest <= 1) {
1374		mlx5hws_err(ctx, "Action must have multiple dests\n");
1375		return NULL;
1376	}
1377
1378	if (flags == (MLX5HWS_ACTION_FLAG_HWS_FDB | MLX5HWS_ACTION_FLAG_SHARED)) {
1379		ft_attr.type = FS_FT_FDB;
1380		ft_attr.level = ctx->caps->fdb_ft.max_level - 1;
1381	} else {
1382		mlx5hws_err(ctx, "Action flags not supported\n");
1383		return NULL;
1384	}
1385
1386	dest_list = kcalloc(num_dest, sizeof(*dest_list), GFP_KERNEL);
1387	if (!dest_list)
1388		return NULL;
1389
1390	for (i = 0; i < num_dest; i++) {
1391		enum mlx5hws_action_type action_type = dests[i].dest->type;
1392		struct mlx5hws_action *reformat_action = dests[i].reformat;
1393
1394		switch (action_type) {
1395		case MLX5HWS_ACTION_TYP_TBL:
1396			dest_list[i].destination_type =
1397				MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1398			dest_list[i].destination_id = dests[i].dest->dest_obj.obj_id;
1399			fte_attr.action_flags |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1400			fte_attr.ignore_flow_level = 1;
1401			if (dests[i].is_wire_ft)
1402				last_dest_idx = i;
1403			break;
1404		case MLX5HWS_ACTION_TYP_VPORT:
1405			dest_list[i].destination_type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1406			dest_list[i].destination_id = dests[i].dest->vport.vport_num;
1407			fte_attr.action_flags |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1408			if (ctx->caps->merged_eswitch) {
1409				dest_list[i].ext_flags |=
1410					MLX5HWS_CMD_EXT_DEST_ESW_OWNER_VHCA_ID;
1411				dest_list[i].esw_owner_vhca_id =
1412					dests[i].dest->vport.esw_owner_vhca_id;
1413			}
1414			break;
1415		default:
1416			mlx5hws_err(ctx, "Unsupported action in dest_array\n");
1417			goto free_dest_list;
1418		}
1419
1420		if (reformat_action) {
1421			mlx5hws_err(ctx, "dest_array with reformat action - unsupported\n");
1422			goto free_dest_list;
1423		}
1424	}
1425
1426	if (last_dest_idx != -1)
1427		swap(dest_list[last_dest_idx], dest_list[num_dest - 1]);
1428
1429	fte_attr.dests_num = num_dest;
1430	fte_attr.dests = dest_list;
1431
1432	fw_island = mlx5hws_cmd_forward_tbl_create(ctx->mdev, &ft_attr, &fte_attr);
1433	if (!fw_island)
1434		goto free_dest_list;
1435
1436	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_DEST_ARRAY);
1437	if (!action)
1438		goto destroy_fw_island;
1439
1440	ret = hws_action_create_stcs(action, fw_island->ft_id);
1441	if (ret)
1442		goto free_action;
1443
1444	action->dest_array.fw_island = fw_island;
1445	action->dest_array.num_dest = num_dest;
1446	action->dest_array.dest_list = dest_list;
1447
1448	return action;
1449
1450free_action:
1451	kfree(action);
1452destroy_fw_island:
1453	mlx5hws_cmd_forward_tbl_destroy(ctx->mdev, fw_island);
1454free_dest_list:
1455	for (i = 0; i < num_dest; i++) {
1456		if (dest_list[i].ext_reformat_id)
1457			mlx5hws_cmd_packet_reformat_destroy(ctx->mdev,
1458							    dest_list[i].ext_reformat_id);
1459	}
1460	kfree(dest_list);
1461	return NULL;
1462}
1463
1464struct mlx5hws_action *
1465mlx5hws_action_create_insert_header(struct mlx5hws_context *ctx,
1466				    u8 num_of_hdrs,
1467				    struct mlx5hws_action_insert_header *hdrs,
1468				    u32 log_bulk_size,
1469				    u32 flags)
1470{
1471	struct mlx5hws_action_reformat_header *reformat_hdrs;
1472	struct mlx5hws_action *action;
1473	int ret;
1474	int i;
1475
1476	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_INSERT_HEADER);
1477	if (!action)
1478		return NULL;
1479
1480	reformat_hdrs = kcalloc(num_of_hdrs, sizeof(*reformat_hdrs), GFP_KERNEL);
1481	if (!reformat_hdrs)
1482		goto free_action;
1483
1484	for (i = 0; i < num_of_hdrs; i++) {
1485		if (hdrs[i].offset % W_SIZE != 0) {
1486			mlx5hws_err(ctx, "Header offset should be in WORD granularity\n");
1487			goto free_reformat_hdrs;
1488		}
1489
1490		action[i].reformat.anchor = hdrs[i].anchor;
1491		action[i].reformat.encap = hdrs[i].encap;
1492		action[i].reformat.offset = hdrs[i].offset;
1493
1494		reformat_hdrs[i].sz = hdrs[i].hdr.sz;
1495		reformat_hdrs[i].data = hdrs[i].hdr.data;
1496	}
1497
1498	ret = hws_action_handle_insert_with_ptr(action, num_of_hdrs,
1499						reformat_hdrs, log_bulk_size);
1500	if (ret) {
1501		mlx5hws_err(ctx, "Failed to create HWS reformat action\n");
1502		goto free_reformat_hdrs;
1503	}
1504
1505	kfree(reformat_hdrs);
1506
1507	return action;
1508
1509free_reformat_hdrs:
1510	kfree(reformat_hdrs);
1511free_action:
1512	kfree(action);
1513	return NULL;
1514}
1515
1516struct mlx5hws_action *
1517mlx5hws_action_create_remove_header(struct mlx5hws_context *ctx,
1518				    struct mlx5hws_action_remove_header_attr *attr,
1519				    u32 flags)
1520{
1521	struct mlx5hws_action *action;
1522
1523	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_REMOVE_HEADER);
1524	if (!action)
1525		return NULL;
1526
1527	/* support only remove anchor with size */
1528	if (attr->size % W_SIZE != 0) {
1529		mlx5hws_err(ctx,
1530			    "Invalid size, HW supports header remove in WORD granularity\n");
1531		goto free_action;
1532	}
1533
1534	if (attr->size > MLX5HWS_ACTION_REMOVE_HEADER_MAX_SIZE) {
1535		mlx5hws_err(ctx, "Header removal size limited to %u bytes\n",
1536			    MLX5HWS_ACTION_REMOVE_HEADER_MAX_SIZE);
1537		goto free_action;
1538	}
1539
1540	action->remove_header.anchor = attr->anchor;
1541	action->remove_header.size = attr->size / W_SIZE;
1542
1543	if (hws_action_create_stcs(action, 0))
1544		goto free_action;
1545
1546	return action;
1547
1548free_action:
1549	kfree(action);
1550	return NULL;
1551}
1552
1553static struct mlx5hws_definer *
1554hws_action_create_dest_match_range_definer(struct mlx5hws_context *ctx)
1555{
1556	struct mlx5hws_definer *definer;
1557	__be32 *tag;
1558	int ret;
1559
1560	definer = kzalloc(sizeof(*definer), GFP_KERNEL);
1561	if (!definer)
1562		return NULL;
1563
1564	definer->dw_selector[0] = MLX5_IFC_DEFINER_FORMAT_OFFSET_OUTER_ETH_PKT_LEN / 4;
1565	/* Set DW0 tag mask */
1566	tag = (__force __be32 *)definer->mask.jumbo;
1567	tag[MLX5HWS_RULE_JUMBO_MATCH_TAG_OFFSET_DW0] = htonl(0xffffUL << 16);
1568
1569	mutex_lock(&ctx->ctrl_lock);
1570
1571	ret = mlx5hws_definer_get_obj(ctx, definer);
1572	if (ret < 0) {
1573		mutex_unlock(&ctx->ctrl_lock);
1574		kfree(definer);
1575		return NULL;
1576	}
1577
1578	mutex_unlock(&ctx->ctrl_lock);
1579	definer->obj_id = ret;
1580
1581	return definer;
1582}
1583
1584static struct mlx5hws_range_action_table *
1585hws_action_create_dest_match_range_table(struct mlx5hws_context *ctx,
1586					 struct mlx5hws_definer *definer,
1587					 u32 miss_ft_id)
1588{
1589	struct mlx5hws_cmd_rtc_create_attr rtc_attr = {0};
1590	struct mlx5hws_range_action_table *table_ste;
1591	struct mlx5hws_pool_attr pool_attr = {0};
1592	struct mlx5hws_pool *ste_pool, *stc_pool;
1593	u32 *rtc_0_id, *rtc_1_id;
1594	u32 obj_id;
1595	int ret;
1596
1597	/* Check if STE range is supported */
1598	if (!IS_BIT_SET(ctx->caps->supp_ste_format_gen_wqe, MLX5_IFC_RTC_STE_FORMAT_RANGE)) {
1599		mlx5hws_err(ctx, "Range STE format not supported\n");
1600		return NULL;
1601	}
1602
1603	table_ste = kzalloc(sizeof(*table_ste), GFP_KERNEL);
1604	if (!table_ste)
1605		return NULL;
1606
1607	mutex_lock(&ctx->ctrl_lock);
1608
1609	pool_attr.table_type = MLX5HWS_TABLE_TYPE_FDB;
1610	pool_attr.pool_type = MLX5HWS_POOL_TYPE_STE;
1611	pool_attr.alloc_log_sz = 1;
1612	table_ste->pool = mlx5hws_pool_create(ctx, &pool_attr);
1613	if (!table_ste->pool) {
1614		mlx5hws_err(ctx, "Failed to allocate memory ste pool\n");
1615		goto free_ste;
1616	}
1617
1618	/* Allocate RTC */
1619	rtc_0_id = &table_ste->rtc_0_id;
1620	rtc_1_id = &table_ste->rtc_1_id;
1621	ste_pool = table_ste->pool;
1622
1623	rtc_attr.log_size = 0;
1624	rtc_attr.log_depth = 0;
1625	rtc_attr.miss_ft_id = miss_ft_id;
1626	rtc_attr.num_hash_definer = 1;
1627	rtc_attr.update_index_mode = MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH;
1628	rtc_attr.access_index_mode = MLX5_IFC_RTC_STE_ACCESS_MODE_BY_HASH;
1629	rtc_attr.match_definer_0 = ctx->caps->trivial_match_definer;
1630	rtc_attr.fw_gen_wqe = true;
1631	rtc_attr.is_scnd_range = true;
1632
1633	obj_id = mlx5hws_pool_get_base_id(ste_pool);
1634
1635	rtc_attr.pd = ctx->pd_num;
1636	rtc_attr.ste_base = obj_id;
1637	rtc_attr.reparse_mode = mlx5hws_context_get_reparse_mode(ctx);
1638	rtc_attr.table_type = mlx5hws_table_get_res_fw_ft_type(MLX5HWS_TABLE_TYPE_FDB, false);
1639
1640	/* STC is a single resource (obj_id), use any STC for the ID */
1641	stc_pool = ctx->stc_pool;
1642	obj_id = mlx5hws_pool_get_base_id(stc_pool);
1643	rtc_attr.stc_base = obj_id;
1644
1645	ret = mlx5hws_cmd_rtc_create(ctx->mdev, &rtc_attr, rtc_0_id);
1646	if (ret) {
1647		mlx5hws_err(ctx, "Failed to create RTC");
1648		goto pool_destroy;
1649	}
1650
1651	/* Create mirror RTC */
1652	obj_id = mlx5hws_pool_get_base_mirror_id(ste_pool);
1653	rtc_attr.ste_base = obj_id;
1654	rtc_attr.table_type = mlx5hws_table_get_res_fw_ft_type(MLX5HWS_TABLE_TYPE_FDB, true);
1655
1656	obj_id = mlx5hws_pool_get_base_mirror_id(stc_pool);
1657	rtc_attr.stc_base = obj_id;
1658
1659	ret = mlx5hws_cmd_rtc_create(ctx->mdev, &rtc_attr, rtc_1_id);
1660	if (ret) {
1661		mlx5hws_err(ctx, "Failed to create mirror RTC");
1662		goto destroy_rtc_0;
1663	}
1664
1665	mutex_unlock(&ctx->ctrl_lock);
1666
1667	return table_ste;
1668
1669destroy_rtc_0:
1670	mlx5hws_cmd_rtc_destroy(ctx->mdev, *rtc_0_id);
1671pool_destroy:
1672	mlx5hws_pool_destroy(table_ste->pool);
1673free_ste:
1674	mutex_unlock(&ctx->ctrl_lock);
1675	kfree(table_ste);
1676	return NULL;
1677}
1678
1679static void hws_action_destroy_dest_match_range_table(
1680	struct mlx5hws_context *ctx,
1681	struct mlx5hws_range_action_table *table_ste)
1682{
1683	mutex_lock(&ctx->ctrl_lock);
1684
1685	mlx5hws_cmd_rtc_destroy(ctx->mdev, table_ste->rtc_1_id);
1686	mlx5hws_cmd_rtc_destroy(ctx->mdev, table_ste->rtc_0_id);
1687	mlx5hws_pool_destroy(table_ste->pool);
1688	kfree(table_ste);
1689
1690	mutex_unlock(&ctx->ctrl_lock);
1691}
1692
1693static int hws_action_create_dest_match_range_fill_table(
1694	struct mlx5hws_context *ctx,
1695	struct mlx5hws_range_action_table *table_ste,
1696	struct mlx5hws_action *hit_ft_action,
1697	struct mlx5hws_definer *range_definer, u32 min, u32 max)
1698{
1699	struct mlx5hws_wqe_gta_data_seg_ste match_wqe_data = {0};
1700	struct mlx5hws_wqe_gta_data_seg_ste range_wqe_data = {0};
1701	struct mlx5hws_wqe_gta_ctrl_seg wqe_ctrl = {0};
1702	u32 no_use, used_rtc_0_id, used_rtc_1_id, ret;
1703	struct mlx5hws_context_common_res *common_res;
1704	struct mlx5hws_send_ste_attr ste_attr = {0};
1705	struct mlx5hws_send_engine *queue;
1706	__be32 *wqe_data_arr;
1707
1708	mutex_lock(&ctx->ctrl_lock);
1709
1710	/* Get the control queue */
1711	queue = &ctx->send_queue[ctx->queues - 1];
1712	if (unlikely(mlx5hws_send_engine_err(queue))) {
1713		ret = -EIO;
1714		goto error;
1715	}
1716
1717	/* Init default send STE attributes */
1718	ste_attr.gta_opcode = MLX5HWS_WQE_GTA_OP_ACTIVATE;
1719	ste_attr.send_attr.opmod = MLX5HWS_WQE_GTA_OPMOD_STE;
1720	ste_attr.send_attr.opcode = MLX5HWS_WQE_OPCODE_TBL_ACCESS;
1721	ste_attr.send_attr.len = MLX5HWS_WQE_SZ_GTA_CTRL + MLX5HWS_WQE_SZ_GTA_DATA;
1722	ste_attr.send_attr.user_data = &no_use;
1723	ste_attr.send_attr.rule = NULL;
1724	ste_attr.send_attr.fence = 1;
1725	ste_attr.send_attr.notify_hw = true;
1726	ste_attr.rtc_0 = table_ste->rtc_0_id;
1727	ste_attr.rtc_1 = table_ste->rtc_1_id;
1728	ste_attr.used_id_rtc_0 = &used_rtc_0_id;
1729	ste_attr.used_id_rtc_1 = &used_rtc_1_id;
1730
1731	common_res = &ctx->common_res;
1732
1733	/* init an empty match STE which will always hit */
1734	ste_attr.wqe_ctrl = &wqe_ctrl;
1735	ste_attr.wqe_data = &match_wqe_data;
1736	ste_attr.send_attr.match_definer_id = ctx->caps->trivial_match_definer;
1737
1738	/* Fill WQE control data */
1739	wqe_ctrl.stc_ix[MLX5HWS_ACTION_STC_IDX_CTRL] =
1740		htonl(common_res->default_stc->nop_ctr.offset);
1741	wqe_ctrl.stc_ix[MLX5HWS_ACTION_STC_IDX_DW5] =
1742		htonl(common_res->default_stc->nop_dw5.offset);
1743	wqe_ctrl.stc_ix[MLX5HWS_ACTION_STC_IDX_DW6] =
1744		htonl(common_res->default_stc->nop_dw6.offset);
1745	wqe_ctrl.stc_ix[MLX5HWS_ACTION_STC_IDX_DW7] =
1746		htonl(common_res->default_stc->nop_dw7.offset);
1747	wqe_ctrl.stc_ix[MLX5HWS_ACTION_STC_IDX_CTRL] |=
1748		htonl(MLX5HWS_ACTION_STC_IDX_LAST_COMBO2 << 29);
1749	wqe_ctrl.stc_ix[MLX5HWS_ACTION_STC_IDX_HIT] =
1750		htonl(hit_ft_action->stc.offset);
1751
1752	wqe_data_arr = (__force __be32 *)&range_wqe_data;
1753
1754	ste_attr.range_wqe_data = &range_wqe_data;
1755	ste_attr.send_attr.len += MLX5HWS_WQE_SZ_GTA_DATA;
1756	ste_attr.send_attr.range_definer_id = mlx5hws_definer_get_id(range_definer);
1757
1758	/* Fill range matching fields,
1759	 * min/max_value_2 corresponds to match_dw_0 in its definer,
1760	 * min_value_2 sets in DW0 in the STE and max_value_2 sets in DW1 in the STE.
1761	 */
1762	wqe_data_arr[MLX5HWS_MATCHER_OFFSET_TAG_DW0] = htonl(min << 16);
1763	wqe_data_arr[MLX5HWS_MATCHER_OFFSET_TAG_DW1] = htonl(max << 16);
1764
1765	/* Send WQEs to FW */
1766	mlx5hws_send_stes_fw(ctx, queue, &ste_attr);
1767
1768	/* Poll for completion */
1769	ret = mlx5hws_send_queue_action(ctx, ctx->queues - 1,
1770					MLX5HWS_SEND_QUEUE_ACTION_DRAIN_SYNC);
1771	if (ret) {
1772		mlx5hws_err(ctx, "Failed to drain control queue");
1773		goto error;
1774	}
1775
1776	mutex_unlock(&ctx->ctrl_lock);
1777
1778	return 0;
1779
1780error:
1781	mutex_unlock(&ctx->ctrl_lock);
1782	return ret;
1783}
1784
1785struct mlx5hws_action *
1786mlx5hws_action_create_dest_match_range(struct mlx5hws_context *ctx,
1787				       u32 field,
1788				       struct mlx5_flow_table *hit_ft,
1789				       struct mlx5_flow_table *miss_ft,
1790				       u32 min, u32 max, u32 flags)
1791{
1792	struct mlx5hws_cmd_stc_modify_attr stc_attr = {0};
1793	struct mlx5hws_range_action_table *table_ste;
1794	struct mlx5hws_action *hit_ft_action;
1795	struct mlx5hws_definer *definer;
1796	struct mlx5hws_action *action;
1797	u32 miss_ft_id = miss_ft->id;
1798	u32 hit_ft_id = hit_ft->id;
1799	int ret;
1800
1801	if (field != MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN ||
1802	    min > 0xffff || max > 0xffff) {
1803		mlx5hws_err(ctx, "Invalid match range parameters\n");
1804		return NULL;
1805	}
1806
1807	action = hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_RANGE);
1808	if (!action)
1809		return NULL;
1810
1811	definer = hws_action_create_dest_match_range_definer(ctx);
1812	if (!definer)
1813		goto free_action;
1814
1815	table_ste = hws_action_create_dest_match_range_table(ctx, definer, miss_ft_id);
1816	if (!table_ste)
1817		goto destroy_definer;
1818
1819	hit_ft_action = mlx5hws_action_create_dest_table_num(ctx, hit_ft_id, flags);
1820	if (!hit_ft_action)
1821		goto destroy_table_ste;
1822
1823	ret = hws_action_create_dest_match_range_fill_table(ctx, table_ste,
1824							    hit_ft_action,
1825							    definer, min, max);
1826	if (ret)
1827		goto destroy_hit_ft_action;
1828
1829	action->range.table_ste = table_ste;
1830	action->range.definer = definer;
1831	action->range.hit_ft_action = hit_ft_action;
1832
1833	/* Allocate STC for jumps to STE */
1834	mutex_lock(&ctx->ctrl_lock);
1835	stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_HIT;
1836	stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_JUMP_TO_STE_TABLE;
1837	stc_attr.reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
1838	stc_attr.ste_table.ste_pool = table_ste->pool;
1839	stc_attr.ste_table.match_definer_id = ctx->caps->trivial_match_definer;
1840
1841	ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, MLX5HWS_TABLE_TYPE_FDB,
1842					      &action->stc);
1843	if (ret)
1844		goto error_unlock;
1845
1846	mutex_unlock(&ctx->ctrl_lock);
1847
1848	return action;
1849
1850error_unlock:
1851	mutex_unlock(&ctx->ctrl_lock);
1852destroy_hit_ft_action:
1853	mlx5hws_action_destroy(hit_ft_action);
1854destroy_table_ste:
1855	hws_action_destroy_dest_match_range_table(ctx, table_ste);
1856destroy_definer:
1857	mlx5hws_definer_free(ctx, definer);
1858free_action:
1859	kfree(action);
1860	mlx5hws_err(ctx, "Failed to create action dest match range");
1861	return NULL;
1862}
1863
1864struct mlx5hws_action *
1865mlx5hws_action_create_last(struct mlx5hws_context *ctx, u32 flags)
1866{
1867	return hws_action_create_generic(ctx, flags, MLX5HWS_ACTION_TYP_LAST);
1868}
1869
1870struct mlx5hws_action *
1871mlx5hws_action_create_flow_sampler(struct mlx5hws_context *ctx,
1872				   u32 sampler_id, u32 flags)
1873{
1874	struct mlx5hws_cmd_ft_create_attr ft_attr = {0};
1875	struct mlx5hws_cmd_set_fte_attr fte_attr = {0};
1876	struct mlx5hws_cmd_forward_tbl *fw_island;
1877	struct mlx5hws_cmd_set_fte_dest dest;
1878	struct mlx5hws_action *action;
1879	int ret;
1880
1881	if (flags != (MLX5HWS_ACTION_FLAG_HWS_FDB | MLX5HWS_ACTION_FLAG_SHARED)) {
1882		mlx5hws_err(ctx, "Unsupported flags for flow sampler\n");
1883		return NULL;
1884	}
1885
1886	ft_attr.type = FS_FT_FDB;
1887	ft_attr.level = ctx->caps->fdb_ft.max_level - 1;
1888
1889	dest.destination_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
1890	dest.destination_id = sampler_id;
1891
1892	fte_attr.dests_num = 1;
1893	fte_attr.dests = &dest;
1894	fte_attr.action_flags = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1895	fte_attr.ignore_flow_level = 1;
1896
1897	fw_island = mlx5hws_cmd_forward_tbl_create(ctx->mdev, &ft_attr, &fte_attr);
1898	if (!fw_island)
1899		return NULL;
1900
1901	action = hws_action_create_generic(ctx, flags,
1902					   MLX5HWS_ACTION_TYP_SAMPLER);
1903	if (!action)
1904		goto destroy_fw_island;
1905
1906	ret = hws_action_create_stcs(action, fw_island->ft_id);
1907	if (ret)
1908		goto free_action;
1909
1910	action->flow_sampler.fw_island = fw_island;
1911
1912	return action;
1913
1914free_action:
1915	kfree(action);
1916destroy_fw_island:
1917	mlx5hws_cmd_forward_tbl_destroy(ctx->mdev, fw_island);
1918	return NULL;
1919}
1920
1921static void hws_action_destroy_hws(struct mlx5hws_action *action)
1922{
1923	u32 ext_reformat_id;
1924	bool shared_arg;
1925	u32 obj_id;
1926	u32 i;
1927
1928	switch (action->type) {
1929	case MLX5HWS_ACTION_TYP_MISS:
1930	case MLX5HWS_ACTION_TYP_TAG:
1931	case MLX5HWS_ACTION_TYP_DROP:
1932	case MLX5HWS_ACTION_TYP_CTR:
1933	case MLX5HWS_ACTION_TYP_TBL:
1934	case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L2_TO_L2:
1935	case MLX5HWS_ACTION_TYP_ASO_METER:
1936	case MLX5HWS_ACTION_TYP_PUSH_VLAN:
1937	case MLX5HWS_ACTION_TYP_REMOVE_HEADER:
1938	case MLX5HWS_ACTION_TYP_VPORT:
1939		hws_action_destroy_stcs(action);
1940		break;
1941	case MLX5HWS_ACTION_TYP_POP_VLAN:
1942		hws_action_destroy_stcs(action);
1943		hws_action_put_shared_stc(action, MLX5HWS_CONTEXT_SHARED_STC_DOUBLE_POP);
1944		break;
1945	case MLX5HWS_ACTION_TYP_DEST_ARRAY:
1946		hws_action_destroy_stcs(action);
1947		mlx5hws_cmd_forward_tbl_destroy(action->ctx->mdev, action->dest_array.fw_island);
1948		for (i = 0; i < action->dest_array.num_dest; i++) {
1949			ext_reformat_id = action->dest_array.dest_list[i].ext_reformat_id;
1950			if (ext_reformat_id)
1951				mlx5hws_cmd_packet_reformat_destroy(action->ctx->mdev,
1952								    ext_reformat_id);
1953		}
1954		kfree(action->dest_array.dest_list);
1955		break;
1956	case MLX5HWS_ACTION_TYP_SAMPLER:
1957		hws_action_destroy_stcs(action);
1958		mlx5hws_cmd_forward_tbl_destroy(action->ctx->mdev,
1959						action->flow_sampler.fw_island);
1960		break;
1961	case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L3_TO_L2:
1962	case MLX5HWS_ACTION_TYP_MODIFY_HDR:
1963		shared_arg = false;
1964		for (i = 0; i < action->modify_header.num_of_patterns; i++) {
1965			hws_action_destroy_stcs(&action[i]);
1966			if (action[i].modify_header.num_of_actions > 1) {
1967				mlx5hws_pat_put_pattern(action[i].ctx,
1968							action[i].modify_header.pat_id);
1969				/* Save shared arg object to be freed after */
1970				obj_id = action[i].modify_header.arg_id;
1971				shared_arg = true;
1972			}
1973		}
1974		if (shared_arg)
1975			mlx5hws_arg_destroy(action->ctx, obj_id);
1976		break;
1977	case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L3:
1978		hws_action_put_shared_stc(action, MLX5HWS_CONTEXT_SHARED_STC_DECAP_L3);
1979		for (i = 0; i < action->reformat.num_of_hdrs; i++)
1980			hws_action_destroy_stcs(&action[i]);
1981		mlx5hws_arg_destroy(action->ctx, action->reformat.arg_id);
1982		break;
1983	case MLX5HWS_ACTION_TYP_INSERT_HEADER:
1984	case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L2:
1985		for (i = 0; i < action->reformat.num_of_hdrs; i++)
1986			hws_action_destroy_stcs(&action[i]);
1987		mlx5hws_arg_destroy(action->ctx, action->reformat.arg_id);
1988		break;
1989	case MLX5HWS_ACTION_TYP_RANGE:
1990		hws_action_destroy_stcs(action);
1991		hws_action_destroy_dest_match_range_table(action->ctx, action->range.table_ste);
1992		mlx5hws_definer_free(action->ctx, action->range.definer);
1993		mlx5hws_action_destroy(action->range.hit_ft_action);
1994		break;
1995	case MLX5HWS_ACTION_TYP_LAST:
1996		break;
1997	default:
1998		pr_warn("HWS: Invalid action type: %d\n", action->type);
1999	}
2000}
2001
2002int mlx5hws_action_destroy(struct mlx5hws_action *action)
2003{
2004	hws_action_destroy_hws(action);
2005
2006	kfree(action);
2007	return 0;
2008}
2009
2010int mlx5hws_action_get_default_stc(struct mlx5hws_context *ctx, u8 tbl_type)
2011__must_hold(&ctx->ctrl_lock)
2012{
2013	struct mlx5hws_cmd_stc_modify_attr stc_attr = {0};
2014	struct mlx5hws_action_default_stc *default_stc;
2015	int ret;
2016
2017	if (ctx->common_res.default_stc) {
2018		ctx->common_res.default_stc->refcount++;
2019		return 0;
2020	}
2021
2022	default_stc = kzalloc(sizeof(*default_stc), GFP_KERNEL);
2023	if (!default_stc)
2024		return -ENOMEM;
2025
2026	stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_NOP;
2027	stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_DW0;
2028	stc_attr.reparse_mode = MLX5_IFC_STC_REPARSE_IGNORE;
2029	ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, tbl_type,
2030					      &default_stc->nop_ctr);
2031	if (ret) {
2032		mlx5hws_err(ctx, "Failed to allocate default counter STC\n");
2033		goto free_default_stc;
2034	}
2035
2036	stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_DW5;
2037	ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, tbl_type,
2038					      &default_stc->nop_dw5);
2039	if (ret) {
2040		mlx5hws_err(ctx, "Failed to allocate default NOP DW5 STC\n");
2041		goto free_nop_ctr;
2042	}
2043
2044	stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_DW6;
2045	ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, tbl_type,
2046					      &default_stc->nop_dw6);
2047	if (ret) {
2048		mlx5hws_err(ctx, "Failed to allocate default NOP DW6 STC\n");
2049		goto free_nop_dw5;
2050	}
2051
2052	stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_DW7;
2053	ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, tbl_type,
2054					      &default_stc->nop_dw7);
2055	if (ret) {
2056		mlx5hws_err(ctx, "Failed to allocate default NOP DW7 STC\n");
2057		goto free_nop_dw6;
2058	}
2059
2060	stc_attr.action_offset = MLX5HWS_ACTION_OFFSET_HIT;
2061	stc_attr.action_type = MLX5_IFC_STC_ACTION_TYPE_ALLOW;
2062
2063	ret = mlx5hws_action_alloc_single_stc(ctx, &stc_attr, tbl_type,
2064					      &default_stc->default_hit);
2065	if (ret) {
2066		mlx5hws_err(ctx, "Failed to allocate default allow STC\n");
2067		goto free_nop_dw7;
2068	}
2069
2070	ctx->common_res.default_stc = default_stc;
2071	ctx->common_res.default_stc->refcount++;
2072
2073	return 0;
2074
2075free_nop_dw7:
2076	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_dw7);
2077free_nop_dw6:
2078	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_dw6);
2079free_nop_dw5:
2080	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_dw5);
2081free_nop_ctr:
2082	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_ctr);
2083free_default_stc:
2084	kfree(default_stc);
2085	return ret;
2086}
2087
2088void mlx5hws_action_put_default_stc(struct mlx5hws_context *ctx, u8 tbl_type)
2089__must_hold(&ctx->ctrl_lock)
2090{
2091	struct mlx5hws_action_default_stc *default_stc;
2092
2093	default_stc = ctx->common_res.default_stc;
2094	if (--default_stc->refcount)
2095		return;
2096
2097	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->default_hit);
2098	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_dw7);
2099	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_dw6);
2100	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_dw5);
2101	mlx5hws_action_free_single_stc(ctx, tbl_type, &default_stc->nop_ctr);
2102	kfree(default_stc);
2103	ctx->common_res.default_stc = NULL;
2104}
2105
2106static void hws_action_modify_write(struct mlx5hws_send_engine *queue,
2107				    u32 arg_idx,
2108				    u8 *arg_data,
2109				    u16 num_of_actions,
2110				    u32 nop_locations)
2111{
2112	u8 *new_arg_data = NULL;
2113	int i, j;
2114
2115	if (unlikely(nop_locations)) {
2116		new_arg_data = kcalloc(num_of_actions,
2117				       MLX5HWS_MODIFY_ACTION_SIZE, GFP_KERNEL);
2118		if (unlikely(!new_arg_data))
2119			return;
2120
2121		for (i = 0, j = 0; j < num_of_actions; i++, j++) {
2122			if (BIT(i) & nop_locations)
2123				j++;
2124			memcpy(&new_arg_data[j * MLX5HWS_MODIFY_ACTION_SIZE],
2125			       &arg_data[i * MLX5HWS_MODIFY_ACTION_SIZE],
2126			       MLX5HWS_MODIFY_ACTION_SIZE);
2127		}
2128	}
2129
2130	mlx5hws_arg_write(queue, NULL, arg_idx,
2131			  new_arg_data ? new_arg_data : arg_data,
2132			  num_of_actions * MLX5HWS_MODIFY_ACTION_SIZE);
2133
2134	kfree(new_arg_data);
2135}
2136
2137void mlx5hws_action_prepare_decap_l3_data(u8 *src, u8 *dst, u16 num_of_actions)
2138{
2139	u8 *e_src;
2140	int i;
2141
2142	/* num_of_actions = remove l3l2 + 4/5 inserts + remove extra 2 bytes
2143	 * copy from end of src to the start of dst.
2144	 * move to the end, 2 is the leftover from 14B or 18B
2145	 */
2146	if (num_of_actions == DECAP_L3_NUM_ACTIONS_W_NO_VLAN)
2147		e_src = src + MLX5HWS_ACTION_HDR_LEN_L2;
2148	else
2149		e_src = src + MLX5HWS_ACTION_HDR_LEN_L2_W_VLAN;
2150
2151	/* Move dst over the first remove action + zero data */
2152	dst += MLX5HWS_ACTION_DOUBLE_SIZE;
2153	/* Move dst over the first insert ctrl action */
2154	dst += MLX5HWS_ACTION_DOUBLE_SIZE / 2;
2155	/* Actions:
2156	 * no vlan: r_h-insert_4b-insert_4b-insert_4b-insert_4b-remove_2b.
2157	 * with vlan: r_h-insert_4b-insert_4b-insert_4b-insert_4b-insert_4b-remove_2b.
2158	 * the loop is without the last insertion.
2159	 */
2160	for (i = 0; i < num_of_actions - 3; i++) {
2161		e_src -= MLX5HWS_ACTION_INLINE_DATA_SIZE;
2162		memcpy(dst, e_src, MLX5HWS_ACTION_INLINE_DATA_SIZE); /* data */
2163		dst += MLX5HWS_ACTION_DOUBLE_SIZE;
2164	}
2165	/* Copy the last 2 bytes after a gap of 2 bytes which will be removed */
2166	e_src -= MLX5HWS_ACTION_INLINE_DATA_SIZE / 2;
2167	dst += MLX5HWS_ACTION_INLINE_DATA_SIZE / 2;
2168	memcpy(dst, e_src, 2);
2169}
2170
2171static int
2172hws_action_get_shared_stc_offset(struct mlx5hws_context_common_res *common_res,
2173				 enum mlx5hws_context_shared_stc_type stc_type)
2174{
2175	return common_res->shared_stc[stc_type]->stc_chunk.offset;
2176}
2177
2178static struct mlx5hws_actions_wqe_setter *
2179hws_action_setter_find_first(struct mlx5hws_actions_wqe_setter *setter,
2180			     u8 req_flags)
2181{
2182	/* Use a new setter if requested flags are taken */
2183	while (setter->flags & req_flags)
2184		setter++;
2185
2186	/* Use current setter in required flags are not used */
2187	return setter;
2188}
2189
2190static void
2191hws_action_apply_stc(struct mlx5hws_actions_apply_data *apply,
2192		     enum mlx5hws_action_stc_idx stc_idx,
2193		     u8 action_idx)
2194{
2195	struct mlx5hws_action *action = apply->rule_action[action_idx].action;
2196
2197	apply->wqe_ctrl->stc_ix[stc_idx] = htonl(action->stc.offset);
2198}
2199
2200static void
2201hws_action_setter_push_vlan(struct mlx5hws_actions_apply_data *apply,
2202			    struct mlx5hws_actions_wqe_setter *setter)
2203{
2204	struct mlx5hws_rule_action *rule_action;
2205
2206	rule_action = &apply->rule_action[setter->idx_double];
2207	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW6] = 0;
2208	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW7] = rule_action->push_vlan.vlan_hdr;
2209
2210	hws_action_apply_stc(apply, MLX5HWS_ACTION_STC_IDX_DW6, setter->idx_double);
2211	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW7] = 0;
2212}
2213
2214static void
2215hws_action_setter_modify_header(struct mlx5hws_actions_apply_data *apply,
2216				struct mlx5hws_actions_wqe_setter *setter)
2217{
2218	struct mlx5hws_rule_action *rule_action;
2219	struct mlx5hws_action *action;
2220	u32 arg_sz, arg_idx;
2221	u8 *single_action;
2222	u8 max_actions;
2223	__be32 stc_idx;
2224
2225	rule_action = &apply->rule_action[setter->idx_double];
2226	action = rule_action->action;
2227
2228	stc_idx = htonl(action->stc.offset);
2229	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW6] = stc_idx;
2230	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW7] = 0;
2231
2232	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW6] = 0;
2233
2234	if (action->modify_header.num_of_actions == 1) {
2235		if (action->modify_header.single_action_type ==
2236		    MLX5_MODIFICATION_TYPE_COPY ||
2237		    action->modify_header.single_action_type ==
2238		    MLX5_MODIFICATION_TYPE_ADD_FIELD) {
2239			apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW7] = 0;
2240			return;
2241		}
2242
2243		if (action->flags & MLX5HWS_ACTION_FLAG_SHARED)
2244			single_action = (u8 *)&action->modify_header.single_action;
2245		else
2246			single_action = rule_action->modify_header.data;
2247
2248		apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW7] =
2249			*(__be32 *)MLX5_ADDR_OF(set_action_in, single_action, data);
2250		return;
2251	}
2252
2253	/* Argument offset multiple with number of args per these actions */
2254	max_actions = action->modify_header.max_num_of_actions;
2255	arg_sz = mlx5hws_arg_get_arg_size(max_actions);
2256	arg_idx = rule_action->modify_header.offset * arg_sz;
2257
2258	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW7] = htonl(arg_idx);
2259
2260	if (!(action->flags & MLX5HWS_ACTION_FLAG_SHARED)) {
2261		apply->require_dep = 1;
2262		hws_action_modify_write(apply->queue,
2263					action->modify_header.arg_id + arg_idx,
2264					rule_action->modify_header.data,
2265					action->modify_header.num_of_actions,
2266					action->modify_header.nop_locations);
2267	}
2268}
2269
2270static void
2271hws_action_setter_insert_ptr(struct mlx5hws_actions_apply_data *apply,
2272			     struct mlx5hws_actions_wqe_setter *setter)
2273{
2274	struct mlx5hws_rule_action *rule_action;
2275	struct mlx5hws_action *action;
2276	u32 arg_idx, arg_sz;
2277	__be32 stc_idx;
2278
2279	rule_action = &apply->rule_action[setter->idx_double];
2280	action = rule_action->action + rule_action->reformat.hdr_idx;
2281
2282	/* Argument offset multiple on args required for header size */
2283	arg_sz = mlx5hws_arg_data_size_to_arg_size(action->reformat.max_hdr_sz);
2284	arg_idx = rule_action->reformat.offset * arg_sz;
2285
2286	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW6] = 0;
2287	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW7] = htonl(arg_idx);
2288
2289	stc_idx = htonl(action->stc.offset);
2290	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW6] = stc_idx;
2291	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW7] = 0;
2292
2293	if (!(action->flags & MLX5HWS_ACTION_FLAG_SHARED)) {
2294		apply->require_dep = 1;
2295		mlx5hws_arg_write(apply->queue, NULL,
2296				  action->reformat.arg_id + arg_idx,
2297				  rule_action->reformat.data,
2298				  action->reformat.header_size);
2299	}
2300}
2301
2302static void
2303hws_action_setter_tnl_l3_to_l2(struct mlx5hws_actions_apply_data *apply,
2304			       struct mlx5hws_actions_wqe_setter *setter)
2305{
2306	struct mlx5hws_rule_action *rule_action;
2307	struct mlx5hws_action *action;
2308	u32 arg_sz, arg_idx;
2309	__be32 stc_idx;
2310
2311	rule_action = &apply->rule_action[setter->idx_double];
2312	action = rule_action->action + rule_action->reformat.hdr_idx;
2313
2314	/* Argument offset multiple on args required for num of actions */
2315	arg_sz = mlx5hws_arg_get_arg_size(action->modify_header.max_num_of_actions);
2316	arg_idx = rule_action->reformat.offset * arg_sz;
2317
2318	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW6] = 0;
2319	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW7] = htonl(arg_idx);
2320
2321	stc_idx = htonl(action->stc.offset);
2322	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW6] = stc_idx;
2323	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW7] = 0;
2324
2325	if (!(action->flags & MLX5HWS_ACTION_FLAG_SHARED)) {
2326		apply->require_dep = 1;
2327		mlx5hws_arg_decapl3_write(apply->queue,
2328					  action->modify_header.arg_id + arg_idx,
2329					  rule_action->reformat.data,
2330					  action->modify_header.num_of_actions);
2331	}
2332}
2333
2334static void
2335hws_action_setter_aso(struct mlx5hws_actions_apply_data *apply,
2336		      struct mlx5hws_actions_wqe_setter *setter)
2337{
2338	struct mlx5hws_rule_action *rule_action;
2339	u32 exe_aso_ctrl;
2340	u32 offset;
2341
2342	rule_action = &apply->rule_action[setter->idx_double];
2343
2344	switch (rule_action->action->type) {
2345	case MLX5HWS_ACTION_TYP_ASO_METER:
2346		/* exe_aso_ctrl format:
2347		 * [STC only and reserved bits 29b][init_color 2b][meter_id 1b]
2348		 */
2349		offset = rule_action->aso_meter.offset / MLX5_ASO_METER_NUM_PER_OBJ;
2350		exe_aso_ctrl = rule_action->aso_meter.offset % MLX5_ASO_METER_NUM_PER_OBJ;
2351		exe_aso_ctrl |= rule_action->aso_meter.init_color <<
2352				MLX5HWS_ACTION_METER_INIT_COLOR_OFFSET;
2353		break;
2354	default:
2355		mlx5hws_err(rule_action->action->ctx,
2356			    "Unsupported ASO action type: %d\n", rule_action->action->type);
2357		return;
2358	}
2359
2360	/* aso_object_offset format: [24B] */
2361	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW6] = htonl(offset);
2362	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW7] = htonl(exe_aso_ctrl);
2363
2364	hws_action_apply_stc(apply, MLX5HWS_ACTION_STC_IDX_DW6, setter->idx_double);
2365	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW7] = 0;
2366}
2367
2368static void
2369hws_action_setter_tag(struct mlx5hws_actions_apply_data *apply,
2370		      struct mlx5hws_actions_wqe_setter *setter)
2371{
2372	struct mlx5hws_rule_action *rule_action;
2373
2374	rule_action = &apply->rule_action[setter->idx_single];
2375	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW5] = htonl(rule_action->tag.value);
2376	hws_action_apply_stc(apply, MLX5HWS_ACTION_STC_IDX_DW5, setter->idx_single);
2377}
2378
2379static void
2380hws_action_setter_ctrl_ctr(struct mlx5hws_actions_apply_data *apply,
2381			   struct mlx5hws_actions_wqe_setter *setter)
2382{
2383	struct mlx5hws_rule_action *rule_action;
2384
2385	rule_action = &apply->rule_action[setter->idx_ctr];
2386	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW0] = htonl(rule_action->counter.offset);
2387	hws_action_apply_stc(apply, MLX5HWS_ACTION_STC_IDX_CTRL, setter->idx_ctr);
2388}
2389
2390static void
2391hws_action_setter_single(struct mlx5hws_actions_apply_data *apply,
2392			 struct mlx5hws_actions_wqe_setter *setter)
2393{
2394	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW5] = 0;
2395	hws_action_apply_stc(apply, MLX5HWS_ACTION_STC_IDX_DW5, setter->idx_single);
2396}
2397
2398static void
2399hws_action_setter_single_double_pop(struct mlx5hws_actions_apply_data *apply,
2400				    struct mlx5hws_actions_wqe_setter *setter)
2401{
2402	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW5] = 0;
2403	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW5] =
2404		htonl(hws_action_get_shared_stc_offset(apply->common_res,
2405						       MLX5HWS_CONTEXT_SHARED_STC_DOUBLE_POP));
2406}
2407
2408static void
2409hws_action_setter_hit(struct mlx5hws_actions_apply_data *apply,
2410		      struct mlx5hws_actions_wqe_setter *setter)
2411{
2412	apply->wqe_data[MLX5HWS_ACTION_OFFSET_HIT_LSB] = 0;
2413	hws_action_apply_stc(apply, MLX5HWS_ACTION_STC_IDX_HIT, setter->idx_hit);
2414}
2415
2416static void
2417hws_action_setter_default_hit(struct mlx5hws_actions_apply_data *apply,
2418			      struct mlx5hws_actions_wqe_setter *setter)
2419{
2420	apply->wqe_data[MLX5HWS_ACTION_OFFSET_HIT_LSB] = 0;
2421	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_HIT] =
2422		htonl(apply->common_res->default_stc->default_hit.offset);
2423}
2424
2425static void
2426hws_action_setter_hit_next_action(struct mlx5hws_actions_apply_data *apply,
2427				  struct mlx5hws_actions_wqe_setter *setter)
2428{
2429	apply->wqe_data[MLX5HWS_ACTION_OFFSET_HIT_LSB] = htonl(apply->next_direct_idx << 6);
2430	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_HIT] = htonl(apply->jump_to_action_stc);
2431}
2432
2433static void
2434hws_action_setter_common_decap(struct mlx5hws_actions_apply_data *apply,
2435			       struct mlx5hws_actions_wqe_setter *setter)
2436{
2437	apply->wqe_data[MLX5HWS_ACTION_OFFSET_DW5] = 0;
2438	apply->wqe_ctrl->stc_ix[MLX5HWS_ACTION_STC_IDX_DW5] =
2439		htonl(hws_action_get_shared_stc_offset(apply->common_res,
2440						       MLX5HWS_CONTEXT_SHARED_STC_DECAP_L3));
2441}
2442
2443static void
2444hws_action_setter_range(struct mlx5hws_actions_apply_data *apply,
2445			struct mlx5hws_actions_wqe_setter *setter)
2446{
2447	/* Always jump to index zero */
2448	apply->wqe_data[MLX5HWS_ACTION_OFFSET_HIT_LSB] = 0;
2449	hws_action_apply_stc(apply, MLX5HWS_ACTION_STC_IDX_HIT, setter->idx_hit);
2450}
2451
2452int mlx5hws_action_template_process(struct mlx5hws_action_template *at)
2453{
2454	struct mlx5hws_actions_wqe_setter *start_setter = at->setters + 1;
2455	enum mlx5hws_action_type *action_type = at->action_type_arr;
2456	struct mlx5hws_actions_wqe_setter *setter = at->setters;
2457	struct mlx5hws_actions_wqe_setter *pop_setter = NULL;
2458	struct mlx5hws_actions_wqe_setter *last_setter;
2459	int i;
2460
2461	/* Note: Given action combination must be valid */
2462
2463	/* Check if action were already processed */
2464	if (at->num_of_action_stes)
2465		return 0;
2466
2467	for (i = 0; i < MLX5HWS_ACTION_MAX_STE; i++)
2468		setter[i].set_hit = &hws_action_setter_hit_next_action;
2469
2470	/* The same action template setters can be used with jumbo or match
2471	 * STE, to support both cases we reserve the first setter for cases
2472	 * with jumbo STE to allow jump to the first action STE.
2473	 * This extra setter can be reduced in some cases on rule creation.
2474	 */
2475	setter = start_setter;
2476	last_setter = start_setter;
2477
2478	for (i = 0; i < at->num_actions; i++) {
2479		switch (action_type[i]) {
2480		case MLX5HWS_ACTION_TYP_DROP:
2481		case MLX5HWS_ACTION_TYP_TBL:
2482		case MLX5HWS_ACTION_TYP_DEST_ARRAY:
2483		case MLX5HWS_ACTION_TYP_SAMPLER:
2484		case MLX5HWS_ACTION_TYP_VPORT:
2485		case MLX5HWS_ACTION_TYP_MISS:
2486			/* Hit action */
2487			last_setter->flags |= ASF_HIT;
2488			last_setter->set_hit = &hws_action_setter_hit;
2489			last_setter->idx_hit = i;
2490			break;
2491
2492		case MLX5HWS_ACTION_TYP_RANGE:
2493			last_setter->flags |= ASF_HIT;
2494			last_setter->set_hit = &hws_action_setter_range;
2495			last_setter->idx_hit = i;
2496			break;
2497
2498		case MLX5HWS_ACTION_TYP_POP_VLAN:
2499			/* Single remove header to header */
2500			if (pop_setter) {
2501				/* We have 2 pops, use the shared */
2502				pop_setter->set_single = &hws_action_setter_single_double_pop;
2503				break;
2504			}
2505			setter = hws_action_setter_find_first(last_setter,
2506							      ASF_SINGLE1 | ASF_MODIFY |
2507							      ASF_INSERT);
2508			setter->flags |= ASF_SINGLE1 | ASF_REMOVE;
2509			setter->set_single = &hws_action_setter_single;
2510			setter->idx_single = i;
2511			pop_setter = setter;
2512			break;
2513
2514		case MLX5HWS_ACTION_TYP_PUSH_VLAN:
2515			/* Double insert inline */
2516			setter = hws_action_setter_find_first(last_setter, ASF_DOUBLE | ASF_REMOVE);
2517			setter->flags |= ASF_DOUBLE | ASF_INSERT;
2518			setter->set_double = &hws_action_setter_push_vlan;
2519			setter->idx_double = i;
2520			break;
2521
2522		case MLX5HWS_ACTION_TYP_MODIFY_HDR:
2523			/* Double modify header list */
2524			setter = hws_action_setter_find_first(last_setter, ASF_DOUBLE | ASF_REMOVE);
2525			setter->flags |= ASF_DOUBLE | ASF_MODIFY;
2526			setter->set_double = &hws_action_setter_modify_header;
2527			setter->idx_double = i;
2528			break;
2529
2530		case MLX5HWS_ACTION_TYP_ASO_METER:
2531			/* Double ASO action */
2532			setter = hws_action_setter_find_first(last_setter, ASF_DOUBLE);
2533			setter->flags |= ASF_DOUBLE;
2534			setter->set_double = &hws_action_setter_aso;
2535			setter->idx_double = i;
2536			break;
2537
2538		case MLX5HWS_ACTION_TYP_REMOVE_HEADER:
2539		case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L2_TO_L2:
2540			/* Single remove header to header */
2541			setter = hws_action_setter_find_first(last_setter,
2542							      ASF_SINGLE1 | ASF_MODIFY);
2543			setter->flags |= ASF_SINGLE1 | ASF_REMOVE;
2544			setter->set_single = &hws_action_setter_single;
2545			setter->idx_single = i;
2546			break;
2547
2548		case MLX5HWS_ACTION_TYP_INSERT_HEADER:
2549		case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L2:
2550			/* Double insert header with pointer */
2551			setter = hws_action_setter_find_first(last_setter, ASF_DOUBLE | ASF_REMOVE);
2552			setter->flags |= ASF_DOUBLE | ASF_INSERT;
2553			setter->set_double = &hws_action_setter_insert_ptr;
2554			setter->idx_double = i;
2555			break;
2556
2557		case MLX5HWS_ACTION_TYP_REFORMAT_L2_TO_TNL_L3:
2558			/* Single remove + Double insert header with pointer */
2559			setter = hws_action_setter_find_first(last_setter,
2560							      ASF_SINGLE1 | ASF_DOUBLE);
2561			setter->flags |= ASF_SINGLE1 | ASF_DOUBLE;
2562			setter->set_double = &hws_action_setter_insert_ptr;
2563			setter->idx_double = i;
2564			setter->set_single = &hws_action_setter_common_decap;
2565			setter->idx_single = i;
2566			break;
2567
2568		case MLX5HWS_ACTION_TYP_REFORMAT_TNL_L3_TO_L2:
2569			/* Double modify header list with remove and push inline */
2570			setter = hws_action_setter_find_first(last_setter, ASF_DOUBLE | ASF_REMOVE);
2571			setter->flags |= ASF_DOUBLE | ASF_MODIFY | ASF_INSERT;
2572			setter->set_double = &hws_action_setter_tnl_l3_to_l2;
2573			setter->idx_double = i;
2574			break;
2575
2576		case MLX5HWS_ACTION_TYP_TAG:
2577			/* Single TAG action, search for any room from the start */
2578			setter = hws_action_setter_find_first(start_setter, ASF_SINGLE1);
2579			setter->flags |= ASF_SINGLE1;
2580			setter->set_single = &hws_action_setter_tag;
2581			setter->idx_single = i;
2582			break;
2583
2584		case MLX5HWS_ACTION_TYP_CTR:
2585			/* Control counter action
2586			 * TODO: Current counter executed first. Support is needed
2587			 *	 for single ation counter action which is done last.
2588			 *	 Example: Decap + CTR
2589			 */
2590			setter = hws_action_setter_find_first(start_setter, ASF_CTR);
2591			setter->flags |= ASF_CTR;
2592			setter->set_ctr = &hws_action_setter_ctrl_ctr;
2593			setter->idx_ctr = i;
2594			break;
2595		default:
2596			pr_warn("HWS: Invalid action type in processingaction template: action_type[%d]=%d\n",
2597				i, action_type[i]);
2598			return -EOPNOTSUPP;
2599		}
2600
2601		last_setter = max(setter, last_setter);
2602	}
2603
2604	/* Set default hit on the last STE if no hit action provided */
2605	if (!(last_setter->flags & ASF_HIT))
2606		last_setter->set_hit = &hws_action_setter_default_hit;
2607
2608	at->num_of_action_stes = last_setter - start_setter + 1;
2609
2610	/* Check if action template doesn't require any action DWs */
2611	at->only_term = (at->num_of_action_stes == 1) &&
2612		!(last_setter->flags & ~(ASF_CTR | ASF_HIT));
2613
2614	return 0;
2615}
2616
2617struct mlx5hws_action_template *
2618mlx5hws_action_template_create(enum mlx5hws_action_type action_type[])
2619{
2620	struct mlx5hws_action_template *at;
2621	u8 num_actions = 0;
2622	int i;
2623
2624	at = kzalloc(sizeof(*at), GFP_KERNEL);
2625	if (!at)
2626		return NULL;
2627
2628	while (action_type[num_actions++] != MLX5HWS_ACTION_TYP_LAST)
2629		;
2630
2631	at->num_actions = num_actions - 1;
2632	at->action_type_arr = kcalloc(num_actions, sizeof(*action_type), GFP_KERNEL);
2633	if (!at->action_type_arr)
2634		goto free_at;
2635
2636	for (i = 0; i < num_actions; i++)
2637		at->action_type_arr[i] = action_type[i];
2638
2639	return at;
2640
2641free_at:
2642	kfree(at);
2643	return NULL;
2644}
2645
2646int mlx5hws_action_template_destroy(struct mlx5hws_action_template *at)
2647{
2648	kfree(at->action_type_arr);
2649	kfree(at);
2650	return 0;
2651}