FILE
ipsec_fs.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3 4#include <linux/netdevice.h> 5#include "en.h" 6#include "en/fs.h" 7#include "eswitch.h" 8#include "ipsec.h" 9#include "fs_core.h" 10#include "lib/ipsec_fs_roce.h" 11#include "lib/fs_chains.h" 12#include "esw/ipsec_fs.h" 13#include "en_rep.h" 14 15#define NUM_IPSEC_FTE BIT(15) 16#define MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE 16 17#define IPSEC_TUNNEL_DEFAULT_TTL 0x40 18 19#define MLX5_IPSEC_FS_SA_SELECTOR_MAX_NUM_GROUPS 16 20 21enum { 22 MLX5_IPSEC_ASO_OK, 23 MLX5_IPSEC_ASO_BAD_REPLY, 24 25 /* For crypto offload, set by driver */ 26 MLX5_IPSEC_ASO_SW_CRYPTO_OFFLOAD = 0xAA, 27}; 28 29struct mlx5e_ipsec_fc { 30 struct mlx5_fc *cnt; 31 struct mlx5_fc *drop; 32}; 33 34struct mlx5e_ipsec_tx { 35 struct mlx5e_ipsec_ft ft; 36 struct mlx5e_ipsec_miss pol; 37 struct mlx5e_ipsec_miss sa; 38 struct mlx5e_ipsec_rule status; 39 struct mlx5_flow_namespace *ns; 40 struct mlx5e_ipsec_fc *fc; 41 struct mlx5_fs_chains *chains; 42 u8 allow_tunnel_mode : 1; 43}; 44 45struct mlx5e_ipsec_status_checks { 46 struct mlx5_flow_group *pass_group; 47 struct mlx5_flow_handle *packet_offload_pass_rule; 48 struct mlx5_flow_handle *crypto_offload_pass_rule; 49 struct mlx5_flow_group *drop_all_group; 50 struct mlx5e_ipsec_drop all; 51}; 52 53struct mlx5e_ipsec_rx { 54 struct mlx5e_ipsec_ft ft; 55 struct mlx5e_ipsec_miss pol; 56 struct mlx5e_ipsec_miss sa; 57 struct mlx5e_ipsec_miss sa_sel; 58 struct mlx5e_ipsec_status_checks status_checks; 59 struct mlx5e_ipsec_fc *fc; 60 struct mlx5_fs_chains *chains; 61 struct mlx5_flow_table *pol_miss_ft; 62 struct mlx5_flow_handle *pol_miss_rule; 63 u8 allow_tunnel_mode : 1; 64 u8 ttc_rules_added : 1; 65}; 66 67/* IPsec RX flow steering */ 68static enum mlx5_traffic_types family2tt(u32 family) 69{ 70 if (family == AF_INET) 71 return MLX5_TT_IPV4_IPSEC_ESP; 72 return MLX5_TT_IPV6_IPSEC_ESP; 73} 74 75static struct mlx5e_ipsec_rx *ipsec_rx(struct mlx5e_ipsec *ipsec, u32 family, int type) 76{ 77 if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) 78 return ipsec->rx_esw; 79 80 if (family == AF_INET) 81 return ipsec->rx_ipv4; 82 83 return ipsec->rx_ipv6; 84} 85 86static struct mlx5e_ipsec_tx *ipsec_tx(struct mlx5e_ipsec *ipsec, int type) 87{ 88 if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) 89 return ipsec->tx_esw; 90 91 return ipsec->tx; 92} 93 94static struct mlx5_fs_chains * 95ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft, 96 enum mlx5_flow_namespace_type ns, int base_prio, 97 int base_level, struct mlx5_flow_table **root_ft) 98{ 99 struct mlx5_chains_attr attr = {}; 100 struct mlx5_fs_chains *chains; 101 struct mlx5_flow_table *ft; 102 int err; 103 104 attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED | 105 MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 106 attr.max_grp_num = 2; 107 attr.default_ft = miss_ft; 108 attr.ns = ns; 109 attr.fs_base_prio = base_prio; 110 attr.fs_base_level = base_level; 111 chains = mlx5_chains_create(mdev, &attr); 112 if (IS_ERR(chains)) 113 return chains; 114 115 /* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */ 116 ft = mlx5_chains_get_table(chains, 0, 1, 0); 117 if (IS_ERR(ft)) { 118 err = PTR_ERR(ft); 119 goto err_chains_get; 120 } 121 122 *root_ft = ft; 123 return chains; 124 125err_chains_get: 126 mlx5_chains_destroy(chains); 127 return ERR_PTR(err); 128} 129 130static void ipsec_chains_destroy(struct mlx5_fs_chains *chains) 131{ 132 mlx5_chains_put_table(chains, 0, 1, 0); 133 mlx5_chains_destroy(chains); 134} 135 136static struct mlx5_flow_table * 137ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio) 138{ 139 return mlx5_chains_get_table(chains, 0, prio + 1, 0); 140} 141 142static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio) 143{ 144 mlx5_chains_put_table(chains, 0, prio + 1, 0); 145} 146 147static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, 148 int level, int prio, 149 int num_reserved_entries, 150 int max_num_groups, u32 flags) 151{ 152 struct mlx5_flow_table_attr ft_attr = {}; 153 154 ft_attr.autogroup.num_reserved_entries = num_reserved_entries; 155 ft_attr.autogroup.max_num_groups = max_num_groups; 156 ft_attr.max_fte = NUM_IPSEC_FTE; 157 ft_attr.level = level; 158 ft_attr.prio = prio; 159 ft_attr.flags = flags; 160 161 return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 162} 163 164static void ipsec_rx_status_drop_destroy(struct mlx5e_ipsec *ipsec, 165 struct mlx5e_ipsec_rx *rx) 166{ 167 mlx5_del_flow_rules(rx->status_checks.all.rule); 168 mlx5_fc_destroy(ipsec->mdev, rx->status_checks.all.fc); 169 mlx5_destroy_flow_group(rx->status_checks.drop_all_group); 170} 171 172static void ipsec_rx_status_pass_destroy(struct mlx5e_ipsec *ipsec, 173 struct mlx5e_ipsec_rx *rx) 174{ 175 mlx5_del_flow_rules(rx->status_checks.packet_offload_pass_rule); 176 mlx5_del_flow_rules(rx->status_checks.crypto_offload_pass_rule); 177} 178 179static void ipsec_rx_rule_add_match_obj(struct mlx5e_ipsec_sa_entry *sa_entry, 180 struct mlx5e_ipsec_rx *rx, 181 struct mlx5_flow_spec *spec) 182{ 183 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 184 185 if (rx == ipsec->rx_esw) { 186 mlx5_esw_ipsec_rx_rule_add_match_obj(sa_entry, spec); 187 } else { 188 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 189 misc_parameters_2.metadata_reg_c_2); 190 MLX5_SET(fte_match_param, spec->match_value, 191 misc_parameters_2.metadata_reg_c_2, 192 sa_entry->ipsec_obj_id | BIT(31)); 193 194 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 195 } 196} 197 198static int rx_add_rule_drop_auth_trailer(struct mlx5e_ipsec_sa_entry *sa_entry, 199 struct mlx5e_ipsec_rx *rx) 200{ 201 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 202 struct mlx5_flow_table *ft = rx->ft.status; 203 struct mlx5_core_dev *mdev = ipsec->mdev; 204 struct mlx5_flow_destination dest = {}; 205 struct mlx5_flow_act flow_act = {}; 206 struct mlx5_flow_handle *rule; 207 struct mlx5_fc *flow_counter; 208 struct mlx5_flow_spec *spec; 209 int err; 210 211 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 212 if (!spec) 213 return -ENOMEM; 214 215 flow_counter = mlx5_fc_create(mdev, true); 216 if (IS_ERR(flow_counter)) { 217 err = PTR_ERR(flow_counter); 218 mlx5_core_err(mdev, 219 "Failed to add ipsec rx status drop rule counter, err=%d\n", err); 220 goto err_cnt; 221 } 222 sa_entry->ipsec_rule.auth.fc = flow_counter; 223 224 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 225 flow_act.flags = FLOW_ACT_NO_APPEND; 226 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 227 dest.counter = flow_counter; 228 if (rx == ipsec->rx_esw) 229 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 230 231 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.ipsec_syndrome); 232 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.ipsec_syndrome, 1); 233 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 234 ipsec_rx_rule_add_match_obj(sa_entry, rx, spec); 235 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 236 if (IS_ERR(rule)) { 237 err = PTR_ERR(rule); 238 mlx5_core_err(mdev, 239 "Failed to add ipsec rx status drop rule, err=%d\n", err); 240 goto err_rule; 241 } 242 sa_entry->ipsec_rule.auth.rule = rule; 243 244 flow_counter = mlx5_fc_create(mdev, true); 245 if (IS_ERR(flow_counter)) { 246 err = PTR_ERR(flow_counter); 247 mlx5_core_err(mdev, 248 "Failed to add ipsec rx status drop rule counter, err=%d\n", err); 249 goto err_cnt_2; 250 } 251 sa_entry->ipsec_rule.trailer.fc = flow_counter; 252 253 dest.counter = flow_counter; 254 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.ipsec_syndrome, 2); 255 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 256 if (IS_ERR(rule)) { 257 err = PTR_ERR(rule); 258 mlx5_core_err(mdev, 259 "Failed to add ipsec rx status drop rule, err=%d\n", err); 260 goto err_rule_2; 261 } 262 sa_entry->ipsec_rule.trailer.rule = rule; 263 264 kvfree(spec); 265 return 0; 266 267err_rule_2: 268 mlx5_fc_destroy(mdev, sa_entry->ipsec_rule.trailer.fc); 269err_cnt_2: 270 mlx5_del_flow_rules(sa_entry->ipsec_rule.auth.rule); 271err_rule: 272 mlx5_fc_destroy(mdev, sa_entry->ipsec_rule.auth.fc); 273err_cnt: 274 kvfree(spec); 275 return err; 276} 277 278static int rx_add_rule_drop_replay(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5e_ipsec_rx *rx) 279{ 280 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 281 struct mlx5_flow_table *ft = rx->ft.status; 282 struct mlx5_core_dev *mdev = ipsec->mdev; 283 struct mlx5_flow_destination dest = {}; 284 struct mlx5_flow_act flow_act = {}; 285 struct mlx5_flow_handle *rule; 286 struct mlx5_fc *flow_counter; 287 struct mlx5_flow_spec *spec; 288 int err; 289 290 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 291 if (!spec) 292 return -ENOMEM; 293 294 flow_counter = mlx5_fc_create(mdev, true); 295 if (IS_ERR(flow_counter)) { 296 err = PTR_ERR(flow_counter); 297 mlx5_core_err(mdev, 298 "Failed to add ipsec rx status drop rule counter, err=%d\n", err); 299 goto err_cnt; 300 } 301 302 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 303 flow_act.flags = FLOW_ACT_NO_APPEND; 304 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 305 dest.counter = flow_counter; 306 if (rx == ipsec->rx_esw) 307 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 308 309 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4); 310 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 1); 311 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 312 ipsec_rx_rule_add_match_obj(sa_entry, rx, spec); 313 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 314 if (IS_ERR(rule)) { 315 err = PTR_ERR(rule); 316 mlx5_core_err(mdev, 317 "Failed to add ipsec rx status drop rule, err=%d\n", err); 318 goto err_rule; 319 } 320 321 sa_entry->ipsec_rule.replay.rule = rule; 322 sa_entry->ipsec_rule.replay.fc = flow_counter; 323 324 kvfree(spec); 325 return 0; 326 327err_rule: 328 mlx5_fc_destroy(mdev, flow_counter); 329err_cnt: 330 kvfree(spec); 331 return err; 332} 333 334static int ipsec_rx_status_drop_all_create(struct mlx5e_ipsec *ipsec, 335 struct mlx5e_ipsec_rx *rx) 336{ 337 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 338 struct mlx5_flow_table *ft = rx->ft.status; 339 struct mlx5_core_dev *mdev = ipsec->mdev; 340 struct mlx5_flow_destination dest = {}; 341 struct mlx5_flow_act flow_act = {}; 342 struct mlx5_flow_handle *rule; 343 struct mlx5_fc *flow_counter; 344 struct mlx5_flow_spec *spec; 345 struct mlx5_flow_group *g; 346 u32 *flow_group_in; 347 int err = 0; 348 349 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 350 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 351 if (!flow_group_in || !spec) { 352 err = -ENOMEM; 353 goto err_out; 354 } 355 356 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); 357 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); 358 g = mlx5_create_flow_group(ft, flow_group_in); 359 if (IS_ERR(g)) { 360 err = PTR_ERR(g); 361 mlx5_core_err(mdev, 362 "Failed to add ipsec rx status drop flow group, err=%d\n", err); 363 goto err_out; 364 } 365 366 flow_counter = mlx5_fc_create(mdev, false); 367 if (IS_ERR(flow_counter)) { 368 err = PTR_ERR(flow_counter); 369 mlx5_core_err(mdev, 370 "Failed to add ipsec rx status drop rule counter, err=%d\n", err); 371 goto err_cnt; 372 } 373 374 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 375 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 376 dest.counter = flow_counter; 377 if (rx == ipsec->rx_esw) 378 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 379 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 380 if (IS_ERR(rule)) { 381 err = PTR_ERR(rule); 382 mlx5_core_err(mdev, 383 "Failed to add ipsec rx status drop rule, err=%d\n", err); 384 goto err_rule; 385 } 386 387 rx->status_checks.drop_all_group = g; 388 rx->status_checks.all.rule = rule; 389 rx->status_checks.all.fc = flow_counter; 390 391 kvfree(flow_group_in); 392 kvfree(spec); 393 return 0; 394 395err_rule: 396 mlx5_fc_destroy(mdev, flow_counter); 397err_cnt: 398 mlx5_destroy_flow_group(g); 399err_out: 400 kvfree(flow_group_in); 401 kvfree(spec); 402 return err; 403} 404 405static int ipsec_rx_status_pass_group_create(struct mlx5e_ipsec *ipsec, 406 struct mlx5e_ipsec_rx *rx) 407{ 408 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 409 struct mlx5_flow_table *ft = rx->ft.status; 410 struct mlx5_flow_group *fg; 411 void *match_criteria; 412 u32 *flow_group_in; 413 int err = 0; 414 415 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 416 if (!flow_group_in) 417 return -ENOMEM; 418 419 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 420 MLX5_MATCH_MISC_PARAMETERS_2); 421 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 422 match_criteria); 423 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 424 misc_parameters_2.ipsec_syndrome); 425 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 426 misc_parameters_2.metadata_reg_c_4); 427 428 MLX5_SET(create_flow_group_in, flow_group_in, 429 start_flow_index, ft->max_fte - 3); 430 MLX5_SET(create_flow_group_in, flow_group_in, 431 end_flow_index, ft->max_fte - 2); 432 433 fg = mlx5_create_flow_group(ft, flow_group_in); 434 if (IS_ERR(fg)) { 435 err = PTR_ERR(fg); 436 mlx5_core_warn(ipsec->mdev, 437 "Failed to create rx status pass flow group, err=%d\n", 438 err); 439 } 440 rx->status_checks.pass_group = fg; 441 442 kvfree(flow_group_in); 443 return err; 444} 445 446static struct mlx5_flow_handle * 447ipsec_rx_status_pass_create(struct mlx5e_ipsec *ipsec, 448 struct mlx5e_ipsec_rx *rx, 449 struct mlx5_flow_destination *dest, 450 u8 aso_ok) 451{ 452 struct mlx5_flow_act flow_act = {}; 453 struct mlx5_flow_handle *rule; 454 struct mlx5_flow_spec *spec; 455 int err; 456 457 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 458 if (!spec) 459 return ERR_PTR(-ENOMEM); 460 461 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 462 misc_parameters_2.ipsec_syndrome); 463 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 464 misc_parameters_2.metadata_reg_c_4); 465 MLX5_SET(fte_match_param, spec->match_value, 466 misc_parameters_2.ipsec_syndrome, 0); 467 MLX5_SET(fte_match_param, spec->match_value, 468 misc_parameters_2.metadata_reg_c_4, aso_ok); 469 if (rx == ipsec->rx_esw) 470 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 471 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 472 flow_act.flags = FLOW_ACT_NO_APPEND | FLOW_ACT_IGNORE_FLOW_LEVEL; 473 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 474 MLX5_FLOW_CONTEXT_ACTION_COUNT; 475 rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); 476 if (IS_ERR(rule)) { 477 err = PTR_ERR(rule); 478 mlx5_core_warn(ipsec->mdev, 479 "Failed to add ipsec rx status pass rule, err=%d\n", err); 480 goto err_rule; 481 } 482 483 kvfree(spec); 484 return rule; 485 486err_rule: 487 kvfree(spec); 488 return ERR_PTR(err); 489} 490 491static void mlx5_ipsec_rx_status_destroy(struct mlx5e_ipsec *ipsec, 492 struct mlx5e_ipsec_rx *rx) 493{ 494 ipsec_rx_status_pass_destroy(ipsec, rx); 495 mlx5_destroy_flow_group(rx->status_checks.pass_group); 496 ipsec_rx_status_drop_destroy(ipsec, rx); 497} 498 499static int mlx5_ipsec_rx_status_create(struct mlx5e_ipsec *ipsec, 500 struct mlx5e_ipsec_rx *rx, 501 struct mlx5_flow_destination *dest) 502{ 503 struct mlx5_flow_destination pol_dest[2]; 504 struct mlx5_flow_handle *rule; 505 int err; 506 507 err = ipsec_rx_status_drop_all_create(ipsec, rx); 508 if (err) 509 return err; 510 511 err = ipsec_rx_status_pass_group_create(ipsec, rx); 512 if (err) 513 goto err_pass_group_create; 514 515 rule = ipsec_rx_status_pass_create(ipsec, rx, dest, 516 MLX5_IPSEC_ASO_SW_CRYPTO_OFFLOAD); 517 if (IS_ERR(rule)) { 518 err = PTR_ERR(rule); 519 goto err_crypto_offload_pass_create; 520 } 521 rx->status_checks.crypto_offload_pass_rule = rule; 522 523 pol_dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 524 pol_dest[0].ft = rx->ft.pol; 525 pol_dest[1] = dest[1]; 526 rule = ipsec_rx_status_pass_create(ipsec, rx, pol_dest, 527 MLX5_IPSEC_ASO_OK); 528 if (IS_ERR(rule)) { 529 err = PTR_ERR(rule); 530 goto err_packet_offload_pass_create; 531 } 532 rx->status_checks.packet_offload_pass_rule = rule; 533 534 return 0; 535 536err_packet_offload_pass_create: 537 mlx5_del_flow_rules(rx->status_checks.crypto_offload_pass_rule); 538err_crypto_offload_pass_create: 539 mlx5_destroy_flow_group(rx->status_checks.pass_group); 540err_pass_group_create: 541 ipsec_rx_status_drop_destroy(ipsec, rx); 542 return err; 543} 544 545static int ipsec_miss_create(struct mlx5_core_dev *mdev, 546 struct mlx5_flow_table *ft, 547 struct mlx5e_ipsec_miss *miss, 548 struct mlx5_flow_destination *dest) 549{ 550 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 551 MLX5_DECLARE_FLOW_ACT(flow_act); 552 struct mlx5_flow_spec *spec; 553 u32 *flow_group_in; 554 int err = 0; 555 556 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 557 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 558 if (!flow_group_in || !spec) { 559 err = -ENOMEM; 560 goto out; 561 } 562 563 /* Create miss_group */ 564 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); 565 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); 566 miss->group = mlx5_create_flow_group(ft, flow_group_in); 567 if (IS_ERR(miss->group)) { 568 err = PTR_ERR(miss->group); 569 mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", 570 err); 571 goto out; 572 } 573 574 /* Create miss rule */ 575 miss->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); 576 if (IS_ERR(miss->rule)) { 577 mlx5_destroy_flow_group(miss->group); 578 err = PTR_ERR(miss->rule); 579 mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", 580 err); 581 goto out; 582 } 583out: 584 kvfree(flow_group_in); 585 kvfree(spec); 586 return err; 587} 588 589static struct mlx5_flow_destination 590ipsec_rx_decrypted_pkt_def_dest(struct mlx5_ttc_table *ttc, u32 family) 591{ 592 struct mlx5_flow_destination dest; 593 594 if (!mlx5_ttc_has_esp_flow_group(ttc)) 595 return mlx5_ttc_get_default_dest(ttc, family2tt(family)); 596 597 dest.ft = mlx5_get_ttc_flow_table(ttc); 598 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 599 600 return dest; 601} 602 603static void ipsec_rx_update_default_dest(struct mlx5e_ipsec_rx *rx, 604 struct mlx5_flow_destination *old_dest, 605 struct mlx5_flow_destination *new_dest) 606{ 607 mlx5_modify_rule_destination(rx->pol_miss_rule, new_dest, old_dest); 608 mlx5_modify_rule_destination(rx->status_checks.crypto_offload_pass_rule, 609 new_dest, old_dest); 610} 611 612static void handle_ipsec_rx_bringup(struct mlx5e_ipsec *ipsec, u32 family) 613{ 614 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); 615 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); 616 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 617 struct mlx5_flow_destination old_dest, new_dest; 618 619 old_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family); 620 621 mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, ns, &old_dest, family, 622 MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, MLX5E_NIC_PRIO); 623 624 new_dest.ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); 625 new_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 626 ipsec_rx_update_default_dest(rx, &old_dest, &new_dest); 627} 628 629static void handle_ipsec_rx_cleanup(struct mlx5e_ipsec *ipsec, u32 family) 630{ 631 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); 632 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 633 struct mlx5_flow_destination old_dest, new_dest; 634 635 old_dest.ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); 636 old_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 637 new_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family); 638 ipsec_rx_update_default_dest(rx, &old_dest, &new_dest); 639 640 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, ipsec->mdev); 641} 642 643static void ipsec_mpv_work_handler(struct work_struct *_work) 644{ 645 struct mlx5e_ipsec_mpv_work *work = container_of(_work, struct mlx5e_ipsec_mpv_work, work); 646 struct mlx5e_ipsec *ipsec = work->slave_priv->ipsec; 647 648 switch (work->event) { 649 case MPV_DEVCOM_IPSEC_MASTER_UP: 650 mutex_lock(&ipsec->tx->ft.mutex); 651 if (ipsec->tx->ft.refcnt) 652 mlx5_ipsec_fs_roce_tx_create(ipsec->mdev, ipsec->roce, ipsec->tx->ft.pol, 653 true); 654 mutex_unlock(&ipsec->tx->ft.mutex); 655 656 mutex_lock(&ipsec->rx_ipv4->ft.mutex); 657 if (ipsec->rx_ipv4->ft.refcnt) 658 handle_ipsec_rx_bringup(ipsec, AF_INET); 659 mutex_unlock(&ipsec->rx_ipv4->ft.mutex); 660 661 mutex_lock(&ipsec->rx_ipv6->ft.mutex); 662 if (ipsec->rx_ipv6->ft.refcnt) 663 handle_ipsec_rx_bringup(ipsec, AF_INET6); 664 mutex_unlock(&ipsec->rx_ipv6->ft.mutex); 665 break; 666 case MPV_DEVCOM_IPSEC_MASTER_DOWN: 667 mutex_lock(&ipsec->tx->ft.mutex); 668 if (ipsec->tx->ft.refcnt) 669 mlx5_ipsec_fs_roce_tx_destroy(ipsec->roce, ipsec->mdev); 670 mutex_unlock(&ipsec->tx->ft.mutex); 671 672 mutex_lock(&ipsec->rx_ipv4->ft.mutex); 673 if (ipsec->rx_ipv4->ft.refcnt) 674 handle_ipsec_rx_cleanup(ipsec, AF_INET); 675 mutex_unlock(&ipsec->rx_ipv4->ft.mutex); 676 677 mutex_lock(&ipsec->rx_ipv6->ft.mutex); 678 if (ipsec->rx_ipv6->ft.refcnt) 679 handle_ipsec_rx_cleanup(ipsec, AF_INET6); 680 mutex_unlock(&ipsec->rx_ipv6->ft.mutex); 681 break; 682 } 683 684 complete(&work->master_priv->ipsec->comp); 685} 686 687static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, 688 struct mlx5e_ipsec_rx *rx, u32 family) 689{ 690 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 691 692 if (rx->ttc_rules_added) 693 mlx5_ttc_destroy_ipsec_rules(ttc); 694 mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); 695} 696 697static void ipsec_rx_policy_destroy(struct mlx5e_ipsec_rx *rx) 698{ 699 if (rx->chains) { 700 ipsec_chains_destroy(rx->chains); 701 } else { 702 mlx5_del_flow_rules(rx->pol.rule); 703 mlx5_destroy_flow_group(rx->pol.group); 704 mlx5_destroy_flow_table(rx->ft.pol); 705 } 706 707 if (rx->pol_miss_rule) { 708 mlx5_del_flow_rules(rx->pol_miss_rule); 709 mlx5_destroy_flow_table(rx->pol_miss_ft); 710 } 711} 712 713static void ipsec_rx_sa_selector_destroy(struct mlx5_core_dev *mdev, 714 struct mlx5e_ipsec_rx *rx) 715{ 716 mlx5_del_flow_rules(rx->sa_sel.rule); 717 mlx5_fc_destroy(mdev, rx->sa_sel.fc); 718 rx->sa_sel.fc = NULL; 719 mlx5_destroy_flow_group(rx->sa_sel.group); 720 mlx5_destroy_flow_table(rx->ft.sa_sel); 721} 722 723static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 724 struct mlx5e_ipsec_rx *rx, u32 family) 725{ 726 /* disconnect */ 727 if (rx != ipsec->rx_esw) 728 ipsec_rx_ft_disconnect(ipsec, rx, family); 729 730 mlx5_del_flow_rules(rx->sa.rule); 731 mlx5_destroy_flow_group(rx->sa.group); 732 mlx5_destroy_flow_table(rx->ft.sa); 733 if (rx->allow_tunnel_mode) 734 mlx5_eswitch_unblock_encap(mdev); 735 mlx5_ipsec_rx_status_destroy(ipsec, rx); 736 mlx5_destroy_flow_table(rx->ft.status); 737 738 ipsec_rx_sa_selector_destroy(mdev, rx); 739 740 ipsec_rx_policy_destroy(rx); 741 742 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, mdev); 743 744#ifdef CONFIG_MLX5_ESWITCH 745 if (rx == ipsec->rx_esw) 746 mlx5_chains_put_table(esw_chains(ipsec->mdev->priv.eswitch), 747 0, 1, 0); 748#endif 749} 750 751static void ipsec_rx_create_attr_set(struct mlx5e_ipsec *ipsec, 752 struct mlx5e_ipsec_rx *rx, 753 u32 family, 754 struct mlx5e_ipsec_rx_create_attr *attr) 755{ 756 if (rx == ipsec->rx_esw) { 757 /* For packet offload in switchdev mode, RX & TX use FDB namespace */ 758 attr->ns = ipsec->tx_esw->ns; 759 mlx5_esw_ipsec_rx_create_attr_set(ipsec, attr); 760 return; 761 } 762 763 attr->ns = mlx5e_fs_get_ns(ipsec->fs, false); 764 attr->ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 765 attr->family = family; 766 attr->prio = MLX5E_NIC_PRIO; 767 attr->pol_level = MLX5E_ACCEL_FS_POL_FT_LEVEL; 768 attr->pol_miss_level = MLX5E_ACCEL_FS_POL_MISS_FT_LEVEL; 769 attr->sa_level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; 770 attr->status_level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; 771 attr->chains_ns = MLX5_FLOW_NAMESPACE_KERNEL; 772} 773 774static int ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, 775 struct mlx5e_ipsec_rx *rx, 776 struct mlx5e_ipsec_rx_create_attr *attr, 777 struct mlx5_flow_destination *dest) 778{ 779 struct mlx5_flow_table *ft; 780 int err; 781 782 if (rx == ipsec->rx_esw) 783 return mlx5_esw_ipsec_rx_status_pass_dest_get(ipsec, dest); 784 785 *dest = ipsec_rx_decrypted_pkt_def_dest(attr->ttc, attr->family); 786 err = mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, attr->ns, dest, 787 attr->family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, 788 attr->prio); 789 if (err) 790 return err; 791 792 ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, attr->family); 793 if (ft) { 794 dest->type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 795 dest->ft = ft; 796 } 797 798 return 0; 799} 800 801static void ipsec_rx_sa_miss_dest_get(struct mlx5e_ipsec *ipsec, 802 struct mlx5e_ipsec_rx *rx, 803 struct mlx5e_ipsec_rx_create_attr *attr, 804 struct mlx5_flow_destination *dest, 805 struct mlx5_flow_destination *miss_dest) 806{ 807 if (rx == ipsec->rx_esw) 808 *miss_dest = *dest; 809 else 810 *miss_dest = 811 mlx5_ttc_get_default_dest(attr->ttc, 812 family2tt(attr->family)); 813} 814 815static void ipsec_rx_default_dest_get(struct mlx5e_ipsec *ipsec, 816 struct mlx5e_ipsec_rx *rx, 817 struct mlx5_flow_destination *dest) 818{ 819 dest->type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 820 dest->ft = rx->pol_miss_ft; 821} 822 823static void ipsec_rx_ft_connect(struct mlx5e_ipsec *ipsec, 824 struct mlx5e_ipsec_rx *rx, 825 struct mlx5e_ipsec_rx_create_attr *attr) 826{ 827 struct mlx5_flow_destination dest = {}; 828 struct mlx5_ttc_table *ttc, *inner_ttc; 829 830 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 831 dest.ft = rx->ft.sa; 832 if (mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest)) 833 return; 834 835 ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 836 inner_ttc = mlx5e_fs_get_ttc(ipsec->fs, true); 837 rx->ttc_rules_added = !mlx5_ttc_create_ipsec_rules(ttc, inner_ttc); 838} 839 840static int ipsec_rx_chains_create_miss(struct mlx5e_ipsec *ipsec, 841 struct mlx5e_ipsec_rx *rx, 842 struct mlx5e_ipsec_rx_create_attr *attr, 843 struct mlx5_flow_destination *dest) 844{ 845 struct mlx5_flow_table_attr ft_attr = {}; 846 MLX5_DECLARE_FLOW_ACT(flow_act); 847 struct mlx5_flow_handle *rule; 848 struct mlx5_flow_table *ft; 849 int err; 850 851 if (rx == ipsec->rx_esw) { 852 /* No need to create miss table for switchdev mode, 853 * just set it to the root chain table. 854 */ 855 rx->pol_miss_ft = dest->ft; 856 return 0; 857 } 858 859 ft_attr.max_fte = 1; 860 ft_attr.autogroup.max_num_groups = 1; 861 ft_attr.level = attr->pol_miss_level; 862 ft_attr.prio = attr->prio; 863 864 ft = mlx5_create_auto_grouped_flow_table(attr->ns, &ft_attr); 865 if (IS_ERR(ft)) 866 return PTR_ERR(ft); 867 868 rule = mlx5_add_flow_rules(ft, NULL, &flow_act, dest, 1); 869 if (IS_ERR(rule)) { 870 err = PTR_ERR(rule); 871 goto err_rule; 872 } 873 874 rx->pol_miss_ft = ft; 875 rx->pol_miss_rule = rule; 876 877 return 0; 878 879err_rule: 880 mlx5_destroy_flow_table(ft); 881 return err; 882} 883 884static int ipsec_rx_policy_create(struct mlx5e_ipsec *ipsec, 885 struct mlx5e_ipsec_rx *rx, 886 struct mlx5e_ipsec_rx_create_attr *attr, 887 struct mlx5_flow_destination *dest) 888{ 889 struct mlx5_flow_destination default_dest; 890 struct mlx5_core_dev *mdev = ipsec->mdev; 891 struct mlx5_flow_table *ft; 892 int err; 893 894 err = ipsec_rx_chains_create_miss(ipsec, rx, attr, dest); 895 if (err) 896 return err; 897 898 ipsec_rx_default_dest_get(ipsec, rx, &default_dest); 899 900 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 901 rx->chains = ipsec_chains_create(mdev, 902 default_dest.ft, 903 attr->chains_ns, 904 attr->prio, 905 attr->sa_level, 906 &rx->ft.pol); 907 if (IS_ERR(rx->chains)) 908 err = PTR_ERR(rx->chains); 909 } else { 910 ft = ipsec_ft_create(attr->ns, attr->pol_level, 911 attr->prio, 1, 2, 0); 912 if (IS_ERR(ft)) { 913 err = PTR_ERR(ft); 914 goto err_out; 915 } 916 rx->ft.pol = ft; 917 918 err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, 919 &default_dest); 920 if (err) 921 mlx5_destroy_flow_table(rx->ft.pol); 922 } 923 924 if (!err) 925 return 0; 926 927err_out: 928 if (rx->pol_miss_rule) { 929 mlx5_del_flow_rules(rx->pol_miss_rule); 930 mlx5_destroy_flow_table(rx->pol_miss_ft); 931 } 932 return err; 933} 934 935static int ipsec_rx_sa_selector_create(struct mlx5e_ipsec *ipsec, 936 struct mlx5e_ipsec_rx *rx, 937 struct mlx5e_ipsec_rx_create_attr *attr) 938{ 939 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 940 struct mlx5_core_dev *mdev = ipsec->mdev; 941 struct mlx5_flow_act flow_act = {}; 942 struct mlx5_flow_destination dest; 943 struct mlx5_flow_handle *rule; 944 struct mlx5_flow_table *ft; 945 struct mlx5_flow_group *fg; 946 u32 *flow_group_in; 947 struct mlx5_fc *fc; 948 int err; 949 950 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 951 if (!flow_group_in) 952 return -ENOMEM; 953 954 ft = ipsec_ft_create(attr->ns, attr->status_level, attr->prio, 1, 955 MLX5_IPSEC_FS_SA_SELECTOR_MAX_NUM_GROUPS, 0); 956 if (IS_ERR(ft)) { 957 err = PTR_ERR(ft); 958 mlx5_core_err(mdev, "Failed to create RX SA selector flow table, err=%d\n", 959 err); 960 goto err_ft; 961 } 962 963 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 964 ft->max_fte - 1); 965 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 966 ft->max_fte - 1); 967 fg = mlx5_create_flow_group(ft, flow_group_in); 968 if (IS_ERR(fg)) { 969 err = PTR_ERR(fg); 970 mlx5_core_err(mdev, "Failed to create RX SA selector miss group, err=%d\n", 971 err); 972 goto err_fg; 973 } 974 975 fc = mlx5_fc_create(mdev, false); 976 if (IS_ERR(fc)) { 977 err = PTR_ERR(fc); 978 mlx5_core_err(mdev, 979 "Failed to create ipsec RX SA selector miss rule counter, err=%d\n", 980 err); 981 goto err_cnt; 982 } 983 984 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 985 dest.counter = fc; 986 flow_act.action = 987 MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_DROP; 988 989 rule = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1); 990 if (IS_ERR(rule)) { 991 err = PTR_ERR(rule); 992 mlx5_core_err(mdev, "Failed to create RX SA selector miss drop rule, err=%d\n", 993 err); 994 goto err_rule; 995 } 996 997 rx->ft.sa_sel = ft; 998 rx->sa_sel.group = fg; 999 rx->sa_sel.fc = fc; 1000 rx->sa_sel.rule = rule; 1001 1002 kvfree(flow_group_in); 1003 1004 return 0; 1005 1006err_rule: 1007 mlx5_fc_destroy(mdev, fc); 1008err_cnt: 1009 mlx5_destroy_flow_group(fg); 1010err_fg: 1011 mlx5_destroy_flow_table(ft); 1012err_ft: 1013 kvfree(flow_group_in); 1014 return err; 1015} 1016 1017/* The decryption processing is as follows: 1018 * 1019 * +----------+ +-------------+ 1020 * | | | | 1021 * | Kernel <--------------+----------+ policy miss <------------+ 1022 * | | ^ | | ^ 1023 * +----^-----+ | +-------------+ | 1024 * | crypto | 1025 * miss offload ok allow/default 1026 * ^ ^ ^ 1027 * | | packet | 1028 * +----+---------+ +----+-------------+ offload ok +------+---+ 1029 * | | | | (no UPSPEC) | | 1030 * | SA (decrypt) +-----> status +--->------->----+ policy | 1031 * | | | | | | 1032 * +--------------+ ++---------+-------+ +-^----+---+ 1033 * | | | | 1034 * v packet +-->->---+ v 1035 * | offload ok match | 1036 * fails (with UPSPEC) | block 1037 * | | +-------------+-+ | 1038 * v v | | miss v 1039 * drop +---> SA sel +--------->drop 1040 * | | 1041 * +---------------+ 1042 */ 1043 1044static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 1045 struct mlx5e_ipsec_rx *rx, u32 family) 1046{ 1047 struct mlx5_flow_destination dest[2], miss_dest; 1048 struct mlx5e_ipsec_rx_create_attr attr; 1049 struct mlx5_flow_table *ft; 1050 u32 flags = 0; 1051 int err; 1052 1053 ipsec_rx_create_attr_set(ipsec, rx, family, &attr); 1054 1055 err = ipsec_rx_status_pass_dest_get(ipsec, rx, &attr, &dest[0]); 1056 if (err) 1057 return err; 1058 1059 ft = ipsec_ft_create(attr.ns, attr.status_level, attr.prio, 3, 4, 0); 1060 if (IS_ERR(ft)) { 1061 err = PTR_ERR(ft); 1062 goto err_fs_ft_status; 1063 } 1064 rx->ft.status = ft; 1065 1066 err = ipsec_rx_sa_selector_create(ipsec, rx, &attr); 1067 if (err) 1068 goto err_fs_ft_sa_sel; 1069 1070 /* Create FT */ 1071 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) 1072 rx->allow_tunnel_mode = 1073 mlx5_eswitch_block_encap(mdev, rx == ipsec->rx_esw); 1074 1075 if (rx->allow_tunnel_mode) 1076 flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1077 ft = ipsec_ft_create(attr.ns, attr.sa_level, attr.prio, 1, 2, flags); 1078 if (IS_ERR(ft)) { 1079 err = PTR_ERR(ft); 1080 goto err_fs_ft; 1081 } 1082 rx->ft.sa = ft; 1083 1084 ipsec_rx_sa_miss_dest_get(ipsec, rx, &attr, &dest[0], &miss_dest); 1085 err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, &miss_dest); 1086 if (err) 1087 goto err_fs; 1088 1089 err = ipsec_rx_policy_create(ipsec, rx, &attr, &dest[0]); 1090 if (err) 1091 goto err_policy; 1092 1093 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1094 dest[1].counter = rx->fc->cnt; 1095 err = mlx5_ipsec_rx_status_create(ipsec, rx, dest); 1096 if (err) 1097 goto err_add; 1098 1099 /* connect */ 1100 if (rx != ipsec->rx_esw) 1101 ipsec_rx_ft_connect(ipsec, rx, &attr); 1102 return 0; 1103 1104err_add: 1105 ipsec_rx_policy_destroy(rx); 1106err_policy: 1107 mlx5_del_flow_rules(rx->sa.rule); 1108 mlx5_destroy_flow_group(rx->sa.group); 1109err_fs: 1110 mlx5_destroy_flow_table(rx->ft.sa); 1111 if (rx->allow_tunnel_mode) 1112 mlx5_eswitch_unblock_encap(mdev); 1113err_fs_ft: 1114 ipsec_rx_sa_selector_destroy(mdev, rx); 1115err_fs_ft_sa_sel: 1116 mlx5_destroy_flow_table(rx->ft.status); 1117err_fs_ft_status: 1118 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, mdev); 1119 return err; 1120} 1121 1122static int rx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 1123 struct mlx5e_ipsec_rx *rx, u32 family) 1124{ 1125 int err; 1126 1127 if (rx->ft.refcnt) 1128 goto skip; 1129 1130 err = mlx5_eswitch_block_mode(mdev); 1131 if (err) 1132 return err; 1133 1134 err = rx_create(mdev, ipsec, rx, family); 1135 if (err) { 1136 mlx5_eswitch_unblock_mode(mdev); 1137 return err; 1138 } 1139 1140skip: 1141 rx->ft.refcnt++; 1142 return 0; 1143} 1144 1145static void rx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx, 1146 u32 family) 1147{ 1148 if (--rx->ft.refcnt) 1149 return; 1150 1151 rx_destroy(ipsec->mdev, ipsec, rx, family); 1152 mlx5_eswitch_unblock_mode(ipsec->mdev); 1153} 1154 1155static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, 1156 struct mlx5e_ipsec *ipsec, u32 family, 1157 int type) 1158{ 1159 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 1160 int err; 1161 1162 mutex_lock(&rx->ft.mutex); 1163 err = rx_get(mdev, ipsec, rx, family); 1164 mutex_unlock(&rx->ft.mutex); 1165 if (err) 1166 return ERR_PTR(err); 1167 1168 return rx; 1169} 1170 1171static struct mlx5_flow_table *rx_ft_get_policy(struct mlx5_core_dev *mdev, 1172 struct mlx5e_ipsec *ipsec, 1173 u32 family, u32 prio, int type) 1174{ 1175 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 1176 struct mlx5_flow_table *ft; 1177 int err; 1178 1179 mutex_lock(&rx->ft.mutex); 1180 err = rx_get(mdev, ipsec, rx, family); 1181 if (err) 1182 goto err_get; 1183 1184 ft = rx->chains ? ipsec_chains_get_table(rx->chains, prio) : rx->ft.pol; 1185 if (IS_ERR(ft)) { 1186 err = PTR_ERR(ft); 1187 goto err_get_ft; 1188 } 1189 1190 mutex_unlock(&rx->ft.mutex); 1191 return ft; 1192 1193err_get_ft: 1194 rx_put(ipsec, rx, family); 1195err_get: 1196 mutex_unlock(&rx->ft.mutex); 1197 return ERR_PTR(err); 1198} 1199 1200static void rx_ft_put(struct mlx5e_ipsec *ipsec, u32 family, int type) 1201{ 1202 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 1203 1204 mutex_lock(&rx->ft.mutex); 1205 rx_put(ipsec, rx, family); 1206 mutex_unlock(&rx->ft.mutex); 1207} 1208 1209static void rx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 family, u32 prio, int type) 1210{ 1211 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 1212 1213 mutex_lock(&rx->ft.mutex); 1214 if (rx->chains) 1215 ipsec_chains_put_table(rx->chains, prio); 1216 1217 rx_put(ipsec, rx, family); 1218 mutex_unlock(&rx->ft.mutex); 1219} 1220 1221static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 1222{ 1223 struct mlx5_flow_destination dest = {}; 1224 struct mlx5_flow_act flow_act = {}; 1225 struct mlx5_flow_handle *fte; 1226 struct mlx5_flow_spec *spec; 1227 int err; 1228 1229 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1230 if (!spec) 1231 return -ENOMEM; 1232 1233 /* create fte */ 1234 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | 1235 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1236 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1237 dest.counter = tx->fc->cnt; 1238 fte = mlx5_add_flow_rules(tx->ft.status, spec, &flow_act, &dest, 1); 1239 if (IS_ERR(fte)) { 1240 err = PTR_ERR(fte); 1241 mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err); 1242 goto err_rule; 1243 } 1244 1245 kvfree(spec); 1246 tx->status.rule = fte; 1247 return 0; 1248 1249err_rule: 1250 kvfree(spec); 1251 return err; 1252} 1253 1254/* IPsec TX flow steering */ 1255static void tx_destroy(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx, 1256 struct mlx5_ipsec_fs *roce) 1257{ 1258 mlx5_ipsec_fs_roce_tx_destroy(roce, ipsec->mdev); 1259 if (tx->chains) { 1260 ipsec_chains_destroy(tx->chains); 1261 } else { 1262 mlx5_del_flow_rules(tx->pol.rule); 1263 mlx5_destroy_flow_group(tx->pol.group); 1264 mlx5_destroy_flow_table(tx->ft.pol); 1265 } 1266 1267 if (tx == ipsec->tx_esw) { 1268 mlx5_del_flow_rules(tx->sa.rule); 1269 mlx5_destroy_flow_group(tx->sa.group); 1270 } 1271 mlx5_destroy_flow_table(tx->ft.sa); 1272 if (tx->allow_tunnel_mode) 1273 mlx5_eswitch_unblock_encap(ipsec->mdev); 1274 mlx5_del_flow_rules(tx->status.rule); 1275 mlx5_destroy_flow_table(tx->ft.status); 1276} 1277 1278static void ipsec_tx_create_attr_set(struct mlx5e_ipsec *ipsec, 1279 struct mlx5e_ipsec_tx *tx, 1280 struct mlx5e_ipsec_tx_create_attr *attr) 1281{ 1282 if (tx == ipsec->tx_esw) { 1283 mlx5_esw_ipsec_tx_create_attr_set(ipsec, attr); 1284 return; 1285 } 1286 1287 attr->prio = 0; 1288 attr->pol_level = 0; 1289 attr->sa_level = 1; 1290 attr->cnt_level = 2; 1291 attr->chains_ns = MLX5_FLOW_NAMESPACE_EGRESS_IPSEC; 1292} 1293 1294static int tx_create(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx, 1295 struct mlx5_ipsec_fs *roce) 1296{ 1297 struct mlx5_core_dev *mdev = ipsec->mdev; 1298 struct mlx5e_ipsec_tx_create_attr attr; 1299 struct mlx5_flow_destination dest = {}; 1300 struct mlx5_flow_table *ft; 1301 u32 flags = 0; 1302 int err; 1303 1304 ipsec_tx_create_attr_set(ipsec, tx, &attr); 1305 ft = ipsec_ft_create(tx->ns, attr.cnt_level, attr.prio, 1, 1, 0); 1306 if (IS_ERR(ft)) 1307 return PTR_ERR(ft); 1308 tx->ft.status = ft; 1309 1310 err = ipsec_counter_rule_tx(mdev, tx); 1311 if (err) 1312 goto err_status_rule; 1313 1314 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) 1315 tx->allow_tunnel_mode = 1316 mlx5_eswitch_block_encap(mdev, tx == ipsec->tx_esw); 1317 1318 if (tx->allow_tunnel_mode) 1319 flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1320 ft = ipsec_ft_create(tx->ns, attr.sa_level, attr.prio, 1, 4, flags); 1321 if (IS_ERR(ft)) { 1322 err = PTR_ERR(ft); 1323 goto err_sa_ft; 1324 } 1325 tx->ft.sa = ft; 1326 1327 if (tx == ipsec->tx_esw) { 1328 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1329 dest.vport.num = MLX5_VPORT_UPLINK; 1330 err = ipsec_miss_create(mdev, tx->ft.sa, &tx->sa, &dest); 1331 if (err) 1332 goto err_sa_miss; 1333 memset(&dest, 0, sizeof(dest)); 1334 } 1335 1336 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 1337 tx->chains = ipsec_chains_create( 1338 mdev, tx->ft.sa, attr.chains_ns, attr.prio, attr.pol_level, 1339 &tx->ft.pol); 1340 if (IS_ERR(tx->chains)) { 1341 err = PTR_ERR(tx->chains); 1342 goto err_pol_ft; 1343 } 1344 1345 goto connect_roce; 1346 } 1347 1348 ft = ipsec_ft_create(tx->ns, attr.pol_level, attr.prio, 1, 2, 0); 1349 if (IS_ERR(ft)) { 1350 err = PTR_ERR(ft); 1351 goto err_pol_ft; 1352 } 1353 tx->ft.pol = ft; 1354 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1355 dest.ft = tx->ft.sa; 1356 err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); 1357 if (err) { 1358 mlx5_destroy_flow_table(tx->ft.pol); 1359 goto err_pol_ft; 1360 } 1361 1362connect_roce: 1363 err = mlx5_ipsec_fs_roce_tx_create(mdev, roce, tx->ft.pol, false); 1364 if (err) 1365 goto err_roce; 1366 return 0; 1367 1368err_roce: 1369 if (tx->chains) { 1370 ipsec_chains_destroy(tx->chains); 1371 } else { 1372 mlx5_del_flow_rules(tx->pol.rule); 1373 mlx5_destroy_flow_group(tx->pol.group); 1374 mlx5_destroy_flow_table(tx->ft.pol); 1375 } 1376err_pol_ft: 1377 if (tx == ipsec->tx_esw) { 1378 mlx5_del_flow_rules(tx->sa.rule); 1379 mlx5_destroy_flow_group(tx->sa.group); 1380 } 1381err_sa_miss: 1382 mlx5_destroy_flow_table(tx->ft.sa); 1383err_sa_ft: 1384 if (tx->allow_tunnel_mode) 1385 mlx5_eswitch_unblock_encap(mdev); 1386 mlx5_del_flow_rules(tx->status.rule); 1387err_status_rule: 1388 mlx5_destroy_flow_table(tx->ft.status); 1389 return err; 1390} 1391 1392static void ipsec_esw_tx_ft_policy_set(struct mlx5_core_dev *mdev, 1393 struct mlx5_flow_table *ft) 1394{ 1395#ifdef CONFIG_MLX5_ESWITCH 1396 struct mlx5_eswitch *esw = mdev->priv.eswitch; 1397 struct mlx5e_rep_priv *uplink_rpriv; 1398 struct mlx5e_priv *priv; 1399 1400 esw->offloads.ft_ipsec_tx_pol = ft; 1401 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); 1402 priv = netdev_priv(uplink_rpriv->netdev); 1403 if (!priv->channels.num) 1404 return; 1405 1406 mlx5e_rep_deactivate_channels(priv); 1407 mlx5e_rep_activate_channels(priv); 1408#endif 1409} 1410 1411static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 1412 struct mlx5e_ipsec_tx *tx) 1413{ 1414 int err; 1415 1416 if (tx->ft.refcnt) 1417 goto skip; 1418 1419 err = mlx5_eswitch_block_mode(mdev); 1420 if (err) 1421 return err; 1422 1423 err = tx_create(ipsec, tx, ipsec->roce); 1424 if (err) { 1425 mlx5_eswitch_unblock_mode(mdev); 1426 return err; 1427 } 1428 1429 if (tx == ipsec->tx_esw) 1430 ipsec_esw_tx_ft_policy_set(mdev, tx->ft.pol); 1431 1432skip: 1433 tx->ft.refcnt++; 1434 return 0; 1435} 1436 1437static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) 1438{ 1439 if (--tx->ft.refcnt) 1440 return; 1441 1442 if (tx == ipsec->tx_esw) { 1443 mlx5_esw_ipsec_restore_dest_uplink(ipsec->mdev); 1444 ipsec_esw_tx_ft_policy_set(ipsec->mdev, NULL); 1445 } 1446 1447 tx_destroy(ipsec, tx, ipsec->roce); 1448 mlx5_eswitch_unblock_mode(ipsec->mdev); 1449} 1450 1451static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev, 1452 struct mlx5e_ipsec *ipsec, 1453 u32 prio, int type) 1454{ 1455 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 1456 struct mlx5_flow_table *ft; 1457 int err; 1458 1459 mutex_lock(&tx->ft.mutex); 1460 err = tx_get(mdev, ipsec, tx); 1461 if (err) 1462 goto err_get; 1463 1464 ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol; 1465 if (IS_ERR(ft)) { 1466 err = PTR_ERR(ft); 1467 goto err_get_ft; 1468 } 1469 1470 mutex_unlock(&tx->ft.mutex); 1471 return ft; 1472 1473err_get_ft: 1474 tx_put(ipsec, tx); 1475err_get: 1476 mutex_unlock(&tx->ft.mutex); 1477 return ERR_PTR(err); 1478} 1479 1480static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, 1481 struct mlx5e_ipsec *ipsec, int type) 1482{ 1483 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 1484 int err; 1485 1486 mutex_lock(&tx->ft.mutex); 1487 err = tx_get(mdev, ipsec, tx); 1488 mutex_unlock(&tx->ft.mutex); 1489 if (err) 1490 return ERR_PTR(err); 1491 1492 return tx; 1493} 1494 1495static void tx_ft_put(struct mlx5e_ipsec *ipsec, int type) 1496{ 1497 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 1498 1499 mutex_lock(&tx->ft.mutex); 1500 tx_put(ipsec, tx); 1501 mutex_unlock(&tx->ft.mutex); 1502} 1503 1504static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio, int type) 1505{ 1506 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 1507 1508 mutex_lock(&tx->ft.mutex); 1509 if (tx->chains) 1510 ipsec_chains_put_table(tx->chains, prio); 1511 1512 tx_put(ipsec, tx); 1513 mutex_unlock(&tx->ft.mutex); 1514} 1515 1516static void setup_fte_addr4(struct mlx5_flow_spec *spec, 1517 struct mlx5e_ipsec_addr *addrs) 1518{ 1519 __be32 *saddr = &addrs->saddr.a4; 1520 __be32 *smask = &addrs->smask.m4; 1521 __be32 *daddr = &addrs->daddr.a4; 1522 __be32 *dmask = &addrs->dmask.m4; 1523 1524 if (!*saddr && !*daddr) 1525 return; 1526 1527 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1528 1529 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 1530 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 1531 1532 if (*saddr) { 1533 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1534 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); 1535 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1536 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), smask, 4); 1537 } 1538 1539 if (*daddr) { 1540 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1541 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); 1542 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1543 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), dmask, 4); 1544 } 1545} 1546 1547static void setup_fte_addr6(struct mlx5_flow_spec *spec, 1548 struct mlx5e_ipsec_addr *addrs) 1549{ 1550 __be32 *saddr = addrs->saddr.a6; 1551 __be32 *smask = addrs->smask.m6; 1552 __be32 *daddr = addrs->daddr.a6; 1553 __be32 *dmask = addrs->dmask.m6; 1554 1555 if (addr6_all_zero(saddr) && addr6_all_zero(daddr)) 1556 return; 1557 1558 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1559 1560 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 1561 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 1562 1563 if (!addr6_all_zero(saddr)) { 1564 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1565 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); 1566 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1567 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), dmask, 16); 1568 } 1569 1570 if (!addr6_all_zero(daddr)) { 1571 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1572 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); 1573 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1574 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), smask, 16); 1575 } 1576} 1577 1578static void setup_fte_esp(struct mlx5_flow_spec *spec) 1579{ 1580 /* ESP header */ 1581 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1582 1583 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 1584 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); 1585} 1586 1587static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi, bool encap) 1588{ 1589 /* SPI number */ 1590 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 1591 1592 if (encap) { 1593 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1594 misc_parameters.inner_esp_spi); 1595 MLX5_SET(fte_match_param, spec->match_value, 1596 misc_parameters.inner_esp_spi, spi); 1597 } else { 1598 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1599 misc_parameters.outer_esp_spi); 1600 MLX5_SET(fte_match_param, spec->match_value, 1601 misc_parameters.outer_esp_spi, spi); 1602 } 1603} 1604 1605static void setup_fte_no_frags(struct mlx5_flow_spec *spec) 1606{ 1607 /* Non fragmented */ 1608 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1609 1610 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); 1611 MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); 1612} 1613 1614static void setup_fte_reg_a(struct mlx5_flow_spec *spec) 1615{ 1616 /* Add IPsec indicator in metadata_reg_a */ 1617 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1618 1619 MLX5_SET(fte_match_param, spec->match_criteria, 1620 misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); 1621 MLX5_SET(fte_match_param, spec->match_value, 1622 misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); 1623} 1624 1625static void setup_fte_reg_c4(struct mlx5_flow_spec *spec, u32 reqid) 1626{ 1627 /* Pass policy check before choosing this SA */ 1628 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 1629 1630 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1631 misc_parameters_2.metadata_reg_c_4); 1632 MLX5_SET(fte_match_param, spec->match_value, 1633 misc_parameters_2.metadata_reg_c_4, reqid); 1634} 1635 1636static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec) 1637{ 1638 switch (upspec->proto) { 1639 case IPPROTO_UDP: 1640 if (upspec->dport) { 1641 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1642 udp_dport, upspec->dport_mask); 1643 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1644 udp_dport, upspec->dport); 1645 } 1646 if (upspec->sport) { 1647 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1648 udp_sport, upspec->sport_mask); 1649 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1650 udp_sport, upspec->sport); 1651 } 1652 break; 1653 case IPPROTO_TCP: 1654 if (upspec->dport) { 1655 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1656 tcp_dport, upspec->dport_mask); 1657 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1658 tcp_dport, upspec->dport); 1659 } 1660 if (upspec->sport) { 1661 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1662 tcp_sport, upspec->sport_mask); 1663 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1664 tcp_sport, upspec->sport); 1665 } 1666 break; 1667 default: 1668 return; 1669 } 1670 1671 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1672 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol); 1673 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto); 1674} 1675 1676static enum mlx5_flow_namespace_type ipsec_fs_get_ns(struct mlx5e_ipsec *ipsec, 1677 int type, u8 dir) 1678{ 1679 if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) 1680 return MLX5_FLOW_NAMESPACE_FDB; 1681 1682 if (dir == XFRM_DEV_OFFLOAD_IN) 1683 return MLX5_FLOW_NAMESPACE_KERNEL; 1684 1685 return MLX5_FLOW_NAMESPACE_EGRESS; 1686} 1687 1688static int setup_modify_header(struct mlx5e_ipsec *ipsec, int type, u32 val, u8 dir, 1689 struct mlx5_flow_act *flow_act) 1690{ 1691 enum mlx5_flow_namespace_type ns_type = ipsec_fs_get_ns(ipsec, type, dir); 1692 u8 action[3][MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1693 struct mlx5_core_dev *mdev = ipsec->mdev; 1694 struct mlx5_modify_hdr *modify_hdr; 1695 u8 num_of_actions = 1; 1696 1697 MLX5_SET(set_action_in, action[0], action_type, MLX5_ACTION_TYPE_SET); 1698 switch (dir) { 1699 case XFRM_DEV_OFFLOAD_IN: 1700 MLX5_SET(set_action_in, action[0], field, 1701 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1702 1703 num_of_actions++; 1704 MLX5_SET(set_action_in, action[1], action_type, MLX5_ACTION_TYPE_SET); 1705 MLX5_SET(set_action_in, action[1], field, MLX5_ACTION_IN_FIELD_METADATA_REG_C_2); 1706 MLX5_SET(set_action_in, action[1], data, val); 1707 MLX5_SET(set_action_in, action[1], offset, 0); 1708 MLX5_SET(set_action_in, action[1], length, 32); 1709 1710 if (type == XFRM_DEV_OFFLOAD_CRYPTO) { 1711 num_of_actions++; 1712 MLX5_SET(set_action_in, action[2], action_type, 1713 MLX5_ACTION_TYPE_SET); 1714 MLX5_SET(set_action_in, action[2], field, 1715 MLX5_ACTION_IN_FIELD_METADATA_REG_C_4); 1716 MLX5_SET(set_action_in, action[2], data, 1717 MLX5_IPSEC_ASO_SW_CRYPTO_OFFLOAD); 1718 MLX5_SET(set_action_in, action[2], offset, 0); 1719 MLX5_SET(set_action_in, action[2], length, 32); 1720 } 1721 break; 1722 case XFRM_DEV_OFFLOAD_OUT: 1723 MLX5_SET(set_action_in, action[0], field, 1724 MLX5_ACTION_IN_FIELD_METADATA_REG_C_4); 1725 break; 1726 default: 1727 return -EINVAL; 1728 } 1729 1730 MLX5_SET(set_action_in, action[0], data, val); 1731 MLX5_SET(set_action_in, action[0], offset, 0); 1732 MLX5_SET(set_action_in, action[0], length, 32); 1733 1734 modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, num_of_actions, action); 1735 if (IS_ERR(modify_hdr)) { 1736 mlx5_core_err(mdev, "Failed to allocate modify_header %pe\n", 1737 modify_hdr); 1738 return PTR_ERR(modify_hdr); 1739 } 1740 1741 flow_act->modify_hdr = modify_hdr; 1742 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1743 return 0; 1744} 1745 1746static int 1747setup_pkt_tunnel_reformat(struct mlx5_core_dev *mdev, 1748 struct mlx5_accel_esp_xfrm_attrs *attrs, 1749 struct mlx5_pkt_reformat_params *reformat_params) 1750{ 1751 struct ip_esp_hdr *esp_hdr; 1752 struct ipv6hdr *ipv6hdr; 1753 struct ethhdr *eth_hdr; 1754 struct iphdr *iphdr; 1755 char *reformatbf; 1756 size_t bfflen; 1757 void *hdr; 1758 1759 bfflen = sizeof(*eth_hdr); 1760 1761 if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) { 1762 bfflen += sizeof(*esp_hdr) + 8; 1763 1764 switch (attrs->addrs.family) { 1765 case AF_INET: 1766 bfflen += sizeof(*iphdr); 1767 break; 1768 case AF_INET6: 1769 bfflen += sizeof(*ipv6hdr); 1770 break; 1771 default: 1772 return -EINVAL; 1773 } 1774 } 1775 1776 reformatbf = kzalloc(bfflen, GFP_KERNEL); 1777 if (!reformatbf) 1778 return -ENOMEM; 1779 1780 eth_hdr = (struct ethhdr *)reformatbf; 1781 switch (attrs->addrs.family) { 1782 case AF_INET: 1783 eth_hdr->h_proto = htons(ETH_P_IP); 1784 break; 1785 case AF_INET6: 1786 eth_hdr->h_proto = htons(ETH_P_IPV6); 1787 break; 1788 default: 1789 goto free_reformatbf; 1790 } 1791 1792 ether_addr_copy(eth_hdr->h_dest, attrs->dmac); 1793 ether_addr_copy(eth_hdr->h_source, attrs->smac); 1794 1795 switch (attrs->dir) { 1796 case XFRM_DEV_OFFLOAD_IN: 1797 reformat_params->type = MLX5_REFORMAT_TYPE_L3_ESP_TUNNEL_TO_L2; 1798 break; 1799 case XFRM_DEV_OFFLOAD_OUT: 1800 reformat_params->type = MLX5_REFORMAT_TYPE_L2_TO_L3_ESP_TUNNEL; 1801 reformat_params->param_0 = attrs->authsize; 1802 1803 hdr = reformatbf + sizeof(*eth_hdr); 1804 switch (attrs->addrs.family) { 1805 case AF_INET: 1806 iphdr = (struct iphdr *)hdr; 1807 memcpy(&iphdr->saddr, &attrs->addrs.saddr.a4, 4); 1808 memcpy(&iphdr->daddr, &attrs->addrs.daddr.a4, 4); 1809 iphdr->version = 4; 1810 iphdr->ihl = 5; 1811 iphdr->ttl = IPSEC_TUNNEL_DEFAULT_TTL; 1812 iphdr->protocol = IPPROTO_ESP; 1813 hdr += sizeof(*iphdr); 1814 break; 1815 case AF_INET6: 1816 ipv6hdr = (struct ipv6hdr *)hdr; 1817 memcpy(&ipv6hdr->saddr, &attrs->addrs.saddr.a6, 16); 1818 memcpy(&ipv6hdr->daddr, &attrs->addrs.daddr.a6, 16); 1819 ipv6hdr->nexthdr = IPPROTO_ESP; 1820 ipv6hdr->version = 6; 1821 ipv6hdr->hop_limit = IPSEC_TUNNEL_DEFAULT_TTL; 1822 hdr += sizeof(*ipv6hdr); 1823 break; 1824 default: 1825 goto free_reformatbf; 1826 } 1827 1828 esp_hdr = (struct ip_esp_hdr *)hdr; 1829 esp_hdr->spi = htonl(attrs->spi); 1830 break; 1831 default: 1832 goto free_reformatbf; 1833 } 1834 1835 reformat_params->size = bfflen; 1836 reformat_params->data = reformatbf; 1837 return 0; 1838 1839free_reformatbf: 1840 kfree(reformatbf); 1841 return -EINVAL; 1842} 1843 1844static int get_reformat_type(struct mlx5_accel_esp_xfrm_attrs *attrs) 1845{ 1846 switch (attrs->dir) { 1847 case XFRM_DEV_OFFLOAD_IN: 1848 if (attrs->encap) 1849 return MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP; 1850 return MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; 1851 case XFRM_DEV_OFFLOAD_OUT: 1852 if (attrs->addrs.family == AF_INET) { 1853 if (attrs->encap) 1854 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4; 1855 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; 1856 } 1857 1858 if (attrs->encap) 1859 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6; 1860 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; 1861 default: 1862 WARN_ON(true); 1863 } 1864 1865 return -EINVAL; 1866} 1867 1868static int 1869setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs, 1870 struct mlx5_pkt_reformat_params *reformat_params) 1871{ 1872 struct udphdr *udphdr; 1873 char *reformatbf; 1874 size_t bfflen; 1875 __be32 spi; 1876 void *hdr; 1877 1878 reformat_params->type = get_reformat_type(attrs); 1879 if (reformat_params->type < 0) 1880 return reformat_params->type; 1881 1882 switch (attrs->dir) { 1883 case XFRM_DEV_OFFLOAD_IN: 1884 break; 1885 case XFRM_DEV_OFFLOAD_OUT: 1886 bfflen = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE; 1887 if (attrs->encap) 1888 bfflen += sizeof(*udphdr); 1889 1890 reformatbf = kzalloc(bfflen, GFP_KERNEL); 1891 if (!reformatbf) 1892 return -ENOMEM; 1893 1894 hdr = reformatbf; 1895 if (attrs->encap) { 1896 udphdr = (struct udphdr *)reformatbf; 1897 udphdr->source = attrs->sport; 1898 udphdr->dest = attrs->dport; 1899 hdr += sizeof(*udphdr); 1900 } 1901 1902 /* convert to network format */ 1903 spi = htonl(attrs->spi); 1904 memcpy(hdr, &spi, sizeof(spi)); 1905 1906 reformat_params->param_0 = attrs->authsize; 1907 reformat_params->size = bfflen; 1908 reformat_params->data = reformatbf; 1909 break; 1910 default: 1911 return -EINVAL; 1912 } 1913 1914 return 0; 1915} 1916 1917static int setup_pkt_reformat(struct mlx5e_ipsec *ipsec, 1918 struct mlx5_accel_esp_xfrm_attrs *attrs, 1919 struct mlx5_flow_act *flow_act) 1920{ 1921 enum mlx5_flow_namespace_type ns_type = ipsec_fs_get_ns(ipsec, attrs->type, 1922 attrs->dir); 1923 struct mlx5_pkt_reformat_params reformat_params = {}; 1924 struct mlx5_core_dev *mdev = ipsec->mdev; 1925 struct mlx5_pkt_reformat *pkt_reformat; 1926 int ret; 1927 1928 switch (attrs->mode) { 1929 case XFRM_MODE_TRANSPORT: 1930 ret = setup_pkt_transport_reformat(attrs, &reformat_params); 1931 break; 1932 case XFRM_MODE_TUNNEL: 1933 ret = setup_pkt_tunnel_reformat(mdev, attrs, &reformat_params); 1934 break; 1935 default: 1936 ret = -EINVAL; 1937 } 1938 1939 if (ret) 1940 return ret; 1941 1942 pkt_reformat = 1943 mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); 1944 kfree(reformat_params.data); 1945 if (IS_ERR(pkt_reformat)) 1946 return PTR_ERR(pkt_reformat); 1947 1948 flow_act->pkt_reformat = pkt_reformat; 1949 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 1950 return 0; 1951} 1952 1953static int rx_add_rule_sa_selector(struct mlx5e_ipsec_sa_entry *sa_entry, 1954 struct mlx5e_ipsec_rx *rx, 1955 struct upspec *upspec) 1956{ 1957 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 1958 struct mlx5_core_dev *mdev = ipsec->mdev; 1959 struct mlx5_flow_destination dest[2]; 1960 struct mlx5_flow_act flow_act = {}; 1961 struct mlx5_flow_handle *rule; 1962 struct mlx5_flow_spec *spec; 1963 int err = 0; 1964 1965 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1966 if (!spec) 1967 return -ENOMEM; 1968 1969 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1970 misc_parameters_2.ipsec_syndrome); 1971 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1972 misc_parameters_2.metadata_reg_c_4); 1973 MLX5_SET(fte_match_param, spec->match_value, 1974 misc_parameters_2.ipsec_syndrome, 0); 1975 MLX5_SET(fte_match_param, spec->match_value, 1976 misc_parameters_2.metadata_reg_c_4, 0); 1977 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1978 1979 ipsec_rx_rule_add_match_obj(sa_entry, rx, spec); 1980 1981 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1982 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1983 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; 1984 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1985 dest[0].ft = rx->ft.sa_sel; 1986 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1987 dest[1].counter = rx->fc->cnt; 1988 1989 rule = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); 1990 if (IS_ERR(rule)) { 1991 err = PTR_ERR(rule); 1992 mlx5_core_err(mdev, 1993 "Failed to add ipsec rx pass rule, err=%d\n", 1994 err); 1995 goto err_add_status_pass_rule; 1996 } 1997 1998 sa_entry->ipsec_rule.status_pass = rule; 1999 2000 MLX5_SET(fte_match_param, spec->match_criteria, 2001 misc_parameters_2.ipsec_syndrome, 0); 2002 MLX5_SET(fte_match_param, spec->match_criteria, 2003 misc_parameters_2.metadata_reg_c_4, 0); 2004 2005 setup_fte_upper_proto_match(spec, upspec); 2006 2007 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2008 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2009 dest[0].ft = rx->ft.pol; 2010 2011 rule = mlx5_add_flow_rules(rx->ft.sa_sel, spec, &flow_act, &dest[0], 1); 2012 if (IS_ERR(rule)) { 2013 err = PTR_ERR(rule); 2014 mlx5_core_err(mdev, 2015 "Failed to add ipsec rx sa selector rule, err=%d\n", 2016 err); 2017 goto err_add_sa_sel_rule; 2018 } 2019 2020 sa_entry->ipsec_rule.sa_sel = rule; 2021 2022 kvfree(spec); 2023 return 0; 2024 2025err_add_sa_sel_rule: 2026 mlx5_del_flow_rules(sa_entry->ipsec_rule.status_pass); 2027err_add_status_pass_rule: 2028 kvfree(spec); 2029 return err; 2030} 2031 2032static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 2033{ 2034 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 2035 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 2036 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 2037 struct mlx5_flow_destination dest[2]; 2038 struct mlx5_flow_act flow_act = {}; 2039 struct mlx5_flow_handle *rule; 2040 struct mlx5_flow_spec *spec; 2041 struct mlx5e_ipsec_rx *rx; 2042 struct mlx5_fc *counter; 2043 int err = 0; 2044 2045 rx = rx_ft_get(mdev, ipsec, attrs->addrs.family, attrs->type); 2046 if (IS_ERR(rx)) 2047 return PTR_ERR(rx); 2048 2049 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2050 if (!spec) { 2051 err = -ENOMEM; 2052 goto err_alloc; 2053 } 2054 2055 if (attrs->addrs.family == AF_INET) 2056 setup_fte_addr4(spec, &attrs->addrs); 2057 else 2058 setup_fte_addr6(spec, &attrs->addrs); 2059 2060 setup_fte_spi(spec, attrs->spi, attrs->encap); 2061 if (!attrs->encap) 2062 setup_fte_esp(spec); 2063 setup_fte_no_frags(spec); 2064 2065 if (!attrs->drop) { 2066 if (rx != ipsec->rx_esw) 2067 err = setup_modify_header(ipsec, attrs->type, 2068 sa_entry->ipsec_obj_id | BIT(31), 2069 XFRM_DEV_OFFLOAD_IN, &flow_act); 2070 else 2071 err = mlx5_esw_ipsec_rx_setup_modify_header(sa_entry, &flow_act); 2072 2073 if (err) 2074 goto err_mod_header; 2075 } 2076 2077 switch (attrs->type) { 2078 case XFRM_DEV_OFFLOAD_PACKET: 2079 err = setup_pkt_reformat(ipsec, attrs, &flow_act); 2080 if (err) 2081 goto err_pkt_reformat; 2082 break; 2083 default: 2084 break; 2085 } 2086 2087 counter = mlx5_fc_create(mdev, true); 2088 if (IS_ERR(counter)) { 2089 err = PTR_ERR(counter); 2090 goto err_add_cnt; 2091 } 2092 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 2093 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 2094 flow_act.flags |= FLOW_ACT_NO_APPEND; 2095 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | 2096 MLX5_FLOW_CONTEXT_ACTION_COUNT; 2097 if (attrs->drop) 2098 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 2099 else 2100 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2101 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2102 dest[0].ft = rx->ft.status; 2103 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2104 dest[1].counter = counter; 2105 rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2); 2106 if (IS_ERR(rule)) { 2107 err = PTR_ERR(rule); 2108 mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); 2109 goto err_add_flow; 2110 } 2111 2112 if (attrs->upspec.proto && attrs->type == XFRM_DEV_OFFLOAD_PACKET) { 2113 err = rx_add_rule_sa_selector(sa_entry, rx, &attrs->upspec); 2114 if (err) 2115 goto err_add_sa_sel; 2116 } 2117 2118 if (attrs->type == XFRM_DEV_OFFLOAD_PACKET) 2119 err = rx_add_rule_drop_replay(sa_entry, rx); 2120 if (err) 2121 goto err_add_replay; 2122 2123 err = rx_add_rule_drop_auth_trailer(sa_entry, rx); 2124 if (err) 2125 goto err_drop_reason; 2126 2127 kvfree(spec); 2128 2129 sa_entry->ipsec_rule.rule = rule; 2130 sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 2131 sa_entry->ipsec_rule.fc = counter; 2132 sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; 2133 return 0; 2134 2135err_drop_reason: 2136 if (sa_entry->ipsec_rule.replay.rule) { 2137 mlx5_del_flow_rules(sa_entry->ipsec_rule.replay.rule); 2138 mlx5_fc_destroy(mdev, sa_entry->ipsec_rule.replay.fc); 2139 } 2140err_add_replay: 2141 if (sa_entry->ipsec_rule.sa_sel) { 2142 mlx5_del_flow_rules(sa_entry->ipsec_rule.sa_sel); 2143 mlx5_del_flow_rules(sa_entry->ipsec_rule.status_pass); 2144 } 2145err_add_sa_sel: 2146 mlx5_del_flow_rules(rule); 2147err_add_flow: 2148 mlx5_fc_destroy(mdev, counter); 2149err_add_cnt: 2150 if (flow_act.pkt_reformat) 2151 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 2152err_pkt_reformat: 2153 if (flow_act.modify_hdr) 2154 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 2155err_mod_header: 2156 kvfree(spec); 2157err_alloc: 2158 rx_ft_put(ipsec, attrs->addrs.family, attrs->type); 2159 return err; 2160} 2161 2162static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 2163{ 2164 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 2165 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 2166 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 2167 struct mlx5_flow_destination dest[2]; 2168 struct mlx5_flow_act flow_act = {}; 2169 struct mlx5_flow_handle *rule; 2170 struct mlx5_flow_spec *spec; 2171 struct mlx5e_ipsec_tx *tx; 2172 struct mlx5_fc *counter; 2173 int err; 2174 2175 tx = tx_ft_get(mdev, ipsec, attrs->type); 2176 if (IS_ERR(tx)) 2177 return PTR_ERR(tx); 2178 2179 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2180 if (!spec) { 2181 err = -ENOMEM; 2182 goto err_alloc; 2183 } 2184 2185 setup_fte_no_frags(spec); 2186 setup_fte_upper_proto_match(spec, &attrs->upspec); 2187 2188 switch (attrs->type) { 2189 case XFRM_DEV_OFFLOAD_CRYPTO: 2190 if (attrs->addrs.family == AF_INET) 2191 setup_fte_addr4(spec, &attrs->addrs); 2192 else 2193 setup_fte_addr6(spec, &attrs->addrs); 2194 setup_fte_spi(spec, attrs->spi, false); 2195 setup_fte_esp(spec); 2196 setup_fte_reg_a(spec); 2197 break; 2198 case XFRM_DEV_OFFLOAD_PACKET: 2199 setup_fte_reg_c4(spec, attrs->reqid); 2200 err = setup_pkt_reformat(ipsec, attrs, &flow_act); 2201 if (err) 2202 goto err_pkt_reformat; 2203 break; 2204 default: 2205 break; 2206 } 2207 2208 counter = mlx5_fc_create(mdev, true); 2209 if (IS_ERR(counter)) { 2210 err = PTR_ERR(counter); 2211 goto err_add_cnt; 2212 } 2213 2214 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 2215 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 2216 flow_act.flags |= FLOW_ACT_NO_APPEND; 2217 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | 2218 MLX5_FLOW_CONTEXT_ACTION_COUNT; 2219 if (attrs->drop) 2220 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 2221 else 2222 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2223 2224 dest[0].ft = tx->ft.status; 2225 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2226 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2227 dest[1].counter = counter; 2228 rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, dest, 2); 2229 if (IS_ERR(rule)) { 2230 err = PTR_ERR(rule); 2231 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 2232 goto err_add_flow; 2233 } 2234 2235 kvfree(spec); 2236 sa_entry->ipsec_rule.rule = rule; 2237 sa_entry->ipsec_rule.fc = counter; 2238 sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; 2239 return 0; 2240 2241err_add_flow: 2242 mlx5_fc_destroy(mdev, counter); 2243err_add_cnt: 2244 if (flow_act.pkt_reformat) 2245 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 2246err_pkt_reformat: 2247 kvfree(spec); 2248err_alloc: 2249 tx_ft_put(ipsec, attrs->type); 2250 return err; 2251} 2252 2253static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 2254{ 2255 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 2256 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 2257 struct mlx5e_ipsec *ipsec = pol_entry->ipsec; 2258 struct mlx5_flow_destination dest[2] = {}; 2259 struct mlx5_flow_act flow_act = {}; 2260 struct mlx5_flow_handle *rule; 2261 struct mlx5_flow_spec *spec; 2262 struct mlx5_flow_table *ft; 2263 struct mlx5e_ipsec_tx *tx; 2264 int err, dstn = 0; 2265 2266 ft = tx_ft_get_policy(mdev, ipsec, attrs->prio, attrs->type); 2267 if (IS_ERR(ft)) 2268 return PTR_ERR(ft); 2269 2270 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2271 if (!spec) { 2272 err = -ENOMEM; 2273 goto err_alloc; 2274 } 2275 2276 tx = ipsec_tx(ipsec, attrs->type); 2277 if (attrs->addrs.family == AF_INET) 2278 setup_fte_addr4(spec, &attrs->addrs); 2279 else 2280 setup_fte_addr6(spec, &attrs->addrs); 2281 2282 setup_fte_no_frags(spec); 2283 setup_fte_upper_proto_match(spec, &attrs->upspec); 2284 2285 switch (attrs->action) { 2286 case XFRM_POLICY_ALLOW: 2287 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2288 if (!attrs->reqid) 2289 break; 2290 2291 err = setup_modify_header(ipsec, attrs->type, attrs->reqid, 2292 XFRM_DEV_OFFLOAD_OUT, &flow_act); 2293 if (err) 2294 goto err_mod_header; 2295 break; 2296 case XFRM_POLICY_BLOCK: 2297 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | 2298 MLX5_FLOW_CONTEXT_ACTION_COUNT; 2299 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2300 dest[dstn].counter = tx->fc->drop; 2301 dstn++; 2302 break; 2303 default: 2304 WARN_ON(true); 2305 err = -EINVAL; 2306 goto err_mod_header; 2307 } 2308 2309 flow_act.flags |= FLOW_ACT_NO_APPEND; 2310 if (tx == ipsec->tx_esw && tx->chains) 2311 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 2312 dest[dstn].ft = tx->ft.sa; 2313 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2314 dstn++; 2315 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 2316 if (IS_ERR(rule)) { 2317 err = PTR_ERR(rule); 2318 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 2319 goto err_action; 2320 } 2321 2322 kvfree(spec); 2323 pol_entry->ipsec_rule.rule = rule; 2324 pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 2325 return 0; 2326 2327err_action: 2328 if (flow_act.modify_hdr) 2329 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 2330err_mod_header: 2331 kvfree(spec); 2332err_alloc: 2333 tx_ft_put_policy(ipsec, attrs->prio, attrs->type); 2334 return err; 2335} 2336 2337static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 2338{ 2339 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 2340 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 2341 struct mlx5e_ipsec *ipsec = pol_entry->ipsec; 2342 struct mlx5_flow_destination dest[2]; 2343 struct mlx5_flow_act flow_act = {}; 2344 struct mlx5_flow_handle *rule; 2345 struct mlx5_flow_spec *spec; 2346 struct mlx5_flow_table *ft; 2347 struct mlx5e_ipsec_rx *rx; 2348 int err, dstn = 0; 2349 2350 ft = rx_ft_get_policy(mdev, pol_entry->ipsec, attrs->addrs.family, 2351 attrs->prio, attrs->type); 2352 if (IS_ERR(ft)) 2353 return PTR_ERR(ft); 2354 2355 rx = ipsec_rx(pol_entry->ipsec, attrs->addrs.family, attrs->type); 2356 2357 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2358 if (!spec) { 2359 err = -ENOMEM; 2360 goto err_alloc; 2361 } 2362 2363 if (attrs->addrs.family == AF_INET) 2364 setup_fte_addr4(spec, &attrs->addrs); 2365 else 2366 setup_fte_addr6(spec, &attrs->addrs); 2367 2368 setup_fte_no_frags(spec); 2369 setup_fte_upper_proto_match(spec, &attrs->upspec); 2370 2371 switch (attrs->action) { 2372 case XFRM_POLICY_ALLOW: 2373 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2374 break; 2375 case XFRM_POLICY_BLOCK: 2376 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 2377 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 2378 dest[dstn].counter = rx->fc->drop; 2379 dstn++; 2380 break; 2381 default: 2382 WARN_ON(true); 2383 err = -EINVAL; 2384 goto err_action; 2385 } 2386 2387 flow_act.flags |= FLOW_ACT_NO_APPEND; 2388 if (rx == ipsec->rx_esw && rx->chains) 2389 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 2390 ipsec_rx_default_dest_get(ipsec, rx, &dest[dstn]); 2391 dstn++; 2392 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 2393 if (IS_ERR(rule)) { 2394 err = PTR_ERR(rule); 2395 mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err); 2396 goto err_action; 2397 } 2398 2399 kvfree(spec); 2400 pol_entry->ipsec_rule.rule = rule; 2401 return 0; 2402 2403err_action: 2404 kvfree(spec); 2405err_alloc: 2406 rx_ft_put_policy(pol_entry->ipsec, attrs->addrs.family, attrs->prio, 2407 attrs->type); 2408 return err; 2409} 2410 2411static void ipsec_fs_destroy_single_counter(struct mlx5_core_dev *mdev, 2412 struct mlx5e_ipsec_fc *fc) 2413{ 2414 mlx5_fc_destroy(mdev, fc->drop); 2415 mlx5_fc_destroy(mdev, fc->cnt); 2416 kfree(fc); 2417} 2418 2419static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) 2420{ 2421 struct mlx5_core_dev *mdev = ipsec->mdev; 2422 2423 ipsec_fs_destroy_single_counter(mdev, ipsec->tx->fc); 2424 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_ipv4->fc); 2425 if (ipsec->is_uplink_rep) { 2426 ipsec_fs_destroy_single_counter(mdev, ipsec->tx_esw->fc); 2427 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_esw->fc); 2428 } 2429} 2430 2431static struct mlx5e_ipsec_fc *ipsec_fs_init_single_counter(struct mlx5_core_dev *mdev) 2432{ 2433 struct mlx5e_ipsec_fc *fc; 2434 struct mlx5_fc *counter; 2435 int err; 2436 2437 fc = kzalloc(sizeof(*fc), GFP_KERNEL); 2438 if (!fc) 2439 return ERR_PTR(-ENOMEM); 2440 2441 counter = mlx5_fc_create(mdev, false); 2442 if (IS_ERR(counter)) { 2443 err = PTR_ERR(counter); 2444 goto err_cnt; 2445 } 2446 fc->cnt = counter; 2447 2448 counter = mlx5_fc_create(mdev, false); 2449 if (IS_ERR(counter)) { 2450 err = PTR_ERR(counter); 2451 goto err_drop; 2452 } 2453 fc->drop = counter; 2454 2455 return fc; 2456 2457err_drop: 2458 mlx5_fc_destroy(mdev, fc->cnt); 2459err_cnt: 2460 kfree(fc); 2461 return ERR_PTR(err); 2462} 2463 2464static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) 2465{ 2466 struct mlx5_core_dev *mdev = ipsec->mdev; 2467 struct mlx5e_ipsec_fc *fc; 2468 int err; 2469 2470 fc = ipsec_fs_init_single_counter(mdev); 2471 if (IS_ERR(fc)) { 2472 err = PTR_ERR(fc); 2473 goto err_rx_cnt; 2474 } 2475 ipsec->rx_ipv4->fc = fc; 2476 2477 fc = ipsec_fs_init_single_counter(mdev); 2478 if (IS_ERR(fc)) { 2479 err = PTR_ERR(fc); 2480 goto err_tx_cnt; 2481 } 2482 ipsec->tx->fc = fc; 2483 2484 if (ipsec->is_uplink_rep) { 2485 fc = ipsec_fs_init_single_counter(mdev); 2486 if (IS_ERR(fc)) { 2487 err = PTR_ERR(fc); 2488 goto err_rx_esw_cnt; 2489 } 2490 ipsec->rx_esw->fc = fc; 2491 2492 fc = ipsec_fs_init_single_counter(mdev); 2493 if (IS_ERR(fc)) { 2494 err = PTR_ERR(fc); 2495 goto err_tx_esw_cnt; 2496 } 2497 ipsec->tx_esw->fc = fc; 2498 } 2499 2500 /* Both IPv4 and IPv6 point to same flow counters struct. */ 2501 ipsec->rx_ipv6->fc = ipsec->rx_ipv4->fc; 2502 return 0; 2503 2504err_tx_esw_cnt: 2505 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_esw->fc); 2506err_rx_esw_cnt: 2507 ipsec_fs_destroy_single_counter(mdev, ipsec->tx->fc); 2508err_tx_cnt: 2509 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_ipv4->fc); 2510err_rx_cnt: 2511 return err; 2512} 2513 2514void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) 2515{ 2516 struct mlx5_core_dev *mdev = priv->mdev; 2517 struct mlx5e_ipsec *ipsec = priv->ipsec; 2518 struct mlx5e_ipsec_hw_stats *stats; 2519 struct mlx5e_ipsec_fc *fc; 2520 u64 packets, bytes; 2521 2522 stats = (struct mlx5e_ipsec_hw_stats *)ipsec_stats; 2523 2524 stats->ipsec_rx_pkts = 0; 2525 stats->ipsec_rx_bytes = 0; 2526 stats->ipsec_rx_drop_pkts = 0; 2527 stats->ipsec_rx_drop_bytes = 0; 2528 stats->ipsec_rx_drop_mismatch_sa_sel = 0; 2529 stats->ipsec_tx_pkts = 0; 2530 stats->ipsec_tx_bytes = 0; 2531 stats->ipsec_tx_drop_pkts = 0; 2532 stats->ipsec_tx_drop_bytes = 0; 2533 2534 fc = ipsec->rx_ipv4->fc; 2535 mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_rx_pkts, &stats->ipsec_rx_bytes); 2536 mlx5_fc_query(mdev, fc->drop, &stats->ipsec_rx_drop_pkts, 2537 &stats->ipsec_rx_drop_bytes); 2538 if (ipsec->rx_ipv4->sa_sel.fc) 2539 mlx5_fc_query(mdev, ipsec->rx_ipv4->sa_sel.fc, 2540 &stats->ipsec_rx_drop_mismatch_sa_sel, &bytes); 2541 2542 fc = ipsec->tx->fc; 2543 mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_tx_pkts, &stats->ipsec_tx_bytes); 2544 mlx5_fc_query(mdev, fc->drop, &stats->ipsec_tx_drop_pkts, 2545 &stats->ipsec_tx_drop_bytes); 2546 2547 if (ipsec->is_uplink_rep) { 2548 fc = ipsec->rx_esw->fc; 2549 if (!mlx5_fc_query(mdev, fc->cnt, &packets, &bytes)) { 2550 stats->ipsec_rx_pkts += packets; 2551 stats->ipsec_rx_bytes += bytes; 2552 } 2553 2554 if (!mlx5_fc_query(mdev, fc->drop, &packets, &bytes)) { 2555 stats->ipsec_rx_drop_pkts += packets; 2556 stats->ipsec_rx_drop_bytes += bytes; 2557 } 2558 2559 fc = ipsec->tx_esw->fc; 2560 if (!mlx5_fc_query(mdev, fc->cnt, &packets, &bytes)) { 2561 stats->ipsec_tx_pkts += packets; 2562 stats->ipsec_tx_bytes += bytes; 2563 } 2564 2565 if (!mlx5_fc_query(mdev, fc->drop, &packets, &bytes)) { 2566 stats->ipsec_tx_drop_pkts += packets; 2567 stats->ipsec_tx_drop_bytes += bytes; 2568 } 2569 2570 if (ipsec->rx_esw->sa_sel.fc && 2571 !mlx5_fc_query(mdev, ipsec->rx_esw->sa_sel.fc, 2572 &packets, &bytes)) 2573 stats->ipsec_rx_drop_mismatch_sa_sel += packets; 2574 } 2575} 2576 2577#ifdef CONFIG_MLX5_ESWITCH 2578static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) 2579{ 2580 struct mlx5_eswitch *esw = mdev->priv.eswitch; 2581 int err = 0; 2582 2583 if (esw) { 2584 err = mlx5_esw_lock(esw); 2585 if (err) 2586 return err; 2587 } 2588 2589 if (mdev->num_block_ipsec) { 2590 err = -EBUSY; 2591 goto unlock; 2592 } 2593 2594 mdev->num_block_tc++; 2595 2596unlock: 2597 if (esw) 2598 mlx5_esw_unlock(esw); 2599 2600 return err; 2601} 2602#else 2603static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) 2604{ 2605 if (mdev->num_block_ipsec) 2606 return -EBUSY; 2607 2608 mdev->num_block_tc++; 2609 return 0; 2610} 2611#endif 2612 2613static void mlx5e_ipsec_unblock_tc_offload(struct mlx5_core_dev *mdev) 2614{ 2615 mdev->num_block_tc--; 2616} 2617 2618int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 2619{ 2620 int err; 2621 2622 if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) { 2623 err = mlx5e_ipsec_block_tc_offload(sa_entry->ipsec->mdev); 2624 if (err) 2625 return err; 2626 } 2627 2628 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 2629 err = tx_add_rule(sa_entry); 2630 else 2631 err = rx_add_rule(sa_entry); 2632 2633 if (err) 2634 goto err_out; 2635 2636 return 0; 2637 2638err_out: 2639 if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) 2640 mlx5e_ipsec_unblock_tc_offload(sa_entry->ipsec->mdev); 2641 return err; 2642} 2643 2644void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 2645{ 2646 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 2647 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 2648 2649 mlx5_del_flow_rules(ipsec_rule->rule); 2650 mlx5_fc_destroy(mdev, ipsec_rule->fc); 2651 if (ipsec_rule->pkt_reformat) 2652 mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); 2653 2654 if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) 2655 mlx5e_ipsec_unblock_tc_offload(mdev); 2656 2657 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { 2658 tx_ft_put(sa_entry->ipsec, sa_entry->attrs.type); 2659 return; 2660 } 2661 2662 if (ipsec_rule->modify_hdr) 2663 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 2664 2665 mlx5_del_flow_rules(ipsec_rule->trailer.rule); 2666 mlx5_fc_destroy(mdev, ipsec_rule->trailer.fc); 2667 2668 mlx5_del_flow_rules(ipsec_rule->auth.rule); 2669 mlx5_fc_destroy(mdev, ipsec_rule->auth.fc); 2670 2671 if (ipsec_rule->sa_sel) { 2672 mlx5_del_flow_rules(ipsec_rule->sa_sel); 2673 mlx5_del_flow_rules(ipsec_rule->status_pass); 2674 } 2675 2676 if (ipsec_rule->replay.rule) { 2677 mlx5_del_flow_rules(ipsec_rule->replay.rule); 2678 mlx5_fc_destroy(mdev, ipsec_rule->replay.fc); 2679 } 2680 mlx5_esw_ipsec_rx_id_mapping_remove(sa_entry); 2681 rx_ft_put(sa_entry->ipsec, sa_entry->attrs.addrs.family, 2682 sa_entry->attrs.type); 2683} 2684 2685int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 2686{ 2687 int err; 2688 2689 err = mlx5e_ipsec_block_tc_offload(pol_entry->ipsec->mdev); 2690 if (err) 2691 return err; 2692 2693 if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 2694 err = tx_add_policy(pol_entry); 2695 else 2696 err = rx_add_policy(pol_entry); 2697 2698 if (err) 2699 goto err_out; 2700 2701 return 0; 2702 2703err_out: 2704 mlx5e_ipsec_unblock_tc_offload(pol_entry->ipsec->mdev); 2705 return err; 2706} 2707 2708void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 2709{ 2710 struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; 2711 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 2712 2713 mlx5_del_flow_rules(ipsec_rule->rule); 2714 2715 mlx5e_ipsec_unblock_tc_offload(pol_entry->ipsec->mdev); 2716 2717 if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { 2718 rx_ft_put_policy(pol_entry->ipsec, 2719 pol_entry->attrs.addrs.family, 2720 pol_entry->attrs.prio, pol_entry->attrs.type); 2721 return; 2722 } 2723 2724 if (ipsec_rule->modify_hdr) 2725 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 2726 2727 tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio, pol_entry->attrs.type); 2728} 2729 2730void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) 2731{ 2732 if (!ipsec->tx) 2733 return; 2734 2735 if (ipsec->roce) 2736 mlx5_ipsec_fs_roce_cleanup(ipsec->roce); 2737 2738 ipsec_fs_destroy_counters(ipsec); 2739 mutex_destroy(&ipsec->tx->ft.mutex); 2740 WARN_ON(ipsec->tx->ft.refcnt); 2741 kfree(ipsec->tx); 2742 2743 mutex_destroy(&ipsec->rx_ipv4->ft.mutex); 2744 WARN_ON(ipsec->rx_ipv4->ft.refcnt); 2745 kfree(ipsec->rx_ipv4); 2746 2747 mutex_destroy(&ipsec->rx_ipv6->ft.mutex); 2748 WARN_ON(ipsec->rx_ipv6->ft.refcnt); 2749 kfree(ipsec->rx_ipv6); 2750 2751 if (ipsec->is_uplink_rep) { 2752 xa_destroy(&ipsec->ipsec_obj_id_map); 2753 2754 mutex_destroy(&ipsec->tx_esw->ft.mutex); 2755 WARN_ON(ipsec->tx_esw->ft.refcnt); 2756 kfree(ipsec->tx_esw); 2757 2758 mutex_destroy(&ipsec->rx_esw->ft.mutex); 2759 WARN_ON(ipsec->rx_esw->ft.refcnt); 2760 kfree(ipsec->rx_esw); 2761 } 2762} 2763 2764int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec, 2765 struct mlx5_devcom_comp_dev **devcom) 2766{ 2767 struct mlx5_core_dev *mdev = ipsec->mdev; 2768 struct mlx5_flow_namespace *ns, *ns_esw; 2769 int err = -ENOMEM; 2770 2771 ns = mlx5_get_flow_namespace(ipsec->mdev, 2772 MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); 2773 if (!ns) 2774 return -EOPNOTSUPP; 2775 2776 if (ipsec->is_uplink_rep) { 2777 ns_esw = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_FDB); 2778 if (!ns_esw) 2779 return -EOPNOTSUPP; 2780 2781 ipsec->tx_esw = kzalloc(sizeof(*ipsec->tx_esw), GFP_KERNEL); 2782 if (!ipsec->tx_esw) 2783 return -ENOMEM; 2784 2785 ipsec->rx_esw = kzalloc(sizeof(*ipsec->rx_esw), GFP_KERNEL); 2786 if (!ipsec->rx_esw) 2787 goto err_rx_esw; 2788 } 2789 2790 ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); 2791 if (!ipsec->tx) 2792 goto err_tx; 2793 2794 ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); 2795 if (!ipsec->rx_ipv4) 2796 goto err_rx_ipv4; 2797 2798 ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); 2799 if (!ipsec->rx_ipv6) 2800 goto err_rx_ipv6; 2801 2802 err = ipsec_fs_init_counters(ipsec); 2803 if (err) 2804 goto err_counters; 2805 2806 mutex_init(&ipsec->tx->ft.mutex); 2807 mutex_init(&ipsec->rx_ipv4->ft.mutex); 2808 mutex_init(&ipsec->rx_ipv6->ft.mutex); 2809 ipsec->tx->ns = ns; 2810 2811 if (ipsec->is_uplink_rep) { 2812 mutex_init(&ipsec->tx_esw->ft.mutex); 2813 mutex_init(&ipsec->rx_esw->ft.mutex); 2814 ipsec->tx_esw->ns = ns_esw; 2815 xa_init_flags(&ipsec->ipsec_obj_id_map, XA_FLAGS_ALLOC1); 2816 } else if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ROCE) { 2817 ipsec->roce = mlx5_ipsec_fs_roce_init(mdev, devcom); 2818 } else { 2819 mlx5_core_warn(mdev, "IPsec was initialized without RoCE support\n"); 2820 } 2821 2822 return 0; 2823 2824err_counters: 2825 kfree(ipsec->rx_ipv6); 2826err_rx_ipv6: 2827 kfree(ipsec->rx_ipv4); 2828err_rx_ipv4: 2829 kfree(ipsec->tx); 2830err_tx: 2831 kfree(ipsec->rx_esw); 2832err_rx_esw: 2833 kfree(ipsec->tx_esw); 2834 return err; 2835} 2836 2837void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry) 2838{ 2839 struct mlx5e_ipsec_sa_entry sa_entry_shadow = {}; 2840 int err; 2841 2842 memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry)); 2843 memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule)); 2844 2845 err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow); 2846 if (err) 2847 return; 2848 2849 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 2850 memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry)); 2851} 2852 2853bool mlx5e_ipsec_fs_tunnel_allowed(struct mlx5e_ipsec_sa_entry *sa_entry) 2854{ 2855 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 2856 struct xfrm_state *x = sa_entry->x; 2857 bool from_fdb; 2858 2859 if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) { 2860 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, x->xso.type); 2861 2862 from_fdb = (tx == ipsec->tx_esw); 2863 } else { 2864 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, x->props.family, 2865 x->xso.type); 2866 2867 from_fdb = (rx == ipsec->rx_esw); 2868 } 2869 2870 return mlx5_eswitch_block_encap(ipsec->mdev, from_fdb); 2871} 2872 2873void mlx5e_ipsec_handle_mpv_event(int event, struct mlx5e_priv *slave_priv, 2874 struct mlx5e_priv *master_priv) 2875{ 2876 struct mlx5e_ipsec_mpv_work *work; 2877 2878 reinit_completion(&master_priv->ipsec->comp); 2879 2880 if (!slave_priv->ipsec) { 2881 complete(&master_priv->ipsec->comp); 2882 return; 2883 } 2884 2885 work = &slave_priv->ipsec->mpv_work; 2886 2887 INIT_WORK(&work->work, ipsec_mpv_work_handler); 2888 work->event = event; 2889 work->slave_priv = slave_priv; 2890 work->master_priv = master_priv; 2891 queue_work(slave_priv->ipsec->wq, &work->work); 2892} 2893 2894void mlx5e_ipsec_send_event(struct mlx5e_priv *priv, int event) 2895{ 2896 if (!priv->ipsec || mlx5_devcom_comp_get_size(priv->devcom) < 2) 2897 return; /* IPsec not supported or no peers */ 2898 2899 mlx5_devcom_send_event(priv->devcom, event, event, priv); 2900 wait_for_completion(&priv->ipsec->comp); 2901} 2902 2903void mlx5e_ipsec_disable_events(struct mlx5e_priv *priv) 2904{ 2905 struct mlx5_devcom_comp_dev *tmp = NULL; 2906 struct mlx5e_priv *peer_priv; 2907 2908 if (!priv->devcom) 2909 return; 2910 2911 if (!mlx5_devcom_for_each_peer_begin(priv->devcom)) 2912 goto out; 2913 2914 peer_priv = mlx5_devcom_get_next_peer_data(priv->devcom, &tmp); 2915 if (peer_priv && peer_priv->ipsec) 2916 complete_all(&peer_priv->ipsec->comp); 2917 2918 mlx5_devcom_for_each_peer_end(priv->devcom); 2919out: 2920 mlx5_devcom_unregister_component(priv->devcom); 2921 priv->devcom = NULL; 2922}
~/
Read-only file preview.