fs_tcp.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
  3
  4#include <mlx5_core.h>
  5#include "en_accel/fs_tcp.h"
  6#include "fs_core.h"
  7
  8enum accel_fs_tcp_type {
  9	ACCEL_FS_IPV4_TCP,
 10	ACCEL_FS_IPV6_TCP,
 11	ACCEL_FS_TCP_NUM_TYPES,
 12};
 13
 14struct mlx5e_accel_fs_tcp {
 15	struct mlx5e_flow_table tables[ACCEL_FS_TCP_NUM_TYPES];
 16	struct mlx5_flow_handle *default_rules[ACCEL_FS_TCP_NUM_TYPES];
 17};
 18
 19static enum mlx5_traffic_types fs_accel2tt(enum accel_fs_tcp_type i)
 20{
 21	switch (i) {
 22	case ACCEL_FS_IPV4_TCP:
 23		return MLX5_TT_IPV4_TCP;
 24	default: /* ACCEL_FS_IPV6_TCP */
 25		return MLX5_TT_IPV6_TCP;
 26	}
 27}
 28
 29static void accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct sock *sk)
 30{
 31	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
 32	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
 33	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
 34	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
 35	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
 36			    outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
 37	       &inet_sk(sk)->inet_daddr, 4);
 38	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
 39			    outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 40	       &inet_sk(sk)->inet_rcv_saddr, 4);
 41	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
 42			 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
 43	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
 44			 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 45}
 46
 47#if IS_ENABLED(CONFIG_IPV6)
 48static void accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct sock *sk)
 49{
 50	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
 51	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
 52	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
 53	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
 54	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
 55			    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
 56	       &sk->sk_v6_daddr, 16);
 57	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
 58			    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 59	       &inet6_sk(sk)->saddr, 16);
 60	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 61			    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
 62	       0xff, 16);
 63	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 64			    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 65	       0xff, 16);
 66}
 67#endif
 68
 69void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule)
 70{
 71	mlx5_del_flow_rules(rule);
 72}
 73
 74struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs,
 75					       struct sock *sk, u32 tirn,
 76					       u32 flow_tag)
 77{
 78	struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs);
 79	struct mlx5_flow_destination dest = {};
 80	struct mlx5e_flow_table *ft = NULL;
 81	MLX5_DECLARE_FLOW_ACT(flow_act);
 82	struct mlx5_flow_handle *flow;
 83	struct mlx5_flow_spec *spec;
 84
 85	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 86	if (!spec)
 87		return ERR_PTR(-ENOMEM);
 88
 89	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 90
 91	switch (sk->sk_family) {
 92	case AF_INET:
 93		accel_fs_tcp_set_ipv4_flow(spec, sk);
 94		ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP];
 95		fs_dbg(fs, "%s flow is %pI4:%d -> %pI4:%d\n", __func__,
 96		       &inet_sk(sk)->inet_rcv_saddr,
 97		       inet_sk(sk)->inet_sport,
 98		       &inet_sk(sk)->inet_daddr,
 99		       inet_sk(sk)->inet_dport);
100		break;
101#if IS_ENABLED(CONFIG_IPV6)
102	case AF_INET6:
103		if (!ipv6_only_sock(sk) &&
104		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
105			accel_fs_tcp_set_ipv4_flow(spec, sk);
106			ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP];
107		} else {
108			accel_fs_tcp_set_ipv6_flow(spec, sk);
109			ft = &fs_tcp->tables[ACCEL_FS_IPV6_TCP];
110		}
111		break;
112#endif
113	default:
114		break;
115	}
116
117	if (!ft) {
118		flow = ERR_PTR(-EINVAL);
119		goto out;
120	}
121
122	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
123			 outer_headers.tcp_dport);
124	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
125			 outer_headers.tcp_sport);
126	MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
127		 ntohs(inet_sk(sk)->inet_sport));
128	MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
129		 ntohs(inet_sk(sk)->inet_dport));
130
131	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
132	dest.tir_num = tirn;
133	if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG) {
134		spec->flow_context.flow_tag = flow_tag;
135		spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
136	}
137
138	flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);
139
140	if (IS_ERR(flow))
141		fs_err(fs, "mlx5_add_flow_rules() failed, flow is %pe\n", flow);
142
143out:
144	kvfree(spec);
145	return flow;
146}
147
148static int accel_fs_tcp_add_default_rule(struct mlx5e_flow_steering *fs,
149					 enum accel_fs_tcp_type type)
150{
151	struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs);
152	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
153	struct mlx5e_flow_table *accel_fs_t;
154	struct mlx5_flow_destination dest;
155	MLX5_DECLARE_FLOW_ACT(flow_act);
156	struct mlx5_flow_handle *rule;
157	int err = 0;
158
159	accel_fs_t = &fs_tcp->tables[type];
160
161	dest = mlx5_ttc_get_default_dest(ttc, fs_accel2tt(type));
162	rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1);
163	if (IS_ERR(rule)) {
164		err = PTR_ERR(rule);
165		fs_err(fs, "%s: add default rule failed, accel_fs type=%d, err %d\n",
166		       __func__, type, err);
167		return err;
168	}
169
170	fs_tcp->default_rules[type] = rule;
171	return 0;
172}
173
174#define MLX5E_ACCEL_FS_TCP_NUM_GROUPS	(2)
175#define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE	(BIT(16) - 1)
176#define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE	(BIT(0))
177#define MLX5E_ACCEL_FS_TCP_TABLE_SIZE	(MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\
178					 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE)
179static int accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft,
180				      enum accel_fs_tcp_type type)
181{
182	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
183	void *outer_headers_c;
184	int ix = 0;
185	u32 *in;
186	int err;
187	u8 *mc;
188
189	ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
190	in = kvzalloc(inlen, GFP_KERNEL);
191	if  (!in || !ft->g) {
192		kfree(ft->g);
193		ft->g = NULL;
194		kvfree(in);
195		return -ENOMEM;
196	}
197
198	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
199	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
200	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
201	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
202
203	switch (type) {
204	case ACCEL_FS_IPV4_TCP:
205	case ACCEL_FS_IPV6_TCP:
206		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
207		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
208		break;
209	default:
210		err = -EINVAL;
211		goto out;
212	}
213
214	switch (type) {
215	case ACCEL_FS_IPV4_TCP:
216		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
217				 src_ipv4_src_ipv6.ipv4_layout.ipv4);
218		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
219				 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
220		break;
221	case ACCEL_FS_IPV6_TCP:
222		memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
223				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
224		       0xff, 16);
225		memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
226				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
227		       0xff, 16);
228		break;
229	default:
230		err = -EINVAL;
231		goto out;
232	}
233
234	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
235	MLX5_SET_CFG(in, start_flow_index, ix);
236	ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE;
237	MLX5_SET_CFG(in, end_flow_index, ix - 1);
238	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
239	if (IS_ERR(ft->g[ft->num_groups]))
240		goto err;
241	ft->num_groups++;
242
243	/* Default Flow Group */
244	memset(in, 0, inlen);
245	MLX5_SET_CFG(in, start_flow_index, ix);
246	ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE;
247	MLX5_SET_CFG(in, end_flow_index, ix - 1);
248	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
249	if (IS_ERR(ft->g[ft->num_groups]))
250		goto err;
251	ft->num_groups++;
252
253	kvfree(in);
254	return 0;
255
256err:
257	err = PTR_ERR(ft->g[ft->num_groups]);
258	ft->g[ft->num_groups] = NULL;
259out:
260	kvfree(in);
261
262	return err;
263}
264
265static int accel_fs_tcp_create_table(struct mlx5e_flow_steering *fs, enum accel_fs_tcp_type type)
266{
267	struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs);
268	struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false);
269	struct mlx5e_flow_table *ft = &accel_tcp->tables[type];
270	struct mlx5_flow_table_attr ft_attr = {};
271	int err;
272
273	ft->num_groups = 0;
274
275	ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE;
276	ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL;
277	ft_attr.prio = MLX5E_NIC_PRIO;
278
279	ft->t = mlx5_create_flow_table(ns, &ft_attr);
280	if (IS_ERR(ft->t)) {
281		err = PTR_ERR(ft->t);
282		ft->t = NULL;
283		return err;
284	}
285
286	fs_dbg(fs, "Created fs accel table id %u level %u\n",
287	       ft->t->id, ft->t->level);
288
289	err = accel_fs_tcp_create_groups(ft, type);
290	if (err)
291		goto err;
292
293	err = accel_fs_tcp_add_default_rule(fs, type);
294	if (err)
295		goto err;
296
297	return 0;
298err:
299	mlx5e_destroy_flow_table(ft);
300	return err;
301}
302
303static int accel_fs_tcp_disable(struct mlx5e_flow_steering *fs)
304{
305	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
306	int err, i;
307
308	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
309		/* Modify ttc rules destination to point back to the indir TIRs */
310		err = mlx5_ttc_fwd_default_dest(ttc, fs_accel2tt(i));
311		if (err) {
312			fs_err(fs,
313			       "%s: modify ttc[%d] default destination failed, err(%d)\n",
314			       __func__, fs_accel2tt(i), err);
315			return err;
316		}
317	}
318
319	return 0;
320}
321
322static int accel_fs_tcp_enable(struct mlx5e_flow_steering *fs)
323{
324	struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs);
325	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
326	struct mlx5_flow_destination dest = {};
327	int err, i;
328
329	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
330	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
331		dest.ft = accel_tcp->tables[i].t;
332
333		/* Modify ttc rules destination to point on the accel_fs FTs */
334		err = mlx5_ttc_fwd_dest(ttc, fs_accel2tt(i), &dest);
335		if (err) {
336			fs_err(fs, "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
337			       __func__, fs_accel2tt(i), err);
338			return err;
339		}
340	}
341	return 0;
342}
343
344static void accel_fs_tcp_destroy_table(struct mlx5e_flow_steering *fs, int i)
345{
346	struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs);
347
348	if (IS_ERR_OR_NULL(fs_tcp->tables[i].t))
349		return;
350
351	mlx5_del_flow_rules(fs_tcp->default_rules[i]);
352	mlx5e_destroy_flow_table(&fs_tcp->tables[i]);
353	fs_tcp->tables[i].t = NULL;
354}
355
356void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs)
357{
358	struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs);
359	int i;
360
361	if (!accel_tcp)
362		return;
363
364	accel_fs_tcp_disable(fs);
365
366	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++)
367		accel_fs_tcp_destroy_table(fs, i);
368
369	kfree(accel_tcp);
370	mlx5e_fs_set_accel_tcp(fs, NULL);
371}
372
373int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs)
374{
375	struct mlx5e_accel_fs_tcp *accel_tcp;
376	int i, err;
377
378	if (!MLX5_CAP_FLOWTABLE_NIC_RX(mlx5e_fs_get_mdev(fs), ft_field_support.outer_ip_version))
379		return -EOPNOTSUPP;
380
381	accel_tcp = kzalloc(sizeof(*accel_tcp), GFP_KERNEL);
382	if (!accel_tcp)
383		return -ENOMEM;
384	mlx5e_fs_set_accel_tcp(fs, accel_tcp);
385
386	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
387		err = accel_fs_tcp_create_table(fs, i);
388		if (err)
389			goto err_destroy_tables;
390	}
391
392	err = accel_fs_tcp_enable(fs);
393	if (err)
394		goto err_destroy_tables;
395
396	return 0;
397
398err_destroy_tables:
399	while (--i >= 0)
400		accel_fs_tcp_destroy_table(fs, i);
401	kfree(accel_tcp);
402	mlx5e_fs_set_accel_tcp(fs, NULL);
403	return err;
404}