fs_core.h

  1/*
  2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32
 33#ifndef _MLX5_FS_CORE_
 34#define _MLX5_FS_CORE_
 35
 36#include <linux/refcount.h>
 37#include <linux/mlx5/fs.h>
 38#include <linux/rhashtable.h>
 39#include <linux/llist.h>
 40#include <steering/sws/fs_dr.h>
 41#include <steering/hws/fs_hws.h>
 42
 43#define FDB_TC_MAX_CHAIN 3
 44#define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1)
 45#define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1)
 46
 47/* The index of the last real chain (FT) + 1 as chain zero is valid as well */
 48#define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1)
 49
 50#define FDB_TC_MAX_PRIO 16
 51#define FDB_TC_LEVELS_PER_PRIO 2
 52
 53struct mlx5_flow_definer {
 54	enum mlx5_flow_namespace_type ns_type;
 55	u32 id;
 56};
 57
 58enum mlx5_flow_resource_owner {
 59	MLX5_FLOW_RESOURCE_OWNER_FW,
 60	MLX5_FLOW_RESOURCE_OWNER_SW,
 61	MLX5_FLOW_RESOURCE_OWNER_HWS,
 62};
 63
 64struct mlx5_modify_hdr {
 65	enum mlx5_flow_namespace_type ns_type;
 66	enum mlx5_flow_resource_owner owner;
 67	union {
 68		struct mlx5_fs_dr_action fs_dr_action;
 69		struct mlx5_fs_hws_action fs_hws_action;
 70		u32 id;
 71	};
 72};
 73
 74struct mlx5_pkt_reformat {
 75	enum mlx5_flow_namespace_type ns_type;
 76	int reformat_type; /* from mlx5_ifc */
 77	enum mlx5_flow_resource_owner owner;
 78	union {
 79		struct mlx5_fs_dr_action fs_dr_action;
 80		struct mlx5_fs_hws_action fs_hws_action;
 81		u32 id;
 82	};
 83};
 84
 85/* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only,
 86 * and those are in parallel to one another when going over them to connect
 87 * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one
 88 * parallel namespace will not automatically connect to the first flow table
 89 * found in any prio in any next namespace, but skip the entire containing
 90 * TYPE_PRIO_CHAINS prio.
 91 *
 92 * This is used to implement tc chains, each chain of prios is a different
 93 * namespace inside a containing TYPE_PRIO_CHAINS prio.
 94 */
 95
 96enum fs_node_type {
 97	FS_TYPE_NAMESPACE,
 98	FS_TYPE_PRIO,
 99	FS_TYPE_PRIO_CHAINS,
100	FS_TYPE_FLOW_TABLE,
101	FS_TYPE_FLOW_GROUP,
102	FS_TYPE_FLOW_ENTRY,
103	FS_TYPE_FLOW_DEST
104};
105
106enum fs_flow_table_op_mod {
107	FS_FT_OP_MOD_NORMAL,
108	FS_FT_OP_MOD_LAG_DEMUX,
109};
110
111enum fs_fte_status {
112	FS_FTE_STATUS_EXISTING = 1UL << 0,
113};
114
115enum mlx5_flow_steering_mode {
116	MLX5_FLOW_STEERING_MODE_DMFS,
117	MLX5_FLOW_STEERING_MODE_SMFS,
118	MLX5_FLOW_STEERING_MODE_HMFS,
119};
120
121enum mlx5_flow_steering_capabilty {
122	MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0,
123	MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1,
124	MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2,
125	MLX5_FLOW_STEERING_CAP_DUPLICATE_MATCH = 1UL << 3,
126};
127
128struct mlx5_flow_steering {
129	struct mlx5_core_dev *dev;
130	enum   mlx5_flow_steering_mode	mode;
131	struct kmem_cache		*fgs_cache;
132	struct kmem_cache               *ftes_cache;
133	struct mlx5_flow_root_namespace *root_ns;
134	struct mlx5_flow_root_namespace *fdb_root_ns;
135	struct mlx5_flow_namespace	**fdb_sub_ns;
136	struct xarray			esw_egress_root_ns;
137	struct xarray			esw_ingress_root_ns;
138	struct mlx5_flow_root_namespace	*sniffer_tx_root_ns;
139	struct mlx5_flow_root_namespace	*sniffer_rx_root_ns;
140	struct mlx5_flow_root_namespace	*rdma_rx_root_ns;
141	struct mlx5_flow_root_namespace	*rdma_tx_root_ns;
142	struct mlx5_flow_root_namespace	*egress_root_ns;
143	struct mlx5_flow_root_namespace	*port_sel_root_ns;
144	struct mlx5_flow_root_namespace **rdma_transport_rx_root_ns;
145	struct mlx5_flow_root_namespace **rdma_transport_tx_root_ns;
146	int rdma_transport_rx_vports;
147	int rdma_transport_tx_vports;
148};
149
150struct fs_node {
151	struct list_head	list;
152	struct list_head	children;
153	enum fs_node_type	type;
154	struct fs_node		*parent;
155	struct fs_node		*root;
156	/* lock the node for writing and traversing */
157	struct rw_semaphore	lock;
158	refcount_t		refcount;
159	bool			active;
160	void			(*del_hw_func)(struct fs_node *);
161	void			(*del_sw_func)(struct fs_node *);
162	atomic_t		version;
163};
164
165struct mlx5_flow_rule {
166	struct fs_node				node;
167	struct mlx5_flow_table			*ft;
168	struct mlx5_flow_destination		dest_attr;
169	/* next_ft should be accessed under chain_lock and only of
170	 * destination type is FWD_NEXT_fT.
171	 */
172	struct list_head			next_ft;
173	u32					sw_action;
174};
175
176struct mlx5_flow_handle {
177	int num_rules;
178	struct mlx5_flow_rule *rule[] __counted_by(num_rules);
179};
180
181/* Type of children is mlx5_flow_group */
182struct mlx5_flow_table {
183	struct fs_node			node;
184	union {
185		struct mlx5_fs_dr_table		fs_dr_table;
186		struct mlx5_fs_hws_table	fs_hws_table;
187	};
188	u32				id;
189	u16				vport;
190	u16				esw_owner_vhca_id;
191	unsigned int			max_fte;
192	unsigned int			level;
193	enum fs_flow_table_type		type;
194	enum fs_flow_table_op_mod	op_mod;
195	struct {
196		bool			active;
197		unsigned int		required_groups;
198		unsigned int		group_size;
199		unsigned int		num_groups;
200		unsigned int		max_fte;
201	} autogroup;
202	/* Protect fwd_rules */
203	struct mutex			lock;
204	/* FWD rules that point on this flow table */
205	struct list_head		fwd_rules;
206	u32				flags;
207	struct rhltable			fgs_hash;
208	enum mlx5_flow_table_miss_action def_miss_action;
209	struct mlx5_flow_namespace	*ns;
210};
211
212struct mlx5_ft_underlay_qp {
213	struct list_head list;
214	u32 qpn;
215};
216
217#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_e00
218/* Calculate the fte_match_param length and without the reserved length.
219 * Make sure the reserved field is the last.
220 */
221#define MLX5_ST_SZ_DW_MATCH_PARAM					    \
222	((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
223	 BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) !=		     \
224			   MLX5_FLD_SZ_BYTES(fte_match_param,		     \
225					     MLX5_FTE_MATCH_PARAM_RESERVED) +\
226			   MLX5_BYTE_OFF(fte_match_param,		     \
227					 MLX5_FTE_MATCH_PARAM_RESERVED)))
228
229struct fs_fte_action {
230	int				modify_mask;
231	u32				dests_size;
232	u32				fwd_dests;
233	struct mlx5_flow_context	flow_context;
234	struct mlx5_flow_act		action;
235};
236
237struct fs_fte_dup {
238	struct list_head children;
239	struct fs_fte_action act_dests;
240};
241
242/* Type of children is mlx5_flow_rule */
243struct fs_fte {
244	struct fs_node			node;
245	union {
246		struct mlx5_fs_dr_rule		fs_dr_rule;
247		struct mlx5_fs_hws_rule		fs_hws_rule;
248	};
249	u32				val[MLX5_ST_SZ_DW_MATCH_PARAM];
250	struct fs_fte_action		act_dests;
251	struct fs_fte_dup		*dup;
252	u32				index;
253	enum fs_fte_status		status;
254	struct rhash_head		hash;
255};
256
257/* Type of children is mlx5_flow_table/namespace */
258struct fs_prio {
259	struct fs_node			node;
260	unsigned int			num_levels;
261	unsigned int			start_level;
262	unsigned int			prio;
263	unsigned int			num_ft;
264};
265
266/* Type of children is fs_prio */
267struct mlx5_flow_namespace {
268	/* parent == NULL => root ns */
269	struct	fs_node			node;
270	enum mlx5_flow_table_miss_action def_miss_action;
271};
272
273struct mlx5_flow_group_mask {
274	u8	match_criteria_enable;
275	u32	match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
276};
277
278/* Type of children is fs_fte */
279struct mlx5_flow_group {
280	struct fs_node			node;
281	union {
282		struct mlx5_fs_dr_matcher	fs_dr_matcher;
283		struct mlx5_fs_hws_matcher	fs_hws_matcher;
284	};
285	struct mlx5_flow_group_mask	mask;
286	u32				start_index;
287	u32				max_ftes;
288	struct ida			fte_allocator;
289	u32				id;
290	struct rhashtable		ftes_hash;
291	struct rhlist_head		hash;
292};
293
294struct mlx5_flow_root_namespace {
295	struct mlx5_flow_namespace	ns;
296	enum   mlx5_flow_steering_mode	mode;
297	union {
298		struct mlx5_fs_dr_domain	fs_dr_domain;
299		struct mlx5_fs_hws_context	fs_hws_context;
300	};
301	enum   fs_flow_table_type	table_type;
302	struct mlx5_core_dev		*dev;
303	struct mlx5_flow_table		*root_ft;
304	/* Should be held when chaining flow tables */
305	struct mutex			chain_lock;
306	struct list_head		underlay_qpns;
307	const struct mlx5_flow_cmds	*cmds;
308};
309
310enum mlx5_fc_type {
311	MLX5_FC_TYPE_ACQUIRED = 0,
312	MLX5_FC_TYPE_LOCAL,
313};
314
315struct mlx5_fc_cache {
316	u64 packets;
317	u64 bytes;
318	u64 lastuse;
319};
320
321struct mlx5_fc {
322	u32 id;
323	bool aging;
324	enum mlx5_fc_type type;
325	struct mlx5_fc_bulk *bulk;
326	struct mlx5_fc_cache cache;
327	refcount_t fc_local_refcount;
328	/* last{packets,bytes} are used for calculating deltas since last reading. */
329	u64 lastpackets;
330	u64 lastbytes;
331};
332
333struct mlx5_fc_bulk {
334	struct mlx5_fs_bulk fs_bulk;
335	u32 base_id;
336	struct mlx5_fs_hws_data hws_data;
337	struct mlx5_fc fcs[];
338};
339
340u32 mlx5_fc_get_base_id(struct mlx5_fc *counter);
341int mlx5_init_fc_stats(struct mlx5_core_dev *dev);
342void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev);
343void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
344			      struct delayed_work *dwork,
345			      unsigned long delay);
346void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
347				      unsigned long interval);
348
349const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void);
350
351int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns,
352				 struct mlx5_flow_root_namespace *peer_ns,
353				 u16 peer_vhca_id);
354
355int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns,
356				 enum mlx5_flow_steering_mode mode);
357
358int mlx5_fs_core_alloc(struct mlx5_core_dev *dev);
359void mlx5_fs_core_free(struct mlx5_core_dev *dev);
360int mlx5_fs_core_init(struct mlx5_core_dev *dev);
361void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev);
362
363int mlx5_fs_vport_egress_acl_ns_add(struct mlx5_flow_steering *steering,
364				    u16 vport_idx);
365int mlx5_fs_vport_ingress_acl_ns_add(struct mlx5_flow_steering *steering,
366				     u16 vport_idx);
367void mlx5_fs_vport_egress_acl_ns_remove(struct mlx5_flow_steering *steering,
368					int vport_idx);
369void mlx5_fs_vport_ingress_acl_ns_remove(struct mlx5_flow_steering *steering,
370					 int vport_idx);
371
372u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type);
373
374struct mlx5_flow_root_namespace *find_root(struct fs_node *node);
375
376int mlx5_fs_get_packet_reformat_id(struct mlx5_pkt_reformat *pkt_reformat,
377				   u32 *id);
378
379#define fs_get_obj(v, _node)  {v = container_of((_node), typeof(*v), node); }
380
381#define fs_list_for_each_entry(pos, root)		\
382	list_for_each_entry(pos, root, node.list)
383
384#define fs_list_for_each_entry_safe(pos, tmp, root)		\
385	list_for_each_entry_safe(pos, tmp, root, node.list)
386
387#define fs_for_each_ns_or_ft_reverse(pos, prio)				\
388	list_for_each_entry_reverse(pos, &(prio)->node.children, list)
389
390#define fs_for_each_ns_or_ft(pos, prio)					\
391	list_for_each_entry(pos, (&(prio)->node.children), list)
392
393#define fs_for_each_prio(pos, ns)			\
394	fs_list_for_each_entry(pos, &(ns)->node.children)
395
396#define fs_for_each_ns(pos, prio)			\
397	fs_list_for_each_entry(pos, &(prio)->node.children)
398
399#define fs_for_each_ft(pos, prio)			\
400	fs_list_for_each_entry(pos, &(prio)->node.children)
401
402#define fs_for_each_ft_safe(pos, tmp, prio)			\
403	fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children)
404
405#define fs_for_each_fg(pos, ft)			\
406	fs_list_for_each_entry(pos, &(ft)->node.children)
407
408#define fs_for_each_fte(pos, fg)			\
409	fs_list_for_each_entry(pos, &(fg)->node.children)
410
411#define fs_for_each_dst(pos, fte)			\
412	fs_list_for_each_entry(pos, &(fte)->node.children)
413
414#define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) (		\
415	(type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) :		\
416	(type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) :		\
417	(type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) :		\
418	(type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) :		\
419	(type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) :		\
420	(type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) :		\
421	(type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) :		\
422	(type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) :		\
423	(type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) :      \
424	(type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) :      \
425	(type == FS_FT_FDB_RX) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) :      \
426	(type == FS_FT_FDB_TX) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) :      \
427	(type == FS_FT_RDMA_TRANSPORT_RX) ? MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(mdev, cap) :      \
428	(type == FS_FT_RDMA_TRANSPORT_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(mdev, cap) :      \
429	(BUILD_BUG_ON_ZERO(FS_FT_RDMA_TRANSPORT_TX != FS_FT_MAX_TYPE))\
430	)
431
432#endif