qp.h

  1/*
  2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32
 33#ifndef MLX5_QP_H
 34#define MLX5_QP_H
 35
 36#include <linux/mlx5/device.h>
 37#include <linux/mlx5/driver.h>
 38
 39#define MLX5_TERMINATE_SCATTER_LIST_LKEY cpu_to_be32(0x100)
 40/* UMR (3 WQE_BB's) + SIG (3 WQE_BB's) + PSV (mem) + PSV (wire) */
 41#define MLX5_SIG_WQE_SIZE	(MLX5_SEND_WQE_BB * 8)
 42#define MLX5_DIF_SIZE		8
 43#define MLX5_STRIDE_BLOCK_OP	0x400
 44#define MLX5_CPY_GRD_MASK	0xc0
 45#define MLX5_CPY_APP_MASK	0x30
 46#define MLX5_CPY_REF_MASK	0x0f
 47#define MLX5_BSF_INC_REFTAG	(1 << 6)
 48#define MLX5_BSF_INL_VALID	(1 << 15)
 49#define MLX5_BSF_REFRESH_DIF	(1 << 14)
 50#define MLX5_BSF_REPEAT_BLOCK	(1 << 7)
 51#define MLX5_BSF_APPTAG_ESCAPE	0x1
 52#define MLX5_BSF_APPREF_ESCAPE	0x2
 53
 54enum mlx5_qp_optpar {
 55	MLX5_QP_OPTPAR_ALT_ADDR_PATH		= 1 << 0,
 56	MLX5_QP_OPTPAR_RRE			= 1 << 1,
 57	MLX5_QP_OPTPAR_RAE			= 1 << 2,
 58	MLX5_QP_OPTPAR_RWE			= 1 << 3,
 59	MLX5_QP_OPTPAR_PKEY_INDEX		= 1 << 4,
 60	MLX5_QP_OPTPAR_Q_KEY			= 1 << 5,
 61	MLX5_QP_OPTPAR_RNR_TIMEOUT		= 1 << 6,
 62	MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH	= 1 << 7,
 63	MLX5_QP_OPTPAR_SRA_MAX			= 1 << 8,
 64	MLX5_QP_OPTPAR_RRA_MAX			= 1 << 9,
 65	MLX5_QP_OPTPAR_PM_STATE			= 1 << 10,
 66	MLX5_QP_OPTPAR_RETRY_COUNT		= 1 << 12,
 67	MLX5_QP_OPTPAR_RNR_RETRY		= 1 << 13,
 68	MLX5_QP_OPTPAR_ACK_TIMEOUT		= 1 << 14,
 69	MLX5_QP_OPTPAR_LAG_TX_AFF		= 1 << 15,
 70	MLX5_QP_OPTPAR_PRI_PORT			= 1 << 16,
 71	MLX5_QP_OPTPAR_SRQN			= 1 << 18,
 72	MLX5_QP_OPTPAR_CQN_RCV			= 1 << 19,
 73	MLX5_QP_OPTPAR_DC_HS			= 1 << 20,
 74	MLX5_QP_OPTPAR_DC_KEY			= 1 << 21,
 75	MLX5_QP_OPTPAR_COUNTER_SET_ID		= 1 << 25,
 76};
 77
 78enum mlx5_qp_state {
 79	MLX5_QP_STATE_RST			= 0,
 80	MLX5_QP_STATE_INIT			= 1,
 81	MLX5_QP_STATE_RTR			= 2,
 82	MLX5_QP_STATE_RTS			= 3,
 83	MLX5_QP_STATE_SQER			= 4,
 84	MLX5_QP_STATE_SQD			= 5,
 85	MLX5_QP_STATE_ERR			= 6,
 86	MLX5_QP_STATE_SQ_DRAINING		= 7,
 87	MLX5_QP_STATE_SUSPENDED			= 9,
 88	MLX5_QP_NUM_STATE,
 89	MLX5_QP_STATE,
 90	MLX5_QP_STATE_BAD,
 91};
 92
 93enum {
 94	MLX5_SQ_STATE_NA	= MLX5_SQC_STATE_ERR + 1,
 95	MLX5_SQ_NUM_STATE	= MLX5_SQ_STATE_NA + 1,
 96	MLX5_RQ_STATE_NA	= MLX5_RQC_STATE_ERR + 1,
 97	MLX5_RQ_NUM_STATE	= MLX5_RQ_STATE_NA + 1,
 98};
 99
100enum {
101	MLX5_QP_ST_RC				= 0x0,
102	MLX5_QP_ST_UC				= 0x1,
103	MLX5_QP_ST_UD				= 0x2,
104	MLX5_QP_ST_XRC				= 0x3,
105	MLX5_QP_ST_MLX				= 0x4,
106	MLX5_QP_ST_DCI				= 0x5,
107	MLX5_QP_ST_DCT				= 0x6,
108	MLX5_QP_ST_QP0				= 0x7,
109	MLX5_QP_ST_QP1				= 0x8,
110	MLX5_QP_ST_RAW_ETHERTYPE		= 0x9,
111	MLX5_QP_ST_RAW_IPV6			= 0xa,
112	MLX5_QP_ST_SNIFFER			= 0xb,
113	MLX5_QP_ST_SYNC_UMR			= 0xe,
114	MLX5_QP_ST_PTP_1588			= 0xd,
115	MLX5_QP_ST_REG_UMR			= 0xc,
116	MLX5_QP_ST_MAX
117};
118
119enum {
120	MLX5_QP_PM_MIGRATED			= 0x3,
121	MLX5_QP_PM_ARMED			= 0x0,
122	MLX5_QP_PM_REARM			= 0x1
123};
124
125enum {
126	MLX5_NON_ZERO_RQ	= 0x0,
127	MLX5_SRQ_RQ		= 0x1,
128	MLX5_CRQ_RQ		= 0x2,
129	MLX5_ZERO_LEN_RQ	= 0x3
130};
131
132/* TODO REM */
133enum {
134	/* params1 */
135	MLX5_QP_BIT_SRE				= 1 << 15,
136	MLX5_QP_BIT_SWE				= 1 << 14,
137	MLX5_QP_BIT_SAE				= 1 << 13,
138	/* params2 */
139	MLX5_QP_BIT_RRE				= 1 << 15,
140	MLX5_QP_BIT_RWE				= 1 << 14,
141	MLX5_QP_BIT_RAE				= 1 << 13,
142	MLX5_QP_BIT_RIC				= 1 <<	4,
143	MLX5_QP_BIT_CC_SLAVE_RECV		= 1 <<  2,
144	MLX5_QP_BIT_CC_SLAVE_SEND		= 1 <<  1,
145	MLX5_QP_BIT_CC_MASTER			= 1 <<  0
146};
147
148enum {
149	MLX5_WQE_CTRL_CQ_UPDATE		= 2 << 2,
150	MLX5_WQE_CTRL_CQ_UPDATE_AND_EQE	= 3 << 2,
151	MLX5_WQE_CTRL_SOLICITED		= 1 << 1,
152	MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE = 1 << 5,
153};
154
155enum {
156	MLX5_SEND_WQE_DS	= 16,
157	MLX5_SEND_WQE_BB	= 64,
158};
159
160#define MLX5_SEND_WQEBB_NUM_DS	(MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
161
162enum {
163	MLX5_SEND_WQE_MAX_WQEBBS	= 16,
164};
165
166#define MLX5_SEND_WQE_MAX_SIZE (MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQE_BB)
167
168enum {
169	MLX5_WQE_FMR_PERM_LOCAL_READ	= 1 << 27,
170	MLX5_WQE_FMR_PERM_LOCAL_WRITE	= 1 << 28,
171	MLX5_WQE_FMR_PERM_REMOTE_READ	= 1 << 29,
172	MLX5_WQE_FMR_PERM_REMOTE_WRITE	= 1 << 30,
173	MLX5_WQE_FMR_PERM_ATOMIC	= 1 << 31
174};
175
176enum {
177	MLX5_FENCE_MODE_NONE			= 0 << 5,
178	MLX5_FENCE_MODE_INITIATOR_SMALL		= 1 << 5,
179	MLX5_FENCE_MODE_FENCE			= 2 << 5,
180	MLX5_FENCE_MODE_STRONG_ORDERING		= 3 << 5,
181	MLX5_FENCE_MODE_SMALL_AND_FENCE		= 4 << 5,
182};
183
184enum {
185	MLX5_RCV_DBR	= 0,
186	MLX5_SND_DBR	= 1,
187};
188
189enum {
190	MLX5_FLAGS_INLINE	= 1<<7,
191	MLX5_FLAGS_CHECK_FREE   = 1<<5,
192};
193
194struct mlx5_wqe_fmr_seg {
195	__be32			flags;
196	__be32			mem_key;
197	__be64			buf_list;
198	__be64			start_addr;
199	__be64			reg_len;
200	__be32			offset;
201	__be32			page_size;
202	u32			reserved[2];
203};
204
205struct mlx5_wqe_ctrl_seg {
206	__be32			opmod_idx_opcode;
207	__be32			qpn_ds;
208
209	struct_group(trailer,
210
211	u8			signature;
212	u8			rsvd[2];
213	u8			fm_ce_se;
214	union {
215		__be32		general_id;
216		__be32		imm;
217		__be32		umr_mkey;
218		__be32		tis_tir_num;
219	};
220
221	); /* end of trailer group */
222};
223
224#define MLX5_WQE_CTRL_DS_MASK 0x3f
225#define MLX5_WQE_CTRL_QPN_MASK 0xffffff00
226#define MLX5_WQE_CTRL_QPN_SHIFT 8
227#define MLX5_WQE_DS_UNITS 16
228#define MLX5_WQE_CTRL_OPCODE_MASK 0xff
229#define MLX5_WQE_CTRL_WQE_INDEX_MASK 0x00ffff00
230#define MLX5_WQE_CTRL_WQE_INDEX_SHIFT 8
231
232enum {
233	MLX5_ETH_WQE_L3_INNER_CSUM      = 1 << 4,
234	MLX5_ETH_WQE_L4_INNER_CSUM      = 1 << 5,
235	MLX5_ETH_WQE_L3_CSUM            = 1 << 6,
236	MLX5_ETH_WQE_L4_CSUM            = 1 << 7,
237};
238
239enum {
240	MLX5_ETH_WQE_TRAILER_HDR_OUTER_IP_ASSOC = 1 << 26,
241	MLX5_ETH_WQE_TRAILER_HDR_OUTER_L4_ASSOC = 1 << 27,
242	MLX5_ETH_WQE_TRAILER_HDR_INNER_IP_ASSOC = 3 << 26,
243	MLX5_ETH_WQE_TRAILER_HDR_INNER_L4_ASSOC = 1 << 28,
244	MLX5_ETH_WQE_INSERT_TRAILER     = 1 << 30,
245};
246
247enum {
248	MLX5_ETH_WQE_SWP_INNER_L3_IPV6  = 1 << 0,
249	MLX5_ETH_WQE_SWP_INNER_L4_UDP   = 1 << 1,
250	MLX5_ETH_WQE_SWP_OUTER_L3_IPV6  = 1 << 4,
251	MLX5_ETH_WQE_SWP_OUTER_L4_UDP   = 1 << 5,
252};
253
254/* Metadata bits 0-7 are used by timestamping */
255/* Base shift for metadata bits used by IPsec and MACsec */
256#define MLX5_ETH_WQE_FT_META_SHIFT 8
257
258enum {
259	MLX5_ETH_WQE_FT_META_IPSEC = BIT(0) << MLX5_ETH_WQE_FT_META_SHIFT,
260	MLX5_ETH_WQE_FT_META_MACSEC = BIT(1) << MLX5_ETH_WQE_FT_META_SHIFT,
261	MLX5_ETH_WQE_FT_META_MACSEC_FS_ID_MASK =
262		GENMASK(5, 2) << MLX5_ETH_WQE_FT_META_SHIFT,
263};
264
265struct mlx5_wqe_eth_seg {
266	u8              swp_outer_l4_offset;
267	u8              swp_outer_l3_offset;
268	u8              swp_inner_l4_offset;
269	u8              swp_inner_l3_offset;
270	u8              cs_flags;
271	u8              swp_flags;
272	__be16          mss;
273	__be32          flow_table_metadata;
274	union {
275		struct {
276			__be16 sz;
277			union {
278				u8     start[2];
279				DECLARE_FLEX_ARRAY(u8, data);
280			};
281		} inline_hdr;
282		__be32 trailer;
283	};
284};
285
286struct mlx5_wqe_xrc_seg {
287	__be32			xrc_srqn;
288	u8			rsvd[12];
289};
290
291struct mlx5_wqe_masked_atomic_seg {
292	__be64			swap_add;
293	__be64			compare;
294	__be64			swap_add_mask;
295	__be64			compare_mask;
296};
297
298struct mlx5_base_av {
299	union {
300		struct {
301			__be32	qkey;
302			__be32	reserved;
303		} qkey;
304		__be64	dc_key;
305	} key;
306	__be32	dqp_dct;
307	u8	stat_rate_sl;
308	u8	fl_mlid;
309	union {
310		__be16	rlid;
311		__be16  udp_sport;
312	};
313};
314
315struct mlx5_av {
316	union {
317		struct {
318			__be32	qkey;
319			__be32	reserved;
320		} qkey;
321		__be64	dc_key;
322	} key;
323	__be32	dqp_dct;
324	u8	stat_rate_sl;
325	u8	fl_mlid;
326	union {
327		__be16	rlid;
328		__be16  udp_sport;
329	};
330	u8	reserved0[4];
331	u8	rmac[6];
332	u8	tclass;
333	u8	hop_limit;
334	__be32	grh_gid_fl;
335	u8	rgid[16];
336};
337
338struct mlx5_ib_ah {
339	struct ib_ah		ibah;
340	struct mlx5_av		av;
341	u8			xmit_port;
342};
343
344static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
345{
346	return container_of(ibah, struct mlx5_ib_ah, ibah);
347}
348
349struct mlx5_wqe_datagram_seg {
350	struct mlx5_av	av;
351};
352
353struct mlx5_wqe_raddr_seg {
354	__be64			raddr;
355	__be32			rkey;
356	u32			reserved;
357};
358
359struct mlx5_wqe_atomic_seg {
360	__be64			swap_add;
361	__be64			compare;
362};
363
364struct mlx5_wqe_data_seg {
365	__be32			byte_count;
366	__be32			lkey;
367	__be64			addr;
368};
369
370struct mlx5_wqe_umr_ctrl_seg {
371	u8		flags;
372	u8		rsvd0[3];
373	__be16		xlt_octowords;
374	union {
375		__be16	xlt_offset;
376		__be16	bsf_octowords;
377	};
378	__be64		mkey_mask;
379	__be32		xlt_offset_47_16;
380	u8		rsvd1[28];
381};
382
383struct mlx5_seg_set_psv {
384	__be32		psv_num;
385	__be16		syndrome;
386	__be16		status;
387	__be32		transient_sig;
388	__be32		ref_tag;
389};
390
391struct mlx5_seg_get_psv {
392	u8		rsvd[19];
393	u8		num_psv;
394	__be32		l_key;
395	__be64		va;
396	__be32		psv_index[4];
397};
398
399struct mlx5_seg_check_psv {
400	u8		rsvd0[2];
401	__be16		err_coalescing_op;
402	u8		rsvd1[2];
403	__be16		xport_err_op;
404	u8		rsvd2[2];
405	__be16		xport_err_mask;
406	u8		rsvd3[7];
407	u8		num_psv;
408	__be32		l_key;
409	__be64		va;
410	__be32		psv_index[4];
411};
412
413struct mlx5_rwqe_sig {
414	u8	rsvd0[4];
415	u8	signature;
416	u8	rsvd1[11];
417};
418
419struct mlx5_wqe_signature_seg {
420	u8	rsvd0[4];
421	u8	signature;
422	u8	rsvd1[11];
423};
424
425#define MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK 0x3ff
426
427struct mlx5_wqe_inline_seg {
428	__be32	byte_count;
429	__be32	data[];
430};
431
432enum mlx5_sig_type {
433	MLX5_DIF_CRC = 0x1,
434	MLX5_DIF_IPCS = 0x2,
435};
436
437struct mlx5_bsf_inl {
438	__be16		vld_refresh;
439	__be16		dif_apptag;
440	__be32		dif_reftag;
441	u8		sig_type;
442	u8		rp_inv_seed;
443	u8		rsvd[3];
444	u8		dif_inc_ref_guard_check;
445	__be16		dif_app_bitmask_check;
446};
447
448struct mlx5_bsf {
449	struct mlx5_bsf_basic {
450		u8		bsf_size_sbs;
451		u8		check_byte_mask;
452		union {
453			u8	copy_byte_mask;
454			u8	bs_selector;
455			u8	rsvd_wflags;
456		} wire;
457		union {
458			u8	bs_selector;
459			u8	rsvd_mflags;
460		} mem;
461		__be32		raw_data_size;
462		__be32		w_bfs_psv;
463		__be32		m_bfs_psv;
464	} basic;
465	struct mlx5_bsf_ext {
466		__be32		t_init_gen_pro_size;
467		__be32		rsvd_epi_size;
468		__be32		w_tfs_psv;
469		__be32		m_tfs_psv;
470	} ext;
471	struct mlx5_bsf_inl	w_inl;
472	struct mlx5_bsf_inl	m_inl;
473};
474
475struct mlx5_mtt {
476	__be64		ptag;
477};
478
479struct mlx5_klm {
480	__be32		bcount;
481	__be32		key;
482	__be64		va;
483};
484
485struct mlx5_ksm {
486	__be32		reserved;
487	__be32		key;
488	__be64		va;
489};
490
491struct mlx5_stride_block_entry {
492	__be16		stride;
493	__be16		bcount;
494	__be32		key;
495	__be64		va;
496};
497
498struct mlx5_stride_block_ctrl_seg {
499	__be32		bcount_per_cycle;
500	__be32		op;
501	__be32		repeat_count;
502	u16		rsvd;
503	__be16		num_entries;
504};
505
506struct mlx5_wqe_flow_update_ctrl_seg {
507	__be32		flow_idx_update;
508	__be32		dest_handle;
509	u8		reserved0[40];
510};
511
512struct mlx5_wqe_header_modify_argument_update_seg {
513	u8		argument_list[64];
514};
515
516struct mlx5_core_qp {
517	struct mlx5_core_rsc_common	common; /* must be first */
518	void (*event)		(struct mlx5_core_qp *, int);
519	int			qpn;
520	struct mlx5_rsc_debug	*dbg;
521	int			pid;
522	u16			uid;
523};
524
525struct mlx5_core_dct {
526	struct mlx5_core_qp	mqp;
527	struct completion	drained;
528};
529
530int mlx5_debug_qp_add(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
531void mlx5_debug_qp_remove(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
532
533static inline const char *mlx5_qp_type_str(int type)
534{
535	switch (type) {
536	case MLX5_QP_ST_RC: return "RC";
537	case MLX5_QP_ST_UC: return "C";
538	case MLX5_QP_ST_UD: return "UD";
539	case MLX5_QP_ST_XRC: return "XRC";
540	case MLX5_QP_ST_MLX: return "MLX";
541	case MLX5_QP_ST_QP0: return "QP0";
542	case MLX5_QP_ST_QP1: return "QP1";
543	case MLX5_QP_ST_RAW_ETHERTYPE: return "RAW_ETHERTYPE";
544	case MLX5_QP_ST_RAW_IPV6: return "RAW_IPV6";
545	case MLX5_QP_ST_SNIFFER: return "SNIFFER";
546	case MLX5_QP_ST_SYNC_UMR: return "SYNC_UMR";
547	case MLX5_QP_ST_PTP_1588: return "PTP_1588";
548	case MLX5_QP_ST_REG_UMR: return "REG_UMR";
549	default: return "Invalid transport type";
550	}
551}
552
553static inline const char *mlx5_qp_state_str(int state)
554{
555	switch (state) {
556	case MLX5_QP_STATE_RST:
557	return "RST";
558	case MLX5_QP_STATE_INIT:
559	return "INIT";
560	case MLX5_QP_STATE_RTR:
561	return "RTR";
562	case MLX5_QP_STATE_RTS:
563	return "RTS";
564	case MLX5_QP_STATE_SQER:
565	return "SQER";
566	case MLX5_QP_STATE_SQD:
567	return "SQD";
568	case MLX5_QP_STATE_ERR:
569	return "ERR";
570	case MLX5_QP_STATE_SQ_DRAINING:
571	return "SQ_DRAINING";
572	case MLX5_QP_STATE_SUSPENDED:
573	return "SUSPENDED";
574	default: return "Invalid QP state";
575	}
576}
577
578static inline int mlx5_get_qp_default_ts(struct mlx5_core_dev *dev)
579{
580	u8 supported_ts_cap = mlx5_get_roce_state(dev) ?
581			      MLX5_CAP_ROCE(dev, qp_ts_format) :
582			      MLX5_CAP_GEN(dev, sq_ts_format);
583
584	return supported_ts_cap ? MLX5_TIMESTAMP_FORMAT_DEFAULT :
585	       MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
586}
587
588#endif /* MLX5_QP_H */