hns_roce_device.h

   1/*
   2 * Copyright (c) 2016 Hisilicon Limited.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#ifndef _HNS_ROCE_DEVICE_H
  34#define _HNS_ROCE_DEVICE_H
  35
  36#include <linux/pci.h>
  37#include <rdma/ib_verbs.h>
  38#include <rdma/hns-abi.h>
  39#include "hns_roce_debugfs.h"
  40
  41#define PCI_REVISION_ID_HIP08			0x21
  42#define PCI_REVISION_ID_HIP09			0x30
  43
  44#define HNS_ROCE_MAX_MSG_LEN			0x80000000
  45
  46#define HNS_ROCE_IB_MIN_SQ_STRIDE		6
  47
  48#define BA_BYTE_LEN				8
  49
  50#define HNS_ROCE_MIN_CQE_NUM			0x40
  51#define HNS_ROCE_MIN_SRQ_WQE_NUM		1
  52
  53#define HNS_ROCE_MAX_IRQ_NUM			128
  54
  55#define HNS_ROCE_SGE_IN_WQE			2
  56#define HNS_ROCE_SGE_SHIFT			4
  57
  58#define EQ_ENABLE				1
  59#define EQ_DISABLE				0
  60
  61#define HNS_ROCE_CEQ				0
  62#define HNS_ROCE_AEQ				1
  63
  64#define HNS_ROCE_CEQE_SIZE 0x4
  65#define HNS_ROCE_AEQE_SIZE 0x10
  66
  67#define HNS_ROCE_V3_EQE_SIZE 0x40
  68
  69#define HNS_ROCE_V2_CQE_SIZE 32
  70#define HNS_ROCE_V3_CQE_SIZE 64
  71
  72#define HNS_ROCE_V2_QPC_SZ 256
  73#define HNS_ROCE_V3_QPC_SZ 512
  74
  75#define HNS_ROCE_MAX_PORTS			6
  76#define HNS_ROCE_GID_SIZE			16
  77#define HNS_ROCE_SGE_SIZE			16
  78#define HNS_ROCE_DWQE_SIZE			65536
  79
  80#define HNS_ROCE_HOP_NUM_0			0xff
  81
  82#define MR_TYPE_MR				0x00
  83#define MR_TYPE_FRMR				0x01
  84#define MR_TYPE_DMA				0x03
  85
  86#define HNS_ROCE_FRMR_MAX_PA			512
  87#define HNS_ROCE_FRMR_ALIGN_SIZE		128
  88
  89#define PKEY_ID					0xffff
  90#define NODE_DESC_SIZE				64
  91#define DB_REG_OFFSET				0x1000
  92
  93/* Configure to HW for PAGE_SIZE larger than 4KB */
  94#define PG_SHIFT_OFFSET				(PAGE_SHIFT - 12)
  95
  96#define ATOMIC_WR_LEN				8
  97
  98#define HNS_ROCE_IDX_QUE_ENTRY_SZ		4
  99#define SRQ_DB_REG				0x230
 100
 101#define HNS_ROCE_QP_BANK_NUM 8
 102#define HNS_ROCE_CQ_BANK_NUM 4
 103
 104#define CQ_BANKID_SHIFT 2
 105#define CQ_BANKID_MASK GENMASK(1, 0)
 106
 107#define HNS_ROCE_MAX_CQ_COUNT 0xFFFF
 108#define HNS_ROCE_MAX_CQ_PERIOD 0xFFFF
 109
 110enum {
 111	SERV_TYPE_RC,
 112	SERV_TYPE_UC,
 113	SERV_TYPE_RD,
 114	SERV_TYPE_UD,
 115	SERV_TYPE_XRC = 5,
 116};
 117
 118enum hns_roce_event {
 119	HNS_ROCE_EVENT_TYPE_PATH_MIG                  = 0x01,
 120	HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED           = 0x02,
 121	HNS_ROCE_EVENT_TYPE_COMM_EST                  = 0x03,
 122	HNS_ROCE_EVENT_TYPE_SQ_DRAINED                = 0x04,
 123	HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR            = 0x05,
 124	HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR    = 0x06,
 125	HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR     = 0x07,
 126	HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH           = 0x08,
 127	HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH        = 0x09,
 128	HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR           = 0x0a,
 129	HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR           = 0x0b,
 130	HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW               = 0x0c,
 131	HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID             = 0x0d,
 132	HNS_ROCE_EVENT_TYPE_PORT_CHANGE               = 0x0f,
 133	/* 0x10 and 0x11 is unused in currently application case */
 134	HNS_ROCE_EVENT_TYPE_DB_OVERFLOW               = 0x12,
 135	HNS_ROCE_EVENT_TYPE_MB                        = 0x13,
 136	HNS_ROCE_EVENT_TYPE_FLR			      = 0x15,
 137	HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION	      = 0x16,
 138	HNS_ROCE_EVENT_TYPE_INVALID_XRCETH	      = 0x17,
 139};
 140
 141enum {
 142	HNS_ROCE_CAP_FLAG_REREG_MR		= BIT(0),
 143	HNS_ROCE_CAP_FLAG_ROCE_V1_V2		= BIT(1),
 144	HNS_ROCE_CAP_FLAG_RQ_INLINE		= BIT(2),
 145	HNS_ROCE_CAP_FLAG_CQ_RECORD_DB		= BIT(3),
 146	HNS_ROCE_CAP_FLAG_QP_RECORD_DB		= BIT(4),
 147	HNS_ROCE_CAP_FLAG_SRQ			= BIT(5),
 148	HNS_ROCE_CAP_FLAG_XRC			= BIT(6),
 149	HNS_ROCE_CAP_FLAG_MW			= BIT(7),
 150	HNS_ROCE_CAP_FLAG_FRMR                  = BIT(8),
 151	HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL		= BIT(9),
 152	HNS_ROCE_CAP_FLAG_ATOMIC		= BIT(10),
 153	HNS_ROCE_CAP_FLAG_DIRECT_WQE		= BIT(12),
 154	HNS_ROCE_CAP_FLAG_SDI_MODE		= BIT(14),
 155	HNS_ROCE_CAP_FLAG_STASH			= BIT(17),
 156	HNS_ROCE_CAP_FLAG_CQE_INLINE		= BIT(19),
 157	HNS_ROCE_CAP_FLAG_BOND                  = BIT(21),
 158	HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB         = BIT(22),
 159};
 160
 161#define HNS_ROCE_DB_TYPE_COUNT			2
 162#define HNS_ROCE_DB_UNIT_SIZE			4
 163
 164enum {
 165	HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4
 166};
 167
 168enum hns_roce_reset_stage {
 169	HNS_ROCE_STATE_NON_RST,
 170	HNS_ROCE_STATE_RST_BEF_DOWN,
 171	HNS_ROCE_STATE_RST_DOWN,
 172	HNS_ROCE_STATE_RST_UNINIT,
 173	HNS_ROCE_STATE_RST_INIT,
 174	HNS_ROCE_STATE_RST_INITED,
 175};
 176
 177enum hns_roce_instance_state {
 178	HNS_ROCE_STATE_NON_INIT,
 179	HNS_ROCE_STATE_INIT,
 180	HNS_ROCE_STATE_INITED,
 181	HNS_ROCE_STATE_UNINIT,
 182	HNS_ROCE_STATE_BOND_UNINIT,
 183};
 184
 185enum {
 186	HNS_ROCE_RST_DIRECT_RETURN		= 0,
 187};
 188
 189#define HNS_ROCE_CMD_SUCCESS			1
 190
 191#define HNS_ROCE_MAX_HOP_NUM			3
 192/* The minimum page size is 4K for hardware */
 193#define HNS_HW_PAGE_SHIFT			12
 194#define HNS_HW_PAGE_SIZE			(1 << HNS_HW_PAGE_SHIFT)
 195
 196#define HNS_HW_MAX_PAGE_SHIFT			27
 197#define HNS_HW_MAX_PAGE_SIZE			(1 << HNS_HW_MAX_PAGE_SHIFT)
 198
 199struct hns_roce_uar {
 200	u64		pfn;
 201	unsigned long	index;
 202	unsigned long	logic_idx;
 203};
 204
 205enum hns_roce_mmap_type {
 206	HNS_ROCE_MMAP_TYPE_DB = 1,
 207	HNS_ROCE_MMAP_TYPE_DWQE,
 208};
 209
 210struct hns_user_mmap_entry {
 211	struct rdma_user_mmap_entry rdma_entry;
 212	enum hns_roce_mmap_type mmap_type;
 213	u64 address;
 214};
 215
 216struct hns_roce_ucontext {
 217	struct ib_ucontext	ibucontext;
 218	struct hns_roce_uar	uar;
 219	struct list_head	page_list;
 220	struct mutex		page_mutex;
 221	struct hns_user_mmap_entry *db_mmap_entry;
 222	u32			config;
 223	u8 cq_bank_id;
 224};
 225
 226struct hns_roce_pd {
 227	struct ib_pd		ibpd;
 228	unsigned long		pdn;
 229};
 230
 231struct hns_roce_xrcd {
 232	struct ib_xrcd ibxrcd;
 233	u32 xrcdn;
 234};
 235
 236struct hns_roce_bitmap {
 237	/* Bitmap Traversal last a bit which is 1 */
 238	unsigned long		last;
 239	unsigned long		top;
 240	unsigned long		max;
 241	unsigned long		reserved_top;
 242	unsigned long		mask;
 243	spinlock_t		lock;
 244	unsigned long		*table;
 245};
 246
 247struct hns_roce_ida {
 248	struct ida ida;
 249	u32 min; /* Lowest ID to allocate.  */
 250	u32 max; /* Highest ID to allocate. */
 251};
 252
 253/* For Hardware Entry Memory */
 254struct hns_roce_hem_table {
 255	/* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */
 256	u32		type;
 257	/* HEM array elment num */
 258	unsigned long	num_hem;
 259	/* Single obj size */
 260	unsigned long	obj_size;
 261	unsigned long	table_chunk_size;
 262	struct mutex	mutex;
 263	struct hns_roce_hem **hem;
 264	u64		**bt_l1;
 265	dma_addr_t	*bt_l1_dma_addr;
 266	u64		**bt_l0;
 267	dma_addr_t	*bt_l0_dma_addr;
 268};
 269
 270struct hns_roce_buf_region {
 271	u32 offset; /* page offset */
 272	u32 count; /* page count */
 273	int hopnum; /* addressing hop num */
 274};
 275
 276#define HNS_ROCE_MAX_BT_REGION	3
 277#define HNS_ROCE_MAX_BT_LEVEL	3
 278struct hns_roce_hem_list {
 279	struct list_head root_bt;
 280	/* link all bt dma mem by hop config */
 281	struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL];
 282	struct list_head btm_bt; /* link all bottom bt in @mid_bt */
 283	dma_addr_t root_ba; /* pointer to the root ba table */
 284};
 285
 286enum mtr_type {
 287	MTR_DEFAULT = 0,
 288	MTR_PBL,
 289};
 290
 291struct hns_roce_buf_attr {
 292	struct {
 293		size_t	size;  /* region size */
 294		int	hopnum; /* multi-hop addressing hop num */
 295	} region[HNS_ROCE_MAX_BT_REGION];
 296	unsigned int region_count; /* valid region count */
 297	unsigned int page_shift;  /* buffer page shift */
 298	unsigned int user_access; /* umem access flag */
 299	u64 iova;
 300	enum mtr_type type;
 301	bool mtt_only; /* only alloc buffer-required MTT memory */
 302	bool adaptive; /* adaptive for page_shift and hopnum */
 303};
 304
 305struct hns_roce_hem_cfg {
 306	dma_addr_t	root_ba; /* root BA table's address */
 307	bool		is_direct; /* addressing without BA table */
 308	unsigned int	ba_pg_shift; /* BA table page shift */
 309	unsigned int	buf_pg_shift; /* buffer page shift */
 310	unsigned int	buf_pg_count;  /* buffer page count */
 311	struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
 312	unsigned int	region_count;
 313};
 314
 315/* memory translate region */
 316struct hns_roce_mtr {
 317	struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */
 318	struct ib_umem		*umem; /* user space buffer */
 319	struct hns_roce_buf	*kmem; /* kernel space buffer */
 320	struct hns_roce_hem_cfg  hem_cfg; /* config for hardware addressing */
 321};
 322
 323struct hns_roce_mr {
 324	struct ib_mr		ibmr;
 325	u64			iova; /* MR's virtual original addr */
 326	u64			size; /* Address range of MR */
 327	u32			key; /* Key of MR */
 328	u32			pd;   /* PD num of MR */
 329	u32			access; /* Access permission of MR */
 330	int			enabled; /* MR's active status */
 331	int			type; /* MR's register type */
 332	u32			pbl_hop_num; /* multi-hop number */
 333	struct hns_roce_mtr	pbl_mtr;
 334	u32			npages;
 335	dma_addr_t		*page_list;
 336};
 337
 338struct hns_roce_mr_table {
 339	struct hns_roce_ida mtpt_ida;
 340	struct hns_roce_hem_table	mtpt_table;
 341};
 342
 343struct hns_roce_wq {
 344	u64		*wrid;     /* Work request ID */
 345	spinlock_t	lock;
 346	u32		wqe_cnt;  /* WQE num */
 347	u32		max_gs;
 348	u32		rsv_sge;
 349	u32		offset;
 350	u32		wqe_shift; /* WQE size */
 351	u32		head;
 352	u32		tail;
 353	void __iomem	*db_reg;
 354	u32		ext_sge_cnt;
 355};
 356
 357struct hns_roce_sge {
 358	unsigned int	sge_cnt; /* SGE num */
 359	u32		offset;
 360	u32		sge_shift; /* SGE size */
 361};
 362
 363struct hns_roce_buf_list {
 364	void		*buf;
 365	dma_addr_t	map;
 366};
 367
 368/*
 369 * %HNS_ROCE_BUF_DIRECT indicates that the all memory must be in a continuous
 370 * dma address range.
 371 *
 372 * %HNS_ROCE_BUF_NOSLEEP indicates that the caller cannot sleep.
 373 *
 374 * %HNS_ROCE_BUF_NOFAIL allocation only failed when allocated size is zero, even
 375 * the allocated size is smaller than the required size.
 376 */
 377enum {
 378	HNS_ROCE_BUF_DIRECT = BIT(0),
 379	HNS_ROCE_BUF_NOSLEEP = BIT(1),
 380	HNS_ROCE_BUF_NOFAIL = BIT(2),
 381};
 382
 383struct hns_roce_buf {
 384	struct hns_roce_buf_list	*trunk_list;
 385	u32				ntrunks;
 386	u32				npages;
 387	unsigned int			trunk_shift;
 388	unsigned int			page_shift;
 389};
 390
 391struct hns_roce_db_pgdir {
 392	struct list_head	list;
 393	DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE);
 394	DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
 395	unsigned long		*bits[HNS_ROCE_DB_TYPE_COUNT];
 396	u32			*page;
 397	dma_addr_t		db_dma;
 398};
 399
 400struct hns_roce_user_db_page {
 401	struct list_head	list;
 402	struct ib_umem		*umem;
 403	unsigned long		user_virt;
 404	refcount_t		refcount;
 405};
 406
 407struct hns_roce_db {
 408	u32		*db_record;
 409	union {
 410		struct hns_roce_db_pgdir *pgdir;
 411		struct hns_roce_user_db_page *user_page;
 412	} u;
 413	dma_addr_t	dma;
 414	void		*virt_addr;
 415	unsigned long	index;
 416	unsigned long	order;
 417};
 418
 419struct hns_roce_cq {
 420	struct ib_cq			ib_cq;
 421	struct hns_roce_mtr		mtr;
 422	struct hns_roce_db		db;
 423	u32				flags;
 424	spinlock_t			lock;
 425	u32				cq_depth;
 426	u32				cons_index;
 427	u32				*set_ci_db;
 428	void __iomem			*db_reg;
 429	int				arm_sn;
 430	int				cqe_size;
 431	unsigned long			cqn;
 432	u32				vector;
 433	refcount_t			refcount;
 434	struct completion		free;
 435	struct list_head		sq_list; /* all qps on this send cq */
 436	struct list_head		rq_list; /* all qps on this recv cq */
 437	int				is_armed; /* cq is armed */
 438	struct list_head		node; /* all armed cqs are on a list */
 439};
 440
 441struct hns_roce_idx_que {
 442	struct hns_roce_mtr		mtr;
 443	u32				entry_shift;
 444	unsigned long			*bitmap;
 445	u32				head;
 446	u32				tail;
 447};
 448
 449struct hns_roce_srq {
 450	struct ib_srq		ibsrq;
 451	unsigned long		srqn;
 452	u32			wqe_cnt;
 453	int			max_gs;
 454	u32			rsv_sge;
 455	u32			wqe_shift;
 456	u32			cqn;
 457	u32			xrcdn;
 458	void __iomem		*db_reg;
 459
 460	refcount_t		refcount;
 461	struct completion	free;
 462
 463	struct hns_roce_mtr	buf_mtr;
 464
 465	u64		       *wrid;
 466	struct hns_roce_idx_que idx_que;
 467	spinlock_t		lock;
 468	struct mutex		mutex;
 469	void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event);
 470	struct hns_roce_db	rdb;
 471	u32			cap_flags;
 472};
 473
 474struct hns_roce_uar_table {
 475	struct hns_roce_bitmap bitmap;
 476};
 477
 478struct hns_roce_bank {
 479	struct ida ida;
 480	u32 inuse; /* Number of IDs allocated */
 481	u32 min; /* Lowest ID to allocate.  */
 482	u32 max; /* Highest ID to allocate. */
 483	u32 next; /* Next ID to allocate. */
 484};
 485
 486struct hns_roce_qp_table {
 487	struct hns_roce_hem_table	qp_table;
 488	struct hns_roce_hem_table	irrl_table;
 489	struct hns_roce_hem_table	trrl_table;
 490	struct hns_roce_hem_table	sccc_table;
 491	struct mutex			scc_mutex;
 492	struct hns_roce_bank bank[HNS_ROCE_QP_BANK_NUM];
 493	struct mutex bank_mutex;
 494	struct xarray			dip_xa;
 495};
 496
 497struct hns_roce_cq_table {
 498	struct xarray			array;
 499	struct hns_roce_hem_table	table;
 500	struct hns_roce_bank bank[HNS_ROCE_CQ_BANK_NUM];
 501	struct mutex			bank_mutex;
 502	u32 ctx_num[HNS_ROCE_CQ_BANK_NUM];
 503};
 504
 505struct hns_roce_srq_table {
 506	struct hns_roce_ida		srq_ida;
 507	struct xarray			xa;
 508	struct hns_roce_hem_table	table;
 509};
 510
 511struct hns_roce_av {
 512	u8 port;
 513	u8 gid_index;
 514	u8 stat_rate;
 515	u8 hop_limit;
 516	u32 flowlabel;
 517	u16 udp_sport;
 518	u8 sl;
 519	u8 tclass;
 520	u8 dgid[HNS_ROCE_GID_SIZE];
 521	u8 mac[ETH_ALEN];
 522	u16 vlan_id;
 523	u8 vlan_en;
 524};
 525
 526struct hns_roce_ah {
 527	struct ib_ah		ibah;
 528	struct hns_roce_av	av;
 529};
 530
 531struct hns_roce_cmd_context {
 532	struct completion	done;
 533	int			result;
 534	int			next;
 535	u64			out_param;
 536	u16			token;
 537	u16			busy;
 538};
 539
 540enum hns_roce_cmdq_state {
 541	HNS_ROCE_CMDQ_STATE_NORMAL,
 542	HNS_ROCE_CMDQ_STATE_FATAL_ERR,
 543};
 544
 545struct hns_roce_cmdq {
 546	struct dma_pool		*pool;
 547	struct semaphore	poll_sem;
 548	/*
 549	 * Event mode: cmd register mutex protection,
 550	 * ensure to not exceed max_cmds and user use limit region
 551	 */
 552	struct semaphore	event_sem;
 553	int			max_cmds;
 554	spinlock_t		context_lock;
 555	int			free_head;
 556	struct hns_roce_cmd_context *context;
 557	/*
 558	 * Process whether use event mode, init default non-zero
 559	 * After the event queue of cmd event ready,
 560	 * can switch into event mode
 561	 * close device, switch into poll mode(non event mode)
 562	 */
 563	u8			use_events;
 564	enum hns_roce_cmdq_state state;
 565};
 566
 567struct hns_roce_cmd_mailbox {
 568	void		       *buf;
 569	dma_addr_t		dma;
 570};
 571
 572struct hns_roce_mbox_msg {
 573	u64 in_param;
 574	u64 out_param;
 575	u8 cmd;
 576	u32 tag;
 577	u16 token;
 578	u8 event_en;
 579};
 580
 581struct hns_roce_dev;
 582
 583enum {
 584	HNS_ROCE_FLUSH_FLAG = 0,
 585	HNS_ROCE_STOP_FLUSH_FLAG = 1,
 586};
 587
 588struct hns_roce_work {
 589	struct hns_roce_dev *hr_dev;
 590	struct work_struct work;
 591	int event_type;
 592	int sub_type;
 593	u32 queue_num;
 594};
 595
 596enum hns_roce_cong_type {
 597	CONG_TYPE_DCQCN,
 598	CONG_TYPE_LDCP,
 599	CONG_TYPE_HC3,
 600	CONG_TYPE_DIP,
 601};
 602
 603struct hns_roce_qp {
 604	struct ib_qp		ibqp;
 605	struct hns_roce_wq	rq;
 606	struct hns_roce_db	rdb;
 607	struct hns_roce_db	sdb;
 608	unsigned long		en_flags;
 609	enum ib_sig_type	sq_signal_bits;
 610	struct hns_roce_wq	sq;
 611
 612	struct hns_roce_mtr	mtr;
 613
 614	u32			buff_size;
 615	struct mutex		mutex;
 616	u8			port;
 617	u8			phy_port;
 618	u8			sl;
 619	u8			resp_depth;
 620	u8			state;
 621	u32                     atomic_rd_en;
 622	u32			qkey;
 623	void			(*event)(struct hns_roce_qp *qp,
 624					 enum hns_roce_event event_type);
 625	unsigned long		qpn;
 626
 627	u32			xrcdn;
 628
 629	refcount_t		refcount;
 630	struct completion	free;
 631
 632	struct hns_roce_sge	sge;
 633	u32			next_sge;
 634	enum ib_mtu		path_mtu;
 635	u32			max_inline_data;
 636	u8			free_mr_en;
 637
 638	/* 0: flush needed, 1: unneeded */
 639	unsigned long		flush_flag;
 640	struct hns_roce_work	flush_work;
 641	struct list_head	node; /* all qps are on a list */
 642	struct list_head	rq_node; /* all recv qps are on a list */
 643	struct list_head	sq_node; /* all send qps are on a list */
 644	struct hns_user_mmap_entry *dwqe_mmap_entry;
 645	u32			config;
 646	enum hns_roce_cong_type	cong_type;
 647	u8			tc_mode;
 648	u8			priority;
 649	spinlock_t flush_lock;
 650	struct hns_roce_dip *dip;
 651};
 652
 653struct hns_roce_ib_iboe {
 654	spinlock_t		lock;
 655	struct net_device      *netdevs[HNS_ROCE_MAX_PORTS];
 656	struct notifier_block	nb;
 657	u8			phy_port[HNS_ROCE_MAX_PORTS];
 658};
 659
 660struct hns_roce_ceqe {
 661	__le32	comp;
 662	__le32	rsv[15];
 663};
 664
 665#define CEQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_ceqe, h, l)
 666
 667#define CEQE_CQN CEQE_FIELD_LOC(23, 0)
 668#define CEQE_OWNER CEQE_FIELD_LOC(31, 31)
 669
 670struct hns_roce_aeqe {
 671	__le32 asyn;
 672	union {
 673		struct {
 674			__le32 num;
 675			u32 rsv0;
 676			u32 rsv1;
 677		} queue_event;
 678
 679		struct {
 680			__le64  out_param;
 681			__le16  token;
 682			u8	status;
 683			u8	rsv0;
 684		} __packed cmd;
 685	 } event;
 686	__le32 rsv[12];
 687};
 688
 689#define AEQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_aeqe, h, l)
 690
 691#define AEQE_EVENT_TYPE AEQE_FIELD_LOC(7, 0)
 692#define AEQE_SUB_TYPE AEQE_FIELD_LOC(15, 8)
 693#define AEQE_OWNER AEQE_FIELD_LOC(31, 31)
 694#define AEQE_EVENT_QUEUE_NUM AEQE_FIELD_LOC(55, 32)
 695
 696struct hns_roce_eq {
 697	struct hns_roce_dev		*hr_dev;
 698	void __iomem			*db_reg;
 699
 700	int				type_flag; /* Aeq:1 ceq:0 */
 701	int				eqn;
 702	u32				entries;
 703	int				eqe_size;
 704	int				irq;
 705	u32				cons_index;
 706	int				over_ignore;
 707	int				coalesce;
 708	int				arm_st;
 709	int				hop_num;
 710	struct hns_roce_mtr		mtr;
 711	u16				eq_max_cnt;
 712	u32				eq_period;
 713	int				shift;
 714	int				event_type;
 715	int				sub_type;
 716	struct work_struct		work;
 717};
 718
 719struct hns_roce_eq_table {
 720	struct hns_roce_eq	*eq;
 721};
 722
 723struct hns_roce_caps {
 724	u64		fw_ver;
 725	u8		num_ports;
 726	int		gid_table_len[HNS_ROCE_MAX_PORTS];
 727	int		pkey_table_len[HNS_ROCE_MAX_PORTS];
 728	int		local_ca_ack_delay;
 729	int		num_uars;
 730	u32		phy_num_uars;
 731	u32		max_sq_sg;
 732	u32		max_sq_inline;
 733	u32		max_rq_sg;
 734	u32		rsv0;
 735	u32		num_qps;
 736	u32		reserved_qps;
 737	u32		num_srqs;
 738	u32		max_wqes;
 739	u32		max_srq_wrs;
 740	u32		max_srq_sges;
 741	u32		max_sq_desc_sz;
 742	u32		max_rq_desc_sz;
 743	u32		rsv2;
 744	int		max_qp_init_rdma;
 745	int		max_qp_dest_rdma;
 746	u32		num_cqs;
 747	u32		max_cqes;
 748	u32		min_cqes;
 749	u32		min_wqes;
 750	u32		reserved_cqs;
 751	u32		reserved_srqs;
 752	int		num_aeq_vectors;
 753	int		num_comp_vectors;
 754	int		num_other_vectors;
 755	u32		num_mtpts;
 756	u32		rsv1;
 757	u32		num_srqwqe_segs;
 758	u32		num_idx_segs;
 759	int		reserved_mrws;
 760	int		reserved_uars;
 761	int		num_pds;
 762	int		reserved_pds;
 763	u32		num_xrcds;
 764	u32		reserved_xrcds;
 765	u32		mtt_entry_sz;
 766	u32		cqe_sz;
 767	u32		page_size_cap;
 768	u32		reserved_lkey;
 769	int		mtpt_entry_sz;
 770	int		qpc_sz;
 771	int		irrl_entry_sz;
 772	int		trrl_entry_sz;
 773	int		cqc_entry_sz;
 774	int		sccc_sz;
 775	int		qpc_timer_entry_sz;
 776	int		cqc_timer_entry_sz;
 777	int		srqc_entry_sz;
 778	int		idx_entry_sz;
 779	u32		pbl_ba_pg_sz;
 780	u32		pbl_buf_pg_sz;
 781	u32		pbl_hop_num;
 782	int		aeqe_depth;
 783	int		ceqe_depth;
 784	u32		aeqe_size;
 785	u32		ceqe_size;
 786	enum ib_mtu	max_mtu;
 787	u32		qpc_bt_num;
 788	u32		qpc_timer_bt_num;
 789	u32		srqc_bt_num;
 790	u32		cqc_bt_num;
 791	u32		cqc_timer_bt_num;
 792	u32		mpt_bt_num;
 793	u32		eqc_bt_num;
 794	u32		smac_bt_num;
 795	u32		sgid_bt_num;
 796	u32		sccc_bt_num;
 797	u32		gmv_bt_num;
 798	u32		qpc_ba_pg_sz;
 799	u32		qpc_buf_pg_sz;
 800	u32		qpc_hop_num;
 801	u32		srqc_ba_pg_sz;
 802	u32		srqc_buf_pg_sz;
 803	u32		srqc_hop_num;
 804	u32		cqc_ba_pg_sz;
 805	u32		cqc_buf_pg_sz;
 806	u32		cqc_hop_num;
 807	u32		mpt_ba_pg_sz;
 808	u32		mpt_buf_pg_sz;
 809	u32		mpt_hop_num;
 810	u32		mtt_ba_pg_sz;
 811	u32		mtt_buf_pg_sz;
 812	u32		mtt_hop_num;
 813	u32		wqe_sq_hop_num;
 814	u32		wqe_sge_hop_num;
 815	u32		wqe_rq_hop_num;
 816	u32		sccc_ba_pg_sz;
 817	u32		sccc_buf_pg_sz;
 818	u32		sccc_hop_num;
 819	u32		qpc_timer_ba_pg_sz;
 820	u32		qpc_timer_buf_pg_sz;
 821	u32		qpc_timer_hop_num;
 822	u32		cqc_timer_ba_pg_sz;
 823	u32		cqc_timer_buf_pg_sz;
 824	u32		cqc_timer_hop_num;
 825	u32		cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */
 826	u32		cqe_buf_pg_sz;
 827	u32		cqe_hop_num;
 828	u32		srqwqe_ba_pg_sz;
 829	u32		srqwqe_buf_pg_sz;
 830	u32		srqwqe_hop_num;
 831	u32		idx_ba_pg_sz;
 832	u32		idx_buf_pg_sz;
 833	u32		idx_hop_num;
 834	u32		eqe_ba_pg_sz;
 835	u32		eqe_buf_pg_sz;
 836	u32		eqe_hop_num;
 837	u32		gmv_entry_num;
 838	u32		gmv_entry_sz;
 839	u32		gmv_ba_pg_sz;
 840	u32		gmv_buf_pg_sz;
 841	u32		gmv_hop_num;
 842	u32		sl_num;
 843	u32		llm_buf_pg_sz;
 844	u32		chunk_sz; /* chunk size in non multihop mode */
 845	u64		flags;
 846	u16		default_ceq_max_cnt;
 847	u16		default_ceq_period;
 848	u16		default_aeq_max_cnt;
 849	u16		default_aeq_period;
 850	u16		default_aeq_arm_st;
 851	u16		default_ceq_arm_st;
 852	u8		cong_cap;
 853	enum hns_roce_cong_type default_cong_type;
 854	u32             max_ack_req_msg_len;
 855};
 856
 857enum hns_roce_device_state {
 858	HNS_ROCE_DEVICE_STATE_INITED,
 859	HNS_ROCE_DEVICE_STATE_RST_DOWN,
 860	HNS_ROCE_DEVICE_STATE_UNINIT,
 861};
 862
 863enum hns_roce_hw_pkt_stat_index {
 864	HNS_ROCE_HW_RX_RC_PKT_CNT,
 865	HNS_ROCE_HW_RX_UC_PKT_CNT,
 866	HNS_ROCE_HW_RX_UD_PKT_CNT,
 867	HNS_ROCE_HW_RX_XRC_PKT_CNT,
 868	HNS_ROCE_HW_RX_PKT_CNT,
 869	HNS_ROCE_HW_RX_ERR_PKT_CNT,
 870	HNS_ROCE_HW_RX_CNP_PKT_CNT,
 871	HNS_ROCE_HW_TX_RC_PKT_CNT,
 872	HNS_ROCE_HW_TX_UC_PKT_CNT,
 873	HNS_ROCE_HW_TX_UD_PKT_CNT,
 874	HNS_ROCE_HW_TX_XRC_PKT_CNT,
 875	HNS_ROCE_HW_TX_PKT_CNT,
 876	HNS_ROCE_HW_TX_ERR_PKT_CNT,
 877	HNS_ROCE_HW_TX_CNP_PKT_CNT,
 878	HNS_ROCE_HW_TRP_GET_MPT_ERR_PKT_CNT,
 879	HNS_ROCE_HW_TRP_GET_IRRL_ERR_PKT_CNT,
 880	HNS_ROCE_HW_ECN_DB_CNT,
 881	HNS_ROCE_HW_RX_BUF_CNT,
 882	HNS_ROCE_HW_TRP_RX_SOF_CNT,
 883	HNS_ROCE_HW_CQ_CQE_CNT,
 884	HNS_ROCE_HW_CQ_POE_CNT,
 885	HNS_ROCE_HW_CQ_NOTIFY_CNT,
 886	HNS_ROCE_HW_CNT_TOTAL
 887};
 888
 889enum hns_roce_sw_dfx_stat_index {
 890	HNS_ROCE_DFX_AEQE_CNT,
 891	HNS_ROCE_DFX_CEQE_CNT,
 892	HNS_ROCE_DFX_CMDS_CNT,
 893	HNS_ROCE_DFX_CMDS_ERR_CNT,
 894	HNS_ROCE_DFX_MBX_POSTED_CNT,
 895	HNS_ROCE_DFX_MBX_POLLED_CNT,
 896	HNS_ROCE_DFX_MBX_EVENT_CNT,
 897	HNS_ROCE_DFX_QP_CREATE_ERR_CNT,
 898	HNS_ROCE_DFX_QP_MODIFY_ERR_CNT,
 899	HNS_ROCE_DFX_CQ_CREATE_ERR_CNT,
 900	HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT,
 901	HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT,
 902	HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT,
 903	HNS_ROCE_DFX_XRCD_ALLOC_ERR_CNT,
 904	HNS_ROCE_DFX_MR_REG_ERR_CNT,
 905	HNS_ROCE_DFX_MR_REREG_ERR_CNT,
 906	HNS_ROCE_DFX_AH_CREATE_ERR_CNT,
 907	HNS_ROCE_DFX_MMAP_ERR_CNT,
 908	HNS_ROCE_DFX_UCTX_ALLOC_ERR_CNT,
 909	HNS_ROCE_DFX_CNT_TOTAL
 910};
 911
 912struct hns_roce_hw {
 913	int (*cmq_init)(struct hns_roce_dev *hr_dev);
 914	void (*cmq_exit)(struct hns_roce_dev *hr_dev);
 915	int (*hw_profile)(struct hns_roce_dev *hr_dev);
 916	int (*hw_init)(struct hns_roce_dev *hr_dev);
 917	void (*hw_exit)(struct hns_roce_dev *hr_dev);
 918	int (*post_mbox)(struct hns_roce_dev *hr_dev,
 919			 struct hns_roce_mbox_msg *mbox_msg);
 920	int (*poll_mbox_done)(struct hns_roce_dev *hr_dev);
 921	bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy);
 922	int (*set_gid)(struct hns_roce_dev *hr_dev, int gid_index,
 923		       const union ib_gid *gid, const struct ib_gid_attr *attr);
 924	int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port,
 925		       const u8 *addr);
 926	int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf,
 927			  struct hns_roce_mr *mr);
 928	int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev,
 929				struct hns_roce_mr *mr, int flags,
 930				void *mb_buf);
 931	int (*frmr_write_mtpt)(void *mb_buf, struct hns_roce_mr *mr);
 932	void (*write_cqc)(struct hns_roce_dev *hr_dev,
 933			  struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
 934			  dma_addr_t dma_handle);
 935	int (*set_hem)(struct hns_roce_dev *hr_dev,
 936		       struct hns_roce_hem_table *table, int obj, u32 step_idx);
 937	int (*clear_hem)(struct hns_roce_dev *hr_dev,
 938			 struct hns_roce_hem_table *table, int obj,
 939			 u32 step_idx);
 940	int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 941			 int attr_mask, enum ib_qp_state cur_state,
 942			 enum ib_qp_state new_state, struct ib_udata *udata);
 943	int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,
 944			 struct hns_roce_qp *hr_qp);
 945	void (*dereg_mr)(struct hns_roce_dev *hr_dev);
 946	int (*init_eq)(struct hns_roce_dev *hr_dev);
 947	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 948	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
 949	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
 950	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
 951	int (*query_mpt)(struct hns_roce_dev *hr_dev, u32 key, void *buffer);
 952	int (*query_srqc)(struct hns_roce_dev *hr_dev, u32 srqn, void *buffer);
 953	int (*query_sccc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
 954	int (*query_hw_counter)(struct hns_roce_dev *hr_dev,
 955				u64 *stats, u32 port, int *hw_counters);
 956	int (*get_dscp)(struct hns_roce_dev *hr_dev, u8 dscp,
 957			u8 *tc_mode, u8 *priority);
 958	const struct ib_device_ops *hns_roce_dev_ops;
 959	const struct ib_device_ops *hns_roce_dev_srq_ops;
 960};
 961
 962struct hns_roce_dev {
 963	struct ib_device	ib_dev;
 964	struct pci_dev		*pci_dev;
 965	struct device		*dev;
 966	struct hns_roce_uar     priv_uar;
 967	const char		*irq_names[HNS_ROCE_MAX_IRQ_NUM];
 968	spinlock_t		sm_lock;
 969	bool			active;
 970	bool			is_reset;
 971	bool			dis_db;
 972	unsigned long		reset_cnt;
 973	struct hns_roce_ib_iboe iboe;
 974	enum hns_roce_device_state state;
 975	struct list_head	qp_list; /* list of all qps on this dev */
 976	spinlock_t		qp_list_lock; /* protect qp_list */
 977
 978	struct list_head        pgdir_list;
 979	struct mutex            pgdir_mutex;
 980	int			irq[HNS_ROCE_MAX_IRQ_NUM];
 981	u8 __iomem		*reg_base;
 982	void __iomem		*mem_base;
 983	struct hns_roce_caps	caps;
 984	struct xarray		qp_table_xa;
 985
 986	unsigned char	dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN];
 987	u64			sys_image_guid;
 988	u32                     vendor_id;
 989	u32                     vendor_part_id;
 990	u32                     hw_rev;
 991	void __iomem            *priv_addr;
 992
 993	struct hns_roce_cmdq	cmd;
 994	struct hns_roce_ida pd_ida;
 995	struct hns_roce_ida xrcd_ida;
 996	struct hns_roce_ida uar_ida;
 997	struct hns_roce_mr_table  mr_table;
 998	struct hns_roce_cq_table  cq_table;
 999	struct hns_roce_srq_table srq_table;
1000	struct hns_roce_qp_table  qp_table;
1001	struct hns_roce_eq_table  eq_table;
1002	struct hns_roce_hem_table  qpc_timer_table;
1003	struct hns_roce_hem_table  cqc_timer_table;
1004	/* GMV is the memory area that the driver allocates for the hardware
1005	 * to store SGID, SMAC and VLAN information.
1006	 */
1007	struct hns_roce_hem_table  gmv_table;
1008
1009	int			cmd_mod;
1010	int			loop_idc;
1011	u32			sdb_offset;
1012	u32			odb_offset;
1013	const struct hns_roce_hw *hw;
1014	void			*priv;
1015	struct workqueue_struct *irq_workq;
1016	struct work_struct ecc_work;
1017	u32 func_num;
1018	u32 is_vf;
1019	u32 cong_algo_tmpl_id;
1020	u64 dwqe_page;
1021	struct hns_roce_dev_debugfs dbgfs;
1022	atomic64_t *dfx_cnt;
1023};
1024
1025enum hns_roce_trace_type {
1026	TRACE_SQ,
1027	TRACE_RQ,
1028	TRACE_SRQ,
1029};
1030
1031static inline const char *trace_type_to_str(enum hns_roce_trace_type type)
1032{
1033	switch (type) {
1034	case TRACE_SQ:
1035		return "SQ";
1036	case TRACE_RQ:
1037		return "RQ";
1038	case TRACE_SRQ:
1039		return "SRQ";
1040	default:
1041		return "UNKNOWN";
1042	}
1043}
1044
1045static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
1046{
1047	return container_of(ib_dev, struct hns_roce_dev, ib_dev);
1048}
1049
1050static inline struct hns_roce_ucontext
1051			*to_hr_ucontext(struct ib_ucontext *ibucontext)
1052{
1053	return container_of(ibucontext, struct hns_roce_ucontext, ibucontext);
1054}
1055
1056static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd)
1057{
1058	return container_of(ibpd, struct hns_roce_pd, ibpd);
1059}
1060
1061static inline struct hns_roce_xrcd *to_hr_xrcd(struct ib_xrcd *ibxrcd)
1062{
1063	return container_of(ibxrcd, struct hns_roce_xrcd, ibxrcd);
1064}
1065
1066static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah)
1067{
1068	return container_of(ibah, struct hns_roce_ah, ibah);
1069}
1070
1071static inline struct hns_roce_mr *to_hr_mr(struct ib_mr *ibmr)
1072{
1073	return container_of(ibmr, struct hns_roce_mr, ibmr);
1074}
1075
1076static inline struct hns_roce_qp *to_hr_qp(struct ib_qp *ibqp)
1077{
1078	return container_of(ibqp, struct hns_roce_qp, ibqp);
1079}
1080
1081static inline struct hns_roce_cq *to_hr_cq(struct ib_cq *ib_cq)
1082{
1083	return container_of(ib_cq, struct hns_roce_cq, ib_cq);
1084}
1085
1086static inline struct hns_roce_srq *to_hr_srq(struct ib_srq *ibsrq)
1087{
1088	return container_of(ibsrq, struct hns_roce_srq, ibsrq);
1089}
1090
1091static inline struct hns_user_mmap_entry *
1092to_hns_mmap(struct rdma_user_mmap_entry *rdma_entry)
1093{
1094	return container_of(rdma_entry, struct hns_user_mmap_entry, rdma_entry);
1095}
1096
1097static inline void hns_roce_write64_k(__le32 val[2], void __iomem *dest)
1098{
1099	writeq(*(u64 *)val, dest);
1100}
1101
1102static inline struct hns_roce_qp
1103	*__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn)
1104{
1105	return xa_load(&hr_dev->qp_table_xa, qpn);
1106}
1107
1108static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf,
1109					unsigned int offset)
1110{
1111	return (char *)(buf->trunk_list[offset >> buf->trunk_shift].buf) +
1112			(offset & ((1 << buf->trunk_shift) - 1));
1113}
1114
1115static inline dma_addr_t hns_roce_buf_dma_addr(struct hns_roce_buf *buf,
1116					       unsigned int offset)
1117{
1118	return buf->trunk_list[offset >> buf->trunk_shift].map +
1119			(offset & ((1 << buf->trunk_shift) - 1));
1120}
1121
1122static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, u32 idx)
1123{
1124	return hns_roce_buf_dma_addr(buf, idx << buf->page_shift);
1125}
1126
1127#define hr_hw_page_align(x)		ALIGN(x, 1 << HNS_HW_PAGE_SHIFT)
1128
1129static inline u64 to_hr_hw_page_addr(u64 addr)
1130{
1131	return addr >> HNS_HW_PAGE_SHIFT;
1132}
1133
1134static inline u32 to_hr_hw_page_shift(u32 page_shift)
1135{
1136	return page_shift - HNS_HW_PAGE_SHIFT;
1137}
1138
1139static inline u32 to_hr_hem_hopnum(u32 hopnum, u32 count)
1140{
1141	if (count > 0)
1142		return hopnum == HNS_ROCE_HOP_NUM_0 ? 0 : hopnum;
1143
1144	return 0;
1145}
1146
1147static inline u32 to_hr_hem_entries_size(u32 count, u32 buf_shift)
1148{
1149	return hr_hw_page_align(count << buf_shift);
1150}
1151
1152static inline u32 to_hr_hem_entries_count(u32 count, u32 buf_shift)
1153{
1154	return hr_hw_page_align(count << buf_shift) >> buf_shift;
1155}
1156
1157static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift)
1158{
1159	if (!count)
1160		return 0;
1161
1162	return ilog2(to_hr_hem_entries_count(count, buf_shift));
1163}
1164
1165#define DSCP_SHIFT 2
1166
1167static inline u8 get_tclass(const struct ib_global_route *grh)
1168{
1169	return grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP ?
1170	       grh->traffic_class >> DSCP_SHIFT : grh->traffic_class;
1171}
1172
1173static inline struct net_device *get_hr_netdev(struct hns_roce_dev *hr_dev,
1174					       u8 port)
1175{
1176	return hr_dev->iboe.netdevs[port];
1177}
1178
1179static inline u8 get_hr_bus_num(struct hns_roce_dev *hr_dev)
1180{
1181	return hr_dev->pci_dev->bus->number;
1182}
1183
1184void hns_roce_init_uar_table(struct hns_roce_dev *dev);
1185int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
1186
1187int hns_roce_cmd_init(struct hns_roce_dev *hr_dev);
1188void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev);
1189void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
1190			u64 out_param);
1191int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);
1192void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);
1193
1194/* hns roce hw need current block and next block addr from mtt */
1195#define MTT_MIN_COUNT	 2
1196static inline dma_addr_t hns_roce_get_mtr_ba(struct hns_roce_mtr *mtr)
1197{
1198	return mtr->hem_cfg.root_ba;
1199}
1200
1201int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1202		      u32 offset, u64 *mtt_buf, int mtt_max);
1203int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1204			struct hns_roce_buf_attr *buf_attr,
1205			unsigned int page_shift, struct ib_udata *udata,
1206			unsigned long user_addr);
1207void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
1208			  struct hns_roce_mtr *mtr);
1209int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1210		     dma_addr_t *pages, unsigned int page_cnt);
1211
1212void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
1213void hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
1214void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);
1215int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);
1216void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev);
1217void hns_roce_init_xrcd_table(struct hns_roce_dev *hr_dev);
1218
1219void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);
1220void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);
1221
1222void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev);
1223
1224int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
1225		       struct ib_udata *udata);
1226int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
1227static inline int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)
1228{
1229	return 0;
1230}
1231
1232int hns_roce_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
1233int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
1234
1235struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc);
1236struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1237				   u64 virt_addr, int access_flags,
1238				   struct ib_dmah *dmah,
1239				   struct ib_udata *udata);
1240struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,
1241				     u64 length, u64 virt_addr,
1242				     int mr_access_flags, struct ib_pd *pd,
1243				     struct ib_udata *udata);
1244struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1245				u32 max_num_sg);
1246int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1247		       unsigned int *sg_offset);
1248int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
1249unsigned long key_to_hw_index(u32 key);
1250
1251void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf);
1252struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,
1253					u32 page_shift, u32 flags);
1254
1255int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
1256			   int buf_cnt, struct hns_roce_buf *buf,
1257			   unsigned int page_shift);
1258int hns_roce_get_umem_bufs(dma_addr_t *bufs,
1259			   int buf_cnt, struct ib_umem *umem,
1260			   unsigned int page_shift);
1261
1262int hns_roce_create_srq(struct ib_srq *srq,
1263			struct ib_srq_init_attr *srq_init_attr,
1264			struct ib_udata *udata);
1265int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
1266
1267int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);
1268int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);
1269
1270int hns_roce_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *init_attr,
1271		       struct ib_udata *udata);
1272int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1273		       int attr_mask, struct ib_udata *udata);
1274void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
1275void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n);
1276void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n);
1277void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n);
1278bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,
1279			  struct ib_cq *ib_cq);
1280void hns_roce_lock_cqs(struct hns_roce_cq *send_cq,
1281		       struct hns_roce_cq *recv_cq);
1282void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1283			 struct hns_roce_cq *recv_cq);
1284void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);
1285void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
1286			 struct ib_udata *udata);
1287__be32 send_ieth(const struct ib_send_wr *wr);
1288int to_hr_qp_type(int qp_type);
1289
1290int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
1291		       struct uverbs_attr_bundle *attrs);
1292
1293int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
1294int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
1295			 struct hns_roce_db *db);
1296void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
1297			    struct hns_roce_db *db);
1298int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
1299		      int order);
1300void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db);
1301
1302void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);
1303void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);
1304void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp);
1305void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);
1306void hns_roce_flush_cqe(struct hns_roce_dev *hr_dev, u32 qpn);
1307void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);
1308void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
1309int hns_roce_init(struct hns_roce_dev *hr_dev);
1310void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup);
1311int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
1312int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
1313int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
1314int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);
1315int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr);
1316int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr);
1317int hns_roce_fill_res_srq_entry(struct sk_buff *msg, struct ib_srq *ib_srq);
1318int hns_roce_fill_res_srq_entry_raw(struct sk_buff *msg, struct ib_srq *ib_srq);
1319struct hns_user_mmap_entry *
1320hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
1321				size_t length,
1322				enum hns_roce_mmap_type mmap_type);
1323bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl);
1324void hns_roce_put_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx);
1325void hns_roce_get_cq_bankid_for_uctx(struct hns_roce_ucontext *uctx);
1326
1327#endif /* _HNS_ROCE_DEVICE_H */